KeyDB

KeyDB copied to clipboard

Describe the bug

Almost every day at least once rdb_bgsave get stuck. I will check info persistence and see rdb_changes_since_last_save: VERY_BIG and rdb_current_bgsave_time_sec: VERY_BIG doing a kill -9 on the bgsave forked process causes it to try again and it always completes within 2 seconds and carries on fine for hours after that.

To reproduce

Enable automative bgsaves, wait up to a day or so.

Expected behavior

It should keep saving.

Additional information

We are in an active-active setup, only one of them has this issue. The one with the issue is:

# Server
redis_version:6.2.2
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:295716e5acce2b10
redis_mode:standalone
os:Linux 5.10.0-9-rt-amd64 x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:10.2.1
process_id:1320323
process_supervised:no
run_id:ff4aad529775ac39d5dcfc88311a6373de212437
tcp_port:6379
server_time_usec:1653133994192371
uptime_in_seconds:1014553
uptime_in_days:11
hz:10
configured_hz:10
lru_clock:8966826
executable:/usr/bin/keydb-server
config_file:/etc/keydb/keydb.conf

I find it unusual this is only happening on one node. Information that could help to further troubleshoot:

1. config files
2. logfile for the node that does not complete bgsave.
3. full output of INFO would be helpful here as well (you mentioned it takes 2sec to sync so I assume this is a small rdb)

info while stuck:

# Server
redis_version:6.2.2
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:295716e5acce2b10
redis_mode:standalone
os:Linux 5.10.0-9-rt-amd64 x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:10.2.1
process_id:1320323
process_supervised:no
run_id:ff4aad529775ac39d5dcfc88311a6373de212437
tcp_port:6379
server_time_usec:1654776892557233
uptime_in_seconds:2657451
uptime_in_days:30
hz:10
configured_hz:10
lru_clock:10609724
executable:/usr/bin/keydb-server
config_file:/etc/keydb/keydb.conf

# Clients
connected_clients:70
cluster_connections:0
maxclients:10000
client_recent_max_input_buffer:520
client_recent_max_output_buffer:0
blocked_clients:1
tracking_clients:0
clients_in_timeout_table:0
current_client_thread:1
thread_0_clients:36
thread_1_clients:35

# Memory
used_memory:95553944
used_memory_human:91.13M
used_memory_rss:109268992
used_memory_rss_human:104.21M
used_memory_peak:356930144
used_memory_peak_human:340.40M
used_memory_peak_perc:26.77%
used_memory_overhead:32217500
used_memory_startup:2082360
used_memory_dataset:63336444
used_memory_dataset_perc:67.76%
allocator_allocated:96251152
allocator_active:100855808
allocator_resident:110661632
total_system_memory:135115661312
total_system_memory_human:125.84G
used_memory_lua:45056
used_memory_lua_human:44.00K
used_memory_scripts:464
used_memory_scripts_human:464B
number_of_cached_scripts:2
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
allocator_frag_ratio:1.05
allocator_frag_bytes:4604656
allocator_rss_ratio:1.10
allocator_rss_bytes:9805824
rss_overhead_ratio:0.99
rss_overhead_bytes:-1392640
mem_fragmentation_ratio:1.14
mem_fragmentation_bytes:13716832
mem_not_counted_for_evict:2554
mem_replication_backlog:1048576
mem_clients_slaves:20512
mem_clients_normal:1415684
mem_aof_buffer:2560
mem_allocator:jemalloc-5.2.1
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:0

# Persistence
loading:0
current_cow_size:2957312
current_cow_size_age:1519489
current_fork_perc:88.11
current_save_keys_processed:454657
current_save_keys_total:516010
rdb_changes_since_last_save:63673452
rdb_bgsave_in_progress:1
rdb_last_save_time:1653257172
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:2
rdb_current_bgsave_time_sec:1519490
rdb_last_cow_size:3530752
aof_enabled:1
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:2854912
module_fork_in_progress:0
module_fork_last_cow_size:0
aof_current_size:6542502079
aof_base_size:61834100
aof_pending_rewrite:0
aof_buffer_length:0
aof_rewrite_buffer_length:0
aof_pending_bio_fsync:0
aof_delayed_fsync:1

# Stats
total_connections_received:305891
total_commands_processed:936297144
instantaneous_ops_per_sec:3231
total_net_input_bytes:79386568847
total_net_output_bytes:60803272546
instantaneous_input_kbps:193.18
instantaneous_output_kbps:274.79
rejected_connections:0
sync_full:4
sync_partial_ok:0
sync_partial_err:0
expired_keys:175234
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:0
evicted_keys:0
keyspace_hits:473539316
keyspace_misses:104662786
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:5676
total_forks:1830
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:80815081
dump_payload_sanitizations:0
total_reads_processed:727946844
total_writes_processed:778464231

# Replication
role:active-replica
master_global_link_status:up
master_host:fdaa:0:3d68:a7b:aa4:0:7189:2
master_port:6379
master_link_status:up
master_last_io_seconds_ago:1
master_sync_in_progress:0
slave_read_repl_offset:33226902450
slave_repl_offset:33226902450
slave_priority:100
slave_read_only:0
replica_announced:1
connected_slaves:1
slave0:ip=fdaa:0:3d68:a7b:aa4:0:7189:2,port=6379,state=online,offset=22643799530,lag=1
master_failover_state:no-failover
master_replid:d70b514d7d02cd634082c053e06fa41e22194b9d
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:22643799530
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:22642750955
repl_backlog_histlen:1048576

# CPU
used_cpu_sys:40780.228280
used_cpu_user:25975.251073
used_cpu_sys_children:311.399316
used_cpu_user_children:1807.596686
server_threads:2
long_lock_waits:8360
used_cpu_sys_main_thread:13456.317667
used_cpu_user_main_thread:9265.490196

# Modules

# Errorstats
errorstat_ERR:count=30
errorstat_LOADING:count=24
errorstat_WRONGTYPE:count=80815027

# Cluster
cluster_enabled:0

# Keyspace
db0:keys=360454,expires=19308,avg_ttl=12800129
db1:keys=70,expires=0,avg_ttl=0
db2:keys=66592,expires=0,avg_ttl=0
db3:keys=96574,expires=0,avg_ttl=0

Replica info at same time showing it is not stuck:

redis_version:6.2.0
redis_git_sha1:971725cc
redis_git_dirty:1
redis_build_id:d1d3c4923bd2f5d3
redis_mode:standalone
os:Linux 5.12.2 x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:7.5.0
process_id:533
process_supervised:no
run_id:81e3bed46f69c0b56545eae52ddc7ab7cf59ca5c
tcp_port:6379
server_time_usec:1654776941910164
uptime_in_seconds:2657399
uptime_in_days:30
hz:10
configured_hz:10
lru_clock:10609773
executable:/fly/keydb-server
config_file:/etc/keydb.conf

# Clients
connected_clients:2
cluster_connections:0
maxclients:10000
client_recent_max_input_buffer:488
client_recent_max_output_buffer:0
blocked_clients:0
tracking_clients:0
clients_in_timeout_table:0
current_client_thread:0
thread_0_clients:3

# Memory
used_memory:93522488
used_memory_human:89.19M
used_memory_rss:100499456
used_memory_rss_human:95.84M
used_memory_peak:135327288
used_memory_peak_human:129.06M
used_memory_peak_perc:69.11%
used_memory_overhead:30262632
used_memory_startup:1520272
used_memory_dataset:63259856
used_memory_dataset_perc:68.76%
allocator_allocated:93739248
allocator_active:94609408
allocator_resident:98201600
total_system_memory:231907328
total_system_memory_human:221.16M
used_memory_lua:45056
used_memory_lua_human:44.00K
used_memory_scripts:464
used_memory_scripts_human:464B
number_of_cached_scripts:2
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
allocator_frag_ratio:1.01
allocator_frag_bytes:870160
allocator_rss_ratio:1.04
allocator_rss_bytes:3592192
rss_overhead_ratio:1.02
rss_overhead_bytes:2297856
mem_fragmentation_ratio:1.07
mem_fragmentation_bytes:6978784
mem_not_counted_for_evict:3066
mem_replication_backlog:1048576
mem_clients_slaves:20512
mem_clients_normal:41024
mem_aof_buffer:3072
mem_allocator:jemalloc-5.2.1
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:0

# Persistence
loading:0
current_cow_size:0
current_cow_size_age:0
current_fork_perc:0.00
current_save_keys_processed:0
current_save_keys_total:0
rdb_changes_since_last_save:3890
rdb_bgsave_in_progress:0
rdb_last_save_time:1654776885
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:1
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:1339392
aof_enabled:1
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:2
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:1511424
module_fork_in_progress:0
module_fork_last_cow_size:0
aof_current_size:72788619
aof_base_size:64082262
aof_pending_rewrite:0
aof_buffer_length:0
aof_rewrite_buffer_length:0
aof_pending_bio_fsync:0
aof_delayed_fsync:0

# Stats
total_connections_received:523559
total_commands_processed:222297819
instantaneous_ops_per_sec:4
total_net_input_bytes:22434997299
total_net_output_bytes:33792384260
instantaneous_input_kbps:0.30
instantaneous_output_kbps:12.99
rejected_connections:0
sync_full:1
sync_partial_ok:0
sync_partial_err:0
expired_keys:175233
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:0
evicted_keys:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:3436
total_forks:13138
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:268893
total_error_replies:268917
dump_payload_sanitizations:0
total_reads_processed:60743935
total_writes_processed:60203639

# Replication
role:active-replica
master_global_link_status:up
master_host:fdaa:0:3d68:a7b:1221:0:a:102
master_port:6379
master_link_status:up
master_last_io_seconds_ago:1
master_sync_in_progress:0
slave_repl_offset:22644515092
slave_priority:100
slave_read_only:0
replica_announced:1
connected_slaves:1
slave0:ip=fdaa:0:3d68:a7b:1221:0:a:102,port=6379,state=online,offset=33227976360,lag=0
master_failover_state:no-failover
master_replid:c644127f2fc475191294e463449c44e875731fa1
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:33227976360
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:33226927785
repl_backlog_histlen:1048576

# CPU
used_cpu_sys:13293.101419
used_cpu_user:13414.255387
used_cpu_sys_children:2331.536828
used_cpu_user_children:17470.860287
server_threads:1
long_lock_waits:0
used_cpu_sys_main_thread:12847.837068
used_cpu_user_main_thread:13303.328620

# Modules

# Errorstats
errorstat_ERR:count=268895
errorstat_LOADING:count=22

# Cluster
cluster_enabled:0

# Keyspace
db0:keys=360453,expires=19307,avg_ttl=12763539
db1:keys=70,expires=0,avg_ttl=0
db2:keys=66592,expires=0,avg_ttl=0
db3:keys=96578,expires=0,avg_ttl=0

info right after doing a kill -9 on the bgsave child (regular kill not enough), showing it is now unstuck:

# Server
redis_version:6.2.2
redis_git_sha1:00000000
redis_git_dirty:0
redis_build_id:295716e5acce2b10
redis_mode:standalone
os:Linux 5.10.0-9-rt-amd64 x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:10.2.1
process_id:1320323
process_supervised:no
run_id:ff4aad529775ac39d5dcfc88311a6373de212437
tcp_port:6379
server_time_usec:1654777027209185
uptime_in_seconds:2657586
uptime_in_days:30
hz:10
configured_hz:10
lru_clock:10609859
executable:/usr/bin/keydb-server
config_file:/etc/keydb/keydb.conf

# Clients
connected_clients:70
cluster_connections:0
maxclients:10000
client_recent_max_input_buffer:520
client_recent_max_output_buffer:0
blocked_clients:1
tracking_clients:0
clients_in_timeout_table:0
current_client_thread:0
thread_0_clients:36
thread_1_clients:35

# Memory
used_memory:95713888
used_memory_human:91.28M
used_memory_rss:109248512
used_memory_rss_human:104.19M
used_memory_peak:356930144
used_memory_peak_human:340.40M
used_memory_peak_perc:26.82%
used_memory_overhead:32215180
used_memory_startup:2082360
used_memory_dataset:63498708
used_memory_dataset_perc:67.82%
allocator_allocated:96300568
allocator_active:100917248
allocator_resident:110723072
total_system_memory:135115661312
total_system_memory_human:125.84G
used_memory_lua:45056
used_memory_lua_human:44.00K
used_memory_scripts:464
used_memory_scripts_human:464B
number_of_cached_scripts:2
maxmemory:0
maxmemory_human:0B
maxmemory_policy:noeviction
allocator_frag_ratio:1.05
allocator_frag_bytes:4616680
allocator_rss_ratio:1.10
allocator_rss_bytes:9805824
rss_overhead_ratio:0.99
rss_overhead_bytes:-1474560
mem_fragmentation_ratio:1.14
mem_fragmentation_bytes:13577392
mem_not_counted_for_evict:252
mem_replication_backlog:1048576
mem_clients_slaves:20512
mem_clients_normal:1415668
mem_aof_buffer:256
mem_allocator:jemalloc-5.2.1
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:0

# Persistence
loading:0
current_cow_size:0
current_cow_size_age:0
current_fork_perc:0.00
current_save_keys_processed:0
current_save_keys_total:0
rdb_changes_since_last_save:1362
rdb_bgsave_in_progress:0
rdb_last_save_time:1654777020
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:1
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:4378624
aof_enabled:1
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:2
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:4497408
module_fork_in_progress:0
module_fork_last_cow_size:0
aof_current_size:64116022
aof_base_size:64055555
aof_pending_rewrite:0
aof_buffer_length:0
aof_rewrite_buffer_length:0
aof_pending_bio_fsync:0
aof_delayed_fsync:1

# Stats
total_connections_received:305898
total_commands_processed:936345329
instantaneous_ops_per_sec:1008
total_net_input_bytes:79391159563
total_net_output_bytes:60806241424
instantaneous_input_kbps:112.55
instantaneous_output_kbps:66.30
rejected_connections:0
sync_full:4
sync_partial_ok:0
sync_partial_err:0
expired_keys:175237
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:0
evicted_keys:0
keyspace_hits:473557079
keyspace_misses:104666821
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:6453
total_forks:1832
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:80817722
dump_payload_sanitizations:0
total_reads_processed:727979265
total_writes_processed:778500751

# Replication
role:active-replica
master_global_link_status:up
master_host:fdaa:0:3d68:a7b:aa4:0:7189:2
master_port:6379
master_link_status:up
master_last_io_seconds_ago:0
master_sync_in_progress:0
slave_read_repl_offset:33229408575
slave_repl_offset:33229408575
slave_priority:100
slave_read_only:0
replica_announced:1
connected_slaves:1
slave0:ip=fdaa:0:3d68:a7b:aa4:0:7189:2,port=6379,state=online,offset=22645453410,lag=1
master_failover_state:no-failover
master_replid:d70b514d7d02cd634082c053e06fa41e22194b9d
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:22645466658
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:22644418083
repl_backlog_histlen:1048576

# CPU
used_cpu_sys:40783.932378
used_cpu_user:25976.392775
used_cpu_sys_children:311.901372
used_cpu_user_children:1810.654782
server_threads:2
long_lock_waits:8360
used_cpu_sys_main_thread:17021.812133
used_cpu_user_main_thread:13646.821876

# Modules

# Errorstats
errorstat_ERR:count=30
errorstat_LOADING:count=24
errorstat_WRONGTYPE:count=80817668

# Cluster
cluster_enabled:0

# Keyspace
db0:keys=360452,expires=19306,avg_ttl=12678239
db1:keys=70,expires=0,avg_ttl=0
db2:keys=66592,expires=0,avg_ttl=0
db3:keys=96576,expires=0,avg_ttl=0

Logs:

May 22 21:58:46 scarlet redis[2036425]: RDB: 2 MB of memory used by copy-on-write
May 22 21:58:46 scarlet redis[1320323]: Background saving terminated with success
May 22 22:02:48 scarlet redis[1320323]: 10000 changes in 60 seconds. Saving...
May 22 22:02:48 scarlet redis[1320323]: Background saving started by pid 2036522
May 22 22:02:49 scarlet redis[2036522]: DB saved on disk
May 22 22:02:49 scarlet redis[2036522]: RDB: 2 MB of memory used by copy-on-write
May 22 22:02:49 scarlet redis[1320323]: Background saving terminated with success
May 22 22:06:10 scarlet redis[1320323]: 10000 changes in 60 seconds. Saving...
May 22 22:06:10 scarlet redis[1320323]: Background saving started by pid 2036587
May 22 22:06:11 scarlet redis[2036587]: DB saved on disk
May 22 22:06:11 scarlet redis[2036587]: RDB: 3 MB of memory used by copy-on-write
May 22 22:06:12 scarlet redis[1320323]: Background saving terminated with success
May 22 22:08:31 scarlet redis[1320323]: Starting automatic rewriting of AOF on 100% growth
May 22 22:08:31 scarlet redis[1320323]: Background append only file rewriting started by pid 2036645
May 22 22:08:32 scarlet redis[1320323]: AOF rewrite child asks to stop sending diffs.
May 22 22:08:32 scarlet redis[2036645]: Parent agreed to stop sending diffs. Finalizing AOF...
May 22 22:08:32 scarlet redis[2036645]: Concatenating 0.00 MB of AOF diff received from parent.
May 22 22:08:32 scarlet redis[2036645]: SYNC append only file rewrite performed
May 22 22:08:32 scarlet redis[2036645]: AOF rewrite: 2 MB of memory used by copy-on-write
May 22 22:08:32 scarlet redis[1320323]: Background AOF rewrite terminated with success
May 22 22:08:32 scarlet redis[1320323]: Residual parent diff successfully flushed to the rewritten AOF (0.00 MB)
May 22 22:08:32 scarlet redis[1320323]: Background AOF rewrite finished successfully
May 22 22:10:02 scarlet redis[1320323]: 10000 changes in 60 seconds. Saving...
May 22 22:10:02 scarlet redis[1320323]: Background saving started by pid 2036694
Jun  4 14:23:13 scarlet redis[1320323]: Possible SECURITY ATTACK detected. It looks like somebody is sending POST or Host: commands to KeyDB. This is likely due to an attacker attempting to use Cross Protocol Scripting to compromise your KeyDB instance. Connection aborted.
Jun  5 01:32:56 scarlet redis[1320323]: Asynchronous AOF fsync is taking too long (disk is busy?). Writing the AOF buffer without waiting for fsync to complete, this may slow down KeyDB.
Jun  9 12:16:59 scarlet redis[1320323]: Background saving terminated by signal 9
Jun  9 12:16:59 scarlet redis[1320323]: 1 changes in 900 seconds. Saving...
Jun  9 12:16:59 scarlet redis[1320323]: Background saving started by pid 2871798
Jun  9 12:17:00 scarlet redis[2871798]: DB saved on disk
Jun  9 12:17:00 scarlet redis[2871798]: RDB: 4 MB of memory used by copy-on-write
Jun  9 12:17:00 scarlet redis[1320323]: Background saving terminated with success
Jun  9 12:17:00 scarlet redis[1320323]: Starting automatic rewriting of AOF on 10481% growth
Jun  9 12:17:00 scarlet redis[1320323]: Background append only file rewriting started by pid 2871810
Jun  9 12:17:02 scarlet redis[1320323]: AOF rewrite child asks to stop sending diffs.
Jun  9 12:17:02 scarlet redis[2871810]: Parent agreed to stop sending diffs. Finalizing AOF...
Jun  9 12:17:02 scarlet redis[2871810]: Concatenating 0.04 MB of AOF diff received from parent.
Jun  9 12:17:02 scarlet redis[2871810]: SYNC append only file rewrite performed
Jun  9 12:17:02 scarlet redis[2871810]: AOF rewrite: 4 MB of memory used by copy-on-write
Jun  9 12:17:02 scarlet redis[1320323]: Background AOF rewrite terminated with success
Jun  9 12:17:02 scarlet redis[1320323]: Residual parent diff successfully flushed to the rewritten AOF (0.00 MB)
Jun  9 12:17:02 scarlet redis[1320323]: Background AOF rewrite finished successfully

Redacted config with comments removed:

bind 127.0.0.1 -::1 REDACTED_INTERNAL_IPV6
protected-mode yes
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
pidfile /var/run/keydb/keydb-server.pid
loglevel notice
logfile ""
syslog-enabled yes
databases 16
always-show-logo yes
set-proc-title yes
proc-title-template "{title} {listen-addr} {server-mode}"

save 900 1
save 300 10
save 60 10000

stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
rdb-del-sync-files no
dir /var/lib/keydb

replicaof REDACTED_INTERAL_IPV6 6379
replica-serve-stale-data yes
replica-read-only yes

repl-diskless-sync no
repl-diskless-sync-delay 5
repl-diskless-load disabled
repl-disable-tcp-nodelay no
replica-priority 100

acllog-max-len 128

lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
lazyfree-lazy-user-del no
lazyfree-lazy-user-flush no

oom-score-adj no
oom-score-adj-values 0 200 800

disable-thp yes

appendonly yes
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes

lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128

latency-monitor-threshold 0

notify-keyspace-events ""

hash-max-ziplist-entries 512
hash-max-ziplist-value 64

list-max-ziplist-size -2
list-compress-depth 0

set-max-intset-entries 512

zset-max-ziplist-entries 128
zset-max-ziplist-value 64

hll-sparse-max-bytes 3000

stream-node-max-bytes 4096
stream-node-max-entries 100

activerehashing yes

client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60

hz 10
dynamic-hz yes

aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes

jemalloc-bg-thread yes

server-threads 2

active-replica yes
multi-master yes

This does not seem to happen on 6.3 but now being stung by #453

closing as duplicate of #378 and is fixed in 6.3