KeyDB
KeyDB copied to clipboard
Snapchat
[CRASH] '((tls_connection*)conn)->el == serverTL->el' is not true
Crash report
Paste the complete crash log between the quotes below. Please include a few lines from the log preceding the crash report to provide some context.
=== KEYDB BUG REPORT START: Cut & paste starting from here ===
17:28:S 07 Jun 2023 20:02:45.188 # === ASSERTION FAILED ===
17:28:S 07 Jun 2023 20:02:45.188 # ==> tls.cpp:1104 '((tls_connection*)conn)->el == serverTL->el' is not true
------ STACK TRACE ------
Backtrace:
keydb-server 0.0.0.0:17379 [cluster](connTLSSetReadHandler(connection*, void (*)(connection*), bool)+0x63) [0x558386af3733]
keydb-server 0.0.0.0:17379 [cluster](createClient(connection*, int)+0xaa) [0x558386a392ca]
keydb-server 0.0.0.0:17379 [cluster](replicationCreateMasterClient(redisMaster*, connection*, int)+0x58) [0x558386a520b8]
keydb-server 0.0.0.0:17379 [cluster](readSyncBulkPayload(connection*)+0x5f7) [0x558386a57bc7]
keydb-server 0.0.0.0:17379 [cluster](tlsHandleEvent(tls_connection*, int)+0x21d) [0x558386af45ad]
keydb-server 0.0.0.0:17379 [cluster](ProcessEventCore+0xe8) [0x558386a1a5d8]
keydb-server 0.0.0.0:17379 [cluster](aeProcessEvents+0x17d) [0x558386a1a8cd]
keydb-server 0.0.0.0:17379 [cluster](aeMain+0x4b) [0x558386a1adeb]
keydb-server 0.0.0.0:17379 [cluster](workerThreadMain(void*)+0x74) [0x558386a22bf4]
/lib/x86_64-linux-gnu/libpthread.so.0(+0x8609) [0x7f5111067609]
/lib/x86_64-linux-gnu/libc.so.6(clone+0x43) [0x7f5110f8c133]
------ INFO OUTPUT ------
# Server
redis_version:6.2.2
redis_git_sha1:78ebe79e
redis_git_dirty:1
redis_build_id:9711844e424f6cb8
redis_mode:cluster
os:Linux 5.10.149-133.644.amzn2.x86_64 x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:9.4.0
process_id:17
process_supervised:no
run_id:97007c7b25f64e704ddd05b65cbdb15cf4abc025
tcp_port:17379
server_time_usec:1686168165189049
uptime_in_seconds:35617
uptime_in_days:0
hz:40
configured_hz:40
lru_clock:8446565
executable:/data/keydb-server
config_file:/redis-conf/redis.conf
availability_zone:us-east-1d
# Clients
connected_clients:594
cluster_connections:424
maxclients:80000
client_recent_max_input_buffer:80
client_recent_max_output_buffer:0
blocked_clients:0
tracking_clients:0
clients_in_timeout_table:0
current_client_thread:3
thread_0_clients:96
thread_1_clients:101
thread_2_clients:110
thread_3_clients:96
thread_4_clients:92
thread_5_clients:99
# Memory
used_memory:30033508080
used_memory_human:27.97G
used_memory_rss:31836147712
used_memory_rss_human:29.65G
used_memory_peak:32689237384
used_memory_peak_human:30.44G
used_memory_peak_perc:91.88%
used_memory_overhead:6960908576
used_memory_startup:31417816
used_memory_dataset:23072599504
used_memory_dataset_perc:76.90%
allocator_allocated:30034025128
allocator_active:30110724096
allocator_resident:30796124160
total_system_memory:66362789888
total_system_memory_human:61.81G
used_memory_lua:37888
used_memory_lua_human:37.00K
used_memory_scripts:0
used_memory_scripts_human:0B
number_of_cached_scripts:0
maxmemory:45742698086
maxmemory_human:42.60G
maxmemory_policy:allkeys-lfu
allocator_frag_ratio:1.00
allocator_frag_bytes:76698968
allocator_rss_ratio:1.02
allocator_rss_bytes:685400064
rss_overhead_ratio:1.03
rss_overhead_bytes:1040023552
mem_fragmentation_ratio:1.06
mem_fragmentation_bytes:1804533696
mem_not_counted_for_evict:0
mem_replication_backlog:0
mem_clients_slaves:0
mem_clients_normal:12154248
mem_aof_buffer:0
mem_allocator:jemalloc-5.2.1
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:121716266
# Persistence
loading:0
current_cow_size:0
current_cow_size_age:0
current_fork_perc:0.00
current_save_keys_processed:0
current_save_keys_total:0
rdb_changes_since_last_save:86707963
rdb_bgsave_in_progress:0
rdb_last_save_time:1686132563
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:0
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:1257472
aof_enabled:0
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:0
module_fork_in_progress:0
module_fork_last_cow_size:0
# Stats
total_connections_received:110055
total_commands_processed:665362548
instantaneous_ops_per_sec:62
total_net_input_bytes:78134179898
total_net_output_bytes:323532867073
instantaneous_input_kbps:9986.09
instantaneous_output_kbps:328.59
rejected_connections:0
sync_full:2
sync_partial_ok:0
sync_partial_err:2
expired_keys:8344387
expired_stale_perc:8.58
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:35814
evicted_keys:0
keyspace_hits:564269349
keyspace_misses:70442624
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:1372
total_forks:1
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:500731
dump_payload_sanitizations:0
total_reads_processed:576778001
total_writes_processed:646275012
# Replication
role:slave
master_global_link_status:down
master_host:10.88.169.207
master_port:17379
master_link_status:down
master_last_io_seconds_ago:-1
master_sync_in_progress:1
slave_read_repl_offset:1
slave_repl_offset:1
master_sync_total_bytes:19880092134
master_sync_read_bytes:1594490880
master_sync_left_bytes:18285601254
master_sync_perc:8.02
master_sync_last_io_seconds_ago:0
master_link_down_since_seconds:1218
slave_priority:100
slave_read_only:1
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:fd510a2695e15950b8233d540e57c50d682dace0
master_replid2:1a86f6674c776c295fbe99527ea1df8cf3e3794a
master_repl_offset:34663744784
second_repl_offset:34663720551
repl_backlog_active:0
repl_backlog_size:2000000000
repl_backlog_first_byte_offset:32663744785
repl_backlog_histlen:2000000000
# CPU
used_cpu_sys:9260.620731
used_cpu_user:15597.600662
used_cpu_sys_children:0.000000
used_cpu_user_children:0.002974
server_threads:6
long_lock_waits:22588
used_cpu_sys_main_thread:1225.531674
used_cpu_user_main_thread:2562.485528
# Modules
module:name=statsd,ver=1,api=1,filters=0,usedby=[],using=[],options=[]
# Commandstats
cmdstat_asking:calls=371421,usec=150291,usec_per_call=0.40,rejected_calls=0,failed_calls=0
cmdstat_replconf:calls=67647,usec=51685,usec_per_call=0.76,rejected_calls=0,failed_calls=0
cmdstat_ping:calls=637902,usec=313188,usec_per_call=0.49,rejected_calls=0,failed_calls=0
cmdstat_dbsize:calls=3332,usec=996,usec_per_call=0.30,rejected_calls=0,failed_calls=0
cmdstat_command:calls=51,usec=31971,usec_per_call=626.88,rejected_calls=0,failed_calls=0
cmdstat_replping:calls=2,usec=2,usec_per_call=1.00,rejected_calls=0,failed_calls=0
cmdstat_setex:calls=28891221,usec=139822972,usec_per_call=4.84,rejected_calls=102779,failed_calls=0
cmdstat_get:calls=576735829,usec=1077307597,usec_per_call=1.87,rejected_calls=352838,failed_calls=0
cmdstat_select:calls=46,usec=35,usec_per_call=0.76,rejected_calls=0,failed_calls=0
cmdstat_info:calls=93084,usec=4760747,usec_per_call=51.14,rejected_calls=0,failed_calls=0
cmdstat_readonly:calls=305884,usec=104852,usec_per_call=0.34,rejected_calls=39323,failed_calls=0
cmdstat_psync:calls=2,usec=1547,usec_per_call=773.50,rejected_calls=0,failed_calls=0
cmdstat_set:calls=28513,usec=110035,usec_per_call=3.86,rejected_calls=0,failed_calls=0
cmdstat_cluster:calls=34859,usec=88378656,usec_per_call=2535.32,rejected_calls=80,failed_calls=32
cmdstat_client:calls=65,usec=101,usec_per_call=1.55,rejected_calls=0,failed_calls=0
cmdstat_restore-asking:calls=57605076,usec=126206688,usec_per_call=2.19,rejected_calls=0,failed_calls=0
cmdstat_del:calls=586337,usec=1791096,usec_per_call=3.05,rejected_calls=5679,failed_calls=0
cmdstat_config:calls=1277,usec=39837,usec_per_call=31.20,rejected_calls=0,failed_calls=0
# Errorstats
errorstat_ERR:count=32
errorstat_LOADING:count=45621
errorstat_MOVED:count=455078
# Cluster
cluster_enabled:1
# Keyspace
db0:keys=60870778,expires=60870778,avg_ttl=45027707
------ CLIENT LIST OUTPUT ------
...
------ MODULES INFO OUTPUT ------
------ FAST MEMORY TEST ------
17:28:S 07 Jun 2023 20:02:45.191 # main thread terminated
17:28:S 07 Jun 2023 20:02:45.191 # Bio thread for job type #0 terminated
17:28:S 07 Jun 2023 20:02:45.191 # Bio thread for job type #1 terminated
17:28:S 07 Jun 2023 20:02:45.192 # Bio thread for job type #2 terminated
Fast memory test PASSED, however your memory can still be broken. Please run a memory test for several hours if possible.
=== KEYDB BUG REPORT END. Make sure to include from START to END. ===
Aditional information
- OS distribution and version
- Steps to reproduce (if any)
