KeyDB icon indicating copy to clipboard operation
KeyDB copied to clipboard
verified publisher icon Snapchat

[CRASH] KeyDB 6.3.1 with Redis Gears 1.2.5

Open artursmolarek opened this issue 3 years ago • 0 comments

Crash report

KeyDB was installed using Helm chart. It uses a custom docker image. I'm trying to spin up 3 nodes cluster (multi master replication). It fails during the startup of the second node. The first one starts with no issues.

=== KEYDB BUG REPORT START: Cut & paste starting from here ===
1:10:S 01 Oct 2022 18:27:10.871 # KeyDB 6.3.1 crashed by signal: 11, si_code: 1
1:10:S 01 Oct 2022 18:27:10.871 # Accessing address: 0x2a0
1:10:S 01 Oct 2022 18:27:10.871 # Crashed running the instruction at: 0x55fc93b45d63

------ STACK TRACE ------
EIP:
keydb-server 0.0.0.0:6379(unlinkClient(client*)+0x473) [0x55fc93b45d63]

Backtrace:
/lib/x86_64-linux-gnu/libpthread.so.0(+0x12980) [0x7fd78b4ba980]
keydb-server 0.0.0.0:6379(unlinkClient(client*)+0x473) [0x55fc93b45d63]
keydb-server 0.0.0.0:6379(freeClient(client*)+0x2aa) [0x55fc93b4636a]
keydb-server 0.0.0.0:6379(RM_FreeThreadSafeContext(RedisModuleCtx*)+0x15) [0x55fc93bbd995]
/modules/redisgears.so(+0x88359) [0x7fd7838a1359]
/modules/redisgears.so(+0xa9154) [0x7fd7838c2154]
/lib/x86_64-linux-gnu/libpthread.so.0(+0x76db) [0x7fd78b4af6db]
/lib/x86_64-linux-gnu/libc.so.6(clone+0x3f) [0x7fd78b1d861f]

------ REGISTERS ------
1:10:S 01 Oct 2022 18:27:10.871 #
RAX:0000000000000000 RBX:00007fd77bb27a80
RCX:0000000000000000 RDX:ffffffffffffe720
RDI:00007fd77bb27a80 RSI:00007fd77bb27a80
RBP:0000000000000000 RSP:00007fd7838162b0
R8 :00007fd785600900 R9 :00007fd77b80c5e0
R10:0000000000000005 R11:0000000000000030
R12:0000000000000000 R13:00007fd785440190
R14:00007fd7854401e8 R15:00007ffd83446d00
RIP:000055fc93b45d63 EFL:0000000000010246
CSGSFS:002b000000000033
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162bf) -> 000055fc93bbd34c
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162be) -> ffffffffffffffff
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162bd) -> 000055fc93b09b01
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162bc) -> 00007fd77bb2bd1c
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162bb) -> 000055fc93b4636a
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162ba) -> 00007fd7854401e8
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162b9) -> 00007fd785440190
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162b8) -> 00007fd77bb27b40
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162b7) -> 00007fd77bb2bd1c
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162b6) -> 00007fd77bb27a80
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162b5) -> e9b4bfd1846a1d00
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162b4) -> 00007fd77bb2bd1c
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162b3) -> 00007fd77b89c550
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162b2) -> 0000000000000000
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162b1) -> 00007fd77bb27a80
1:10:S 01 Oct 2022 18:27:10.871 # (00007fd7838162b0) -> 00007fd77b89c580

------ INFO OUTPUT ------
# Server
redis_version:6.3.1
redis_git_sha1:ee16abf0
redis_git_dirty:1
redis_build_id:46da54e1e1000581
redis_mode:standalone
os:Linux 5.4.0-126-generic x86_64
arch_bits:64
multiplexing_api:epoll
atomicvar_api:atomic-builtin
gcc_version:7.5.0
process_id:1
process_supervised:no
run_id:c24921069ffc3222189779f51dfeeb49059ecfea
tcp_port:6379
server_time_usec:1664648830869844
uptime_in_seconds:1
uptime_in_days:0
hz:10
configured_hz:10
lru_clock:3704446
executable:/data/keydb-server
config_file:/etc/keydb/redis.conf

# Clients
connected_clients:1
cluster_connections:0
maxclients:10000
client_recent_max_input_buffer:8
client_recent_max_output_buffer:0
blocked_clients:0
tracking_clients:0
clients_in_timeout_table:0
current_client_thread:-1150477577
thread_0_clients:1
thread_1_clients:0

# Memory
used_memory:18620528
used_memory_human:17.76M
used_memory_rss:66523136
used_memory_rss_human:63.44M
used_memory_peak:18620528
used_memory_peak_human:17.76M
used_memory_peak_perc:100.01%
used_memory_overhead:11833344
used_memory_startup:10759048
used_memory_dataset:6787184
used_memory_dataset_perc:86.33%
allocator_allocated:19032688
allocator_active:20516864
allocator_resident:53829632
total_system_memory:8296624128
total_system_memory_human:7.73G
used_memory_lua:37888
used_memory_lua_human:37.00K
used_memory_scripts:0
used_memory_scripts_human:0B
number_of_cached_scripts:0
maxmemory:7340032000
maxmemory_human:6.84G
maxmemory_policy:allkeys-random
allocator_frag_ratio:1.08
allocator_frag_bytes:1484176
allocator_rss_ratio:2.62
allocator_rss_bytes:33312768
rss_overhead_ratio:1.24
rss_overhead_bytes:12693504
mem_fragmentation_ratio:3.57
mem_fragmentation_bytes:47903984
mem_not_counted_for_evict:1048576
mem_replication_backlog:1048576
mem_clients_slaves:0
mem_clients_normal:20488
mem_aof_buffer:0
mem_allocator:jemalloc-5.2.1
active_defrag_running:0
lazyfree_pending_objects:0
lazyfreed_objects:0
storage_provider:none

# Persistence
loading:0
current_cow_size:0
current_cow_size_age:0
current_fork_perc:0.00
current_save_keys_processed:0
current_save_keys_total:0
rdb_changes_since_last_save:0
rdb_bgsave_in_progress:0
rdb_last_save_time:1664648829
rdb_last_bgsave_status:ok
rdb_last_bgsave_time_sec:-1
rdb_current_bgsave_time_sec:-1
rdb_last_cow_size:0
aof_enabled:0
aof_rewrite_in_progress:0
aof_rewrite_scheduled:0
aof_last_rewrite_time_sec:-1
aof_current_rewrite_time_sec:-1
aof_last_bgrewrite_status:ok
aof_last_write_status:ok
aof_last_cow_size:0
module_fork_in_progress:0
module_fork_last_cow_size:0

# Stats
total_connections_received:0
total_commands_processed:2
instantaneous_ops_per_sec:0
total_net_input_bytes:233047
total_net_output_bytes:0
instantaneous_input_kbps:34.11
instantaneous_output_kbps:0.00
rejected_connections:0
sync_full:0
sync_partial_ok:0
sync_partial_err:0
expired_keys:0
expired_stale_perc:0.00
expired_time_cap_reached_count:0
expire_cycle_cpu_milliseconds:0
evicted_keys:0
keyspace_hits:0
keyspace_misses:0
pubsub_channels:0
pubsub_patterns:0
latest_fork_usec:0
total_forks:0
migrate_cached_sockets:0
slave_expires_tracked_keys:0
active_defrag_hits:0
active_defrag_misses:0
active_defrag_key_hits:0
active_defrag_key_misses:0
tracking_total_keys:0
tracking_total_items:0
tracking_total_prefixes:0
unexpected_error_replies:0
total_error_replies:0
dump_payload_sanitizations:0
total_reads_processed:0
total_writes_processed:0
instantaneous_lock_contention:0
avg_lock_contention:0.046875
storage_provider_read_hits:0
storage_provider_read_misses:0

# Replication
role:active-replica
master_global_link_status:down
connected_masters:1
master_host:keydb-2.keydb-headless
master_port:6379
master_link_status:down
master_last_io_seconds_ago:-1
master_sync_in_progress:0
slave_read_repl_offset:1
slave_repl_offset:1
master_link_down_since_seconds:-1
master_1_host:keydb-0.keydb-headless
master_1_port:6379
master_1_link_status:up
master_1_last_io_seconds_ago:0
master_1_sync_in_progress:0
slave_read_repl_offset:1547888
slave_repl_offset:1547888
slave_priority:100
slave_read_only:0
replica_announced:1
connected_slaves:0
master_failover_state:no-failover
master_replid:83e0a89f7f33d393a0d2d041c37de4e702b67308
master_replid2:0000000000000000000000000000000000000000
master_repl_offset:496099
second_repl_offset:-1
repl_backlog_active:1
repl_backlog_size:1048576
repl_backlog_first_byte_offset:496100
repl_backlog_histlen:0

# CPU
used_cpu_sys:0.041853
used_cpu_user:0.991078
used_cpu_sys_children:0.000615
used_cpu_user_children:0.000000
server_threads:2
long_lock_waits:0
used_cpu_sys_main_thread:0.000000
used_cpu_user_main_thread:0.001915

# Modules
module:name=rg,ver=10205,api=1,filters=1,usedby=[rg],using=[rg],options=[]

# Commandstats
cmdstat_info:calls=1,usec=14,usec_per_call=14.00,rejected_calls=0,failed_calls=0
cmdstat_config:calls=1,usec=9,usec_per_call=9.00,rejected_calls=0,failed_calls=0

# Errorstats

# Cluster
cluster_enabled:0

# Keyspace
db0:keys=105,expires=0,avg_ttl=0,cached_keys=105

# KeyDB
mvcc_depth:0

------ CLIENT LIST OUTPUT ------
id=13 addr=10.42.7.221:6379 laddr=10.42.14.76:35895 fd=26 name= age=0 idle=0 flags=M db=0 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=4 argv-mem=0 obl=0 oll=0 omem=0 tot-mem=20488 events=r cmd=NULL user=(superuser) redir=-1

Additional information

  1. The docker image was built using x86_64_v6.3.1

artursmolarek avatar Oct 01 '22 18:10 artursmolarek