rspamd
rspamd copied to clipboard
Published
20 hours ago •
rspamd
rspamd
[BUG] Many crashes
Describe the bug
2023-09-27T08:50:32.769475+00:00 mail1.example.com rspamd[1263707]: <f3a935>; lua; once_received.lua:52: error looking up 222.60.120.89.in-addr.arpa: server fail
2023-09-27T08:57:47.607169+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_crash_sig_handler: caught fatal signal 11(Segmentation fault), pid: 1263707, trace:
2023-09-27T08:57:47.607381+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 0: 00007FAA420476A4: _ZN6rspamd8symcache16symcache_runtime18process_item_rdepsEP11rspamd_taskPNS0_10cache_itemE()+0x4
2023-09-27T08:57:47.607709+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 1: 00007FAA41D4D436: ev_invoke_pending()+0x56
2023-09-27T08:57:47.608064+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 2: 00007FAA41D521DD: ev_run()+0x7cd
2023-09-27T08:57:47.608222+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 3: 0000560003D53263: <unknown>
2023-09-27T08:57:47.608678+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 4: 00007FAA42034B85: rspamd_fork_worker()+0x7c5
2023-09-27T08:57:47.608823+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 5: 0000560003D588B4: <unknown>
2023-09-27T08:57:47.609172+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 6: 00007FAA41D4D436: ev_invoke_pending()+0x56
2023-09-27T08:57:47.609629+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 7: 00007FAA41D521DD: ev_run()+0x7cd
2023-09-27T08:57:47.609786+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 8: 0000560003D5139F: <unknown>
2023-09-27T08:57:47.610146+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 9: 00007FAA41755083: __libc_start_main()+0xf3
2023-09-27T08:57:47.610301+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 10: 0000560003D518AE: <unknown>
2023-09-27T08:57:47.610386+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_crash_sig_handler: please see Rspamd FAQ to learn how to dump core files and how to fill a bug report
2023-09-27T08:57:47.615434+00:00 mail1.example.com rspamd[152]: <>; ; rspamd_srv_handler: cannot read from worker's srv pipe connection closed; command = heartbeat
2023-09-27T08:57:47.615544+00:00 mail1.example.com rspamd[152]: <7b628b>; main; rspamd_check_termination_clause: rspamd_proxy process 1263707 terminated abnormally with exit code 0 by signal: Segmentation fault but NOT created core file (throttled=no); core file limits: 0 current,
2023-09-27T08:57:47.606726+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_crash_sig_handler: caught fatal signal 11(Segmentation fault), pid: 1263707, trace:
2023-09-27T08:57:47.607351+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 0: 00007FAA420476A4: _ZN6rspamd8symcache16symcache_runtime18process_item_rdepsEP11rspamd_taskPNS0_10cache_itemE()+0x4
2023-09-27T08:57:47.607659+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 1: 00007FAA41D4D436: ev_invoke_pending()+0x56
2023-09-27T08:57:47.608015+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 2: 00007FAA41D521DD: ev_run()+0x7cd
2023-09-27T08:57:47.608194+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 3: 0000560003D53263: <unknown>
2023-09-27T08:57:47.608626+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 4: 00007FAA42034B85: rspamd_fork_worker()+0x7c5
2023-09-27T08:57:47.608796+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 5: 0000560003D588B4: <unknown>
2023-09-27T08:57:47.609148+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 6: 00007FAA41D4D436: ev_invoke_pending()+0x56
2023-09-27T08:57:47.609573+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 7: 00007FAA41D521DD: ev_run()+0x7cd
2023-09-27T08:57:47.609749+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 8: 0000560003D5139F: <unknown>
2023-09-27T08:57:47.610094+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 9: 00007FAA41755083: __libc_start_main()+0xf3
2023-09-27T08:57:47.610276+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_print_crash: 10: 0000560003D518AE: <unknown>
2023-09-27T08:57:47.610364+00:00 mail1.example.com rspamd[1263707]: <>; ; rspamd_crash_sig_handler: please see Rspamd FAQ to learn how to dump core files and how to fill a bug report
2023-09-27T08:57:47.615109+00:00 mail1.example.com rspamd[152]: <>; ; rspamd_srv_handler: cannot read from worker's srv pipe connection closed; command = heartbeat
2023-09-27T08:57:47.615518+00:00 mail1.example.com rspamd[152]: <7b628b>; main; rspamd_check_termination_clause: rspamd_proxy process 1263707 terminated abnormally with exit code 0 by signal: Segmentation fault but NOT created core file (throttled=no); core file limits: 0 current,
2023-09-27T08:57:49.619061+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_redis_init: cannot init redis backend for BAYES_SPAM
2023-09-27T08:57:49.619184+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_stat_cache_redis_init: cannot init redis cache for BAYES_SPAM
2023-09-27T08:57:49.619240+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_stat_init: error adding cache redis for symbol BAYES_SPAM
2023-09-27T08:57:49.619284+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_stat_init: cannot init backend redis for statfile BAYES_SPAM
2023-09-27T08:57:49.619366+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_redis_init: cannot init redis backend for BAYES_HAM
2023-09-27T08:57:49.619412+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_stat_cache_redis_init: cannot init redis cache for BAYES_HAM
2023-09-27T08:57:49.619454+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_stat_init: error adding cache redis for symbol BAYES_HAM
2023-09-27T08:57:49.619495+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_stat_init: cannot init backend redis for statfile BAYES_HAM
2023-09-27T08:57:49.618737+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_redis_init: cannot init redis backend for BAYES_SPAM
2023-09-27T08:57:49.619170+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_stat_cache_redis_init: cannot init redis cache for BAYES_SPAM
2023-09-27T08:57:49.619228+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_stat_init: error adding cache redis for symbol BAYES_SPAM
2023-09-27T08:57:49.619273+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_stat_init: cannot init backend redis for statfile BAYES_SPAM
2023-09-27T08:57:49.619354+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_redis_init: cannot init redis backend for BAYES_HAM
2023-09-27T08:57:49.619401+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_stat_cache_redis_init: cannot init redis cache for BAYES_HAM
2023-09-27T08:57:49.619443+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_stat_init: error adding cache redis for symbol BAYES_HAM
2023-09-27T08:57:49.619484+00:00 mail1.example.com rspamd[1280437]: <hr3nxr>; cfg; rspamd_stat_init: cannot init backend redis for statfile BAYES_HAM
2023-09-27T09:22:36.534457+00:00 mail1.example.com rspamd[1280437]: <0980d5>; lua; once_received.lua:52: error looking up 222.60.120.89.in-addr.arpa: server fail
2023-09-27T09:22:36.534182+00:00 mail1.example.com rspamd[1280437]: <0980d5>; lua; once_received.lua:52: error looking up 222.60.120.89.in-addr.arpa: server fail
2023-09-27T09:25:07.486098+00:00 mail1.example.com rspamd[1280437]: <0f25b9>; lua; once_received.lua:52: error looking up 222.60.120.89.in-addr.arpa: server fail
Expected behavior Should not crash.
Versions
dpkg -l | grep rspamd
ii rspamd 3.6-2~bbb8660e5~focal amd64 Rapid spam filtering system
Please check this entry in the FAQ: https://rspamd.com/doc/faq.html#how-to-figure-out-why-rspamd-process-crashed
Hello,
this is the call stack:
Reading symbols from /usr/bin/rspamd...
Reading symbols from /usr/lib/debug/.build-id/ea/b9441b7c1ba7a66b631253a29a7f1fddbd1175.debug...
[New LWP 2197]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `rspamd: rspamd_proxy process (localhost:11332)'.
Program terminated with signal SIGSEGV, Segmentation fault.
#0 rspamd::symcache::symcache_runtime::process_item_rdeps (this=0x0, task=0x0, item=0x0) at ./src/libserver/symcache/symcache_runtime.cxx:785
785 ./src/libserver/symcache/symcache_runtime.cxx: No such file or directory.
(gdb) bt
#0 rspamd::symcache::symcache_runtime::process_item_rdeps (this=0x0, task=0x0, item=0x0) at ./src/libserver/symcache/symcache_runtime.cxx:785
#1 0x00007fe787cbc436 in ev_invoke_pending (loop=0x7fe783e40200) at ./contrib/libev/ev.c:3807
#2 0x00007fe787cc11dd in ev_run (loop=0x7fe783e40200, flags=0) at ./contrib/libev/ev.c:4229
#3 0x00005596b4991263 in ev_loop (flags=0, loop=<optimized out>) at ./contrib/libev/ev.h:830
#4 start_rspamd_proxy (worker=0x7fe783f58dc0) at ./src/rspamd_proxy.c:2411
#5 0x00007fe787fa3b85 in rspamd_handle_child_fork (listen_sockets=<optimized out>, cf=0x7fe783f4a280, rspamd_main=0x7fe783f1c140, wrk=0x7fe783f58dc0) at ./src/libserver/worker_util.c:1185
#6 rspamd_fork_worker (rspamd_main=0x7fe783f1c140, cf=0x7fe783f4a280, index=<optimized out>, ev_base=<optimized out>, term_handler=<optimized out>, listen_sockets=<optimized out>) at ./src/libserver/worker_util.c:1302
#7 0x00005596b499699d in spawn_worker_type (rspamd_main=0x7fe783f1c140, event_loop=0x7fe787cc60a0 <default_loop_struct>, cf=0x7fe783f4a280) at ./src/rspamd.c:631
#8 0x00005596b4996be2 in spawn_workers (rspamd_main=0x7fe783f1c140, ev_base=0x7fe787cc60a0 <default_loop_struct>) at ./src/rspamd.c:734
#9 0x00005596b498f2e1 in main (argc=<optimized out>, argv=<optimized out>, env=<optimized out>) at ./src/rspamd.c:1676
(gdb) info locals
cache_ptr = <error reading variable cache_ptr (Cannot access memory at address 0xf8)>
__func__ = <optimized out>
(gdb) info registers
rax 0x0 0
rbx 0x7fe783e40200 140632326930944
rcx 0x34 52
rdx 0x0 0
rsi 0x0 0
rdi 0x0 0
rbp 0x0 0x0
rsp 0x7ffc639e45a8 0x7ffc639e45a8
r8 0x7fe788503130 140632401129776
r9 0xa0 160
r10 0x7fe7840008c0 140632328767680
r11 0x1a0 416
r12 0x0 0
r13 0x0 0
r14 0x7fe783e40200 140632326930944
r15 0x0 0
rip 0x7fe787fb66a4 0x7fe787fb66a4 <rspamd::symcache::symcache_runtime::process_item_rdeps(rspamd_task*, rspamd::symcache::cache_item*)+4>
eflags 0x10202 [ IF RF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x0 0
(gdb) up
#1 0x00007fe787cbc436 in ev_invoke_pending (loop=0x7fe783e40200) at ./contrib/libev/ev.c:3807
3807 ./contrib/libev/ev.c: No such file or directory.
(gdb) info locals
p = <optimized out>
(gdb) info registers
rax 0x0 0
rbx 0x7fe783e40200 140632326930944
rcx 0x34 52
rdx 0x0 0
rsi 0x0 0
rdi 0x0 0
rbp 0x0 0x0
rsp 0x7ffc639e45b0 0x7ffc639e45b0
r8 0x7fe788503130 140632401129776
r9 0xa0 160
r10 0x7fe7840008c0 140632328767680
r11 0x1a0 416
r12 0x0 0
r13 0x0 0
r14 0x7fe783e40200 140632326930944
r15 0x0 0
rip 0x7fe787cbc436 0x7fe787cbc436 <ev_invoke_pending+86>
eflags 0x10202 [ IF RF ]
cs 0x33 51
ss 0x2b 43
ds 0x0 0
es 0x0 0
fs 0x0 0
gs 0x0 0
Do you need more info ?
Thank you.
It looks like it is related to some of your custom lua code that has some issues. I cannot say more I'm afraid.
I upgraded to the latest version:
# dpkg -l | grep rspamd
ii rspamd 3.7.4-1~a07208b9f~focal amd64 Rapid spam filtering system
ii rspamd-dbg 3.7.4-1~a07208b9f~focal amd64 debugging symbols for rspamd
Remade the entire configuration and it is still crashing.
I'm using rspamd only for DKIM and my configuration files are (all others are unmodified):
### cat actions.conf ###
reject = null;
### cat logging.inc ###
type = "syslog";
facility = "LOG_DAEMON";
level = "warning";
#debug_modules = ["dkim", "milter"];
### cat milter_headers.conf ###
#extended_spam_headers = true;
#authenticated_headers = ["authentication-results"];
#use = ["x-spamd-bar", "x-spam-level", "authentication-results"];
#use = ["x-spam-level", "authentication-results", "remove-headers"];
use = ["x-spam-level", "authentication-results"];
#routines {
# remove-headers {
# headers {
# "User-Agent" = 0;
# "Received" = 0;
# "X-Originating-IP" = 0;
# "X-Mailer" = 0;
# }
# }
#}
### cat worker-controller.inc ###
password = "$2$...";
enable_password = "$2$...";
secure_ip = "";
### cat worker-normal.inc ###
enabled = false;
### cat worker-proxy.inc ###
upstream "local" {
default = yes;
self_scan = yes;
}
Thanks.