kamailio Service crashed for IMS VoLTE MT call

Description

Troubleshooting

Reproduction

Debugging Data

root@imstester04:~/core_dump_test# gdb kamailio core._usr_local_sbin_kamailio.0.d660bb15-1623-4c0c-ae29-64c78dc89d8c.3799655.792169952
GNU gdb (Ubuntu 9.2-0ubuntu1~20.04.2) 9.2
Copyright (C) 2020 Free Software Foundation, Inc.
License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Type "show copying" and "show warranty" for details.
This GDB was configured as "x86_64-linux-gnu".
Type "show configuration" for configuration details.
For bug reporting instructions, please see:
<http://www.gnu.org/software/gdb/bugs/>.
Find the GDB manual and other documentation resources online at:
    <http://www.gnu.org/software/gdb/documentation/>.

For help, type "help".
Type "apropos word" to search for commands related to "word"...
Reading symbols from kamailio...
[New LWP 3799655]
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".
Core was generated by `/usr/local/sbin/kamailio -P /var/run/kamailio_pcscf/kamailio_pcscf.pid -f /etc/'.
Program terminated with signal SIGABRT, Aborted.
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
50      ../sysdeps/unix/sysv/linux/raise.c: No such file or directory.
(gdb) bt full
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
        set = {__val = {0, 19989655252, 140370540584975, 1725509713, 94706844523408, 341, 64424509453, 21474836492, 532575944712, 1065151889412, 140368121167872, 28800, 94706843944736, 560, 71148340464, 0}}
        pid = <optimized out>
        tid = <optimized out>
        ret = <optimized out>
#1  0x00007faa94995859 in __GI_abort () at abort.c:79
        save_stage = 1
        act = {__sigaction_handler = {sa_handler = 0x0, sa_sigaction = 0x0}, sa_mask = {__val = {0, 140366190491840, 140366190223360, 0, 0, 0, 0, 0, 94706838593952, 94706840315664, 94706840316072,
              140366318452096, 2001317716380379136, 0, 94706840315898, 0}}, sa_flags = -1483647068, sa_restorer = 0x7faa94a8b410 <__syslog>}
        sigs = {__val = {32, 0 <repeats 15 times>}}
#2  0x00005622a778e1c7 in qm_debug_check_frag (qm=0x7fa98ce82000, f=0x7fa9948cbd80, file=0x7faa908fc84e "rom %s:%u\n", line=171, efile=0x5622a7932729 "core/mem/q_malloc.c", eline=546)
    at core/mem/q_malloc.c:143
        p = 0x0
        __func__ = "qm_debug_check_frag"
#3  0x00005622a77923f8 in qm_free (qmp=0x7fa98ce82000, p=0x7fa9948cbdc0, file=0x7faa908fc84e "rom %s:%u\n", func=0x7faa908fc460 "%s%s%sadd_blind failed\n", line=171, mname=0x7faa908fc28e "")
    at core/mem/q_malloc.c:546
        qm = 0x7fa98ce82000
        f = 0x7fa9948cbd80
        size = 0
        next = 0x0
        prev = 0x0
        __func__ = "qm_free"
#4  0x00007faa907e3749 in free_cell_helper (dead_cell=0x7fa994875c90, silent=0, fname=0x7faa90915362 <__func__.11447+2> "_init_timers", fline=651) at h_table.c:126
        b = 0x0
        i = 0
        rpl = 0x7ffca5bc65e0
        tt = 0x1
        foo = 0x0
        cbs = 0x7fa98cfb76d0
        cbs_tmp = 0x7faa907e1041 <t_write_req+481>
        __func__ = "free_cell_helper"
#5  0x00007faa9089f28f in wait_handler (ti=872968975, wait_tl=0x7fa994875d48, data=0x7fa994875c90) at timer.c:651
        p_cell = 0x7fa994875c90
        ret = 0
        unlinked = 0
        rcount = 1
        __func__ = "wait_handler"
#6  0x00005622a7522813 in timer_list_expire (t=872968975, h=0x7fa98cf0a950, slow_l=0x7fa98cf0dd50, slow_mark=6936) at core/timer.c:861
        tl = 0x7fa994875d48
        ret = 0
#7  0x00005622a7522d52 in timer_handler () at core/timer.c:925
        saved_ticks = 872968975
        run_slow_timer = 0
        i = 792
        __func__ = "timer_handler"
#8  0x00005622a7523280 in timer_main () at core/timer.c:963
No locals.
#9  0x00005622a744501a in main_loop () at main.c:1932
        i = 64
        pid = 0
        si = 0x0
        si_desc = "udp receiver child=63 sock=10.51.0.10:20004\000\324\002\000\000\003\000\000\000\345\002\000\000\000h\214\017\334\033\306\033\000\000\000\000\000\000\000\000\210��\247\"V\000\000\000\000\000\000\000\000\000\000\303W\221\247\"V\000\000\345\002\000\000\000\000\000\000\020\264\250\224\252\177\000\000\340i\274\245\374\177\000\000\061\237_\247\"V\000"
        nrprocs = 64
        woneinit = 1
        __func__ = "main_loop"
#10 0x00005622a7450425 in main (argc=13, argv=0x7ffca5bc6f48) at main.c:3257
        cfg_stream = 0x5622a7b512a0
--Type <RET> for more, q to quit, c to continue without paging--c
        c = -1
        r = 0
        tmp = 0x7ffca5bc7e58 ""
        tmp_len = 32682
        port = 5060
        proto = 0
        aproto = 0
        ahost = 0x0
        aport = 0
        options = 0x5622a78c0368 ":f:cm:M:dVIhEeb:B:l:L:n:vKrRDTN:W:w:t:u:g:P:G:SQ:O:a:A:x:X:Y:"
        ret = -1
        seed = 3374692201
        rfd = 4
        debug_save = 0
        debug_flag = 0
        dont_fork_cnt = 0
        n_lst = 0x0
        p = 0x7faa94cfa410 ""
        st = {st_dev = 25, st_ino = 754, st_nlink = 2, st_mode = 16832, st_uid = 0, st_gid = 0, __pad0 = 0, st_rdev = 0, st_size = 40, st_blksize = 4096, st_blocks = 0, st_atim = {tv_sec = 1717582113, tv_nsec = 155483622}, st_mtim = {tv_sec = 1717582113, tv_nsec = 155483622}, st_ctim = {tv_sec = 1717582113, tv_nsec = 155483622}, __glibc_reserved = {0, 0, 0}}
        l1 = 4096
        tbuf = "\020n\274\245\374\177\000\000\273\327��\252\177", '\000' <repeats 18 times>, "��\277X��\377\377\000\000\000\000\000\000\000\000\004\000\000\000\000\000\000\000\001\000\000\000\000\000\000\000\000\207��\252\177\000\000@#\227\224\252\177\000\000`\201��\252\177\000\000\001l\274\245\374\177\000\000\350y��\252\177", '\000' <repeats 26 times>, "\220\274\266\224\252\177\000\000\004\000\000\024\000\000\000\000 \224\267\224\252\177", '\000' <repeats 130 times>...
        option_index = 0
        long_options = {{name = 0x5622a78c28f6 "help", has_arg = 0, flag = 0x0, val = 104}, {name = 0x5622a78bd54c "version", has_arg = 0, flag = 0x0, val = 118}, {name = 0x5622a78c28fb "alias", has_arg = 1, flag = 0x0, val = 1024}, {name = 0x5622a78c2901 "subst", has_arg = 1, flag = 0x0, val = 1025}, {name = 0x5622a78c2907 "substdef", has_arg = 1, flag = 0x0, val = 1026}, {name = 0x5622a78c2910 "substdefs", has_arg = 1, flag = 0x0, val = 1027}, {name = 0x5622a78c291a "server-id", has_arg = 1, flag = 0x0, val = 1028}, {name = 0x5622a78c2924 "loadmodule", has_arg = 1, flag = 0x0, val = 1029}, {name = 0x5622a78c292f "modparam", has_arg = 1, flag = 0x0, val = 1030}, {name = 0x5622a78c2938 "log-engine", has_arg = 1, flag = 0x0, val = 1031}, {name = 0x5622a78c2943 "debug", has_arg = 1, flag = 0x0, val = 1032}, {name = 0x5622a78c2949 "cfg-print", has_arg = 0, flag = 0x0, val = 1033}, {name = 0x5622a78c2953 "atexit", has_arg = 1, flag = 0x0, val = 1034}, {name = 0x5622a78c295a "all-errors", has_arg = 0, flag = 0x0, val = 1035}, {name = 0x0, has_arg = 0, flag = 0x0, val = 0}}
        __func__ = "main"
(gdb)
(gdb)
(gdb)

Log Messages

SIP Traffic

Possible Solutions

Additional Information

Kamailio Version - output of kamailio -v

`Kamailio release 5.8.2``


* **Operating System**:

<!--
Details about the operating system, the type: Linux (e.g.,: Debian 8.4, Ubuntu 16.04, CentOS 7.1, ...), MacOS, xBSD, Solaris, ...;
Kernel details (output of `lsb_release -a` and `uname -a`)
-->

(paste your output here)

Sep 12 '24 07:09 citicims

Thanks for the report. Does the crash happened only once, or do you observe it frequently?

Sep 16 '24 13:09 henningw

The crash is not happening once only, we observe that it cannot run for more than 1 or 2 hours, and crash within 1 or 2 hours

Sep 17 '24 01:09 citicims

This issue is stale because it has been open 6 weeks with no activity. Remove stale label or comment or this will be closed in 2 weeks.

Oct 29 '24 03:10 github-actions[bot]

I didnt receive any further action or finding from support. Please help!

Oct 29 '24 03:10 citicims

Dear Support, please provide solution or finding for helping this issue rather than let it to close. Thx!

Nov 13 '24 03:11 citicims

This support is provided from the open source project. If you don't receive a reply, it could mean that nobody know about this particular problem or just that nobody found time yet to look into it. If you are looking for commercial support options, please have a look to the business directory.

Nov 13 '24 09:11 henningw

This issue is stale because it has been open 6 weeks with no activity. Remove stale label or comment or this will be closed in 2 weeks.

Dec 26 '24 03:12 github-actions[bot]