abort: quic_bugcheck in msquic 2.3.5

[qzhuyan]

Describe the bug

(gdb) bt full
#0  __pthread_kill_implementation (threadid=281472009564288, signo=signo@entry=6, no_tid=no_tid@entry=0) at ./nptl/pthread_kill.c:44
        tid = 457805
        ret = 0
        pd = 0xffff4f24f080
        old_mask = {__val = {281472878291084, 1102416563, 281472051204736, 281472048519040, 281472046898844, 281470726134240, 68719476736, 281470726133888, 281470726133920, 281470726134256, 281470726134248, 281472009560736, 281472047985312, 281470781056112, 281472051154944, 281472009558608}}
        ret = <optimized out>
#1  0x0000ffff82f0f254 in __pthread_kill_internal (signo=6, threadid=<optimized out>) at ./nptl/pthread_kill.c:78
No locals.
#2  0x0000ffff82eca67c in __GI_raise (sig=sig@entry=6) at ../sysdeps/posix/raise.c:26
        ret = <optimized out>
#3  0x0000ffff82eb7130 in __GI_abort () at ./stdlib/abort.c:79
        save_stage = 1
        act = {__sigaction_handler = {sa_handler = 0xffff835a0000 <__asan::AsanBuggyPcPool+120>, sa_sigaction = 0xffff835a0000 <__asan::AsanBuggyPcPool+120>}, sa_mask = {__val = {18446744073709547520, 281472658317312, 281472886308384, 281472884235020, 281472884235096, 281472884234868, 18150945922037014272, 281472051008640, 281472051009184, 1, 281470658482680, 281472046536976,
              18150945922037014272, 281472051008640, 281472009555968, 1}}, sa_flags = -23260680, sa_restorer = 0xffff515aeea8 <CxPlatSocketContextUninitialize+1420>}
        sigs = {__val = {32, 68719476736, 35183843042613, 281470744340904, 281470744340904, 281472051017152, 281472885456896, 281472051009184, 1, 281470658482680, 281472009561200, 281472884210692, 281472885456896, 18150945922037014272, 281472885456896, 281472884234856}}
#4  0x0000ffff51591914 in quic_bugcheck (File=0xffff519d5480 "/home/ubuntu/repo/quic/msquic/src/platform/datapath_epoll.c", Line=1142, Expr=0xffff519d56a0 "CxPlatEventQEnqueue( SocketContext->DatapathPartition->EventQ, &SocketContext->ShutdownSqe.Sqe, &SocketContext->ShutdownSqe)") at /home/ubuntu/repo/quic/msquic/src/platform/platform_posix.c:93
No locals.
#5  0x0000ffff515aeeb8 in CxPlatSocketContextUninitialize (SocketContext=<optimized out>) at /home/ubuntu/repo/quic/msquic/src/platform/datapath_epoll.c:1142
No locals.
#6  0x0000ffff515adc20 in SocketDelete (Socket=0xffff1aebd040) at /home/ubuntu/repo/quic/msquic/src/platform/datapath_epoll.c:1598
        i = 0
        SocketCount = <optimized out>
        __head = <optimized out>
#7  0x0000ffff515970e0 in CxPlatSocketDelete (Socket=<optimized out>) at /home/ubuntu/repo/quic/msquic/src/platform/datapath_linux.c:286
No locals.
#8  0x0000ffff51489024 in QuicBindingUninitialize (Binding=0xffff62614040) at /home/ubuntu/repo/quic/msquic/src/core/binding.c:205
        __head = <optimized out>
        __head = <optimized out>
#9  0x0000ffff51417200 in QuicLibraryReleaseBinding (Binding=0xffff62614040) at /home/ubuntu/repo/quic/msquic/src/core/library.c:2151
        Uninitialize = 1 '\001'
        __head = <optimized out>
#10 0x0000ffff5149c07c in QuicConnFree (Connection=0xfffefe9d0d00) at /home/ubuntu/repo/quic/msquic/src/core/connection.c:365
        Path = 0xfffefe9d0e30
        __head = <optimized out>
#11 0x0000ffff5146f190 in QuicConnRelease (Ref=<optimized out>, Connection=<optimized out>) at /home/ubuntu/repo/quic/msquic/src/core/connection.h:1085
No locals.
#12 QuicWorkerProcessConnection (Worker=0xffff756955f0, Connection=0xfffefe9d0d00, ThreadID=457805, TimeNow=<optimized out>) at /home/ubuntu/repo/quic/msquic/src/core/worker.c:558
        StillHasWorkToDo = <optimized out>
        DoneWithConnection = 1 '\001'
        __head = <optimized out>
        __head = <optimized out>
        __head = <optimized out>
#13 0x0000ffff5146fcfc in QuicWorkerLoop (Context=0xffff756955f0, State=0xffff4f24e740) at /home/ubuntu/repo/quic/msquic/src/core/worker.c:658
        Worker = <optimized out>
        Connection = 0xfffefe9d0d00
        Operation = <optimized out>
#14 0x0000ffff51588e28 in CxPlatRunExecutionContexts (Worker=<optimized out>, State=0xffff4f24e740) at /home/ubuntu/repo/quic/msquic/src/platform/platform_worker.c:395
        Next = 0x0
        Context = 0xffff756955f0
        Ready = <optimized out>
        NextTime = 18446744073709551615
        EC = 0xffff756b35f0
#15 0x0000ffff51589cb0 in CxPlatWorkerThread (Context=0xffff5f712c30) at /home/ubuntu/repo/quic/msquic/src/platform/platform_worker.c:492
        Worker = <optimized out>
        State = {TimeNow = 364643558352, LastWorkTime = 364640543362, WaitTime = 2993, NoWorkCount = 0, ThreadID = 457805}
        Shutdown = <optimized out>
        __head = <optimized out>
        __head = <optimized out>
#16 0x0000ffff82f0d5c8 in start_thread (arg=0x0) at ./nptl/pthread_create.c:442
        ret = <optimized out>
        pd = 0x0
        out = <optimized out>
        unwind_buf = {cancel_jmp_buf = {{jmp_buf = {281472009564288, 281472629971952, 281472629969358, 8448192, 281472629969359, 0, 281472001114112, 8448192, 281472629993600, 281472001114112, 281472009562016, 16510817446987276462, 0, 16510817447862571794, 4637880783447523328, 0, 0, 0, 0, 0, 0, 0}, mask_was_saved = 0}}, priv = {pad = {0x0, 0x0, 0x0, 0x0}, data = {prev = 0x0,
              cleanup = 0x0, canceltype = 0}}}
        not_first_call = 0
#17 0x0000ffff82f75edc in thread_start () at ../sysdeps/unix/sysv/linux/aarch64/clone.S:79
No locals

Affected OS

• [ ] Windows
• [X] Linux
• [ ] macOS
• [ ] Other (specify below)

Additional OS information

ubuntu22.04 ARM64

MsQuic version

v2.3

Steps taken to reproduce bug

Run property based test + ASAN check, happened 2 times out of ~18000 run.

Test just doing random send data over stream, create/shutdown streams, create/close connection, new stream over a connection at random timing.

Expected behavior

Test should pass without abortion

Actual outcome

Test failed due to quic_bugcheck abortion.

Additional details

No such issue with msquic 2.2.3. Maybe it is fault at caller like use after close the handle? Will try to collect logs

[qzhuyan]

msquic.log.gz log attached

correct me if I am wrong, I don't see the application double close the connection, so I think it is indeed a bug in Msquic?