bdwgc icon indicating copy to clipboard operation
bdwgc copied to clipboard
verified publisher icon ivmai

Clang address sanitizer crashes when used with `-DREDIRECT_MALLOC=GC_malloc_uncollectable`

Open jeaye opened this issue 2 months ago • 6 comments

Hey Ivan! I've found in jank that I can't combine ASan and -DREDIRECT_MALLOC=GC_malloc_uncollectable. I end up getting a crash prior to the entry point of my program (back trace included below).

I've scanned through the issues and docs and didn't find anything suggesting that these two can or can't be used together, so primarily I'd like to check with you if this is considered a bug or an expected incompatibility.

(gdb) bt
#0  0x0000000000000000 in ?? ()
#1  0x00005555558cb0ca in ___interceptor_pthread_setcancelstate ()
    at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/asan/../sanitizer_common/sanitizer_common_interceptors.inc:6811
#2  0x0000555557be3360 in GC_init () at /home/jeaye/projects/jank/compiler+runtime/third-party/bdwgc/misc.c:1066
#3  0x0000555557bdb908 in GC_generic_malloc_inner_small (lb=87, kind=2) at /home/jeaye/projects/jank/compiler+runtime/third-party/bdwgc/malloc.c:202
#4  0x0000555557bdc54f in GC_generic_malloc_uncollectable (lb=87, kind=2) at /home/jeaye/projects/jank/compiler+runtime/third-party/bdwgc/malloc.c:463
#5  0x0000555557bdc67a in GC_malloc_uncollectable (lb=88) at /home/jeaye/projects/jank/compiler+runtime/third-party/bdwgc/malloc.c:506
#6  0x0000555557bdc6b5 in malloc (lb=88) at /home/jeaye/projects/jank/compiler+runtime/third-party/bdwgc/malloc.c:553
#7  0x00007ffff7fc6b72 in _dl_exception_create_format () from /lib64/ld-linux-x86-64.so.2
#8  0x00007ffff7fcdb6c in ?? () from /lib64/ld-linux-x86-64.so.2
#9  0x00007fffee8c3086 in ?? () from /usr/lib/libc.so.6
#10 0x00007fffee7ebe00 in ?? () from /usr/lib/libc.so.6
#11 0x00007ffff7fc4456 in _dl_catch_exception () from /lib64/ld-linux-x86-64.so.2
#12 0x00007ffff7fc45a9 in ?? () from /lib64/ld-linux-x86-64.so.2
#13 0x00007fffee7eb7f3 in ?? () from /usr/lib/libc.so.6
#14 0x00007fffee7ebe9e in dlsym () from /usr/lib/libc.so.6
#15 0x0000555555918334 in GetFuncAddr () at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/interception/interception_linux.cpp:42
#16 InterceptFunction () at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/interception/interception_linux.cpp:61
#17 0x00005555558f1427 in InitializeCommonInterceptors ()
    at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/asan/../sanitizer_common/sanitizer_common_interceptors.inc:10454
#18 0x00005555558ef6df in InitializeAsanInterceptors ()
    at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/asan/asan_interceptors.cpp:803
#19 0x0000555555912373 in AsanInitInternal () at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/asan/asan_rtl.cpp:469
#20 0x0000555555912784 in AsanInitFromRtl () at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/asan/asan_rtl.cpp:567
#21 __asan_init () at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/asan/asan_rtl.cpp:678
#22 0x00007ffff7fc83cf in ?? () from /lib64/ld-linux-x86-64.so.2
#23 0x00007ffff7fe0ba0 in ?? () from /lib64/ld-linux-x86-64.so.2
#24 0x0000000000000003 in ?? ()
#25 0x00007fffffffccf7 in ?? ()
#26 0x00007fffffffcd2d in ?? ()
#27 0x00007fffffffcd3c in ?? ()
#28 0x0000000000000000 in ?? ()

Details

This is on Linux x86_64 and Clang/LLVM 22 (head) on bdwgc's latest master (19a7f495cd7c48cdf6f56c593c6dd57187afa079). I'm building bdwgc alongside my project with the following cmake config:

  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -DREDIRECT_MALLOC=GC_malloc_uncollectable")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -DREDIRECT_MALLOC=GC_malloc_uncollectable")
  set(BUILD_SHARED_LIBS OFF)
  set(CMAKE_CXX_CLANG_TIDY "")
  set(enable_cplusplus ON CACHE BOOL "Enable C++")
  set(build_cord OFF CACHE BOOL "Build cord")
  set(enable_docs OFF CACHE BOOL "Enable docs")
  set(enable_large_config ON CACHE BOOL "Optimize for large heap or root set")
  set(enable_throw_bad_alloc_library ON CACHE BOOL "Enable C++ gctba library build")
  set(enable_gc_debug OFF CACHE BOOL "Support for pointer back-tracing")

  add_subdirectory(third-party/bdwgc EXCLUDE_FROM_ALL)

jeaye avatar Oct 16 '25 23:10 jeaye

Hello @jeaye, While nobody reported it, it seems the expected behavior. I think to fix the issue we need same code in malloc (see file malloc.c) as in REDIR_MALLOC_AND_LINUX_THREADS block of calloc. Could you please try it?

ivmai avatar Oct 27 '25 13:10 ivmai

E.g. could you please try the following:

diff --git a/malloc.c b/malloc.c
index ba6c1edf..b536ea52 100644
--- a/malloc.c
+++ b/malloc.c
@@ -536,26 +536,6 @@ GC_malloc_atomic_uncollectable(size_t lb)
 #    define REDIRECT_MALLOC_F REDIRECT_MALLOC
 #  endif
 
-void *
-malloc(size_t lb)
-{
-  /*
-   * It might help to manually inline the `GC_malloc` call here.
-   * But any decent compiler should reduce the extra procedure call
-   * to at most a jump instruction in this case.
-   */
-#  if defined(SOLARIS) && defined(THREADS) && defined(I386)
-  /*
-   * Thread initialization can call `malloc` before we are ready for.
-   * It is not clear that this is enough to help matters.  The thread
-   * implementation may well call `malloc` at other inopportune times.
-   */
-  if (UNLIKELY(!GC_is_initialized))
-    return sbrk(lb);
-#  endif
-  return (void *)REDIRECT_MALLOC_F(lb);
-}
-
 #  ifdef REDIR_MALLOC_AND_LINUX_THREADS
 #    ifdef HAVE_LIBPTHREAD_SO
 STATIC ptr_t GC_libpthread_start = NULL;
@@ -604,6 +584,40 @@ GC_init_lib_bounds(void)
 }
 #  endif /* REDIR_MALLOC_AND_LINUX_THREADS */
 
+void *
+malloc(size_t lb)
+{
+  /*
+   * It might help to manually inline the `GC_malloc` call here.
+   * But any decent compiler should reduce the extra procedure call
+   * to at most a jump instruction in this case.
+   */
+#  if defined(SOLARIS) && defined(THREADS) && defined(I386)
+  /*
+   * Thread initialization can call `malloc` before we are ready for.
+   * It is not clear that this is enough to help matters.  The thread
+   * implementation may well call `malloc` at other inopportune times.
+   */
+  if (UNLIKELY(!GC_is_initialized))
+    return sbrk(lb);
+#  endif
+#  ifdef REDIR_MALLOC_AND_LINUX_THREADS
+  {
+    ptr_t caller = (ptr_t)__builtin_return_address(0);
+
+    GC_init_lib_bounds();
+    if (ADDR_INSIDE(caller, GC_libld_start, GC_libld_end)
+#    ifdef HAVE_LIBPTHREAD_SO
+        || ADDR_INSIDE(caller, GC_libpthread_start, GC_libpthread_end)
+#    endif
+    ) {
+      return GC_generic_malloc_uncollectable(lb, UNCOLLECTABLE);
+    }
+  }
+#  endif
+  return (void *)REDIRECT_MALLOC_F(lb);
+}
+
 void *
 calloc(size_t n, size_t lb)
 {

Note: please check that REDIR_MALLOC_AND_LINUX_THREADS is really defined.

ivmai avatar Oct 28 '25 11:10 ivmai

Thanks, Ivan! I will give this a go and report back.

jeaye avatar Oct 28 '25 17:10 jeaye

Reporting back! Thanks again for taking the time to offer support. :)

So far, the crash still exists. Here's what I've done:

  1. Applied your provided patch (on top of 19a7f495cd7c48cdf6f56c593c6dd57187afa079)
  2. Added a preprocessor check to ensure REDIR_MALLOC_AND_LINUX_THREADS was defined
  3. Added -DREDIR_MALLOC_AND_LINUX_THREADS since it was not defined
  4. Experienced the same crash on start (backtrace below)

Looking at the code, I think the most relevant thing to point out is that HAVE_LIBPTHREAD_SO was not defined for me. I tried manually linking in libpthread.so.0 and providing -DHAVE_LIBPTHREAD_SO, but that had no effect. Do you think this is related?

Backtrace

(gdb) bt
#0  0x0000000000000000 in ?? ()
#1  0x00005555558cc01a in ___interceptor_pthread_setcancelstate ()
    at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/asan/../sanitizer_common/sanitizer_common_interceptors.inc:6811
#2  0x0000555557b6d4d8 in GC_init_lib_bounds () at /home/jeaye/projects/jank/compiler+runtime/third-party/bdwgc/malloc.c:557
#3  0x0000555557b6d549 in malloc (lb=88) at /home/jeaye/projects/jank/compiler+runtime/third-party/bdwgc/malloc.c:609
#4  0x00007ffff7fc6b72 in _dl_exception_create_format () from /lib64/ld-linux-x86-64.so.2
#5  0x00007ffff7fcdb6c in ?? () from /lib64/ld-linux-x86-64.so.2
#6  0x00007fffeeac3086 in ?? () from /usr/lib/libc.so.6
#7  0x00007fffee9ebe00 in ?? () from /usr/lib/libc.so.6
#8  0x00007ffff7fc4456 in _dl_catch_exception () from /lib64/ld-linux-x86-64.so.2
#9  0x00007ffff7fc45a9 in ?? () from /lib64/ld-linux-x86-64.so.2
#10 0x00007fffee9eb7f3 in ?? () from /usr/lib/libc.so.6
#11 0x00007fffee9ebe9e in dlsym () from /usr/lib/libc.so.6
#12 0x0000555555919284 in GetFuncAddr () at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/interception/interception_linux.cpp:42
#13 InterceptFunction () at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/interception/interception_linux.cpp:61
#14 0x00005555558f2377 in InitializeCommonInterceptors ()
    at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/asan/../sanitizer_common/sanitizer_common_interceptors.inc:10454
#15 0x00005555558f062f in InitializeAsanInterceptors ()
    at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/asan/asan_interceptors.cpp:803
#16 0x00005555559132c3 in AsanInitInternal () at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/asan/asan_rtl.cpp:469
#17 0x00005555559136d4 in AsanInitFromRtl () at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/asan/asan_rtl.cpp:567
#18 __asan_init () at /home/jeaye/projects/jank/compiler+runtime/build/llvm/compiler-rt/lib/asan/asan_rtl.cpp:678
#19 0x00007ffff7fc83cf in ?? () from /lib64/ld-linux-x86-64.so.2
#20 0x00007ffff7fe0ba0 in ?? () from /lib64/ld-linux-x86-64.so.2
#21 0x0000000000000003 in ?? ()
#22 0x00007fffffffcceb in ?? ()
#23 0x00007fffffffcd21 in ?? ()
#24 0x00007fffffffcd30 in ?? ()
#25 0x0000000000000000 in ?? ()

CMake setup

  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -DREDIRECT_MALLOC=GC_malloc_uncollectable -DREDIR_MALLOC_AND_LINUX_THREADS")
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -DREDIRECT_MALLOC=GC_malloc_uncollectable -DREDIR_MALLOC_AND_LINUX_THREADS")
  set(BUILD_SHARED_LIBS OFF)
  set(CMAKE_CXX_CLANG_TIDY "")
  set(enable_cplusplus ON CACHE BOOL "Enable C++")
  set(build_cord OFF CACHE BOOL "Build cord")
  set(enable_docs OFF CACHE BOOL "Enable docs")
  set(enable_threads ON CACHE BOOL "Enable multi-threading support")
  set(enable_large_config ON CACHE BOOL "Optimize for large heap or root set")
  set(enable_throw_bad_alloc_library ON CACHE BOOL "Enable C++ gctba library build")
  set(enable_gc_debug OFF CACHE BOOL "Support for pointer back-tracing")

  add_subdirectory(third-party/bdwgc EXCLUDE_FROM_ALL)

jeaye avatar Nov 04 '25 05:11 jeaye

HAVE_LIBPTHREAD_SO was not defined for me

I think it is correct.

ivmai avatar Nov 04 '25 18:11 ivmai 

#2  0x0000555557b6d4d8 in GC_init_lib_bounds () at /home/jeaye/projects/jank/compiler+runtime/third-party/bdwgc/malloc.c:557
#3  0x0000555557b6d549 in malloc (lb=88) at /home/jeaye/projects/jank/compiler+runtime/third-party/bdwgc/malloc.c:609
#4  0x00007ffff7fc6b72 in _dl_exception_create_format () from /lib64/ld-linux-x86-64.so.2

The problem here is that we should call GC_init_lib_bounds earlier than InitializeCommonInterceptors.

I wonder if GC_wrap_dlopen is called before you get the crash?

ivmai avatar Nov 04 '25 19:11 ivmai

I think to fix the issue we need same code in malloc (see file malloc.c) as in REDIR_MALLOC_AND_LINUX_THREADS block of calloc. Could you please try it?

This was wrong advice, because GC_init cannot be called during InitializeAsanInterceptors(). At the same time we cannot call GC_init before it.

The only advice here is to redirect to sbrk if GC_is_initialized is false. The client code is responsible to call GC_init explicitly in this case.

ivmai avatar Dec 15 '25 17:12 ivmai