react-native icon indicating copy to clipboard operation
react-native copied to clipboard

How to symbolicate Hermes crash stack symbol?

Open iCodeWoods opened this issue 11 months ago • 13 comments

Description

As the document states, there are Hermes symbols in the assets, but only certain versions have them (e.g., 0.70.7), while others don't (e.g., 0.71.7).

How can I obtain the Hermes dSYM for version 0.71.7?

image

image

Below is the crash report.

Exception Type:  SIGABRT
Exception Codes: #0 at 0x1e9f0e558
Crashed Thread:  2

Thread 2 Crashed:
0   libsystem_kernel.dylib               0x00000001e9f0e558 __pthread_kill + 8
1   libsystem_c.dylib                    0x00000001b24f3178 abort + 176
2   libsystem_malloc.dylib               0x00000001b945b0e4 malloc_vreport + 904
3   libsystem_malloc.dylib               0x00000001b945b38c malloc_zone_error + 100
4   libsystem_malloc.dylib               0x00000001b9455658 nanov2_guard_corruption_detected + 40
5   libsystem_malloc.dylib               0x00000001b9453810 nanov2_allocate_outlined + 400
6   hermes                               0x00000001068d1ab0 0x1066a8000 + 2267824
7   hermes                               0x00000001066e1ef0 0x1066a8000 + 237296
8   hermes                               0x00000001066c8250 0x1066a8000 + 131664
9   hermes                               0x00000001067054dc 0x1066a8000 + 382172
10  hermes                               0x000000010670c8a4 0x1066a8000 + 411812
11  hermes                               0x00000001066ccc7c 0x1066a8000 + 150652
12  hermes                               0x0000000106707c00 0x1066a8000 + 392192
13  hermes                               0x0000000106708164 0x1066a8000 + 393572
14  hermes                               0x00000001066f0290 0x1066a8000 + 295568
15  hermes                               0x00000001066ed71c 0x1066a8000 + 284444
16  hermes                               0x00000001066cf2d8 0x1066a8000 + 160472
17  hermes                               0x00000001066ceb40 0x1066a8000 + 158528
18  hermes                               0x00000001066ee284 0x1066a8000 + 287364
19  hermes                               0x00000001066ed71c 0x1066a8000 + 284444
20  hermes                               0x00000001066cf2d8 0x1066a8000 + 160472
21  hermes                               0x00000001066ceb40 0x1066a8000 + 158528
22  hermes                               0x00000001066b3ce4 0x1066a8000 + 48356
23  Skype4Life                           0x0000000102b617fc facebook::jsi::WithRuntimeDecorator<facebook::react::(anonymous namespace)::ReentrancyCheck, facebook::jsi::Runtime, facebook::jsi::Runtime>::call(facebook::jsi::Function const&, facebook::jsi::Value const&, facebook::jsi::Value const*, unsigned long) (decorator.h:337)
24  Skype4Life                           0x0000000102b9118c facebook::jsi::Value facebook::jsi::Function::call<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, facebook::jsi::Value>(facebook::jsi::Runtime&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&&&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&&&, facebook::jsi::Value&&) const (jsi-inl.h:257)
25  Skype4Life                           0x0000000102b90ff0 std::__1::__function::__func<facebook::react::JSIExecutor::callFunction(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, folly::dynamic const&)::$_5, std::__1::allocator<facebook::react::JSIExecutor::callFunction(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, folly::dynamic const&)::$_5>, void ()>::operator()() (JSIExecutor.cpp:253)
26  Skype4Life                           0x0000000102a96724 void std::__1::__invoke_void_return_wrapper<void, true>::__call<void (*&)(std::__1::function<void ()> const&, std::__1::function<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > ()>), std::__1::function<void ()> const&, std::__1::function<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > ()> >(void (*&&&)(std::__1::function<void ()> const&, std::__1::function<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > ()>), std::__1::function<void ()> const&&&, std::__1::function<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > ()>&&) (type_traits:3918)
27  Skype4Life                           0x0000000102b8e1e8 facebook::react::JSIExecutor::callFunction(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, folly::dynamic const&) (function.h:505)
28  Skype4Life                           0x0000000102b3d684 std::__1::__function::__func<facebook::react::NativeToJsBridge::runOnExecutorQueue(std::__1::function<void (facebook::react::JSExecutor*)>)::$_8, std::__1::allocator<facebook::react::NativeToJsBridge::runOnExecutorQueue(std::__1::function<void (facebook::react::JSExecutor*)>)::$_8>, void ()>::operator()() (function.h:505)
29  Skype4Life                           0x0000000102a99304 facebook::react::tryAndReturnError(std::__1::function<void ()> const&) (function.h:505)
30  Skype4Life                           0x0000000102aa53dc facebook::react::RCTMessageThread::tryFunc(std::__1::function<void ()> const&) (RCTMessageThread.mm:69)
31  Skype4Life                           0x0000000102aa5190 ___ZN8facebook5react16RCTMessageThread8runAsyncENSt3__18functionIFvvEEE_block_invoke (function.h:505)
32  CoreFoundation                       0x00000001ab0066e0 __CFRUNLOOP_IS_CALLING_OUT_TO_A_BLOCK__ + 24
33  CoreFoundation                       0x00000001ab06d210 __CFRunLoopDoBlocks + 360
34  CoreFoundation                       0x00000001ab03d0e8 __CFRunLoopRun + 844
35  CoreFoundation                       0x00000001ab0423ec CFRunLoopRunSpecific + 608
36  Skype4Life                           0x0000000102a8edc4 +[RCTCxxBridge runRunLoop] (RCTCxxBridge.mm:336)
37  Foundation                           0x00000001a52dc544 __NSThread__start__ + 712
38  libsystem_pthread.dylib              0x000000020ad516b8 _pthread_start + 144
39  libsystem_pthread.dylib              0x000000020ad50b88 thread_start + 4

React Native Version

0.71.7

Output of npx react-native info

info Fetching system and libraries information...
System:
    OS: macOS 13.4
    CPU: (10) x64 Apple M1 Pro
    Memory: 26.28 MB / 16.00 GB
    Shell: 5.9 - /bin/zsh
  Binaries:
    Node: 14.21.3 - ~/.nvm/versions/node/v14.21.3/bin/node
    Yarn: 1.22.19 - /opt/homebrew/bin/yarn
    npm: 6.14.18 - ~/.nvm/versions/node/v14.21.3/bin/npm
    Watchman: 2023.04.10.00 - /opt/homebrew/bin/watchman
  Managers:
    CocoaPods: 1.11.3 - /usr/local/bin/pod
  SDKs:
    iOS SDK:
      Platforms: DriverKit 22.4, iOS 16.4, macOS 13.3, tvOS 16.4, watchOS 9.4
    Android SDK: Not Found
  IDEs:
    Android Studio: Not Found
    Xcode: 14.3.1/14E300c - /usr/bin/xcodebuild
  Languages:
    Java: Not Found
  npmPackages:
    @react-native-community/cli: Not Found
    react: Not Found
    react-native: Not Found
    react-native-macos: Not Found
  npmGlobalPackages:
    *react-native*: Not Found
info React Native v0.72.1 is now available (your project is running on v0.71.7).
info Changelog: https://github.com/facebook/react-native/releases/tag/v0.72.1
info Diff: https://react-native-community.github.io/upgrade-helper/?from=0.71.7
info For more info, check out "https://reactnative.dev/docs/upgrading".

Steps to reproduce

Check https://github.com/facebook/react-native/releases?page=2 and there is no hermes artifacts in v0.71.7

Snack, code example, screenshot, or link to a repository

image

iCodeWoods avatar Jul 04 '23 08:07 iCodeWoods

:warning: Newer Version of React Native is Available!
:information_source: You are on a supported minor version, but it looks like there's a newer patch available - 0.71.11. Please upgrade to the highest patch for your minor or latest and verify if the issue persists (alternatively, create a new project and repro the issue in it). If it does not repro, please let us know so we can close out this issue. This helps us ensure we are looking at issues that still exist in the most recent releases.

github-actions[bot] avatar Jul 04 '23 08:07 github-actions[bot]

Even if we use the newer patch - 0.71.11, it's still don't have hermes artifacts in the assets.

image

iCodeWoods avatar Jul 04 '23 08:07 iCodeWoods

For context, those artifacts are only for Android debug symbols. They've been removed from 0.71 as they're distributed on Maven Central alongside the React Native artifacts.

As for iOS, I'm unsure where the dSYM are stored (cc @cipolleschi).

cortinico avatar Jul 04 '23 10:07 cortinico

hum... I don't think we are generating dSYM anymore, unfortunately. Probably due to some bug or unwanted changes. I'll have to look into that more deeply. @dmytrorykun, do you have more information about this?

cipolleschi avatar Jul 04 '23 13:07 cipolleschi

do you have more information about this?

@cipolleschi nope, looks like a bug to me.

dmytrorykun avatar Jul 04 '23 13:07 dmytrorykun

Hi @cortinico @cipolleschi @dmytrorykun Thanks for your reply. The ultimate goal of wanting the dSYM for Hermes is to resolve crashes. Do you have any thoughts on the crash stack trace i provided earlier? It contains several Hermes stack frames, but symbolizing them may not be very helpful. I have encountered similar issues before where crashes triggered by bridge messages end up with stack traces within React Native, but it's difficult to determine which bridge specifically caused the issue. Do you have any suggestions or recommendations for addressing this?

iCodeWoods avatar Jul 05 '23 06:07 iCodeWoods

I understand that it's difficult to debug the issue without being able to reproduce it.

However, at least I would like to ask if there are any ideas regarding the crash triggered by +[RCTCxxBridge runRunLoop].

For example, should I fork a copy of the code and add some logs in this area? (But I think this may impact performance, so it may not be a good idea.)

iCodeWoods avatar Jul 05 '23 06:07 iCodeWoods

Can you reproduce the crash locally in a Debug version of the app? Or are these crashes happening only in production? If you manage to reproduce the crash locally, does it also happen if you disable Hermes? (to rule out that the problem is in hermes).

Some logs would help: for example, the app is calling something in Javascript that is then forwarded to native, as shown by lines 25 and 27 (JSIExecutor::callFunction) but we can't have the function invoked in the stacktrace...

cipolleschi avatar Jul 05 '23 10:07 cipolleschi

Hi @cipolleschi Thanks for your reply.

It's an issue that occurs in the production environment and cannot be reproduced locally. Before enabling Hermes, there was another issue in the production environment, but the quantity was only half of this one. After enabling Hermes, the previous issue disappeared, but this new issue appeared, with a quantity twice as much as before.

I would like to know if React Native has any commonly used or general solutions for this type of crash. Although the crash appears in the React Native stack trace, it may not indicate the specific cause, and it is possible that the issue is not related to React Native itself, but rather to the caller's code.

iCodeWoods avatar Jul 06 '23 02:07 iCodeWoods

Yeah, I understand. If a crash is happening with both engines, it makes me think that it is either something in the app or in React Native. Looking at these lines, it seems that the symbols for Hermes are actually created.

But then, here those symbols are stripped. I think that the idea was to strip them from there and to create a separate .tar with them.

So, one thing you can do while we work to restore them is to:

  1. clone react-native repo
  2. checkout the branch 0.71-stable
  3. run yarn in the main folder
  4. cd packages/rn-tester
  5. bundle install
  6. bundle exec pod install --> this will download the right version of hermes locally
  7. cd ../react-native/sdks
  8. BUILD_TYPE="Release" ./utils/build-ios-framework.sh --> This should build hermes for iPhone, iPhoneSimulator, Catalyst.

This should build the framework in the /tmp/hermes/hermes-runtime-darwin/destroot/ folder and you can look whether there is a dSYM folder that you can use. We never update Hermes after 71.4, so the latest you can build from 0.71 is the same you should have in prod.

I really hope this helps.

cipolleschi avatar Jul 06 '23 12:07 cipolleschi

Hi @cipolleschi Thanks for your help. I encountered some issues when using the command.

➜  hermes-engine git:(0.71-stable) BUILD_TYPE="Release" ./utils/build-ios-framework.sh
Building hermesc
CMake Error: The source directory "/Users/linzhennan/react-native/sdks/hermes-engine" does not appear to contain CMakeLists.txt.
Specify --help for usage, or press the help button on the CMake GUI.
make: Makefile: No such file or directory
make: *** No rule to make target `Makefile'.  Stop.
Host hermesc is required to build apple frameworks!
Building Release framework for iphoneos with architectures: arm64
CMake Error: The source directory "/Users/linzhennan/react-native/sdks/hermes-engine" does not appear to contain CMakeLists.txt.
Specify --help for usage, or press the help button on the CMake GUI.
Error: could not load cache
Building hermesc
CMake Error: The source directory "/Users/linzhennan/react-native/sdks/hermes-engine" does not appear to contain CMakeLists.txt.
Specify --help for usage, or press the help button on the CMake GUI.
make: Makefile: No such file or directory
make: *** No rule to make target `Makefile'.  Stop.
Host hermesc is required to build apple frameworks!
Building Release framework for iphonesimulator with architectures: x86_64;arm64
CMake Error: The source directory "/Users/linzhennan/react-native/sdks/hermes-engine" does not appear to contain CMakeLists.txt.
Specify --help for usage, or press the help button on the CMake GUI.
Error: could not load cache
Building hermesc
CMake Error: The source directory "/Users/linzhennan/react-native/sdks/hermes-engine" does not appear to contain CMakeLists.txt.
Specify --help for usage, or press the help button on the CMake GUI.
make: Makefile: No such file or directory
make: *** No rule to make target `Makefile'.  Stop.
Host hermesc is required to build apple frameworks!
Building Release framework for catalyst with architectures: x86_64;arm64
CMake Error: The source directory "/Users/linzhennan/react-native/sdks/hermes-engine" does not appear to contain CMakeLists.txt.
Specify --help for usage, or press the help button on the CMake GUI.
Error: could not load cache
/Users/linzhennan/react-native/sdks/hermes-engine/utils/build-apple-framework.sh: line 124: pushd: /Users/linzhennan/react-native/sdks/hermes-engine/utils/../destroot/Library/Frameworks: No such file or directory```

iCodeWoods avatar Jul 07 '23 07:07 iCodeWoods

I think you are running the command from the wrong folder:

hermes-engine is a supporting folder, where react-native specifies the build logic for hermes hermes is the folder that actually contains the hermes code.

So... if you run BUILD_TYPE="Release" ./utils/build-ios-framework.sh from sdks/hermes it should work.

Warning: it will take some time to build hermes, unfortunately... 😓

cipolleschi avatar Jul 07 '23 08:07 cipolleschi

@iCodeWoods I make a test today and I can confirm that running

BUILD_TYPE="Release" ./utils/build-ios-framework.sh
BUILD_TYPE="Release" ./utils/build-mac-framework.sh

Creates also the dSYMs:

Screenshot 2023-07-07 at 18 46 12

I'll be on holiday the next week, and back the week after. Once back, I'll have a look at how to create the archive and ship the symbols!

cipolleschi avatar Jul 07 '23 16:07 cipolleschi

Hey there, starting from 0.73, we are publishing the dSYMs for both Hermes in Release and Hermes in Debug. You can find the link in the GitHub releases.

In any case, they follow the format (replace the <> with the proper value. All letters are lowercase):

https://repo1.maven.org/maven2/com/facebook/react/react-native-artifacts/<version>/react-native-artifacts-<version>-hermes-framework-dSYM-<build_type>.tar.gz

We don't have dSYMs for versions lower than 0.72, unfortunately.

cipolleschi avatar Oct 05 '23 08:10 cipolleschi

@cipolleschi are the hermes dsyms for 0.73.1 missing or not intended to be published?

danilobuerger avatar Dec 29 '23 20:12 danilobuerger

Yes they have been published here: https://repo1.maven.org/maven2/com/facebook/react/react-native-artifacts/0.73.1

Probably we forgot to update the release notes

cipolleschi avatar Dec 29 '23 20:12 cipolleschi

@cipolleschi thats really weird. if you go to https://repo1.maven.org/maven2/com/facebook/react/react-native-artifacts/ it doesn't show that folder...

danilobuerger avatar Dec 29 '23 21:12 danilobuerger

Yeah.. that's weird. Perhaps sonatype is having issues in synching the indexes. That per of the infra is a bit out of our control... 🤷

cipolleschi avatar Dec 29 '23 21:12 cipolleschi

if you go to repo1.maven.org/maven2/com/facebook/react/react-native-artifacts it doesn't show that folder...

I believe things were down over the NYE. Everything should be back to normal now 👍

cortinico avatar Jan 02 '24 09:01 cortinico

Hey @iCodeWoods , were you able to resolve or find the root cause of this issue ? We have a similar crash that got reported on our Instabug.

"react-native": "0.70.8"

SIGABRT: MyApp(1131,0x16e1b3000) malloc: Heap corruption detected, free list is damaged at 0x301f44d80 *** Incorrect guard value: 6091085824
com.facebook.react.JavaScript
0x1052eb7c8
Thread 9 com.facebook.react.JavaScript [Crashed]:

0    libsystem_kernel.dylib                   0x1dbbb2974     __pthread_kill + 8
1    libsystem_c.dylib                        0x19b603b80     abort + 179
2    libsystem_malloc.dylib                   0x1a382ec68     malloc_vreport + 895
3    libsystem_malloc.dylib                   0x1a382ef10     malloc_zone_error + 103
4    libsystem_malloc.dylib                   0x1a3824a44     nanov2_guard_corruption_detected + 43
5    libsystem_malloc.dylib                   0x1a380af84     nanov2_allocate_outlined + 403
6    libc++abi.dylib                          0x1ef556b84     operator new(unsigned long) + 51
7    hermes                                   0x1052eb7c8     0x105130000
8    hermes                                   0x1052d93e8     0x105130000
9    hermes                                   0x1052d967c     0x105130000
10   hermes                                   0x1052de6a4     0x105130000
11   hermes                                   0x1052de218     0x105130000
12   hermes                                   0x1052dec7c     0x105130000
13   hermes                                   0x1052de304     0x105130000
14   hermes                                   0x1052de528     0x105130000
15   hermes                                   0x1052e347c     0x105130000
16   hermes                                   0x1052e34f8     0x105130000
17   hermes                                   0x1052e616c     0x105130000
18   hermes                                   0x1052d8e7c     0x105130000
19   hermes                                   0x1052d7854     0x105130000
20   hermes                                   0x105159c34     0x105130000
21   hermes                                   0x105176dbc     0x105130000
22   hermes                                   0x10517618c     0x105130000
23   hermes                                   0x10515859c     0x105130000
24   hermes                                   0x105157e04     0x105130000
25   hermes                                   0x105176c94     0x105130000
26   hermes                                   0x10517618c     0x105130000
27   hermes                                   0x10515859c     0x105130000
28   hermes                                   0x105157e04     0x105130000
29   hermes                                   0x105176c94     0x105130000
30   hermes                                   0x10517618c     0x105130000
31   hermes                                   0x105158fcc     0x105130000
32   hermes                                   0x1051fcc6c     0x105130000
33   hermes                                   0x1051582f4     0x105130000
34   hermes                                   0x105176c70     0x105130000
35   hermes                                   0x10517618c     0x105130000
36   hermes                                   0x10515859c     0x105130000
37   hermes                                   0x105156eb0     0x105130000
38   hermes                                   0x105228704     0x105130000
39   hermes                                   0x1051582f4     0x105130000
40   hermes                                   0x105176c70     0x105130000
41   hermes                                   0x10517618c     0x105130000
42   hermes                                   0x10515859c     0x105130000
43   hermes                                   0x105157e04     0x105130000
44   hermes                                   0x105176c94     0x105130000
45   hermes                                   0x10517618c     0x105130000
46   hermes                                   0x10515859c     0x105130000
47   hermes                                   0x105157e04     0x105130000
48   hermes                                   0x10513cd58     0x105130000
49   MyApp                                    0x1024c2258     facebook::jsi::WithRuntimeDecorator<facebook::react::(anonymous namespace)::ReentrancyCheck, facebook::jsi::Runtime, facebook::jsi::Runtime>::call(facebook::jsi::Function const&, facebook::jsi::Value const&, facebook::jsi::Value const*, unsigned long) (decorator.h:709)

cc: @cipolleschi

dgrover-spotnana avatar Apr 16 '24 12:04 dgrover-spotnana

@dgrover-spotnana unfortunately, we don't have dSYMs for React Native 0.70 and that version is not supported anymore, so it's not possible to symbolicate Hermes symbols for that version.

cipolleschi avatar Apr 24 '24 15:04 cipolleschi