cmssw
cmssw copied to clipboard
cms-sw
SIGSEGV in (anonymous namespace)::maybeConfigPileUp
RelVal 13034.412 crashed in (anonymous namespace)::maybeConfigPileUp:
A fatal system signal has occurred: segmentation violation
The following is the call stack containing the origin of the signal.
Thu Oct 10 04:57:38 CEST 2024
Thread 7 (Thread 0x14619e7ff700 (LWP 1328548) "cmsRun"):
#0 0x00001462250c141d in syscall () from /lib64/libc.so.6
#1 0x0000146227125fd2 in tbb::detail::r1::futex_wait (comparand=2, futex=0x146221574024) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/semaphore.h:100
#2 tbb::detail::r1::binary_semaphore::P (this=0x146221574024) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/semaphore.h:253
#3 tbb::detail::r1::rml::internal::thread_monitor::wait (this=0x146221574020) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/rml_thread_monitor.h:235
#4 tbb::detail::r1::rml::private_worker::run (this=0x146221574000) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/private_server.cpp:273
#5 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x146221574000) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#6 0x00001462254661ca in start_thread () from /lib64/libpthread.so.0
#7 0x00001462250c18d3 in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x14619f7a0700 (LWP 1328547) "cmsRun"):
#0 0x00001462251baac1 in poll () from /lib64/libc.so.6
#1 0x0000146220cc29f7 in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x0000146220cc2bf4 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x00001462272dfec1 in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_2_GPU_X_2024-10-09-2300/external/el8_amd64_gcc12/lib/libRIO.so
#5 0x0000146227384046 in TKey::TKey(TObject const*, char const*, int, TDirectory*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_2_GPU_X_2024-10-09-2300/external/el8_amd64_gcc12/lib/libRIO.so
#6 0x00001462273420a5 in TFile::CreateKey(TDirectory*, TObject const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_2_GPU_X_2024-10-09-2300/external/el8_amd64_gcc12/lib/libRIO.so
#7 0x0000146227333286 in TDirectoryFile::WriteTObject(TObject const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_2_GPU_X_2024-10-09-2300/external/el8_amd64_gcc12/lib/libRIO.so
#8 0x0000146226e40c04 in TObject::Write(char const*, int, int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_2_GPU_X_2024-10-09-2300/external/el8_amd64_gcc12/lib/libCore.so
#9 0x000014619ec32666 in (anonymous namespace)::maybeConfigPileUp(edm::ParameterSet const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, int, int, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libMixingBase.so
#10 0x000014619ec3312f in edm::MixingCache::Config::Config(edm::ParameterSet const&, unsigned int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libMixingBase.so
#11 0x000014619ec33817 in edm::BMixingModule::initializeGlobalCache(edm::ParameterSet const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libMixingBase.so
#12 0x000014619ecde586 in edm::WorkerMaker<edm::MixingModule>::makeModule(edm::ParameterSet const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/pluginSimGeneralMixingModulePlugins.so
#13 0x0000146227e345a7 in edm::Maker::makeModule(edm::MakeModuleParams const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x0000146227da1ed8 in edm::Factory::makeModule(edm::MakeModuleParams const&, edm::ModuleTypeResolverMaker const*, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x0000146227db204f in edm::ModuleRegistry::getModule(edm::MakeModuleParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x0000146227e34da8 in edm::WorkerRegistry::getWorker(edm::WorkerParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#17 0x0000146227e35085 in edm::WorkerManager::getWorker(edm::ParameterSet&, edm::ProductRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#18 0x0000146227e0f940 in edm::(anonymous namespace)::getWorker(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::ParameterSet&, edm::WorkerManager&, edm::ProductRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#19 0x0000146227e1debe in edm::StreamSchedule::fillWorkers(edm::ParameterSet&, edm::ProductRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, std::vector<edm::WorkerInPath, std::allocator<edm::WorkerInPath> >&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, edm::ConditionalTaskHelper const&, std::unordered_set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#20 0x0000146227e1ebe9 in edm::StreamSchedule::fillTrigPath(edm::ParameterSet&, edm::ProductRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::shared_ptr<edm::HLTGlobalStatus>, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, edm::ConditionalTaskHelper const&, std::unordered_set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#21 0x0000146227e13867 in edm::StreamSchedule::StreamSchedule(std::shared_ptr<edm::TriggerResultInserter>, std::vector<edm::propagate_const<std::shared_ptr<edm::PathStatusInserter> >, std::allocator<edm::propagate_const<std::shared_ptr<edm::PathStatusInserter> > > >&, std::vector<edm::propagate_const<std::shared_ptr<edm::EndPathStatusInserter> >, std::allocator<edm::propagate_const<std::shared_ptr<edm::EndPathStatusInserter> > > >&, std::shared_ptr<edm::ModuleRegistry>, edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::ProductRegistry&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::StreamID, edm::ProcessContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#22 0x0000146227df8f75 in edm::Schedule::Schedule(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::ProductRegistry&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#23 0x0000146227e0cb00 in edm::ScheduleItems::initModules(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#24 0x0000146227d79c47 in tbb::detail::d1::function_task<edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#25 0x0000146227123b3b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x146223bc3300, waiter=..., this=0x146223bca900) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#26 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x146223bca900) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#27 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/arena.cpp:137
#28 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/market.cpp:599
#29 0x0000146227125cee in tbb::detail::r1::rml::private_worker::run (this=0x146221574100) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#30 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x146221574100) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#31 0x00001462254661ca in start_thread () from /lib64/libpthread.so.0
#32 0x00001462250c18d3 in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x1461a01a1700 (LWP 1328546) "cmsRun"):
#0 0x0000146225190098 in nanosleep () from /lib64/libc.so.6
#1 0x000014622518ff9e in sleep () from /lib64/libc.so.6
#2 0x0000146220cbf3a0 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x00001462250c141d in syscall () from /lib64/libc.so.6
#5 0x0000146227125fd2 in tbb::detail::r1::futex_wait (comparand=2, futex=0x1462215740a4) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/semaphore.h:100
#6 tbb::detail::r1::binary_semaphore::P (this=0x1462215740a4) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/semaphore.h:253
#7 tbb::detail::r1::rml::internal::thread_monitor::wait (this=0x1462215740a0) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/rml_thread_monitor.h:235
#8 tbb::detail::r1::rml::private_worker::run (this=0x146221574080) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/private_server.cpp:273
#9 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x146221574080) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#10 0x00001462254661ca in start_thread () from /lib64/libpthread.so.0
#11 0x00001462250c18d3 in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x1461f13de700 (LWP 1328438) "cuda-EvtHandlr"):
#0 0x00001462251baac1 in poll () from /lib64/libc.so.6
#1 0x000014621386889f in ?? () from /.singularity.d/libs/libcuda.so.1
#2 0x0000146213936dcf in ?? () from /.singularity.d/libs/libcuda.so.1
#3 0x0000146213863373 in ?? () from /.singularity.d/libs/libcuda.so.1
#4 0x00001462254661ca in start_thread () from /lib64/libpthread.so.0
#5 0x00001462250c18d3 in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x1461f8bfe700 (LWP 1328437) "cuda0000340000e"):
#0 0x00001462251baac1 in poll () from /lib64/libc.so.6
#1 0x000014621386889f in ?? () from /.singularity.d/libs/libcuda.so.1
#2 0x0000146213936dcf in ?? () from /.singularity.d/libs/libcuda.so.1
#3 0x0000146213863373 in ?? () from /.singularity.d/libs/libcuda.so.1
#4 0x00001462254661ca in start_thread () from /lib64/libpthread.so.0
#5 0x00001462250c18d3 in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x1461f8dff700 (LWP 1328431) "cmsRun"):
#0 0x00001462254706a2 in waitpid () from /lib64/libpthread.so.0
#1 0x0000146220cbf4f7 in edm::service::cmssw_stacktrace_fork() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x0000146220cc27da in edm::service::InitRootHandlers::stacktraceHelperThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 0x0000146225ad8a73 in std::execute_native_thread_routine (__p=0x1462192dcb90) at ../../../../../libstdc++-v3/src/c++11/thread.cc:82
#4 0x00001462254661ca in start_thread () from /lib64/libpthread.so.0
#5 0x00001462250c18d3 in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x146226b55680 (LWP 1328224) "cmsRun"):
#0 0x0000146225190098 in nanosleep () from /lib64/libc.so.6
#1 0x000014622518ff9e in sleep () from /lib64/libc.so.6
#2 0x0000146220cbf3a0 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014622546c48c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#5 0x00001462278d4ec3 in ROOT::TReentrantRWLock<std::mutex, ROOT::Internal::RecurseCountsTBBUnique>::WriteLock() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_2_GPU_X_2024-10-09-2300/external/el8_amd64_gcc12/lib/libThread.so
#6 0x0000146227398ab5 in TStreamerInfo::BuildCheck(TFile*, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_2_GPU_X_2024-10-09-2300/external/el8_amd64_gcc12/lib/libRIO.so
#7 0x0000146227342a15 in TFile::ReadStreamerInfo() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_2_GPU_X_2024-10-09-2300/external/el8_amd64_gcc12/lib/libRIO.so
#8 0x000014622734e285 in TFile::Init(bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_2_GPU_X_2024-10-09-2300/external/el8_amd64_gcc12/lib/libRIO.so
#9 0x00001462212dfe78 in TStorageFactoryFile::Initialize(char const*, char const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libIOPoolTFileAdaptor.so
#10 0x00001462212e0494 in TStorageFactoryFile::TStorageFactoryFile(char const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_GPU_X_2024-10-08-2300/lib/el8_amd64_gcc12/libIOPoolTFileAdaptor.so
#11 0x000014621e3b020d in ?? ()
#12 0x0000000000000000 in ?? ()
Current Modules:
Module: none (crashed)
A fatal system signal has occurred: segmentation violation
timeout: the monitored command dumped core
Happened in CMSSW_14_2_X_2024-10-13-0000/el8_amd64_gcc12 RelVal 12634.21 log
Thread 4 (Thread 0x14ef67bdd700 (LWP 1310764) "cmsRun"):
#0 0x000014efb49baac1 in poll () from /lib64/libc.so.6
#1 0x000014efb021b9f7 in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x000014efb021bbf4 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014efb6adfec1 in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libRIO.so
#5 0x000014efb6b84046 in TKey::TKey(TObject const*, char const*, int, TDirectory*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libRIO.so
#6 0x000014efb6b420a5 in TFile::CreateKey(TDirectory*, TObject const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libRIO.so
#7 0x000014efb6b33286 in TDirectoryFile::WriteTObject(TObject const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libRIO.so
#8 0x000014efb6640c04 in TObject::Write(char const*, int, int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libCore.so
#9 0x000014ef5ceef666 in (anonymous namespace)::maybeConfigPileUp(edm::ParameterSet const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, int, int, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libMixingBase.so
#10 0x000014ef5cef012f in edm::MixingCache::Config::Config(edm::ParameterSet const&, unsigned int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libMixingBase.so
#11 0x000014ef5cef0817 in edm::BMixingModule::initializeGlobalCache(edm::ParameterSet const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libMixingBase.so
#12 0x000014ef5cf9b586 in edm::WorkerMaker<edm::MixingModule>::makeModule(edm::ParameterSet const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/pluginSimGeneralMixingModulePlugins.so
#13 0x000014efb76346a7 in edm::Maker::makeModule(edm::MakeModuleParams const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x000014efb75a1ed8 in edm::Factory::makeModule(edm::MakeModuleParams const&, edm::ModuleTypeResolverMaker const*, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x000014efb75b204f in edm::ModuleRegistry::getModule(edm::MakeModuleParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x000014efb7634ea8 in edm::WorkerRegistry::getWorker(edm::WorkerParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#17 0x000014efb7635185 in edm::WorkerManager::getWorker(edm::ParameterSet&, edm::ProductRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#18 0x000014efb76352a0 in edm::WorkerManager::addToUnscheduledWorkers(edm::ParameterSet&, edm::ProductRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::less<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#19 0x000014efb7614230 in edm::StreamSchedule::StreamSchedule(std::shared_ptr<edm::TriggerResultInserter>, std::vector<edm::propagate_const<std::shared_ptr<edm::PathStatusInserter> >, std::allocator<edm::propagate_const<std::shared_ptr<edm::PathStatusInserter> > > >&, std::vector<edm::propagate_const<std::shared_ptr<edm::EndPathStatusInserter> >, std::allocator<edm::propagate_const<std::shared_ptr<edm::EndPathStatusInserter> > > >&, std::shared_ptr<edm::ModuleRegistry>, edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::ProductRegistry&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::StreamID, edm::ProcessContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#20 0x000014efb75f90ad in edm::Schedule::Schedule(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::ProductRegistry&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#21 0x000014efb760cc10 in edm::ScheduleItems::initModules(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#22 0x000014efb7579c97 in tbb::detail::d1::function_task<edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#23 0x000014efb6969b3b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x14efb33c7600, waiter=..., this=0x14efb33c9400) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#24 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x14efb33c9400) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#25 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/arena.cpp:137
#26 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/market.cpp:599
#27 0x000014efb696bcee in tbb::detail::r1::rml::private_worker::run (this=0x14efb14b4100) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#28 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x14efb14b4100) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#29 0x000014efb4c661ca in start_thread () from /lib64/libpthread.so.0
#30 0x000014efb48c18d3 in clone () from /lib64/libc.so.6
Other threads are sleeping, except Thread 1, which is reading ROOT file:
Thread 1 (Thread 0x14efb62e5680 (LWP 1310648) "cmsRun"):
#0 0x000014efb4990098 in nanosleep () from /lib64/libc.so.6
#1 0x000014efb498ff9e in sleep () from /lib64/libc.so.6
#2 0x000014efb02183a0 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014efb70f4def in ROOT::TReentrantRWLock<std::mutex, ROOT::Internal::RecurseCountsTBBUnique>::WriteLock() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libThread.so
#5 0x000014efb67021a6 in TClass::LoadClassInfo() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libCore.so
#6 0x000014efb6702518 in TClass::GetListOfBases() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libCore.so
#7 0x000014efb67123fc in TClass::GetCheckSum(TClass::ECheckSum, bool&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libCore.so
#8 0x000014efb6712cee in TClass::GetCheckSum(TClass::ECheckSum) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libCore.so
#9 0x000014efb6b99a65 in TStreamerInfo::BuildCheck(TFile*, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libRIO.so
#10 0x000014efb6b42a15 in TFile::ReadStreamerInfo() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libRIO.so
#11 0x000014efb6b4e285 in TFile::Init(bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libRIO.so
#12 0x000014efb135fe78 in TStorageFactoryFile::Initialize(char const*, char const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libIOPoolTFileAdaptor.so
#13 0x000014efb1360434 in TStorageFactoryFile::TStorageFactoryFile(char const*, char const*, char const*, int, int, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libIOPoolTFileAdaptor.so
#14 0x000014efb02ed0b9 in ?? ()
#15 0x000014ef00000000 in ?? ()
#16 0x000014ef6dfbb480 in ?? ()
#17 0x000014ef8f80f7e0 in ?? ()
#18 0x00007ffc7fb78c50 in ?? ()
#19 0x000014ef671be1d1 in ?? () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#20 0x00007ffc7fb78d89 in ?? ()
#21 0x000014ef6e1041a0 in ?? ()
#22 0x000014efa850e2d2 in TClingCallFunc::IFacePtr() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libCling.so
#23 0x0000006500000000 in ?? ()
#24 0x000014ef913bb3e0 in ?? ()
#25 0x000014ef6e21b980 in ?? ()
#26 0x00007ffc7fb78d50 in ?? ()
#27 0x00007ffc7fb78d58 in ?? ()
#28 0x000014ef912a6b00 in ?? ()
#29 0x000014efb6b57233 in long TPluginHandler::ExecPluginImpl<char const*, char const*, char const*, int, int>(char const* const&, char const* const&, char const* const&, int const&, int const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libRIO.so
#30 0x000014efb6b5369c in TFile::Open(char const*, char const*, char const*, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/external/el8_amd64_gcc12/lib/libRIO.so
#31 0x000014ef6717d66e in edm::InputFile::InputFile(char const*, char const*, edm::InputType) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#32 0x000014ef671ac904 in edm::RootInputFileSequence::initTheFile(bool, bool, edm::InputSource*, char const*, edm::InputType) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#33 0x000014ef671adfd8 in edm::RootPrimaryFileSequence::RootPrimaryFileSequence(edm::ParameterSet const&, edm::PoolSource&, edm::InputFileCatalog const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#34 0x000014ef6718287f in edm::PoolSource::PoolSource(edm::ParameterSet const&, edm::InputSourceDescription const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#35 0x000014ef6718369e in edmplugin::PluginFactory<edm::InputSource* (edm::ParameterSet const&, edm::InputSourceDescription const&)>::PMaker<edm::PoolSource>::create(edm::ParameterSet const&, edm::InputSourceDescription const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#36 0x000014efb75acfc5 in edm::InputSourceFactory::makeInputSource(edm::ParameterSet const&, edm::InputSourceDescription const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#37 0x000014efb754515c in edm::makeInput(unsigned int, edm::ParameterSet&, edm::CommonParams const&, std::shared_ptr<edm::ProductRegistry>, std::shared_ptr<edm::BranchIDListHelper>, std::shared_ptr<edm::ProcessBlockHelper> const&, std::shared_ptr<edm::ThinnedAssociationsHelper>, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::PreallocationConfiguration const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#38 0x000014efb7578f00 in tbb::detail::d1::function_task<edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy)::{lambda()#2}>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#39 0x000014efb69723e1 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x14efb33c9380) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#40 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x14efb33c9380) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#41 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#42 0x000014efb7546ff1 in edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#43 0x000014efb754a0ee in edm::EventProcessor::EventProcessor(std::shared_ptr<edm::ProcessDesc>, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week1/el8_amd64_gcc12/cms/cmssw/CMSSW_14_2_X_2024-10-13-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#44 0x0000000000408227 in tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const ()
#45 0x000014efb695e9ad in tbb::detail::r1::task_arena_impl::execute (ta=..., d=...) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e785b749a0b6cb9c66dc1d78066210e0/tbb-v2021.9.0/src/tbb/arena.cpp:688
#46 0x000000000040a0f2 in main::{lambda()#1}::operator()() const ()
#47 0x0000000000405100 in main ()
New categories assigned: simulation
@civanch,@kpedro88,@mdhildreth you have been requested to review this Pull request/Issue and eventually sign? Thanks
A new Issue was created by @iarspider.
@Dr15Jones, @antoniovilela, @makortel, @mandrenguyen, @rappoccio, @sextonkennedy, @smuzaffar can you please review it and eventually sign/assign? Thanks.
cms-bot commands are listed here
@pcanal Any thoughts on the stack traces? One thread is opening a TFile, and another thread is writing into another TFile.
But why are we writing into a TFile from edm::MixingCache::Config constructor?
Here is the code https://github.com/cms-sw/cmssw/blob/5c7c91f006f718a3cdb0a5b2839a114a82a5212b/Mixing/Base/src/BMixingModule.cc#L147-L156
The writing is controlled with this condition https://github.com/cms-sw/cmssw/blob/5c7c91f006f718a3cdb0a5b2839a114a82a5212b/Mixing/Base/src/BMixingModule.cc#L86-L87 and indeed that's how the MixingModule is configured in this workflow https://github.com/cms-sw/cmssw/blob/master/SimGeneral/MixingModule/python/mix_Run3_Flat55To75_PoissonOOTPU_cfi.py https://github.com/cms-sw/cmssw/blob/master/SimGeneral/MixingModule/python/mix_probFunction_25ns_PoissonOOTPU_cfi.py#L40
It seems to me the use of the binned probabilities and writing to a file have been strongly coupled. I can understand writing the resulting histogram into a file can be useful in testing, but why is it done in production?
I bet the problem is here https://github.com/cms-sw/cmssw/blob/5c7c91f006f718a3cdb0a5b2839a114a82a5212b/Mixing/Base/src/BMixingModule.cc#L153
what if the histogram attached itself to the wrong file? I.e. not to the one created by the mixing module but the one opened by the Source?
what if the histogram attached itself to the wrong file?
That would be surprising. The line before open/create the file and change gDirectory to point to it and since gDirectory is thread local, nothing can change it in between. (That is unless threading support was not enabled and 2 thread are opening files are the same time).
#3 <signal handler called>
#4 0x00001462272dfec1 in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_2_GPU_X_2024-10-09-2300/external/el8_amd64_gcc12/lib/libRIO.so
#5 0x0000146227384046 in TKey::TKey(TObject const*, char const*, int, TDirectory*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02858/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_14_2_GPU_X_2024-10-09-2300/external/el8_amd64_gcc12/lib/libRIO.so
from this stack trace, we can not tell whether the problem is with the StreamerInfo (since the other thread is stuck while building one) or with the user objects.
Another occurrence: https://cmssdt.cern.ch/SDT/cgi-bin/logreader/el8_amd64_gcc12/CMSSW_15_1_X_2025-04-22-1100/pyRelValMatrixLogs/run/11834.0_TTbar_14TeV+2022PU/step3_TTbar_14TeV+2022PU.log#/132-132
Thread 3 (Thread 0x14b1213b1700 (LWP 910319) "cmsRun"):
#0 0x000014b17bb5cac1 in poll () from /lib64/libc.so.6
#1 0x000014b174e72a47 in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x000014b174e72c44 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014b171adef61 in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#5 0x000014b171ad9155 in TBufferFile::WriteFastArray(void*, TClass const*, long long, TMemberStreamer*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#6 0x000014b171d57710 in int TStreamerInfo::WriteBufferAux<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#7 0x000014b171bae24d in TStreamerInfoActions::GenericWriteAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#8 0x000014b171adf176 in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#9 0x000014b17c948a6b in TStreamerBase::WriteBuffer(TBuffer&, char*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libCore.so
#10 0x000014b171d5a251 in int TStreamerInfo::WriteBufferAux<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#11 0x000014b171bae24d in TStreamerInfoActions::GenericWriteAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#12 0x000014b171adf176 in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#13 0x000014b171b830e6 in TKey::TKey(TObject const*, char const*, int, TDirectory*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#14 0x000014b171b40f15 in TFile::CreateKey(TDirectory*, TObject const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#15 0x000014b171b32086 in TDirectoryFile::WriteTObject(TObject const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#16 0x000014b17c8445b4 in TObject::Write(char const*, int, int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libCore.so
#17 0x000014b11cf9c37c in (anonymous namespace)::maybeConfigPileUp(edm::ParameterSet const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, int, int, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libMixingBase.so
#18 0x000014b11cf9cdae in edm::MixingCache::Config::Config(edm::ParameterSet const&, unsigned int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libMixingBase.so
#19 0x000014b11cf9d097 in edm::BMixingModule::initializeGlobalCache(edm::ParameterSet const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libMixingBase.so
#20 0x000014b11d048d96 in edm::WorkerMaker<edm::MixingModule>::makeModule(edm::ParameterSet const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/pluginSimGeneralMixingModulePlugins.so
#21 0x000014b17ce2dc28 in edm::Maker::makeModule(edm::MakeModuleParams const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so
#22 0x000014b17cda6b88 in edm::Factory::makeModule(edm::MakeModuleParams const&, edm::ModuleTypeResolverMaker const*, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so
#23 0x000014b17cdb228f in edm::ModuleRegistry::getModule(edm::MakeModuleParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so
#24 0x000014b17ce2e42a in edm::WorkerRegistry::getWorker(edm::WorkerParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so
#25 0x000014b17ce2e705 in edm::WorkerManager::getWorker(edm::ParameterSet&, edm::SignallingProductRegistryFiller&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so
#26 0x000014b17ce0c010 in edm::(anonymous namespace)::getWorker(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::ParameterSet&, edm::WorkerManager&, edm::SignallingProductRegistryFiller&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so
#27 0x000014b17ce1687f in edm::StreamSchedule::fillWorkers(edm::ParameterSet&, edm::SignallingProductRegistryFiller&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, std::vector<edm::WorkerInPath, std::allocator<edm::WorkerInPath> >&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, edm::ConditionalTaskHelper const&, std::unordered_set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so
#28 0x000014b17ce1754a in edm::StreamSchedule::fillTrigPath(edm::ParameterSet&, edm::SignallingProductRegistryFiller&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::shared_ptr<edm::HLTGlobalStatus>, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, edm::ConditionalTaskHelper const&, std::unordered_set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so
#29 0x000014b17ce18b74 in edm::StreamSchedule::StreamSchedule(std::shared_ptr<edm::TriggerResultInserter>, std::vector<edm::propagate_const<std::shared_ptr<edm::PathStatusInserter> >, std::allocator<edm::propagate_const<std::shared_ptr<edm::PathStatusInserter> > > >&, std::vector<edm::propagate_const<std::shared_ptr<edm::EndPathStatusInserter> >, std::allocator<edm::propagate_const<std::shared_ptr<edm::EndPathStatusInserter> > > >&, std::shared_ptr<edm::ModuleRegistry>, edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::SignallingProductRegistryFiller&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::StreamID, edm::ProcessContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so
#30 0x000014b17cdfa16a in edm::Schedule::Schedule(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::SignallingProductRegistryFiller&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so
#31 0x000014b17ce0e211 in edm::ScheduleItems::initModules(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so
#32 0x000014b17cd79647 in tbb::detail::d2::function_task<edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so
#33 0x000014b17cb6e60a in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=<optimized out>, waiter=..., this=0x14b179f81e80) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/task_dispatcher.h:334
#34 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x14b179f81e80) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/task_dispatcher.h:470
#35 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/arena.cpp:215
#36 tbb::detail::r1::thread_dispatcher_client::process (td=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/thread_dispatcher_client.h:41
#37 tbb::detail::r1::thread_dispatcher::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/thread_dispatcher.cpp:195
#38 0x000014b17cb666b8 in tbb::detail::r1::rml::private_worker::run (this=0x14b175728100) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/private_server.cpp:271
#39 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x14b175728100) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/private_server.cpp:221
#40 0x000014b17b3e81ca in start_thread () from /lib64/libpthread.so.0
#41 0x000014b17ba638d3 in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x14b17c4c8580 (LWP 910270) "cmsRun"):
#0 0x000014b17bb32098 in nanosleep () from /lib64/libc.so.6
#1 0x000014b17bb31f9e in sleep () from /lib64/libc.so.6
#2 0x000014b174e6f870 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014b17c90dba6 in TClass::GetClass(char const*, bool, bool, unsigned long, unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libCore.so
#5 0x000014b17c938d33 in TProtoClass::FillTClass(TClass*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libCore.so
#6 0x000014b17c91907d in TClass::Init(char const*, short, std::type_info const*, TVirtualIsAProxy*, char const*, char const*, int, int, ClassInfo_t*, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libCore.so
#7 0x000014b17c91a408 in TClass::TClass(char const*, short, std::type_info const&, TVirtualIsAProxy*, char const*, char const*, int, int, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libCore.so
#8 0x000014b17c91a96b in ROOT::CreateClass(char const*, short, std::type_info const&, TVirtualIsAProxy*, char const*, char const*, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libCore.so
#9 0x000014b17c92b392 in ROOT::TGenericClassInfo::GetClass() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libCore.so
#10 0x000014b17c90ddd2 in TClass::GetClass(char const*, bool, bool, unsigned long, unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libCore.so
#11 0x000014b171b97beb in TStreamerInfo::BuildCheck(TFile*, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#12 0x000014b171b41885 in TFile::ReadStreamerInfo() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#13 0x000014b171b4d11a in TFile::Init(bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_15_1_X_2025-04-22-1100/external/el8_amd64_gcc12/lib/libRIO.so
#14 0x000014b175481dca in TStorageFactoryFile::Initialize(char const*, char const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-21-1100/lib/el8_amd64_gcc12/libIOPoolTFileAdaptor.so
#15 0x000014b175482394 in TStorageFactoryFile::TStorageFactoryFile(char const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02886/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-21-1100/lib/el8_amd64_gcc12/libIOPoolTFileAdaptor.so
#16 0x000014b173e5920c in ?? ()
#17 0x0000000000000000 in ?? ()
Other threads are idle.
Another occurance: link
Thread 3 (Thread 0x14565dee2700 (LWP 406098) "cmsRun"):
#0 0x00001456b5b25ac1 in poll () from /lib64/libc.so.6
#1 0x00001456b1319a47 in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x00001456b1319c44 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x00001456b0805f61 in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#5 0x00001456b08aa0e6 in TKey::TKey(TObject const*, char const*, int, TDirectory*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#6 0x00001456b0867f15 in TFile::CreateKey(TDirectory*, TObject const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#7 0x00001456b0859086 in TDirectoryFile::WriteTObject(TObject const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#8 0x00001456b65115b4 in TObject::Write(char const*, int, int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#9 0x0000145659ac837c in (anonymous namespace)::maybeConfigPileUp(edm::ParameterSet const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, int, int, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libMixingBase.so
#10 0x0000145659ac8dae in edm::MixingCache::Config::Config(edm::ParameterSet const&, unsigned int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libMixingBase.so
#11 0x0000145659ac9097 in edm::BMixingModule::initializeGlobalCache(edm::ParameterSet const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libMixingBase.so
#12 0x0000145659b74d96 in edm::WorkerMaker<edm::MixingModule>::makeModule(edm::ParameterSet const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/pluginSimGeneralMixingModulePlugins.so
#13 0x00001456b6a1f678 in edm::Maker::makeModule(edm::MakeModuleParams const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x00001456b6996568 in edm::Factory::makeModule(edm::MakeModuleParams const&, edm::ModuleTypeResolverMaker const*, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x00001456b69a57bf in edm::ModuleRegistry::getModule(edm::MakeModuleParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x00001456b6a1fe7a in edm::WorkerRegistry::getWorker(edm::WorkerParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#17 0x00001456b6a20155 in edm::WorkerManager::getWorker(edm::ParameterSet&, edm::SignallingProductRegistryFiller&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#18 0x00001456b69fda90 in edm::(anonymous namespace)::getWorker(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::ParameterSet&, edm::WorkerManager&, edm::SignallingProductRegistryFiller&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#19 0x00001456b6a083cf in edm::StreamSchedule::fillWorkers(edm::ParameterSet&, edm::SignallingProductRegistryFiller&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, std::vector<edm::WorkerInPath, std::allocator<edm::WorkerInPath> >&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, edm::ConditionalTaskHelper const&, std::unordered_set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#20 0x00001456b6a0909a in edm::StreamSchedule::fillTrigPath(edm::ParameterSet&, edm::SignallingProductRegistryFiller&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::shared_ptr<edm::HLTGlobalStatus>, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, edm::ConditionalTaskHelper const&, std::unordered_set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#21 0x00001456b6a0a6c4 in edm::StreamSchedule::StreamSchedule(std::shared_ptr<edm::TriggerResultInserter>, std::vector<edm::propagate_const<std::shared_ptr<edm::PathStatusInserter> >, std::allocator<edm::propagate_const<std::shared_ptr<edm::PathStatusInserter> > > >&, std::vector<edm::propagate_const<std::shared_ptr<edm::EndPathStatusInserter> >, std::allocator<edm::propagate_const<std::shared_ptr<edm::EndPathStatusInserter> > > >&, std::shared_ptr<edm::ModuleRegistry>, edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::SignallingProductRegistryFiller&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::StreamID, edm::ProcessContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#22 0x00001456b69ec0d2 in edm::Schedule::Schedule(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::SignallingProductRegistryFiller&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#23 0x00001456b69ffc91 in edm::ScheduleItems::initModules(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#24 0x00001456b696bfc7 in tbb::detail::d2::function_task<edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#25 0x00001456b6b7e60a in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=<optimized out>, waiter=..., this=0x1456b3b81e80) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/task_dispatcher.h:334
#26 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x1456b3b81e80) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/task_dispatcher.h:470
#27 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/arena.cpp:215
#28 tbb::detail::r1::thread_dispatcher_client::process (td=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/thread_dispatcher_client.h:41
#29 tbb::detail::r1::thread_dispatcher::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/thread_dispatcher.cpp:195
#30 0x00001456b6b766b8 in tbb::detail::r1::rml::private_worker::run (this=0x1456b18cf100) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/private_server.cpp:271
#31 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x1456b18cf100) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-10fdc07bd3d228f4c274203a8cd20b5c/tbb-v2022.0.0/src/tbb/private_server.cpp:221
#32 0x00001456b52321ca in start_thread () from /lib64/libpthread.so.0
#33 0x00001456b5a2c8d3 in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x1456b4499580 (LWP 405194) "cmsRun"):
#0 0x00001456b5afb098 in nanosleep () from /lib64/libc.so.6
#1 0x00001456b5afaf9e in sleep () from /lib64/libc.so.6
#2 0x00001456b1316870 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014569b178957 in clang::serialization::MultiOnDiskHashTable<clang::serialization::reader::ASTDeclContextNameLookupTrait>::find(clang::DeclarationName const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#5 0x000014569b186f62 in clang::ASTReader::FindExternalVisibleDeclsByName(clang::DeclContext const*, clang::DeclarationName) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#6 0x000014569b2782a8 in clang::MultiplexExternalSemaSource::FindExternalVisibleDeclsByName(clang::DeclContext const*, clang::DeclarationName) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#7 0x000014569d3ce4e6 in clang::DeclContext::lookup(clang::DeclarationName) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#8 0x000014569b7e65a4 in LookupDirect(clang::Sema&, clang::LookupResult&, clang::DeclContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#9 0x000014569b7e71ce in CppNamespaceLookup(clang::Sema&, clang::LookupResult&, clang::ASTContext&, clang::DeclContext*, (anonymous namespace)::UnqualUsingDirectiveSet&) [clone .constprop.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#10 0x000014569b7df113 in clang::Sema::CppLookupName(clang::LookupResult&, clang::Scope*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#11 0x000014569b7e0145 in clang::Sema::LookupName(clang::LookupResult&, clang::Scope*, bool, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#12 0x000014569a698db7 in cling::utils::Lookup::Named(clang::Sema*, clang::DeclarationName const&, clang::DeclContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#13 0x000014569a698e31 in cling::utils::Lookup::Named(clang::Sema*, llvm::StringRef, clang::DeclContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#14 0x000014569a59c6bb in cling::quickFindDecl(llvm::StringRef, clang::Decl const*&, clang::Parser&, cling::LookupHelper::DiagSetting) [clone .isra.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#15 0x000014569a5a36d9 in cling::LookupHelper::findScope(llvm::StringRef, cling::LookupHelper::DiagSetting, clang::Type const**, bool) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#16 0x000014569a4bfa72 in TCling::CheckClassInfo(char const*, bool, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCling.so
#17 0x00001456b65e5ef2 in TClass::Init(char const*, short, std::type_info const*, TVirtualIsAProxy*, char const*, char const*, int, int, ClassInfo_t*, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#18 0x00001456b65e7408 in TClass::TClass(char const*, short, std::type_info const&, TVirtualIsAProxy*, char const*, char const*, int, int, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#19 0x00001456b65e796b in ROOT::CreateClass(char const*, short, std::type_info const&, TVirtualIsAProxy*, char const*, char const*, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#20 0x00001456b65f8392 in ROOT::TGenericClassInfo::GetClass() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#21 0x00001456b0ba7cb1 in TBranchElement::Dictionary() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libTree.so
#22 0x00001456b65dadd2 in TClass::GetClass(char const*, bool, bool, unsigned long, unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#23 0x00001456b08bebeb in TStreamerInfo::BuildCheck(TFile*, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#24 0x00001456b0868885 in TFile::ReadStreamerInfo() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#25 0x00001456b087411a in TFile::Init(bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#26 0x00001456b1287daa in TStorageFactoryFile::Initialize(char const*, char const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libIOPoolTFileAdaptor.so
#27 0x00001456b1288374 in TStorageFactoryFile::TStorageFactoryFile(char const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02887/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-04-29-2300/lib/el8_amd64_gcc12/libIOPoolTFileAdaptor.so
#28 0x00001456ad68b20c in ?? ()
#29 0x0000000000000000 in ?? ()
other threads are idle.
This is (hopefully) fixed by https://github.com/root-project/root/pull/18522
@makortel , I guess we need https://github.com/root-project/root/pull/18522 fix for root 6.32 ... right?
@pcanal , do we have a backport of https://github.com/root-project/root/pull/18522 for root 6.32? If not , can you please backport it
thanks @pcanal . I have opened https://github.com/cms-sw/cmsdist/pull/9889 to test root 6-32-00-patches branch changes for 15.1.X
Happened again: link
Thread 4 (Thread 0x150aef622700 (LWP 2682602) "cmsRun"):
#0 0x0000150b4635bba1 in poll () from /lib64/libc.so.6
#1 0x0000150b419021ce in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x0000150b419023df in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x0000150b40edfd31 in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#5 0x0000150b40ed9ce9 in TBufferFile::WriteFastArray(void*, TClass const*, long long, TMemberStreamer*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#6 0x0000150b4115cf75 in int TStreamerInfo::WriteBufferAux<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#7 0x0000150b40fb2f3c in TStreamerInfoActions::GenericWriteAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#8 0x0000150b40edff4e in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#9 0x0000150b46d51e7c in TStreamerBase::WriteBuffer(TBuffer&, char*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#10 0x0000150b4115fd45 in int TStreamerInfo::WriteBufferAux<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#11 0x0000150b40fb2f3c in TStreamerInfoActions::GenericWriteAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#12 0x0000150b40edff4e in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#13 0x0000150b40f877c3 in TKey::TKey(TObject const*, char const*, int, TDirectory*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#14 0x0000150b40f44b1e in TFile::CreateKey(TDirectory*, TObject const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#15 0x0000150b40f355ef in TDirectoryFile::WriteTObject(TObject const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#16 0x0000150b46c463c3 in TObject::Write(char const*, int, int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#17 0x0000150aea6aba05 in (anonymous namespace)::maybeConfigPileUp(edm::ParameterSet const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, int, int, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libMixingBase.so
#18 0x0000150aea6ac48e in edm::MixingCache::Config::Config(edm::ParameterSet const&, unsigned int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libMixingBase.so
#19 0x0000150aea6ac780 in edm::BMixingModule::initializeGlobalCache(edm::ParameterSet const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libMixingBase.so
#20 0x0000150aea75ac97 in edm::ModuleMaker<edm::MixingModule>::makeModule(edm::ParameterSet const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/pluginSimGeneralMixingModulePlugins.so
#21 0x0000150b471c2af7 in edm::ModuleMakerBase::makeModule(edm::MakeModuleParams const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#22 0x0000150b471bacd9 in edm::ModuleHolderFactory::makeModule(edm::MakeModuleParams const&, edm::ModuleTypeResolverMaker const*, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#23 0x0000150b471c2d1e in edm::ModuleRegistry::getModule(edm::MakeModuleParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#24 0x0000150b472438c2 in edm::WorkerRegistry::getWorker(edm::WorkerParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#25 0x0000150b47243a5c in edm::WorkerManager::getWorker(edm::ParameterSet&, edm::SignallingProductRegistryFiller&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#26 0x0000150b47268327 in edm::(anonymous namespace)::getWorker(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::ParameterSet&, edm::WorkerManager&, edm::SignallingProductRegistryFiller&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, bool) [clone .constprop.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#27 0x0000150b4723307f in edm::StreamSchedule::fillWorkers(edm::ParameterSet&, edm::SignallingProductRegistryFiller&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, edm::ConditionalTaskHelper const&, std::unordered_set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#28 0x0000150b47233d85 in edm::StreamSchedule::fillTrigPath(edm::ParameterSet&, edm::SignallingProductRegistryFiller&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, int, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::shared_ptr<edm::HLTGlobalStatus>, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, edm::ConditionalTaskHelper const&, std::unordered_set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#29 0x0000150b47228978 in edm::StreamSchedule::StreamSchedule(std::shared_ptr<edm::TriggerResultInserter>, std::vector<edm::propagate_const<std::shared_ptr<edm::PathStatusInserter> >, std::allocator<edm::propagate_const<std::shared_ptr<edm::PathStatusInserter> > > >&, std::vector<edm::propagate_const<std::shared_ptr<edm::EndPathStatusInserter> >, std::allocator<edm::propagate_const<std::shared_ptr<edm::EndPathStatusInserter> > > >&, std::shared_ptr<edm::ModuleRegistry>, edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::SignallingProductRegistryFiller&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::StreamID, edm::ProcessContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#30 0x0000150b4720fcea in edm::Schedule::Schedule(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::SignallingProductRegistryFiller&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#31 0x0000150b472224e4 in edm::ScheduleItems::initModules(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#32 0x0000150b4717ff47 in tbb::detail::d2::function_task<edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#33 0x0000150b4750e7b3 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=<optimized out>, waiter=..., this=0x150b449d1e80) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-d05167f8f23dd5d0fceb24869ee49ed9/tbb-v2022.0.0/src/tbb/task_dispatcher.h:334
#34 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x150b449d1e80) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-d05167f8f23dd5d0fceb24869ee49ed9/tbb-v2022.0.0/src/tbb/task_dispatcher.h:470
#35 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-d05167f8f23dd5d0fceb24869ee49ed9/tbb-v2022.0.0/src/tbb/arena.cpp:215
#36 tbb::detail::r1::thread_dispatcher_client::process (td=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-d05167f8f23dd5d0fceb24869ee49ed9/tbb-v2022.0.0/src/tbb/thread_dispatcher_client.h:41
#37 tbb::detail::r1::thread_dispatcher::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-d05167f8f23dd5d0fceb24869ee49ed9/tbb-v2022.0.0/src/tbb/thread_dispatcher.cpp:195
#38 0x0000150b47506961 in tbb::detail::r1::rml::private_worker::run (this=0x150b420f4080) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-d05167f8f23dd5d0fceb24869ee49ed9/tbb-v2022.0.0/src/tbb/private_server.cpp:271
#39 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x150b420f4080) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-d05167f8f23dd5d0fceb24869ee49ed9/tbb-v2022.0.0/src/tbb/private_server.cpp:221
#40 0x0000150b45be81ca in start_thread () from /lib64/libpthread.so.0
#41 0x0000150b462628d3 in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x150b46832580 (LWP 2682282) "cmsRun"):
#0 0x0000150b46331178 in nanosleep () from /lib64/libc.so.6
#1 0x0000150b4633107e in sleep () from /lib64/libc.so.6
#2 0x0000150b418fef5c in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x0000150b46308349 in __memset_avx512_unaligned_erms () from /lib64/libc.so.6
#5 0x0000150b46c692e1 in TStorage::ObjectAlloc(unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#6 0x0000150b46d19b94 in ROOT::CreateClass(char const*, short, std::type_info const&, TVirtualIsAProxy*, char const*, char const*, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#7 0x0000150b46d33b9e in ROOT::TGenericClassInfo::GetClass() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#8 0x0000150aea652497 in ROOT::edmcLcLWrapperlEvectorlEPCaloHitgRsPgR_Dictionary() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02896/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libSimDataFormatsCaloHit.so
#9 0x0000150b46d1a832 in TClass::GetClass(char const*, bool, bool, unsigned long, unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#10 0x0000150b40f9c656 in TStreamerInfo::BuildCheck(TFile*, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#11 0x0000150b40f454d7 in TFile::ReadStreamerInfo() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#12 0x0000150b40f50e14 in TFile::Init(bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#13 0x0000150b41ade779 in TStorageFactoryFile::Initialize(char const*, char const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libIOPoolTFileAdaptor.so
#14 0x0000150b41aded65 in TStorageFactoryFile::TStorageFactoryFile(char const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libIOPoolTFileAdaptor.so
#15 0x0000150b3f86f20c in ?? ()
#16 0x0000150af3ccd9a0 in ?? ()
#17 0x0000150b46ea1da0 in gThreadTsd () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#18 0x00007ffe3ed059a0 in ?? ()
#19 0x0000150b46bf452c in ROOT::TVirtualRWMutex::UnLock() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#20 0x0000150b40f5ae64 in long TPluginHandler::ExecPluginImpl<char const*, char const*, char const*, int>(char const* const&, char const* const&, char const* const&, int const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#21 0x0000150b40f56c9e in TFile::Open(char const*, char const*, char const*, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
#22 0x0000150aee1c08ba in edm::InputFile::InputFile(char const*, char const*, edm::InputType) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#23 0x0000150aee1f5845 in edm::RootInputFileSequence::initTheFile(bool, bool, edm::InputSource*, char const*, edm::InputType) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#24 0x0000150aee1f6b28 in edm::RootPrimaryFileSequence::RootPrimaryFileSequence(edm::ParameterSet const&, edm::PoolSource&, edm::InputFileCatalog const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#25 0x0000150aee1c3ec1 in edm::PoolSource::PoolSource(edm::ParameterSet const&, edm::InputSourceDescription const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#26 0x0000150aee1c4ebf in edmplugin::PluginFactory<edm::InputSource* (edm::ParameterSet const&, edm::InputSourceDescription const&)>::PMaker<edm::PoolSource>::create(edm::ParameterSet const&, edm::InputSourceDescription const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#27 0x0000150b471b0649 in edm::InputSourceFactory::makeInputSource(edm::ParameterSet const&, edm::InputSourceDescription const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#28 0x0000150b4714abea in edm::makeInput(unsigned int, edm::ParameterSet&, edm::CommonParams const&, std::shared_ptr<edm::BranchIDListHelper>, std::shared_ptr<edm::ProcessBlockHelper> const&, std::shared_ptr<edm::ThinnedAssociationsHelper>, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::PreallocationConfiguration const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#29 0x0000150b4717e833 in tbb::detail::d2::function_task<edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy)::{lambda()#2}>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#30 0x0000150b4750bbc3 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x150b449db100, this=0x150b449d1e00) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-d05167f8f23dd5d0fceb24869ee49ed9/tbb-v2022.0.0/src/tbb/task_dispatcher.h:334
#31 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x150b449d1e00) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-d05167f8f23dd5d0fceb24869ee49ed9/tbb-v2022.0.0/src/tbb/task_dispatcher.h:470
#32 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-d05167f8f23dd5d0fceb24869ee49ed9/tbb-v2022.0.0/src/tbb/task_dispatcher.cpp:168
#33 0x0000150b4714e34f in edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#34 0x0000150b47151b52 in edm::EventProcessor::EventProcessor(std::shared_ptr<edm::ProcessDesc>, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#35 0x00000000004084ac in tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const ()
#36 0x0000150b474fa111 in tbb::detail::r1::task_arena_impl::execute (ta=..., d=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2022.0.0-d05167f8f23dd5d0fceb24869ee49ed9/tbb-v2022.0.0/src/tbb/arena.cpp:821
#37 0x000000000040a438 in main::{lambda()#1}::operator()() const ()
#38 0x00000000004051c8 in main ()
(other threads were idle)
I am a little baffled by the 2nd stack trace which either inaccurate of something went very wrong earlier ... it says:
#14 0x0000150b41aded65 in TStorageFactoryFile::TStorageFactoryFile(char const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/lib/el8_amd64_gcc12/libIOPoolTFileAdaptor.so
#15 0x0000150b3f86f20c in ?? ()
#16 0x0000150af3ccd9a0 in ?? ()
#17 0x0000150b46ea1da0 in gThreadTsd () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#18 0x00007ffe3ed059a0 in ?? ()
#19 0x0000150b46bf452c in ROOT::TVirtualRWMutex::UnLock() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libCore.so
#20 0x0000150b40f5ae64 in long TPluginHandler::ExecPluginImpl<char const*, char const*, char const*, int>(char const* const&, char const* const&, char const* const&, int const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_15_1_FP_X_2025-06-29-2300/external/el8_amd64_gcc12/lib/libRIO.so
Which seems to say we are opening a file while executing the mutex unlock routine .. !?
Happened again in a different workflow, and only when running on Nvidia_t4 GPU in el9: link
Thread 3 (Thread 0x14653e255640 (LWP 2572233) "cmsRun"):
#0 0x00001465b9046fdf in poll () from /lib64/libc.so.6
#1 0x00001465b45f6297 in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02902/el9_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-08-11-2300/lib/el9_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x00001465b45f6494 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02902/el9_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-08-11-2300/lib/el9_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x00001465b3c30f31 in TBufferFile::WriteClassBuffer(TClass const*, void*) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libRIO.so
#5 0x00001465b3cd4f06 in TKey::TKey(TObject const*, char const*, int, TDirectory*) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libRIO.so
#6 0x00001465b3c92d45 in TFile::CreateKey(TDirectory*, TObject const*, char const*, int) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libRIO.so
#7 0x00001465b3c83f66 in TDirectoryFile::WriteTObject(TObject const*, char const*, char const*, int) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libRIO.so
#8 0x00001465b98ea464 in TObject::Write(char const*, int, int) const () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCore.so
#9 0x000014653bca52ec in (anonymous namespace)::maybeConfigPileUp(edm::ParameterSet const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, int, int, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02902/el9_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-08-11-2300/lib/el9_amd64_gcc12/libMixingBase.so
#10 0x000014653bca5d1e in edm::MixingCache::Config::Config(edm::ParameterSet const&, unsigned int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02902/el9_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-08-11-2300/lib/el9_amd64_gcc12/libMixingBase.so
#11 0x000014653bca6007 in edm::BMixingModule::initializeGlobalCache(edm::ParameterSet const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02902/el9_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-08-11-2300/lib/el9_amd64_gcc12/libMixingBase.so
#12 0x000014653bd53433 in edm::ModuleMaker<edm::MixingModule>::makeModule(edm::ParameterSet const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02902/el9_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-08-11-2300/lib/el9_amd64_gcc12/pluginSimGeneralMixingModulePlugins.so
#13 0x00001465b9f23067 in edm::ModuleMakerBase::makeModule(edm::MakeModuleParams const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from .../CMSSW_15_1_X_2025-08-12-2300/lib/el9_amd64_gcc12/libFWCoreFramework.so
#14 0x00001465b9f1a628 in edm::ModuleHolderFactory::makeModule(edm::MakeModuleParams const&, edm::ModuleTypeResolverMaker const*, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from .../CMSSW_15_1_X_2025-08-12-2300/lib/el9_amd64_gcc12/libFWCoreFramework.so
#15 0x00001465b9f233c7 in edm::ModuleRegistry::getModule(edm::MakeModuleParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) () from .../CMSSW_15_1_X_2025-08-12-2300/lib/el9_amd64_gcc12/libFWCoreFramework.so
#16 0x00001465b9f7cf05 in edm::(anonymous namespace)::getModule(edm::ParameterSet&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::ModuleRegistry&, edm::SignallingProductRegistryFiller&, edm::ActivityRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>) () from .../CMSSW_15_1_X_2025-08-12-2300/lib/el9_amd64_gcc12/libFWCoreFramework.so
#17 0x00001465b9f80fa8 in edm::(anonymous namespace)::fillModulesInPath(edm::ParameterSet&, edm::ModuleRegistry&, edm::SignallingProductRegistryFiller&, edm::ActivityRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, edm::(anonymous namespace)::ConditionalTaskHelper const&, std::unordered_set<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > >&) () from .../CMSSW_15_1_X_2025-08-12-2300/lib/el9_amd64_gcc12/libFWCoreFramework.so
#18 0x00001465b9f83b2d in edm::ScheduleBuilder::ScheduleBuilder(edm::ModuleRegistry&, edm::ParameterSet&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, edm::PreallocationConfiguration const&, edm::SignallingProductRegistryFiller&, edm::ActivityRegistry&, std::shared_ptr<edm::ProcessConfiguration const>) () from .../CMSSW_15_1_X_2025-08-12-2300/lib/el9_amd64_gcc12/libFWCoreFramework.so
#19 0x00001465b9f6f066 in edm::Schedule::Schedule(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::SignallingProductRegistryFiller&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from .../CMSSW_15_1_X_2025-08-12-2300/lib/el9_amd64_gcc12/libFWCoreFramework.so
#20 0x00001465b9f8a6f5 in edm::ScheduleItems::initModules(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from .../CMSSW_15_1_X_2025-08-12-2300/lib/el9_amd64_gcc12/libFWCoreFramework.so
#21 0x00001465b9eec6c7 in tbb::detail::d2::function_task<edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from .../CMSSW_15_1_X_2025-08-12-2300/lib/el9_amd64_gcc12/libFWCoreFramework.so
#22 0x00001465b93c15da in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=<optimized out>, waiter=..., this=0x1465b7db3e00) at .../tbb/v2022.0.0-2c8b19d7f71a88d9ed1f550c4776837f/tbb-v2022.0.0/src/tbb/task_dispatcher.h:334
#23 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x1465b7db3e00) at .../tbb/v2022.0.0-2c8b19d7f71a88d9ed1f550c4776837f/tbb-v2022.0.0/src/tbb/task_dispatcher.h:470
#24 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at .../tbb/v2022.0.0-2c8b19d7f71a88d9ed1f550c4776837f/tbb-v2022.0.0/src/tbb/arena.cpp:215
#25 tbb::detail::r1::thread_dispatcher_client::process (td=..., this=<optimized out>) at .../tbb/v2022.0.0-2c8b19d7f71a88d9ed1f550c4776837f/tbb-v2022.0.0/src/tbb/thread_dispatcher_client.h:41
#26 tbb::detail::r1::thread_dispatcher::process (this=<optimized out>, j=...) at .../tbb/v2022.0.0-2c8b19d7f71a88d9ed1f550c4776837f/tbb-v2022.0.0/src/tbb/thread_dispatcher.cpp:195
#27 0x00001465b93b9688 in tbb::detail::r1::rml::private_worker::run (this=0x1465b4df3100) at .../tbb/v2022.0.0-2c8b19d7f71a88d9ed1f550c4776837f/tbb-v2022.0.0/src/tbb/private_server.cpp:271
#28 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x1465b4df3100) at .../tbb/v2022.0.0-2c8b19d7f71a88d9ed1f550c4776837f/tbb-v2022.0.0/src/tbb/private_server.cpp:221
#29 0x00001465b8fcf19a in start_thread () from /lib64/libc.so.6
#30 0x00001465b9053504 in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x1465b86b0540 (LWP 2572146) "cmsRun"):
#0 0x00001465b9019655 in clock_nanosleep@GLIBC_2.2.5 () from /lib64/libc.so.6
#1 0x00001465b901e247 in nanosleep () from /lib64/libc.so.6
#2 0x00001465b901e17e in sleep () from /lib64/libc.so.6
#3 0x00001465b45f30c0 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02902/el9_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-08-11-2300/lib/el9_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x00001465a369d267 in clang::ASTReader::readTypeRecord(unsigned int) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#6 0x00001465a369d7fc in clang::ASTReader::GetType(unsigned int) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#7 0x00001465a36ec292 in clang::ASTDeclReader::Visit(clang::Decl*) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#8 0x00001465a36ecc0d in clang::ASTReader::ReadDeclRecord(unsigned int) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#9 0x00001465a367f332 in clang::ASTRecordReader::readDecl() () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#10 0x00001465a36a994f in clang::serialization::AbstractTypeReader<clang::ASTRecordReader>::read(clang::Type::TypeClass) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#11 0x00001465a369d46f in clang::ASTReader::readTypeRecord(unsigned int) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#12 0x00001465a369d7fc in clang::ASTReader::GetType(unsigned int) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#13 0x00001465a36a9fba in clang::serialization::AbstractTypeReader<clang::ASTRecordReader>::read(clang::Type::TypeClass) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#14 0x00001465a369d46f in clang::ASTReader::readTypeRecord(unsigned int) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#15 0x00001465a369d7fc in clang::ASTReader::GetType(unsigned int) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#16 0x00001465a36a0b22 in clang::ASTRecordReader::readTypeSourceInfo() () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#17 0x00001465a36a2fbd in clang::ASTReader::GetExternalCXXBaseSpecifiers(unsigned long) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#18 0x00001465a379bf05 in clang::MultiplexExternalSemaSource::GetExternalCXXBaseSpecifiers(unsigned long) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#19 0x00001465a58efb71 in clang::CXXRecordDecl::DefinitionData::getBasesSlowCase() const () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#20 0x00001465a2a1f655 in TClingBaseClassInfo::TClingBaseClassInfo(cling::Interpreter*, TClingClassInfo*) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#21 0x00001465a29be313 in TCling::CreateListOfBaseClasses(TClass*) const () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCling.so
#22 0x00001465b99ac144 in TClass::GetListOfBases() () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCore.so
#23 0x00001465b99b427c in TClass::GetCheckSum(TClass::ECheckSum, bool&) const () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCore.so
#24 0x00001465b99b4b6e in TClass::GetCheckSum(TClass::ECheckSum) const () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCore.so
#25 0x00001465b3cea985 in TStreamerInfo::BuildCheck(TFile*, bool) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libRIO.so
#26 0x00001465b3c93d75 in TFile::ReadStreamerInfo() () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libRIO.so
#27 0x00001465b3c9ef3a in TFile::Init(bool) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libRIO.so
#28 0x00001465b47d4b50 in TStorageFactoryFile::Initialize(char const*, char const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02902/el9_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-08-11-2300/lib/el9_amd64_gcc12/libIOPoolTFileAdaptor.so
#29 0x00001465b47d5090 in TStorageFactoryFile::TStorageFactoryFile(char const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02902/el9_amd64_gcc12/cms/cmssw/CMSSW_15_1_X_2025-08-11-2300/lib/el9_amd64_gcc12/libIOPoolTFileAdaptor.so
#30 0x00001465b13a420c in ?? ()
#31 0x00001465b9b48da0 in gThreadTsd () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCore.so
#32 0x00001465b4a318e0 in ROOT::TReentrantRWLock<std::mutex, ROOT::Internal::RecurseCountsTBBUnique>::WriteUnLock(ROOT::TVirtualRWMutex::Hint_t*) () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libThread.so
#33 0x00001465b9899edc in ROOT::TVirtualRWMutex::UnLock() () from .../CMSSW_15_1_X_2025-08-12-2300/external/el9_amd64_gcc12/lib/libCore.so
#34 0x00007ffc7fed4789 in ?? ()
#35 0x0000146540219520 in ?? ()
#36 0x00000003aa2c5e00 in ?? ()
#37 0x00007ffc7fed4660 in ?? ()
#38 0x0000000000000000 in ?? ()
This is (hopefully) fixed by root-project/root#18522
I guess we can say that that PR did not fix the problem (or we have another one).
Although I could repeat my question (to @cms-sw/simulation-l2 I guess)
But why are we writing into a
TFilefromedm::MixingCache::Configconstructor?
I had to double check this: https://github.com/cms-sw/cmssw/blob/ab50343a75a8e1ae33dd3dcbeafa91390b9c22d7/Mixing/Base/src/BMixingModule.cc#L151
When the pileup parameters are provided via Python lists, they are turned into a histogram and then written to a ROOT file.
This code was introduced in 2010 (https://github.com/cms-sw/cmssw/commit/110f5fc7109537073865bc57dda0618ffb765791) and almost certainly has not been critically examined since then... most likely this option could be disabled by default in production and left there for debugging (unless there is some production use of this file of which I am unaware).
This error still happens, e.g. CMSSW_16_0_ROOT636_X-2025-10-09-1100 for el8_amd64_gcc14 - RelVal 11846.19
Thread 17 (Thread 0x148ad288f700 (LWP 1599084) "cmsRun"):
#0 0x0000148b2095bbb1 in poll () from /lib64/libc.so.6
#1 0x0000148b1a52b65e in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/pluginFWCoreServicesPlugins.so
#2 0x0000148b1a52b863 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x0000148b19922b61 in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/external/el8_amd64_gcc14/lib/libRIO.so
#5 0x0000148b199d5dce in TKey::TKey(TObject const*, char const*, int, TDirectory*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/external/el8_amd64_gcc14/lib/libRIO.so
#6 0x0000148b1998a1be in TFile::CreateKey(TDirectory*, TObject const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/external/el8_amd64_gcc14/lib/libRIO.so
#7 0x0000148b19978bc0 in TDirectoryFile::WriteTObject(TObject const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/external/el8_amd64_gcc14/lib/libRIO.so
#8 0x0000148b2124a37b in TObject::Write(char const*, int, int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/external/el8_amd64_gcc14/lib/libCore.so
#9 0x0000148abdffe475 in (anonymous namespace)::maybeConfigPileUp(edm::ParameterSet const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, int, int, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/libMixingBase.so
#10 0x0000148abdfff0da in edm::MixingCache::Config::Config(edm::ParameterSet const&, unsigned int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/libMixingBase.so
#11 0x0000148abe0001d0 in edm::BMixingModule::initializeGlobalCache(edm::ParameterSet const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/libMixingBase.so
#12 0x0000148abe0b17c7 in edm::ModuleMaker<edm::MixingModule>::makeModule(edm::ParameterSet const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/pluginSimGeneralMixingModulePlugins.so
#13 0x0000148b217bb48d in edm::ModuleMakerBase::makeModule(edm::MakeModuleParams const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/libFWCoreFramework.so
#14 0x0000148b217b24a9 in edm::ModuleHolderFactory::makeModule(edm::MakeModuleParams const&, edm::ModuleTypeResolverMaker const*, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/libFWCoreFramework.so
#15 0x0000148b217bc31f in edm::ModuleRegistry::getModule(edm::MakeModuleParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/libFWCoreFramework.so
#16 0x0000148b2181124e in edm::(anonymous namespace)::getModule(edm::ParameterSet&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::ModuleRegistry&, edm::SignallingProductRegistryFiller&, edm::ActivityRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/libFWCoreFramework.so
#17 0x0000148b2181c66f in edm::ScheduleBuilder::ScheduleBuilder(edm::ModuleRegistry&, edm::ParameterSet&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, edm::PreallocationConfiguration const&, edm::SignallingProductRegistryFiller&, edm::ActivityRegistry&, std::shared_ptr<edm::ProcessConfiguration const>) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/libFWCoreFramework.so
#18 0x0000148b21808e8a in edm::Schedule::Schedule(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::SignallingProductRegistryFiller&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/libFWCoreFramework.so
#19 0x0000148b2181f168 in edm::ScheduleItems::initModules(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/libFWCoreFramework.so
#20 0x0000148b21782ca6 in tbb::detail::d2::function_task<edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy)::{lambda()#2}>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02910/el8_amd64_gcc14/cms/cmssw/CMSSW_16_0_ROOT636_X_2025-10-09-1100/lib/el8_amd64_gcc14/libFWCoreFramework.so
#21 0x0000148b219da123 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=<optimized out>, this=0x148b1efd1e80, waiter=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc14/external/tbb/v2022.0.0-55f685bd9510a32cc19002d62d7b507b/tbb-v2022.0.0/src/tbb/task_dispatcher.h:334
#22 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, this=0x148b1efd1e80, waiter=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc14/external/tbb/v2022.0.0-55f685bd9510a32cc19002d62d7b507b/tbb-v2022.0.0/src/tbb/task_dispatcher.h:470
#23 tbb::detail::r1::arena::process (this=<optimized out>, tls=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc14/external/tbb/v2022.0.0-55f685bd9510a32cc19002d62d7b507b/tbb-v2022.0.0/src/tbb/arena.cpp:215
#24 tbb::detail::r1::thread_dispatcher_client::process (this=<optimized out>, td=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc14/external/tbb/v2022.0.0-55f685bd9510a32cc19002d62d7b507b/tbb-v2022.0.0/src/tbb/thread_dispatcher_client.h:41
#25 tbb::detail::r1::thread_dispatcher::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc14/external/tbb/v2022.0.0-55f685bd9510a32cc19002d62d7b507b/tbb-v2022.0.0/src/tbb/thread_dispatcher.cpp:195
#26 0x0000148b219d1d61 in tbb::detail::r1::rml::private_worker::run (this=0x148b1bbb5100) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc14/external/tbb/v2022.0.0-55f685bd9510a32cc19002d62d7b507b/tbb-v2022.0.0/src/tbb/private_server.cpp:271
#27 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x148b1bbb5100) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc14/external/tbb/v2022.0.0-55f685bd9510a32cc19002d62d7b507b/tbb-v2022.0.0/src/tbb/private_server.cpp:221
#28 0x0000148b201e81ca in start_thread () from /lib64/libpthread.so.0
#29 0x0000148b208628d3 in clone () from /lib64/libc.so.6
Failed again: link
Thread 3 (Thread 0x14916faec700 (LWP 2250791) "cmsRun"):
#0 0x00001491ecca1bb1 in poll () from /lib64/libc.so.6
#1 0x00001491e7f5b88e in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/pluginFWCoreServicesPlugins.so
#2 0x00001491e7f5ba93 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x00001491e7458921 in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libRIO.so
#5 0x00001491e74528fa in TBufferFile::WriteFastArray(void*, TClass const*, long long, TMemberStreamer*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libRIO.so
#6 0x00001491e76db1da in int TStreamerInfo::WriteBufferAux<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libRIO.so
#7 0x00001491e752f22c in TStreamerInfoActions::GenericWriteAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libRIO.so
#8 0x00001491e7458b3e in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libRIO.so
#9 0x00001491ed64458c in TStreamerBase::WriteBuffer(TBuffer&, char*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libCore.so
#10 0x00001491e76dda0e in int TStreamerInfo::WriteBufferAux<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libRIO.so
#11 0x00001491e752f22c in TStreamerInfoActions::GenericWriteAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libRIO.so
#12 0x00001491e7458b3e in TBufferFile::WriteClassBuffer(TClass const*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libRIO.so
#13 0x00001491e7501f43 in TKey::TKey(TObject const*, char const*, int, TDirectory*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libRIO.so
#14 0x00001491e74bcbee in TFile::CreateKey(TDirectory*, TObject const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libRIO.so
#15 0x00001491e74ad23f in TDirectoryFile::WriteTObject(TObject const*, char const*, char const*, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libRIO.so
#16 0x00001491ed538f03 in TObject::Write(char const*, int, int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/external/el8_amd64_gcc13/lib/libCore.so
#17 0x000014916dd1e564 in (anonymous namespace)::maybeConfigPileUp(edm::ParameterSet const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, int, int, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/libMixingBase.so
#18 0x000014916dd1eff3 in edm::MixingCache::Config::Config(edm::ParameterSet const&, unsigned int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/libMixingBase.so
#19 0x000014916dd1f3a0 in edm::BMixingModule::initializeGlobalCache(edm::ParameterSet const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/libMixingBase.so
#20 0x000014916ddd2c54 in edm::ModuleMaker<edm::MixingModule>::makeModule(edm::ParameterSet const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/pluginSimGeneralMixingModulePlugins.so
#21 0x00001491ed9d98ad in edm::ModuleMakerBase::makeModule(edm::MakeModuleParams const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/libFWCoreFramework.so
#22 0x00001491ed9d0f49 in edm::ModuleHolderFactory::makeModule(edm::MakeModuleParams const&, edm::ModuleTypeResolverMaker const*, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/libFWCoreFramework.so
#23 0x00001491ed9d9df6 in edm::ModuleRegistry::getModule(edm::MakeModuleParams const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&, edm::signalslot::Signal<void (edm::ModuleDescription const&)>&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/libFWCoreFramework.so
#24 0x00001491eda3230a in edm::(anonymous namespace)::getModule(edm::ParameterSet&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, edm::ModuleRegistry&, edm::SignallingProductRegistryFiller&, edm::ActivityRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/libFWCoreFramework.so
#25 0x00001491eda32585 in edm::(anonymous namespace)::fillModulesInPath(edm::ParameterSet&, edm::ModuleRegistry&, edm::SignallingProductRegistryFiller&, edm::ActivityRegistry&, edm::PreallocationConfiguration const*, std::shared_ptr<edm::ProcessConfiguration const>, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, bool, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/libFWCoreFramework.so
#26 0x00001491eda351e9 in edm::ScheduleBuilder::ScheduleBuilder(edm::ModuleRegistry&, edm::ParameterSet&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, edm::PreallocationConfiguration const&, edm::SignallingProductRegistryFiller&, edm::ActivityRegistry&, std::shared_ptr<edm::ProcessConfiguration const>) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/libFWCoreFramework.so
#27 0x00001491eda245ed in edm::Schedule::Schedule(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::SignallingProductRegistryFiller&, edm::ExceptionToActionTable const&, std::shared_ptr<edm::ActivityRegistry>, std::shared_ptr<edm::ProcessConfiguration const>, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/libFWCoreFramework.so
#28 0x00001491eda3b9b4 in edm::ScheduleItems::initModules(edm::ParameterSet&, edm::service::TriggerNamesService const&, edm::PreallocationConfiguration const&, edm::ProcessContext const*, edm::ModuleTypeResolverMaker const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/libFWCoreFramework.so
#29 0x00001491ed9a0b08 in tbb::detail::d2::function_task<edm::EventProcessor::init(std::shared_ptr<edm::ProcessDesc>&, edm::ServiceToken const&, edm::serviceregistry::ServiceLegacy)::{lambda()#2}>::execute(tbb::detail::d1::execution_data&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02914/el8_amd64_gcc13/cms/cmssw/CMSSW_16_0_X_2025-11-05-1100/lib/el8_amd64_gcc13/libFWCoreFramework.so
#30 0x00001491ed1d21fb in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=<optimized out>, waiter=..., this=0x1491eb123980) at src/tbb/task_dispatcher.h:344
#31 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x1491eb123980) at src/tbb/task_dispatcher.h:487
#32 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at src/tbb/arena.cpp:216
#33 tbb::detail::r1::thread_dispatcher_client::process (td=..., this=<optimized out>) at src/tbb/thread_dispatcher_client.h:41
#34 tbb::detail::r1::thread_dispatcher::process (this=<optimized out>, j=...) at src/tbb/thread_dispatcher.cpp:195
#35 0x00001491ed1c5d25 in tbb::detail::r1::rml::private_worker::run (this=0x1491e869b100) at src/tbb/private_server.cpp:271
#36 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x1491e869b100) at src/tbb/private_server.cpp:221
#37 0x00001491ec0531ca in start_thread () from /lib64/libpthread.so.0
#38 0x00001491ecba88d3 in clone () from /lib64/libc.so.6
@iarspider we believe this problem is related to a thread safety problem. Therefore when reporting a new failure, we really need to know what all the threads are doing, not just the one which exhibited the crash.