cmssw
cmssw copied to clipboard
cms-sw
Segfault in HGCalCLUEAlgoT<...>::makeClusters()
Worklofw 25234.911 step 2 failed in CMSSW_13_2_ROOT628_X_2023-06-19-2300 on el8_amd64_gcc11
Thread 5 (Thread 0x14b2e93ff700 (LWP 1285997) "cmsRun"):
#2 0x000014b343915660 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014b289b5cc1f in TICLLayerTileProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginRecoHGCalTICLPlugins.so
#5 0x000014b34a60f39d in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/libFWCoreFramework.so
#6 0x000014b34a5ec6c2 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/libFWCoreFramework.so
Thread 4 (Thread 0x14b2ea327700 (LWP 1285995) "cmsRun"):
#2 0x000014b343915660 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014b34782ca17 in write () from /lib64/libpthread.so.0
#5 0x000014b347ea7d7e in (anonymous namespace)::xwrite (__n=39, __s=0x14b29b0d3000 " 1 0 0 3.96875 -2.30174 0.250806 0.4 0\n0000000e+01\" rms=\"0.", '0' <repeats 16 times>, "e+00\" purity=\"5.1485347747802734e-01\" nType=\"1\"/>\n <Node pos=\"r\" depth=\"2\" NCoef=\"0\" IVar=\"1\" Cut=\"8.961122483015060"..., __fd=28) at basic_file.cc:127
#6 std::__basic_file<char>::xsputn (this=this@entry=0x14b29b106f80, __s=__s@entry=0x14b29b0d3000 " 1 0 0 3.96875 -2.30174 0.250806 0.4 0\n0000000e+01\" rms=\"0.", '0' <repeats 16 times>, "e+00\" purity=\"5.1485347747802734e-01\" nType=\"1\"/>\n <Node pos=\"r\" depth=\"2\" NCoef=\"0\" IVar=\"1\" Cut=\"8.961122483015060"..., __n=__n@entry=39) at basic_file.cc:343
#7 0x000014b347ee5ad5 in std::basic_filebuf<char, std::char_traits<char> >::_M_convert_to_external (this=this@entry=0x14b29b106f18, __ibuf=0x14b29b0d3000 " 1 0 0 3.96875 -2.30174 0.250806 0.4 0\n0000000e+01\" rms=\"0.", '0' <repeats 16 times>, "e+00\" purity=\"5.1485347747802734e-01\" nType=\"1\"/>\n <Node pos=\"r\" depth=\"2\" NCoef=\"0\" IVar=\"1\" Cut=\"8.961122483015060"..., __ilen=39) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_amd64_gcc11/external/gcc/11.2.1-f9b9dfdd886f71cd63f5538223d8f161/gcc-11.2.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/bits/fstream.tcc:642
#8 0x000014b347ee5e98 in std::basic_filebuf<char, std::char_traits<char> >::overflow (this=0x14b29b106f18, __c=-1) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_amd64_gcc11/external/gcc/11.2.1-f9b9dfdd886f71cd63f5538223d8f161/gcc-11.2.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/streambuf:534
#9 0x000014b347ee3cfd in std::basic_filebuf<char, std::char_traits<char> >::sync (this=<optimized out>) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_amd64_gcc11/external/gcc/11.2.1-f9b9dfdd886f71cd63f5538223d8f161/gcc-11.2.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/bits/fstream.tcc:1020
#10 std::basic_filebuf<char, std::char_traits<char> >::sync (this=<optimized out>) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_amd64_gcc11/external/gcc/11.2.1-f9b9dfdd886f71cd63f5538223d8f161/gcc-11.2.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/bits/fstream.tcc:1012
#11 0x000014b347f0a573 in std::basic_streambuf<char, std::char_traits<char> >::pubsync (this=<optimized out>) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_amd64_gcc11/external/gcc/11.2.1-f9b9dfdd886f71cd63f5538223d8f161/gcc-11.2.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/streambuf:276
#12 std::ostream::flush (this=0x14b29b106f10) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_amd64_gcc11/external/gcc/11.2.1-f9b9dfdd886f71cd63f5538223d8f161/gcc-11.2.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/bits/ostream.tcc:219
#13 0x000014b2825bcb4f in Phase2L1GMT::Isolation::isolation_allmu_alltrk(std::vector<l1t::TrackerMuon, std::allocator<l1t::TrackerMuon> >&, std::vector<Phase2L1GMT::ConvertedTTTrack, std::allocator<Phase2L1GMT::ConvertedTTTrack> >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginPhase2L1GMTPlugins.so
#14 0x000014b2825be8c3 in Phase2L1GMT::Node::processEvent(std::vector<edm::Ptr<TTTrack<edm::Ref<edm::DetSetVector<Phase2TrackerDigi>, Phase2TrackerDigi, edm::refhelper::FindForDetSetVector<Phase2TrackerDigi> > > >, std::allocator<edm::Ptr<TTTrack<edm::Ref<edm::DetSetVector<Phase2TrackerDigi>, Phase2TrackerDigi, edm::refhelper::FindForDetSetVector<Phase2TrackerDigi> > > > > > const&, BXVector<edm::Ref<BXVector<l1t::RegionalMuonCand>, l1t::RegionalMuonCand, edm::refhelper::FindUsingAdvance<BXVector<l1t::RegionalMuonCand>, l1t::RegionalMuonCand> > > const&, std::vector<edm::Ref<std::vector<l1t::MuonStub, std::allocator<l1t::MuonStub> >, l1t::MuonStub, edm::refhelper::FindUsingAdvance<std::vector<l1t::MuonStub, std::allocator<l1t::MuonStub> >, l1t::MuonStub> >, std::allocator<edm::Ref<std::vector<l1t::MuonStub, std::allocator<l1t::MuonStub> >, l1t::MuonStub, edm::refhelper::FindUsingAdvance<std::vector<l1t::MuonStub, std::allocator<l1t::MuonStub> >, l1t::MuonStub> > > > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginPhase2L1GMTPlugins.so
#15 0x000014b2825c0a54 in Phase2L1TGMTProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginPhase2L1GMTPlugins.so
#16 0x000014b34a60f39d in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/libFWCoreFramework.so
#17 0x000014b34a5ec6c2 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/libFWCoreFramework.so
Thread 3 (Thread 0x14b2ead28700 (LWP 1285994) "cmsRun"):
#2 0x000014b343915660 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014b34782ca17 in write () from /lib64/libpthread.so.0
#5 0x000014b347ea7d7e in (anonymous namespace)::xwrite (__n=37, __s=0x14b29af3e000 " 0 0 0 4.96875 1.57847 1.61298 4.7 0\n-01\" nType=\"1\"/>\n", ' ' <repeats 12 times>, "<Node pos=\"r\" depth=\"3\" NCoef=\"0\" IVar=\"-1\" Cut=\"0.", '0' <repeats 16 times>, "e+00\" cType=\"1\" res=\"-9.9", '0' <repeats 15 times>, "e+01\" rms=\"0.", '0' <repeats 14 times>..., __fd=24) at basic_file.cc:127
#6 std::__basic_file<char>::xsputn (this=this@entry=0x14b29b106580, __s=__s@entry=0x14b29af3e000 " 0 0 0 4.96875 1.57847 1.61298 4.7 0\n-01\" nType=\"1\"/>\n", ' ' <repeats 12 times>, "<Node pos=\"r\" depth=\"3\" NCoef=\"0\" IVar=\"-1\" Cut=\"0.", '0' <repeats 16 times>, "e+00\" cType=\"1\" res=\"-9.9", '0' <repeats 15 times>, "e+01\" rms=\"0.", '0' <repeats 14 times>..., __n=__n@entry=37) at basic_file.cc:343
#7 0x000014b347ee5ad5 in std::basic_filebuf<char, std::char_traits<char> >::_M_convert_to_external (this=this@entry=0x14b29b106518, __ibuf=0x14b29af3e000 " 0 0 0 4.96875 1.57847 1.61298 4.7 0\n-01\" nType=\"1\"/>\n", ' ' <repeats 12 times>, "<Node pos=\"r\" depth=\"3\" NCoef=\"0\" IVar=\"-1\" Cut=\"0.", '0' <repeats 16 times>, "e+00\" cType=\"1\" res=\"-9.9", '0' <repeats 15 times>, "e+01\" rms=\"0.", '0' <repeats 14 times>..., __ilen=37) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_amd64_gcc11/external/gcc/11.2.1-f9b9dfdd886f71cd63f5538223d8f161/gcc-11.2.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/bits/fstream.tcc:642
#8 0x000014b347ee5e98 in std::basic_filebuf<char, std::char_traits<char> >::overflow (this=0x14b29b106518, __c=-1) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_amd64_gcc11/external/gcc/11.2.1-f9b9dfdd886f71cd63f5538223d8f161/gcc-11.2.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/streambuf:534
#9 0x000014b347ee3cfd in std::basic_filebuf<char, std::char_traits<char> >::sync (this=<optimized out>) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_amd64_gcc11/external/gcc/11.2.1-f9b9dfdd886f71cd63f5538223d8f161/gcc-11.2.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/bits/fstream.tcc:1020
#10 std::basic_filebuf<char, std::char_traits<char> >::sync (this=<optimized out>) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_amd64_gcc11/external/gcc/11.2.1-f9b9dfdd886f71cd63f5538223d8f161/gcc-11.2.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/bits/fstream.tcc:1012
#11 0x000014b347f0a573 in std::basic_streambuf<char, std::char_traits<char> >::pubsync (this=<optimized out>) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_amd64_gcc11/external/gcc/11.2.1-f9b9dfdd886f71cd63f5538223d8f161/gcc-11.2.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/streambuf:276
#12 std::ostream::flush (this=0x14b29b106510) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el8_amd64_gcc11/external/gcc/11.2.1-f9b9dfdd886f71cd63f5538223d8f161/gcc-11.2.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/bits/ostream.tcc:219
#13 0x000014b2825bcb4f in Phase2L1GMT::Isolation::isolation_allmu_alltrk(std::vector<l1t::TrackerMuon, std::allocator<l1t::TrackerMuon> >&, std::vector<Phase2L1GMT::ConvertedTTTrack, std::allocator<Phase2L1GMT::ConvertedTTTrack> >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginPhase2L1GMTPlugins.so
#14 0x000014b2825be8c3 in Phase2L1GMT::Node::processEvent(std::vector<edm::Ptr<TTTrack<edm::Ref<edm::DetSetVector<Phase2TrackerDigi>, Phase2TrackerDigi, edm::refhelper::FindForDetSetVector<Phase2TrackerDigi> > > >, std::allocator<edm::Ptr<TTTrack<edm::Ref<edm::DetSetVector<Phase2TrackerDigi>, Phase2TrackerDigi, edm::refhelper::FindForDetSetVector<Phase2TrackerDigi> > > > > > const&, BXVector<edm::Ref<BXVector<l1t::RegionalMuonCand>, l1t::RegionalMuonCand, edm::refhelper::FindUsingAdvance<BXVector<l1t::RegionalMuonCand>, l1t::RegionalMuonCand> > > const&, std::vector<edm::Ref<std::vector<l1t::MuonStub, std::allocator<l1t::MuonStub> >, l1t::MuonStub, edm::refhelper::FindUsingAdvance<std::vector<l1t::MuonStub, std::allocator<l1t::MuonStub> >, l1t::MuonStub> >, std::allocator<edm::Ref<std::vector<l1t::MuonStub, std::allocator<l1t::MuonStub> >, l1t::MuonStub, edm::refhelper::FindUsingAdvance<std::vector<l1t::MuonStub, std::allocator<l1t::MuonStub> >, l1t::MuonStub> > > > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginPhase2L1GMTPlugins.so
#15 0x000014b2825c0a54 in Phase2L1TGMTProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginPhase2L1GMTPlugins.so
#16 0x000014b34a60f39d in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/libFWCoreFramework.so
#17 0x000014b34a5ec6c2 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/libFWCoreFramework.so
Thread 1 (Thread 0x14b346974680 (LWP 1269271) "cmsRun"):
#3 0x000014b343918acb in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x000014b2cec28bf8 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#6 0x000014b2cec2f27b in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#7 0x000014b3486d99cd in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x14b283c19f00, this=0x14b345dc9380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_1_0_pre3-el8_amd64_gcc11/build/CMSSW_13_1_0_pre3-build/BUILD/el8_amd64_gcc11/external/tbb/v2021.8.0-4e779f195a25a0aba119b27519937ba0/tbb-v2021.8.0/src/tbb/task_dispatcher.h:322
#8 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x14b345dc9380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_1_0_pre3-el8_amd64_gcc11/build/CMSSW_13_1_0_pre3-build/BUILD/el8_amd64_gcc11/external/tbb/v2021.8.0-4e779f195a25a0aba119b27519937ba0/tbb-v2021.8.0/src/tbb/task_dispatcher.h:458
#9 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_1_0_pre3-el8_amd64_gcc11/build/CMSSW_13_1_0_pre3-build/BUILD/el8_amd64_gcc11/external/tbb/v2021.8.0-4e779f195a25a0aba119b27519937ba0/tbb-v2021.8.0/src/tbb/task_dispatcher.cpp:168
#10 0x000014b2cec2e7e0 in tbb::detail::d1::task_arena_function<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#11 0x000014b3486c80ee in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_1_0_pre3-el8_amd64_gcc11/build/CMSSW_13_1_0_pre3-build/BUILD/el8_amd64_gcc11/external/tbb/v2021.8.0-4e779f195a25a0aba119b27519937ba0/tbb-v2021.8.0/src/tbb/arena.cpp:767
#12 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_1_0_pre3-el8_amd64_gcc11/build/CMSSW_13_1_0_pre3-build/BUILD/el8_amd64_gcc11/external/tbb/v2021.8.0-4e779f195a25a0aba119b27519937ba0/tbb-v2021.8.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#13 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_1_0_pre3-el8_amd64_gcc11/build/CMSSW_13_1_0_pre3-build/BUILD/el8_amd64_gcc11/external/tbb/v2021.8.0-4e779f195a25a0aba119b27519937ba0/tbb-v2021.8.0/src/tbb/arena.cpp:768
#14 0x000014b2cec1e0c4 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#15 0x000014b2cec3f357 in HGCalLayerClusterProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#16 0x000014b34a60f39d in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/libFWCoreFramework.so
#17 0x000014b34a5ec6c2 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02790/el8_amd64_gcc11/cms/cmssw/CMSSW_13_2_ROOT628_X_2023-06-19-2300/lib/el8_amd64_gcc11/libFWCoreFramework.so
Current Modules:
Module: HGCalLayerClusterProducer:hgcalLayerClustersHSci (crashed)
Module: Phase2L1TGMTProducer:l1tTkMuonsGmt
Module: TICLLayerTileProducer:ticlLayerTileProducer
Module: Phase2L1TGMTProducer:l1tTkMuonsGmt
https://cmssdt.cern.ch/SDT/cgi-bin/logreader/el8_amd64_gcc11/CMSSW_13_2_ROOT628_X_2023-06-19-2300/pyRelValMatrixLogs/run/25234.911_TTbar_14TeV+2026D99_DD4hep/step2_TTbar_14TeV+2026D99_DD4hep.log#/
A new Issue was created by @makortel Matti Kortelainen.
@Dr15Jones, @perrotta, @dpiparo, @rappoccio, @makortel, @smuzaffar can you please review it and eventually sign/assign? Thanks.
cms-bot commands are listed here
New categories assigned: upgrade,reconstruction
@AdrianoDee,@mandrenguyen,@clacaputo,@srimanob you have been requested to review this Pull request/Issue and eventually sign? Thanks
@makortel thanks for reporting on this. We will follow up on this ASAP. @OlivieFranklova @felicepantaleo
Seen again in WF25234.911 step3 on slc7_amd64_gcc11 CMSSW_13_3_X_2023-09-13-2300. Apparently very rare, but this at least lets us eliminate some possible thread interactions.
Thread 5 (Thread 0x1478cb3ff700 (LWP 3500077) "cmsRun"):
#2 0x000014792b07cb10 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x00001479310c8ddd in poll () from /lib64/libc.so.6
#5 0x00001478ce104c18 in frontier_read () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libfrontier_client.so.2
#6 0x00001478ce101d97 in http_read () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libfrontier_client.so.2
#7 0x00001478ce102e7b in frontierHttpClnt_read () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libfrontier_client.so.2
#8 0x00001478ce10f51f in get_data () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libfrontier_client.so.2
#9 0x00001478ce10fd47 in frontier_postRawData () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libfrontier_client.so.2
#10 0x00001478ce10f987 in frontier_getRawData () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libfrontier_client.so.2
#11 0x00001478ce1071d5 in frontier::Session::getData(std::vector<frontier::Request const*, std::allocator<frontier::Request const*> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libfrontier_client.so.2
#12 0x00001478ce17354b in coral::FrontierAccess::Statement::execute(coral::AttributeList const&, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/liblcg_FrontierAccess.so
#13 0x00001478ce162be2 in coral::FrontierAccess::Query::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/liblcg_FrontierAccess.so
#14 0x000014790532f2f0 in cond::persistency::PAYLOAD::Table::select(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >&, cond::Binary&, cond::Binary&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/libCondCoreCondDB.so
#15 0x00001478cdf170da in std::unique_ptr<CSCDBNoiseMatrix, std::default_delete<CSCDBNoiseMatrix> > cond::persistency::Session::fetchPayload<CSCDBNoiseMatrix>(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginCondCoreCSCPlugins.so
#16 0x00001478cdf0e56d in cond::persistency::PayloadProxy<CSCDBNoiseMatrix>::loadPayload() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginCondCoreCSCPlugins.so
#17 0x00001478cdf08a07 in cond::ProductResolver<CSCDBNoiseMatrixRcd, CSCDBNoiseMatrix, cond::DefaultInitializer<CSCDBNoiseMatrix> >::prefetch(edm::eventsetup::DataKey const&, edm::EventSetupRecordDetails) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginCondCoreCSCPlugins.so
#18 0x000014793396c13e in edm::eventsetup::ESSourceProductResolverBase::doPrefetchAndSignals(edm::eventsetup::EventSetupRecordImpl const&, edm::eventsetup::DataKey const&, edm::ESParentContext const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/libFWCoreFramework.so
Thread 4 (Thread 0x1478cc38a700 (LWP 3500076) "cmsRun"):
#3 0x000014792b08034b in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x000014788f052b78 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#6 0x000014788f0591cb in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#7 0x00001479321e6a7d in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x14786132ad00, this=0x14792e5c9480) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/slc7_amd64_gcc11/external/tbb/v2021.9.0-5caec19d753f5e33a53af823382dc5ea/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#8 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x14792e5c9480) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/slc7_amd64_gcc11/external/tbb/v2021.9.0-5caec19d753f5e33a53af823382dc5ea/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#9 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/slc7_amd64_gcc11/external/tbb/v2021.9.0-5caec19d753f5e33a53af823382dc5ea/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#10 0x000014788f058738 in tbb::detail::d1::task_arena_function<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#11 0x00001479321d416e in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/slc7_amd64_gcc11/external/tbb/v2021.9.0-5caec19d753f5e33a53af823382dc5ea/tbb-v2021.9.0/src/tbb/arena.cpp:757
#12 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/slc7_amd64_gcc11/external/tbb/v2021.9.0-5caec19d753f5e33a53af823382dc5ea/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#13 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/slc7_amd64_gcc11/external/tbb/v2021.9.0-5caec19d753f5e33a53af823382dc5ea/tbb-v2021.9.0/src/tbb/arena.cpp:758
#14 0x000014788f047fd4 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#15 0x000014788f0692d7 in HGCalLayerClusterProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginRecoLocalCaloHGCalRecProducersPlugins.so
Thread 3 (Thread 0x1478ccd8b700 (LWP 3500074) "cmsRun"):
#2 0x000014792b07cb10 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014792b9b6986 in dqm::impl::MonitorElement::accessMut() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/libDQMServicesCore.so
#5 0x000014792b9c061c in dqm::impl::MonitorElement::Fill(double, double, double) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/libDQMServicesCore.so
#6 0x000014789def4a7e in EcalSelectiveReadoutValidation::analyzeEE(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginValidationEcalDigisAuto.so
#7 0x000014789def5177 in EcalSelectiveReadoutValidation::analyze(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginValidationEcalDigisAuto.so
Thread 1 (Thread 0x14792f214780 (LWP 3496275) "cmsRun"):
#2 0x000014792b07cb10 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 imalloc_fastpath (fallback_alloc=0x14793265ed10 <fallback_impl<false>(std::size_t)>, size=40) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:309
#5 newImpl<false> (size=40) at src/jemalloc_cpp.cpp:109
#6 operator new (size=40) at src/jemalloc_cpp.cpp:114
#7 0x0000147902277522 in EcalEBTriggerPrimitiveDigi::EcalEBTriggerPrimitiveDigi() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/libDataFormatsEcalDigi.so
#8 0x000014790227757d in EcalEBTriggerPrimitiveDigi* std::__uninitialized_default_n_1<false>::__uninit_default_n<EcalEBTriggerPrimitiveDigi*, unsigned long>(EcalEBTriggerPrimitiveDigi*, unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/libDataFormatsEcalDigi.so
#9 0x000014790226ee58 in ROOT::Detail::TCollectionProxyInfo::Pushback<std::vector<EcalEBTriggerPrimitiveDigi, std::allocator<EcalEBTriggerPrimitiveDigi> > >::resize(void*, unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/libDataFormatsEcalDigi.so
#10 0x0000147932f7fc15 in TGenCollectionProxy::Allocate(unsigned int, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libRIO.so
#11 0x0000147933033ba2 in int TStreamerInfoActions::ReadSTL<&TStreamerInfoActions::ReadSTLMemberWiseSameClass, &TStreamerInfoActions::ReadSTLObjectWiseFastArray>(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libRIO.so
#12 0x0000147932ee6365 in TBufferFile::ReadClassBuffer(TClass const*, void*, TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libRIO.so
#13 0x00001479331388c4 in int TStreamerInfo::ReadBuffer<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libRIO.so
#14 0x0000147932fb112d in TStreamerInfoActions::GenericReadAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libRIO.so
#15 0x0000147932ede0e5 in TBufferFile::ApplySequence(TStreamerInfoActions::TActionSequence const&, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libRIO.so
#16 0x00001479336289d7 in TBranchElement::ReadLeavesMember(TBuffer&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libTree.so
#17 0x0000147933621049 in TBranch::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libTree.so
#18 0x000014793363b5f5 in TBranchElement::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libTree.so
#19 0x000014793363b5a8 in TBranchElement::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/external/slc7_amd64_gcc11/lib/libTree.so
#20 0x00001478cb95b9c4 in edm::RootTree::getEntry(TBranch*, long long) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginIOPoolInput.so
#21 0x00001478cb93e08c in edm::RootDelayedReader::getProduct_(edm::BranchID const&, edm::EDProductGetter const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/pluginIOPoolInput.so
#22 0x000014793395e455 in edm::DelayedReader::getProduct(edm::BranchID const&, edm::EDProductGetter const*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/libFWCoreFramework.so
#23 0x0000147933a0608b in edm::DelayedReaderInputProductResolver::prefetchAsync_(edm::WaitingTaskHolder, edm::Principal const&, bool, edm::ServiceToken const&, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const::{lambda()#1}::operator()() const::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02802/slc7_amd64_gcc11/cms/cmssw/CMSSW_13_3_X_2023-09-13-2300/lib/slc7_amd64_gcc11/libFWCoreFramework.so
Current Modules:
Module: HGCalLayerClusterProducer:hgcalLayerClustersHSci (crashed)
Module: EcalSelectiveReadoutValidation:ecalSelectiveReadoutValidation
Module: none
Module: none
Happened again: link
Thu Oct 19 05:50:08 CEST 2023
Thread 8 (Thread 0x1486ea5ff700 (LWP 3425861) "cmsRun"):
#0 0x000014877853c45c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000014873f03a84e in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#2 0x000014873f03ae03 in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#3 0x000014873f038578 in std::_Function_handler<void (), tsl::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#4 0x0000148731daaa02 in tsl::(anonymous namespace)::PThread::ThreadFn(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#5 0x00001487785361ca in start_thread () from /lib64/libpthread.so.0
#6 0x00001487781a2e73 in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x1486eb0f8700 (LWP 3425858) "cmsRun"):
#0 0x000014877853c45c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000014873f03a84e in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#2 0x000014873f03ae03 in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#3 0x000014873f038578 in std::_Function_handler<void (), tsl::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#4 0x0000148731daaa02 in tsl::(anonymous namespace)::PThread::ThreadFn(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#5 0x00001487785361ca in start_thread () from /lib64/libpthread.so.0
#6 0x00001487781a2e73 in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x1486eb2f9700 (LWP 3425857) "cmsRun"):
#0 0x000014877853c45c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000014873f03a84e in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#2 0x000014873f03ae03 in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#3 0x000014873f038578 in std::_Function_handler<void (), tsl::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#4 0x0000148731daaa02 in tsl::(anonymous namespace)::PThread::ThreadFn(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#5 0x00001487785361ca in start_thread () from /lib64/libpthread.so.0
#6 0x00001487781a2e73 in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x14872a570700 (LWP 3421483) "cmsRun"):
#0 0x0000148778262868 in nanosleep () from /lib64/libc.so.6
#1 0x000014877826276e in sleep () from /lib64/libc.so.6
#2 0x0000148773e22d20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x0000148746e47fba in AdaptiveVertexFitter::reWeightTracks(std::vector<ReferenceCountingPointer<LinearizedTrackState<5u> >, std::allocator<ReferenceCountingPointer<LinearizedTrackState<5u> > > > const&, CachingVertex<5u> const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libRecoVertexAdaptiveVertexFit.so
#5 0x0000148746e48e65 in AdaptiveVertexFitter::reWeightTracks(std::vector<ReferenceCountingPointer<VertexTrack<5u> >, std::allocator<ReferenceCountingPointer<VertexTrack<5u> > > > const&, CachingVertex<5u> const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libRecoVertexAdaptiveVertexFit.so
#6 0x0000148746e4a2ed in AdaptiveVertexFitter::fit(std::vector<ReferenceCountingPointer<VertexTrack<5u> >, std::allocator<ReferenceCountingPointer<VertexTrack<5u> > > > const&, VertexState const&, bool) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libRecoVertexAdaptiveVertexFit.so
#7 0x0000148746e4acff in AdaptiveVertexFitter::vertex(std::vector<reco::TransientTrack, std::allocator<reco::TransientTrack> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libRecoVertexAdaptiveVertexFit.so
#8 0x00001486ec7ebbd4 in AdaptiveVertexReconstructor::vertices(std::vector<reco::TransientTrack, std::allocator<reco::TransientTrack> > const&, std::vector<reco::TransientTrack, std::allocator<reco::TransientTrack> > const&, reco::BeamSpot const&, bool, bool) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libRecoVertexAdaptiveVertexFinder.so
#9 0x00001486ec7ec80b in AdaptiveVertexReconstructor::vertices(std::vector<reco::TransientTrack, std::allocator<reco::TransientTrack> > const&, reco::BeamSpot const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libRecoVertexAdaptiveVertexFinder.so
#10 0x00001486ec80316e in ConfigurableAdaptiveReconstructor::vertices(std::vector<reco::TransientTrack, std::allocator<reco::TransientTrack> > const&, reco::BeamSpot const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libRecoVertexConfigurableVertexReco.so
#11 0x00001486ec803cce in ConfigurableVertexReconstructor::vertices(std::vector<reco::TransientTrack, std::allocator<reco::TransientTrack> > const&, reco::BeamSpot const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libRecoVertexConfigurableVertexReco.so
#12 0x00001486ec8381d9 in TemplatedInclusiveVertexFinder<std::vector<reco::Track, std::allocator<reco::Track> >, reco::Vertex>::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginRecoVertexAdaptiveVertexFinderPlugins.so
#13 0x000014877aca60d1 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x000014877ac8a84e in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#15 0x000014877ac168e9 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x000014877ac16e84 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#17 0x000014877add2f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#18 0x00001487793f795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x1486c0633e00, waiter=..., this=0x148776bc9500) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#19 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x148776bc9500) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#20 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/arena.cpp:137
#21 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/market.cpp:599
#22 0x00001487793f9b0e in tbb::detail::r1::rml::private_worker::run (this=0x14877442e000) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#23 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x14877442e000) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#24 0x00001487785361ca in start_thread () from /lib64/libpthread.so.0
#25 0x00001487781a2e73 in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x14872b487700 (LWP 3421481) "cmsRun"):
#0 0x000014877828d251 in poll () from /lib64/libc.so.6
#1 0x0000148773e5addf in full_read.constprop () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x0000148773e2639c in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 0x0000148773e26520 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x00001486ed9a28d8 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#6 0x00001486ed9a2dab in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#7 0x0000148779400281 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x148776bc9400) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#8 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x148776bc9400) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#9 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#10 0x00001486ed99fd17 in tbb::detail::d1::task_arena_function<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#11 0x00001487793ed28e in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/arena.cpp:757
#12 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#13 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/arena.cpp:758
#14 0x00001486ed991854 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#15 0x00001486ed9ae8cf in HGCalLayerClusterProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#16 0x000014877aca60d1 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#17 0x000014877ac8a84e in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#18 0x000014877ac168e9 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#19 0x000014877ac16e84 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#20 0x000014877add2f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#21 0x00001487793f795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x1486c0696300, waiter=..., this=0x148776bc9400) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#22 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x148776bc9400) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#23 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/arena.cpp:137
#24 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/market.cpp:599
#25 0x00001487793f9b0e in tbb::detail::r1::rml::private_worker::run (this=0x14877442e100) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#26 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x14877442e100) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#27 0x00001487785361ca in start_thread () from /lib64/libpthread.so.0
#28 0x00001487781a2e73 in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x14872be88700 (LWP 3421480) "cmsRun"):
#0 0x0000148778262868 in nanosleep () from /lib64/libc.so.6
#1 0x000014877826276e in sleep () from /lib64/libc.so.6
#2 0x0000148773e22d20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x00001487014b5a8f in std::_Rb_tree<edm::Ptr<reco::Candidate>, edm::Ptr<reco::Candidate>, std::_Identity<edm::Ptr<reco::Candidate> >, std::less<edm::Ptr<reco::Candidate> >, std::allocator<edm::Ptr<reco::Candidate> > >::find(edm::Ptr<reco::Candidate> const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginCommonToolsCandAlgos_plugins.so
#5 0x00001487014b8061 in CandPtrProjector::produce(edm::StreamID, edm::Event&, edm::EventSetup const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginCommonToolsCandAlgos_plugins.so
#6 0x000014877ac918a6 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#7 0x000014877ac847fe in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#8 0x000014877ac168e9 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#9 0x000014877ac16e84 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#10 0x000014877add2f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#11 0x00001487793f795b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x1486c05d2000, waiter=..., this=0x148776bc9480) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#12 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x148776bc9480) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#13 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/arena.cpp:137
#14 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/market.cpp:599
#15 0x00001487793f9b0e in tbb::detail::r1::rml::private_worker::run (this=0x14877442e080) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#16 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x14877442e080) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#17 0x00001487785361ca in start_thread () from /lib64/libpthread.so.0
#18 0x00001487781a2e73 in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x14874e3d6700 (LWP 3420652) "cmsRun"):
#0 0x0000148778540672 in waitpid () from /lib64/libpthread.so.0
#1 0x0000148773e22e77 in edm::service::cmssw_stacktrace_fork() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x0000148773e262ca in edm::service::InitRootHandlers::stacktraceHelperThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 0x0000148778bcaa73 in std::execute_native_thread_routine (__p=0x14876b764650) at ../../../../../libstdc++-v3/src/c++11/thread.cc:82
#4 0x00001487785361ca in start_thread () from /lib64/libpthread.so.0
#5 0x00001487781a2e73 in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x1487776c1640 (LWP 3413660) "cmsRun"):
#0 0x0000148778262868 in nanosleep () from /lib64/libc.so.6
#1 0x000014877826276e in sleep () from /lib64/libc.so.6
#2 0x0000148773e22d20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 ph_merge_children (cmp=<optimized out>, offset=40, phn=0x14864dd52880) at include/jemalloc/internal/ph.h:241
#5 ph_remove (cmp=<optimized out>, offset=40, phn=0x14864dd52880, ph=ph@entry=0x148777414890) at include/jemalloc/internal/ph.h:401
#6 je_edata_heap_remove (ph=ph@entry=0x148777414890, phn=0x14864dd52880, phn@entry=0x0) at src/edata.c:6
#7 0x0000148779809250 in arena_bin_slabs_nonfull_remove (slab=0x0, bin=0x1487774147c8) at src/arena.c:587
#8 arena_dissociate_bin_slab (bin=bin@entry=0x1487774147c8, slab=slab@entry=0x0, arena=arena@entry=0x148779861fb8 <pac_dalloc_impl+72>) at src/arena.c:1311
#9 je_arena_dalloc_bin_locked_handle_newly_empty (tsdn=tsdn@entry=0x1487776bb750, arena=arena@entry=0x1487774010c0, slab=slab@entry=0x14864dd52880, bin=bin@entry=0x1487774147c8) at src/arena.c:1356
#10 0x000014877986dfe4 in arena_dalloc_bin_locked_step (ptr=<optimized out>, slab=<optimized out>, binind=<optimized out>, info=<synthetic pointer>, bin=0x1487774147c8, arena=<optimized out>, tsdn=<optimized out>) at include/jemalloc/internal/arena_inlines_b.h:524
#11 tcache_bin_flush_impl (small=true, nflush=<optimized out>, ptrs=0x7ffdb2a293a0, binind=<optimized out>, cache_bin=<optimized out>, tcache=<optimized out>, tsd=<optimized out>) at src/tcache.c:448
#12 tcache_bin_flush_bottom (small=true, rem=<optimized out>, binind=0, cache_bin=<optimized out>, tcache=<optimized out>, tsd=<optimized out>) at src/tcache.c:519
#13 je_tcache_bin_flush_small (tsd=<optimized out>, tcache=<optimized out>, cache_bin=<optimized out>, binind=binind@entry=3, rem=<optimized out>) at src/tcache.c:529
#14 0x000014877986e1c0 in tcache_gc_small (tsd=<optimized out>, tcache_slow=0x1487776bb850, tcache=<optimized out>, szind=<optimized out>) at src/tcache.c:148
#15 0x000014877986ec72 in je_tcache_gc_dalloc_event_handler () at src/tcache.c:223
#16 0x00001487798714a3 in je_te_event_trigger (tsd=0x1487776bb750, ctx=<optimized out>) at src/thread_event.c:299
#17 0x0000148779803f05 in te_event_advance (is_alloc=false, usize=48, tsd=0x1487776bb750) at include/jemalloc/internal/thread_event.h:287
#18 thread_dalloc_event (usize=48, tsd=0x1487776bb750) at include/jemalloc/internal/thread_event.h:293
#19 isfree (slow_path=<optimized out>, tcache=<optimized out>, usize=<optimized out>, ptr=<optimized out>, tsd=<optimized out>) at src/jemalloc.c:2991
#20 je_sdallocx_default (ptr=<optimized out>, size=<optimized out>, flags=<optimized out>) at src/jemalloc.c:3928
#21 0x00001487798727fe in sizedDeleteImpl (size=<optimized out>, ptr=<optimized out>) at src/jemalloc_cpp.cpp:195
#22 operator delete (ptr=<optimized out>, size=<optimized out>) at src/jemalloc_cpp.cpp:200
#23 0x000014874a0f360d in HGCalCell::cellUV2XY2(int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libGeometryHGCalCommonData.so
#24 0x000014874a0fb581 in HGCalDDDConstants::locateCell(int, int, int, int, int, int, bool, bool, bool, bool) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libGeometryHGCalCommonData.so
#25 0x000014874a334de6 in HGCalGeometry::getPosition(DetId const&, bool) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libGeometryHGCalGeometry.so
#26 0x00001486fcc1536a in hgcal::RecHitTools::getPosition(DetId const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libRecoLocalCaloHGCalRecAlgos.so
#27 0x00001486fd2f5995 in HGVHistoProducerAlgo::fill_generic_cluster_histos(HGVHistoProducerAlgoHistograms const&, int, edm::Handle<std::vector<reco::CaloCluster, std::allocator<reco::CaloCluster> > >, std::vector<reco::CaloCluster, std::allocator<reco::CaloCluster> > const&, edm::Handle<std::vector<CaloParticle, std::allocator<CaloParticle> > >, std::vector<CaloParticle, std::allocator<CaloParticle> > const&, std::vector<unsigned long, std::allocator<unsigned long> > const&, std::vector<unsigned long, std::allocator<unsigned long> > const&, std::unordered_map<DetId, HGCRecHit const*, std::hash<DetId>, std::equal_to<DetId>, std::allocator<std::pair<DetId const, HGCRecHit const*> > > const&, std::map<double, double, std::less<double>, std::allocator<std::pair<double const, double> > >, unsigned int, std::vector<int, std::allocator<int> >, edm::AssociationMap<edm::OneToManyWithQualityGeneric<std::vector<reco::CaloCluster, std::allocator<reco::CaloCluster> >, std::vector<CaloParticle, std::allocator<CaloParticle> >, float, unsigned int, edm::RefProd<std::vector<reco::CaloCluster, std::allocator<reco::CaloCluster> > >, edm::RefProd<std::vector<CaloParticle, std::allocator<CaloParticle> > >, edm::Ref<std::vector<reco::CaloCluster, std::allocator<reco::CaloCluster> >, reco::CaloCluster, edm::refhelper::FindUsingAdvance<std::vector<reco::CaloCluster, std::allocator<reco::CaloCluster> >, reco::CaloCluster> >, edm::Ref<std::vector<CaloParticle, std::allocator<CaloParticle> >, CaloParticle, edm::refhelper::FindUsingAdvance<std::vector<CaloParticle, std::allocator<CaloParticle> >, CaloParticle> > > > const&, edm::AssociationMap<edm::OneToManyWithQualityGeneric<std::vector<CaloParticle, std::allocator<CaloParticle> >, std::vector<reco::CaloCluster, std::allocator<reco::CaloCluster> >, std::pair<float, float>, unsigned int, edm::RefProd<std::vector<CaloParticle, std::allocator<CaloParticle> > >, edm::RefProd<std::vector<reco::CaloCluster, std::allocator<reco::CaloCluster> > >, edm::Ref<std::vector<CaloParticle, std::allocator<CaloParticle> >, CaloParticle, edm::refhelper::FindUsingAdvance<std::vector<CaloParticle, std::allocator<CaloParticle> >, CaloParticle> >, edm::Ref<std::vector<reco::CaloCluster, std::allocator<reco::CaloCluster> >, reco::CaloCluster, edm::refhelper::FindUsingAdvance<std::vector<reco::CaloCluster, std::allocator<reco::CaloCluster> >, reco::CaloCluster> > > > const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/biglib/el8_amd64_gcc12/pluginSimulation.so
#28 0x00001486fd351d76 in HGCalValidator::dqmAnalyze(edm::Event const&, edm::EventSetup const&, HGCalValidatorHistograms const&) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/biglib/el8_amd64_gcc12/pluginSimulation.so
#29 0x000014877ac918a6 in edm::global::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#30 0x000014877ac847fe in edm::WorkerT<edm::global::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#31 0x000014877ac168e9 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#32 0x000014877ac16e84 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#33 0x000014877add2f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#34 0x0000148779400281 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x148776bc9380) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#35 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x148776bc9380) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#36 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#37 0x000014877ab9b1bb in edm::FinalWaitingTask::wait() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#38 0x000014877aba4bc5 in edm::EventProcessor::processRuns() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#39 0x000014877aba518f in edm::EventProcessor::runToCompletion() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_CUDART_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#40 0x00000000004074ef in tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const ()
#41 0x00001487793ec9ad in tbb::detail::r1::task_arena_impl::execute (ta=..., d=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-d7f1d991adf5a79d0785f12eb7ec307e/tbb-v2021.9.0/src/tbb/arena.cpp:688
#42 0x0000000000408ed2 in main::{lambda()#1}::operator()() const ()
#43 0x000000000040517c in main ()
Current Modules:
Module: HGCalLayerClusterProducer:hgcalLayerClustersHSci (crashed)
Module: HGCalValidator:hgcalValidator
Module: InclusiveVertexFinder:inclusiveVertexFinder
Module: CandPtrProjector:pfCandsNoJets
Happened again: https://cmssdt.cern.ch/SDT/cgi-bin/logreader/el8_amd64_gcc12/CMSSW_13_3_ROOT630_X_2023-10-19-2300/pyRelValMatrixLogs/run/25234.911_TTbar_14TeV+2026D99_DD4hep/step3_TTbar_14TeV+2026D99_DD4hep.log#/126-126
A fatal system signal has occurred: segmentation violation
The following is the call stack containing the origin of the signal.
Fri Oct 20 04:37:23 CEST 2023
Thread 8 (Thread 0x14adcdfff700 (LWP 730842) "cmsRun"):
#0 0x000014ae6482a45c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000014ae2282684e in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#2 0x000014ae22826e03 in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#3 0x000014ae22824578 in std::_Function_handler<void (), tsl::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#4 0x000014ae15596a02 in tsl::(anonymous namespace)::PThread::ThreadFn(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#5 0x000014ae648241ca in start_thread () from /lib64/libpthread.so.0
#6 0x000014ae64490e73 in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x14adce9ff700 (LWP 730840) "cmsRun"):
#0 0x000014ae6482a45c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000014ae2282684e in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#2 0x000014ae22826e03 in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#3 0x000014ae22824578 in std::_Function_handler<void (), tsl::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#4 0x000014ae15596a02 in tsl::(anonymous namespace)::PThread::ThreadFn(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#5 0x000014ae648241ca in start_thread () from /lib64/libpthread.so.0
#6 0x000014ae64490e73 in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x14adcee1f700 (LWP 730839) "cmsRun"):
#0 0x000014ae6482a45c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x000014ae2282684e in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WaitForWork(Eigen::EventCount::Waiter*, tsl::thread::EigenEnvironment::Task*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#2 0x000014ae22826e03 in Eigen::ThreadPoolTempl<tsl::thread::EigenEnvironment>::WorkerLoop(int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#3 0x000014ae22824578 in std::_Function_handler<void (), tsl::thread::EigenEnvironment::CreateThread(std::function<void ()>)::{lambda()#1}>::_M_invoke(std::_Any_data const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#4 0x000014ae15596a02 in tsl::(anonymous namespace)::PThread::ThreadFn(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#5 0x000014ae648241ca in start_thread () from /lib64/libpthread.so.0
#6 0x000014ae64490e73 in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x14ae0d7ff700 (LWP 728884) "cmsRun"):
#0 0x000014ae64550868 in nanosleep () from /lib64/libc.so.6
#1 0x000014ae6455076e in sleep () from /lib64/libc.so.6
#2 0x000014ae5e3d3d20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014ae289dcf26 in boost::archive::detail::iserializer<eos::portable_iarchive, EcalPulseCovariance>::load_object_data(boost::archive::detail::basic_iarchive&, void*, unsigned int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/libCondFormatsEcalObjects.so
#5 0x000014ae33274c07 in boost::archive::detail::basic_iarchive::load_object(void*, boost::archive::detail::basic_iserializer const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libboost_serialization.so.1.80.0
#6 0x000014ae289d63b9 in boost::archive::detail::iserializer<eos::portable_iarchive, std::vector<EcalPulseCovariance, std::allocator<EcalPulseCovariance> > >::load_object_data(boost::archive::detail::basic_iarchive&, void*, unsigned int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/libCondFormatsEcalObjects.so
#7 0x000014ae33274c07 in boost::archive::detail::basic_iarchive::load_object(void*, boost::archive::detail::basic_iserializer const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libboost_serialization.so.1.80.0
#8 0x000014ae289bac3a in boost::archive::detail::iserializer<eos::portable_iarchive, EcalContainer<EBDetId, EcalPulseCovariance> >::load_object_data(boost::archive::detail::basic_iarchive&, void*, unsigned int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/libCondFormatsEcalObjects.so
#9 0x000014ae33274c07 in boost::archive::detail::basic_iarchive::load_object(void*, boost::archive::detail::basic_iserializer const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libboost_serialization.so.1.80.0
#10 0x000014ae28998789 in void EcalCondObjectContainer<EcalPulseCovariance>::serialize<eos::portable_iarchive>(eos::portable_iarchive&, unsigned int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/libCondFormatsEcalObjects.so
#11 0x000014ae33274c07 in boost::archive::detail::basic_iarchive::load_object(void*, boost::archive::detail::basic_iserializer const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libboost_serialization.so.1.80.0
#12 0x000014ae1029f458 in std::unique_ptr<EcalCondObjectContainer<EcalPulseCovariance>, std::default_delete<EcalCondObjectContainer<EcalPulseCovariance> > > cond::persistency::Session::fetchPayload<EcalCondObjectContainer<EcalPulseCovariance> >(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginCondCoreEcalPlugins.so
#13 0x000014ae1028b47b in cond::persistency::PayloadProxy<EcalCondObjectContainer<EcalPulseCovariance> >::loadPayload() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginCondCoreEcalPlugins.so
#14 0x000014ae10289367 in cond::ProductResolver<EcalPulseCovariancesRcd, EcalCondObjectContainer<EcalPulseCovariance>, cond::DefaultInitializer<EcalCondObjectContainer<EcalPulseCovariance> > >::prefetch(edm::eventsetup::DataKey const&, edm::EventSetupRecordDetails) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginCondCoreEcalPlugins.so
#15 0x000014ae66eb0fee in edm::eventsetup::ESSourceProductResolverBase::doPrefetchAndSignals(edm::eventsetup::EventSetupRecordImpl const&, edm::eventsetup::DataKey const&, edm::ESParentContext const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#16 0x000014ae66eb10b5 in edm::SerialTaskQueue::QueuedTask<edm::eventsetup::ESSourceProductResolverBase::prefetchAsyncImplTemplate<edm::eventsetup::ESSourceProductResolverNonConcurrentBase::prefetchAsyncImpl(edm::WaitingTaskHolder, edm::eventsetup::EventSetupRecordImpl const&, edm::eventsetup::DataKey const&, edm::EventSetupImpl const*, edm::ServiceToken const&, edm::ESParentContext const&)::{lambda(auto:1&, auto:2)#1}, edm::eventsetup::ESSourceProductResolverNonConcurrentBase::prefetchAsyncImpl(edm::WaitingTaskHolder, edm::eventsetup::EventSetupRecordImpl const&, edm::eventsetup::DataKey const&, edm::EventSetupImpl const*, edm::ServiceToken const&, edm::ESParentContext const&)::{lambda()#2}>(edm::eventsetup::ESSourceProductResolverNonConcurrentBase::prefetchAsyncImpl(edm::WaitingTaskHolder, edm::eventsetup::EventSetupRecordImpl const&, edm::eventsetup::DataKey const&, edm::EventSetupImpl const*, edm::ServiceToken const&, edm::ESParentContext const&)::{lambda(auto:1&, auto:2)#1}, edm::eventsetup::ESSourceProductResolverNonConcurrentBase::prefetchAsyncImpl(edm::WaitingTaskHolder, edm::eventsetup::EventSetupRecordImpl const&, edm::eventsetup::DataKey const&, edm::EventSetupImpl const*, edm::ServiceToken const&, edm::ESParentContext const&)::{lambda()#2}, edm::WaitingTaskHolder, edm::eventsetup::EventSetupRecordImpl const&, edm::eventsetup::DataKey const&, edm::ESParentContext const&)::{lambda()#1}>::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#17 0x000014ae670e71d0 in tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#18 0x000014ae656e595b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x14ad9caac700, waiter=..., this=0x14ae62dc9500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#19 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x14ae62dc9500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#20 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/arena.cpp:137
#21 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/market.cpp:599
#22 0x000014ae656e7b0e in tbb::detail::r1::rml::private_worker::run (this=0x14ae5f4dd000) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#23 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x14ae5f4dd000) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#24 0x000014ae648241ca in start_thread () from /lib64/libpthread.so.0
#25 0x000014ae64490e73 in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x14ae0ec5c700 (LWP 728883) "cmsRun"):
#0 0x000014ae64550868 in nanosleep () from /lib64/libc.so.6
#1 0x000014ae6455076e in sleep () from /lib64/libc.so.6
#2 0x000014ae5e3d3d20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014ae1363ef90 in l1t::L1Candidate::L1Candidate(l1t::L1Candidate const&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libDataFormatsL1THGCal.so
#5 0x000014ae13644a20 in l1t::HGCalTower::HGCalTower(l1t::HGCalTower const&) [clone .lto_priv.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libDataFormatsL1THGCal.so
#6 0x000014ae136578c6 in std::__detail::_Hash_node<std::pair<unsigned short const, l1t::HGCalTower>, false>* std::__detail::_Hashtable_alloc<std::allocator<std::__detail::_Hash_node<std::pair<unsigned short const, l1t::HGCalTower>, false> > >::_M_allocate_node<std::pair<unsigned short const, l1t::HGCalTower> const&>(std::pair<unsigned short const, l1t::HGCalTower> const&) [clone .isra.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libDataFormatsL1THGCal.so
#7 0x000014ae13645c53 in ROOT::Detail::TCollectionProxyInfo::MapInsert<std::unordered_map<unsigned short, l1t::HGCalTower, std::hash<unsigned short>, std::equal_to<unsigned short>, std::allocator<std::pair<unsigned short const, l1t::HGCalTower> > > >::feed(void*, void*, unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libDataFormatsL1THGCal.so
#8 0x000014ae66499772 in TGenCollectionProxy::Commit(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libRIO.so
#9 0x000014ae66648eb4 in int TStreamerInfo::ReadBuffer<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libRIO.so
#10 0x000014ae664fcfcc in TStreamerInfoActions::VectorLooper::GenericRead(TBuffer&, void*, void const*, TStreamerInfoActions::TLoopConfiguration const*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libRIO.so
#11 0x000014ae663f2d0c in TBufferFile::ApplySequence(TStreamerInfoActions::TActionSequence const&, void*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libRIO.so
#12 0x000014ae6654a50f in int TStreamerInfoActions::ReadSTL<&TStreamerInfoActions::ReadSTLMemberWiseSameClass, &TStreamerInfoActions::ReadSTLObjectWiseFastArray>(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libRIO.so
#13 0x000014ae663fb5be in TBufferFile::ReadClassBuffer(TClass const*, void*, TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libRIO.so
#14 0x000014ae66647f4c in int TStreamerInfo::ReadBuffer<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libRIO.so
#15 0x000014ae664c9acd in TStreamerInfoActions::GenericReadAction(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libRIO.so
#16 0x000014ae663f32c5 in TBufferFile::ApplySequence(TStreamerInfoActions::TActionSequence const&, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libRIO.so
#17 0x000014ae66b5cb87 in TBranchElement::ReadLeavesMember(TBuffer&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libTree.so
#18 0x000014ae66b55429 in TBranch::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libTree.so
#19 0x000014ae66b67d44 in TBranchElement::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libTree.so
#20 0x000014ae66b67cfd in TBranchElement::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libTree.so
#21 0x000014ae0de2e3fc in edm::RootTree::getEntry(TBranch*, long long) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#22 0x000014ae0de0f1ec in edm::RootDelayedReader::getProduct_(edm::BranchID const&, edm::EDProductGetter const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginIOPoolInput.so
#23 0x000014ae66ea2b5f in edm::DelayedReader::getProduct(edm::BranchID const&, edm::EDProductGetter const*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#24 0x000014ae66f4a3bb in edm::DelayedReaderInputProductResolver::prefetchAsync_(edm::WaitingTaskHolder, edm::Principal const&, bool, edm::ServiceToken const&, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const::{lambda()#1}::operator()() const::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#25 0x000014ae66f4b7ec in edm::DelayedReaderInputProductResolver::prefetchAsync_(edm::WaitingTaskHolder, edm::Principal const&, bool, edm::ServiceToken const&, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#26 0x000014ae66f4b938 in edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::DelayedReaderInputProductResolver::prefetchAsync_(edm::WaitingTaskHolder, edm::Principal const&, bool, edm::ServiceToken const&, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::DelayedReaderInputProductResolver::prefetchAsync_(edm::WaitingTaskHolder, edm::Principal const&, bool, edm::ServiceToken const&, edm::SharedResourcesAcquirer*, edm::ModuleCallingContext const*) const::{lambda()#1}&)::{lambda()#1}>::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#27 0x000014ae670e71d0 in tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#28 0x000014ae656e595b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x14ad9caac100, waiter=..., this=0x14ae62dc9480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#29 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x14ae62dc9480) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#30 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/arena.cpp:137
#31 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/market.cpp:599
#32 0x000014ae656e7b0e in tbb::detail::r1::rml::private_worker::run (this=0x14ae5f4dd100) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#33 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x14ae5f4dd100) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#34 0x000014ae648241ca in start_thread () from /lib64/libpthread.so.0
#35 0x000014ae64490e73 in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x14ae0f65d700 (LWP 728882) "cmsRun"):
#0 0x000014ae6457b251 in poll () from /lib64/libc.so.6
#1 0x000014ae5e40bddf in full_read.constprop () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x000014ae5e3d739c in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 0x000014ae5e3d7520 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x000014add14dcc6d in void std::vector<int, std::allocator<int> >::_M_realloc_insert<int>(__gnu_cxx::__normal_iterator<int*, std::vector<int, std::allocator<int> > >, int&&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#6 0x000014add14e29bc in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#7 0x000014add14e2c83 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#8 0x000014ae656ee281 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x14ae62dc9400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#9 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x14ae62dc9400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#10 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#11 0x000014add14dfd17 in tbb::detail::d1::task_arena_function<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#12 0x000014ae656db28e in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/arena.cpp:757
#13 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#14 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/arena.cpp:758
#15 0x000014add14d1854 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#16 0x000014add14ee8cf in HGCalLayerClusterProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#17 0x000014ae66fc10d1 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#18 0x000014ae66fa584e in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#19 0x000014ae66f318e9 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#20 0x000014ae66f31e84 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#21 0x000014ae670e5f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#22 0x000014ae656e595b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x14ad9cb5c200, waiter=..., this=0x14ae62dc9400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#23 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x14ae62dc9400) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#24 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/arena.cpp:137
#25 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/market.cpp:599
#26 0x000014ae656e7b0e in tbb::detail::r1::rml::private_worker::run (this=0x14ae5f4dd080) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#27 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x14ae5f4dd080) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#28 0x000014ae648241ca in start_thread () from /lib64/libpthread.so.0
#29 0x000014ae64490e73 in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x14ae33c08700 (LWP 728399) "cmsRun"):
#0 0x000014ae6482e672 in waitpid () from /lib64/libpthread.so.0
#1 0x000014ae5e3d3e77 in edm::service::cmssw_stacktrace_fork() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x000014ae5e3d72ca in edm::service::InitRootHandlers::stacktraceHelperThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 0x000014ae64eb8a73 in std::execute_native_thread_routine (__p=0x14ae570608c0) at ../../../../../libstdc++-v3/src/c++11/thread.cc:82
#4 0x000014ae648241ca in start_thread () from /lib64/libpthread.so.0
#5 0x000014ae64490e73 in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x14ae639af640 (LWP 722518) "cmsRun"):
#0 0x000014ae64550868 in nanosleep () from /lib64/libc.so.6
#1 0x000014ae6455076e in sleep () from /lib64/libc.so.6
#2 0x000014ae5e3d3d20 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000014ae64580c67 in mmap64 () from /lib64/libc.so.6
#5 0x000014ae1f69f2df in Xbyak::MmapAllocator::alloc(unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#6 0x000014ae1f69a0ab in Xbyak::CodeArray::CodeArray(unsigned long, void*, Xbyak::Allocator*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#7 0x000014ae1fd52abe in dnnl::impl::cpu::x64::avx_gemm_f32::xbyak_gemm_t::xbyak_gemm_t(char, char, float, bool, void*, unsigned long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#8 0x000014ae1fd5795d in std::once_flag::_Prepare_execution::_Prepare_execution<std::call_once<dnnl::impl::cpu::x64::avx_gemm_f32::get_xbyak_gemm(bool, bool, float, bool)::{lambda()#2}>(std::once_flag&, dnnl::impl::cpu::x64::avx_gemm_f32::get_xbyak_gemm(bool, bool, float, bool)::{lambda()#2}&&)::{lambda()#1}>(dnnl::impl::cpu::x64::avx_gemm_f32::get_xbyak_gemm(bool, bool, float, bool)::{lambda()#2}&)::{lambda()#1}::_FUN() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#9 0x000014ae6482be67 in __pthread_once_slow () from /lib64/libpthread.so.0
#10 0x000014ae1fd39caf in dnnl::impl::cpu::x64::avx_gemm_f32::get_xbyak_gemm(bool, bool, float, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#11 0x000014ae1fd39e7a in dnnl::impl::cpu::x64::avx_gemm_f32::sgemm_nocopy_driver(char const*, char const*, long, long, long, float const*, float const*, long, float const*, long, float const*, float*, long, float const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#12 0x000014ae1fd3affb in dnnl::impl::cpu::x64::jit_avx_gemm_f32(int, char const*, char const*, long const*, long const*, long const*, float const*, float const*, long const*, float const*, long const*, float const*, float*, long const*, float const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#13 0x000014ae1fe5f63d in dnnl_status_t dnnl::impl::cpu::x64::gemm_driver<float, float, float>(char const*, char const*, char const*, long const*, long const*, long const*, float const*, float const*, long const*, float const*, float const*, long const*, float const*, float const*, float*, long const*, float const*, bool, dnnl::impl::cpu::x64::pack_type, dnnl::impl::cpu::x64::gemm_pack_storage_t*, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#14 0x000014ae1f984567 in dnnl::impl::cpu::extended_sgemm(char const*, char const*, long const*, long const*, long const*, float const*, float const*, long const*, float const*, long const*, float const*, float*, long const*, float const*, bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#15 0x000014ae1f625d23 in dnnl_sgemm () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#16 0x000014ae1cf9eec7 in Eigen::internal::TensorContractionKernel<float, float, float, long, Eigen::internal::blas_data_mapper<float, long, 0, 0, 1>, Eigen::internal::TensorContractionInputMapper<float, long, 1, Eigen::TensorEvaluator<Eigen::TensorReshapingOp<Eigen::DSizes<long, 2> const, Eigen::TensorMap<Eigen::Tensor<float const, 4, 1, long>, 16, Eigen::MakePointer> const> const, Eigen::ThreadPoolDevice>, Eigen::array<long, 1ul>, Eigen::array<long, 1ul>, 4, true, false, 0, Eigen::MakePointer>, Eigen::internal::TensorContractionInputMapper<float, long, 0, Eigen::TensorEvaluator<Eigen::TensorReshapingOp<Eigen::DSizes<long, 2> const, Eigen::TensorImagePatchOp<-1l, -1l, Eigen::TensorMap<Eigen::Tensor<float const, 4, 1, long>, 16, Eigen::MakePointer> const> const> const, Eigen::ThreadPoolDevice>, Eigen::array<long, 1ul>, Eigen::array<long, 1ul>, 4, true, false, 0, Eigen::MakePointer> >::invoke(Eigen::internal::blas_data_mapper<float, long, 0, 0, 1> const&, Eigen::internal::ColMajorBlock<float, long> const&, Eigen::internal::ColMajorBlock<float, long> const&, long, long, long, float, float) [clone .isra.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#17 0x000014ae1cfef67d in void Eigen::TensorContractionEvaluatorBase<Eigen::TensorEvaluator<Eigen::TensorContractionOp<Eigen::array<Eigen::IndexPair<long>, 1ul> const, Eigen::TensorReshapingOp<Eigen::DSizes<long, 2> const, Eigen::TensorImagePatchOp<-1l, -1l, Eigen::TensorMap<Eigen::Tensor<float const, 4, 1, long>, 16, Eigen::MakePointer> const> const> const, Eigen::TensorReshapingOp<Eigen::DSizes<long, 2> const, Eigen::TensorMap<Eigen::Tensor<float const, 4, 1, long>, 16, Eigen::MakePointer> const> const, tensorflow::LaunchFusedConv2DWithOutputKernel<float>::OutputKernelWrapper const> const, Eigen::ThreadPoolDevice> >::evalGemmPartial<true, true, false, 0, true>(float*, long, long, int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#18 0x000014ae1d020e75 in Eigen::internal::TensorExecutor<Eigen::TensorAssignOp<Eigen::TensorMap<Eigen::Tensor<float, 4, 1, long>, 16, Eigen::MakePointer>, Eigen::TensorReshapingOp<Eigen::DSizes<long, 4> const, Eigen::TensorContractionOp<Eigen::array<Eigen::IndexPair<long>, 1ul> const, Eigen::TensorReshapingOp<Eigen::DSizes<long, 2> const, Eigen::TensorImagePatchOp<-1l, -1l, Eigen::TensorMap<Eigen::Tensor<float const, 4, 1, long>, 16, Eigen::MakePointer> const> const> const, Eigen::TensorReshapingOp<Eigen::DSizes<long, 2> const, Eigen::TensorMap<Eigen::Tensor<float const, 4, 1, long>, 16, Eigen::MakePointer> const> const, tensorflow::LaunchFusedConv2DWithOutputKernel<float>::OutputKernelWrapper const> const> const> const, Eigen::ThreadPoolDevice, true, (Eigen::internal::TiledEvaluation)0>::run(Eigen::TensorAssignOp<Eigen::TensorMap<Eigen::Tensor<float, 4, 1, long>, 16, Eigen::MakePointer>, Eigen::TensorReshapingOp<Eigen::DSizes<long, 4> const, Eigen::TensorContractionOp<Eigen::array<Eigen::IndexPair<long>, 1ul> const, Eigen::TensorReshapingOp<Eigen::DSizes<long, 2> const, Eigen::TensorImagePatchOp<-1l, -1l, Eigen::TensorMap<Eigen::Tensor<float const, 4, 1, long>, 16, Eigen::MakePointer> const> const> const, Eigen::TensorReshapingOp<Eigen::DSizes<long, 2> const, Eigen::TensorMap<Eigen::Tensor<float const, 4, 1, long>, 16, Eigen::MakePointer> const> const, tensorflow::LaunchFusedConv2DWithOutputKernel<float>::OutputKernelWrapper const> const> const> const&, Eigen::ThreadPoolDevice const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#19 0x000014ae1d021c74 in void tensorflow::functor::SpatialConvolutionFunc<Eigen::ThreadPoolDevice, Eigen::TensorMap<Eigen::Tensor<float const, 4, 1, long>, 16, Eigen::MakePointer>, Eigen::TensorMap<Eigen::Tensor<float const, 4, 1, long>, 16, Eigen::MakePointer>, Eigen::TensorMap<Eigen::Tensor<float, 4, 1, long>, 16, Eigen::MakePointer>, tensorflow::LaunchFusedConv2DWithOutputKernel<float>::OutputKernelWrapper>(Eigen::ThreadPoolDevice const&, Eigen::TensorMap<Eigen::Tensor<float, 4, 1, long>, 16, Eigen::MakePointer>, Eigen::TensorMap<Eigen::Tensor<float const, 4, 1, long>, 16, Eigen::MakePointer>, Eigen::TensorMap<Eigen::Tensor<float const, 4, 1, long>, 16, Eigen::MakePointer>, int, int, int, int, Eigen::PaddingType const&, tensorflow::LaunchFusedConv2DWithOutputKernel<float>::OutputKernelWrapper const&, int, int, int, int) [clone .isra.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#20 0x000014ae1d02389d in void tensorflow::LaunchFusedConv2DWithOutputKernel<float>::operator()<tensorflow::BiasAddOutputKernel<float, tensorflow::Relu> >(tensorflow::BiasAddOutputKernel<float, tensorflow::Relu> const&, tensorflow::OpKernelContext*, tensorflow::Tensor const&, tensorflow::Tensor const&, tensorflow::Tensor*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#21 0x000014ae1d027cb5 in tensorflow::FusedConv2DOp<Eigen::ThreadPoolDevice, float>::Compute(tensorflow::OpKernelContext*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#22 0x000014ae1567e5fb in tensorflow::ThreadPoolDevice::Compute(tensorflow::OpKernel*, tensorflow::OpKernelContext*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#23 0x000014ae1576c621 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ProcessInline(tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*, long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#24 0x000014ae1576d706 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::Process(tensorflow::SimplePropagatorState::TaggedNode, long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#25 0x000014ae2282757c in tsl::thread::ThreadPool::Schedule(std::function<void ()>) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#26 0x000014ae229658f8 in std::_Function_handler<void (std::function<void ()>), tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&)::{lambda(std::function<void ()>)#6}>::_M_invoke(std::_Any_data const&, std::function<void ()>&&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#27 0x000014ae15752004 in void tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::RunTask<std::_Bind<void (tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::*(tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>*, tensorflow::SimplePropagatorState::TaggedNode, long))(tensorflow::SimplePropagatorState::TaggedNode, long)> >(std::_Bind<void (tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::*(tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>*, tensorflow::SimplePropagatorState::TaggedNode, long))(tensorflow::SimplePropagatorState::TaggedNode, long)>&&, int) [clone .constprop.0] () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#28 0x000014ae15757a4e in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ScheduleReady(absl::lts_20220623::InlinedVector<tensorflow::SimplePropagatorState::TaggedNode, 8ul, std::allocator<tensorflow::SimplePropagatorState::TaggedNode> >*, tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#29 0x000014ae157616d5 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::NodeDone(tsl::Status const&, absl::lts_20220623::InlinedVector<tensorflow::SimplePropagatorState::TaggedNode, 8ul, std::allocator<tensorflow::SimplePropagatorState::TaggedNode> >*, tensorflow::NodeExecStatsInterface*, tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#30 0x000014ae1576b93c in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ProcessInline(tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*, long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#31 0x000014ae1576d706 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::Process(tensorflow::SimplePropagatorState::TaggedNode, long) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#32 0x000014ae2282757c in tsl::thread::ThreadPool::Schedule(std::function<void ()>) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#33 0x000014ae229658f8 in std::_Function_handler<void (std::function<void ()>), tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&)::{lambda(std::function<void ()>)#6}>::_M_invoke(std::_Any_data const&, std::function<void ()>&&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#34 0x000014ae15757951 in tensorflow::(anonymous namespace)::ExecutorState<tensorflow::SimplePropagatorState>::ScheduleReady(absl::lts_20220623::InlinedVector<tensorflow::SimplePropagatorState::TaggedNode, 8ul, std::allocator<tensorflow::SimplePropagatorState::TaggedNode> >*, tensorflow::SimplePropagatorState::TaggedNodeReadyQueue*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#35 0x000014ae1575c5e8 in tensorflow::(anonymous namespace)::ExecutorImpl::RunAsync(tensorflow::Executor::Args const&, std::function<void (tsl::Status const&)>) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_framework.so.2
#36 0x000014ae2297826a in tensorflow::DirectSession::RunInternal(long, tensorflow::RunOptions const&, tensorflow::CallFrameInterface*, tensorflow::DirectSession::ExecutorsAndKeys*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#37 0x000014ae2297af9e in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/external/el8_amd64_gcc12/lib/libtensorflow_cc.so.2
#38 0x000014ae27e494d5 in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/libPhysicsToolsTensorFlow.so
#39 0x000014ae27e49589 in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tsl::thread::ThreadPoolInterface*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/libPhysicsToolsTensorFlow.so
#40 0x000014adba810712 in ticl::PatternRecognitionbyCLUE3D<TICLGenericTile<std::array<TICLLayerTileT<ticl::TileConstants>, 104ul> > >::energyRegressionAndID(std::vector<reco::CaloCluster, std::allocator<reco::CaloCluster> > const&, tensorflow::Session const*, std::vector<ticl::Trackster, std::allocator<ticl::Trackster> >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/pluginRecoHGCalTICLPlugins.so
#41 0x000014adba811567 in ticl::PatternRecognitionbyCLUE3D<TICLGenericTile<std::array<TICLLayerTileT<ticl::TileConstants>, 104ul> > >::makeTracksters(ticl::PatternRecognitionAlgoBaseT<TICLGenericTile<std::array<TICLLayerTileT<ticl::TileConstants>, 104ul> > >::Inputs const&, std::vector<ticl::Trackster, std::allocator<ticl::Trackster> >&, std::unordered_map<int, std::vector<int, std::allocator<int> >, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, std::vector<int, std::allocator<int> > > > >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/pluginRecoHGCalTICLPlugins.so
#42 0x000014adba855924 in TrackstersProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/pluginRecoHGCalTICLPlugins.so
#43 0x000014ae66fc10d1 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#44 0x000014ae66fa584e in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#45 0x000014ae66f318e9 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#46 0x000014ae66f31e84 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#47 0x000014ae670e5f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw/CMSSW_13_3_ROOT630_X_2023-10-18-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#48 0x000014ae656ee281 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x14ae62dc9380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#49 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x14ae62dc9380) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#50 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#51 0x000014ae66eb61bb in edm::FinalWaitingTask::wait() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#52 0x000014ae66ebfbc5 in edm::EventProcessor::processRuns() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#53 0x000014ae66ec018f in edm::EventProcessor::runToCompletion() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02807/el8_amd64_gcc12/cms/cmssw-patch/CMSSW_13_3_ROOT630_X_2023-10-19-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#54 0x00000000004074ef in tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const ()
#55 0x000014ae656da9ad in tbb::detail::r1::task_arena_impl::execute (ta=..., d=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_13_3_0_pre3-el8_amd64_gcc12/build/CMSSW_13_3_0_pre3-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-e755918dac6a30ec36eff63ac4f7ddec/tbb-v2021.9.0/src/tbb/arena.cpp:688
#56 0x0000000000408ed2 in main::{lambda()#1}::operator()() const ()
#57 0x000000000040517c in main ()
Current Modules:
Module: HGCalLayerClusterProducer:hgcalLayerClustersHSci (crashed)
Module: none
Module: TrackstersProducer:ticlTrackstersCLUE3DHigh
Module: none
New stack trace, somewhat different from the old ones. WF 25234.911, el9_amd64_gcc12, CMSSW_14_0_X_2023-11-19-2300
https://cmssdt.cern.ch/SDT/cgi-bin/buildlogs/raw/el9_amd64_gcc12/CMSSW_14_0_X_2023-11-19-2300/pyRelValMatrixLogs/run/25234.911_TTbar_14TeV+2026D99_DD4hep/step3_TTbar_14TeV+2026D99_DD4hep.log
Thread 5 (Thread 0x154f9f3ff640 (LWP 3576526) "cmsRun"):
##3 0x0000154fe93daaa0 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x0000154fa8fcb970 in tensorflow::DeviceAttributes::~DeviceAttributes() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libtensorflow_framework.so.2
#6 0x0000154fb5923629 in tensorflow::grappler::VirtualCluster::VirtualCluster(tensorflow::DeviceSet const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libtensorflow_cc.so.2
#7 0x0000154fb579f2e0 in tensorflow::GraphExecutionState::OptimizeGraph(tensorflow::BuildGraphOptions const&, tensorflow::Graph const&, tensorflow::FunctionLibraryDefinition const*, std::unique_ptr<tensorflow::Graph, std::default_delete<tensorflow::Graph> >*, std::unique_ptr<tensorflow::FunctionLibraryDefinition, std::default_delete<tensorflow::FunctionLibraryDefinition> >*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libtensorflow_cc.so.2
#8 0x0000154fb579ff5f in tensorflow::GraphExecutionState::BuildGraph(tensorflow::BuildGraphOptions const&, std::unique_ptr<tensorflow::ClientGraph, std::default_delete<tensorflow::ClientGraph> >*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libtensorflow_cc.so.2
#9 0x0000154fb6104166 in tensorflow::DirectSession::CreateGraphs(tensorflow::BuildGraphOptions const&, std::unordered_map<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::unique_ptr<tensorflow::Graph, std::default_delete<tensorflow::Graph> >, std::hash<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::equal_to<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const, std::unique_ptr<tensorflow::Graph, std::default_delete<tensorflow::Graph> > > > >*, std::unique_ptr<tensorflow::FunctionLibraryDefinition, std::default_delete<tensorflow::FunctionLibraryDefinition> >*, tensorflow::DirectSession::RunStateArgs*, absl::lts_20220623::InlinedVector<tensorflow::DataType, 4ul, std::allocator<tensorflow::DataType> >*, absl::lts_20220623::InlinedVector<tensorflow::DataType, 4ul, std::allocator<tensorflow::DataType> >*, long*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libtensorflow_cc.so.2
#10 0x0000154fb6106602 in tensorflow::DirectSession::CreateExecutors(tensorflow::CallableOptions const&, std::unique_ptr<tensorflow::DirectSession::ExecutorsAndKeys, std::default_delete<tensorflow::DirectSession::ExecutorsAndKeys> >*, std::unique_ptr<tensorflow::DirectSession::FunctionInfo, std::default_delete<tensorflow::DirectSession::FunctionInfo> >*, tensorflow::DirectSession::RunStateArgs*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libtensorflow_cc.so.2
#11 0x0000154fb61093cd in tensorflow::DirectSession::GetOrCreateExecutors(absl::lts_20220623::Span<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const>, absl::lts_20220623::Span<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const>, absl::lts_20220623::Span<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const>, tensorflow::DirectSession::ExecutorsAndKeys**, tensorflow::DirectSession::RunStateArgs*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libtensorflow_cc.so.2
#12 0x0000154fb610d82a in tensorflow::DirectSession::Run(tensorflow::RunOptions const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tensorflow::RunMetadata*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libtensorflow_cc.so.2
#13 0x0000154fbb5da4d5 in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tsl::thread::ThreadPoolOptions const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/libPhysicsToolsTensorFlow.so
#14 0x0000154fbb5da589 in tensorflow::run(tensorflow::Session*, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, tensorflow::Tensor> > > const&, std::vector<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, std::allocator<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > > > const&, std::vector<tensorflow::Tensor, std::allocator<tensorflow::Tensor> >*, tsl::thread::ThreadPoolInterface*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/libPhysicsToolsTensorFlow.so
#15 0x0000154f4dc93f62 in ticl::PatternRecognitionbyCLUE3D<TICLGenericTile<std::array<TICLLayerTileT<ticl::TileConstants>, 104ul> > >::energyRegressionAndID(std::vector<reco::CaloCluster, std::allocator<reco::CaloCluster> > const&, tensorflow::Session const*, std::vector<ticl::Trackster, std::allocator<ticl::Trackster> >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginRecoHGCalTICLPlugins.so
#16 0x0000154f4dc94877 in ticl::PatternRecognitionbyCLUE3D<TICLGenericTile<std::array<TICLLayerTileT<ticl::TileConstants>, 104ul> > >::makeTracksters(ticl::PatternRecognitionAlgoBaseT<TICLGenericTile<std::array<TICLLayerTileT<ticl::TileConstants>, 104ul> > >::Inputs const&, std::vector<ticl::Trackster, std::allocator<ticl::Trackster> >&, std::unordered_map<int, std::vector<int, std::allocator<int> >, std::hash<int>, std::equal_to<int>, std::allocator<std::pair<int const, std::vector<int, std::allocator<int> > > > >&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginRecoHGCalTICLPlugins.so
#17 0x0000154f4dcf3aea in TrackstersProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginRecoHGCalTICLPlugins.so
Thread 4 (Thread 0x154fa02bb640 (LWP 3576525) "cmsRun"):
#3 0x0000154fe93de500 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x0000154feda9c47f in cache_bin_alloc_impl (adjust_low_water=false, success=<synthetic pointer>, bin=<optimized out>) at include/jemalloc/internal/cache_bin.h:361
#6 cache_bin_alloc_easy (success=<synthetic pointer>, bin=<optimized out>) at include/jemalloc/internal/cache_bin.h:399
#7 imalloc_fastpath (fallback_alloc=0x154feda9c3e0 <fallback_impl<false>(std::size_t)>, size=4) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:326
#8 newImpl<false> (size=4) at src/jemalloc_cpp.cpp:109
#9 operator new (size=4) at src/jemalloc_cpp.cpp:114
#10 0x0000154f6406a3f8 in void std::vector<int, std::allocator<int> >::_M_realloc_insert<int>(__gnu_cxx::__normal_iterator<int*, std::vector<int, std::allocator<int> > >, int&&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#11 0x0000154f64065975 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::findAndAssignClusters(unsigned int, float) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#12 0x0000154f64072f39 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#13 0x0000154f640732cb in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#14 0x0000154fed629241 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x154febfd9400) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el9_amd64_gcc12/external/tbb/v2021.9.0-eb2105f6bbb3f8753e00290bed91f88a/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#15 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x154febfd9400) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el9_amd64_gcc12/external/tbb/v2021.9.0-eb2105f6bbb3f8753e00290bed91f88a/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#16 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el9_amd64_gcc12/external/tbb/v2021.9.0-eb2105f6bbb3f8753e00290bed91f88a/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#17 0x0000154f6406bd47 in tbb::detail::d1::task_arena_function<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#18 0x0000154fed61624e in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el9_amd64_gcc12/external/tbb/v2021.9.0-eb2105f6bbb3f8753e00290bed91f88a/tbb-v2021.9.0/src/tbb/arena.cpp:757
#19 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el9_amd64_gcc12/external/tbb/v2021.9.0-eb2105f6bbb3f8753e00290bed91f88a/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#20 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/toolconf/BUILD/el9_amd64_gcc12/external/tbb/v2021.9.0-eb2105f6bbb3f8753e00290bed91f88a/tbb-v2021.9.0/src/tbb/arena.cpp:758
#21 0x0000154f6405e064 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#22 0x0000154f6407b22f in HGCalLayerClusterProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
Thread 3 (Thread 0x154fa0cbc640 (LWP 3576524) "cmsRun"):
#3 0x0000154fe93daaa0 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x0000154fee32c0b0 in TBufferFile::ReadVersion(unsigned int*, unsigned int*, TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libRIO.so
#6 0x0000154fee32e00f in TBufferFile::ReadClassBuffer(TClass const*, void*, TClass const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libRIO.so
#7 0x0000154fee5767ac in int TStreamerInfo::ReadBuffer<char**>(TBuffer&, char** const&, TStreamerInfo::TCompInfo* const*, int, int, int, int, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libRIO.so
#8 0x0000154fee42b92c in TStreamerInfoActions::VectorLooper::GenericRead(TBuffer&, void*, void const*, TStreamerInfoActions::TLoopConfiguration const*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libRIO.so
#9 0x0000154fee32570c in TBufferFile::ApplySequence(TStreamerInfoActions::TActionSequence const&, void*, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libRIO.so
#10 0x0000154fee478e6f in int TStreamerInfoActions::ReadSTL<&TStreamerInfoActions::ReadSTLMemberWiseSameClass, &TStreamerInfoActions::ReadSTLObjectWiseFastArray>(TBuffer&, void*, TStreamerInfoActions::TConfiguration const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libRIO.so
#11 0x0000154fee325cc5 in TBufferFile::ApplySequence(TStreamerInfoActions::TActionSequence const&, void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libRIO.so
#12 0x0000154feea733c7 in TBranchElement::ReadLeavesMember(TBuffer&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libTree.so
#13 0x0000154feea6bcc9 in TBranch::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libTree.so
#14 0x0000154feea86fc4 in TBranchElement::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libTree.so
#15 0x0000154feea86f7d in TBranchElement::GetEntry(long long, int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libTree.so
#16 0x0000154f9f88b3ec in edm::RootTree::getEntry(TBranch*, long long) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginIOPoolInput.so
#17 0x0000154f9f86c25c in edm::RootDelayedReader::getProduct_(edm::BranchID const&, edm::EDProductGetter const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginIOPoolInput.so
Thread 1 (Thread 0x154feca14640 (LWP 3575545) "cmsRun"):
#3 0x0000154fe93daaa0 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x0000154fed4588a8 in std::basic_streambuf<char, std::char_traits<char> >::xsgetn (this=0x7ffcf64a21f0, __s=0x7ffcf64a1bfc "", __n=4) at /data/cmsbld/jenkins/workspace/jenkins-test-bootstrap/bootstrap/BUILD/el9_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/gcc-12.3.1/obj/x86_64-redhat-linux-gnu/libstdc++-v3/include/bits/streambuf.tcc:49
#6 0x0000154fbc403ea2 in boost::archive::detail::iserializer<eos::portable_iarchive, EcalPulseCovariance>::load_object_data(boost::archive::detail::basic_iarchive&, void*, unsigned int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/libCondFormatsEcalObjects.so
#7 0x0000154fc3908c07 in boost::archive::detail::basic_iarchive::load_object(void*, boost::archive::detail::basic_iserializer const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libboost_serialization.so.1.80.0
#8 0x0000154fbc3fd389 in boost::archive::detail::iserializer<eos::portable_iarchive, std::vector<EcalPulseCovariance, std::allocator<EcalPulseCovariance> > >::load_object_data(boost::archive::detail::basic_iarchive&, void*, unsigned int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/libCondFormatsEcalObjects.so
#9 0x0000154fc3908c07 in boost::archive::detail::basic_iarchive::load_object(void*, boost::archive::detail::basic_iserializer const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libboost_serialization.so.1.80.0
#10 0x0000154fbc3e1c5a in boost::archive::detail::iserializer<eos::portable_iarchive, EcalContainer<EBDetId, EcalPulseCovariance> >::load_object_data(boost::archive::detail::basic_iarchive&, void*, unsigned int) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/libCondFormatsEcalObjects.so
#11 0x0000154fc3908c07 in boost::archive::detail::basic_iarchive::load_object(void*, boost::archive::detail::basic_iserializer const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libboost_serialization.so.1.80.0
#12 0x0000154fbc3bf779 in void EcalCondObjectContainer<EcalPulseCovariance>::serialize<eos::portable_iarchive>(eos::portable_iarchive&, unsigned int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/libCondFormatsEcalObjects.so
#13 0x0000154fc3908c07 in boost::archive::detail::basic_iarchive::load_object(void*, boost::archive::detail::basic_iserializer const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/external/el9_amd64_gcc12/lib/libboost_serialization.so.1.80.0
#14 0x0000154fa18f2b68 in std::unique_ptr<EcalCondObjectContainer<EcalPulseCovariance>, std::default_delete<EcalCondObjectContainer<EcalPulseCovariance> > > cond::persistency::Session::fetchPayload<EcalCondObjectContainer<EcalPulseCovariance> >(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginCondCoreEcalPlugins.so
#15 0x0000154fa18dea9b in cond::persistency::PayloadProxy<EcalCondObjectContainer<EcalPulseCovariance> >::loadPayload() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginCondCoreEcalPlugins.so
#16 0x0000154fa18ddc77 in cond::ProductResolver<EcalPulseCovariancesRcd, EcalCondObjectContainer<EcalPulseCovariance>, cond::DefaultInitializer<EcalCondObjectContainer<EcalPulseCovariance> > >::prefetch(edm::eventsetup::DataKey const&, edm::EventSetupRecordDetails) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/pluginCondCoreEcalPlugins.so
#17 0x0000154feefbfeee in edm::eventsetup::ESSourceProductResolverBase::doPrefetchAndSignals(edm::eventsetup::EventSetupRecordImpl const&, edm::eventsetup::DataKey const&, edm::ESParentContext const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el9_amd64_gcc12/cms/cmssw/CMSSW_14_0_X_2023-11-19-2300/lib/el9_amd64_gcc12/libFWCoreFramework.so
#18 0x0000154feefbffa2 in
Current Modules:
Module: HGCalLayerClusterProducer:hgcalLayerClustersHSci (crashed)
Module: none
Module: none
Module: TrackstersProducer:ticlTrackstersCLUE3DHigh
@dan131riley can we have the input file so we can reproduce the issue?
Probably the same issue is seed in UBSANR630_X IB. Also I found an old issue that looks similar: #41731.
Occurred in CMSSW_14_1_X_2024-02-23-2300 on el8_ppc64le_gcc12
Thread 5 (Thread 0x10085dc391f0 (LWP 118785) "cmsRun"):
#2 0x000010000ab85a3c in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 std::locale::operator= (this=0x10085dc374a0, __other=...) at ../../../../../libstdc++-v3/src/c++98/locale.cc:121
#5 0x00001000029398e4 in std::ios_base::_M_init (this=0x10085dc373d0) at ../../../../../libstdc++-v3/src/c++98/ios_locale.cc:44
#6 0x00001000029c3af8 in std::basic_ios<char, std::char_traits<char> >::init (this=<optimized out>, __sb=<optimized out>) at /scratch/cmsbuild/jenkins_a/workspace/jenkins-test-bootstrap/bootstrap/BUILD/el8_ppc64le_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/gcc-12.3.1/obj/powerpc64le-redhat-linux-gnu/libstdc++-v3/include/bits/basic_ios.tcc:129
#7 0x00001008321228e0 in HGCalDDDConstants::locateCell(int, int, int, int, int, int, bool, bool, bool, bool) const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libGeometryHGCalCommonData.so
#8 0x0000100831cbcb8c in HGCalGeometry::getPosition(DetId const&, bool) const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libGeometryHGCalGeometry.so
#9 0x000010097b91ce14 in HGCalTriggerGeometryV9Imp3::getModulePosition(unsigned int) const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginL1TriggerL1THGCalPlugins_geometries.so
#10 0x00001008369b05b4 in HGCalConcentratorTrigSumImpl::doSum(unsigned int, std::vector<l1t::HGCalTriggerCell, std::allocator<l1t::HGCalTriggerCell> > const&, std::vector<l1t::HGCalTriggerSums, std::allocator<l1t::HGCalTriggerSums> >&) const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libL1TriggerL1THGCal.so
#11 0x00001008db3a6ce0 in HGCalConcentratorProcessorSelection::run(edm::Handle<BXVector<l1t::HGCalTriggerCell> > const&, std::tuple<BXVector<l1t::HGCalTriggerCell>, BXVector<l1t::HGCalTriggerSums>, BXVector<l1t::HGCalConcentratorData> >&) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginL1TriggerL1THGCalPlugins_fe_be.so
#12 0x00001008368d75bc in HGCalConcentratorProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginL1TriggerL1THGCalPlugins.so
#13 0x000010000044c264 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
Thread 4 (Thread 0x10085d2291f0 (LWP 118784) "cmsRun"):
#3 0x000010000ab8b71c in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x00001008903fcdb4 in int& std::vector<int, std::allocator<int> >::emplace_back<int>(int&&) [clone .isra.0] () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#6 0x000010089040d6b0 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#7 0x00001008904144f0 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#8 0x0000100890414870 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#9 0x00001000025c7bc4 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x100881288900, this=0x100003823e00) at /scratch/cmsbuild/jenkins_b/workspace/auto-builds/CMSSW_14_0_0_pre3-el8_ppc64le_gcc12/build/CMSSW_14_0_0_pre3-build/BUILD/el8_ppc64le_gcc12/external/tbb/v2021.9.0-16196f107c456d12eb9d248792d8cb7b/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#10 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x100003823e00) at /scratch/cmsbuild/jenkins_b/workspace/auto-builds/CMSSW_14_0_0_pre3-el8_ppc64le_gcc12/build/CMSSW_14_0_0_pre3-build/BUILD/el8_ppc64le_gcc12/external/tbb/v2021.9.0-16196f107c456d12eb9d248792d8cb7b/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#11 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /scratch/cmsbuild/jenkins_b/workspace/auto-builds/CMSSW_14_0_0_pre3-el8_ppc64le_gcc12/build/CMSSW_14_0_0_pre3-build/BUILD/el8_ppc64le_gcc12/external/tbb/v2021.9.0-16196f107c456d12eb9d248792d8cb7b/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#12 0x000010089040208c in tbb::detail::d1::task_arena_function<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#13 0x00001000025aa838 in operator() (__closure=<optimized out>) at /scratch/cmsbuild/jenkins_b/workspace/auto-builds/CMSSW_14_0_0_pre3-el8_ppc64le_gcc12/build/CMSSW_14_0_0_pre3-build/BUILD/el8_ppc64le_gcc12/external/tbb/v2021.9.0-16196f107c456d12eb9d248792d8cb7b/tbb-v2021.9.0/src/tbb/arena.cpp:757
#14 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /scratch/cmsbuild/jenkins_b/workspace/auto-builds/CMSSW_14_0_0_pre3-el8_ppc64le_gcc12/build/CMSSW_14_0_0_pre3-build/BUILD/el8_ppc64le_gcc12/external/tbb/v2021.9.0-16196f107c456d12eb9d248792d8cb7b/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#15 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /scratch/cmsbuild/jenkins_b/workspace/auto-builds/CMSSW_14_0_0_pre3-el8_ppc64le_gcc12/build/CMSSW_14_0_0_pre3-build/BUILD/el8_ppc64le_gcc12/external/tbb/v2021.9.0-16196f107c456d12eb9d248792d8cb7b/tbb-v2021.9.0/src/tbb/arena.cpp:758
#16 0x00001008903ffdfc in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#17 0x000010089042c660 in HGCalLayerClusterProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#18 0x000010000044c264 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
Thread 3 (Thread 0x10085c8191f0 (LWP 118783) "cmsRun"):
#2 0x000010000ab85a3c in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000010000037ba70 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#5 0x000010000037ba54 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#6 0x000010000037ba54 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#7 0x000010000037ba54 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#8 0x000010000037ba54 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#9 0x000010000037ba54 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#10 0x000010000037ba54 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#11 0x000010000037ba54 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#12 0x000010000037ba54 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#13 0x000010000037ba54 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#14 0x000010000037ba54 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#15 0x000010000037ba54 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#16 0x000010000037ba54 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_alternative() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#17 0x000010000037bd90 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_M_disjunction() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#18 0x000010000037c2b0 in std::__detail::_Compiler<std::__cxx11::regex_traits<char> >::_Compiler(char const*, char const*, std::locale const&, std::regex_constants::syntax_option_type) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
#19 0x000010082d87dba8 in dd4hep::dd::isMatch(std::basic_string_view<char, std::char_traits<char> >, std::basic_string_view<char, std::char_traits<char> >) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/external/el8_ppc64le_gcc12/lib/libDDCore.so.1.27
#20 0x000010082d35b430 in auto cms::DDFilteredView::history()::{lambda(auto:1 const&)#1}::operator()<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > >(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libDetectorDescriptionDDCMS.so
#21 0x000010082d35dde4 in cms::DDFilteredView::history() () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libDetectorDescriptionDDCMS.so
#22 0x0000100835438464 in DTGeometryBuilderFromDD4hep::buildLayer(cms::DDFilteredView&, DTSuperLayer*, MuonGeometryConstants const&) const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libGeometryDTGeometryBuilder.so
#23 0x0000100835438bc4 in DTGeometryBuilderFromDD4hep::buildGeometry(cms::DDFilteredView&, DTGeometry&, MuonGeometryConstants const&) const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libGeometryDTGeometryBuilder.so
#24 0x0000100835438d34 in DTGeometryBuilderFromDD4hep::build(DTGeometry&, cms::DDDetector const*, MuonGeometryConstants const&, std::vector<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, dd4hep::SpecPar const*>, std::allocator<std::pair<std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >, dd4hep::SpecPar const*> > > const&) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libGeometryDTGeometryBuilder.so
#25 0x00001008353d9190 in DTGeometryESModule::setupDD4hepGeometry(MuonNumberingRecord const&, std::shared_ptr<edm::ESProductHost<DTGeometry, MuonNumberingRecord, DTRecoGeometryRcd> >&) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginDTGeometryPlugins.so
#26 0x00001008353dcb74 in DTGeometryESModule::produce(MuonGeometryRecord const&) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginDTGeometryPlugins.so
#27 0x00001008353f3f08 in edm::eventsetup::CallbackBase<edm::ESProducer, edm::ESProducer::setWhatProduced<DTGeometryESModule, std::shared_ptr<DTGeometry>, MuonGeometryRecord, edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> >(DTGeometryESModule*, std::shared_ptr<DTGeometry> (DTGeometryESModule::*)(MuonGeometryRecord const&), edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> const&, edm::es::Label const&)::{lambda(MuonGeometryRecord const&)#1}, std::shared_ptr<DTGeometry>, MuonGeometryRecord, edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> >::makeProduceTask<edm::eventsetup::Callback<edm::ESProducer, edm::ESProducer::setWhatProduced<DTGeometryESModule, std::shared_ptr<DTGeometry>, MuonGeometryRecord, edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> >(DTGeometryESModule*, std::shared_ptr<DTGeometry> (DTGeometryESModule::*)(MuonGeometryRecord const&), edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> const&, edm::es::Label const&)::{lambda(MuonGeometryRecord const&)#1}, std::shared_ptr<DTGeometry>, MuonGeometryRecord, edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> >::prefetchAsync(edm::WaitingTaskHolder, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, edm::ServiceToken const&, edm::ESParentContext const&)::{lambda(auto:1&&, auto:2&&, auto:3&&, auto:4&&)#1}::operator()<tbb::detail::d1::task_group*&, edm::ServiceWeakToken&, edm::eventsetup::EventSetupRecordImpl const*&, edm::EventSetupImpl const*&>(tbb::detail::d1::task_group*&, edm::ServiceWeakToken&, edm::eventsetup::EventSetupRecordImpl const*&, edm::EventSetupImpl const*&) const::{lambda(MuonGeometryRecord const&)#1}>(tbb::detail::d1::task_group*, edm::ServiceWeakToken const&, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, bool, tbb::detail::d1::task_group*&)::{lambda(std::__exception_ptr::exception_ptr const*)#1}::operator()(std::__exception_ptr::exception_ptr const*) const::{lambda()#2}::operator()() const::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginDTGeometryPlugins.so
#28 0x00001008353f54b0 in edm::convertException::wrap<edm::eventsetup::CallbackBase<edm::ESProducer, edm::ESProducer::setWhatProduced<DTGeometryESModule, std::shared_ptr<DTGeometry>, MuonGeometryRecord, edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> >(DTGeometryESModule*, std::shared_ptr<DTGeometry> (DTGeometryESModule::*)(MuonGeometryRecord const&), edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> const&, edm::es::Label const&)::{lambda(MuonGeometryRecord const&)#1}, std::shared_ptr<DTGeometry>, MuonGeometryRecord, edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> >::makeProduceTask<edm::eventsetup::Callback<edm::ESProducer, edm::ESProducer::setWhatProduced<DTGeometryESModule, std::shared_ptr<DTGeometry>, MuonGeometryRecord, edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> >(DTGeometryESModule*, std::shared_ptr<DTGeometry> (DTGeometryESModule::*)(MuonGeometryRecord const&), edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> const&, edm::es::Label const&)::{lambda(MuonGeometryRecord const&)#1}, std::shared_ptr<DTGeometry>, MuonGeometryRecord, edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> >::prefetchAsync(edm::WaitingTaskHolder, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, edm::ServiceToken const&, edm::ESParentContext const&)::{lambda(auto:1&&, auto:2&&, auto:3&&, auto:4&&)#1}::operator()<tbb::detail::d1::task_group*&, edm::ServiceWeakToken&, edm::eventsetup::EventSetupRecordImpl const*&, edm::EventSetupImpl const*&>(tbb::detail::d1::task_group*&, edm::ServiceWeakToken&, edm::eventsetup::EventSetupRecordImpl const*&, edm::EventSetupImpl const*&) const::{lambda(MuonGeometryRecord const&)#1}>(tbb::detail::d1::task_group*, edm::ServiceWeakToken const&, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, bool, tbb::detail::d1::task_group*&)::{lambda(std::__exception_ptr::exception_ptr const*)#1}::operator()(std::__exception_ptr::exception_ptr const*) const::{lambda()#2}::operator()() const::{lambda()#1}> () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginDTGeometryPlugins.so
#29 0x00001008353f5648 in edm::eventsetup::CallbackBase<edm::ESProducer, edm::ESProducer::setWhatProduced<DTGeometryESModule, std::shared_ptr<DTGeometry>, MuonGeometryRecord, edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> >(DTGeometryESModule*, std::shared_ptr<DTGeometry> (DTGeometryESModule::*)(MuonGeometryRecord const&), edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> const&, edm::es::Label const&)::{lambda(MuonGeometryRecord const&)#1}, std::shared_ptr<DTGeometry>, MuonGeometryRecord, edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> >::makeProduceTask<edm::eventsetup::Callback<edm::ESProducer, edm::ESProducer::setWhatProduced<DTGeometryESModule, std::shared_ptr<DTGeometry>, MuonGeometryRecord, edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> >(DTGeometryESModule*, std::shared_ptr<DTGeometry> (DTGeometryESModule::*)(MuonGeometryRecord const&), edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> const&, edm::es::Label const&)::{lambda(MuonGeometryRecord const&)#1}, std::shared_ptr<DTGeometry>, MuonGeometryRecord, edm::eventsetup::CallbackSimpleDecorator<MuonGeometryRecord> >::prefetchAsync(edm::WaitingTaskHolder, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, edm::ServiceToken const&, edm::ESParentContext const&)::{lambda(auto:1&&, auto:2&&, auto:3&&, auto:4&&)#1}::operator()<tbb::detail::d1::task_group*&, edm::ServiceWeakToken&, edm::eventsetup::EventSetupRecordImpl const*&, edm::EventSetupImpl const*&>(tbb::detail::d1::task_group*&, edm::ServiceWeakToken&, edm::eventsetup::EventSetupRecordImpl const*&, edm::EventSetupImpl const*&) const::{lambda(MuonGeometryRecord const&)#1}>(tbb::detail::d1::task_group*, edm::ServiceWeakToken const&, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, bool, tbb::detail::d1::task_group*&)::{lambda(std::__exception_ptr::exception_ptr const*)#1}::operator()(std::__exception_ptr::exception_ptr const*) const::{lambda()#2}::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginDTGeometryPlugins.so
Thread 1 (Thread 0x1000027ccfe0 (LWP 117524) "cmsRun"):
#2 0x000010000ab85a3c in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x000010000293b400 in std::locale::id::_M_id (this=0x100002b15808 <std::__gnu_cxx_ldbl128::num_put<char, std::ostreambuf_iterator<char, std::char_traits<char> > >::id>) at ../../../../../libstdc++-v3/src/c++98/locale.cc:505
#5 0x00001000029d2a28 in std::use_facet<std::__gnu_cxx_ldbl128::num_put<char, std::ostreambuf_iterator<char, std::char_traits<char> > > > (__loc=...) at /scratch/cmsbuild/jenkins_a/workspace/jenkins-test-bootstrap/bootstrap/BUILD/el8_ppc64le_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/gcc-12.3.1/obj/powerpc64le-redhat-linux-gnu/libstdc++-v3/include/bits/locale_classes.tcc:145
#6 0x00001000029c38d4 in std::basic_ios<char, std::char_traits<char> >::_M_cache_locale (this=0x3fffeb98abb0, __loc=...) at /scratch/cmsbuild/jenkins_a/workspace/jenkins-test-bootstrap/bootstrap/BUILD/el8_ppc64le_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/gcc-12.3.1/obj/powerpc64le-redhat-linux-gnu/libstdc++-v3/include/bits/basic_ios.tcc:165
#7 0x00001000029c3b08 in std::basic_ios<char, std::char_traits<char> >::init (this=0x3fffeb98abb0, __sb=0x0) at /scratch/cmsbuild/jenkins_a/workspace/jenkins-test-bootstrap/bootstrap/BUILD/el8_ppc64le_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/gcc-12.3.1/obj/powerpc64le-redhat-linux-gnu/libstdc++-v3/include/bits/basic_ios.tcc:132
#8 0x00001008321228e0 in HGCalDDDConstants::locateCell(int, int, int, int, int, int, bool, bool, bool, bool) const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libGeometryHGCalCommonData.so
#9 0x0000100831cbcb8c in HGCalGeometry::getPosition(DetId const&, bool) const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libGeometryHGCalGeometry.so
#10 0x0000100863ceb4a0 in hgcal::RecHitTools::getPosition(DetId const&) const () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libRecoLocalCaloHGCalRecAlgos.so
#11 0x000010089042b420 in HGCalLayerClusterProducer::calculatePosition(std::unordered_map<unsigned int, HGCRecHit const*, std::hash<unsigned int>, std::equal_to<unsigned int>, std::allocator<std::pair<unsigned int const, HGCRecHit const*> > >&, std::vector<std::pair<DetId, float>, std::allocator<std::pair<DetId, float> > > const&) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#12 0x000010089042c8c0 in HGCalLayerClusterProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#13 0x000010000044c264 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/ppc64le/nweek-02825/el8_ppc64le_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-23-2300/lib/el8_ppc64le_gcc12/libFWCoreFramework.so
Current Modules:
Module: HGCalLayerClusterProducer:hgcalLayerClustersHSci (crashed)
Module: HGCalLayerClusterProducer:hgcalLayerClustersEE
Module: none
Module: HGCalConcentratorProducer:l1tHGCalConcentratorProducer
https://cmssdt.cern.ch/SDT/cgi-bin/logreader/el8_ppc64le_gcc12/CMSSW_14_1_X_2024-02-23-2300/pyRelValMatrixLogs/run/25234.911_TTbar_14TeV+2026D99_DD4hep/step2_TTbar_14TeV+2026D99_DD4hep.log#/
Now also a floating point exception in CMSSW_14_1_NONLTO_X_2024-03-24-0000 (although I'm puzzled what enabled those)
A fatal system signal has occurred: floating point exception
The following is the call stack containing the origin of the signal.
Thread 5 (Thread 0x1553627ff700 (LWP 2675198) "cmsRun"):
#2 0x00001553b11f3f80 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x00001552e83e893a in HGCFEElectronics<HGCDataFrame<DetId, HGCSample> >::runShaperWithToT(HGCDataFrame<DetId, HGCSample>&, std::array<float, 15ul>&, std::array<float, 15ul>&, CLHEP::HepRandomEngine*, unsigned int, float, unsigned int, float, int, float, float, std::array<float, 6ul> const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libSimCalorimetryHGCalSimProducers.so
#5 0x00001552e84238a4 in HGCDigitizerBase::runSimple(std::unique_ptr<edm::SortedCollection<HGCDataFrame<DetId, HGCSample>, edm::StrictWeakOrdering<HGCDataFrame<DetId, HGCSample> > >, std::default_delete<edm::SortedCollection<HGCDataFrame<DetId, HGCSample>, edm::StrictWeakOrdering<HGCDataFrame<DetId, HGCSample> > > > >&, std::unordered_map<unsigned int, hgc_digi::HGCCellInfo, std::hash<unsigned int>, std::equal_to<unsigned int>, std::allocator<std::pair<unsigned int const, hgc_digi::HGCCellInfo> > >&, CaloSubdetectorGeometry const*, std::unordered_set<DetId, std::hash<DetId>, std::equal_to<DetId>, std::allocator<DetId> > const&, CLHEP::HepRandomEngine*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimCalorimetryHGCalSimProducersAuto.so
#6 0x00001552e8415e78 in HGCDigitizer::finalizeEvent(edm::Event&, edm::EventSetup const&, CLHEP::HepRandomEngine*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimCalorimetryHGCalSimProducersAuto.so
#7 0x00001552e8408c84 in HGCDigiProducer::finalizeEvent(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimCalorimetryHGCalSimProducersAuto.so
#8 0x00001552ee76385b in edm::MixingModule::finalizeEvent(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimGeneralMixingModulePlugins.so
#9 0x00001552ee6a89eb in edm::BMixingModule::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libMixingBase.so
#10 0x00001553baf1aab2 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00001553baefc0d6 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#12 0x00001553bae4d4a5 in decltype ({parm#1}()) edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#13 0x00001553bae4d6f5 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#14 0x00001553bae4fde9 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
Thread 4 (Thread 0x155363634700 (LWP 2675197) "cmsRun"):
#3 0x00001553b11f8178 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x000015535175f6ac in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#6 0x0000155351765734 in void tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#7 0x0000155351765bb9 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#8 0x00001553b94db281 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x1553b6bc9500) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-2bdfc55ee8e4ee32defbb06c66a16b5f/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#9 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x1553b6bc9500) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-2bdfc55ee8e4ee32defbb06c66a16b5f/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#10 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-2bdfc55ee8e4ee32defbb06c66a16b5f/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#11 0x00001553517566fb in tbb::detail::d1::task_arena_function<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#12 0x00001553b94c828e in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-2bdfc55ee8e4ee32defbb06c66a16b5f/tbb-v2021.9.0/src/tbb/arena.cpp:757
#13 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-2bdfc55ee8e4ee32defbb06c66a16b5f/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#14 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-2bdfc55ee8e4ee32defbb06c66a16b5f/tbb-v2021.9.0/src/tbb/arena.cpp:758
#15 0x00001553517552a8 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#16 0x00001553517791c7 in HGCalLayerClusterProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#17 0x00001553baf1aab2 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#18 0x00001553baefc0d6 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
Thread 3 (Thread 0x155364035700 (LWP 2675196) "cmsRun"):
#2 0x00001553b11f3f80 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x00001552e8422d15 in HGCDigitizerBase::updateOutput(std::unique_ptr<edm::SortedCollection<HGCDataFrame<DetId, HGCSample>, edm::StrictWeakOrdering<HGCDataFrame<DetId, HGCSample> > >, std::default_delete<edm::SortedCollection<HGCDataFrame<DetId, HGCSample>, edm::StrictWeakOrdering<HGCDataFrame<DetId, HGCSample> > > > >&, HGCDataFrame<DetId, HGCSample> const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimCalorimetryHGCalSimProducersAuto.so
#5 0x00001552e84289f3 in HGCHEbackDigitizer::runRealisticDigitizer(std::unique_ptr<edm::SortedCollection<HGCDataFrame<DetId, HGCSample>, edm::StrictWeakOrdering<HGCDataFrame<DetId, HGCSample> > >, std::default_delete<edm::SortedCollection<HGCDataFrame<DetId, HGCSample>, edm::StrictWeakOrdering<HGCDataFrame<DetId, HGCSample> > > > >&, std::unordered_map<unsigned int, hgc_digi::HGCCellInfo, std::hash<unsigned int>, std::equal_to<unsigned int>, std::allocator<std::pair<unsigned int const, hgc_digi::HGCCellInfo> > >&, CaloSubdetectorGeometry const*, std::unordered_set<DetId, std::hash<DetId>, std::equal_to<DetId>, std::allocator<DetId> > const&, CLHEP::HepRandomEngine*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimCalorimetryHGCalSimProducersAuto.so
#6 0x00001552e8415e78 in HGCDigitizer::finalizeEvent(edm::Event&, edm::EventSetup const&, CLHEP::HepRandomEngine*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimCalorimetryHGCalSimProducersAuto.so
#7 0x00001552e8408c84 in HGCDigiProducer::finalizeEvent(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimCalorimetryHGCalSimProducersAuto.so
#8 0x00001552ee76385b in edm::MixingModule::finalizeEvent(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimGeneralMixingModulePlugins.so
#9 0x00001552ee6a89eb in edm::BMixingModule::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libMixingBase.so
#10 0x00001553baf1aab2 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00001553baefc0d6 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
Thread 1 (Thread 0x1553b779f680 (LWP 2675122) "cmsRun"):
#2 0x00001553b11f3f80 in sig_pause_for_stacktrace () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x00001552e83e850b in HGCFEElectronics<HGCDataFrame<DetId, HGCSample> >::runShaperWithToT(HGCDataFrame<DetId, HGCSample>&, std::array<float, 15ul>&, std::array<float, 15ul>&, CLHEP::HepRandomEngine*, unsigned int, float, unsigned int, float, int, float, float, std::array<float, 6ul> const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libSimCalorimetryHGCalSimProducers.so
#5 0x00001552e84238a4 in HGCDigitizerBase::runSimple(std::unique_ptr<edm::SortedCollection<HGCDataFrame<DetId, HGCSample>, edm::StrictWeakOrdering<HGCDataFrame<DetId, HGCSample> > >, std::default_delete<edm::SortedCollection<HGCDataFrame<DetId, HGCSample>, edm::StrictWeakOrdering<HGCDataFrame<DetId, HGCSample> > > > >&, std::unordered_map<unsigned int, hgc_digi::HGCCellInfo, std::hash<unsigned int>, std::equal_to<unsigned int>, std::allocator<std::pair<unsigned int const, hgc_digi::HGCCellInfo> > >&, CaloSubdetectorGeometry const*, std::unordered_set<DetId, std::hash<DetId>, std::equal_to<DetId>, std::allocator<DetId> > const&, CLHEP::HepRandomEngine*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimCalorimetryHGCalSimProducersAuto.so
#6 0x00001552e8415e78 in HGCDigitizer::finalizeEvent(edm::Event&, edm::EventSetup const&, CLHEP::HepRandomEngine*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimCalorimetryHGCalSimProducersAuto.so
#7 0x00001552e8408c84 in HGCDigiProducer::finalizeEvent(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimCalorimetryHGCalSimProducersAuto.so
#8 0x00001552ee76385b in edm::MixingModule::finalizeEvent(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/pluginSimGeneralMixingModulePlugins.so
#9 0x00001552ee6a89eb in edm::BMixingModule::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libMixingBase.so
#10 0x00001553baf1aab2 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
#11 0x00001553baefc0d6 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_NONLTO_X_2024-03-24-0000/lib/el8_amd64_gcc12/libFWCoreFramework.so
Current Modules:
Module: HGCalLayerClusterProducer:hgcalLayerClustersHSci (crashed)
Module: MixingModule:mix
Module: MixingModule:mix
Module: MixingModule:mix
https://cmssdt.cern.ch/SDT/cgi-bin/logreader/el8_amd64_gcc12/CMSSW_14_1_NONLTO_X_2024-03-24-0000/pyRelValMatrixLogs/run/25234.911_TTbar_14TeV+2026D99_DD4hep/step2_TTbar_14TeV+2026D99_DD4hep.log#/
Now also a floating point exception in CMSSW_14_1_NONLTO_X_2024-03-24-0000 (although I'm puzzled what enabled those)
we have seen external packages messing with the FPE state, that's why we added it to the signals we trap in #39474
Now also a floating point exception in CMSSW_14_1_NONLTO_X_2024-03-24-0000 (although I'm puzzled what enabled those)
we have seen external packages messing with the FPE state, that's why we added it to the signals we trap in #39474
Thanks Dan.
Occured in CMSSW_14_1_X_2024-05-05-2300 for slc7_amd64_gcc12:
Thread 3 (Thread 0x14f25be42700 (LWP 2227974) "cmsRun"):
#0 0x000014f2addc7ddd in poll () from /lib64/libc.so.6
#1 0x000014f2a5eb111f in full_read.constprop () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02836/slc7_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-05-0000/lib/slc7_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x000014f2a5e655dc in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02836/slc7_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-05-0000/lib/slc7_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 0x000014f2a5e65f40 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02836/slc7_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-05-0000/lib/slc7_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x000014f2af36f11f in cache_bin_alloc_impl (adjust_low_water=false, success=<synthetic pointer>, bin=<optimized out>) at include/jemalloc/internal/cache_bin.h:361
#6 cache_bin_alloc_easy (success=<synthetic pointer>, bin=<optimized out>) at include/jemalloc/internal/cache_bin.h:399
#7 imalloc_fastpath (fallback_alloc=0x14f2af36f080 <fallback_impl<false>(std::size_t)>, size=64) at include/jemalloc/internal/jemalloc_internal_inlines_c.h:326
#8 newImpl<false> (size=64) at src/jemalloc_cpp.cpp:109
#9 operator new (size=64) at src/jemalloc_cpp.cpp:114
#10 0x000014f24afce8d1 in void std::vector<std::pair<DetId, float>, std::allocator<std::pair<DetId, float> > >::_M_realloc_insert<DetId&, float>(__gnu_cxx::__normal_iterator<std::pair<DetId, float>*, std::vector<std::pair<DetId, float>, std::allocator<std::pair<DetId, float> > > >, DetId&, float&&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02836/slc7_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-05-0000/lib/slc7_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#11 0x000014f24afc9e16 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::getClusters(bool) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02836/slc7_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-05-0000/lib/slc7_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#12 0x000014f24afe0b78 in HGCalLayerClusterProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02836/slc7_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-05-0000/lib/slc7_amd64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#13 0x000014f2b07cd995 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02836/slc7_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-05-0000/lib/slc7_amd64_gcc12/libFWCoreFramework.so
#14 0x000014f2b07b4a8e in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02836/slc7_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-05-0000/lib/slc7_amd64_gcc12/libFWCoreFramework.so
#15 0x000014f2b07406d9 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02836/slc7_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-05-0000/lib/slc7_amd64_gcc12/libFWCoreFramework.so
#16 0x000014f2b0742d78 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02836/slc7_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-05-0000/lib/slc7_amd64_gcc12/libFWCoreFramework.so
#17 0x000014f2b08e6f28 in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02836/slc7_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-05-05-0000/lib/slc7_amd64_gcc12/libFWCoreConcurrency.so
#18 0x000014f2aeef596b in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x14f231eb7b00, waiter=..., this=0x14f2ab3c9500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre3-slc7_amd64_gcc12/build/CMSSW_14_1_0_pre3-build/BUILD/slc7_amd64_gcc12/external/tbb/v2021.9.0-5849be8e21b090e14f1b189539cee138/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#19 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (t=0x0, waiter=..., this=0x14f2ab3c9500) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre3-slc7_amd64_gcc12/build/CMSSW_14_1_0_pre3-build/BUILD/slc7_amd64_gcc12/external/tbb/v2021.9.0-5849be8e21b090e14f1b189539cee138/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#20 tbb::detail::r1::arena::process (tls=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre3-slc7_amd64_gcc12/build/CMSSW_14_1_0_pre3-build/BUILD/slc7_amd64_gcc12/external/tbb/v2021.9.0-5849be8e21b090e14f1b189539cee138/tbb-v2021.9.0/src/tbb/arena.cpp:137
#21 tbb::detail::r1::market::process (this=<optimized out>, j=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre3-slc7_amd64_gcc12/build/CMSSW_14_1_0_pre3-build/BUILD/slc7_amd64_gcc12/external/tbb/v2021.9.0-5849be8e21b090e14f1b189539cee138/tbb-v2021.9.0/src/tbb/market.cpp:599
#22 0x000014f2aeef7b1e in tbb::detail::r1::rml::private_worker::run (this=0x14f2a8c39080) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre3-slc7_amd64_gcc12/build/CMSSW_14_1_0_pre3-build/BUILD/slc7_amd64_gcc12/external/tbb/v2021.9.0-5849be8e21b090e14f1b189539cee138/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#23 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x14f2a8c39080) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_1_0_pre3-slc7_amd64_gcc12/build/CMSSW_14_1_0_pre3-build/BUILD/slc7_amd64_gcc12/external/tbb/v2021.9.0-5849be8e21b090e14f1b189539cee138/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#24 0x000014f2ae0a9ea5 in start_thread () from /lib64/libpthread.so.0
#25 0x000014f2addd2b0d in clone () from /lib64/libc.so.6
Occured in CMSSW_14_1_DBG_X_2024-05-09-2300 for el8_amd64_gcc12:
Thread 1 (Thread 0x155187ae2640 (LWP 2067337) "cmsRun"):
#0 0x00001551886af301 in poll () from /lib64/libc.so.6
#1 0x0000155181898457 in (anonymous namespace)::full_read (len=1, timeout_s=300, inbuf=0x7ffd183c3c46 "", fd=7) at src/FWCore/Services/plugins/InitRootHandlers.cc:382
#2 edm::service::InitRootHandlers::stacktraceFromThread () at src/FWCore/Services/plugins/InitRootHandlers.cc:702
#3 0x0000155181898654 in (anonymous namespace)::sig_dostack_then_abort (sig=11) at src/FWCore/Services/plugins/InitRootHandlers.cc:548
#4 <signal handler called>
#5 HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}::operator()(unsigned long) const () at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/include/c++/12.3.1/bits/new_allocator.h:173
#6 0x000015512506fb4b in tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>::operator()(tbb::detail::d1::blocked_range<unsigned long> const&) const (r=..., this=0x15510be7ae58) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/parallel_for.h:211
#7 tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>::run_body(tbb::detail::d1::blocked_range<unsigned long>&) (r=..., this=0x15510be7ae00) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/parallel_for.h:122
#8 tbb::detail::d1::dynamic_grainsize_mode<tbb::detail::d1::adaptive_mode<tbb::detail::d1::auto_partition_type> >::work_balance<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) (ed=..., range=..., start=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/partitioner.h:435
#9 tbb::detail::d1::partition_type_base<tbb::detail::d1::auto_partition_type>::execute<tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>, tbb::detail::d1::blocked_range<unsigned long> >(tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>&, tbb::detail::d1::blocked_range<unsigned long>&, tbb::detail::d1::execution_data&) (ed=..., range=..., start=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/partitioner.h:289
#10 tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) (this=0x15510be7ae00, ed=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/parallel_for.h:175
#11 0x0000155189823281 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x155186fc9380) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#12 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x155186fc9380) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#13 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#14 0x0000155125070227 in tbb::detail::d1::task_arena_function<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}, void>::operator()() const (this=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/detail/_task.h:191
#15 0x000015518981028e in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/tbb-v2021.9.0/src/tbb/arena.cpp:757
#16 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#17 tbb::detail::r1::isolate_within_arena (d=..., isolation=isolation@entry=0) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/tbb-v2021.9.0/src/tbb/arena.cpp:758
#18 0x0000155125057272 in tbb::detail::d1::isolate_impl<void, HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}>(HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}&) (f=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/task_arena.h:203
#19 tbb::detail::d1::isolate<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}>(HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}&&) (f=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/task_arena.h:445
#20 HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters (this=<optimized out>) at src/RecoLocalCalo/HGCalRecProducers/plugins/HGCalCLUEAlgo.cc:100
#21 0x000015512507f2e5 in HGCalLayerClusterProducer::produce (this=0x1551275ed000, evt=..., es=...) at src/RecoLocalCalo/HGCalRecProducers/plugins/HGCalLayerClusterProducer.cc:248
#22 0x000015518b123c43 in edm::stream::EDProducerAdaptorBase::doEvent (this=0x1551275c8fc0, info=..., act=<optimized out>, mcc=0x1550857282e8) at src/FWCore/Framework/src/stream/EDProducerAdaptorBase.cc:82
#23 0x000015518b1018dc in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo (this=0x1550857282c0, info=..., mcc=0x1550857282e8) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/gcc/12.3.1-40d504be6370b5a30e3947a6e575ca28/include/c++/12.3.1/bits/shared_ptr_base.h:1665
#24 0x000015518b08a53a in edm::workerhelper::CallImpl<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::call (mcc=0x1550857282e8, info=..., iWorker=0x1550857282c0) at src/FWCore/Framework/interface/maker/Worker.h:696
#25 edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}::operator()() const (__closure=<synthetic pointer>) at src/FWCore/Framework/interface/maker/Worker.h:1251
#26 edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) (iFunc=...) at src/FWCore/Utilities/interface/ConvertException.h:21
#27 edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> > (context=0x1550856a9a90, parentContext=..., streamID=..., transitionInfo=..., this=0x1550857282c0) at src/FWCore/Framework/interface/maker/Worker.h:1250
#28 edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> > (this=0x1550857282c0, iEPtr=..., transitionInfo=..., streamID=..., parentContext=..., context=0x1550856a9a90) at src/FWCore/Framework/interface/maker/Worker.h:1164
#29 0x000015518b08becd in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute (this=0x15500eb9b510) at src/FWCore/Framework/interface/maker/Worker.h:495
#30 0x000015518adc4138 in operator() (__closure=0x15510be7b840) at src/FWCore/Concurrency/src/WaitingTaskList.cc:206
#31 tbb::detail::d2::(anonymous namespace)::task_ptr_or_nullptr_impl<const edm::WaitingTaskList::announce()::<lambda()>&> (f=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/task_group.h:115
#32 tbb::detail::d2::(anonymous namespace)::task_ptr_or_nullptr<const edm::WaitingTaskList::announce()::<lambda()>&> (f=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/task_group.h:125
#33 tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::<lambda()> >::execute(tbb::detail::d1::execution_data &) (this=0x15510be7b800, ed=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/task_group.h:452
#34 0x0000155189823281 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x155186fc9380) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#35 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x155186fc9380) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#36 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#37 0x000015518b0091ab in tbb::detail::d1::wait (ctx=..., wait_ctx=...) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/detail/_task.h:197
#38 tbb::detail::d1::task_group_base::wait()::{lambda()#1}::operator()() const (__closure=<optimized out>) at /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/external/tbb/v2021.9.0-a7089dd5ec356e9a0bc222e109b15cef/include/oneapi/tbb/task_group.h:582
#39 tbb::detail::d0::try_call_proxy<tbb::detail::d1::task_group_base::wait()::{lambda()#1}>::on_completion<tbb::detail::d1::task_group_base::wait()::{lambda()#2}>(tbb::detail::d1::task_group_b
(stacktrace is incomplete: Waiting for stacktrace completion failed: timed out waiting for GDB to complete)
Occured in CMSSW_14_1_ROOT6_X_2024-06-05-2300 for el8_aarch64_gcc12:
Thread 4 (Thread 0x40005fe39220 (LWP 3393752) "cmsRun"):
#0 0x00004000150ea954 in poll () from /lib64/libc.so.6
#1 0x000040001bb0f700 in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02840/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-06-03-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x000040001bb0f934 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02840/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-06-03-2300/lib/el8_aarch64_gcc12/pluginFWCoreServicesPlugins.so
#3 <signal handler called>
#4 0x00004000ad20b3b4 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}::operator()(unsigned long) const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02840/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_ROOT6_X_2024-06-05-2300/lib/el8_aarch64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#5 0x00004000ad20b734 in tbb::detail::d1::start_for<tbb::detail::d1::blocked_range<unsigned long>, tbb::detail::d1::parallel_for_body_wrapper<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}::operator()() const::{lambda(unsigned long)#1}, unsigned long>, tbb::detail::d1::auto_partitioner const>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02840/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_ROOT6_X_2024-06-05-2300/lib/el8_aarch64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#6 0x0000400014a58774 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=0x4000eb741100, this=0x4000159eb380) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#7 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x4000159eb380) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#8 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#9 0x00004000ad209f40 in tbb::detail::d1::task_arena_function<HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters()::{lambda()#1}, void>::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02840/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_ROOT6_X_2024-06-05-2300/lib/el8_aarch64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#10 0x0000400014a51724 in operator() (__closure=<optimized out>) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/arena.cpp:757
#11 tbb::detail::d0::try_call_proxy<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> >::on_completion<tbb::detail::r1::isolate_within_arena(tbb::detail::d1::delegate_base&, intptr_t)::<lambda()> > (on_completion_body=..., this=<optimized out>) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/../../include/oneapi/tbb/detail/_template_helpers.h:230
#12 tbb::detail::r1::isolate_within_arena (d=..., isolation=<optimized out>) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/arena.cpp:758
#13 0x00004000ad1f9ff0 in HGCalCLUEAlgoT<HGCalLayerTilesT<HGCalScintillatorTilesConstants, PhiWrapper>, HGCalScintillatorStrategy>::makeClusters() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02840/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_ROOT6_X_2024-06-05-2300/lib/el8_aarch64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#14 0x00004000ad21b79c in HGCalLayerClusterProducer::produce(edm::Event&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02840/el8_aarch64_gcc12/cms/cmssw-patch/CMSSW_14_1_ROOT6_X_2024-06-05-2300/lib/el8_aarch64_gcc12/pluginRecoLocalCaloHGCalRecProducersPlugins.so
#15 0x0000400012fd8754 in edm::stream::EDProducerAdaptorBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02840/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-06-03-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#16 0x0000400012fb5570 in edm::WorkerT<edm::stream::EDProducerAdaptorBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02840/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-06-03-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#17 0x0000400012f40e08 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02840/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-06-03-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#18 0x0000400012f42b04 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute() () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02840/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-06-03-2300/lib/el8_aarch64_gcc12/libFWCoreFramework.so
#19 0x000040001350527c in tbb::detail::d1::function_task<edm::WaitingTaskList::announce()::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/aarch64/nweek-02840/el8_aarch64_gcc12/cms/cmssw/CMSSW_14_1_ROOT6_X_2024-06-03-2300/lib/el8_aarch64_gcc12/libFWCoreConcurrency.so
#20 0x0000400014a52dfc in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::outermost_worker_waiter> (t=0x4000eb740d00, waiter=<synthetic pointer>..., this=0x4000159eb380) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#21 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::outermost_worker_waiter> (waiter=<synthetic pointer>..., t=0x0, this=0x4000159eb380) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#22 tbb::detail::r1::arena::process (tls=..., this=0x4000159ead00) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/arena.cpp:137
#23 tbb::detail::r1::market::process (this=0x4000159eb580, j=...) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/market.cpp:599
#24 0x0000400014a5af2c in tbb::detail::r1::rml::private_worker::run (this=0x40001ad28100) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/private_server.cpp:271
#25 tbb::detail::r1::rml::private_worker::thread_routine (arg=0x40001ad28100) at /data/cmsbld/jenkins_a/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_aarch64_gcc12/external/tbb/v2021.9.0-8fa0764296b216b253841e2f8e4e6b9c/tbb-v2021.9.0/src/tbb/private_server.cpp:221
#26 0x0000400014fe78b8 in start_thread () from /lib64/libpthread.so.0
#27 0x0000400015043afc in thread_start () from /lib64/libc.so.6
this should be fixed now by https://github.com/cms-sw/cmssw/pull/45178 Let us know if this is not the case.