cmssw
cmssw copied to clipboard
Published
20 hours ago •
cms-sw
cms-sw
Segfault in gbl::GblTrajectory::prepare()
[Find reproducible example below] Stack trace from CMSSW_14_0_0 caused by gbl::GblTrajectory::prepare(), executed on EL8 (el8_amd64_gcc12):
Thread 2 (Thread 0x145ebacb4640 (LWP 224) "cmsRun"):
#0 0x0000145ee311830f in wait4 () from /lib64/libc.so.6
#1 0x0000145ede9f6e17 in edm::service::cmssw_stacktrace_fork() () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x0000145ede9f9cea in edm::service::InitRootHandlers::stacktraceHelperThread() () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 0x0000145ee34d8a73 in std::execute_native_thread_routine (__p=0x145ed2eaf290) at ../../../../../libstdc++-v3/src/c++11/thread.cc:82
#4 0x0000145ee309f802 in start_thread () from /lib64/libc.so.6
#5 0x0000145ee303f450 in clone3 () from /lib64/libc.so.6
Thread 1 (Thread 0x145ee44bb640 (LWP 214) "cmsRun"):
#0 0x0000145ee31426ff in poll () from /lib64/libc.so.6
#1 0x0000145edea456af in full_read.constprop () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#2 0x0000145ede9f9dbc in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 0x0000145ede9fa720 in sig_dostack_then_abort () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 atomic_load_p (mo=atomic_memory_order_relaxed, a=0x846f8) at include/jemalloc/internal/atomic.h:83
#6 rtree_leaf_elm_bits_read (dependent=true, elm=0x846f8, rtree=<optimized out>, tsdn=0x145ee44b5738) at include/jemalloc/internal/rtree.h:182
#7 rtree_leaf_elm_read (dependent=true, elm=0x846f8, rtree=<optimized out>, tsdn=0x145ee44b5738) at include/jemalloc/internal/rtree.h:242
#8 rtree_metadata_read (rtree=<optimized out>, key=4611594290014846845, rtree_ctx=0x145ee44b58e8, tsdn=0x145ee44b5738) at include/jemalloc/internal/rtree.h:446
#9 emap_alloc_ctx_lookup (alloc_ctx=<synthetic pointer>, ptr=0x3fffac92d08dff7d, emap=<optimized out>, tsdn=0x145ee44b5738) at include/jemalloc/internal/emap.h:238
#10 ifree (slow_path=false, tcache=0x145ee44b5a90, ptr=0x3fffac92d08dff7d, tsd=0x145ee44b5738) at src/jemalloc.c:2877
#11 je_free_default (ptr=0x3fffac92d08dff7d) at src/jemalloc.c:3014
#12 0x0000145e89d48b8a in gbl::GblTrajectory::prepare() () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/external/el8_amd64_gcc12/lib/libGBL.so.2.4
#13 0x0000145e89d4b1c0 in gbl::GblTrajectory::construct() () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/external/el8_amd64_gcc12/lib/libGBL.so.2.4
#14 0x0000145e834451fe in MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) () from /afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/CMSSW_14_0_0/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so
#15 0x0000145e83445850 in MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) () from /afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/CMSSW_14_0_0/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so
#16 0x0000145e8a58947f in AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so
#17 0x0000145ee543da9e in edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/libFWCoreFramework.so
#18 0x0000145ee542820e in edm::WorkerT<edm::one::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/libFWCoreFramework.so
#19 0x0000145ee53ba669 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/libFWCoreFramework.so
#20 0x0000145ee53bb73f in edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/libFWCoreFramework.so
#21 0x0000145ee50e51d0 in tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#22 0x0000145ee45da281 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x145ee19b3e00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_0_0_pre2_SKYLAKEAVX512-el8_amd64_gcc12/build/CMSSW_14_0_0_pre2_SKYLAKEAVX512-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-5109e373398eaf79a8268d6a86a2f6e2/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#23 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x145ee19b3e00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_0_0_pre2_SKYLAKEAVX512-el8_amd64_gcc12/build/CMSSW_14_0_0_pre2_SKYLAKEAVX512-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-5109e373398eaf79a8268d6a86a2f6e2/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#24 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_0_0_pre2_SKYLAKEAVX512-el8_amd64_gcc12/build/CMSSW_14_0_0_pre2_SKYLAKEAVX512-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-5109e373398eaf79a8268d6a86a2f6e2/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#25 0x0000145ee533dceb in edm::FinalWaitingTask::wait() () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/libFWCoreFramework.so
#26 0x0000145ee534765a in edm::EventProcessor::processRuns() () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/libFWCoreFramework.so
#27 0x0000145ee5347bb1 in edm::EventProcessor::runToCompletion() () from /cvmfs/cms.cern.ch/el8_amd64_gcc12/cms/cmssw/CMSSW_14_0_0/lib/el8_amd64_gcc12/libFWCoreFramework.so
#28 0x00000000004074ef in tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const ()
#29 0x0000145ee45c69ad in tbb::detail::r1::task_arena_impl::execute (ta=..., d=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_0_0_pre2_SKYLAKEAVX512-el8_amd64_gcc12/build/CMSSW_14_0_0_pre2_SKYLAKEAVX512-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-5109e373398eaf79a8268d6a86a2f6e2/tbb-v2021.9.0/src/tbb/arena.cpp:688
#30 0x0000000000408ed2 in main::{lambda()#1}::operator()() const ()
#31 0x000000000040517c in main ()
Module: AlignmentProducerAsAnalyzer:AlignmentProducer (crashed)
######### TO REPRODUCE ######### cd /afs/cern.ch/cms/CAF/CMSALCA/ALCA_TRACKERALIGN/MP/MPproduction/CMSSW_14_0_0 cmsenv cd /afs/cern.ch/work/s/sroychow/public/TkDPG/segfaultmille/scripts cmsRun mille_failing.py
FYI: @henriettepetersen @sroychow
A new Issue was created by @TomasKello.
@rappoccio, @sextonkennedy, @makortel, @smuzaffar, @antoniovilela, @Dr15Jones can you please review it and eventually sign/assign? Thanks.
cms-bot commands are listed here
this is very reminiscent of https://github.com/cms-sw/cmssw/issues/43801. Have you checked if it persists in recent IBs ?
New categories assigned: alca
@saumyaphor4252,@perrotta,@consuegs you have been requested to review this Pull request/Issue and eventually sign? Thanks
Could you run the job in CMSSW_14_1_ASAN_X_2024-02-26-2300 and report here what ASAN reports?
(I'd also suggest to use the "code block" formatting for the output, i.e. start and end the block with three backuotes ```. I find the regular text formatting difficult to read for this kind of logs)
Here is report when running with CMSSW_14_1_ASAN_X_2024-02-26-2300
Using Global Tag: 140X_dataRun3_v1
# MSG-i trackselectionRefitting: g4Refitting=False
A fatal system signal has occurred: segmentation violation
The following is the call stack containing the origin of the signal.
Tue Feb 27 15:48:44 CET 2024
Thread 16 (Thread 0x7f94db2f6700 (LWP 2930428) "cmsRun"):
#0 0x00007f9535a1fda6 in do_futex_wait.constprop () from /lib64/libpthread.so.0
#1 0x00007f9535a1fe98 in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x00007f9530133706 in XrdCl::JobManager::RunJobs() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdCl.so.3
#3 0x00007f95301337b9 in RunRunnerThread () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdCl.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 15 (Thread 0x7f94dbaf7700 (LWP 2930427) "cmsRun"):
#0 0x00007f9535a1fda6 in do_futex_wait.constprop () from /lib64/libpthread.so.0
#1 0x00007f9535a1fe98 in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x00007f9530133706 in XrdCl::JobManager::RunJobs() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdCl.so.3
#3 0x00007f95301337b9 in RunRunnerThread () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdCl.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 14 (Thread 0x7f94dc2f8700 (LWP 2930426) "cmsRun"):
#0 0x00007f9535a1fda6 in do_futex_wait.constprop () from /lib64/libpthread.so.0
#1 0x00007f9535a1fe98 in __new_sem_wait_slow.constprop.0 () from /lib64/libpthread.so.0
#2 0x00007f9530133706 in XrdCl::JobManager::RunJobs() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdCl.so.3
#3 0x00007f95301337b9 in RunRunnerThread () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdCl.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 13 (Thread 0x7f94dcaf9700 (LWP 2930425) "cmsRun"):
#0 0x00007f9535a21180 in nanosleep () from /lib64/libpthread.so.0
#1 0x00007f9530222f08 in XrdSysTimer::Wait(int) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#2 0x00007f95300a803c in XrdCl::TaskManager::RunTasks() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdCl.so.3
#3 0x00007f95300a8179 in RunRunnerThread () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdCl.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 12 (Thread 0x7f94dd2fa700 (LWP 2930424) "cmsRun"):
#0 0x00007f9535779247 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f953021d282 in XrdSys::IOEvents::PollE::Begin(XrdSysSemaphore*, int&, char const**) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#2 0x00007f95302191ed in XrdSys::IOEvents::BootStrap::Start(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#3 0x00007f9530222617 in XrdSysThread_Xeq () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 11 (Thread 0x7f94ddafb700 (LWP 2930423) "cmsRun"):
#0 0x00007f9535779247 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f953021d282 in XrdSys::IOEvents::PollE::Begin(XrdSysSemaphore*, int&, char const**) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#2 0x00007f95302191ed in XrdSys::IOEvents::BootStrap::Start(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#3 0x00007f9530222617 in XrdSysThread_Xeq () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7f94de2fc700 (LWP 2930422) "cmsRun"):
#0 0x00007f9535779247 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f953021d282 in XrdSys::IOEvents::PollE::Begin(XrdSysSemaphore*, int&, char const**) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#2 0x00007f95302191ed in XrdSys::IOEvents::BootStrap::Start(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#3 0x00007f9530222617 in XrdSysThread_Xeq () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7f94deafd700 (LWP 2930421) "cmsRun"):
#0 0x00007f9535779247 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f953021d282 in XrdSys::IOEvents::PollE::Begin(XrdSysSemaphore*, int&, char const**) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#2 0x00007f95302191ed in XrdSys::IOEvents::BootStrap::Start(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#3 0x00007f9530222617 in XrdSysThread_Xeq () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7f94df2fe700 (LWP 2930420) "cmsRun"):
#0 0x00007f9535779247 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f953021d282 in XrdSys::IOEvents::PollE::Begin(XrdSysSemaphore*, int&, char const**) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#2 0x00007f95302191ed in XrdSys::IOEvents::BootStrap::Start(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#3 0x00007f9530222617 in XrdSysThread_Xeq () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7f94dfaff700 (LWP 2930419) "cmsRun"):
#0 0x00007f9535779247 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f953021d282 in XrdSys::IOEvents::PollE::Begin(XrdSysSemaphore*, int&, char const**) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#2 0x00007f95302191ed in XrdSys::IOEvents::BootStrap::Start(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#3 0x00007f9530222617 in XrdSysThread_Xeq () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7f94e0300700 (LWP 2930418) "cmsRun"):
#0 0x00007f9535779247 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f953021d282 in XrdSys::IOEvents::PollE::Begin(XrdSysSemaphore*, int&, char const**) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#2 0x00007f95302191ed in XrdSys::IOEvents::BootStrap::Start(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#3 0x00007f9530222617 in XrdSysThread_Xeq () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7f94e0b01700 (LWP 2930417) "cmsRun"):
#0 0x00007f9535779247 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f953021d282 in XrdSys::IOEvents::PollE::Begin(XrdSysSemaphore*, int&, char const**) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#2 0x00007f95302191ed in XrdSys::IOEvents::BootStrap::Start(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#3 0x00007f9530222617 in XrdSysThread_Xeq () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f94e1302700 (LWP 2930416) "cmsRun"):
#0 0x00007f9535779247 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f953021d282 in XrdSys::IOEvents::PollE::Begin(XrdSysSemaphore*, int&, char const**) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#2 0x00007f95302191ed in XrdSys::IOEvents::BootStrap::Start(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#3 0x00007f9530222617 in XrdSysThread_Xeq () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f94e1b03700 (LWP 2930415) "cmsRun"):
#0 0x00007f9535779247 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f953021d282 in XrdSys::IOEvents::PollE::Begin(XrdSysSemaphore*, int&, char const**) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#2 0x00007f95302191ed in XrdSys::IOEvents::BootStrap::Start(void*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#3 0x00007f9530222617 in XrdSysThread_Xeq () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libXrdUtils.so.3
#4 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#5 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f9513ca2700 (LWP 2929786) "cmsRun"):
#0 0x00007f9535a21672 in waitpid () from /lib64/libpthread.so.0
#1 0x00007f95393f78ef in __interceptor_waitpid (pid=<optimized out>, status=0x7f9513c9ae00, options=<optimized out>) at ../../../../libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:2518
#2 0x00007f952f522962 in edm::service::cmssw_stacktrace_fork() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 0x00007f952f526a1a in edm::service::InitRootHandlers::stacktraceHelperThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 0x00007f95360aba73 in std::execute_native_thread_routine (__p=0x602000179290) at ../../../../../libstdc++-v3/src/c++11/thread.cc:82
#5 0x00007f9535a171ca in start_thread () from /lib64/libpthread.so.0
#6 0x00007f9535683e73 in clone () from /lib64/libc.so.6
Thread 1 (Thread 0x7f9534ba06c0 (LWP 2929564) "cmsRun"):
#0 0x00007f953576e301 in poll () from /lib64/libc.so.6
#1 0x00007f95393ef782 in __interceptor_poll (fds=0x7ffc30c81550, nfds=1, timeout=299999) at ../../../../libsanitizer/sanitizer_common/sanitizer_common_interceptors.inc:4044
#2 0x00007f952f523b34 in full_read.constprop () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#3 0x00007f952f526bf4 in edm::service::InitRootHandlers::stacktraceFromThread() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 0x00007f952f5314fe in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#5 <signal handler called>
#6 0x00007f95393a32c5 in __sanitizer::atomic_compare_exchange_strong<__sanitizer::atomic_uint8_t> (mo=__sanitizer::memory_order_acquire, xchg=3 '\003', cmp=<synthetic pointer>, a=0x8e405) at ../../../../libsanitizer/sanitizer_common/sanitizer_atomic_clang.h:74
#7 __asan::Allocator::AtomicallySetQuarantineFlagIfAllocated (this=0x7f95394b5dc0 <__asan::instance>, stack=0x7ffc30c82370, ptr=0x8e415, m=0x8e405) at ../../../../libsanitizer/asan/asan_allocator.cpp:620
#8 __asan::Allocator::Deallocate (alloc_type=__asan::FROM_MALLOC, stack=0x7ffc30c82370, delete_alignment=0, delete_size=0, ptr=0x8e415, this=0x7f95394b5dc0 <__asan::instance>) at ../../../../libsanitizer/asan/asan_allocator.cpp:696
#9 __asan::asan_free (ptr=ptr@entry=0x8e415, stack=stack@entry=0x7ffc30c82370, alloc_type=alloc_type@entry=__asan::FROM_MALLOC) at ../../../../libsanitizer/asan/asan_allocator.cpp:955
#10 0x00007f953943188f in __interceptor_free (ptr=0x8e415) at ../../../../libsanitizer/asan/asan_malloc_linux.cpp:53
#11 0x00007f94d49edb8a in gbl::GblTrajectory::prepare() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libGBL.so.2.4
#12 0x00007f94d49f01c0 in gbl::GblTrajectory::construct() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libGBL.so.2.4
#13 0x00007f94d1f3a0cf in gbl::GblTrajectory::GblTrajectory<Eigen::Matrix<double, -1, -1, 0, -1, -1>, Eigen::Matrix<double, -1, 1, 0, -1, 1>, Eigen::Matrix<double, -1, 1, 0, -1, 1>, (void*)0>(std::vector<std::pair<std::vector<gbl::GblPoint, std::allocator<gbl::GblPoint> >, Eigen::Matrix<double, -1, -1, 0, -1, -1> >, std::allocator<std::pair<std::vector<gbl::GblPoint, std::allocator<gbl::GblPoint> >, Eigen::Matrix<double, -1, -1, 0, -1, -1> > > > const&, Eigen::MatrixBase<Eigen::Matrix<double, -1, -1, 0, -1, -1> > const&, Eigen::MatrixBase<Eigen::Matrix<double, -1, 1, 0, -1, 1> > const&, Eigen::MatrixBase<Eigen::Matrix<double, -1, 1, 0, -1, 1> > const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so
#14 0x00007f94d1ec1493 in MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so
#15 0x00007f94d1ec235e in MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so
#16 0x00007f94d5f8b939 in AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so
#17 0x00007f95391954ff in edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#18 0x00007f9539143039 in edm::WorkerT<edm::one::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#19 0x00007f9538e0c798 in decltype ({parm#1}()) edm::convertException::wrap<edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}>(edm::Worker::runModule<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*)::{lambda()#1}) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#20 0x00007f9538e0ce43 in std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#21 0x00007f9538e14f03 in edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}::operator()() const () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#22 0x00007f9538e1b379 in void edm::SerialTaskQueueChain::actionToRun<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#23 0x00007f9538e1b8a5 in edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#24 0x00007f9537fb5249 in tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so
#25 0x00007f95368f6281 in tbb::detail::r1::task_dispatcher::local_wait_for_all<false, tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x7f95323c3e00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_0_0_pre2_SKYLAKEAVX512-el8_amd64_gcc12/build/CMSSW_14_0_0_pre2_SKYLAKEAVX512-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-5109e373398eaf79a8268d6a86a2f6e2/tbb-v2021.9.0/src/tbb/task_dispatcher.h:322
#26 tbb::detail::r1::task_dispatcher::local_wait_for_all<tbb::detail::r1::external_waiter> (waiter=..., t=<optimized out>, this=0x7f95323c3e00) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_0_0_pre2_SKYLAKEAVX512-el8_amd64_gcc12/build/CMSSW_14_0_0_pre2_SKYLAKEAVX512-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-5109e373398eaf79a8268d6a86a2f6e2/tbb-v2021.9.0/src/tbb/task_dispatcher.h:458
#27 tbb::detail::r1::task_dispatcher::execute_and_wait (t=<optimized out>, wait_ctx=..., w_ctx=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_0_0_pre2_SKYLAKEAVX512-el8_amd64_gcc12/build/CMSSW_14_0_0_pre2_SKYLAKEAVX512-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-5109e373398eaf79a8268d6a86a2f6e2/tbb-v2021.9.0/src/tbb/task_dispatcher.cpp:168
#28 0x00007f9538b8fe1c in edm::FinalWaitingTask::wait() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#29 0x00007f9538b313b3 in edm::EventProcessor::processRuns() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#30 0x00007f9538b616d6 in edm::EventProcessor::runToCompletion() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_ASAN_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
#31 0x000000000040bb65 in tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const ()
#32 0x00007f95368e29ad in tbb::detail::r1::task_arena_impl::execute (ta=..., d=...) at /data/cmsbld/jenkins/workspace/auto-builds/CMSSW_14_0_0_pre2_SKYLAKEAVX512-el8_amd64_gcc12/build/CMSSW_14_0_0_pre2_SKYLAKEAVX512-build/BUILD/el8_amd64_gcc12/external/tbb/v2021.9.0-5109e373398eaf79a8268d6a86a2f6e2/tbb-v2021.9.0/src/tbb/arena.cpp:688
#33 0x000000000040f71b in main::{lambda()#1}::operator()() const ()
#34 0x00000000004083b5 in main ()
Current Modules:
Module: AlignmentProducerAsAnalyzer:AlignmentProducer (crashed)
A fatal system signal has occurred: segmentation violation
I have also put the relevant log files in the following area,
/afs/cern.ch/work/s/sroychow/public/TkDPG/segfaultmille/scripts
Thanks, so ASAN crashes within ASAN code itself. That points towards a pretty bad memory corruption.
I ran the job (thanks for the easy and quick reproducer!) in UBSAN, but it crashed as in the issue description without adding any information. (ok, it did report
/data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/cms/vdt/0.4.3-5a80085534117eaccb28e669c6da4b6f/include/vdt/exp.h:144:38: runtime error: left shift of negative value -74
#0 0x7f06bb94c1e2 in vdt::fast_expf(float) /data/cmsbld/jenkins/workspace/build-any-ib/w/el8_amd64_gcc12/cms/vdt/0.4.3-5a80085534117eaccb28e669c6da4b6f/include/vdt/exp.h:144
#1 0x7f06bb94c1e2 in VVIObjFDetails::expint(float) src/RecoLocalTracker/SiPixelRecHits/src/VVIObjF.cc:512
#2 0x7f06bbc59f51 in VVIObjF::VVIObjF(float) src/RecoLocalTracker/SiPixelRecHits/src/VVIObjF.cc:208
#3 0x7f06bbc13368 in SiPixelTemplateReco::PixelTempReco1D(int, float, float, float, float, SiPixelTemplateReco::ClusMatrix&, SiPixelTemplate&, float&, float&, float&, float&, float&, float&, int&, int, bool, std::vector<std::pair<int, int>, std::allocator<std::pair<int, int> > >&, float&, int&, int&) src/RecoLocalTracker/SiPixelRecHits/src/SiPixelTemplateReco.cc:1185
...
but I think that is reported also UBSAN IBs, so I didn't worry about it; even if undefined behavior is by definition undefined...)
I'm checking with valgrind now.
My valgrind is still running, but it is already showing things like
==4269== Warning: set address range perms: large range [0x37875000, 0x49823000) (defined)
==4269== Invalid read of size 8
==4269== at 0x6821DB80: gbl::GblTrajectory::prepare() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4269== by 0x682201BF: gbl::GblTrajectory::construct() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4269== by 0x6BAE218D: MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4269== by 0x6BAE27DF: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4269== by 0x623CC47E: AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==4269== by 0x4C6E5CD: edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4C58E6D: edm::WorkerT<edm::one::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEB5F8: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEC6CE: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x42181CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:322)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:458)
==4269== by 0x6404280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==4269== by 0x4B6ECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78649: edm::EventProcessor::processRuns() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78BA0: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4074EE: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x63F09AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==4269== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== Address 0x7afbf6c8 is 8 bytes before a block of size 144 alloc'd
==4269== at 0x403A76B: malloc (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==4269== by 0x50E827AB: Eigen::PlainObjectBase<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::resize(long, long) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/lwtnn/2.13-2a9158f4cc92ca328e394928acaf9101/lib/liblwtnn.so)
==4269== by 0x6821D5D2: gbl::GblTrajectory::prepare() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4269== by 0x682201BF: gbl::GblTrajectory::construct() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4269== by 0x6BAE218D: MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4269== by 0x6BAE27DF: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4269== by 0x623CC47E: AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==4269== by 0x4C6E5CD: edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4C58E6D: edm::WorkerT<edm::one::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEB5F8: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEC6CE: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x42181CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:322)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:458)
==4269== by 0x6404280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==4269== by 0x4B6ECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78649: edm::EventProcessor::processRuns() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78BA0: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4074EE: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x63F09AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==4269== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269==
==4269== Invalid free() / delete / delete[] / realloc()
==4269== at 0x403D9E4: free (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==4269== by 0x6821DB89: gbl::GblTrajectory::prepare() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4269== by 0x682201BF: gbl::GblTrajectory::construct() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4269== by 0x6BAE218D: MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4269== by 0x6BAE27DF: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4269== by 0x623CC47E: AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==4269== by 0x4C6E5CD: edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4C58E6D: edm::WorkerT<edm::one::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEB5F8: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEC6CE: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x42181CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:322)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:458)
==4269== by 0x6404280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==4269== by 0x4B6ECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78649: edm::EventProcessor::processRuns() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78BA0: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4074EE: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x63F09AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==4269== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== Address 0x3 is not stack'd, malloc'd or (recently) free'd
==4269== Mismatched free() / delete / delete []
==4269== at 0x403D9E4: free (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==4269== by 0x6821E471: gbl::GblTrajectory::prepare() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4269== by 0x682201BF: gbl::GblTrajectory::construct() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4269== by 0x6BAE218D: MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4269== by 0x6BAE27DF: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4269== by 0x623CC47E: AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==4269== by 0x4C6E5CD: edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4C58E6D: edm::WorkerT<edm::one::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEB5F8: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEC6CE: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x42181CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:322)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:458)
==4269== by 0x6404280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==4269== by 0x4B6ECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78649: edm::EventProcessor::processRuns() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78BA0: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4074EE: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x63F09AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==4269== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== Address 0x74e3f210 is 0 bytes inside a block of size 64 alloc'd
==4269== at 0x403AED1: operator new(unsigned long) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==4269== by 0x34998FBC: TrackerGeomBuilderFromGeometricDet::buildSilicon(std::vector<GeometricDet const*, std::allocator<GeometricDet const*> > const&, TrackerGeometry*, GeomDetEnumerators::SubDetector, std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libGeometryTrackerGeometryBuilder.so)
==4269== by 0x3499B97E: TrackerGeomBuilderFromGeometricDet::build(GeometricDet const*, PTrackerParameters const&, TrackerTopology const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libGeometryTrackerGeometryBuilder.so)
==4269== by 0x623C8C49: AlignmentProducerBase::createGeometries(edm::EventSetup const&, TrackerTopology const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==4269== by 0x623CB03F: AlignmentProducerBase::initAlignmentAlgorithm(edm::EventSetup const&, bool) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==4269== by 0x623CC80B: AlignmentProducerBase::beginRunImpl(edm::Run const&, edm::EventSetup const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==4269== by 0x4C6F118: edm::one::EDProducerBase::doBeginRun(edm::RunTransitionInfo const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4C590BF: edm::WorkerT<edm::one::EDProducerBase>::implDoBegin(edm::RunTransitionInfo const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BAC81E: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)0> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)0>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)0>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BB0A6D: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)0> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::RunPrincipal, (edm::BranchActionType)0> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x42181CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:322)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:458)
==4269== by 0x6404280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==4269== by 0x4B6ECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78649: edm::EventProcessor::processRuns() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78BA0: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4074EE: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x63F09AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==4269== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== Invalid read of size 8
==4269== at 0x3495EF40: SiStripDetCabling::nApvPairs(unsigned int) const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libCalibFormatsSiStripObjects.so)
==4269== by 0x349727D6: SiStripQuality::addInvalidConnectionFromCabling() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libCalibFormatsSiStripObjects.so)
==4269== by 0x349729CF: SiStripQuality::add(SiStripDetCabling const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libCalibFormatsSiStripObjects.so)
==4269== by 0x4EFDA816: SiStripQualityESProducer::produce(SiStripQualityRcd const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginCalibTrackerSiStripRealESProducersPlugins.so)
==4269== by 0x4EFE1D71: edm::eventsetup::CallbackBase<edm::ESProducer, edm::ESProducer::setWhatProduced<SiStripQualityESProducer, std::unique_ptr<SiStripQuality, std::default_delete<SiStripQuality> >, SiStripQualityRcd, edm::eventsetup::CallbackSimpleDecorator<SiStripQualityRcd> >(SiStripQualityESProducer*, std::unique_ptr<SiStripQuality, std::default_delete<SiStripQuality> > (SiStripQualityESProducer::*)(SiStripQualityRcd const&), edm::eventsetup::CallbackSimpleDecorator<SiStripQualityRcd> const&, edm::es::Label const&)::{lambda(SiStripQualityRcd const&)#1}, std::unique_ptr<SiStripQuality, std::default_delete<SiStripQuality> >, SiStripQualityRcd, edm::eventsetup::CallbackSimpleDecorator<SiStripQualityRcd> >::makeProduceTask<edm::eventsetup::Callback<edm::ESProducer, edm::ESProducer::setWhatProduced<SiStripQualityESProducer, std::unique_ptr<SiStripQuality, std::default_delete<SiStripQuality> >, SiStripQualityRcd, edm::eventsetup::CallbackSimpleDecorator<SiStripQualityRcd> >(SiStripQualityESProducer*, std::unique_ptr<SiStripQuality, std::default_delete<SiStripQuality> > (SiStripQualityESProducer::*)(SiStripQualityRcd const&), edm::eventsetup::CallbackSimpleDecorator<SiStripQualityRcd> const&, edm::es::Label const&)::{lambda(SiStripQualityRcd const&)#1}, std::unique_ptr<SiStripQuality, std::default_delete<SiStripQuality> >, SiStripQualityRcd, edm::eventsetup::CallbackSimpleDecorator<SiStripQualityRcd> >::prefetchAsync(edm::WaitingTaskHolder, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, edm::ServiceToken const&, edm::ESParentContext const&)::{lambda(auto:1&&, auto:2&&, auto:3&&, auto:4&&)#1}::operator()<tbb::detail::d1::task_group*&, edm::ServiceWeakToken&, edm::eventsetup::EventSetupRecordImpl const*&, edm::EventSetupImpl const*&>(tbb::detail::d1::task_group*&, edm::ServiceWeakToken&, edm::eventsetup::EventSetupRecordImpl const*&, edm::EventSetupImpl const*&) const::{lambda(SiStripQualityRcd const&)#1}>(tbb::detail::d1::task_group*, edm::ServiceWeakToken const&, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, bool, tbb::detail::d1::task_group*&)::{lambda(std::__exception_ptr::exception_ptr const*)#1}::operator()(std::__exception_ptr::exception_ptr const*) const::{lambda()#2}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginCalibTrackerSiStripRealESProducersPlugins.so)
==4269== by 0x4EFE20D7: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::eventsetup::CallbackBase<edm::ESProducer, edm::ESProducer::setWhatProduced<SiStripQualityESProducer, std::unique_ptr<SiStripQuality, std::default_delete<SiStripQuality> >, SiStripQualityRcd, edm::eventsetup::CallbackSimpleDecorator<SiStripQualityRcd> >(SiStripQualityESProducer*, std::unique_ptr<SiStripQuality, std::default_delete<SiStripQuality> > (SiStripQualityESProducer::*)(SiStripQualityRcd const&), edm::eventsetup::CallbackSimpleDecorator<SiStripQualityRcd> const&, edm::es::Label const&)::{lambda(SiStripQualityRcd const&)#1}, std::unique_ptr<SiStripQuality, std::default_delete<SiStripQuality> >, SiStripQualityRcd, edm::eventsetup::CallbackSimpleDecorator<SiStripQualityRcd> >::makeProduceTask<edm::eventsetup::Callback<edm::ESProducer, edm::ESProducer::setWhatProduced<SiStripQualityESProducer, std::unique_ptr<SiStripQuality, std::default_delete<SiStripQuality> >, SiStripQualityRcd, edm::eventsetup::CallbackSimpleDecorator<SiStripQualityRcd> >(SiStripQualityESProducer*, std::unique_ptr<SiStripQuality, std::default_delete<SiStripQuality> > (SiStripQualityESProducer::*)(SiStripQualityRcd const&), edm::eventsetup::CallbackSimpleDecorator<SiStripQualityRcd> const&, edm::es::Label const&)::{lambda(SiStripQualityRcd const&)#1}, std::unique_ptr<SiStripQuality, std::default_delete<SiStripQuality> >, SiStripQualityRcd, edm::eventsetup::CallbackSimpleDecorator<SiStripQualityRcd> >::prefetchAsync(edm::WaitingTaskHolder, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, edm::ServiceToken const&, edm::ESParentContext const&)::{lambda(auto:1&&, auto:2&&, auto:3&&, auto:4&&)#1}::operator()<tbb::detail::d1::task_group*&, edm::ServiceWeakToken&, edm::eventsetup::EventSetupRecordImpl const*&, edm::EventSetupImpl const*&>(tbb::detail::d1::task_group*&, edm::ServiceWeakToken&, edm::eventsetup::EventSetupRecordImpl const*&, edm::EventSetupImpl const*&) const::{lambda(SiStripQualityRcd const&)#1}>(tbb::detail::d1::task_group*, edm::ServiceWeakToken const&, edm::eventsetup::EventSetupRecordImpl const*, edm::EventSetupImpl const*, bool, tbb::detail::d1::task_group*&)::{lambda(std::__exception_ptr::exception_ptr const*)#1}::operator()(std::__exception_ptr::exception_ptr const*) const::{lambda()#2}>(tbb::detail::d1::task_group&, tbb::detail::d1::task_group*&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginCalibTrackerSiStripRealESProducersPlugins.so)
==4269== by 0x42181CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:322)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:458)
==4269== by 0x6404280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==4269== by 0x4B6ECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78649: edm::EventProcessor::processRuns() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78BA0: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4074EE: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x63F09AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==4269== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== Address 0x78b44fd0 is 0 bytes inside an unallocated block of size 32 in arena "client"
and (I'm guessing at this stage the memory has corrupted enough for anything to happen)
==4269== Invalid read of size 1
==4269== at 0x4219220: edm::WaitingTaskList::announce() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x4BEB681: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEC6CE: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x42181CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:322)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:458)
==4269== by 0x6404280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==4269== by 0x4B6ECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78649: edm::EventProcessor::processRuns() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78BA0: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4074EE: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x63F09AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==4269== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== Address 0x82f659f is 31 bytes inside a block of size 1,272 free'd
==4269== at 0x403D9E4: free (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==4269== by 0x6821DB89: gbl::GblTrajectory::prepare() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4269== by 0x682201BF: gbl::GblTrajectory::construct() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4269== by 0x6BAE218D: MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgor
ithmAuto.so)
==4269== by 0x6BAE27DF: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4269== by 0x623CC47E: AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==4269== by 0x4C6E5CD: edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4C58E6D: edm::WorkerT<edm::one::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEB5F8: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::Occurrenc
eTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEC6CE: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda(
)#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x42181CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:322)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:458)
==4269== by 0x6404280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==4269== by 0x4B6ECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78649: edm::EventProcessor::processRuns() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78BA0: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4074EE: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x63F09AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==4269== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== Block was alloc'd at
==4269== at 0x403AED1: operator new(unsigned long) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==4269== by 0x40736C: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x63F09AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==4269== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== Invalid read of size 1
==4269== at 0x6401D39: tbb::detail::r1::spawn(tbb::detail::d1::task&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:32)
==4269== by 0x421CA77: ??? (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x421929A: edm::WaitingTaskList::announce() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x4BEB681: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEC6CE: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x42181CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:322)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:458)
==4269== by 0x6404280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==4269== by 0x4B6ECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78649: edm::EventProcessor::processRuns() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78BA0: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4074EE: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x63F09AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==4269== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== Address 0x82f659f is 31 bytes inside a block of size 1,272 free'd
==4269== at 0x403D9E4: free (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==4269== by 0x6821DB89: gbl::GblTrajectory::prepare() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4269== by 0x682201BF: gbl::GblTrajectory::construct() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4269== by 0x6BAE218D: MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4269== by 0x6BAE27DF: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4269== by 0x623CC47E: AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==4269== by 0x4C6E5CD: edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4C58E6D: edm::WorkerT<edm::one::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEB5F8: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4BEC6CE: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x42181CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:322)
==4269== by 0x6404280: UnknownInlinedFun (task_dispatcher.h:458)
==4269== by 0x6404280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==4269== by 0x4B6ECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78649: edm::EventProcessor::processRuns() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4B78BA0: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==4269== by 0x4074EE: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x63F09AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==4269== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== Block was alloc'd at
==4269== at 0x403AED1: operator new(unsigned long) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==4269== by 0x40736C: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x63F09AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==4269== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
==4269== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRun)
The gbl::GblTrajectory::prepare() looks quite complicated, so I'm wondering if further debugging would need a debug build of gbl external.
cmsRunGlibC crashed as
Thread 1 (Thread 0x7f14d853ac40 (LWP 11778) "cmsRunGlibC"):
#3 0x00007f14d6183730 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x00007f14d907fe31 in free () from /lib64/libc.so.6
#6 0x00007f1490b89b8a in gbl::GblTrajectory::prepare() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libGBL.so.2.4
#7 0x00007f1490b8c1c0 in gbl::GblTrajectory::construct() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libGBL.so.2.4
#8 0x00007f1490a7f18e in MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so
#9 0x00007f1490a7f7e0 in MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024
-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so
#10 0x00007f14913f547f in AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so
#11 0x00007f14dbe885ce in edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
cmsRunTC died with SIGABRT and
src/tcmalloc.cc:300] Attempt to free invalid pointer 0xbfc2f0b690d1919c
Thread 1 "cmsRunTC" received signal SIGABRT, Aborted. [5/1521]
0x00007ffff52b1acf in raise () from /lib64/libc.so.6
(gdb) where
#0 0x00007ffff52b1acf in raise () from /lib64/libc.so.6
#1 0x00007ffff5284ea5 in abort () from /lib64/libc.so.6
#2 0x00007ffff5e31fa9 in tcmalloc::Log (mode=<optimized out>, mode@entry=tcmalloc::kCrash, filename=filename@entry=0x7ffff5e42676 "src/tcmalloc.cc", line=<optimized out>, line@entry=300, a=..., b=..., c=...,
d=...) at src/internal_logging.cc:118
#3 0x00007ffff5e327d9 in (anonymous namespace)::InvalidFree (ptr=<optimized out>) at src/internal_logging.h:67
#4 0x00007fffae3d2b8a in gbl::GblTrajectory::prepare() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libGBL.so.2.4
#5 0x00007fffae3d51c0 in gbl::GblTrajectory::construct() () from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/external/el8_amd64_gcc12/lib/libGBL.so.2.4
#6 0x00007fffae2c818e in MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) ()
from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so
#7 0x00007fffae2c87e0 in MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) ()
from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so
#8 0x00007fffaec3e47f in AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) ()
from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so
#9 0x00007ffff7cd65ce in edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) ()
from /cvmfs/cms-ib.cern.ch/sw/x86_64/week0/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
With debug build of gbl cmsRunGlibC shows
Thread 1 (Thread 0x7fe4e6e4bc40 (LWP 17805) "cmsRunGlibC"):
#3 0x00007fe4e4a94730 in sig_dostack_then_abort () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so
#4 <signal handler called>
#5 0x00007fe4e7990e31 in free () from /lib64/libc.so.6
#6 0x00007fe47b2995df in Eigen::internal::handmade_aligned_free (ptr=0x262851e0) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/el8_amd64_gcc12/external/eigen/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e-cf66836cbd6ad8d696d804181e36d61c/include/eigen3/Eigen/src/Core/util/Memory.h:120
#7 0x00007fe47b299640 in Eigen::internal::aligned_free (ptr=0x262851e0) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/el8_amd64_gcc12/external/eigen/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e-cf66836cbd6ad8d696d804181e36d61c/include/eigen3/Eigen/src/Core/util/Memory.h:208
#8 0x00007fe47b29a3ac in Eigen::internal::conditional_aligned_free<true> (ptr=0x262851e0) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/el8_amd64_gcc12/external/eigen/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e-cf66836cbd6ad8d696d804181e36d61c/include/eigen3/Eigen/src/Core/util/Memory.h:272
#9 0x00007fe47b299db0 in Eigen::internal::conditional_aligned_delete_auto<double, true> (ptr=0x262851e0, size=4) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/el8_amd64_gcc12/external/eigen/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e-cf66836cbd6ad8d696d804181e36d61c/include/eigen3/Eigen/src/Core/util/Memory.h:498
#10 0x00007fe47b2998bb in Eigen::DenseStorage<double, -1, -1, -1, 0>::~DenseStorage (this=0x7ffd7928c3c0, __in_chrg=<optimized out>) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/el8_amd64_gcc12/external/eigen/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e-cf66836cbd6ad8d696d804181e36d61c/include/eigen3/Eigen/src/Core/DenseStorage.h:442
#11 0x00007fe47b299668 in Eigen::PlainObjectBase<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::~PlainObjectBase (this=0x7ffd7928c3c0, __in_chrg=<optimized out>) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/el8_amd64_gcc12/external/eigen/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e-cf66836cbd6ad8d696d804181e36d61c/include/eigen3/Eigen/src/Core/PlainObjectBase.h:100
#12 0x00007fe47b299684 in Eigen::Matrix<double, -1, -1, 0, -1, -1>::~Matrix (this=0x7ffd7928c3c0, __in_chrg=<optimized out>) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/el8_amd64_gcc12/external/eigen/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e-cf66836cbd6ad8d696d804181e36d61c/include/eigen3/Eigen/src/Core/Matrix.h:180
#13 0x00007fe47b2a03f0 in Eigen::SelfAdjointEigenSolver<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::~SelfAdjointEigenSolver (this=0x7ffd7928c3c0, __in_chrg=<optimized out>) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/el8_amd64_gcc12/external/eigen/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e-cf66836cbd6ad8d696d804181e36d61c/include/eigen3/Eigen/src/Eigenvalues/SelfAdjointEigenSolver.h:78
#14 0x00007fe47b29d090 in gbl::GblMeasurement::GblMeasurement (this=0x27caa830, aProjection=..., aResiduals=..., aPrecision=..., minPrecision=0.29999999999999999) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/gbl/V02-04-01-5fcbb233a060467b13cbdfca73266fc9/gbl-V02-04-01/cpp/src/GblMeasurement.cpp:62
#15 0x00007fe47b477b35 in void std::__new_allocator<gbl::GblMeasurement>::construct<gbl::GblMeasurement, Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&>(gbl::GblMeasurement*, Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&) [clone .constprop.0] [clone .isra.0] () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#16 0x00007fe47b477c50 in gbl::GblMeasurement& std::vector<gbl::GblMeasurement, std::allocator<gbl::GblMeasurement> >::emplace_back<Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&>(Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&) [clone .constprop.0] () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#17 0x00007fe47b468aa9 in ReferenceTrajectory::addMaterialEffectsLocalGbl(std::vector<CLHEP::HepMatrix, std::allocator<CLHEP::HepMatrix> > const&, std::vector<CLHEP::HepMatrix, std::allocator<CLHEP::HepMatrix> > const&, std::vector<CLHEP::HepSymMatrix, std::allocator<CLHEP::HepSymMatrix> > const&, std::vector<CLHEP::HepSymMatrix, std::allocator<CLHEP::HepSymMatrix> > const&) () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#18 0x00007fe47b45f6b1 in ReferenceTrajectory::construct(TrajectoryStateOnSurface const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, MagneticField const*, reco::BeamSpot const&) () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#19 0x00007fe47b460976 in ReferenceTrajectory::ReferenceTrajectory(TrajectoryStateOnSurface const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, MagneticField const*, reco::BeamSpot const&, ReferenceTrajectoryBase::Config const&) () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#20 0x00007fe47b473a07 in TwoBodyDecayTrajectory::construct(TwoBodyDecayTrajectoryState const&, std::pair<std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > > const&, MagneticField const*, reco::BeamSpot const&) () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#21 0x00007fe47b474ec6 in TwoBodyDecayTrajectory::TwoBodyDecayTrajectory(TwoBodyDecayTrajectoryState const&, std::pair<std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > > const&, MagneticField const*, reco::BeamSpot const&, ReferenceTrajectoryBase::Config const&) () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#22 0x00007fe47acc54e3 in TwoBodyDecayTrajectoryFactory::constructTrajectories(std::vector<std::pair<Trajectory const*, reco::Track const*>, std::allocator<std::pair<Trajectory const*, reco::Track const*> > > const&, TwoBodyDecay const&, MagneticField const*, reco::BeamSpot const&, bool) const () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentReferenceTrajectoriesPlugins.so
#23 0x00007fe47acc2b6f in TwoBodyDecayTrajectoryFactory::trajectories(edm::EventSetup const&, std::vector<std::pair<Trajectory const*, reco::Track const*>, std::allocator<std::pair<Trajectory const*, reco::Track const*> > > const&, reco::BeamSpot const&) const () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentReferenceTrajectoriesPlugins.so
#24 0x00007fe47aee476e in MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so
#25 0x00007fe47bc8647f in AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so
#26 0x00007fe4ea7995ce in edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
Running the cmsRunGlibC through valgrind becomes interesting
==18087== Invalid read of size 8
==18087== at 0x6850A5D4: Eigen::internal::handmade_aligned_free(void*) (Memory.h:120)
==18087== by 0x6850A63F: Eigen::internal::aligned_free(void*) (Memory.h:208)
==18087== by 0x6850B3AB: void Eigen::internal::conditional_aligned_free<true>(void*) (Memory.h:272)
==18087== by 0x6850ADAF: void Eigen::internal::conditional_aligned_delete_auto<double, true>(double*, unsigned long) (Memory.h:498)
==18087== by 0x6850A8BA: Eigen::DenseStorage<double, -1, -1, -1, 0>::~DenseStorage() (DenseStorage.h:442)
==18087== by 0x6850A667: Eigen::PlainObjectBase<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::~PlainObjectBase() (PlainObjectBase.h:100)
==18087== by 0x6850A683: Eigen::Matrix<double, -1, -1, 0, -1, -1>::~Matrix() (Matrix.h:180)
==18087== by 0x685113EF: Eigen::SelfAdjointEigenSolver<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::~SelfAdjointEigenSolver() (SelfAdjointEigenSolver.h:78)
==18087== by 0x6850E08F: gbl::GblMeasurement::GblMeasurement(Eigen::Matrix<double, -1, -1, 0, -1, -1> const&, Eigen::Matrix<double, -1, 1, 0, -1, 1> const&, Eigen::Matrix<double, -1, -1, 0, -1, -1> const&, double) (GblMeasurement.cpp:62)
==18087== by 0x681FEB34: void std::__new_allocator<gbl::GblMeasurement>::construct<gbl::GblMeasurement, Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&>(gbl::GblMeasurement*, Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&) [clone .constprop.0] [clone .isra.0] (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681FEC4F: gbl::GblMeasurement& std::vector<gbl::GblMeasurement, std::allocator<gbl::GblMeasurement> >::emplace_back<Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&>(Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&) [clone .constprop.0] (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681EFAA8: ReferenceTrajectory::addMaterialEffectsLocalGbl(std::vector<CLHEP::HepMatrix, std::allocator<CLHEP::HepMatrix> > const&, std::vector<CLHEP::HepMatrix, std::allocator<CLHEP::HepMatrix> > const&, std::vector<CLHEP::HepSymMatrix, std::allocator<CLHEP::HepSymMatrix> > const&, std::vector<CLHEP::HepSymMatrix, std::allocator<CLHEP::HepSymMatrix> > const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681E66B0: ReferenceTrajectory::construct(TrajectoryStateOnSurface const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, MagneticField const*, reco::BeamSpot const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681E7975: ReferenceTrajectory::ReferenceTrajectory(TrajectoryStateOnSurface const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, MagneticField const*, reco::BeamSpot const&, ReferenceTrajectoryBase::Config const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681FAA06: TwoBodyDecayTrajectory::construct(TwoBodyDecayTrajectoryState const&, std::pair<std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > > const&, MagneticField const*, reco::BeamSpot const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681FBEC5: TwoBodyDecayTrajectory::TwoBodyDecayTrajectory(TwoBodyDecayTrajectoryState const&, std::pair<std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > > const&, MagneticField const*, reco::BeamSpot const&, ReferenceTrajectoryBase::Config const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x6C5A24E2: TwoBodyDecayTrajectoryFactory::constructTrajectories(std::vector<std::pair<Trajectory const*, reco::Track const*>, std::allocator<std::pair<Trajectory const*, reco::Track const*> > > const&, TwoBodyDecay const&, MagneticField const*, reco::BeamSpot const&, bool) const (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentReferenceTrajectoriesPlugins.so)
==18087== by 0x6C59FB6E: TwoBodyDecayTrajectoryFactory::trajectories(edm::EventSetup const&, std::vector<std::pair<Trajectory const*, reco::Track const*>, std::allocator<std::pair<Trajectory const*, reco::Track const*> > > const&, reco::BeamSpot const&) const (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentReferenceTrajectoriesPlugins.so)
==18087== by 0x6C34E76D: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==18087== by 0x623CC47E: AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==18087== Address 0x78f43ff8 is 8 bytes before a block of size 32 alloc'd
==18087== at 0x403A76B: malloc (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==18087== by 0x512C27AB: Eigen::PlainObjectBase<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::resize(long, long) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/lwtnn/2.13-2a9158f4cc92ca328e394928acaf9101/lib/liblwtnn.so)
==18087== by 0x685155A2: void Eigen::PlainObjectBase<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::_init2<long, long>(long, long, std::enable_if<true, long>::type*) (PlainObjectBase.h:832)
==18087== by 0x68512A17: Eigen::Matrix<double, -1, -1, 0, -1, -1>::Matrix<long, long>(long const&, long const&) (Matrix.h:332)
==18087== by 0x685117B6: Eigen::SelfAdjointEigenSolver<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::SelfAdjointEigenSolver<Eigen::Matrix<double, -1, -1, 0, -1, -1> >(Eigen::EigenBase<Eigen::Matrix<double, -1, -1, 0, -1, -1> > const&, int) (SelfAdjointEigenSolver.h:176)
==18087== by 0x6850DEFF: gbl::GblMeasurement::GblMeasurement(Eigen::Matrix<double, -1, -1, 0, -1, -1> const&, Eigen::Matrix<double, -1, 1, 0, -1, 1> const&, Eigen::Matrix<double, -1, -1, 0, -1, -1> const&, double) (GblMeasurement.cpp:54)
==18087== by 0x681FEB34: void std::__new_allocator<gbl::GblMeasurement>::construct<gbl::GblMeasurement, Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&>(gbl::GblMeasurement*, Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&) [clone .constprop.0] [clone .isra.0] (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681FEC4F: gbl::GblMeasurement& std::vector<gbl::GblMeasurement, std::allocator<gbl::GblMeasurement> >::emplace_back<Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&>(Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&) [clone .constprop.0] (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681EFAA8: ReferenceTrajectory::addMaterialEffectsLocalGbl(std::vector<CLHEP::HepMatrix, std::allocator<CLHEP::HepMatrix> > const&, std::vector<CLHEP::HepMatrix, std::allocator<CLHEP::HepMatrix> > const&, std::vector<CLHEP::HepSymMatrix, std::allocator<CLHEP::HepSymMatrix> > const&, std::vector<CLHEP::HepSymMatrix, std::allocator<CLHEP::HepSymMatrix> > const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681E66B0: ReferenceTrajectory::construct(TrajectoryStateOnSurface const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, MagneticField const*, reco::BeamSpot const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681E7975: ReferenceTrajectory::ReferenceTrajectory(TrajectoryStateOnSurface const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, MagneticField const*, reco::BeamSpot const&, ReferenceTrajectoryBase::Config const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681FAA06: TwoBodyDecayTrajectory::construct(TwoBodyDecayTrajectoryState const&, std::pair<std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > > const&, MagneticField const*, reco::BeamSpot const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681FBEC5: TwoBodyDecayTrajectory::TwoBodyDecayTrajectory(TwoBodyDecayTrajectoryState const&, std::pair<std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > > const&, MagneticField const*, reco::BeamSpot const&, ReferenceTrajectoryBase::Config const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x6C5A24E2: TwoBodyDecayTrajectoryFactory::constructTrajectories(std::vector<std::pair<Trajectory const*, reco::Track const*>, std::allocator<std::pair<Trajectory const*, reco::Track const*> > > const&, TwoBodyDecay const&, MagneticField const*, reco::BeamSpot const&, bool) const (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentReferenceTrajectoriesPlugins.so)
==18087== by 0x6C59FB6E: TwoBodyDecayTrajectoryFactory::trajectories(edm::EventSetup const&, std::vector<std::pair<Trajectory const*, reco::Track const*>, std::allocator<std::pair<Trajectory const*, reco::Track const*> > > const&, reco::BeamSpot const&) const (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentReferenceTrajectoriesPlugins.so)
==18087== by 0x6C34E76D: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==18087== by 0x623CC47E: AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==18087== by 0x4C6E5CD: edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==18087== Invalid free() / delete / delete[] / realloc()
==18087== at 0x403D9E4: free (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==18087== by 0x6850A5DE: Eigen::internal::handmade_aligned_free(void*) (Memory.h:120)
==18087== by 0x6850A63F: Eigen::internal::aligned_free(void*) (Memory.h:208)
==18087== by 0x6850B3AB: void Eigen::internal::conditional_aligned_free<true>(void*) (Memory.h:272)
==18087== by 0x6850ADAF: void Eigen::internal::conditional_aligned_delete_auto<double, true>(double*, unsigned long) (Memory.h:498)
==18087== by 0x6850A8BA: Eigen::DenseStorage<double, -1, -1, -1, 0>::~DenseStorage() (DenseStorage.h:442)
==18087== by 0x6850A667: Eigen::PlainObjectBase<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::~PlainObjectBase() (PlainObjectBase.h:100)
==18087== by 0x6850A683: Eigen::Matrix<double, -1, -1, 0, -1, -1>::~Matrix() (Matrix.h:180)
==18087== by 0x685113EF: Eigen::SelfAdjointEigenSolver<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::~SelfAdjointEigenSolver() (SelfAdjointEigenSolver.h:78)
==18087== by 0x6850E08F: gbl::GblMeasurement::GblMeasurement(Eigen::Matrix<double, -1, -1, 0, -1, -1> const&, Eigen::Matrix<double, -1, 1, 0, -1, 1> const&, Eigen::Matrix<double, -1, -1, 0, -1, -1> const&, double) (GblMeasurement.cpp:62)
==18087== by 0x681FEB34: void std::__new_allocator<gbl::GblMeasurement>::construct<gbl::GblMeasurement, Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&>(gbl::GblMeasurement*, Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&) [clone .constprop.0] [clone .isra.0] (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681FEC4F: gbl::GblMeasurement& std::vector<gbl::GblMeasurement, std::allocator<gbl::GblMeasurement> >::emplace_back<Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&>(Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&) [clone .constprop.0] (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681EFAA8: ReferenceTrajectory::addMaterialEffectsLocalGbl(std::vector<CLHEP::HepMatrix, std::allocator<CLHEP::HepMatrix> > const&, std::vector<CLHEP::HepMatrix, std::allocator<CLHEP::HepMatrix> > const&, std::vector<CLHEP::HepSymMatrix, std::allocator<CLHEP::HepSymMatrix> > const&, std::vector<CLHEP::HepSymMatrix, std::allocator<CLHEP::HepSymMatrix> > const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681E66B0: ReferenceTrajectory::construct(TrajectoryStateOnSurface const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, MagneticField const*, reco::BeamSpot const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681E7975: ReferenceTrajectory::ReferenceTrajectory(TrajectoryStateOnSurface const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, MagneticField const*, reco::BeamSpot const&, ReferenceTrajectoryBase::Config const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681FAA06: TwoBodyDecayTrajectory::construct(TwoBodyDecayTrajectoryState const&, std::pair<std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > > const&, MagneticField const*, reco::BeamSpot const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x681FBEC5: TwoBodyDecayTrajectory::TwoBodyDecayTrajectory(TwoBodyDecayTrajectoryState const&, std::pair<std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > > const&, MagneticField const*, reco::BeamSpot const&, ReferenceTrajectoryBase::Config const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so)
==18087== by 0x6C5A24E2: TwoBodyDecayTrajectoryFactory::constructTrajectories(std::vector<std::pair<Trajectory const*, reco::Track const*>, std::allocator<std::pair<Trajectory const*, reco::Track const*> > > const&, TwoBodyDecay const&, MagneticField const*, reco::BeamSpot const&, bool) const (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentReferenceTrajectoriesPlugins.so)
==18087== by 0x6C59FB6E: TwoBodyDecayTrajectoryFactory::trajectories(edm::EventSetup const&, std::vector<std::pair<Trajectory const*, reco::Track const*>, std::allocator<std::pair<Trajectory const*, reco::Track const*> > > const&, reco::BeamSpot const&) const (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentReferenceTrajectoriesPlugins.so)
==18087== by 0x6C34E76D: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==18087== Address 0x3f7799803d000000 is not stack'd, malloc'd or (recently) free'd
Maybe the Eigen memory allocation strategy is still incorrect in some way?
New categories assigned: core
@Dr15Jones,@makortel,@smuzaffar you have been requested to review this Pull request/Issue and eventually sign? Thanks
I thought we had changed the build params for eigen so it wouldn't use the handmade_aligned_free(). This is probably the same kind of alignment disagreement we were seeing before, I think we really need to get rid of eigen's "handmade" allocators, there's just too many opportunities for stuff to go wrong.
Could it be that gbl be compiled using different flags associated with Eigen than the rest of CMSSW?
-DEIGEN_MAX_ALIGN_BYTES=64 -msse3 were added in https://github.com/cms-sw/cmsdist/pull/8979
The call to handmade_aligned_free() is controlled with
EIGEN_DEVICE_FUNC inline void aligned_free(void *ptr)
{
#if (EIGEN_DEFAULT_ALIGN_BYTES==0) || EIGEN_MALLOC_ALREADY_ALIGNED
EIGEN_USING_STD(free)
free(ptr);
#else
handmade_aligned_free(ptr);
#endif
}
https://github.com/cms-externals/eigen-git-mirror/blob/cms/master/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e/Eigen/src/Core/util/Memory.h#L200-L211
EIGEN_MALLOC_ALREADY_ALIGNED is defined as
#ifndef EIGEN_MALLOC_ALREADY_ALIGNED
// ...
#if defined(__GLIBC__) && ((__GLIBC__>=2 && __GLIBC_MINOR__ >= 8) || __GLIBC__>2) \
&& defined(__LP64__) && ! defined( __SANITIZE_ADDRESS__ ) && (EIGEN_DEFAULT_ALIGN_BYTES == 16)
#define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 1
#else
#define EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED 0
#endif
// ...
#if (EIGEN_OS_MAC && (EIGEN_DEFAULT_ALIGN_BYTES == 16)) \
|| (EIGEN_OS_WIN64 && (EIGEN_DEFAULT_ALIGN_BYTES == 16)) \
|| EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED \
|| EIGEN_FREEBSD_MALLOC_ALREADY_ALIGNED
#define EIGEN_MALLOC_ALREADY_ALIGNED 1
#else
#define EIGEN_MALLOC_ALREADY_ALIGNED 0
#endif
https://github.com/cms-externals/eigen-git-mirror/blob/cms/master/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e/Eigen/src/Core/util/Memory.h#L34-L39 https://github.com/cms-externals/eigen-git-mirror/blob/cms/master/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e/Eigen/src/Core/util/Memory.h#L51-L58
Playing with the definitions of the various Eigen macros the EIGEN_DEFAULT_ALIGN_BYTES seems to be 64, so the macro logic in https://github.com/cms-sw/cmssw/issues/44188#issuecomment-1969174886 leads to setting EIGEN_GLIBC_MALLOC_ALREADY_ALIGNED=0.
The EIGEN_DEFAULT_ALIGN_BYTES is defined as
#if EIGEN_IDEAL_MAX_ALIGN_BYTES > EIGEN_MAX_ALIGN_BYTES
#define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_IDEAL_MAX_ALIGN_BYTES
#else
#define EIGEN_DEFAULT_ALIGN_BYTES EIGEN_MAX_ALIGN_BYTES
#endif
https://github.com/cms-externals/eigen-git-mirror/blob/46f072265231c56d9cbad74ab0af2fb2ae576d5a/Eigen/src/Core/util/ConfigureVectorization.h#L178-L182
and EIGEN_IDEAL_MAX_ALIGN_BYTES is defined as
#if defined(EIGEN_DONT_VECTORIZE)
#if defined(EIGEN_GPUCC)
// GPU code is always vectorized and requires memory alignment for
// statically allocated buffers.
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 16
#else
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 0
#endif
#elif defined(__AVX512F__)
// 64 bytes static alignment is preferred only if really required
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 64
#elif defined(__AVX__)
// 32 bytes static alignment is preferred only if really required
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 32
#else
#define EIGEN_IDEAL_MAX_ALIGN_BYTES 16
#endif
https://github.com/cms-externals/eigen-git-mirror/blob/46f072265231c56d9cbad74ab0af2fb2ae576d5a/Eigen/src/Core/util/ConfigureVectorization.h#L57-L73
Compiling -msse3 leads to EIGEN_IDEAL_MAX_ALIGN_BYTES=16, and then the comparison logic above sets EIGEN_DEFAULT_ALIGN_BYTES=EIGEN_MAX_ALIGN_BYTES=64 following our -DEIGEN_MAX_ALIGN_BYTES=64.
(well, this just repeated https://github.com/cms-sw/cmssw/issues/43801#issuecomment-1918108485 in different words)
From https://eigen.tuxfamily.org/dox/TopicPreprocessorDirectives.html
EIGEN_MALLOC_ALREADY_ALIGNED- Can be set to 0 or 1 to tell whether default system malloc already returns aligned buffers. In not defined, then this information is automatically deduced from the compiler and system preprocessor tokens.EIGEN_MAX_ALIGN_BYTES- Must be a power of two, or 0. Defines an upper bound on the memory boundary in bytes on which dynamically and statically allocated data may be aligned by Eigen. If not defined, a default value is automatically computed based on architecture, compiler, and OS. This option is typically used to enforce binary compatibility between code/libraries compiled with different SIMD options. For instance, one may compile AVX code and enforce ABI compatibility with existing SSE code by definingEIGEN_MAX_ALIGN_BYTES=16. In the other way round, since by default AVX implies 32 bytes alignment for best performance, one can compile SSE code to be ABI compatible with AVX code by definingEIGEN_MAX_ALIGN_BYTES=32.
Should we explicitly set
EIGEN_MALLOC_ALREADY_ALIGNED=1?
Not sure if that would be correct for the 64-byte alignment we request though...
https://gitlab.com/libeigen/eigen/-/issues/1409 has some discussion about the use of (or why not) std::aligned_alloc
@Dr15Jones suspected ODR violation, and here is proof
(gdb) where
#0 0x00007ffff4ecc780 in malloc () from /lib64/libc.so.6
#1 0x00007fffb88a17ac in Eigen::PlainObjectBase<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::resize(long, long) () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/external/el8_amd64_gcc12/lib/liblwtnn.so
#2 0x00007fffac85f5a3 in Eigen::PlainObjectBase<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::_init2<long, long> (this=0x7ffffffeec40, rows=2, cols=2)
at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/el8_amd64_gcc12/external/eigen/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e-cf66836cbd6ad8d696d804181e36d61c/include/eigen3/Eigen/src/Core/PlainObjectBase.h:832
#3 0x00007fffac85ca18 in Eigen::Matrix<double, -1, -1, 0, -1, -1>::Matrix<long, long> (this=0x7ffffffeec40, x=@0x7ffffffeebd0: 2, y=@0x7ffffffeebc8: 2) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/el8_amd64_gcc12/external/eigen/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e-cf66836cbd6ad8d696d804181e36d61c/include/eigen3/Eigen/src/Core/Matrix.h:332
#4 0x00007fffac85b7b7 in Eigen::SelfAdjointEigenSolver<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::SelfAdjointEigenSolver<Eigen::Matrix<double, -1, -1, 0, -1, -1> > (this=0x7ffffffeec40, matrix=..., options=128)
at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/el8_amd64_gcc12/external/eigen/3bb6a48d8c171cf20b5f8e48bfb4e424fbd4f79e-cf66836cbd6ad8d696d804181e36d61c/include/eigen3/Eigen/src/Eigenvalues/SelfAdjointEigenSolver.h:176
#5 0x00007fffac857f00 in gbl::GblMeasurement::GblMeasurement (this=0x26ac1c30, aProjection=..., aResiduals=..., aPrecision=..., minPrecision=0.29999999999999999) at /data/cmsbld/jenkins/workspace/ib-run-pr-tests/testBuildDir/BUILD/el8_amd64_gcc12/external/gbl/V02-04-01-5fcbb233a060467b13cbdfca73266fc9/gbl-V02-04-01/cpp/src/GblMeasurement.cpp:54
#6 0x00007fffaca32b35 in void std::__new_allocator<gbl::GblMeasurement>::construct<gbl::GblMeasurement, Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&>(gbl::GblMeasurement*, Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&) [clone .constprop.0] [clone .isra.0] ()
from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#7 0x00007fffaca32c50 in gbl::GblMeasurement& std::vector<gbl::GblMeasurement, std::allocator<gbl::GblMeasurement> >::emplace_back<Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&>(Eigen::MatrixBase<Eigen::Matrix<double, 2, 2, 0, 2, 2> > const&, Eigen::MatrixBase<Eigen::Matrix<double, 2, 1, 0, 2, 1> > const&, Eigen::MatrixBase<Eigen::Inverse<Eigen::Matrix<double, 2, 2, 0, 2, 2> > > const&, double&) [clone .constprop.0] () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#8 0x00007fffaca23aa9 in ReferenceTrajectory::addMaterialEffectsLocalGbl(std::vector<CLHEP::HepMatrix, std::allocator<CLHEP::HepMatrix> > const&, std::vector<CLHEP::HepMatrix, std::allocator<CLHEP::HepMatrix> > const&, std::vector<CLHEP::HepSymMatrix, std::allocator<CLHEP::HepSymMatrix> > const&, std::vector<CLHEP::HepSymMatrix, std::allocator<CLHEP::HepSymMatrix> > const&) () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#9 0x00007fffaca1a6b1 in ReferenceTrajectory::construct(TrajectoryStateOnSurface const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, MagneticField const*, reco::BeamSpot const&) ()
from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#10 0x00007fffaca1b976 in ReferenceTrajectory::ReferenceTrajectory(TrajectoryStateOnSurface const&, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > const&, MagneticField const*, reco::BeamSpot const&, ReferenceTrajectoryBase::Config const&) ()
from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#11 0x00007fffaca2ea07 in TwoBodyDecayTrajectory::construct(TwoBodyDecayTrajectoryState const&, std::pair<std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > > const&, MagneticField const*, reco::BeamSpot const&) () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#12 0x00007fffaca2fec6 in TwoBodyDecayTrajectory::TwoBodyDecayTrajectory(TwoBodyDecayTrajectoryState const&, std::pair<std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > >, std::vector<std::shared_ptr<TrackingRecHit const>, std::allocator<std::shared_ptr<TrackingRecHit const> > > > const&, MagneticField const*, reco::BeamSpot const&, ReferenceTrajectoryBase::Config const&) () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/libAlignmentReferenceTrajectories.so
#13 0x00007fffac2804e3 in TwoBodyDecayTrajectoryFactory::constructTrajectories(std::vector<std::pair<Trajectory const*, reco::Track const*>, std::allocator<std::pair<Trajectory const*, reco::Track const*> > > const&, TwoBodyDecay const&, MagneticField const*, reco::BeamSpot const&, bool) const ()
from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentReferenceTrajectoriesPlugins.so
#14 0x00007fffac27db6f in TwoBodyDecayTrajectoryFactory::trajectories(edm::EventSetup const&, std::vector<std::pair<Trajectory const*, reco::Track const*>, std::allocator<std::pair<Trajectory const*, reco::Track const*> > > const&, reco::BeamSpot const&) const ()
from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentReferenceTrajectoriesPlugins.so
#15 0x00007fffac49f76e in MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) () from /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so
#16 0x00007fffad24147f in AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so
#17 0x00007ffff7cd65ce in edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) () from /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so
The plain malloc() calls via symbols in lwtnn.so!
I suppose we need to add something along-DEIGEN_MAX_ALIGN_BYTES=64 -msse3 to all externals that use Eigen. Based on a quick git grep in cmsdist that would be
-
lwtnn -
opencv -
professor2(doesn't seem to be linked against in CMSSW, but maybe better to include anyway) -
pytorch -
tensorflow(already has it)
I'm testing more consistent build settings in https://github.com/cms-sw/cmsdist/pull/9043
I also tested LD_PRELOAD=.../libGBL.so with cmsRunGlibC. The job proceeded beyond the earlier crash point, but still crashed. Valgrind showed
==28159== Invalid read of size 4
==28159== at 0x4D8D967: gbl::GblTrajectory::prepare() (GblTrajectory.cpp:1266)
==28159== by 0x4D8A6DF: gbl::GblTrajectory::construct() (GblTrajectory.cpp:459)
==28159== by 0x6B89418D: MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==28159== by 0x6B8947DF: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==28159== by 0x61FCB47E: AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==28159== by 0x50EE5CD: edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x50D8E6D: edm::WorkerT<edm::one::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x506B5F8: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x506C6CE: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x421A1CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==28159== by 0x6BD5280: UnknownInlinedFun (task_dispatcher.h:322)
==28159== by 0x6BD5280: UnknownInlinedFun (task_dispatcher.h:458)
==28159== by 0x6BD5280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==28159== by 0x4FEECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x4FF8649: edm::EventProcessor::processRuns() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x4FF8BA0: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x4074EE: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRunGlibC)
==28159== by 0x6BC19AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==28159== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRunGlibC)
==28159== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRunGlibC)
==28159== Address 0x63daf118 is 0 bytes after a block of size 40 alloc'd
==28159== at 0x403AED1: operator new(unsigned long) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==28159== by 0x4D9E581: std::__new_allocator<std::array<unsigned int, 5ul> >::allocate(unsigned long, void const*) (new_allocator.h:137)
==28159== by 0x4D9CC13: std::allocator_traits<std::allocator<std::array<unsigned int, 5ul> > >::allocate(std::allocator<std::array<unsigned int, 5ul> >&, unsigned long) (alloc_traits.h:464)
==28159== by 0x4D9ADD1: std::_Vector_base<std::array<unsigned int, 5ul>, std::allocator<std::array<unsigned int, 5ul> > >::_M_allocate(unsigned long) (stl_vector.h:378)
==28159== by 0x4D97B13: void std::vector<std::array<unsigned int, 5ul>, std::allocator<std::array<unsigned int, 5ul> > >::_M_realloc_insert<std::array<unsigned int, 5ul> const&>(__gnu_cxx::__normal_iterator<std::array<unsigned int, 5ul>*, std::vector<std::array<unsigned int, 5ul>, std::allocator<std::array<unsigned int, 5ul> > > >, std::array<unsigned int, 5ul> const&) (vector.tcc:453)
==28159== by 0x4D931FF: std::vector<std::array<unsigned int, 5ul>, std::allocator<std::array<unsigned int, 5ul> > >::push_back(std::array<unsigned int, 5ul> const&) (stl_vector.h:1287)
==28159== by 0x4D8D48D: gbl::GblTrajectory::prepare() (GblTrajectory.cpp:1208)
==28159== by 0x4D8A6DF: gbl::GblTrajectory::construct() (GblTrajectory.cpp:459)
==28159== by 0x6B89418D: MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==28159== by 0x6B8947DF: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==28159== by 0x61FCB47E: AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==28159== by 0x50EE5CD: edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x50D8E6D: edm::WorkerT<edm::one::EDProducerBase>::implDo(edm::EventTransitionInfo const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x506B5F8: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x506C6CE: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::EventPrincipal, (edm::BranchActionType)1> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x421A1CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==28159== by 0x6BD5280: UnknownInlinedFun (task_dispatcher.h:322)
==28159== by 0x6BD5280: UnknownInlinedFun (task_dispatcher.h:458)
==28159== by 0x6BD5280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==28159== by 0x4FEECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x4FF8649: edm::EventProcessor::processRuns() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x4FF8BA0: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
and
==28159== Invalid read of size 4
==28159== at 0x4D8E039: gbl::GblTrajectory::prepare() (GblTrajectory.cpp:1344)
==28159== by 0x4D8A6DF: gbl::GblTrajectory::construct() (GblTrajectory.cpp:459)
(the rest was the same as in above)
The corresponding code are
while (innerTransLab[iTraj][ifirst]
!= labDer[ilabel]
and ifirst <= ilast) {
++ifirst;
}
https://gitlab.desy.de/claus.kleinwort/general-broken-lines/-/blob/31e726d777fe93cdbed0c363dc15f803f7767f40/cpp/src/GblTrajectory.cpp#L1265-1269 https://gitlab.desy.de/claus.kleinwort/general-broken-lines/-/blob/31e726d777fe93cdbed0c363dc15f803f7767f40/cpp/src/GblTrajectory.cpp#L1343-1346
I suspect the problem is the loop first de-referencing innerTransLab[iTraj][ifirst] and only then checking the index validity ifirst <= ilast.
The crash itself came from
==28159== Invalid read of size 8
==28159== at 0x5FDE2D9: TFree::GetBestFree(TList*, int) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libRIO.so)
==28159== by 0x601DC33: TKey::Create(int, TFile*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libRIO.so)
==28159== by 0x5A0E99D: TBasket::WriteBuffer() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libTree.so)
==28159== by 0x5A1E257: std::_Function_handler<void (), ROOT::Internal::TBranchIMTHelper::Run<TBranch::WriteBasketImpl(TBasket*, int, ROOT::Internal::TBranchIMTHelper*)::{lambda()#1}>(TBranch::WriteBasketImpl(TBasket*, int, ROOT::Internal::TBranchIMTHelper*)::{lambda()#1} const&)::{lambda()#1}>::_M_invoke(std::_Any_data const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libTree.so)
==28159== by 0x7E7560D: tbb::detail::d1::function_task<std::function<void ()> >::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libImt.so)
==28159== by 0x6BD5280: UnknownInlinedFun (task_dispatcher.h:322)
==28159== by 0x6BD5280: UnknownInlinedFun (task_dispatcher.h:458)
==28159== by 0x6BD5280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==28159== by 0x7E75398: ROOT::Experimental::TTaskGroup::Wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libImt.so)
==28159== by 0x5A97E65: TTree::Fill() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libTree.so)
==28159== by 0x674F0323: AlignableDataIORoot::writeAbsRaw(AlignableData<Point3DBase<double, GlobalTag> > const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentAlgorithm.so)
==28159== by 0x674F0E45: AlignableDataIO::writeOrgPos(Alignable*, bool) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentAlgorithm.so)
==28159== by 0x674F0F41: AlignableDataIO::writeOrgPos(std::vector<Alignable*, std::allocator<Alignable*> > const&, bool) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentAlgorithm.so)
==28159== by 0x674F7E6F: AlignmentIORoot::writeAlignableOriginalPositions(std::vector<Alignable*, std::allocator<Alignable*> > const&, char const*, int, bool, int&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentAlgorithm.so)
==28159== by 0x6B8959C2: MillePedeAlignmentAlgorithm::doIO(int) const (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==28159== by 0x6B895FC8: MillePedeAlignmentAlgorithm::terminate() (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==28159== by 0x61FC55B2: AlignmentProducerBase::terminateProcessing(edm::EventSetup const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==28159== by 0x6B84A171: AlignmentProducerAsAnalyzer::endProcessBlockProduce(edm::ProcessBlock&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentCommonAlignmentProducerAsAnalyzerPlugin.so)
==28159== by 0x50EECAA: edm::one::EDProducerBase::doEndProcessBlock(edm::ProcessBlockPrincipal const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x50D909F: edm::WorkerT<edm::one::EDProducerBase>::implDoEndProcessBlock(edm::ProcessBlockPrincipal const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x502CF21: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::ProcessBlockPrincipal, (edm::BranchActionType)3> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::ProcessBlockPrincipal, (edm::BranchActionType)3>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::ProcessBlockPrincipal, (edm::BranchActionType)3>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x50307ED: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::ProcessBlockPrincipal, (edm::BranchActionType)3> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::ProcessBlockPrincipal, (edm::BranchActionType)3> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== Address 0x18 is not stack'd, malloc'd or (recently) free'd
==28159==
A fatal system signal has occurred: segmentation violation
The following is the call stack containing the origin of the signal.
Thread 1: status = VgTs_WaitSys syscall 7 (lwpid 28159)
==28159== at 0x7BC9301: poll (in /usr/lib64/libc-2.28.so)
==28159== by 0xEFC670E: full_read.constprop.0 (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so)
==28159== by 0xEF7ADCB: edm::service::InitRootHandlers::stacktraceFromThread() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so)
==28159== by 0xEF7B72F: sig_dostack_then_abort (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginFWCoreServicesPlugins.so)
==28159== by 0x7897CEF: ??? (in /usr/lib64/libpthread-2.28.so)
==28159== by 0x5FDE2D8: TFree::GetBestFree(TList*, int) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libRIO.so)
==28159== by 0x4868: ???
==28159== by 0x601DC33: TKey::Create(int, TFile*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libRIO.so)
==28159== by 0x5A0E99D: TBasket::WriteBuffer() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libTree.so)
==28159== by 0x5A1E257: std::_Function_handler<void (), ROOT::Internal::TBranchIMTHelper::Run<TBranch::WriteBasketImpl(TBasket*, int, ROOT::Internal::TBranchIMTHelper*)::{lambda()#1}>(TBranch::WriteBasketImpl(TBasket*, int, ROOT::Internal::TBranchIMTHelper*)::{lambda()#1} const&)::{lambda()#1}>::_M_invoke(std::_Any_data const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libTree.so)
==28159== by 0x7E7560D: tbb::detail::d1::function_task<std::function<void ()> >::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libImt.so)
==28159== by 0x6BD5280: UnknownInlinedFun (task_dispatcher.h:322)
==28159== by 0x6BD5280: UnknownInlinedFun (task_dispatcher.h:458)
==28159== by 0x6BD5280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==28159== by 0x7E75398: ROOT::Experimental::TTaskGroup::Wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libImt.so)
==28159== by 0x5A97E65: TTree::Fill() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/lcg/root/6.30.05-09dd1c56883e4f434d3cfa5dffc18d6e/lib/libTree.so)
==28159== by 0x674F0323: AlignableDataIORoot::writeAbsRaw(AlignableData<Point3DBase<double, GlobalTag> > const&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentAlgorithm.so)
==28159== by 0x674F0E45: AlignableDataIO::writeOrgPos(Alignable*, bool) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentAlgorithm.so)
==28159== by 0x674F0F41: AlignableDataIO::writeOrgPos(std::vector<Alignable*, std::allocator<Alignable*> > const&, bool) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentAlgorithm.so)
==28159== by 0x674F7E6F: AlignmentIORoot::writeAlignableOriginalPositions(std::vector<Alignable*, std::allocator<Alignable*> > const&, char const*, int, bool, int&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentAlgorithm.so)
==28159== by 0x6B8959C2: MillePedeAlignmentAlgorithm::doIO(int) const (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==28159== by 0x6B895FC8: MillePedeAlignmentAlgorithm::terminate() (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-27-2300/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==28159== by 0x61FC55B2: AlignmentProducerBase::terminateProcessing(edm::EventSetup const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==28159== by 0x6B84A171: AlignmentProducerAsAnalyzer::endProcessBlockProduce(edm::ProcessBlock&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/pluginAlignmentCommonAlignmentProducerAsAnalyzerPlugin.so)
==28159== by 0x50EECAA: edm::one::EDProducerBase::doEndProcessBlock(edm::ProcessBlockPrincipal const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x50D909F: edm::WorkerT<edm::one::EDProducerBase>::implDoEndProcessBlock(edm::ProcessBlockPrincipal const&, edm::ModuleCallingContext const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x502CF21: std::__exception_ptr::exception_ptr edm::Worker::runModuleAfterAsyncPrefetch<edm::OccurrenceTraits<edm::ProcessBlockPrincipal, (edm::BranchActionType)3> >(std::__exception_ptr::exception_ptr, edm::OccurrenceTraits<edm::ProcessBlockPrincipal, (edm::BranchActionType)3>::TransitionInfoType const&, edm::StreamID, edm::ParentContext const&, edm::OccurrenceTraits<edm::ProcessBlockPrincipal, (edm::BranchActionType)3>::Context const*) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x50307ED: edm::SerialTaskQueue::QueuedTask<edm::SerialTaskQueueChain::push<edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::ProcessBlockPrincipal, (edm::BranchActionType)3> >::execute()::{lambda()#1}&>(tbb::detail::d1::task_group&, edm::Worker::RunModuleTask<edm::OccurrenceTraits<edm::ProcessBlockPrincipal, (edm::BranchActionType)3> >::execute()::{lambda()#1}&)::{lambda()#1}>::execute() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x421A1CF: tbb::detail::d1::function_task<edm::SerialTaskQueue::spawn(edm::SerialTaskQueue::TaskBase&)::{lambda()#1}>::execute(tbb::detail::d1::execution_data&) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreConcurrency.so)
==28159== by 0x6BD5280: UnknownInlinedFun (task_dispatcher.h:322)
==28159== by 0x6BD5280: UnknownInlinedFun (task_dispatcher.h:458)
==28159== by 0x6BD5280: tbb::detail::r1::task_dispatcher::execute_and_wait(tbb::detail::d1::task*, tbb::detail::d1::wait_context&, tbb::detail::d1::task_group_context&) (task_dispatcher.cpp:168)
==28159== by 0x4FEECDA: edm::FinalWaitingTask::wait() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x4FF3CAA: edm::EventProcessor::endProcessBlock(bool, bool) (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x4FF8C80: edm::EventProcessor::runToCompletion() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/lib/el8_amd64_gcc12/libFWCoreFramework.so)
==28159== by 0x4074EE: tbb::detail::d1::task_arena_function<main::{lambda()#1}::operator()() const::{lambda()#1}, void>::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRunGlibC)
==28159== by 0x6BC19AC: tbb::detail::r1::task_arena_impl::execute(tbb::detail::d1::task_arena_base&, tbb::detail::d1::delegate_base&) (arena.cpp:688)
==28159== by 0x408ED1: main::{lambda()#1}::operator()() const (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRunGlibC)
==28159== by 0x40517B: main (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/cms/cmssw/CMSSW_14_1_X_2024-02-26-2300/bin/el8_amd64_gcc12/cmsRunGlibC)
Hmh, building the lwtnn, opencv, and pytorch with -DEIGEN_MAX_ALIGN_BYTES=64 -msse3 in https://github.com/cms-sw/cmsdist/pull/9043 didn't make a difference, the job still fails (log from valgring + cmsRunGlibC)
==4590== Invalid read of size 8
==4590== at 0x6AB59B80: gbl::GblTrajectory::prepare() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4590== by 0x6AB5C1BF: gbl::GblTrajectory::construct() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4590== by 0x6E41F18D: MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-29-1100/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4590== by 0x6E41F7DF: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-29-1100/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4590== by 0x648E547E: AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-29-1100/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==4590== by 0x4C6E49D: edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-29-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so)
...
==4590== Address 0x7d9fd558 is 8 bytes before a block of size 144 alloc'd
==4590== at 0x403A76B: malloc (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==4590== by 0x537C17AB: Eigen::PlainObjectBase<Eigen::Matrix<double, -1, -1, 0, -1, -1> >::resize(long, long) (in /cvmfs/cms-ci.cern.ch/week0/PR_0c0a8c3d/el8_amd64_gcc12/external/lwtnn/2.13-dc060310f9f71ab2d7dcf18f48a8216e/lib/liblwtnn.so)
==4590== by 0x6AB595D2: gbl::GblTrajectory::prepare() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4590== by 0x6AB5C1BF: gbl::GblTrajectory::construct() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4590== by 0x6E41F18D: MillePedeAlignmentAlgorithm::addReferenceTrajectory(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&, ReferenceCountingPointer<ReferenceTrajectoryBase> const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-29-1100/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4590== by 0x6E41F7DF: MillePedeAlignmentAlgorithm::run(edm::EventSetup const&, AlignmentAlgorithmBase::EventInfo const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-29-1100/lib/el8_amd64_gcc12/pluginAlignmentMillePedeAlignmentAlgorithmAuto.so)
==4590== by 0x648E547E: AlignmentProducerBase::processEvent(edm::Event const&, edm::EventSetup const&) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-29-1100/lib/el8_amd64_gcc12/libAlignmentCommonAlignmentProducer.so)
==4590== by 0x4C6E49D: edm::one::EDProducerBase::doEvent(edm::EventTransitionInfo const&, edm::ActivityRegistry*, edm::ModuleCallingContext const*) (in /build/mkortela/debug/el8/issue44188/CMSSW_14_1_X_2024-02-29-1100/lib/el8_amd64_gcc12/libFWCoreFramework.so)
...
==4590== Invalid free() / delete / delete[] / realloc()
==4590== at 0x403D9E4: free (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/valgrind/3.22.0-e989c977f8949fc48f1bc1206f1f87c8/libexec/valgrind/vgpreload_memcheck-amd64-linux.so)
==4590== by 0x6AB59B89: gbl::GblTrajectory::prepare() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
==4590== by 0x6AB5C1BF: gbl::GblTrajectory::construct() (in /cvmfs/cms-ib.cern.ch/sw/x86_64/nweek-02826/el8_amd64_gcc12/external/gbl/V02-04-01-262bd150f85b6d7f92432b0f015e0445/lib/libGBL.so.2.4.1)
I suspect a flaw in my build setup
I suspect a flaw in my build setup
Looking at the build log of lwtnn https://cmssdt.cern.ch/SDT/jenkins-artifacts/pull-request-integration/PR-9dec1c/37832/externals/lwtnn/2.13-dc060310f9f71ab2d7dcf18f48a8216e/log it's clear that the -DEIGEN_MAX_ALIGN_BYTES=64 -msse3 did not propagate to the actual compilation.