chroma
chroma copied to clipboard
Published
20 hours ago •
chroma-core
chroma-core
foyer 0.14.1 panics on cache read
On commit 96243fd4625d269e14b3207f274a6abe0162e055, a call to the query service in chromadb/test/property/test_add.py occasionally flakes. The query service logs a panic on these flakes:
2025-03-20T03:11:46.295961Z ERROR A panic occurred, panic.payload: "called `Result::unwrap()` on an `Err` value: JoinError::Panic(Id(16043), \"range end index 11086 out of range for slice of length 10918\", ...)", panic.location: "/usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/foyer-storage-0.14.1/src/store.rs:161:61", panic.backtrace: 0: chroma_tracing::init_tracer::init_panic_tracing_hook::{{closure}}
at ./chroma/rust/tracing/src/init_tracer.rs:155:55
1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2084:9
2: std::panicking::rust_panic_with_hook
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:808:13
3: std::panicking::begin_panic_handler::{{closure}}
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:674:13
4: std::sys::backtrace::__rust_end_short_backtrace
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:168:18
5: rust_begin_unwind
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:665:5
6: core::panicking::panic_fmt
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panicking.rs:74:14
7: core::result::unwrap_failed
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/result.rs:1679:5
8: core::result::Result<T,E>::unwrap
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/result.rs:1102:23
9: foyer_storage::store::Store<K,V,S>::load::{{closure}}
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/foyer-storage-0.14.1/src/store.rs:161:15
10: <fastrace::future::InSpan<T> as core::future::future::Future>::poll
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/fastrace-0.7.4/src/future.rs:124:19
11: foyer::hybrid::cache::HybridCache<K,V,S>::get::{{closure}}
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/foyer-0.14.1/src/hybrid/cache.rs:283:14
12: <chroma_cache::foyer::FoyerHybridCache<K,V> as chroma_cache::Cache<K,V>>::get::{{closure}}
at ./chroma/rust/cache/src/foyer.rs:457:39
13: <core::pin::Pin<P> as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
14: chroma_blockstore::arrow::provider::BlockManager::get::{{closure}}
at ./chroma/rust/blockstore/src/arrow/provider.rs:323:46
15: chroma_blockstore::arrow::blockfile::ArrowBlockfileReader<K,V>::get_block::{{closure}}
at ./chroma/rust/blockstore/src/arrow/blockfile.rs:386:65
16: chroma_blockstore::arrow::blockfile::ArrowBlockfileReader<K,V>::get::{{closure}}
at ./chroma/rust/blockstore/src/arrow/blockfile.rs:462:53
17: chroma_blockstore::types::reader::BlockfileReader<K,V>::get::{{closure}}
at ./chroma/rust/blockstore/src/types/reader.rs:37:86
18: chroma_segment::blockfile_record::RecordSegmentReader::get_data_for_offset_id::{{closure}}
at ./chroma/rust/segment/src/blockfile_record.rs:799:44
19: <worker::execution::operators::projection::ProjectionOperator as chroma_system::execution::operator::Operator<worker::execution::operators::projection::ProjectionInput,worker::execution::operators::projection::ProjectionOutput>>::run::{{closure}}
at ./chroma/rust/worker/src/execution/operators/projection.rs:153:30
20: <core::pin::Pin<P> as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
21: <worker::execution::operators::knn_projection::KnnProjectionOperator as chroma_system::execution::operator::Operator<worker::execution::operators::knn_projection::KnnProjectionInput,worker::execution::operators::knn_projection::KnnProjectionOutput>>::run::{{closure}}
at ./chroma/rust/worker/src/execution/operators/knn_projection.rs:97:61
22: <core::pin::Pin<P> as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
23: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:297:9
24: <futures_util::future::future::catch_unwind::CatchUnwind<Fut> as core::future::future::Future>::poll::{{closure}}
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/future/future/catch_unwind.rs:37:42
25: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9
26: std::panicking::try::do_call
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40
27: __rust_try
28: std::panicking::try
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19
29: std::panic::catch_unwind
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14
30: <futures_util::future::future::catch_unwind::CatchUnwind<Fut> as core::future::future::Future>::poll
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/future/future/catch_unwind.rs:37:9
31: <chroma_system::execution::operator::Task<Input,Output,Error> as chroma_system::execution::operator::TaskWrapper>::run::{{closure}}
at ./chroma/rust/system/src/execution/operator.rs:146:14
32: <core::pin::Pin<P> as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
33: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9
34: <chroma_system::execution::worker_thread::WorkerThread as chroma_system::types::Handler<alloc::boxed::Box<dyn chroma_system::execution::operator::TaskWrapper>>>::handle::{{closure}}
at ./chroma/rust/system/src/execution/worker_thread.rs:62:43
35: <core::pin::Pin<P> as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
36: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:297:9
37: <futures_util::future::future::catch_unwind::CatchUnwind<Fut> as core::future::future::Future>::poll::{{closure}}
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/future/future/catch_unwind.rs:37:42
38: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9
39: std::panicking::try::do_call
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40
40: __rust_try
41: std::panicking::try
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19
42: std::panic::catch_unwind
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14
43: <futures_util::future::future::catch_unwind::CatchUnwind<Fut> as core::future::future::Future>::poll
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.31/src/future/future/catch_unwind.rs:37:9
44: <core::option::Option<chroma_system::wrapped_message::HandleableMessageImpl<M,<C as chroma_system::types::Handler<M>>::Result>> as chroma_system::wrapped_message::HandleableMessage<C>>::handle_and_reply::{{closure}}
at ./chroma/rust/system/src/wrapped_message.rs:81:18
45: <core::pin::Pin<P> as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
46: chroma_system::wrapped_message::WrappedMessage<C>::handle::{{closure}}
at ./chroma/rust/system/src/wrapped_message.rs:55:61
47: <tracing::instrument::Instrumented<T> as core::future::future::Future>::poll
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tracing-0.1.40/src/instrument.rs:321:9
48: chroma_system::executor::ComponentExecutor<C>::run::{{closure}}
at ./chroma/rust/system/src/executor.rs:98:64
49: chroma_system::system::System::start_component::{{closure}}::{{closure}}
at ./chroma/rust/system/src/system.rs:67:63
50: <core::pin::Pin<P> as core::future::future::Future>::poll
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/future/future.rs:123:9
51: tokio::runtime::scheduler::current_thread::CoreGuard::block_on::{{closure}}::{{closure}}::{{closure}}
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/scheduler/current_thread/mod.rs:729:57
52: tokio::runtime::coop::with_budget
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/coop.rs:107:5
--
63: tokio::runtime::scheduler::current_thread::CoreGuard::enter
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/scheduler/current_thread/mod.rs:807:27
64: tokio::runtime::scheduler::current_thread::CoreGuard::block_on
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/scheduler/current_thread/mod.rs:716:19
65: tokio::runtime::scheduler::current_thread::CurrentThread::block_on::{{closure}}
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/scheduler/current_thread/mod.rs:196:28
66: tokio::runtime::context::runtime::enter_runtime
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/context/runtime.rs:65:16
67: tokio::runtime::scheduler::current_thread::CurrentThread::block_on
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/scheduler/current_thread/mod.rs:184:9
68: tokio::runtime::runtime::Runtime::block_on_inner
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/runtime.rs:368:47
69: tokio::runtime::runtime::Runtime::block_on
at ./usr/local/cargo/registry/src/index.crates.io-6f17d22bba15001f/tokio-1.41.1/src/runtime/runtime.rs:342:13
70: chroma_system::system::System::start_component::{{closure}}
at ./chroma/rust/system/src/system.rs:67:21
71: std::sys::backtrace::__rust_begin_short_backtrace
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/backtrace.rs:152:18
72: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:538:17
73: <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/panic/unwind_safe.rs:272:9
74: std::panicking::try::do_call
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:557:40
75: __rust_try
76: std::panicking::try
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panicking.rs:521:19
77: std::panic::catch_unwind
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/panic.rs:350:14
78: std::thread::Builder::spawn_unchecked_::{{closure}}
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/thread/mod.rs:537:30
79: core::ops::function::FnOnce::call_once{{vtable.shim}}
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/core/src/ops/function.rs:250:5
80: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9
81: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/alloc/src/boxed.rs:2070:9
82: std::sys::pal::unix::thread::Thread::new::thread_start
at ./rustc/eeb90cda1969383f56a2637cbd3037bdf598841c/library/std/src/sys/pal/unix/thread.rs:108:17
We believe this is because we manually insert to both the disk and in-memory cache on cache writes.
