chroma
chroma copied to clipboard
Published
20 hours ago •
chroma-core
chroma-core
[Bug]: RuntimeError: dictionary changed size during iteration
What happened?
I'm doing some batch embeddings creations and get a lot of these exceptions
Versions
0.3.22.dev17
Relevant log output
2023-04-17 11:23:45 2023-04-17 18:23:45 ERROR chromadb.server.fastapi dictionary changed size during iteration
2023-04-17 11:23:45 Traceback (most recent call last):
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/anyio/streams/memory.py", line 94, in receive
2023-04-17 11:23:45 return self.receive_nowait()
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/anyio/streams/memory.py", line 89, in receive_nowait
2023-04-17 11:23:45 raise WouldBlock
2023-04-17 11:23:45 anyio.WouldBlock
2023-04-17 11:23:45
2023-04-17 11:23:45 During handling of the above exception, another exception occurred:
2023-04-17 11:23:45
2023-04-17 11:23:45 Traceback (most recent call last):
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/middleware/base.py", line 43, in call_next
2023-04-17 11:23:45 message = await recv_stream.receive()
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/anyio/streams/memory.py", line 114, in receive
2023-04-17 11:23:45 raise EndOfStream
2023-04-17 11:23:45 anyio.EndOfStream
2023-04-17 11:23:45
2023-04-17 11:23:45 During handling of the above exception, another exception occurred:
2023-04-17 11:23:45
2023-04-17 11:23:45 Traceback (most recent call last):
2023-04-17 11:23:45 File "/chroma/./chromadb/server/fastapi/__init__.py", line 47, in catch_exceptions_middleware
2023-04-17 11:23:45 return await call_next(request)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/middleware/base.py", line 46, in call_next
2023-04-17 11:23:45 raise app_exc
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/middleware/base.py", line 36, in coro
2023-04-17 11:23:45 await self.app(scope, request.receive, send_stream.send)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 75, in __call__
2023-04-17 11:23:45 raise exc
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 64, in __call__
2023-04-17 11:23:45 await self.app(scope, receive, sender)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__
2023-04-17 11:23:45 raise e
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
2023-04-17 11:23:45 await self.app(scope, receive, send)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 680, in __call__
2023-04-17 11:23:45 await route.handle(scope, receive, send)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 275, in handle
2023-04-17 11:23:45 await self.app(scope, receive, send)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 65, in app
2023-04-17 11:23:45 response = await func(request)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/fastapi/routing.py", line 231, in app
2023-04-17 11:23:45 raw_response = await run_endpoint_function(
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/fastapi/routing.py", line 162, in run_endpoint_function
2023-04-17 11:23:45 return await run_in_threadpool(dependant.call, **values)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/concurrency.py", line 41, in run_in_threadpool
2023-04-17 11:23:45 return await anyio.to_thread.run_sync(func, *args)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/anyio/to_thread.py", line 31, in run_sync
2023-04-17 11:23:45 return await get_asynclib().run_sync_in_worker_thread(
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 937, in run_sync_in_worker_thread
2023-04-17 11:23:45 return await future
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 867, in run
2023-04-17 11:23:45 result = context.run(func, *args)
2023-04-17 11:23:45 File "/chroma/./chromadb/server/fastapi/__init__.py", line 158, in add
2023-04-17 11:23:45 result = self._api._add(
2023-04-17 11:23:45 File "/chroma/./chromadb/api/local.py", line 140, in _add
2023-04-17 11:23:45 self._db.add_incremental(collection_uuid, added_uuids, embeddings)
2023-04-17 11:23:45 File "/chroma/./chromadb/db/clickhouse.py", line 542, in add_incremental
2023-04-17 11:23:45 index.add(uuids, embeddings)
2023-04-17 11:23:45 File "/chroma/./chromadb/db/index/hnswlib.py", line 148, in add
2023-04-17 11:23:45 self._save()
2023-04-17 11:23:45 File "/chroma/./chromadb/db/index/hnswlib.py", line 187, in _save
2023-04-17 11:23:45 pickle.dump(self._label_to_id, f, pickle.HIGHEST_PROTOCOL)
2023-04-17 11:23:45 RuntimeError: dictionary changed size during iteration
2023-04-17 11:23:45 2023-04-17 18:23:45 INFO uvicorn.access 172.18.0.1:48976 - "POST /api/v1/collections/mimessage-embeddings/add HTTP/1.1" 500
2023-04-17 11:23:45 2023-04-17 18:23:45 ERROR chromadb.server.fastapi dictionary changed size during iteration
2023-04-17 11:23:45 Traceback (most recent call last):
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/anyio/streams/memory.py", line 94, in receive
2023-04-17 11:23:45 return self.receive_nowait()
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/anyio/streams/memory.py", line 89, in receive_nowait
2023-04-17 11:23:45 raise WouldBlock
2023-04-17 11:23:45 anyio.WouldBlock
2023-04-17 11:23:45
2023-04-17 11:23:45 During handling of the above exception, another exception occurred:
2023-04-17 11:23:45
2023-04-17 11:23:45 Traceback (most recent call last):
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/middleware/base.py", line 43, in call_next
2023-04-17 11:23:45 message = await recv_stream.receive()
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/anyio/streams/memory.py", line 114, in receive
2023-04-17 11:23:45 raise EndOfStream
2023-04-17 11:23:45 anyio.EndOfStream
2023-04-17 11:23:45
2023-04-17 11:23:45 During handling of the above exception, another exception occurred:
2023-04-17 11:23:45
2023-04-17 11:23:45 Traceback (most recent call last):
2023-04-17 11:23:45 File "/chroma/./chromadb/server/fastapi/__init__.py", line 47, in catch_exceptions_middleware
2023-04-17 11:23:45 return await call_next(request)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/middleware/base.py", line 46, in call_next
2023-04-17 11:23:45 raise app_exc
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/middleware/base.py", line 36, in coro
2023-04-17 11:23:45 await self.app(scope, request.receive, send_stream.send)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 75, in __call__
2023-04-17 11:23:45 raise exc
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 64, in __call__
2023-04-17 11:23:45 await self.app(scope, receive, sender)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__
2023-04-17 11:23:45 raise e
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
2023-04-17 11:23:45 await self.app(scope, receive, send)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 680, in __call__
2023-04-17 11:23:45 await route.handle(scope, receive, send)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 275, in handle
2023-04-17 11:23:45 await self.app(scope, receive, send)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 65, in app
2023-04-17 11:23:45 response = await func(request)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/fastapi/routing.py", line 231, in app
2023-04-17 11:23:45 raw_response = await run_endpoint_function(
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/fastapi/routing.py", line 162, in run_endpoint_function
2023-04-17 11:23:45 return await run_in_threadpool(dependant.call, **values)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/starlette/concurrency.py", line 41, in run_in_threadpool
2023-04-17 11:23:45 return await anyio.to_thread.run_sync(func, *args)
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/anyio/to_thread.py", line 31, in run_sync
2023-04-17 11:23:45 return await get_asynclib().run_sync_in_worker_thread(
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 937, in run_sync_in_worker_thread
2023-04-17 11:23:45 return await future
2023-04-17 11:23:45 File "/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 867, in run
2023-04-17 11:23:45 result = context.run(func, *args)
2023-04-17 11:23:45 File "/chroma/./chromadb/server/fastapi/__init__.py", line 158, in add
2023-04-17 11:23:45 result = self._api._add(
2023-04-17 11:23:45 File "/chroma/./chromadb/api/local.py", line 140, in _add
2023-04-17 11:23:45 self._db.add_incremental(collection_uuid, added_uuids, embeddings)
2023-04-17 11:23:45 File "/chroma/./chromadb/db/clickhouse.py", line 542, in add_incremental
2023-04-17 11:23:45 index.add(uuids, embeddings)
2023-04-17 11:23:45 File "/chroma/./chromadb/db/index/hnswlib.py", line 148, in add
2023-04-17 11:23:45 self._save()
2023-04-17 11:23:45 File "/chroma/./chromadb/db/index/hnswlib.py", line 187, in _save
2023-04-17 11:23:45 pickle.dump(self._label_to_id, f, pickle.HIGHEST_PROTOCOL)
2023-04-17 11:23:45 RuntimeError: dictionary changed size during iteration
