[QUESTION]: load failed with "collection has not been loaded into QueryNode"

[sugia]

Is there an existing issue for this?

• [X] I have searched the existing issues

What is your question?

Hi team, I tried to do a load in a collection with 16 partitions, and it gets error: "collection has not been loaded into QueryNode". Also tried to load each partition individually, and they get the same error: "collection has not been loaded into QueryNode". There are 8 partitions each contains about 550,000 entities. And 8 partitions each contains about 110,000 entities, for a total of about 5,000,000 entities.

Anything else?

No response

[csnelsonchu]

I have a similar issues. I have 660,000 entities in my standalone. Seems like every time it get to a larger size it can no longer search any more. It's giving me the following error......

---

RPC error: [wait_for_loading_collection], <BaseException: (code=1, message=collection 430757187117383681 has not been loaded to memory or load failed)>, <Time:{'RPC start': '2022-01-27 12:01:10.776524', 'RPC error': '2022-01-27 12:01:11.282246'}> RPC error: [load_collection], <BaseException: (code=1, message=collection 430757187117383681 has not been loaded to memory or load failed)>, <Time:{'RPC start': '2022-01-27 12:01:10.760610', 'RPC error': '2022-01-27 12:01:11.282924'}>

BaseException Traceback (most recent call last) Input In [5], in 4 connections.connect(host=milvus_host, port=milvus_port) 5 collection = Collection("semantic_search_title_v11") ----> 6 collection.load()

File /opt/conda/lib/python3.9/site-packages/pymilvus/orm/collection.py:457, in Collection.load(self, partition_names, timeout, **kwargs) 455 conn.load_partitions(self._name, partition_names, timeout=timeout, **kwargs) 456 else: --> 457 conn.load_collection(self._name, timeout=timeout, **kwargs)

File /opt/conda/lib/python3.9/site-packages/pymilvus/decorators.py:56, in retry_on_rpc_failure..wrapper..handler(self, *args, **kwargs) 54 time.sleep(wait) 55 except Exception as e: ---> 56 raise e 57 finally: 58 counter += 1

File /opt/conda/lib/python3.9/site-packages/pymilvus/decorators.py:41, in retry_on_rpc_failure..wrapper..handler(self, *args, **kwargs) 39 while True: 40 try: ---> 41 return func(self, *args, **kwargs) 42 except grpc.RpcError as e: 43 # DEADLINE_EXCEEDED means that the task wat not completed 44 # UNAVAILABLE means that the service is not reachable currently 45 # Reference: https://grpc.github.io/grpc/python/grpc.html#grpc-status-code 46 if e.code() != grpc.StatusCode.DEADLINE_EXCEEDED and e.code() != grpc.StatusCode.UNAVAILABLE:

File /opt/conda/lib/python3.9/site-packages/pymilvus/decorators.py:74, in error_handler..handler(*args, **kwargs) 72 record_dict["RPC error"] = str(datetime.datetime.now()) 73 LOGGER.error(f"RPC error: [{func.name}], {e}, Time:{record_dict}") ---> 74 raise e 75 except grpc.FutureTimeoutError as e: 76 record_dict["RPC timeout"] = str(datetime.datetime.now())

File /opt/conda/lib/python3.9/site-packages/pymilvus/decorators.py:70, in error_handler..handler(*args, **kwargs) 68 try: 69 record_dict["RPC start"] = str(datetime.datetime.now()) ---> 70 return func(*args, **kwargs) 71 except BaseException as e: 72 record_dict["RPC error"] = str(datetime.datetime.now())

File /opt/conda/lib/python3.9/site-packages/pymilvus/client/grpc_handler.py:656, in GrpcHandler.load_collection(self, collection_name, timeout, **kwargs) 654 _async = kwargs.get("_async", False) 655 if not _async: --> 656 self.wait_for_loading_collection(collection_name, timeout)

File /opt/conda/lib/python3.9/site-packages/pymilvus/decorators.py:56, in retry_on_rpc_failure..wrapper..handler(self, *args, **kwargs) 54 time.sleep(wait) 55 except Exception as e: ---> 56 raise e 57 finally: 58 counter += 1

File /opt/conda/lib/python3.9/site-packages/pymilvus/decorators.py:41, in retry_on_rpc_failure..wrapper..handler(self, *args, **kwargs) 39 while True: 40 try: ---> 41 return func(self, *args, **kwargs) 42 except grpc.RpcError as e: 43 # DEADLINE_EXCEEDED means that the task wat not completed 44 # UNAVAILABLE means that the service is not reachable currently 45 # Reference: https://grpc.github.io/grpc/python/grpc.html#grpc-status-code 46 if e.code() != grpc.StatusCode.DEADLINE_EXCEEDED and e.code() != grpc.StatusCode.UNAVAILABLE:

File /opt/conda/lib/python3.9/site-packages/pymilvus/decorators.py:74, in error_handler..handler(*args, **kwargs) 72 record_dict["RPC error"] = str(datetime.datetime.now()) 73 LOGGER.error(f"RPC error: [{func.name}], {e}, Time:{record_dict}") ---> 74 raise e 75 except grpc.FutureTimeoutError as e: 76 record_dict["RPC timeout"] = str(datetime.datetime.now())

File /opt/conda/lib/python3.9/site-packages/pymilvus/decorators.py:70, in error_handler..handler(*args, **kwargs) 68 try: 69 record_dict["RPC start"] = str(datetime.datetime.now()) ---> 70 return func(*args, **kwargs) 71 except BaseException as e: 72 record_dict["RPC error"] = str(datetime.datetime.now())

File /opt/conda/lib/python3.9/site-packages/pymilvus/client/grpc_handler.py:676, in GrpcHandler.wait_for_loading_collection(self, collection_name, timeout) 673 @retry_on_rpc_failure(retry_times=10, wait=1) 674 @error_handler 675 def wait_for_loading_collection(self, collection_name, timeout=None): --> 676 return self._wait_for_loading_collection_v2(collection_name, timeout)

File /opt/conda/lib/python3.9/site-packages/pymilvus/client/grpc_handler.py:700, in GrpcHandler._wait_for_loading_collection_v2(self, collection_name, timeout) 697 response = future.result() 699 if response.status.error_code != 0: --> 700 raise BaseException(response.status.error_code, response.status.reason) 702 ol = len(response.collection_names) 703 pl = len(response.inMemory_percentages)

BaseException: <BaseException: (code=1, message=collection 430757187117383681 has not been loaded to memory or load failed)>

[tomjerrygithub]

The same error

BaseException Traceback (most recent call last) in 15 #partition = collection.partition("p_1_1_364611_12081577_celebpat") 16 #partition.load() ---> 17 partition = collection._get_connection().load_partitions('c_1_1_364611_effnet_embd', ["p_1_1_364611_12081577_effnetpat"]) 18 partition 19 #partition.num_entities

~/miniconda3/lib/python3.8/site-packages/pymilvus/decorators.py in handler(self, *args, **kwargs) 54 time.sleep(wait) 55 except Exception as e: ---> 56 raise e 57 finally: 58 counter += 1

~/miniconda3/lib/python3.8/site-packages/pymilvus/decorators.py in handler(self, *args, **kwargs) 39 while True: 40 try: ---> 41 return func(self, *args, **kwargs) 42 except grpc.RpcError as e: 43 # DEADLINE_EXCEEDED means that the task wat not completed

~/miniconda3/lib/python3.8/site-packages/pymilvus/decorators.py in handler(*args, **kwargs) 72 record_dict["RPC error"] = str(datetime.datetime.now()) 73 LOGGER.error(f"RPC error: [{func.name}], {e}, Time:{record_dict}") ---> 74 raise e 75 except grpc.FutureTimeoutError as e: 76 record_dict["RPC timeout"] = str(datetime.datetime.now())

~/miniconda3/lib/python3.8/site-packages/pymilvus/decorators.py in handler(*args, **kwargs) 68 try: 69 record_dict["RPC start"] = str(datetime.datetime.now()) ---> 70 return func(*args, **kwargs) 71 except BaseException as e: 72 record_dict["RPC error"] = str(datetime.datetime.now())

~/miniconda3/lib/python3.8/site-packages/pymilvus/client/grpc_handler.py in load_partitions(self, collection_name, partition_names, timeout, **kwargs) 749 sync = kwargs.get("sync", True) 750 if sync: --> 751 self.wait_for_loading_partitions(collection_name, partition_names) 752 753 @retry_on_rpc_failure(retry_times=10, wait=1)

~/miniconda3/lib/python3.8/site-packages/pymilvus/decorators.py in handler(self, *args, **kwargs) 54 time.sleep(wait) 55 except Exception as e: ---> 56 raise e 57 finally: 58 counter += 1

~/miniconda3/lib/python3.8/site-packages/pymilvus/decorators.py in handler(self, *args, **kwargs) 39 while True: 40 try: ---> 41 return func(self, *args, **kwargs) 42 except grpc.RpcError as e: 43 # DEADLINE_EXCEEDED means that the task wat not completed

~/miniconda3/lib/python3.8/site-packages/pymilvus/decorators.py in handler(*args, **kwargs) 72 record_dict["RPC error"] = str(datetime.datetime.now()) 73 LOGGER.error(f"RPC error: [{func.name}], {e}, Time:{record_dict}") ---> 74 raise e 75 except grpc.FutureTimeoutError as e: 76 record_dict["RPC timeout"] = str(datetime.datetime.now())

~/miniconda3/lib/python3.8/site-packages/pymilvus/decorators.py in handler(*args, **kwargs) 68 try: 69 record_dict["RPC start"] = str(datetime.datetime.now()) ---> 70 return func(*args, **kwargs) 71 except BaseException as e: 72 record_dict["RPC error"] = str(datetime.datetime.now())

~/miniconda3/lib/python3.8/site-packages/pymilvus/client/grpc_handler.py in wait_for_loading_partitions(self, collection_name, partition_names, timeout) 754 @error_handler 755 def wait_for_loading_partitions(self, collection_name, partition_names, timeout=None): --> 756 return self._wait_for_loading_partitions_v2(collection_name, partition_names, timeout) 757 758 # TODO seems not in use

~/miniconda3/lib/python3.8/site-packages/pymilvus/client/grpc_handler.py in _wait_for_loading_partitions_v2(self, collection_name, partition_names, timeout) 794 status = response.status 795 if status.error_code != 0: --> 796 raise BaseException(status.error_code, status.reason) 797 798 ol = len(response.partition_names)

BaseException: <BaseException: (code=1, message=collection 431705768639856641 has not been loaded into QueryNode)>