ragas
ragas copied to clipboard
explodinggradients
test dataset generator not working on Bedrock
[ ] I have checked the documentation and related resources and couldn't resolve my bug.
Describe the bug A clear and concise description of what the bug is.
Ragas version: 0.1.4 Python version:3.10.13
Code to Reproduce Share code to reproduce the issue
testset = test_generator.generate_with_langchain_docs(
documents=documents, test_size=10, distributions=distributions
)
**Error trace**
Filename and doc_id are the same for all nodes.
Generating: 0%
0/10 [00:27<?, ?it/s]
Exception in thread Thread-36:
Traceback (most recent call last):
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
self.run()
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/ragas/executor.py", line 96, in run
results = self.loop.run_until_complete(self._aresults())
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/nest_asyncio.py", line 98, in run_until_complete
return f.result()
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/asyncio/futures.py", line 201, in result
raise self._exception.with_traceback(self._exception_tb)
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/asyncio/tasks.py", line 232, in __step
result = coro.send(None)
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/ragas/executor.py", line 84, in _aresults
raise e
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/ragas/executor.py", line 79, in _aresults
r = await future
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/asyncio/tasks.py", line 571, in _wait_for_one
return f.result() # May raise f.exception().
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/asyncio/futures.py", line 201, in result
raise self._exception.with_traceback(self._exception_tb)
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/asyncio/tasks.py", line 232, in __step
result = coro.send(None)
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/ragas/executor.py", line 38, in sema_coro
return await coro
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/ragas/executor.py", line 112, in wrapped_callable_async
return counter, await callable(*args, **kwargs)
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/ragas/testset/evolutions.py", line 144, in evolve
question=evolved_question,
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/ragas/testset/evolutions.py", line 210, in generate_datarow
current_nodes.nodes[i - 1]
File "/home/ec2-user/anaconda3/envs/python3/lib/python3.10/site-packages/ragas/testset/evolutions.py", line 213, in <listcomp>
]
TypeError: unsupported operand type(s) for -: 'str' and 'int'
---------------------------------------------------------------------------
ExceptionInRunner Traceback (most recent call last)
Cell In[112], line 2
1 # use generator.generate_with_llamaindex_docs if you use llama-index as document loader
----> 2 testset = test_generator.generate_with_langchain_docs(
3 documents=documents, test_size=10, distributions=distributions
4 )
File ~/anaconda3/envs/python3/lib/python3.10/site-packages/ragas/testset/generator.py:179, in TestsetGenerator.generate_with_langchain_docs(self, documents, test_size, distributions, with_debugging_logs, is_async, raise_exceptions, run_config)
174 # chunk documents and add to docstore
175 self.docstore.add_documents(
176 [Document.from_langchain_document(doc) for doc in documents]
177 )
--> 179 return self.generate(
180 test_size=test_size,
181 distributions=distributions,
182 with_debugging_logs=with_debugging_logs,
183 is_async=is_async,
184 raise_exceptions=raise_exceptions,
185 run_config=run_config,
186 )
File ~/anaconda3/envs/python3/lib/python3.10/site-packages/ragas/testset/generator.py:274, in TestsetGenerator.generate(self, test_size, distributions, with_debugging_logs, is_async, raise_exceptions, run_config)
272 test_data_rows = exec.results()
273 if not test_data_rows:
--> 274 raise ExceptionInRunner()
276 except ValueError as e:
277 raise e
ExceptionInRunner: The runner thread which was running the jobs raised an exeception. Read the traceback above to debug it. You can also pass `raise_exceptions=False` incase you want to show only a warning message instead.
Expected behavior A clear and concise description of what you expected to happen.
Additional context Add any other context about the problem here. I was following the tutorials on ragas bedrock application.
This is also occurring in ragas 0.1.4 and Python 3.11.6
Exception in thread Thread-7:
Traceback (most recent call last):
File "/opt/conda/lib/python3.11/threading.py", line 1045, in _bootstrap_inner
self.run()
File "/opt/conda/lib/python3.11/site-packages/ragas/executor.py", line 96, in run
results = self.loop.run_until_complete(self._aresults())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/asyncio/base_events.py", line 653, in run_until_complete
return future.result()
^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/ragas/executor.py", line 84, in _aresults
raise e
File "/opt/conda/lib/python3.11/site-packages/ragas/executor.py", line 79, in _aresults
r = await future
^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/asyncio/tasks.py", line 615, in _wait_for_one
return f.result() # May raise f.exception().
^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/ragas/executor.py", line 38, in sema_coro
return await coro
^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/ragas/executor.py", line 112, in wrapped_callable_async
return counter, await callable(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/ragas/testset/evolutions.py", line 143, in evolve
return await self.generate_datarow(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/conda/lib/python3.11/site-packages/ragas/testset/evolutions.py", line 209, in generate_datarow
selected_nodes = [
^
File "/opt/conda/lib/python3.11/site-packages/ragas/testset/evolutions.py", line 212, in <listcomp>
if i - 1 < len(current_nodes.nodes)
~~^~~
TypeError: unsupported operand type(s) for -: 'str' and 'int'
---------------------------------------------------------------------------
ExceptionInRunner Traceback (most recent call last)
Cell In[16], line 1
----> 1 testset = generator.generate_with_langchain_docs(documents, test_size=3)
File /opt/conda/lib/python3.11/site-packages/ragas/testset/generator.py:179, in TestsetGenerator.generate_with_langchain_docs(self, documents, test_size, distributions, with_debugging_logs, is_async, raise_exceptions, run_config)
174 # chunk documents and add to docstore
175 self.docstore.add_documents(
176 [Document.from_langchain_document(doc) for doc in documents]
177 )
--> 179 return self.generate(
180 test_size=test_size,
181 distributions=distributions,
182 with_debugging_logs=with_debugging_logs,
183 is_async=is_async,
184 raise_exceptions=raise_exceptions,
185 run_config=run_config,
186 )
File /opt/conda/lib/python3.11/site-packages/ragas/testset/generator.py:274, in TestsetGenerator.generate(self, test_size, distributions, with_debugging_logs, is_async, raise_exceptions, run_config)
272 test_data_rows = exec.results()
273 if not test_data_rows:
--> 274 raise ExceptionInRunner()
276 except ValueError as e:
277 raise e
ExceptionInRunner: The runner thread which was running the jobs raised an exeception. Read the traceback above to debug it. You can also pass `raise_exceptions=False` incase you want to show only a warning message instead.
