ragas icon indicating copy to clipboard operation
ragas copied to clipboard

Invalid n value (currently only n = 1 is supported)

Open jidechao opened this issue 7 months ago • 11 comments

[ ] I have checked the documentation and related resources and couldn't resolve my bug.

Describe the bug

openai.BadRequestError: Error code: 400 - {'detail': 'Invalid n value (currently only n = 1 is supported)'}

Ragas version: 0.1.10

Code to Reproduce

from ragas import evaluate
from langchain_openai import ChatOpenAI, OpenAI
from langchain_community.embeddings import HuggingFaceBgeEmbeddings
from datasets import Dataset
from langchain.vectorstores.milvus import Milvus

#embedding(chroma)
model_name = "BAAI/bge-m3"
model_kwargs = {'device': 'cpu'}
encode_kwargs = {'normalize_embeddings': True}
embeddings = HuggingFaceBgeEmbeddings(
    model_name=model_name,
    model_kwargs=model_kwargs,
    encode_kwargs=encode_kwargs
)

import nest_asyncio

nest_asyncio.apply()

llm = ChatOpenAI(model="deepseek-chat",api_key="sk-*******",base_url="https://api.deepseek.com/v1" ,temperature=0, n=1)

from ragas.metrics import (
    faithfulness,
    answer_relevancy,
    context_relevancy,
    context_recall,
    context_precision,
)

data = {
    'question': ['What is the capital of France?'],
    'contexts': [['Paris is the capital of France.']],
    'answer': ['Paris'],
    'ground_truth': ['Paris']
}
dataset = Dataset.from_dict(data)


result = evaluate(
    dataset = dataset, 
    metrics=[
        context_precision,
        context_recall,
        faithfulness,
        answer_relevancy,
    ],
    llm=llm,
    embeddings=embeddings,
)

result

**Error trace**
Exception in thread Thread-11:
Traceback (most recent call last):
  File "/usr/lib/python3.10/threading.py", line 1016, in _bootstrap_inner
    self.run()
  File "/usr/local/lib/python3.10/dist-packages/ragas/executor.py", line 95, in run
    results = self.loop.run_until_complete(self._aresults())
  File "/usr/local/lib/python3.10/dist-packages/nest_asyncio.py", line 98, in run_until_complete
    return f.result()
  File "/usr/lib/python3.10/asyncio/futures.py", line 201, in result
    raise self._exception.with_traceback(self._exception_tb)
  File "/usr/lib/python3.10/asyncio/tasks.py", line 232, in __step
    result = coro.send(None)
  File "/usr/local/lib/python3.10/dist-packages/ragas/executor.py", line 83, in _aresults
    raise e
  File "/usr/local/lib/python3.10/dist-packages/ragas/executor.py", line 78, in _aresults
    r = await future
  File "/usr/lib/python3.10/asyncio/tasks.py", line 571, in _wait_for_one
    return f.result()  # May raise f.exception().
  File "/usr/lib/python3.10/asyncio/futures.py", line 201, in result
    raise self._exception.with_traceback(self._exception_tb)
  File "/usr/lib/python3.10/asyncio/tasks.py", line 232, in __step
    result = coro.send(None)
  File "/usr/local/lib/python3.10/dist-packages/ragas/executor.py", line 37, in sema_coro
    return await coro
  File "/usr/local/lib/python3.10/dist-packages/ragas/executor.py", line 111, in wrapped_callable_async
    return counter, await callable(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/ragas/metrics/base.py", line 125, in ascore
    raise e
  File "/usr/local/lib/python3.10/dist-packages/ragas/metrics/base.py", line 121, in ascore
    score = await self._ascore(row=row, callbacks=group_cm, is_async=is_async)
  File "/usr/local/lib/python3.10/dist-packages/ragas/metrics/_answer_relevance.py", line 152, in _ascore
    result = await self.llm.generate(
  File "/usr/local/lib/python3.10/dist-packages/ragas/llms/base.py", line 93, in generate
    return await agenerate_text_with_retry(
  File "/usr/local/lib/python3.10/dist-packages/tenacity/asyncio/__init__.py", line 189, in async_wrapped
    return await copy(fn, *args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/tenacity/asyncio/__init__.py", line 111, in __call__
    do = await self.iter(retry_state=retry_state)
  File "/usr/local/lib/python3.10/dist-packages/tenacity/asyncio/__init__.py", line 153, in iter
    result = await action(retry_state)
  File "/usr/local/lib/python3.10/dist-packages/tenacity/_utils.py", line 99, in inner
    return call(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/tenacity/__init__.py", line 398, in <lambda>
    self._add_action_func(lambda rs: rs.outcome.result())
  File "/usr/lib/python3.10/concurrent/futures/_base.py", line 451, in result
    return self.__get_result()
  File "/usr/lib/python3.10/concurrent/futures/_base.py", line 403, in __get_result
    raise self._exception
  File "/usr/local/lib/python3.10/dist-packages/tenacity/asyncio/__init__.py", line 114, in __call__
    result = await fn(*args, **kwargs)
  File "/usr/local/lib/python3.10/dist-packages/ragas/llms/base.py", line 170, in agenerate_text
    return await self.langchain_llm.agenerate_prompt(
  File "/usr/local/lib/python3.10/dist-packages/langchain_core/language_models/chat_models.py", line 691, in agenerate_prompt
    return await self.agenerate(
  File "/usr/local/lib/python3.10/dist-packages/langchain_core/language_models/chat_models.py", line 651, in agenerate
    raise exceptions[0]
  File "/usr/lib/python3.10/asyncio/tasks.py", line 232, in __step
    result = coro.send(None)
  File "/usr/local/lib/python3.10/dist-packages/langchain_core/language_models/chat_models.py", line 836, in _agenerate_with_cache
    result = await self._agenerate(
  File "/usr/local/lib/python3.10/dist-packages/langchain_openai/chat_models/base.py", line 674, in _agenerate
    response = await self.async_client.create(**payload)
  File "/usr/local/lib/python3.10/dist-packages/openai/resources/chat/completions.py", line 1289, in create
    return await self._post(
  File "/usr/local/lib/python3.10/dist-packages/openai/_base_client.py", line 1816, in post
    return await self.request(cast_to, opts, stream=stream, stream_cls=stream_cls)
  File "/usr/local/lib/python3.10/dist-packages/openai/_base_client.py", line 1514, in request
    return await self._request(
  File "/usr/local/lib/python3.10/dist-packages/openai/_base_client.py", line 1610, in _request
    raise self._make_status_error_from_response(err.response) from None
openai.BadRequestError: Error code: 400 - {'detail': 'Invalid n value (currently only n = 1 is supported)'}

Expected behavior A clear and concise description of what you expected to happen.

Additional context Add any other context about the problem here.

jidechao avatar Jul 06 '24 07:07 jidechao