ragas
ragas copied to clipboard
feat: introduce components experimental
Experimental feature: Component in metrics
# load desired NLI model as component
from ragas.experimental.metrics.component import TextClassificationNLIComponent
classifier = pipeline(
"text-classification",
model='vectara/hallucination_evaluation_model',
tokenizer=AutoTokenizer.from_pretrained('google/flan-t5-base'),
trust_remote_code=True
)
classifier = classifier(input_pairs, top_k=1)
prompt = "<pad> Determine if the hypothesis is true given the premise?\n\nPremise: {text1}\n\nHypothesis: {text2}"
vectara_hhem = TextClassificationNLIComponent(hf_pipeline=classifier,prompt=prompt,batch_size=8,label="consistent")
# customise faithfulness NLI component
from ragas.dataset_schema import SingleTurnSample
from ragas.experimental.metrics._faithfulness import FaithfulnessExperimental
sample = SingleTurnSample(
user_input = "Where is Eiffer Towerr",
response="The Eiffel Tower is located in Paris.",
retrieved_contexts=["The Eiffel Tower, an iconic symbol of Paris and one of the most recognizable structures in the world, was completed in 1889 as the entrance arch for the 1889 World's Fair. Designed by the engineer Gustave Eiffel, this wrought-iron lattice tower stands 324 meters tall and was initially met with skepticism but has since become a global cultural icon of France. It attracts millions of visitors each year, offering stunning panoramic views of the city from its observation decks."]
)
scorer = FaithfulnessExperimental()
scorer.llm = openai_model
scorer.nli_component = component
# scorer.nli_component.llm = openai_model
await scorer.single_turn_ascore(sample)
Also would suggest to move current version of experimental faithfulness before merging this.