danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Better output in QA evaluations

Browse source
This commit is contained in:
Yiorgis Gozadinos 2025-10-10 11:19:02 +03:00
parent 98e0124ee0
commit dae0992e3f
No known key found for this signature in database
1 changed files with 5 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
src/evaluations/benchmark.py
View file

@ -212,8 +212,6 @@ async def run_qa_benchmark(
return await qa.answer(question) return await qa.answer(question)
for case in evaluation_dataset.cases: for case in evaluation_dataset.cases:
progress.console.print(f"\n[bold]Evaluating case:[/bold] {case.name}")
single_case_dataset = EvalDataset[str, str, dict[str, str]]( single_case_dataset = EvalDataset[str, str, dict[str, str]](
cases=[case], cases=[case],
evaluators=evaluation_dataset.evaluators, evaluators=evaluation_dataset.evaluators,
@ -232,32 +230,24 @@ async def run_qa_benchmark(
result_case = report.cases[0] result_case = report.cases[0]
equivalence = result_case.assertions.get("answer_equivalent") equivalence = result_case.assertions.get("answer_equivalent")
progress.console.print(f"Question: {result_case.inputs}")
progress.console.print(f"Expected: {result_case.expected_output}")
progress.console.print(f"Generated: {result_case.output}")
if equivalence is not None: if equivalence is not None:
progress.console.print(
f"Equivalent: {equivalence.value}"
+ (f" — {equivalence.reason}" if equivalence.reason else "")
)
if equivalence.value: if equivalence.value:
passing_cases += 1 passing_cases += 1
progress.console.print("")
if report.failures: if report.failures:
failures.extend(report.failures) failures.extend(report.failures)
failure = report.failures[0] failure = report.failures[0]
progress.console.print( progress.console.print(
"[red]Failure encountered during case evaluation:[/red]" "[red]Failure encountered during case evaluation:[/red]"
) )
progress.console.print(f"Question: {failure.inputs}")
progress.console.print(f"Error: {failure.error_message}") progress.console.print(f"Error: {failure.error_message}")
progress.console.print("") progress.console.print("")
progress.console.print( progress.update(
f"[green]Accuracy: {(passing_cases / total_processed):.4f} " qa_task,
f"{passing_cases}/{total_processed}[/green]" description="[yellow]Evaluating QA cases...[/yellow] "
f"[green]Accuracy: {(passing_cases / total_processed):.2f} "
f"{passing_cases}/{total_processed}[/green]",
) )
progress.advance(qa_task) progress.advance(qa_task)