From adb7d9093d0c6808aa5b7d20d04878822c03cf68 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Fri, 14 Nov 2025 11:39:13 +0200
Subject: [PATCH] Run entire evaluation dataset as one so that it appears
 properly in logfire

---
 CHANGELOG.md                         |  5 ++
 evaluations/evaluations/benchmark.py | 72 ++++++++--------------------
 2 files changed, 24 insertions(+), 53 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 981575bb..703fa43b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,11 @@
 # Changelog
 ## [Unreleased]
 
+### Changed
+
+- **Evaluations**: Refactored QA benchmark to run entire dataset as single evaluation for better Logfire experiment tracking
+- **Evaluations**: Added `.env` file loading support via `python-dotenv` dependency
+
 ## [0.16.0] - 2025-11-13
 
 ### Added
diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py
index d4f9bb82..a8dda959 100644
--- a/evaluations/evaluations/benchmark.py
+++ b/evaluations/evaluations/benchmark.py
@@ -195,62 +195,28 @@ async def run_qa_benchmark(
         ],
     )
 
-    total_processed = 0
-    passing_cases = 0
-    failures: list[ReportCaseFailure[str, str, dict[str, str]]] = []
+    async with HaikuRAG(spec.db_path, config=config) as rag:
+        system_prompt = WIX_SUPPORT_PROMPT if spec.key == "wix" else None
+        qa = get_qa_agent(rag, system_prompt=system_prompt)
 
-    with Progress(console=console) as progress:
-        qa_task = progress.add_task(
-            "[yellow]Evaluating QA cases...",
-            total=len(evaluation_dataset.cases),
+        async def answer_question(question: str) -> str:
+            return await qa.answer(question)
+
+        report = await evaluation_dataset.evaluate(
+            answer_question,
+            name=f"{spec.key}_qa_evaluation",
+            max_concurrency=1,
+            progress=True,
         )
 
-        async with HaikuRAG(spec.db_path, config=config) as rag:
-            system_prompt = WIX_SUPPORT_PROMPT if spec.key == "wix" else None
-            qa = get_qa_agent(rag, system_prompt=system_prompt)
-
-            async def answer_question(question: str) -> str:
-                return await qa.answer(question)
-
-            for case in evaluation_dataset.cases:
-                single_case_dataset = EvalDataset[str, str, dict[str, str]](
-                    cases=[case],
-                    evaluators=evaluation_dataset.evaluators,
-                )
-
-                report = await single_case_dataset.evaluate(
-                    answer_question,
-                    name="qa_answer",
-                    max_concurrency=1,
-                    progress=False,
-                )
-
-                total_processed += 1
-
-                if report.cases:
-                    result_case = report.cases[0]
-
-                    equivalence = result_case.assertions.get("answer_equivalent")
-                    if equivalence is not None:
-                        if equivalence.value:
-                            passing_cases += 1
-
-                if report.failures:
-                    failures.extend(report.failures)
-                    failure = report.failures[0]
-                    progress.console.print(
-                        "[red]Failure encountered during case evaluation:[/red]"
-                    )
-                    progress.console.print(f"Error: {failure.error_message}")
-                    progress.console.print("")
-
-                progress.update(
-                    qa_task,
-                    description="[yellow]Evaluating QA cases...[/yellow] "
-                    f"[green]Accuracy: {(passing_cases / total_processed):.2f} "
-                    f"{passing_cases}/{total_processed}[/green]",
-                )
-                progress.advance(qa_task)
+    passing_cases = sum(
+        1
+        for case in report.cases
+        if case.assertions.get("answer_equivalent")
+        and case.assertions["answer_equivalent"].value
+    )
+    total_processed = len(report.cases)
+    failures = report.failures
 
     total_cases = total_processed
     accuracy = passing_cases / total_cases if total_cases > 0 else 0