Merge pull request #140 from ggozad/fix/evals-experiment
Run entire evaluation dataset as an experiment instead of individual test cases
This commit is contained in:
Yiorgis Gozadinos
GitHub
commit
8a697249f4
7 changed files with 46 additions and 72 deletions
|
|
@ -1,6 +1,11 @@
|
|||
# Changelog
|
||||
## [Unreleased]
|
||||
|
||||
### Changed
|
||||
|
||||
- **Evaluations**: Refactored QA benchmark to run entire dataset as single evaluation for better Logfire experiment tracking
|
||||
- **Evaluations**: Added `.env` file loading support via `python-dotenv` dependency
|
||||
|
||||
## [0.16.0] - 2025-11-13
|
||||
|
||||
### Added
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from typing import Any, cast
|
|||
|
||||
import logfire
|
||||
import typer
|
||||
from dotenv import load_dotenv
|
||||
from pydantic_ai.models.openai import OpenAIChatModel
|
||||
from pydantic_ai.providers.ollama import OllamaProvider
|
||||
from pydantic_evals import Dataset as EvalDataset
|
||||
|
|
@ -22,6 +23,8 @@ from haiku.rag.config import AppConfig, find_config_file, load_yaml_config
|
|||
from haiku.rag.logging import configure_cli_logging
|
||||
from haiku.rag.qa import get_qa_agent
|
||||
|
||||
load_dotenv()
|
||||
|
||||
QA_JUDGE_MODEL = "qwen3"
|
||||
|
||||
logfire.configure(send_to_logfire="if-token-present", service_name="evals")
|
||||
|
|
@ -192,62 +195,28 @@ async def run_qa_benchmark(
|
|||
],
|
||||
)
|
||||
|
||||
total_processed = 0
|
||||
passing_cases = 0
|
||||
failures: list[ReportCaseFailure[str, str, dict[str, str]]] = []
|
||||
async with HaikuRAG(spec.db_path, config=config) as rag:
|
||||
system_prompt = WIX_SUPPORT_PROMPT if spec.key == "wix" else None
|
||||
qa = get_qa_agent(rag, system_prompt=system_prompt)
|
||||
|
||||
with Progress(console=console) as progress:
|
||||
qa_task = progress.add_task(
|
||||
"[yellow]Evaluating QA cases...",
|
||||
total=len(evaluation_dataset.cases),
|
||||
async def answer_question(question: str) -> str:
|
||||
return await qa.answer(question)
|
||||
|
||||
report = await evaluation_dataset.evaluate(
|
||||
answer_question,
|
||||
name=f"{spec.key}_qa_evaluation",
|
||||
max_concurrency=1,
|
||||
progress=True,
|
||||
)
|
||||
|
||||
async with HaikuRAG(spec.db_path, config=config) as rag:
|
||||
system_prompt = WIX_SUPPORT_PROMPT if spec.key == "wix" else None
|
||||
qa = get_qa_agent(rag, system_prompt=system_prompt)
|
||||
|
||||
async def answer_question(question: str) -> str:
|
||||
return await qa.answer(question)
|
||||
|
||||
for case in evaluation_dataset.cases:
|
||||
single_case_dataset = EvalDataset[str, str, dict[str, str]](
|
||||
cases=[case],
|
||||
evaluators=evaluation_dataset.evaluators,
|
||||
)
|
||||
|
||||
report = await single_case_dataset.evaluate(
|
||||
answer_question,
|
||||
name="qa_answer",
|
||||
max_concurrency=1,
|
||||
progress=False,
|
||||
)
|
||||
|
||||
total_processed += 1
|
||||
|
||||
if report.cases:
|
||||
result_case = report.cases[0]
|
||||
|
||||
equivalence = result_case.assertions.get("answer_equivalent")
|
||||
if equivalence is not None:
|
||||
if equivalence.value:
|
||||
passing_cases += 1
|
||||
|
||||
if report.failures:
|
||||
failures.extend(report.failures)
|
||||
failure = report.failures[0]
|
||||
progress.console.print(
|
||||
"[red]Failure encountered during case evaluation:[/red]"
|
||||
)
|
||||
progress.console.print(f"Error: {failure.error_message}")
|
||||
progress.console.print("")
|
||||
|
||||
progress.update(
|
||||
qa_task,
|
||||
description="[yellow]Evaluating QA cases...[/yellow] "
|
||||
f"[green]Accuracy: {(passing_cases / total_processed):.2f} "
|
||||
f"{passing_cases}/{total_processed}[/green]",
|
||||
)
|
||||
progress.advance(qa_task)
|
||||
passing_cases = sum(
|
||||
1
|
||||
for case in report.cases
|
||||
if case.assertions.get("answer_equivalent")
|
||||
and case.assertions["answer_equivalent"].value
|
||||
)
|
||||
total_processed = len(report.cases)
|
||||
failures = report.failures
|
||||
|
||||
total_cases = total_processed
|
||||
accuracy = passing_cases / total_cases if total_cases > 0 else 0
|
||||
|
|
|
|||
|
|
@ -9,9 +9,10 @@ requires-python = ">=3.12"
|
|||
|
||||
dependencies = [
|
||||
"haiku.rag-slim",
|
||||
"pydantic-ai-slim[evals,logfire]>=1.11.0",
|
||||
"pydantic-ai-slim[evals,logfire]>=1.17.0",
|
||||
"datasets>=4.3.0",
|
||||
"typer>=0.19.2,<0.20.0",
|
||||
"python-dotenv>=1.0.1",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ keywords = ["RAG", "a2a", "agent", "conversational-ai"]
|
|||
dependencies = [
|
||||
"haiku.rag>=0.15.0",
|
||||
"fasta2a>=0.1.0",
|
||||
"pydantic-ai-slim[a2a]>=1.11.1",
|
||||
"pydantic-ai-slim[a2a]>=1.17.0",
|
||||
"rich>=14.2.0",
|
||||
"httpx>=0.28.1",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -7,16 +7,13 @@ requires-python = ">=3.13"
|
|||
dependencies = [
|
||||
"starlette>=0.45.2",
|
||||
"uvicorn[standard]>=0.34.2",
|
||||
"pydantic-ai-slim[ag-ui,openai]>=1.1.0",
|
||||
"pydantic-ai-slim[ag-ui,openai]>=1.17.0",
|
||||
"python-dotenv>=1.0.1",
|
||||
"haiku-rag-slim @ file:///Users/ggozad/dev/open-source/haiku.rag-agui/haiku_rag_slim",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pyright>=1.1.406",
|
||||
"ruff>=0.13.0",
|
||||
]
|
||||
dev = ["pyright>=1.1.406", "ruff>=0.13.0"]
|
||||
|
||||
[tool.hatch.metadata]
|
||||
allow-direct-references = true
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ dependencies = [
|
|||
"lancedb==0.25.2",
|
||||
"pathspec>=0.12.1",
|
||||
"pydantic>=2.12.3",
|
||||
"pydantic-ai-slim[openai,fastmcp,logfire,ag-ui]>=1.11.1",
|
||||
"pydantic-ai-slim[openai,fastmcp,logfire,ag-ui]==1.17.0",
|
||||
"python-dotenv>=1.2.1",
|
||||
"pyyaml>=6.0.3",
|
||||
"rich>=14.2.0",
|
||||
|
|
|
|||
24
uv.lock
24
uv.lock
|
|
@ -1179,6 +1179,7 @@ dependencies = [
|
|||
{ name = "datasets" },
|
||||
{ name = "haiku-rag-slim" },
|
||||
{ name = "pydantic-ai-slim", extra = ["evals", "logfire"] },
|
||||
{ name = "python-dotenv" },
|
||||
{ name = "typer" },
|
||||
]
|
||||
|
||||
|
|
@ -1186,7 +1187,8 @@ dependencies = [
|
|||
requires-dist = [
|
||||
{ name = "datasets", specifier = ">=4.3.0" },
|
||||
{ name = "haiku-rag-slim", editable = "haiku_rag_slim" },
|
||||
{ name = "pydantic-ai-slim", extras = ["evals", "logfire"], specifier = ">=1.11.0" },
|
||||
{ name = "pydantic-ai-slim", extras = ["evals", "logfire"], specifier = ">=1.17.0" },
|
||||
{ name = "python-dotenv", specifier = ">=1.0.1" },
|
||||
{ name = "typer", specifier = ">=0.19.2,<0.20.0" },
|
||||
]
|
||||
|
||||
|
|
@ -1259,7 +1261,7 @@ requires-dist = [
|
|||
{ name = "pydantic-ai-slim", extras = ["google"], marker = "extra == 'google'" },
|
||||
{ name = "pydantic-ai-slim", extras = ["groq"], marker = "extra == 'groq'" },
|
||||
{ name = "pydantic-ai-slim", extras = ["mistral"], marker = "extra == 'mistral'" },
|
||||
{ name = "pydantic-ai-slim", extras = ["openai", "fastmcp", "logfire", "ag-ui"], specifier = ">=1.11.1" },
|
||||
{ name = "pydantic-ai-slim", extras = ["openai", "fastmcp", "logfire", "ag-ui"], specifier = "==1.17.0" },
|
||||
{ name = "pydantic-ai-slim", extras = ["vertexai"], marker = "extra == 'vertexai'" },
|
||||
{ name = "python-dotenv", specifier = ">=1.2.1" },
|
||||
{ name = "pyyaml", specifier = ">=6.0.3" },
|
||||
|
|
@ -3014,7 +3016,7 @@ email = [
|
|||
|
||||
[[package]]
|
||||
name = "pydantic-ai-slim"
|
||||
version = "1.11.1"
|
||||
version = "1.17.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "genai-prices" },
|
||||
|
|
@ -3025,9 +3027,9 @@ dependencies = [
|
|||
{ name = "pydantic-graph" },
|
||||
{ name = "typing-inspection" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/90/a5/fbfcdd3c89549dd44417606af0130f1118aea8e43f4d14723e49218901a6/pydantic_ai_slim-1.11.1.tar.gz", hash = "sha256:242fb5c7a0f812d540f68d4e2e6498730ef11644b55ccf3da38bf9767802f742", size = 298765, upload-time = "2025-11-06T00:48:42.815Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/fe/bc/39f1dca02883372ccccd82e55b21a0bf4d248e69f40a22a6e177a285781a/pydantic_ai_slim-1.17.0.tar.gz", hash = "sha256:7c6a10b0842819cd1328dc6d0b64faba4ae59b78d9a97b53910aff1a28108e0a", size = 301616, upload-time = "2025-11-14T00:40:17.329Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f8/6d/d8ea48afdd8838d6419cdbc08d81753e2e732ff3451e3d83f6b4b56388af/pydantic_ai_slim-1.11.1-py3-none-any.whl", hash = "sha256:00ca8b0a8f677fa9efd077239b66c925423d1dc517dfac7953b62547a66adbf2", size = 397971, upload-time = "2025-11-06T00:48:28.219Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/34/16/be20a655d6a9323165306b0889b2f04b985025a7f195c486e4292fd51f74/pydantic_ai_slim-1.17.0-py3-none-any.whl", hash = "sha256:2fc64bad8b6396a2af32c1ff04d73f4cde62ba15c28329cc98dbae47651cad90", size = 401934, upload-time = "2025-11-14T00:40:03.326Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
|
|
@ -3136,7 +3138,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "pydantic-evals"
|
||||
version = "1.11.1"
|
||||
version = "1.17.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "anyio" },
|
||||
|
|
@ -3146,14 +3148,14 @@ dependencies = [
|
|||
{ name = "pyyaml" },
|
||||
{ name = "rich" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/04/7a/37272a8b371aae31dd4c74eb9ee0b0e4bc55c14aaa36719922261612e77e/pydantic_evals-1.11.1.tar.gz", hash = "sha256:3eec8e7ff260626c2fab4087634fe078fdcc69b75e6a4204a3fb7b416e41e589", size = 46969, upload-time = "2025-11-06T00:48:44.088Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/51/f5/4126cedb4c65e54c13d44b03a9551ed8cf6d4b1c0551e61a2ed9f700c73c/pydantic_evals-1.17.0.tar.gz", hash = "sha256:54e24324fb99b453b27817a8c51510a56282a41bc7968e1e5585355cf0c1aea1", size = 46978, upload-time = "2025-11-14T00:40:18.719Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f5/a9/1ecc57122e81420ff799a3859e2fef4c215ac2c185d07afaa5d5495a68e0/pydantic_evals-1.11.1-py3-none-any.whl", hash = "sha256:76f8c22231e5a0ae7466b212906027773ced7e01cf10120be813fd4f0636c052", size = 56132, upload-time = "2025-11-06T00:48:30.487Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/33/04/2ddffbd5b45388e7e0f6019670e4cd6cf524fef07597938107f0e6aeb431/pydantic_evals-1.17.0-py3-none-any.whl", hash = "sha256:a7447c99ca86bf68880c3078f1b751c418145a725185bce99b604dab9479bf1a", size = 56134, upload-time = "2025-11-14T00:40:06.793Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pydantic-graph"
|
||||
version = "1.11.1"
|
||||
version = "1.17.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "httpx" },
|
||||
|
|
@ -3161,9 +3163,9 @@ dependencies = [
|
|||
{ name = "pydantic" },
|
||||
{ name = "typing-inspection" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6f/b6/1b37a9517bc71fde33184cc6f3f03795c3669b7be5a143a3012fb112742d/pydantic_graph-1.11.1.tar.gz", hash = "sha256:345d6309ac677ef6cf2f5b225e6762afd9b87cc916b943376a5cb555705a7f2b", size = 57964, upload-time = "2025-11-06T00:48:45.028Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/bd/c8/7d41e6f07d2e81851036552a6a4cd62b100509bfeafceb2e46051beea7b0/pydantic_graph-1.17.0.tar.gz", hash = "sha256:0e673f049fa5e86443ea90f0b79d8e24696b2d49545d31556933a08b9363633b", size = 57983, upload-time = "2025-11-14T00:40:19.876Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/50/64/934e1f9be64f44515c501bf528cfc2dd672516530d5a7aa7436f72aba5ef/pydantic_graph-1.11.1-py3-none-any.whl", hash = "sha256:4d52d0c925672439e407d64e663a5e7f011f0bb0941c8b6476911044c7478cd6", size = 72002, upload-time = "2025-11-06T00:48:32.411Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/0f/6c/a3ccda382ad69d5549da186fe349b17efd1f4ad7bf871584186381a1bc31/pydantic_graph-1.17.0-py3-none-any.whl", hash = "sha256:436e11e8ca5bd6a99e5e2ba50a42f958bfff65aeb8a40aef83ae5da1bb8b5891", size = 72003, upload-time = "2025-11-14T00:40:08.329Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
Loading…
Reference in a new issue