diff --git a/CHANGELOG.md b/CHANGELOG.md index 981575bb..703fa43b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog ## [Unreleased] +### Changed + +- **Evaluations**: Refactored QA benchmark to run entire dataset as single evaluation for better Logfire experiment tracking +- **Evaluations**: Added `.env` file loading support via `python-dotenv` dependency + ## [0.16.0] - 2025-11-13 ### Added diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 9f3f4eb4..a8dda959 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -5,6 +5,7 @@ from typing import Any, cast import logfire import typer +from dotenv import load_dotenv from pydantic_ai.models.openai import OpenAIChatModel from pydantic_ai.providers.ollama import OllamaProvider from pydantic_evals import Dataset as EvalDataset @@ -22,6 +23,8 @@ from haiku.rag.config import AppConfig, find_config_file, load_yaml_config from haiku.rag.logging import configure_cli_logging from haiku.rag.qa import get_qa_agent +load_dotenv() + QA_JUDGE_MODEL = "qwen3" logfire.configure(send_to_logfire="if-token-present", service_name="evals") @@ -192,62 +195,28 @@ async def run_qa_benchmark( ], ) - total_processed = 0 - passing_cases = 0 - failures: list[ReportCaseFailure[str, str, dict[str, str]]] = [] + async with HaikuRAG(spec.db_path, config=config) as rag: + system_prompt = WIX_SUPPORT_PROMPT if spec.key == "wix" else None + qa = get_qa_agent(rag, system_prompt=system_prompt) - with Progress(console=console) as progress: - qa_task = progress.add_task( - "[yellow]Evaluating QA cases...", - total=len(evaluation_dataset.cases), + async def answer_question(question: str) -> str: + return await qa.answer(question) + + report = await evaluation_dataset.evaluate( + answer_question, + name=f"{spec.key}_qa_evaluation", + max_concurrency=1, + progress=True, ) - async with HaikuRAG(spec.db_path, config=config) as rag: - system_prompt = WIX_SUPPORT_PROMPT if spec.key == "wix" else None - qa = get_qa_agent(rag, system_prompt=system_prompt) - - async def answer_question(question: str) -> str: - return await qa.answer(question) - - for case in evaluation_dataset.cases: - single_case_dataset = EvalDataset[str, str, dict[str, str]]( - cases=[case], - evaluators=evaluation_dataset.evaluators, - ) - - report = await single_case_dataset.evaluate( - answer_question, - name="qa_answer", - max_concurrency=1, - progress=False, - ) - - total_processed += 1 - - if report.cases: - result_case = report.cases[0] - - equivalence = result_case.assertions.get("answer_equivalent") - if equivalence is not None: - if equivalence.value: - passing_cases += 1 - - if report.failures: - failures.extend(report.failures) - failure = report.failures[0] - progress.console.print( - "[red]Failure encountered during case evaluation:[/red]" - ) - progress.console.print(f"Error: {failure.error_message}") - progress.console.print("") - - progress.update( - qa_task, - description="[yellow]Evaluating QA cases...[/yellow] " - f"[green]Accuracy: {(passing_cases / total_processed):.2f} " - f"{passing_cases}/{total_processed}[/green]", - ) - progress.advance(qa_task) + passing_cases = sum( + 1 + for case in report.cases + if case.assertions.get("answer_equivalent") + and case.assertions["answer_equivalent"].value + ) + total_processed = len(report.cases) + failures = report.failures total_cases = total_processed accuracy = passing_cases / total_cases if total_cases > 0 else 0 diff --git a/evaluations/pyproject.toml b/evaluations/pyproject.toml index f66cc9ae..2eb7b112 100644 --- a/evaluations/pyproject.toml +++ b/evaluations/pyproject.toml @@ -9,9 +9,10 @@ requires-python = ">=3.12" dependencies = [ "haiku.rag-slim", - "pydantic-ai-slim[evals,logfire]>=1.11.0", + "pydantic-ai-slim[evals,logfire]>=1.17.0", "datasets>=4.3.0", "typer>=0.19.2,<0.20.0", + "python-dotenv>=1.0.1", ] [build-system] diff --git a/examples/a2a-server/pyproject.toml b/examples/a2a-server/pyproject.toml index 78bcd6b6..f7090160 100644 --- a/examples/a2a-server/pyproject.toml +++ b/examples/a2a-server/pyproject.toml @@ -11,7 +11,7 @@ keywords = ["RAG", "a2a", "agent", "conversational-ai"] dependencies = [ "haiku.rag>=0.15.0", "fasta2a>=0.1.0", - "pydantic-ai-slim[a2a]>=1.11.1", + "pydantic-ai-slim[a2a]>=1.17.0", "rich>=14.2.0", "httpx>=0.28.1", ] diff --git a/examples/ag-ui-research/backend/pyproject.toml b/examples/ag-ui-research/backend/pyproject.toml index 9513393f..779da262 100644 --- a/examples/ag-ui-research/backend/pyproject.toml +++ b/examples/ag-ui-research/backend/pyproject.toml @@ -7,16 +7,13 @@ requires-python = ">=3.13" dependencies = [ "starlette>=0.45.2", "uvicorn[standard]>=0.34.2", - "pydantic-ai-slim[ag-ui,openai]>=1.1.0", + "pydantic-ai-slim[ag-ui,openai]>=1.17.0", "python-dotenv>=1.0.1", "haiku-rag-slim @ file:///Users/ggozad/dev/open-source/haiku.rag-agui/haiku_rag_slim", ] [dependency-groups] -dev = [ - "pyright>=1.1.406", - "ruff>=0.13.0", -] +dev = ["pyright>=1.1.406", "ruff>=0.13.0"] [tool.hatch.metadata] allow-direct-references = true diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index faca7a43..6279f566 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -27,7 +27,7 @@ dependencies = [ "lancedb==0.25.2", "pathspec>=0.12.1", "pydantic>=2.12.3", - "pydantic-ai-slim[openai,fastmcp,logfire,ag-ui]>=1.11.1", + "pydantic-ai-slim[openai,fastmcp,logfire,ag-ui]==1.17.0", "python-dotenv>=1.2.1", "pyyaml>=6.0.3", "rich>=14.2.0", diff --git a/uv.lock b/uv.lock index d7e02bb2..97fece9c 100644 --- a/uv.lock +++ b/uv.lock @@ -1179,6 +1179,7 @@ dependencies = [ { name = "datasets" }, { name = "haiku-rag-slim" }, { name = "pydantic-ai-slim", extra = ["evals", "logfire"] }, + { name = "python-dotenv" }, { name = "typer" }, ] @@ -1186,7 +1187,8 @@ dependencies = [ requires-dist = [ { name = "datasets", specifier = ">=4.3.0" }, { name = "haiku-rag-slim", editable = "haiku_rag_slim" }, - { name = "pydantic-ai-slim", extras = ["evals", "logfire"], specifier = ">=1.11.0" }, + { name = "pydantic-ai-slim", extras = ["evals", "logfire"], specifier = ">=1.17.0" }, + { name = "python-dotenv", specifier = ">=1.0.1" }, { name = "typer", specifier = ">=0.19.2,<0.20.0" }, ] @@ -1259,7 +1261,7 @@ requires-dist = [ { name = "pydantic-ai-slim", extras = ["google"], marker = "extra == 'google'" }, { name = "pydantic-ai-slim", extras = ["groq"], marker = "extra == 'groq'" }, { name = "pydantic-ai-slim", extras = ["mistral"], marker = "extra == 'mistral'" }, - { name = "pydantic-ai-slim", extras = ["openai", "fastmcp", "logfire", "ag-ui"], specifier = ">=1.11.1" }, + { name = "pydantic-ai-slim", extras = ["openai", "fastmcp", "logfire", "ag-ui"], specifier = "==1.17.0" }, { name = "pydantic-ai-slim", extras = ["vertexai"], marker = "extra == 'vertexai'" }, { name = "python-dotenv", specifier = ">=1.2.1" }, { name = "pyyaml", specifier = ">=6.0.3" }, @@ -3014,7 +3016,7 @@ email = [ [[package]] name = "pydantic-ai-slim" -version = "1.11.1" +version = "1.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "genai-prices" }, @@ -3025,9 +3027,9 @@ dependencies = [ { name = "pydantic-graph" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/90/a5/fbfcdd3c89549dd44417606af0130f1118aea8e43f4d14723e49218901a6/pydantic_ai_slim-1.11.1.tar.gz", hash = "sha256:242fb5c7a0f812d540f68d4e2e6498730ef11644b55ccf3da38bf9767802f742", size = 298765, upload-time = "2025-11-06T00:48:42.815Z" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/bc/39f1dca02883372ccccd82e55b21a0bf4d248e69f40a22a6e177a285781a/pydantic_ai_slim-1.17.0.tar.gz", hash = "sha256:7c6a10b0842819cd1328dc6d0b64faba4ae59b78d9a97b53910aff1a28108e0a", size = 301616, upload-time = "2025-11-14T00:40:17.329Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/6d/d8ea48afdd8838d6419cdbc08d81753e2e732ff3451e3d83f6b4b56388af/pydantic_ai_slim-1.11.1-py3-none-any.whl", hash = "sha256:00ca8b0a8f677fa9efd077239b66c925423d1dc517dfac7953b62547a66adbf2", size = 397971, upload-time = "2025-11-06T00:48:28.219Z" }, + { url = "https://files.pythonhosted.org/packages/34/16/be20a655d6a9323165306b0889b2f04b985025a7f195c486e4292fd51f74/pydantic_ai_slim-1.17.0-py3-none-any.whl", hash = "sha256:2fc64bad8b6396a2af32c1ff04d73f4cde62ba15c28329cc98dbae47651cad90", size = 401934, upload-time = "2025-11-14T00:40:03.326Z" }, ] [package.optional-dependencies] @@ -3136,7 +3138,7 @@ wheels = [ [[package]] name = "pydantic-evals" -version = "1.11.1" +version = "1.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -3146,14 +3148,14 @@ dependencies = [ { name = "pyyaml" }, { name = "rich" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/04/7a/37272a8b371aae31dd4c74eb9ee0b0e4bc55c14aaa36719922261612e77e/pydantic_evals-1.11.1.tar.gz", hash = "sha256:3eec8e7ff260626c2fab4087634fe078fdcc69b75e6a4204a3fb7b416e41e589", size = 46969, upload-time = "2025-11-06T00:48:44.088Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/f5/4126cedb4c65e54c13d44b03a9551ed8cf6d4b1c0551e61a2ed9f700c73c/pydantic_evals-1.17.0.tar.gz", hash = "sha256:54e24324fb99b453b27817a8c51510a56282a41bc7968e1e5585355cf0c1aea1", size = 46978, upload-time = "2025-11-14T00:40:18.719Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f5/a9/1ecc57122e81420ff799a3859e2fef4c215ac2c185d07afaa5d5495a68e0/pydantic_evals-1.11.1-py3-none-any.whl", hash = "sha256:76f8c22231e5a0ae7466b212906027773ced7e01cf10120be813fd4f0636c052", size = 56132, upload-time = "2025-11-06T00:48:30.487Z" }, + { url = "https://files.pythonhosted.org/packages/33/04/2ddffbd5b45388e7e0f6019670e4cd6cf524fef07597938107f0e6aeb431/pydantic_evals-1.17.0-py3-none-any.whl", hash = "sha256:a7447c99ca86bf68880c3078f1b751c418145a725185bce99b604dab9479bf1a", size = 56134, upload-time = "2025-11-14T00:40:06.793Z" }, ] [[package]] name = "pydantic-graph" -version = "1.11.1" +version = "1.17.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx" }, @@ -3161,9 +3163,9 @@ dependencies = [ { name = "pydantic" }, { name = "typing-inspection" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6f/b6/1b37a9517bc71fde33184cc6f3f03795c3669b7be5a143a3012fb112742d/pydantic_graph-1.11.1.tar.gz", hash = "sha256:345d6309ac677ef6cf2f5b225e6762afd9b87cc916b943376a5cb555705a7f2b", size = 57964, upload-time = "2025-11-06T00:48:45.028Z" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/c8/7d41e6f07d2e81851036552a6a4cd62b100509bfeafceb2e46051beea7b0/pydantic_graph-1.17.0.tar.gz", hash = "sha256:0e673f049fa5e86443ea90f0b79d8e24696b2d49545d31556933a08b9363633b", size = 57983, upload-time = "2025-11-14T00:40:19.876Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/50/64/934e1f9be64f44515c501bf528cfc2dd672516530d5a7aa7436f72aba5ef/pydantic_graph-1.11.1-py3-none-any.whl", hash = "sha256:4d52d0c925672439e407d64e663a5e7f011f0bb0941c8b6476911044c7478cd6", size = 72002, upload-time = "2025-11-06T00:48:32.411Z" }, + { url = "https://files.pythonhosted.org/packages/0f/6c/a3ccda382ad69d5549da186fe349b17efd1f4ad7bf871584186381a1bc31/pydantic_graph-1.17.0-py3-none-any.whl", hash = "sha256:436e11e8ca5bd6a99e5e2ba50a42f958bfff65aeb8a40aef83ae5da1bb8b5891", size = 72003, upload-time = "2025-11-14T00:40:08.329Z" }, ] [[package]]