From 7685a592c21ef15e30a00bdc7271519c23730d45 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 28 Apr 2026 12:03:14 +0300 Subject: [PATCH 1/5] bump haiku.skills to 0.16.0 --- haiku_rag_slim/haiku/rag/chat/app.py | 14 +++++++++++++- haiku_rag_slim/pyproject.toml | 2 +- tests/skills/test_analysis.py | 4 ++-- tests/skills/test_rag.py | 4 ++-- uv.lock | 8 ++++---- 5 files changed, 22 insertions(+), 10 deletions(-) diff --git a/haiku_rag_slim/haiku/rag/chat/app.py b/haiku_rag_slim/haiku/rag/chat/app.py index a40f7e03..6d7e3a48 100644 --- a/haiku_rag_slim/haiku/rag/chat/app.py +++ b/haiku_rag_slim/haiku/rag/chat/app.py @@ -401,7 +401,19 @@ class ChatApp(App): """Show the current session state.""" from haiku.skills.chat.app import StateScreen - self.push_screen(StateScreen(self._state)) + self.push_screen(StateScreen(self._state, on_save=self._apply_state_edit)) + + def _apply_state_edit(self, new_state: dict[str, Any]) -> None: + if self._toolset is None: + return + if not isinstance(new_state, dict): + raise ValueError("state must be a JSON object") + for namespace, data in new_state.items(): + current = self._toolset.get_namespace(namespace) + if current is not None: + type(current).model_validate(data) + self._toolset.restore_state_snapshot(new_state) + self._state = self._toolset.build_state_snapshot() def on_citation_widget_selected(self, event: CitationWidget.Selected) -> None: """Handle citation selection.""" diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index 538cbd23..85c71c31 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ dependencies = [ "docling-core>=2.71.0,<2.72", - "haiku.skills>=0.15.0", + "haiku.skills>=0.16.0", "httpx>=0.28.1", "jinja2>=3.1.0", "jsonpatch>=1.33", diff --git a/tests/skills/test_analysis.py b/tests/skills/test_analysis.py index 6935eec8..54a8dd5b 100644 --- a/tests/skills/test_analysis.py +++ b/tests/skills/test_analysis.py @@ -273,10 +273,10 @@ class TestAnalysisLifespan: from pydantic_ai.models.test import TestModel from haiku.rag.skills.analysis import create_skill - from haiku.skills.agent import _run_skill + from haiku.skills.agent import run_skill skill = create_skill(db_path=rag_db) - result, *_ = await _run_skill(TestModel(), skill, "Print the document count.") + result, *_ = await run_skill(TestModel(), skill, "Print the document count.") assert result async def test_lifespan_clears_executions_citations_searches(self, rag_db): diff --git a/tests/skills/test_rag.py b/tests/skills/test_rag.py index 808c0603..7e4d7b7d 100644 --- a/tests/skills/test_rag.py +++ b/tests/skills/test_rag.py @@ -366,10 +366,10 @@ class TestLifespan: from pydantic_ai.models.test import TestModel from haiku.rag.skills.rag import create_skill - from haiku.skills.agent import _run_skill + from haiku.skills.agent import run_skill skill = create_skill(db_path=rag_db) - result, *_ = await _run_skill(TestModel(), skill, "List the documents.") + result, *_ = await run_skill(TestModel(), skill, "List the documents.") assert result async def test_lifespan_clears_citations_and_searches_but_keeps_index(self, rag_db): diff --git a/uv.lock b/uv.lock index e7607d23..e9edf4eb 100644 --- a/uv.lock +++ b/uv.lock @@ -1570,7 +1570,7 @@ requires-dist = [ { name = "cohere", marker = "extra == 'cohere'", specifier = ">=5.21.1" }, { name = "docling", marker = "extra == 'docling'", specifier = ">=2.84.0" }, { name = "docling-core", specifier = ">=2.71.0,<2.72" }, - { name = "haiku-skills", specifier = ">=0.15.0" }, + { name = "haiku-skills", specifier = ">=0.16.0" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "jinja2", specifier = ">=3.1.0" }, { name = "jsonpatch", specifier = ">=1.33" }, @@ -1604,7 +1604,7 @@ provides-extras = ["docling", "voyageai", "mxbai", "cohere", "zeroentropy", "jin [[package]] name = "haiku-skills" -version = "0.15.0" +version = "0.16.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ag-ui-protocol" }, @@ -1614,9 +1614,9 @@ dependencies = [ { name = "pyyaml" }, { name = "skills-ref" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/86/a1/e2bd00a72d002f9db1c53c068167ed436a457dae0f8996399f116c087f6a/haiku_skills-0.15.0.tar.gz", hash = "sha256:ce93e6846e05397f5d96c144f956edd395b9e7308cb5cef6213c49c183a08bbc", size = 252030, upload-time = "2026-04-22T09:00:13.775Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8c/12/98ea5ee4ca14d4053019b4ec4b22e39af8ae1bc054aea107c87f5ecb025e/haiku_skills-0.16.0.tar.gz", hash = "sha256:e7bfa8141523912f3eb4469bf0b611408fa1e1e54707c9b475d0b3e04ea7ffbc", size = 256020, upload-time = "2026-04-28T08:44:23.347Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9b/68/3df2c9761fc0b0592b60f4723c87adeda5f335ea0835b4cf77ca07784379/haiku_skills-0.15.0-py3-none-any.whl", hash = "sha256:a1771b16e0ffe7da775f28d38c704e029f8e8757791616d7f27e73feb2c16fd0", size = 32041, upload-time = "2026-04-22T09:00:12.824Z" }, + { url = "https://files.pythonhosted.org/packages/55/ca/4cc0b026ee3e25d4b7f85267a210d1248988079fcac73649a09e9f9a5873/haiku_skills-0.16.0-py3-none-any.whl", hash = "sha256:50dec4e24594dceadc781242cafb20ced3b1fe743fdb7035a081756fbb402aa1", size = 32794, upload-time = "2026-04-28T08:44:21.855Z" }, ] [[package]] From d00befd0c43fdc5756593518f410ad583514aefa Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 28 Apr 2026 12:09:19 +0300 Subject: [PATCH 2/5] benchmark RAG and analysis skills via --target --- evaluations/evaluations/benchmark.py | 120 +++++++-- evaluations/evaluations/skill_runner.py | 92 +++++++ evaluations/tests/test_benchmark.py | 130 ++++++++++ evaluations/tests/test_skill_runner.py | 313 ++++++++++++++++++++++++ 4 files changed, 640 insertions(+), 15 deletions(-) create mode 100644 evaluations/evaluations/skill_runner.py create mode 100644 evaluations/tests/test_skill_runner.py diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index d6626ce7..1216787b 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -2,7 +2,7 @@ import asyncio import shutil from collections.abc import Mapping from pathlib import Path -from typing import Any, cast +from typing import Any, Literal, cast import logfire import typer @@ -17,6 +17,7 @@ from rich.progress import Progress from evaluations.config import DatasetSpec from evaluations.datasets import DATASETS from evaluations.evaluators import ANSWER_EQUIVALENCE_RUBRIC +from evaluations.skill_runner import SkillFactory, run_skill_question from haiku.rag.client import HaikuRAG from haiku.rag.config import AppConfig, find_config_file, load_yaml_config from haiku.rag.config.models import ModelConfig @@ -24,6 +25,9 @@ from haiku.rag.logging import configure_cli_logging from haiku.rag.agents.qa import get_qa_agent from haiku.rag.utils import get_model, parse_model_option +Target = Literal["qa", "rag-skill", "analysis-skill"] +TARGETS: tuple[Target, ...] = ("qa", "rag-skill", "analysis-skill") + load_dotenv(find_dotenv(usecwd=True)) HF_REPO_ID = "ggozad/haiku-rag-eval-dbs" @@ -39,11 +43,14 @@ def build_experiment_metadata( test_cases: int, config: AppConfig, judge_config: ModelConfig | None = None, + target: Target = "qa", + skill_config: ModelConfig | None = None, ) -> dict[str, Any]: """Build experiment metadata for Logfire tracking.""" metadata: dict[str, Any] = { "dataset": dataset_key, "test_cases": test_cases, + "target": target, "embedder_provider": config.embeddings.model.provider, "embedder_model": config.embeddings.model.name, "embedder_dim": config.embeddings.model.vector_dim, @@ -71,6 +78,16 @@ def build_experiment_metadata( "judge_enable_thinking": judge_config.enable_thinking, } ) + if skill_config is not None: + metadata.update( + { + "skill_provider": skill_config.provider, + "skill_model": skill_config.name, + "skill_temperature": skill_config.temperature, + "skill_max_tokens": skill_config.max_tokens, + "skill_enable_thinking": skill_config.enable_thinking, + } + ) return metadata @@ -260,6 +277,18 @@ async def run_retrieval_benchmark( } +def _skill_factory_for_target(target: Target) -> SkillFactory: + if target == "rag-skill": + from haiku.rag.skills.rag import create_skill + + return create_skill + if target == "analysis-skill": + from haiku.rag.skills.analysis import create_skill + + return create_skill + raise ValueError(f"target {target!r} is not a skill target") + + async def run_qa_benchmark( spec: DatasetSpec, config: AppConfig, @@ -267,6 +296,8 @@ async def run_qa_benchmark( name: str | None = None, db_path: Path | None = None, judge_model: ModelConfig | None = None, + target: Target = "qa", + skill_model: ModelConfig | None = None, ) -> ReportCaseFailure[str, str, dict[str, str]] | None: corpus = spec.qa_loader() if limit is not None: @@ -298,21 +329,49 @@ async def run_qa_benchmark( ) db = spec.db_path(db_path) - async with HaikuRAG(db, config=config) as rag: - qa = get_qa_agent(rag, config, system_prompt=spec.resolve_system_prompt(config)) + skill_config = skill_model or config.qa.model if target != "qa" else None + + eval_name = name if name is not None else f"{spec.key}_qa_evaluation" + experiment_metadata = build_experiment_metadata( + dataset_key=spec.key, + test_cases=len(cases), + config=config, + judge_config=judge_config, + target=target, + skill_config=skill_config, + ) + + if target == "qa": + async with HaikuRAG(db, config=config) as rag: + qa = get_qa_agent( + rag, config, system_prompt=spec.resolve_system_prompt(config) + ) + + async def answer_question(question: str) -> str: + answer, _ = await qa.answer(question) + return answer + + report = await evaluation_dataset.evaluate( + answer_question, + name=eval_name, + max_concurrency=1, + progress=True, + metadata=experiment_metadata, + ) + else: + skill_factory = _skill_factory_for_target(target) + assert skill_config is not None + resolved_skill_model = get_model(skill_config, config) async def answer_question(question: str) -> str: - answer, _ = await qa.answer(question) - return answer - - eval_name = name if name is not None else f"{spec.key}_qa_evaluation" - - experiment_metadata = build_experiment_metadata( - dataset_key=spec.key, - test_cases=len(cases), - config=config, - judge_config=judge_config, - ) + result = await run_skill_question( + skill_factory=skill_factory, + db_path=db, + config=config, + question=question, + skill_model=resolved_skill_model, + ) + return result.answer report = await evaluation_dataset.evaluate( answer_question, @@ -361,6 +420,8 @@ async def evaluate_dataset( vacuum_interval: int = 100, multimodal_only: bool = False, judge_model: ModelConfig | None = None, + target: Target = "qa", + skill_model: ModelConfig | None = None, ) -> None: if not skip_db: console.print(f"Using dataset: {spec.key}", style="bold magenta") @@ -380,7 +441,9 @@ async def evaluate_dataset( ) if not skip_qa: - console.print("\nRunning QA benchmarks...", style="bold yellow") + console.print( + f"\nRunning QA benchmarks (target={target})...", style="bold yellow" + ) await run_qa_benchmark( spec, config, @@ -388,6 +451,8 @@ async def evaluate_dataset( name=name, db_path=db_path, judge_model=judge_model, + target=target, + skill_model=skill_model, ) @@ -462,10 +527,33 @@ def run( "--judge-model", help="Judge model as 'provider:name' (e.g. 'ollama:gpt-oss').", ), + target: str = typer.Option( + "qa", + "--target", + help="What to benchmark: qa | rag-skill | analysis-skill.", + ), + skill_model: str | None = typer.Option( + None, + "--skill-model", + help=( + "Skill model as 'provider:name'. Used when --target is rag-skill or " + "analysis-skill. Defaults to qa.model from the config." + ), + ), ) -> None: spec = _resolve_dataset(dataset) app_config = _load_config(config) + if target not in TARGETS: + raise typer.BadParameter( + f"Unknown target {target!r}. Choose from: {', '.join(TARGETS)}" + ) + target_value = cast(Target, target) judge_model_config = parse_model_option(judge_model) if judge_model else None + skill_model_config = parse_model_option(skill_model) if skill_model else None + if target_value == "qa" and skill_model_config is not None: + raise typer.BadParameter( + "--skill-model is only valid when --target is rag-skill or analysis-skill." + ) asyncio.run( evaluate_dataset( @@ -480,6 +568,8 @@ def run( vacuum_interval=vacuum_interval, multimodal_only=multimodal_only, judge_model=judge_model_config, + target=target_value, + skill_model=skill_model_config, ) ) diff --git a/evaluations/evaluations/skill_runner.py b/evaluations/evaluations/skill_runner.py new file mode 100644 index 00000000..7164604c --- /dev/null +++ b/evaluations/evaluations/skill_runner.py @@ -0,0 +1,92 @@ +from collections.abc import Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Protocol, cast + +from pydantic_ai.models import Model + +from haiku.rag.agents.research.models import Citation +from haiku.rag.config.models import AppConfig +from haiku.rag.store.models.chunk import SearchResult +from haiku.skills import run_skill +from haiku.skills.models import Skill + +SkillFactory = Callable[..., Skill] + + +class _RagLikeState(Protocol): + document_filter: str | None + citation_index: dict[str, Citation] + citations: list[str] + searches: dict[str, list[SearchResult]] + + +@dataclass +class SkillRunResult: + answer: str + cited_uris: list[str] = field(default_factory=list) + cited_chunk_ids: list[str] = field(default_factory=list) + searched_uris: list[str] = field(default_factory=list) + n_searches: int = 0 + + +async def run_skill_question( + skill_factory: SkillFactory, + db_path: Path, + config: AppConfig, + question: str, + skill_model: str | Model, + document_filter: str | None = None, + request_limit: int | None = None, +) -> SkillRunResult: + """Run a single question through a skill and return answer + retrieval data. + + Builds the skill via ``skill_factory(db_path=..., config=...)`` and + invokes it with a fresh state instance derived from + ``skill.state_type``. After the run, citations and searched documents + are extracted from the state for downstream eval scoring. + + The skill must produce a state with RAG-skill-shaped fields (citation + index, searches, optional document filter) — i.e. ``RAGState`` or + ``AnalysisState`` from ``haiku.rag.skills``. + """ + skill = skill_factory(db_path=db_path, config=config) + if request_limit is not None: + skill.request_limit = request_limit + + if skill.state_type is None: + raise ValueError(f"Skill {skill.metadata.name!r} has no state_type") + state = skill.state_type() + typed = cast(_RagLikeState, state) + if document_filter is not None: + typed.document_filter = document_filter + + answer, _, _ = await run_skill(skill_model, skill, question, state=state) + + cited_chunk_ids: list[str] = list(typed.citations) + seen_cited: set[str] = set() + cited_uris: list[str] = [] + for chunk_id in cited_chunk_ids: + citation = typed.citation_index.get(chunk_id) + if citation is None: + continue + if citation.document_uri not in seen_cited: + seen_cited.add(citation.document_uri) + cited_uris.append(citation.document_uri) + + seen_searched: set[str] = set() + searched_uris: list[str] = [] + for results in typed.searches.values(): + for result in results: + uri = result.document_uri + if uri and uri not in seen_searched: + seen_searched.add(uri) + searched_uris.append(uri) + + return SkillRunResult( + answer=answer, + cited_uris=cited_uris, + cited_chunk_ids=cited_chunk_ids, + searched_uris=searched_uris, + n_searches=len(typed.searches), + ) diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py index a13ba514..1595c679 100644 --- a/evaluations/tests/test_benchmark.py +++ b/evaluations/tests/test_benchmark.py @@ -192,3 +192,133 @@ class TestEvaluateDatasetJudgeModel: mock_qa.assert_called_once() assert mock_qa.call_args[1]["judge_model"] is custom_judge + + +class TestExperimentMetadataTargets: + def test_default_target_is_qa(self) -> None: + result = build_experiment_metadata( + dataset_key="test", test_cases=1, config=AppConfig() + ) + assert result["target"] == "qa" + assert "skill_provider" not in result + assert "skill_model" not in result + + def test_skill_target_includes_skill_config(self) -> None: + skill = ModelConfig(provider="ollama", name="gpt-oss-large", temperature=0.2) + result = build_experiment_metadata( + dataset_key="test", + test_cases=1, + config=AppConfig(), + target="rag-skill", + skill_config=skill, + ) + assert result["target"] == "rag-skill" + assert result["skill_provider"] == "ollama" + assert result["skill_model"] == "gpt-oss-large" + assert result["skill_temperature"] == 0.2 + + +class TestEvaluateDatasetTarget: + def _spec(self) -> DatasetSpec: + return DatasetSpec( + key="test", + db_filename="test.lancedb", + document_loader=lambda: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + document_mapper=lambda doc: None, + qa_loader=lambda: [], # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + qa_case_builder=lambda idx, doc: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + ) + + @pytest.mark.asyncio + async def test_threads_target_and_skill_model(self) -> None: + skill = ModelConfig(provider="ollama", name="gpt-oss") + with patch( + "evaluations.benchmark.run_qa_benchmark", new_callable=AsyncMock + ) as mock_qa: + await evaluate_dataset( + spec=self._spec(), + config=AppConfig(), + skip_db=True, + skip_retrieval=True, + skip_qa=False, + limit=None, + name=None, + db_path=None, + target="rag-skill", + skill_model=skill, + ) + + mock_qa.assert_called_once() + assert mock_qa.call_args[1]["target"] == "rag-skill" + assert mock_qa.call_args[1]["skill_model"] is skill + + @pytest.mark.asyncio + async def test_default_target_is_qa(self) -> None: + with patch( + "evaluations.benchmark.run_qa_benchmark", new_callable=AsyncMock + ) as mock_qa: + await evaluate_dataset( + spec=self._spec(), + config=AppConfig(), + skip_db=True, + skip_retrieval=True, + skip_qa=False, + limit=None, + name=None, + db_path=None, + ) + assert mock_qa.call_args[1]["target"] == "qa" + assert mock_qa.call_args[1]["skill_model"] is None + + +class TestRunQaBenchmarkSkillTarget: + def _spec(self, tmp_path: Path) -> DatasetSpec: + return DatasetSpec( + key="test", + db_filename="test.lancedb", + document_loader=lambda: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + document_mapper=lambda doc: None, + qa_loader=lambda: [], # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + qa_case_builder=lambda idx, doc: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + ) + + @pytest.mark.asyncio + async def test_rag_skill_target_uses_run_skill_question( + self, tmp_path: Path + ) -> None: + from evaluations.skill_runner import SkillRunResult + + skill_run = AsyncMock(return_value=SkillRunResult(answer="from skill")) + with ( + patch("evaluations.benchmark.get_model") as mock_get_model, + patch( + "evaluations.benchmark.run_skill_question", new=skill_run + ) as mock_run_skill, + patch("evaluations.benchmark.HaikuRAG") as mock_haiku, + ): + mock_get_model.return_value = "fake-model" + await run_qa_benchmark( + self._spec(tmp_path), + AppConfig(), + db_path=tmp_path / "test.lancedb", + target="rag-skill", + ) + + # When target is rag-skill, HaikuRAG context manager is NOT entered + # (the skill manages its own client via lifespan). + mock_haiku.assert_not_called() + # skill model defaults to qa.model when not provided + skill_call = mock_get_model.call_args_list[-1] + assert skill_call[0][0] == AppConfig().qa.model + assert mock_run_skill is skill_run + + @pytest.mark.asyncio + async def test_analysis_skill_target_resolves_factory(self, tmp_path: Path) -> None: + from evaluations.benchmark import _skill_factory_for_target + from haiku.rag.skills.analysis import create_skill as analysis_factory + from haiku.rag.skills.rag import create_skill as rag_factory + + assert _skill_factory_for_target("rag-skill") is rag_factory + assert _skill_factory_for_target("analysis-skill") is analysis_factory + with pytest.raises(ValueError, match="not a skill target"): + _skill_factory_for_target("qa") # type: ignore[arg-type] diff --git a/evaluations/tests/test_skill_runner.py b/evaluations/tests/test_skill_runner.py new file mode 100644 index 00000000..4ba46e7b --- /dev/null +++ b/evaluations/tests/test_skill_runner.py @@ -0,0 +1,313 @@ +import random +from pathlib import Path +from typing import Any + +import pytest +from pydantic_ai.models.test import TestModel + +from evaluations.skill_runner import SkillRunResult, run_skill_question +from haiku.rag.agents.research.models import Citation +from haiku.rag.client import HaikuRAG +from haiku.rag.config.models import AppConfig +from haiku.rag.embeddings import EmbedderWrapper +from haiku.rag.skills.analysis import ( + AnalysisState, + create_skill as create_analysis_skill, +) +from haiku.rag.skills.rag import RAGState, create_skill as create_rag_skill +from haiku.rag.store.models.chunk import SearchResult + +VECTOR_DIM = 2560 + + +@pytest.fixture(autouse=True) +def mock_embedder(monkeypatch: pytest.MonkeyPatch): + """Deterministic embeddings so search is reproducible.""" + + async def fake_embed_query(self, text): + random.seed(hash(text) % (2**32)) + return [random.random() for _ in range(VECTOR_DIM)] + + async def fake_embed_documents(self, texts): + result = [] + for t in texts: + random.seed(hash(t) % (2**32)) + result.append([random.random() for _ in range(VECTOR_DIM)]) + return result + + monkeypatch.setattr(EmbedderWrapper, "embed_query", fake_embed_query) + monkeypatch.setattr(EmbedderWrapper, "embed_documents", fake_embed_documents) + + +@pytest.fixture +def app_config(): + return AppConfig(environment="skill-runner-test") + + +@pytest.fixture +async def rag_db(tmp_path: Path): + """A small two-document database.""" + db_path = tmp_path / "test.lancedb" + async with HaikuRAG(db_path, create=True) as rag: + await rag.create_document( + "Artificial intelligence is transforming healthcare and finance.", + title="AI Overview", + uri="test://ai", + ) + await rag.create_document( + "Machine learning includes supervised, unsupervised, and reinforcement.", + title="ML Basics", + uri="test://ml", + ) + return db_path + + +class TestRunSkillQuestionMocked: + """Verify the runner reads state correctly without going through a real skill loop.""" + + async def test_extracts_cited_and_searched_uris( + self, monkeypatch: pytest.MonkeyPatch, app_config: AppConfig, rag_db: Path + ) -> None: + async def fake_run_skill( + model: Any, + skill: Any, + request: str, + state: Any = None, + event_sink: Any = None, + ) -> tuple[str, list[Any], list[Any]]: + state.citation_index["c1"] = Citation( + chunk_id="c1", + document_id="d1", + document_uri="test://doc-a", + document_title="A", + content="alpha", + ) + state.citation_index["c2"] = Citation( + chunk_id="c2", + document_id="d2", + document_uri="test://doc-b", + document_title="B", + content="beta", + ) + state.citations = ["c1", "c2"] + state.searches["q1"] = [ + SearchResult(content="x", score=0.9, document_uri="test://doc-a"), + SearchResult(content="y", score=0.8, document_uri="test://doc-c"), + ] + state.searches["q2"] = [ + SearchResult(content="z", score=0.7, document_uri="test://doc-a"), + ] + return "answer", [], [] + + monkeypatch.setattr("evaluations.skill_runner.run_skill", fake_run_skill) + + result = await run_skill_question( + skill_factory=create_rag_skill, + db_path=rag_db, + config=app_config, + question="anything?", + skill_model=TestModel(), + ) + + assert isinstance(result, SkillRunResult) + assert result.answer == "answer" + assert result.cited_chunk_ids == ["c1", "c2"] + assert result.cited_uris == ["test://doc-a", "test://doc-b"] + assert result.searched_uris == ["test://doc-a", "test://doc-c"] + assert result.n_searches == 2 + + async def test_skips_chunks_missing_from_index( + self, monkeypatch: pytest.MonkeyPatch, app_config: AppConfig, rag_db: Path + ) -> None: + async def fake_run_skill( + model: Any, + skill: Any, + request: str, + state: Any = None, + event_sink: Any = None, + ) -> tuple[str, list[Any], list[Any]]: + state.citation_index["c1"] = Citation( + chunk_id="c1", + document_id="d1", + document_uri="test://doc-a", + content="a", + ) + state.citations = ["c1", "missing"] + return "ok", [], [] + + monkeypatch.setattr("evaluations.skill_runner.run_skill", fake_run_skill) + + result = await run_skill_question( + skill_factory=create_rag_skill, + db_path=rag_db, + config=app_config, + question="?", + skill_model=TestModel(), + ) + + assert result.cited_chunk_ids == ["c1", "missing"] + assert result.cited_uris == ["test://doc-a"] + + async def test_document_filter_is_set_on_state( + self, monkeypatch: pytest.MonkeyPatch, app_config: AppConfig, rag_db: Path + ) -> None: + captured: dict = {} + + async def fake_run_skill( + model: Any, + skill: Any, + request: str, + state: Any = None, + event_sink: Any = None, + ) -> tuple[str, list[Any], list[Any]]: + captured["filter"] = state.document_filter + captured["state_type"] = type(state) + return "ok", [], [] + + monkeypatch.setattr("evaluations.skill_runner.run_skill", fake_run_skill) + + await run_skill_question( + skill_factory=create_rag_skill, + db_path=rag_db, + config=app_config, + question="?", + skill_model=TestModel(), + document_filter="uri = 'test://ai'", + ) + + assert captured["filter"] == "uri = 'test://ai'" + assert captured["state_type"] is RAGState + + async def test_request_limit_override( + self, monkeypatch: pytest.MonkeyPatch, app_config: AppConfig, rag_db: Path + ) -> None: + captured: dict = {} + + async def fake_run_skill( + model: Any, + skill: Any, + request: str, + state: Any = None, + event_sink: Any = None, + ) -> tuple[str, list[Any], list[Any]]: + captured["request_limit"] = skill.request_limit + return "ok", [], [] + + monkeypatch.setattr("evaluations.skill_runner.run_skill", fake_run_skill) + + await run_skill_question( + skill_factory=create_rag_skill, + db_path=rag_db, + config=app_config, + question="?", + skill_model=TestModel(), + request_limit=42, + ) + + assert captured["request_limit"] == 42 + + async def test_request_limit_unset_leaves_skill_default( + self, monkeypatch: pytest.MonkeyPatch, app_config: AppConfig, rag_db: Path + ) -> None: + captured: dict = {} + + async def fake_run_skill( + model: Any, + skill: Any, + request: str, + state: Any = None, + event_sink: Any = None, + ) -> tuple[str, list[Any], list[Any]]: + captured["request_limit"] = skill.request_limit + return "ok", [], [] + + monkeypatch.setattr("evaluations.skill_runner.run_skill", fake_run_skill) + + await run_skill_question( + skill_factory=create_rag_skill, + db_path=rag_db, + config=app_config, + question="?", + skill_model=TestModel(), + ) + + assert captured["request_limit"] is None + + async def test_analysis_skill_uses_analysis_state( + self, monkeypatch: pytest.MonkeyPatch, app_config: AppConfig, rag_db: Path + ) -> None: + captured: dict = {} + + async def fake_run_skill( + model: Any, + skill: Any, + request: str, + state: Any = None, + event_sink: Any = None, + ) -> tuple[str, list[Any], list[Any]]: + captured["state_type"] = type(state) + return "ok", [], [] + + monkeypatch.setattr("evaluations.skill_runner.run_skill", fake_run_skill) + + await run_skill_question( + skill_factory=create_analysis_skill, + db_path=rag_db, + config=app_config, + question="?", + skill_model=TestModel(), + ) + + assert captured["state_type"] is AnalysisState + + async def test_raises_when_skill_has_no_state_type( + self, app_config: AppConfig, rag_db: Path + ) -> None: + from haiku.skills.models import Skill, SkillMetadata, SkillSource + + def factory(*, db_path, config) -> Skill: + return Skill( + metadata=SkillMetadata(name="bare", description="No state."), + source=SkillSource.ENTRYPOINT, + instructions="Do nothing.", + ) + + with pytest.raises(ValueError, match="no state_type"): + await run_skill_question( + skill_factory=factory, + db_path=rag_db, + config=app_config, + question="?", + skill_model=TestModel(), + ) + + +class TestRunSkillQuestionEndToEnd: + """Real skill loop against a real LanceDB. Verifies the wiring beyond mocks.""" + + async def test_rag_skill_runs_against_real_db( + self, + allow_model_requests: None, + app_config: AppConfig, + rag_db: Path, + ) -> None: + result = await run_skill_question( + skill_factory=create_rag_skill, + db_path=rag_db, + config=app_config, + question="What is machine learning?", + skill_model=TestModel(), + ) + + assert isinstance(result, SkillRunResult) + assert result.answer + assert result.n_searches >= 1 + assert all(uri.startswith("test://") for uri in result.searched_uris) + + +@pytest.fixture +def allow_model_requests(): + import pydantic_ai.models + + with pydantic_ai.models.override_allow_model_requests(True): + yield From 7d288c525ed88d126fc141dabd79d329873ed5e4 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 28 Apr 2026 12:46:00 +0300 Subject: [PATCH 3/5] citation retrieval scoring --- evaluations/evaluations/benchmark.py | 149 +++++++++++++----- .../evaluations/evaluators/__init__.py | 6 + .../evaluations/evaluators/citation.py | 60 +++++++ evaluations/tests/test_benchmark.py | 91 +++++++++++ evaluations/tests/test_citation_evaluators.py | 82 ++++++++++ 5 files changed, 351 insertions(+), 37 deletions(-) create mode 100644 evaluations/evaluations/evaluators/citation.py create mode 100644 evaluations/tests/test_citation_evaluators.py diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 1216787b..d5979c7a 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -1,6 +1,6 @@ import asyncio import shutil -from collections.abc import Mapping +from collections.abc import Awaitable, Callable, Mapping from pathlib import Path from typing import Any, Literal, cast @@ -8,15 +8,21 @@ import logfire import typer from dotenv import find_dotenv, load_dotenv from huggingface_hub import HfApi, snapshot_download -from pydantic_evals import Case, Dataset as EvalDataset -from pydantic_evals.evaluators import LLMJudge +from pydantic_evals import Case, Dataset as EvalDataset, set_eval_attribute +from pydantic_evals.evaluators import Evaluator, LLMJudge from pydantic_evals.reporting import ReportCaseFailure from rich.console import Console from rich.progress import Progress from evaluations.config import DatasetSpec from evaluations.datasets import DATASETS -from evaluations.evaluators import ANSWER_EQUIVALENCE_RUBRIC +from evaluations.evaluators import ( + ANSWER_EQUIVALENCE_RUBRIC, + CitationMAPEvaluator, + CitationMRREvaluator, + MAPEvaluator, + MRREvaluator, +) from evaluations.skill_runner import SkillFactory, run_skill_question from haiku.rag.client import HaikuRAG from haiku.rag.config import AppConfig, find_config_file, load_yaml_config @@ -25,6 +31,11 @@ from haiku.rag.logging import configure_cli_logging from haiku.rag.agents.qa import get_qa_agent from haiku.rag.utils import get_model, parse_model_option +_CITATION_EVALUATORS: dict[type[Evaluator], type[Evaluator]] = { + MRREvaluator: CitationMRREvaluator, + MAPEvaluator: CitationMAPEvaluator, +} + Target = Literal["qa", "rag-skill", "analysis-skill"] TARGETS: tuple[Target, ...] = ("qa", "rag-skill", "analysis-skill") @@ -289,6 +300,44 @@ def _skill_factory_for_target(target: Target) -> SkillFactory: raise ValueError(f"target {target!r} is not a skill target") +def _citation_evaluator_for(retrieval_evaluator: Evaluator | None) -> Evaluator | None: + """Return the citation-scoring twin of the dataset's retrieval evaluator.""" + if retrieval_evaluator is None: + return None + twin = _CITATION_EVALUATORS.get(type(retrieval_evaluator)) + return twin() if twin is not None else None + + +def _attach_relevant_uris( + cases: list[Case[str, str, dict[str, Any]]], + spec: DatasetSpec, + limit: int | None, +) -> None: + """Augment QA cases with `relevant_uris` joined from retrieval samples. + + Mutates each case's metadata in place. Cases with no matching retrieval + sample (by question) are left untouched. + """ + if spec.retrieval_loader is None or spec.retrieval_mapper is None: + return + corpus = spec.retrieval_loader() + if limit is not None: + corpus = corpus.select(range(min(limit, len(corpus)))) + expected_by_question: dict[str, tuple[str, ...]] = {} + for raw in corpus: + sample = spec.retrieval_mapper(cast(Mapping[str, Any], raw)) + if sample is None or sample.skip: + continue + expected_by_question[sample.question] = sample.expected_uris + for case in cases: + uris = expected_by_question.get(case.inputs) + if uris is None: + continue + metadata = case.metadata if case.metadata is not None else {} + metadata["relevant_uris"] = list(uris) + case.metadata = metadata + + async def run_qa_benchmark( spec: DatasetSpec, config: AppConfig, @@ -309,28 +358,33 @@ async def run_qa_benchmark( ] judge_config = judge_model or config.qa.model - judge = get_model(judge_config, config) + skill_config = (skill_model or config.qa.model) if target != "qa" else None + db = spec.db_path(db_path) + + citation_evaluator: Evaluator | None = None + if target != "qa": + _attach_relevant_uris(cases, spec, limit) + citation_evaluator = _citation_evaluator_for(spec.retrieval_evaluator) + + evaluators: list[Evaluator] = [ + LLMJudge( + rubric=ANSWER_EQUIVALENCE_RUBRIC, + include_input=True, + include_expected_output=True, + model=get_model(judge_config, config), + assertion={ + "evaluation_name": "answer_equivalent", + "include_reason": True, + }, + ), + ] + if citation_evaluator is not None: + evaluators.append(citation_evaluator) evaluation_dataset = EvalDataset[str, str, dict[str, str]]( - name=spec.key, - cases=cases, - evaluators=[ - LLMJudge( - rubric=ANSWER_EQUIVALENCE_RUBRIC, - include_input=True, - include_expected_output=True, - model=judge, - assertion={ - "evaluation_name": "answer_equivalent", - "include_reason": True, - }, - ), - ], + name=spec.key, cases=cases, evaluators=evaluators ) - db = spec.db_path(db_path) - skill_config = skill_model or config.qa.model if target != "qa" else None - eval_name = name if name is not None else f"{spec.key}_qa_evaluation" experiment_metadata = build_experiment_metadata( dataset_key=spec.key, @@ -341,6 +395,15 @@ async def run_qa_benchmark( skill_config=skill_config, ) + async def _evaluate(answer_fn: Callable[[str], Awaitable[str]]): + return await evaluation_dataset.evaluate( + answer_fn, + name=eval_name, + max_concurrency=1, + progress=True, + metadata=experiment_metadata, + ) + if target == "qa": async with HaikuRAG(db, config=config) as rag: qa = get_qa_agent( @@ -351,13 +414,7 @@ async def run_qa_benchmark( answer, _ = await qa.answer(question) return answer - report = await evaluation_dataset.evaluate( - answer_question, - name=eval_name, - max_concurrency=1, - progress=True, - metadata=experiment_metadata, - ) + report = await _evaluate(answer_question) else: skill_factory = _skill_factory_for_target(target) assert skill_config is not None @@ -371,15 +428,10 @@ async def run_qa_benchmark( question=question, skill_model=resolved_skill_model, ) + set_eval_attribute("cited_uris", result.cited_uris) return result.answer - report = await evaluation_dataset.evaluate( - answer_question, - name=eval_name, - max_concurrency=1, - progress=True, - metadata=experiment_metadata, - ) + report = await _evaluate(answer_question) passing_cases = sum( 1 @@ -389,7 +441,6 @@ async def run_qa_benchmark( ) total_processed = len(report.cases) failures = report.failures - accuracy = passing_cases / total_processed if total_processed > 0 else 0 console.print("\n=== QA Benchmark Results ===", style="bold cyan") @@ -397,6 +448,30 @@ async def run_qa_benchmark( console.print(f"Correct answers: {passing_cases}") console.print(f"QA Accuracy: {accuracy:.4f} ({accuracy * 100:.2f}%)") + if citation_evaluator is not None: + score_key = citation_evaluator.get_default_evaluation_name() + scores = [ + case.scores[score_key].value + for case in report.cases + if score_key in case.scores + ] + if scores: + cited_count = sum( + 1 for case in report.cases if case.attributes.get("cited_uris") + ) + mean_citations = sum( + len(case.attributes.get("cited_uris") or []) for case in report.cases + ) / len(report.cases) + mean_score = sum(scores) / len(scores) + console.print( + f"\n=== Citation Retrieval ({score_key}) ===", style="bold cyan" + ) + console.print(f"Mean {score_key}: {mean_score:.4f}") + console.print( + f"Cite rate (≥1 citation): {cited_count / len(report.cases):.2%}" + ) + console.print(f"Mean citations per case: {mean_citations:.2f}") + if failures: console.print("[red]\nSummary of failures:[/red]") for failure in failures: diff --git a/evaluations/evaluations/evaluators/__init__.py b/evaluations/evaluations/evaluators/__init__.py index cf28d8fc..6a855a4f 100644 --- a/evaluations/evaluations/evaluators/__init__.py +++ b/evaluations/evaluations/evaluators/__init__.py @@ -1,3 +1,7 @@ +from evaluations.evaluators.citation import ( + CitationMAPEvaluator, + CitationMRREvaluator, +) from evaluations.evaluators.judge import ( ANSWER_EQUIVALENCE_RUBRIC, LLMJudge, @@ -8,6 +12,8 @@ from evaluations.evaluators.mrr import MRREvaluator __all__ = [ "ANSWER_EQUIVALENCE_RUBRIC", + "CitationMAPEvaluator", + "CitationMRREvaluator", "LLMJudge", "LLMJudgeResponseSchema", "MAPEvaluator", diff --git a/evaluations/evaluations/evaluators/citation.py b/evaluations/evaluations/evaluators/citation.py new file mode 100644 index 00000000..9f226704 --- /dev/null +++ b/evaluations/evaluations/evaluators/citation.py @@ -0,0 +1,60 @@ +from dataclasses import dataclass + +from pydantic_evals.evaluators import Evaluator, EvaluatorContext + + +def _cited_uris(ctx: EvaluatorContext) -> list[str]: + return list(ctx.attributes.get("cited_uris") or []) + + +def _relevant_uris(ctx: EvaluatorContext) -> set[str]: + if ctx.metadata is None: + return set() + return set(ctx.metadata.get("relevant_uris", [])) + + +@dataclass +class CitationMRREvaluator(Evaluator): + """Reciprocal rank over the URIs the skill cited via the `cite` tool. + + Reads ``cited_uris`` from ``ctx.attributes`` (recorded during the task run + via :func:`pydantic_evals.set_eval_attribute`) and ``relevant_uris`` from + ``ctx.metadata``. Returns ``1.0/rank`` of the first cited URI that is in + the relevant set, or ``0.0`` if none match. + + Use for single-document datasets, mirroring :class:`MRREvaluator`. + """ + + evaluation_name: str = "cited_mrr" + + def evaluate(self, ctx: EvaluatorContext) -> float: + relevant = _relevant_uris(ctx) + for rank, uri in enumerate(_cited_uris(ctx), start=1): + if uri in relevant: + return 1.0 / rank + return 0.0 + + +@dataclass +class CitationMAPEvaluator(Evaluator): + """Average precision over the URIs the skill cited via the `cite` tool. + + Same input shape as :class:`CitationMRREvaluator`; use for multi-document + datasets, mirroring :class:`MAPEvaluator`. + """ + + evaluation_name: str = "cited_map" + + def evaluate(self, ctx: EvaluatorContext) -> float: + relevant = _relevant_uris(ctx) + if not relevant: + return 0.0 + precisions: list[float] = [] + found = 0 + for rank, uri in enumerate(_cited_uris(ctx), start=1): + if uri in relevant: + found += 1 + precisions.append(found / rank) + if not precisions: + return 0.0 + return sum(precisions) / len(relevant) diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py index 1595c679..a80cc7fc 100644 --- a/evaluations/tests/test_benchmark.py +++ b/evaluations/tests/test_benchmark.py @@ -322,3 +322,94 @@ class TestRunQaBenchmarkSkillTarget: assert _skill_factory_for_target("analysis-skill") is analysis_factory with pytest.raises(ValueError, match="not a skill target"): _skill_factory_for_target("qa") # type: ignore[arg-type] + + +class TestCitationEvaluatorWiring: + def test_returns_mrr_twin_for_mrr_evaluator(self) -> None: + from evaluations.benchmark import _citation_evaluator_for + from evaluations.evaluators import CitationMRREvaluator, MRREvaluator + + result = _citation_evaluator_for(MRREvaluator()) + assert isinstance(result, CitationMRREvaluator) + + def test_returns_map_twin_for_map_evaluator(self) -> None: + from evaluations.benchmark import _citation_evaluator_for + from evaluations.evaluators import CitationMAPEvaluator, MAPEvaluator + + result = _citation_evaluator_for(MAPEvaluator()) + assert isinstance(result, CitationMAPEvaluator) + + def test_returns_none_for_no_evaluator(self) -> None: + from evaluations.benchmark import _citation_evaluator_for + + assert _citation_evaluator_for(None) is None + + +class TestAttachRelevantUris: + def test_joins_by_question(self) -> None: + from pydantic_evals import Case + + from evaluations.benchmark import _attach_relevant_uris + from evaluations.config import RetrievalSample + from evaluations.evaluators import MRREvaluator + + cases: list[Case[str, str, dict]] = [ + Case(name="c1", inputs="What is X?", expected_output="X is a thing"), + Case( + name="c2", + inputs="What is Y?", + expected_output="Y is another", + metadata={"existing": "value"}, + ), + Case( + name="c3", + inputs="What is Z?", + expected_output="not in retrieval set", + ), + ] + + spec = DatasetSpec( + key="test", + db_filename="test.lancedb", + document_loader=lambda: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + document_mapper=lambda doc: None, + qa_loader=lambda: [], # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + qa_case_builder=lambda idx, doc: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + retrieval_loader=lambda: [ # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + {"q": "What is X?", "uris": ("uri-x",)}, + {"q": "What is Y?", "uris": ("uri-y1", "uri-y2")}, + ], + retrieval_mapper=lambda d: RetrievalSample( + question=d["q"], expected_uris=d["uris"] + ), + retrieval_evaluator=MRREvaluator(), + ) + + _attach_relevant_uris(cases, spec, limit=None) + + assert cases[0].metadata == {"relevant_uris": ["uri-x"]} + assert cases[1].metadata == { + "existing": "value", + "relevant_uris": ["uri-y1", "uri-y2"], + } + # case c3 has no matching retrieval sample — metadata untouched + assert cases[2].metadata is None + + def test_no_op_without_retrieval_loader(self) -> None: + from pydantic_evals import Case + + from evaluations.benchmark import _attach_relevant_uris + + cases: list[Case[str, str, dict]] = [ + Case(name="c1", inputs="q", expected_output="a"), + ] + spec = DatasetSpec( + key="test", + db_filename="test.lancedb", + document_loader=lambda: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + document_mapper=lambda doc: None, + qa_loader=lambda: [], # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + qa_case_builder=lambda idx, doc: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + ) + _attach_relevant_uris(cases, spec, limit=None) + assert cases[0].metadata is None diff --git a/evaluations/tests/test_citation_evaluators.py b/evaluations/tests/test_citation_evaluators.py new file mode 100644 index 00000000..1cd88846 --- /dev/null +++ b/evaluations/tests/test_citation_evaluators.py @@ -0,0 +1,82 @@ +from unittest.mock import MagicMock + +from evaluations.evaluators.citation import ( + CitationMAPEvaluator, + CitationMRREvaluator, +) + + +def _ctx(cited: list[str], relevant: list[str]) -> MagicMock: + ctx = MagicMock() + ctx.metadata = {"relevant_uris": relevant} + ctx.attributes = {"cited_uris": cited} + return ctx + + +class TestCitationMRREvaluator: + def setup_method(self) -> None: + self.evaluator = CitationMRREvaluator() + + def test_first_citation_is_relevant(self) -> None: + assert self.evaluator.evaluate(_ctx(["a", "b"], ["a"])) == 1.0 + + def test_second_citation_is_relevant(self) -> None: + assert self.evaluator.evaluate(_ctx(["a", "b"], ["b"])) == 0.5 + + def test_no_citations(self) -> None: + assert self.evaluator.evaluate(_ctx([], ["a"])) == 0.0 + + def test_no_relevant(self) -> None: + assert self.evaluator.evaluate(_ctx(["a"], [])) == 0.0 + + def test_no_matches(self) -> None: + assert self.evaluator.evaluate(_ctx(["a", "b"], ["c"])) == 0.0 + + def test_metadata_none(self) -> None: + ctx = MagicMock() + ctx.metadata = None + ctx.attributes = {"cited_uris": ["a"]} + assert self.evaluator.evaluate(ctx) == 0.0 + + def test_attribute_missing(self) -> None: + ctx = MagicMock() + ctx.metadata = {"relevant_uris": ["a"]} + ctx.attributes = {} + assert self.evaluator.evaluate(ctx) == 0.0 + + def test_evaluation_name(self) -> None: + assert self.evaluator.evaluation_name == "cited_mrr" + + +class TestCitationMAPEvaluator: + def setup_method(self) -> None: + self.evaluator = CitationMAPEvaluator() + + def test_all_relevant_first(self) -> None: + # Both relevant docs cited at ranks 1 and 2: AP = (1/1 + 2/2) / 2 = 1.0 + assert self.evaluator.evaluate(_ctx(["a", "b"], ["a", "b"])) == 1.0 + + def test_partial_match(self) -> None: + # Cited a, x, b. relevant a, b. P@1 = 1/1, P@3 = 2/3. AP = (1 + 2/3)/2 + assert ( + self.evaluator.evaluate(_ctx(["a", "x", "b"], ["a", "b"])) + == (1.0 + 2 / 3) / 2 + ) + + def test_no_matches(self) -> None: + assert self.evaluator.evaluate(_ctx(["x", "y"], ["a", "b"])) == 0.0 + + def test_no_relevant(self) -> None: + assert self.evaluator.evaluate(_ctx(["a"], [])) == 0.0 + + def test_no_citations(self) -> None: + assert self.evaluator.evaluate(_ctx([], ["a"])) == 0.0 + + def test_metadata_none(self) -> None: + ctx = MagicMock() + ctx.metadata = None + ctx.attributes = {"cited_uris": ["a"]} + assert self.evaluator.evaluate(ctx) == 0.0 + + def test_evaluation_name(self) -> None: + assert self.evaluator.evaluation_name == "cited_map" From 7bfb818d77513a6a3867d1968ff5fd707e4d30ae Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 28 Apr 2026 14:33:25 +0300 Subject: [PATCH 4/5] remove dataset-specific system prompts --- evaluations/evaluations/benchmark.py | 4 +- evaluations/evaluations/config.py | 11 ----- .../evaluations/datasets/open_rag_bench.py | 41 ------------------- evaluations/evaluations/datasets/wix.py | 22 ---------- evaluations/evaluations/optimization.py | 2 +- evaluations/tests/test_config.py | 20 --------- evaluations/tests/test_optimization.py | 1 - 7 files changed, 2 insertions(+), 99 deletions(-) diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index d5979c7a..d963da4a 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -406,9 +406,7 @@ async def run_qa_benchmark( if target == "qa": async with HaikuRAG(db, config=config) as rag: - qa = get_qa_agent( - rag, config, system_prompt=spec.resolve_system_prompt(config) - ) + qa = get_qa_agent(rag, config) async def answer_question(question: str) -> str: answer, _ = await qa.answer(question) diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py index 580235fc..063fb104 100644 --- a/evaluations/evaluations/config.py +++ b/evaluations/evaluations/config.py @@ -7,8 +7,6 @@ from datasets import Dataset from pydantic_evals import Case from pydantic_evals.evaluators import Evaluator -from haiku.rag.config.models import AppConfig - @dataclass class DocumentPayload: @@ -47,7 +45,6 @@ class DatasetSpec: retrieval_mapper: RetrievalMapper | None = None retrieval_evaluator: Evaluator | None = None document_limit: int | None = None - system_prompt: str | None = None def db_path(self, override_path: Path | None = None) -> Path: """Get the database path. @@ -65,11 +62,3 @@ class DatasetSpec: data_dir = get_default_data_dir() return data_dir / "evaluations" / "dbs" / self.db_filename - - def resolve_system_prompt(self, config: AppConfig) -> str | None: - """Resolve the QA system prompt. - - Precedence: config.prompts.qa > spec.system_prompt > None - (get_qa_agent handles the final fallback to QA_SYSTEM_PROMPT) - """ - return config.prompts.qa or self.system_prompt diff --git a/evaluations/evaluations/datasets/open_rag_bench.py b/evaluations/evaluations/datasets/open_rag_bench.py index 79b869e2..f11be906 100644 --- a/evaluations/evaluations/datasets/open_rag_bench.py +++ b/evaluations/evaluations/datasets/open_rag_bench.py @@ -14,46 +14,6 @@ from evaluations.evaluators import MAPEvaluator logger = logging.getLogger(__name__) -ORB_SYSTEM_PROMPT = """You are a knowledgeable assistant that answers questions using a document knowledge base. - -Process: -1. Call search_documents with relevant keywords from the question -2. Review the results ordered by relevance -3. If needed, perform follow-up searches with different keywords (max 3 total) -4. Provide a concise answer based strictly on the retrieved content - -The search tool returns results like: -[chunk_abc123] [rank 1 of 5] -Source: "Document Title" > Section > Subsection -Type: paragraph -Content: -The actual text content here... - -[chunk_def456] [rank 2 of 5] -Source: "Another Document" -Type: table -Content: -| Column 1 | Column 2 | -... - -Each result includes: -- chunk_id in brackets and rank position (rank 1 = most relevant) -- Source: document title and section hierarchy (when available) -- Type: content type like paragraph, table, code, list_item (when available) -- Content: the actual text - -In your response, include the chunk IDs you used in cited_chunks. - -Guidelines: -- Base answers strictly on retrieved content - do not use external knowledge -- Use the Source and Type metadata to understand context -- If multiple results are relevant, synthesize them coherently -- If information is insufficient, say: "I cannot find enough information in the knowledge base to answer this question." -- Be concise and direct - avoid elaboration unless asked -- Results are ordered by relevance, with rank 1 being most relevant -- IMPORTANT: Do not use LaTeX notation (like \\(...\\) or $...$) in your answers. Use plain text or Unicode math symbols instead. -""" - REPO_ID = "vectara/open_ragbench" PDF_SUBDIR = "pdf/arxiv" @@ -266,5 +226,4 @@ OPEN_RAG_BENCH_SPEC = DatasetSpec( retrieval_loader=load_orb_retrieval, retrieval_mapper=map_orb_retrieval, retrieval_evaluator=MAPEvaluator(), - system_prompt=ORB_SYSTEM_PROMPT, ) diff --git a/evaluations/evaluations/datasets/wix.py b/evaluations/evaluations/datasets/wix.py index 0fdcbda6..7e81fb8f 100644 --- a/evaluations/evaluations/datasets/wix.py +++ b/evaluations/evaluations/datasets/wix.py @@ -8,27 +8,6 @@ from pydantic_evals import Case from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample from evaluations.evaluators import MAPEvaluator -WIX_SUPPORT_PROMPT = """You are a WIX technical support expert helping users with questions about the WIX platform. - -Your process: -1. When a user asks a question, use the search_documents tool to find relevant information -2. Search with specific keywords and phrases from the user's question -3. Review the search results ordered by relevance (rank 1 = most relevant) -4. If you need additional context, perform follow-up searches with different keywords -5. Provide a short and to the point comprehensive answer based only on the retrieved documents - -Guidelines: -- Base your answers strictly on the provided document content -- Quote or reference specific information when possible -- If multiple documents contain relevant information, synthesize them coherently -- Indicate when information is incomplete or when you need to search for additional context -- If the retrieved documents don't contain sufficient information, clearly state: "I cannot find enough information in the knowledge base to answer this question." -- For complex questions, consider breaking them down and performing multiple searches -- Stick to the answer, do not ellaborate or provide context unless explicitly asked for it. - -Be concise, and always maintain accuracy over completeness. Prefer short, direct answers that are well-supported by the documents. -""" - def load_wix_corpus() -> Dataset: dataset_dict = load_dataset("Wix/WixQA", "wix_kb_corpus") @@ -102,5 +81,4 @@ WIX_SPEC = DatasetSpec( retrieval_loader=load_wix_qa, retrieval_mapper=map_wix_retrieval, retrieval_evaluator=MAPEvaluator(), - system_prompt=WIX_SUPPORT_PROMPT, ) diff --git a/evaluations/evaluations/optimization.py b/evaluations/evaluations/optimization.py index 6a768406..ae68af96 100644 --- a/evaluations/evaluations/optimization.py +++ b/evaluations/evaluations/optimization.py @@ -236,7 +236,7 @@ def run_optimization( reflect_config = reflect_model or config.qa.model reflection_lm = ReflectionLM(reflect_config, config) - seed_prompt = spec.resolve_system_prompt(config) or QA_SYSTEM_PROMPT + seed_prompt = config.prompts.qa or QA_SYSTEM_PROMPT seed_candidate = {"instructions": seed_prompt} mid = len(cases) // 2 diff --git a/evaluations/tests/test_config.py b/evaluations/tests/test_config.py index 60f96aee..37342701 100644 --- a/evaluations/tests/test_config.py +++ b/evaluations/tests/test_config.py @@ -2,7 +2,6 @@ from pathlib import Path from unittest.mock import patch from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample -from haiku.rag.config.models import AppConfig def _make_spec(**kwargs: object) -> DatasetSpec: @@ -52,25 +51,6 @@ class TestDatasetSpecDefaults: assert spec.retrieval_mapper is None assert spec.retrieval_evaluator is None assert spec.document_limit is None - assert spec.system_prompt is None - - -class TestResolveSystemPrompt: - def test_config_prompt_overrides_spec_prompt(self) -> None: - spec = _make_spec(system_prompt="spec prompt") - config = AppConfig() - config.prompts.qa = "config prompt" - assert spec.resolve_system_prompt(config) == "config prompt" - - def test_spec_prompt_used_when_config_unset(self) -> None: - spec = _make_spec(system_prompt="spec prompt") - config = AppConfig() - assert spec.resolve_system_prompt(config) == "spec prompt" - - def test_returns_none_when_both_unset(self) -> None: - spec = _make_spec() - config = AppConfig() - assert spec.resolve_system_prompt(config) is None class TestDocumentPayload: diff --git a/evaluations/tests/test_optimization.py b/evaluations/tests/test_optimization.py index 7f108072..0e832b19 100644 --- a/evaluations/tests/test_optimization.py +++ b/evaluations/tests/test_optimization.py @@ -291,7 +291,6 @@ class TestRunOptimization: document_mapper=lambda doc: None, qa_loader=lambda: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] qa_case_builder=lambda idx, doc: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - system_prompt="You are a test assistant.", ) def test_returns_results(self, tmp_path: Path, gepa_mock_result: MagicMock) -> None: From 5e31c159078ecd91551771d3a95fd9170a99e669 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 28 Apr 2026 14:44:27 +0300 Subject: [PATCH 5/5] docs --- CHANGELOG.md | 10 ++++++++++ docs/benchmarks.md | 8 ++++++++ evaluations/README.md | 15 +++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6229d49e..4ec86a6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,16 @@ # Changelog ## [Unreleased] +### Added + +- **Skill-based QA evaluation via `evaluations run --target {qa,rag-skill,analysis-skill}`.** Benchmark the RAG and analysis skills end-to-end alongside the existing QA agent path, against the same datasets and judge. `--skill-model "provider:name"` overrides the skill model independently from the judge. +- **Citation retrieval as a second eval metric.** `CitationMRREvaluator` and `CitationMAPEvaluator` score the URIs the skill registered via the `cite` tool against each dataset's gold `expected_uris`, alongside the existing LLMJudge. Console output gains a "Citation Retrieval" summary (mean score, cite rate, mean citations per case). Zero extra skill runs — cited URIs are surfaced via `pydantic_evals.set_eval_attribute`. +- Bumps `haiku.skills` to `>=0.16.0` for the public `run_skill` API and `Skill.request_limit`. + +### Changed + +- **Removed dataset-specific eval system prompts.** `WIX_SUPPORT_PROMPT` and `ORB_SYSTEM_PROMPT` duplicated guidance already in the shipped `QA_SYSTEM_PROMPT` and `SKILL.md`, and ORB's referenced the obsolete `search_documents` tool name. The eval-side machinery for injecting them (`DatasetSpec.system_prompt`, `resolve_system_prompt()`) is removed. `config.prompts.qa` remains as the user-facing override knob. + ## [0.43.1] - 2026-04-25 ### Fixed diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 9add1242..56ce43da 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -61,6 +61,8 @@ evaluations run repliqa --config /path/to/haiku.rag.yaml --db /path/to/custom.la - `--limit N` - Limit number of test cases - `--name NAME` - Override the evaluation name - `--judge-model PROVIDER:NAME` - Override the LLM judge model (default: `config.qa.model`) +- `--target {qa,rag-skill,analysis-skill}` - Choose what to benchmark (default: `qa`). `rag-skill` and `analysis-skill` run the corresponding [skill](skills/index.md) end-to-end against the same datasets and judge as the QA agent. +- `--skill-model PROVIDER:NAME` - Override the skill model independently from the judge (default: `config.qa.model`). Only valid with skill targets. If no config file is specified, the script searches standard locations: `./haiku.rag.yaml`, user config directory, then falls back to defaults. @@ -87,6 +89,12 @@ If no config file is specified, the script searches standard locations: `./haiku For question-answering evaluation, `pydantic-evals` coordinates an LLM judge to determine whether answers are correct. By default the judge uses the same model as QA (`config.qa.model`); override with `--judge-model provider:name`. Accuracy is the fraction of correctly answered questions. +### Citation Retrieval + +When benchmarking a skill (`--target rag-skill` or `--target analysis-skill`), a second metric scores the URIs the skill registered via the `cite` tool against each dataset's gold `expected_uris`, using the same MRR / MAP math as raw retrieval. The score key is `cited_mrr` for single-doc datasets and `cited_map` for multi-doc. Console output also includes the cite rate (% of cases with at least one citation) and the mean number of citations per case. + +This is computed alongside QA accuracy from the same skill run — no extra invocations. The signal complements raw retrieval: where raw retrieval measures whether the retriever surfaced the gold document at any rank, citation retrieval measures whether the skill grounded its answer on it. + ## RepliQA [RepliQA](https://huggingface.co/datasets/ServiceNow/repliqa) contains synthetic news stories with question-answer pairs. We use `News Stories` from `repliqa_3` (1035 documents). Each question has exactly one relevant document, so we use MRR for retrieval evaluation. diff --git a/evaluations/README.md b/evaluations/README.md index 8781a1be..8777c3c3 100644 --- a/evaluations/README.md +++ b/evaluations/README.md @@ -39,6 +39,21 @@ evaluations run repliqa --skip-qa evaluations run repliqa --limit 100 ``` +### Benchmarking the skills + +By default `evaluations run` benchmarks the QA agent. Pass `--target` to +benchmark the RAG or analysis skill instead, against the same datasets and judge: + +```bash +evaluations run wix --target rag-skill +evaluations run wix --target analysis-skill --skill-model ollama:gpt-oss +``` + +`--skill-model "provider:name"` overrides the skill model independently from +the judge (defaults to `qa.model`). For skill targets, a citation retrieval +metric (`cited_mrr` / `cited_map`) is computed alongside QA accuracy from the +URIs the skill registered via the `cite` tool. + ### Pre-built Databases Download pre-built evaluation databases from HuggingFace: