From 044da7ae99b63abacdf225c4067fdd4d4b2dc0c0 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Tue, 28 Jul 2026 11:36:27 +0300 Subject: [PATCH] Open the eval database read-only outside population Retrieval and QA only read from the database, but the benchmark opened it writable, where an embedder identity differing from the stored one aborts instead of warning. Running a pre-built database against a different serving stack then needed a `rebuild --set-embedder` first. Correct the debug-evals skill alongside it: the pydantic-ai span names are `execute_tool {tool_name}` and `invoke_agent agent`, targets are `{rag,analysis}-capability`, and no `skill_model` metadata key exists. --- .claude/skills/debug-evals/SKILL.md | 6 +++--- CHANGELOG.md | 1 + evaluations/evaluations/benchmark.py | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.claude/skills/debug-evals/SKILL.md b/.claude/skills/debug-evals/SKILL.md index 238481f9..4a279e75 100644 --- a/.claude/skills/debug-evals/SKILL.md +++ b/.claude/skills/debug-evals/SKILL.md @@ -31,7 +31,7 @@ A run is one experiment span; its cases are direct children sharing its - `attributes->>'name'` — run label (the `--name` arg, or `{dataset}_qa_evaluation` / `{dataset}_retrieval_evaluation`). - `attributes->>'dataset_name'` — dataset. - `(attributes->>'assertion_pass_rate')::float` — overall judge pass rate (QA runs). - - `attributes->'logfire.experiment.metadata'->'metadata'` — run config: `target` (`rag-skill`|`analysis-skill`), `qa_model`, `embedder_model`, `chunk_size`, `search_limit`, `rerank_model`, `judge_model`, `skill_model`, etc. + - `attributes->'logfire.experiment.metadata'->'metadata'` — run config: `target` (`rag-capability`|`analysis-capability`), `qa_model`, `embedder_model`, `chunk_size`, `search_limit`, `rerank_model`, `judge_model`, `qa_max_searches`, etc. - `trace_id` — scopes the whole run. - Case span: `span_name = 'case: {case_name}'` (scope `pydantic-evals`). - `message` — `case: `. @@ -39,8 +39,8 @@ A run is one experiment span; its cases are direct children sharing its - `attributes->'scores'->'cited_map'->>'value'` — citation average precision (0..1). - `attributes->'scores'->'number_match'->>'value'` — numeric-answer match (datasets that use it). - `duration` — task time in seconds. -- Inside each case the skill under test emits agent spans (scope `pydantic-ai`): - `execute {task}`, `agent run`, `running tool`, `chat {model}`. +- Inside each case the capability under test emits agent spans (scope `pydantic-ai`): + `execute {task}`, `invoke_agent agent`, `execute_tool {tool_name}`, `chat {model}`. The service is `evals` regardless of model, so filter on `service_name = 'evals'` first. `otel_scope_name` separates the layers (`pydantic-evals` for run/case, diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c8577c2..70ecb1d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - `Store.set_haiku_version` stamps the store's own config into a recreated settings row instead of the process-global `Config`. - `check_source_accessible` returns `False` for a URI it cannot resolve (unparseable host, unreadable path) instead of raising and aborting a full rebuild. +- `evaluations run` opens the database read-only outside the population phase, so an embedder identity differing from the stored one warns instead of aborting the run. ### Removed diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index bf2125f5..b23d1826 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -227,7 +227,7 @@ async def run_retrieval_benchmark( ) db = spec.db_path(db_path) - async with HaikuRAG(db, config=config) as rag: + async with HaikuRAG(db, config=config, read_only=True) as rag: async def retrieval_target(question: str) -> list[str]: chunks = await rag.search(query=question, limit=5)