diff --git a/evaluations/configs/multidb.yaml b/evaluations/configs/multidb.yaml new file mode 100644 index 00000000..de089603 --- /dev/null +++ b/evaluations/configs/multidb.yaml @@ -0,0 +1,68 @@ +# Reference config for the `multidb` behaviour families: the acceptance gate for +# lancedb.databases. Build the corpus first, then run with --skip-db: +# uv run python -m evaluations.datasets.multidb --config configs/multidb.yaml +# evaluations run multidb --config configs/multidb.yaml --skip-db --skip-retrieval +# For the no-reranker arm, copy this and drop the reranking block; a reference +# config's filename has to name a dataset, so that arm lives outside configs/. + +environment: development + +storage: + auto_vacuum: false + +lancedb: + # Order matters: RRF ties resolve to the order listed here, so the first + # database wins ties. B2 rotates the order it passes per case. + databases: + northern: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_northern.lancedb + southern: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_southern.lancedb + equipment: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_equipment.lancedb + +processing: + # Pinned: the builder asserts no single chunk holds all twelve monthly + # readings, so S3 cannot be answered from one search hit. + chunk_size: 256 + +search: + # Three databases at limit 5 means RRF fuses 15 candidates to 5, so the + # truncation bites and the no-reranker arm can lose evidence. + limit: 5 + +embeddings: + model: + provider: openai + name: qwen3-embedding-4b + vector_dim: 2560 + base_url: http://vllm:11431/v1 + +reranking: + model: + provider: vllm + name: Qwen/Qwen3-Reranker-4B + base_url: http://vllm:11455 + +qa: + model: + provider: openai + name: Inferact/Qwen3.8-27B-NVFP4 + base_url: http://vllm:11439/v1 + max_tokens: 16384 + extra_body: + chat_template_kwargs: + reasoning_effort: low + +evaluations: + # Scoring is deterministic, but RefusalJudge runs on the B6 and B7 cases, + # which carry answerability labels. + judge: + provider: openai + name: Inferact/Qwen3.8-27B-NVFP4 + base_url: http://vllm:11439/v1 + temperature: 0.6 + max_tokens: 16384 + extra_body: + top_p: 0.95 + top_k: 20 + min_p: 0 + chat_template_kwargs: + reasoning_effort: low diff --git a/evaluations/configs/multidb_surfaces.yaml b/evaluations/configs/multidb_surfaces.yaml new file mode 100644 index 00000000..bc50363d --- /dev/null +++ b/evaluations/configs/multidb_surfaces.yaml @@ -0,0 +1,68 @@ +# Reference config for the `multidb_surfaces` analysis families (sandbox surfaces). +# lancedb.databases. Build the corpus first, then run with --skip-db: +# uv run python -m evaluations.datasets.multidb --config configs/multidb_surfaces.yaml +# evaluations run multidb_surfaces --config configs/multidb_surfaces.yaml --skip-db --skip-retrieval +# For the no-reranker arm, copy this and drop the reranking block; a reference +# config's filename has to name a dataset, so that arm lives outside configs/. + +environment: development + +storage: + auto_vacuum: false + +lancedb: + # Order matters: RRF ties resolve to the order listed here, so the first + # database wins ties. B2 rotates the order it passes per case. + databases: + northern: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_northern.lancedb + southern: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_southern.lancedb + equipment: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_equipment.lancedb + +processing: + # Pinned: the builder asserts no single chunk holds all twelve monthly + # readings, so S3 cannot be answered from one search hit. + chunk_size: 256 + +search: + # Three databases at limit 5 means RRF fuses 15 candidates to 5, so the + # truncation bites and the no-reranker arm can lose evidence. + limit: 5 + +embeddings: + model: + provider: openai + name: qwen3-embedding-4b + vector_dim: 2560 + base_url: http://vllm:11431/v1 + +reranking: + model: + provider: vllm + name: Qwen/Qwen3-Reranker-4B + base_url: http://vllm:11455 + +qa: + model: + provider: openai + name: Inferact/Qwen3.8-27B-NVFP4 + base_url: http://vllm:11439/v1 + max_tokens: 16384 + extra_body: + chat_template_kwargs: + reasoning_effort: low + +evaluations: + # Scoring is deterministic, but RefusalJudge runs on the B6 and B7 cases, + # which carry answerability labels. + judge: + provider: openai + name: Inferact/Qwen3.8-27B-NVFP4 + base_url: http://vllm:11439/v1 + temperature: 0.6 + max_tokens: 16384 + extra_body: + top_p: 0.95 + top_k: 20 + min_p: 0 + chat_template_kwargs: + reasoning_effort: low diff --git a/evaluations/evaluations/capability_runner.py b/evaluations/evaluations/capability_runner.py index 937754ff..ab8d6248 100644 --- a/evaluations/evaluations/capability_runner.py +++ b/evaluations/evaluations/capability_runner.py @@ -55,6 +55,9 @@ class CapabilityRunResult: n_failed_tools: int = 0 n_requests: int = 0 citation_status: str | None = None + # The Python the model itself ran, so a run can report which sandbox + # surfaces it actually reached rather than only how often it ran code. + executed_code: list[str] = field(default_factory=list) class ToolTraffic(NamedTuple): @@ -277,8 +280,9 @@ def _result_from_run( seen_searched.add(uri) searched_uris.append(uri) - executions = getattr(typed, "executions", None) - n_executions = len(executions) if executions is not None else 0 + executions = getattr(typed, "executions", None) or [] + n_executions = len(executions) + executed_code = [entry.code for entry in executions] record = typed.evidence status = ( @@ -304,4 +308,5 @@ def _result_from_run( n_failed_tools=traffic.n_failed_tools, n_requests=traffic.n_requests, citation_status=status, + executed_code=executed_code, ) diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py index 29b96145..8064541e 100644 --- a/evaluations/evaluations/config.py +++ b/evaluations/evaluations/config.py @@ -94,6 +94,9 @@ class DatasetSpec: live: bool = False compaction: bool = False experiment_metadata: dict[str, Any] | None = None + # Called with the report's cases after the run prints, for datasets that + # report something the shared summary cannot express (e.g. hard gates). + report_hook: Callable[[list[Any]], None] | None = None def uses_configured_databases( self, config, override_path: Path | None = None diff --git a/evaluations/evaluations/datasets/multidb.py b/evaluations/evaluations/datasets/multidb.py index 02da0818..7b772478 100644 --- a/evaluations/evaluations/datasets/multidb.py +++ b/evaluations/evaluations/datasets/multidb.py @@ -8,6 +8,7 @@ from datasets import Dataset from pydantic_evals import Case from evaluations.config import DatasetSpec, ScopedQuestion +from evaluations.datasets.multidb_report import render as render_gate_report from evaluations.evaluators.multidb import MultiDBScores from haiku.rag.client import HaikuRAG from haiku.rag.config.models import AppConfig @@ -772,6 +773,16 @@ def _unused_document_loader() -> Dataset: # pragma: no cover - never called ) +def print_gate_report(cases: list[Any]) -> None: + """Print the hard gates, per-family rates and surface coverage. + + Emits a greppable `GATES: PASSED` / `GATES: FAILED` line, since this dataset + is an acceptance gate rather than a rate to watch drift on. + """ + text, _passed = render_gate_report(cases) + print(text) + + MULTIDB_SPEC = DatasetSpec( key="multidb", db_filename="multidb_northern.lancedb", @@ -780,6 +791,7 @@ MULTIDB_SPEC = DatasetSpec( qa_loader=load_behaviour_questions, qa_case_builder=build_multidb_case, qa_evaluator=MultiDBScores(), + report_hook=print_gate_report, ) MULTIDB_SURFACES_SPEC = DatasetSpec( @@ -790,4 +802,5 @@ MULTIDB_SURFACES_SPEC = DatasetSpec( qa_loader=load_surface_questions, qa_case_builder=build_multidb_case, qa_evaluator=MultiDBScores(), + report_hook=print_gate_report, ) diff --git a/evaluations/evaluations/datasets/multidb_report.py b/evaluations/evaluations/datasets/multidb_report.py new file mode 100644 index 00000000..045fdba2 --- /dev/null +++ b/evaluations/evaluations/datasets/multidb_report.py @@ -0,0 +1,131 @@ +from collections import Counter +from dataclasses import dataclass +from typing import Any + +# A sandbox surface is counted as touched when the model's own Python names it. +# Coverage is measured, not required: a surface nothing reaches is a finding about +# whether it earns its place in the VFS, not a failure of the run. +SURFACE_MARKERS: dict[str, tuple[str, ...]] = { + "list_documents()": ("list_documents(",), + "in-code search()": ("search(",), + "content.txt": ("content.txt",), + "items.jsonl": ("items.jsonl",), + "toc.json": ("toc.json",), + "metadata.json": ("metadata.json",), +} + +# Families where exactly one database is correct, so a citation elsewhere is an +# attribution error rather than a difference of opinion. +SINGLE_SOURCE_FAMILIES = ("B1", "B3", "B4") + + +def _flag(case: Any, name: str) -> bool | None: + """Read a boolean assertion, or a score recorded as a number.""" + assertion = (getattr(case, "assertions", None) or {}).get(name) + if assertion is not None: + return bool(assertion.value) + score = (getattr(case, "scores", None) or {}).get(name) + if score is not None: + return bool(score.value >= 1.0) + return None + + +def _meta(case: Any, key: str, default: Any = None) -> Any: + return (getattr(case, "metadata", None) or {}).get(key, default) + + +@dataclass +class Gate: + name: str + checked: int + failures: list[str] + + @property + def passed(self) -> bool: + return not self.failures + + def line(self) -> str: + state = "PASS" if self.passed else f"FAIL ({len(self.failures)})" + detail = "" if self.passed else " " + ", ".join(self.failures[:6]) + return f"{state:<12} {self.name} over {self.checked} case(s){detail}" + + +def scope_gate(cases: list[Any]) -> Gate: + """No scoped case cites a database outside its scope. Includes the empty + scope, which may cite nothing at all.""" + checked, failures = 0, [] + for case in cases: + if _meta(case, "scope") is None: + continue + checked += 1 + if _flag(case, "scope_honoured") is False: + failures.append(case.name) + return Gate("scope leakage", checked, failures) + + +def attribution_gate(cases: list[Any]) -> Gate: + """Every single-source case cites exactly the database that holds the answer. + `cited_map` scores URIs and would pass a wrongly attributed answer.""" + checked, failures = 0, [] + for case in cases: + if _meta(case, "family") not in SINGLE_SOURCE_FAMILIES: + continue + if _meta(case, "expected_sources") is None: + continue + checked += 1 + if _flag(case, "attribution_correct") is False: + failures.append(case.name) + return Gate("attribution errors", checked, failures) + + +def family_rates(cases: list[Any]) -> dict[str, tuple[int, int]]: + """Answered-correctly over scored, per family. Families with no numeric or + ordered-text expectation (the refusal families) are absent rather than 0/0.""" + passed: Counter[str] = Counter() + total: Counter[str] = Counter() + for case in cases: + family = _meta(case, "family") + flag = _flag(case, "answer_correct") + if family is None or flag is None: + continue + total[family] += 1 + passed[family] += int(flag) + return {family: (passed[family], total[family]) for family in sorted(total)} + + +def surface_coverage(cases: list[Any]) -> dict[str, int]: + """How many cases reached each sandbox surface, from the recorded Python.""" + counts = dict.fromkeys(SURFACE_MARKERS, 0) + for case in cases: + code = " ".join( + (getattr(case, "attributes", None) or {}).get("executed_code") or [] + ) + if not code: + continue + for surface, markers in SURFACE_MARKERS.items(): + if any(marker in code for marker in markers): + counts[surface] += 1 + return counts + + +def render(cases: list[Any]) -> tuple[str, bool]: + """The gate report. Returns the text and whether both hard gates passed.""" + gates = [scope_gate(cases), attribution_gate(cases)] + lines = ["", "=== Multi-database gates ===", ""] + lines += [gate.line() for gate in gates] + + lines += ["", "=== Answer rates by family ===", ""] + for family, (passed, total) in family_rates(cases).items(): + share = f"{passed / total:.0%}" if total else "-" + lines.append(f"{family:<4} {passed:>3}/{total:<3} {share}") + + coverage = surface_coverage(cases) + if any(coverage.values()): + lines += ["", "=== Sandbox surfaces reached ===", ""] + for surface, count in coverage.items(): + note = "" if count else " (never reached)" + lines.append(f"{surface:<18} {count:>3} case(s){note}") + + all_passed = all(gate.passed for gate in gates) + lines += ["", f"GATES: {'PASSED' if all_passed else 'FAILED'}", ""] + return "\n".join(lines), all_passed diff --git a/evaluations/evaluations/evaluators/multidb.py b/evaluations/evaluations/evaluators/multidb.py index 97c0691e..2bae9259 100644 --- a/evaluations/evaluations/evaluators/multidb.py +++ b/evaluations/evaluations/evaluators/multidb.py @@ -47,7 +47,7 @@ class NumericAnswer(Evaluator): def get_default_evaluation_name(self) -> str: return "answer_correct" - def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool]: + def evaluate(self, ctx: EvaluatorContext) -> dict[str, float | bool]: meta = ctx.metadata or {} expected = _as_floats(meta.get("expected_value"), meta.get("expected_values")) if not expected: @@ -57,7 +57,7 @@ class NumericAnswer(Evaluator): meta.get("distractor_value"), meta.get("distractor_values") ) correct = expected <= found and not (forbidden & found) - return {"answer_correct": correct} + return {"answer_correct": 1.0 if correct else 0.0} @dataclass @@ -71,7 +71,7 @@ class AttributionGate(Evaluator): def get_default_evaluation_name(self) -> str: return "attribution_correct" - def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool]: + def evaluate(self, ctx: EvaluatorContext) -> dict[str, float | bool]: meta = ctx.metadata or {} expected = meta.get("expected_sources") if expected is None: @@ -90,7 +90,7 @@ class ScopeGate(Evaluator): def get_default_evaluation_name(self) -> str: return "scope_honoured" - def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool]: + def evaluate(self, ctx: EvaluatorContext) -> dict[str, float | bool]: meta = ctx.metadata or {} scope = meta.get("scope") if scope is None: @@ -109,7 +109,7 @@ class TextAnswer(Evaluator): def get_default_evaluation_name(self) -> str: return "answer_correct" - def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool]: + def evaluate(self, ctx: EvaluatorContext) -> dict[str, float | bool]: meta = ctx.metadata or {} required = meta.get("expected_ordered") if required is None: @@ -119,12 +119,12 @@ class TextAnswer(Evaluator): for needle in required: found = haystack.find(str(needle).lower(), cursor) if found < 0: - return {"answer_correct": False} + return {"answer_correct": 0.0} cursor = found + len(str(needle)) for forbidden in meta.get("forbidden_text") or []: if str(forbidden).lower() in haystack: - return {"answer_correct": False} - return {"answer_correct": True} + return {"answer_correct": 0.0} + return {"answer_correct": 1.0} @dataclass @@ -136,7 +136,10 @@ class MultiDBScores(Evaluator): on cases whose metadata does not ask for it. """ - def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool]: + def get_default_evaluation_name(self) -> str: + return "answer_correct" + + def evaluate(self, ctx: EvaluatorContext) -> dict[str, float | bool]: meta = ctx.metadata or {} numeric = bool(meta.get("expected_value") or meta.get("expected_values")) textual = meta.get("expected_ordered") is not None @@ -145,7 +148,7 @@ class MultiDBScores(Evaluator): f"case {meta.get('question_id')!r} asks for both a numeric and an " "ordered-text answer; they share the answer_correct key" ) - scores: dict[str, bool] = {} + scores: dict[str, float | bool] = {} for evaluator in ( NumericAnswer(), TextAnswer(), diff --git a/evaluations/evaluations/qa.py b/evaluations/evaluations/qa.py index d6a5e271..61207706 100644 --- a/evaluations/evaluations/qa.py +++ b/evaluations/evaluations/qa.py @@ -377,6 +377,7 @@ async def run_qa_benchmark( set_eval_attribute("n_executions", result.n_executions) set_eval_attribute("n_requests", result.n_requests) set_eval_attribute("citation_status", result.citation_status) + set_eval_attribute("executed_code", result.executed_code) return result.answer report = await evaluation_dataset.evaluate( @@ -450,6 +451,9 @@ async def run_qa_benchmark( "(PARTIAL excluded)" ) + if spec.report_hook is not None: + spec.report_hook(list(report.cases)) + _print_failures(failures, show_question=True) return failures[0] if failures else None diff --git a/evaluations/tests/test_multidb_evaluators.py b/evaluations/tests/test_multidb_evaluators.py index cf62a149..24d45e6f 100644 --- a/evaluations/tests/test_multidb_evaluators.py +++ b/evaluations/tests/test_multidb_evaluators.py @@ -25,7 +25,7 @@ def test_gold_number_counts_as_correct(): result = NumericAnswer().evaluate( ctx("It sits at 1240 metres.", {"expected_value": 1240}) ) - assert result == {"answer_correct": True} + assert result == {"answer_correct": 1.0} def test_hedging_between_gold_and_distractor_fails(): @@ -37,7 +37,7 @@ def test_hedging_between_gold_and_distractor_fails(): {"expected_value": 1240, "distractor_value": 2310}, ) ) - assert result == {"answer_correct": False} + assert result == {"answer_correct": 0.0} def test_numeric_answer_abstains_without_a_gold_value(): @@ -100,10 +100,10 @@ def test_ordered_text_must_appear_in_order(): meta = {"expected_ordered": ["Overview", "Instruments", "Measurements"]} assert TextAnswer().evaluate( ctx("Overview, then Instruments, then Measurements", meta) - ) == {"answer_correct": True} + ) == {"answer_correct": 1.0} assert TextAnswer().evaluate( ctx("Measurements, then Overview, then Instruments", meta) - ) == {"answer_correct": False} + ) == {"answer_correct": 0.0} def test_forbidden_text_fails_even_when_required_text_is_present(): @@ -116,4 +116,4 @@ def test_forbidden_text_fails_even_when_required_text_is_present(): }, ) ) - assert result == {"answer_correct": False} + assert result == {"answer_correct": 0.0} diff --git a/evaluations/tests/test_multidb_report.py b/evaluations/tests/test_multidb_report.py new file mode 100644 index 00000000..9cd347e4 --- /dev/null +++ b/evaluations/tests/test_multidb_report.py @@ -0,0 +1,151 @@ +from types import SimpleNamespace + +from evaluations.datasets.multidb_report import ( + attribution_gate, + family_rates, + render, + scope_gate, + surface_coverage, +) + + +def case( + name="0_x", + *, + family=None, + scope=None, + expected_sources=None, + scope_honoured=None, + attribution_correct=None, + answer_correct=None, + code=None, +): + assertions = {} + if scope_honoured is not None: + assertions["scope_honoured"] = SimpleNamespace(value=scope_honoured) + if attribution_correct is not None: + assertions["attribution_correct"] = SimpleNamespace(value=attribution_correct) + scores = {} + if answer_correct is not None: + scores["answer_correct"] = SimpleNamespace(value=answer_correct) + metadata = {} + if family is not None: + metadata["family"] = family + if scope is not None: + metadata["scope"] = scope + if expected_sources is not None: + metadata["expected_sources"] = expected_sources + return SimpleNamespace( + name=name, + assertions=assertions, + scores=scores, + metadata=metadata, + attributes={"executed_code": code} if code else {}, + ) + + +def test_scope_gate_counts_only_scoped_cases(): + cases = [ + case("0_b1", family="B1"), + case("1_b4", family="B4", scope=["northern"], scope_honoured=True), + case("2_b4", family="B4", scope=["northern"], scope_honoured=False), + ] + gate = scope_gate(cases) + assert gate.checked == 2 + assert gate.failures == ["2_b4"] + assert gate.passed is False + + +def test_scope_gate_passes_when_nothing_leaks(): + gate = scope_gate([case("0_b4", family="B4", scope=[], scope_honoured=True)]) + assert gate.passed + assert "PASS" in gate.line() + + +def test_attribution_gate_ignores_families_with_more_than_one_answer(): + """B5 is answered by two databases, so it is not an attribution error to cite + both and the gate must not police it.""" + cases = [ + case( + "0_b5", + family="B5", + expected_sources=["northern", "southern"], + attribution_correct=False, + ), + case( + "1_b1", family="B1", expected_sources=["northern"], attribution_correct=True + ), + ] + gate = attribution_gate(cases) + assert gate.checked == 1 + assert gate.passed + + +def test_attribution_gate_reports_the_offending_case(): + gate = attribution_gate( + [ + case( + "3_b3", + family="B3", + expected_sources=["northern"], + attribution_correct=False, + ) + ] + ) + assert gate.failures == ["3_b3"] + assert "FAIL (1)" in gate.line() + assert "3_b3" in gate.line() + + +def test_family_rates_skip_cases_with_no_answer_score(): + """The refusal families have no numeric expectation, so they must not appear + as 0/0 and drag a family rate down.""" + cases = [ + case("0_b1", family="B1", answer_correct=1.0), + case("1_b1", family="B1", answer_correct=0.0), + case("2_b6", family="B6"), + ] + rates = family_rates(cases) + assert rates == {"B1": (1, 2)} + assert "B6" not in rates + + +def test_surface_coverage_counts_cases_not_snippets(): + cases = [ + case("0", code=["print(toc)", "open('/documents/1/toc.json')"]), + case("1", code=["open('/documents/2/content.txt')"]), + case("2"), + ] + coverage = surface_coverage(cases) + assert coverage["toc.json"] == 1 + assert coverage["content.txt"] == 1 + assert coverage["items.jsonl"] == 0 + + +def test_render_says_passed_only_when_both_gates_pass(): + good = [ + case( + "0_b4", + family="B4", + scope=["northern"], + scope_honoured=True, + expected_sources=["northern"], + attribution_correct=True, + answer_correct=1.0, + ), + ] + text, passed = render(good) + assert passed + assert "GATES: PASSED" in text + + bad = good + [case("1_b4", family="B4", scope=["northern"], scope_honoured=False)] + text, passed = render(bad) + assert not passed + assert "GATES: FAILED" in text + assert "1_b4" in text + + +def test_render_flags_a_surface_nothing_reached(): + text, _ = render([case("0", family="S5", code=["open('/documents/1/toc.json')"])]) + assert "never reached" in text + assert "toc.json" in text diff --git a/evaluations/tests/test_reference_configs.py b/evaluations/tests/test_reference_configs.py index 142ccb34..a3f75135 100644 --- a/evaluations/tests/test_reference_configs.py +++ b/evaluations/tests/test_reference_configs.py @@ -44,15 +44,16 @@ def test_filename_names_a_dataset(path: Path) -> None: @pytest.mark.parametrize("path", _config_paths(), ids=lambda p: p.stem) def test_judge_pinned_where_the_judge_runs(path: Path) -> None: - """Datasets without their own qa_evaluator are scored by the LLM judge. + """Wherever a judge runs, its sampling must be frozen so accuracy stays + comparable across runs. - Those configs must carry the frozen judge settings, so accuracy stays - comparable across runs. Datasets that bring a deterministic evaluator - never construct a judge, so a judge block there would be dead config. + A dataset without its own qa_evaluator is scored by the LLM judge and must + carry the block. A dataset with a deterministic evaluator may still need one, + because RefusalJudge runs on any case carrying an answerability label; where + it declares no judge it is asserting that no case does. """ judge = _load(path).evaluations.judge - if DATASETS[path.stem].qa_evaluator is not None: - assert judge is None + if DATASETS[path.stem].qa_evaluator is not None and judge is None: return assert judge is not None