From 18e1127375f80f016bf2a96e448473fd113e165a Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 26 Aug 2026 15:17:42 +0300 Subject: [PATCH] Report the multi-database gates and which surfaces were reached The two hard gates are pass/fail, not rates: any scope leak or attribution error is a bug. The report names the offending cases and prints a greppable GATES: PASSED / FAILED line, since this dataset is an acceptance gate rather than a number to watch drift on. The attribution gate covers only families where exactly one database is correct. B5 is answered by two, so citing both is right there and policing it would report a bug that is not one. Surface coverage comes from the model's own Python, now carried on the run result: counting executions said how often code ran but never which surface it reached. Coverage is measured, not required, so a surface nothing reaches is a finding about whether it earns its place in the VFS. answer_correct is recorded as a score rather than a boolean, since booleans become assertions and the run reads its accuracy headline from scores. The two gates stay boolean assertions, which is what they are. DatasetSpec grows a report_hook so this lives with the dataset instead of becoming a key check in the shared QA reporting. The reference-config invariant said a dataset with a deterministic evaluator must declare no judge. That is not true here: RefusalJudge runs on any case carrying an answerability label, which B6 and B7 do, so its sampling has to be pinned. The assertion now requires a pinned block wherever a judge runs and allows its absence only as the claim that none does. --- evaluations/configs/multidb.yaml | 68 ++++++++ evaluations/configs/multidb_surfaces.yaml | 68 ++++++++ evaluations/evaluations/capability_runner.py | 9 +- evaluations/evaluations/config.py | 3 + evaluations/evaluations/datasets/multidb.py | 13 ++ .../evaluations/datasets/multidb_report.py | 131 +++++++++++++++ evaluations/evaluations/evaluators/multidb.py | 23 +-- evaluations/evaluations/qa.py | 4 + evaluations/tests/test_multidb_evaluators.py | 10 +- evaluations/tests/test_multidb_report.py | 151 ++++++++++++++++++ evaluations/tests/test_reference_configs.py | 13 +- 11 files changed, 470 insertions(+), 23 deletions(-) create mode 100644 evaluations/configs/multidb.yaml create mode 100644 evaluations/configs/multidb_surfaces.yaml create mode 100644 evaluations/evaluations/datasets/multidb_report.py create mode 100644 evaluations/tests/test_multidb_report.py diff --git a/evaluations/configs/multidb.yaml b/evaluations/configs/multidb.yaml new file mode 100644 index 00000000..de089603 --- /dev/null +++ b/evaluations/configs/multidb.yaml @@ -0,0 +1,68 @@ +# Reference config for the `multidb` behaviour families: the acceptance gate for +# lancedb.databases. Build the corpus first, then run with --skip-db: +# uv run python -m evaluations.datasets.multidb --config configs/multidb.yaml +# evaluations run multidb --config configs/multidb.yaml --skip-db --skip-retrieval +# For the no-reranker arm, copy this and drop the reranking block; a reference +# config's filename has to name a dataset, so that arm lives outside configs/. + +environment: development + +storage: + auto_vacuum: false + +lancedb: + # Order matters: RRF ties resolve to the order listed here, so the first + # database wins ties. B2 rotates the order it passes per case. + databases: + northern: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_northern.lancedb + southern: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_southern.lancedb + equipment: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_equipment.lancedb + +processing: + # Pinned: the builder asserts no single chunk holds all twelve monthly + # readings, so S3 cannot be answered from one search hit. + chunk_size: 256 + +search: + # Three databases at limit 5 means RRF fuses 15 candidates to 5, so the + # truncation bites and the no-reranker arm can lose evidence. + limit: 5 + +embeddings: + model: + provider: openai + name: qwen3-embedding-4b + vector_dim: 2560 + base_url: http://vllm:11431/v1 + +reranking: + model: + provider: vllm + name: Qwen/Qwen3-Reranker-4B + base_url: http://vllm:11455 + +qa: + model: + provider: openai + name: Inferact/Qwen3.8-27B-NVFP4 + base_url: http://vllm:11439/v1 + max_tokens: 16384 + extra_body: + chat_template_kwargs: + reasoning_effort: low + +evaluations: + # Scoring is deterministic, but RefusalJudge runs on the B6 and B7 cases, + # which carry answerability labels. + judge: + provider: openai + name: Inferact/Qwen3.8-27B-NVFP4 + base_url: http://vllm:11439/v1 + temperature: 0.6 + max_tokens: 16384 + extra_body: + top_p: 0.95 + top_k: 20 + min_p: 0 + chat_template_kwargs: + reasoning_effort: low diff --git a/evaluations/configs/multidb_surfaces.yaml b/evaluations/configs/multidb_surfaces.yaml new file mode 100644 index 00000000..bc50363d --- /dev/null +++ b/evaluations/configs/multidb_surfaces.yaml @@ -0,0 +1,68 @@ +# Reference config for the `multidb_surfaces` analysis families (sandbox surfaces). +# lancedb.databases. Build the corpus first, then run with --skip-db: +# uv run python -m evaluations.datasets.multidb --config configs/multidb_surfaces.yaml +# evaluations run multidb_surfaces --config configs/multidb_surfaces.yaml --skip-db --skip-retrieval +# For the no-reranker arm, copy this and drop the reranking block; a reference +# config's filename has to name a dataset, so that arm lives outside configs/. + +environment: development + +storage: + auto_vacuum: false + +lancedb: + # Order matters: RRF ties resolve to the order listed here, so the first + # database wins ties. B2 rotates the order it passes per case. + databases: + northern: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_northern.lancedb + southern: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_southern.lancedb + equipment: ${HOME}/.local/share/haiku.rag/evaluations/dbs/multidb_equipment.lancedb + +processing: + # Pinned: the builder asserts no single chunk holds all twelve monthly + # readings, so S3 cannot be answered from one search hit. + chunk_size: 256 + +search: + # Three databases at limit 5 means RRF fuses 15 candidates to 5, so the + # truncation bites and the no-reranker arm can lose evidence. + limit: 5 + +embeddings: + model: + provider: openai + name: qwen3-embedding-4b + vector_dim: 2560 + base_url: http://vllm:11431/v1 + +reranking: + model: + provider: vllm + name: Qwen/Qwen3-Reranker-4B + base_url: http://vllm:11455 + +qa: + model: + provider: openai + name: Inferact/Qwen3.8-27B-NVFP4 + base_url: http://vllm:11439/v1 + max_tokens: 16384 + extra_body: + chat_template_kwargs: + reasoning_effort: low + +evaluations: + # Scoring is deterministic, but RefusalJudge runs on the B6 and B7 cases, + # which carry answerability labels. + judge: + provider: openai + name: Inferact/Qwen3.8-27B-NVFP4 + base_url: http://vllm:11439/v1 + temperature: 0.6 + max_tokens: 16384 + extra_body: + top_p: 0.95 + top_k: 20 + min_p: 0 + chat_template_kwargs: + reasoning_effort: low diff --git a/evaluations/evaluations/capability_runner.py b/evaluations/evaluations/capability_runner.py index 937754ff..ab8d6248 100644 --- a/evaluations/evaluations/capability_runner.py +++ b/evaluations/evaluations/capability_runner.py @@ -55,6 +55,9 @@ class CapabilityRunResult: n_failed_tools: int = 0 n_requests: int = 0 citation_status: str | None = None + # The Python the model itself ran, so a run can report which sandbox + # surfaces it actually reached rather than only how often it ran code. + executed_code: list[str] = field(default_factory=list) class ToolTraffic(NamedTuple): @@ -277,8 +280,9 @@ def _result_from_run( seen_searched.add(uri) searched_uris.append(uri) - executions = getattr(typed, "executions", None) - n_executions = len(executions) if executions is not None else 0 + executions = getattr(typed, "executions", None) or [] + n_executions = len(executions) + executed_code = [entry.code for entry in executions] record = typed.evidence status = ( @@ -304,4 +308,5 @@ def _result_from_run( n_failed_tools=traffic.n_failed_tools, n_requests=traffic.n_requests, citation_status=status, + executed_code=executed_code, ) diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py index 29b96145..8064541e 100644 --- a/evaluations/evaluations/config.py +++ b/evaluations/evaluations/config.py @@ -94,6 +94,9 @@ class DatasetSpec: live: bool = False compaction: bool = False experiment_metadata: dict[str, Any] | None = None + # Called with the report's cases after the run prints, for datasets that + # report something the shared summary cannot express (e.g. hard gates). + report_hook: Callable[[list[Any]], None] | None = None def uses_configured_databases( self, config, override_path: Path | None = None diff --git a/evaluations/evaluations/datasets/multidb.py b/evaluations/evaluations/datasets/multidb.py index 02da0818..7b772478 100644 --- a/evaluations/evaluations/datasets/multidb.py +++ b/evaluations/evaluations/datasets/multidb.py @@ -8,6 +8,7 @@ from datasets import Dataset from pydantic_evals import Case from evaluations.config import DatasetSpec, ScopedQuestion +from evaluations.datasets.multidb_report import render as render_gate_report from evaluations.evaluators.multidb import MultiDBScores from haiku.rag.client import HaikuRAG from haiku.rag.config.models import AppConfig @@ -772,6 +773,16 @@ def _unused_document_loader() -> Dataset: # pragma: no cover - never called ) +def print_gate_report(cases: list[Any]) -> None: + """Print the hard gates, per-family rates and surface coverage. + + Emits a greppable `GATES: PASSED` / `GATES: FAILED` line, since this dataset + is an acceptance gate rather than a rate to watch drift on. + """ + text, _passed = render_gate_report(cases) + print(text) + + MULTIDB_SPEC = DatasetSpec( key="multidb", db_filename="multidb_northern.lancedb", @@ -780,6 +791,7 @@ MULTIDB_SPEC = DatasetSpec( qa_loader=load_behaviour_questions, qa_case_builder=build_multidb_case, qa_evaluator=MultiDBScores(), + report_hook=print_gate_report, ) MULTIDB_SURFACES_SPEC = DatasetSpec( @@ -790,4 +802,5 @@ MULTIDB_SURFACES_SPEC = DatasetSpec( qa_loader=load_surface_questions, qa_case_builder=build_multidb_case, qa_evaluator=MultiDBScores(), + report_hook=print_gate_report, ) diff --git a/evaluations/evaluations/datasets/multidb_report.py b/evaluations/evaluations/datasets/multidb_report.py new file mode 100644 index 00000000..045fdba2 --- /dev/null +++ b/evaluations/evaluations/datasets/multidb_report.py @@ -0,0 +1,131 @@ +from collections import Counter +from dataclasses import dataclass +from typing import Any + +# A sandbox surface is counted as touched when the model's own Python names it. +# Coverage is measured, not required: a surface nothing reaches is a finding about +# whether it earns its place in the VFS, not a failure of the run. +SURFACE_MARKERS: dict[str, tuple[str, ...]] = { + "list_documents()": ("list_documents(",), + "in-code search()": ("search(",), + "content.txt": ("content.txt",), + "items.jsonl": ("items.jsonl",), + "toc.json": ("toc.json",), + "metadata.json": ("metadata.json",), +} + +# Families where exactly one database is correct, so a citation elsewhere is an +# attribution error rather than a difference of opinion. +SINGLE_SOURCE_FAMILIES = ("B1", "B3", "B4") + + +def _flag(case: Any, name: str) -> bool | None: + """Read a boolean assertion, or a score recorded as a number.""" + assertion = (getattr(case, "assertions", None) or {}).get(name) + if assertion is not None: + return bool(assertion.value) + score = (getattr(case, "scores", None) or {}).get(name) + if score is not None: + return bool(score.value >= 1.0) + return None + + +def _meta(case: Any, key: str, default: Any = None) -> Any: + return (getattr(case, "metadata", None) or {}).get(key, default) + + +@dataclass +class Gate: + name: str + checked: int + failures: list[str] + + @property + def passed(self) -> bool: + return not self.failures + + def line(self) -> str: + state = "PASS" if self.passed else f"FAIL ({len(self.failures)})" + detail = "" if self.passed else " " + ", ".join(self.failures[:6]) + return f"{state:<12} {self.name} over {self.checked} case(s){detail}" + + +def scope_gate(cases: list[Any]) -> Gate: + """No scoped case cites a database outside its scope. Includes the empty + scope, which may cite nothing at all.""" + checked, failures = 0, [] + for case in cases: + if _meta(case, "scope") is None: + continue + checked += 1 + if _flag(case, "scope_honoured") is False: + failures.append(case.name) + return Gate("scope leakage", checked, failures) + + +def attribution_gate(cases: list[Any]) -> Gate: + """Every single-source case cites exactly the database that holds the answer. + `cited_map` scores URIs and would pass a wrongly attributed answer.""" + checked, failures = 0, [] + for case in cases: + if _meta(case, "family") not in SINGLE_SOURCE_FAMILIES: + continue + if _meta(case, "expected_sources") is None: + continue + checked += 1 + if _flag(case, "attribution_correct") is False: + failures.append(case.name) + return Gate("attribution errors", checked, failures) + + +def family_rates(cases: list[Any]) -> dict[str, tuple[int, int]]: + """Answered-correctly over scored, per family. Families with no numeric or + ordered-text expectation (the refusal families) are absent rather than 0/0.""" + passed: Counter[str] = Counter() + total: Counter[str] = Counter() + for case in cases: + family = _meta(case, "family") + flag = _flag(case, "answer_correct") + if family is None or flag is None: + continue + total[family] += 1 + passed[family] += int(flag) + return {family: (passed[family], total[family]) for family in sorted(total)} + + +def surface_coverage(cases: list[Any]) -> dict[str, int]: + """How many cases reached each sandbox surface, from the recorded Python.""" + counts = dict.fromkeys(SURFACE_MARKERS, 0) + for case in cases: + code = " ".join( + (getattr(case, "attributes", None) or {}).get("executed_code") or [] + ) + if not code: + continue + for surface, markers in SURFACE_MARKERS.items(): + if any(marker in code for marker in markers): + counts[surface] += 1 + return counts + + +def render(cases: list[Any]) -> tuple[str, bool]: + """The gate report. Returns the text and whether both hard gates passed.""" + gates = [scope_gate(cases), attribution_gate(cases)] + lines = ["", "=== Multi-database gates ===", ""] + lines += [gate.line() for gate in gates] + + lines += ["", "=== Answer rates by family ===", ""] + for family, (passed, total) in family_rates(cases).items(): + share = f"{passed / total:.0%}" if total else "-" + lines.append(f"{family:<4} {passed:>3}/{total:<3} {share}") + + coverage = surface_coverage(cases) + if any(coverage.values()): + lines += ["", "=== Sandbox surfaces reached ===", ""] + for surface, count in coverage.items(): + note = "" if count else " (never reached)" + lines.append(f"{surface:<18} {count:>3} case(s){note}") + + all_passed = all(gate.passed for gate in gates) + lines += ["", f"GATES: {'PASSED' if all_passed else 'FAILED'}", ""] + return "\n".join(lines), all_passed diff --git a/evaluations/evaluations/evaluators/multidb.py b/evaluations/evaluations/evaluators/multidb.py index 97c0691e..2bae9259 100644 --- a/evaluations/evaluations/evaluators/multidb.py +++ b/evaluations/evaluations/evaluators/multidb.py @@ -47,7 +47,7 @@ class NumericAnswer(Evaluator): def get_default_evaluation_name(self) -> str: return "answer_correct" - def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool]: + def evaluate(self, ctx: EvaluatorContext) -> dict[str, float | bool]: meta = ctx.metadata or {} expected = _as_floats(meta.get("expected_value"), meta.get("expected_values")) if not expected: @@ -57,7 +57,7 @@ class NumericAnswer(Evaluator): meta.get("distractor_value"), meta.get("distractor_values") ) correct = expected <= found and not (forbidden & found) - return {"answer_correct": correct} + return {"answer_correct": 1.0 if correct else 0.0} @dataclass @@ -71,7 +71,7 @@ class AttributionGate(Evaluator): def get_default_evaluation_name(self) -> str: return "attribution_correct" - def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool]: + def evaluate(self, ctx: EvaluatorContext) -> dict[str, float | bool]: meta = ctx.metadata or {} expected = meta.get("expected_sources") if expected is None: @@ -90,7 +90,7 @@ class ScopeGate(Evaluator): def get_default_evaluation_name(self) -> str: return "scope_honoured" - def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool]: + def evaluate(self, ctx: EvaluatorContext) -> dict[str, float | bool]: meta = ctx.metadata or {} scope = meta.get("scope") if scope is None: @@ -109,7 +109,7 @@ class TextAnswer(Evaluator): def get_default_evaluation_name(self) -> str: return "answer_correct" - def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool]: + def evaluate(self, ctx: EvaluatorContext) -> dict[str, float | bool]: meta = ctx.metadata or {} required = meta.get("expected_ordered") if required is None: @@ -119,12 +119,12 @@ class TextAnswer(Evaluator): for needle in required: found = haystack.find(str(needle).lower(), cursor) if found < 0: - return {"answer_correct": False} + return {"answer_correct": 0.0} cursor = found + len(str(needle)) for forbidden in meta.get("forbidden_text") or []: if str(forbidden).lower() in haystack: - return {"answer_correct": False} - return {"answer_correct": True} + return {"answer_correct": 0.0} + return {"answer_correct": 1.0} @dataclass @@ -136,7 +136,10 @@ class MultiDBScores(Evaluator): on cases whose metadata does not ask for it. """ - def evaluate(self, ctx: EvaluatorContext) -> dict[str, bool]: + def get_default_evaluation_name(self) -> str: + return "answer_correct" + + def evaluate(self, ctx: EvaluatorContext) -> dict[str, float | bool]: meta = ctx.metadata or {} numeric = bool(meta.get("expected_value") or meta.get("expected_values")) textual = meta.get("expected_ordered") is not None @@ -145,7 +148,7 @@ class MultiDBScores(Evaluator): f"case {meta.get('question_id')!r} asks for both a numeric and an " "ordered-text answer; they share the answer_correct key" ) - scores: dict[str, bool] = {} + scores: dict[str, float | bool] = {} for evaluator in ( NumericAnswer(), TextAnswer(), diff --git a/evaluations/evaluations/qa.py b/evaluations/evaluations/qa.py index d6a5e271..61207706 100644 --- a/evaluations/evaluations/qa.py +++ b/evaluations/evaluations/qa.py @@ -377,6 +377,7 @@ async def run_qa_benchmark( set_eval_attribute("n_executions", result.n_executions) set_eval_attribute("n_requests", result.n_requests) set_eval_attribute("citation_status", result.citation_status) + set_eval_attribute("executed_code", result.executed_code) return result.answer report = await evaluation_dataset.evaluate( @@ -450,6 +451,9 @@ async def run_qa_benchmark( "(PARTIAL excluded)" ) + if spec.report_hook is not None: + spec.report_hook(list(report.cases)) + _print_failures(failures, show_question=True) return failures[0] if failures else None diff --git a/evaluations/tests/test_multidb_evaluators.py b/evaluations/tests/test_multidb_evaluators.py index cf62a149..24d45e6f 100644 --- a/evaluations/tests/test_multidb_evaluators.py +++ b/evaluations/tests/test_multidb_evaluators.py @@ -25,7 +25,7 @@ def test_gold_number_counts_as_correct(): result = NumericAnswer().evaluate( ctx("It sits at 1240 metres.", {"expected_value": 1240}) ) - assert result == {"answer_correct": True} + assert result == {"answer_correct": 1.0} def test_hedging_between_gold_and_distractor_fails(): @@ -37,7 +37,7 @@ def test_hedging_between_gold_and_distractor_fails(): {"expected_value": 1240, "distractor_value": 2310}, ) ) - assert result == {"answer_correct": False} + assert result == {"answer_correct": 0.0} def test_numeric_answer_abstains_without_a_gold_value(): @@ -100,10 +100,10 @@ def test_ordered_text_must_appear_in_order(): meta = {"expected_ordered": ["Overview", "Instruments", "Measurements"]} assert TextAnswer().evaluate( ctx("Overview, then Instruments, then Measurements", meta) - ) == {"answer_correct": True} + ) == {"answer_correct": 1.0} assert TextAnswer().evaluate( ctx("Measurements, then Overview, then Instruments", meta) - ) == {"answer_correct": False} + ) == {"answer_correct": 0.0} def test_forbidden_text_fails_even_when_required_text_is_present(): @@ -116,4 +116,4 @@ def test_forbidden_text_fails_even_when_required_text_is_present(): }, ) ) - assert result == {"answer_correct": False} + assert result == {"answer_correct": 0.0} diff --git a/evaluations/tests/test_multidb_report.py b/evaluations/tests/test_multidb_report.py new file mode 100644 index 00000000..9cd347e4 --- /dev/null +++ b/evaluations/tests/test_multidb_report.py @@ -0,0 +1,151 @@ +from types import SimpleNamespace + +from evaluations.datasets.multidb_report import ( + attribution_gate, + family_rates, + render, + scope_gate, + surface_coverage, +) + + +def case( + name="0_x", + *, + family=None, + scope=None, + expected_sources=None, + scope_honoured=None, + attribution_correct=None, + answer_correct=None, + code=None, +): + assertions = {} + if scope_honoured is not None: + assertions["scope_honoured"] = SimpleNamespace(value=scope_honoured) + if attribution_correct is not None: + assertions["attribution_correct"] = SimpleNamespace(value=attribution_correct) + scores = {} + if answer_correct is not None: + scores["answer_correct"] = SimpleNamespace(value=answer_correct) + metadata = {} + if family is not None: + metadata["family"] = family + if scope is not None: + metadata["scope"] = scope + if expected_sources is not None: + metadata["expected_sources"] = expected_sources + return SimpleNamespace( + name=name, + assertions=assertions, + scores=scores, + metadata=metadata, + attributes={"executed_code": code} if code else {}, + ) + + +def test_scope_gate_counts_only_scoped_cases(): + cases = [ + case("0_b1", family="B1"), + case("1_b4", family="B4", scope=["northern"], scope_honoured=True), + case("2_b4", family="B4", scope=["northern"], scope_honoured=False), + ] + gate = scope_gate(cases) + assert gate.checked == 2 + assert gate.failures == ["2_b4"] + assert gate.passed is False + + +def test_scope_gate_passes_when_nothing_leaks(): + gate = scope_gate([case("0_b4", family="B4", scope=[], scope_honoured=True)]) + assert gate.passed + assert "PASS" in gate.line() + + +def test_attribution_gate_ignores_families_with_more_than_one_answer(): + """B5 is answered by two databases, so it is not an attribution error to cite + both and the gate must not police it.""" + cases = [ + case( + "0_b5", + family="B5", + expected_sources=["northern", "southern"], + attribution_correct=False, + ), + case( + "1_b1", family="B1", expected_sources=["northern"], attribution_correct=True + ), + ] + gate = attribution_gate(cases) + assert gate.checked == 1 + assert gate.passed + + +def test_attribution_gate_reports_the_offending_case(): + gate = attribution_gate( + [ + case( + "3_b3", + family="B3", + expected_sources=["northern"], + attribution_correct=False, + ) + ] + ) + assert gate.failures == ["3_b3"] + assert "FAIL (1)" in gate.line() + assert "3_b3" in gate.line() + + +def test_family_rates_skip_cases_with_no_answer_score(): + """The refusal families have no numeric expectation, so they must not appear + as 0/0 and drag a family rate down.""" + cases = [ + case("0_b1", family="B1", answer_correct=1.0), + case("1_b1", family="B1", answer_correct=0.0), + case("2_b6", family="B6"), + ] + rates = family_rates(cases) + assert rates == {"B1": (1, 2)} + assert "B6" not in rates + + +def test_surface_coverage_counts_cases_not_snippets(): + cases = [ + case("0", code=["print(toc)", "open('/documents/1/toc.json')"]), + case("1", code=["open('/documents/2/content.txt')"]), + case("2"), + ] + coverage = surface_coverage(cases) + assert coverage["toc.json"] == 1 + assert coverage["content.txt"] == 1 + assert coverage["items.jsonl"] == 0 + + +def test_render_says_passed_only_when_both_gates_pass(): + good = [ + case( + "0_b4", + family="B4", + scope=["northern"], + scope_honoured=True, + expected_sources=["northern"], + attribution_correct=True, + answer_correct=1.0, + ), + ] + text, passed = render(good) + assert passed + assert "GATES: PASSED" in text + + bad = good + [case("1_b4", family="B4", scope=["northern"], scope_honoured=False)] + text, passed = render(bad) + assert not passed + assert "GATES: FAILED" in text + assert "1_b4" in text + + +def test_render_flags_a_surface_nothing_reached(): + text, _ = render([case("0", family="S5", code=["open('/documents/1/toc.json')"])]) + assert "never reached" in text + assert "toc.json" in text diff --git a/evaluations/tests/test_reference_configs.py b/evaluations/tests/test_reference_configs.py index 142ccb34..a3f75135 100644 --- a/evaluations/tests/test_reference_configs.py +++ b/evaluations/tests/test_reference_configs.py @@ -44,15 +44,16 @@ def test_filename_names_a_dataset(path: Path) -> None: @pytest.mark.parametrize("path", _config_paths(), ids=lambda p: p.stem) def test_judge_pinned_where_the_judge_runs(path: Path) -> None: - """Datasets without their own qa_evaluator are scored by the LLM judge. + """Wherever a judge runs, its sampling must be frozen so accuracy stays + comparable across runs. - Those configs must carry the frozen judge settings, so accuracy stays - comparable across runs. Datasets that bring a deterministic evaluator - never construct a judge, so a judge block there would be dead config. + A dataset without its own qa_evaluator is scored by the LLM judge and must + carry the block. A dataset with a deterministic evaluator may still need one, + because RefusalJudge runs on any case carrying an answerability label; where + it declares no judge it is asserting that no case does. """ judge = _load(path).evaluations.judge - if DATASETS[path.stem].qa_evaluator is not None: - assert judge is None + if DATASETS[path.stem].qa_evaluator is not None and judge is None: return assert judge is not None