From 9f1bd9940a2c0f41756d896b7b76645725616fa2 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 4 Jun 2026 12:05:06 +0300 Subject: [PATCH 01/13] =?UTF-8?q?Add=20T=C2=B2-RAGBench=20FinQA=20evaluati?= =?UTF-8?q?on=20dataset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 4 + evaluations/evaluations/datasets/__init__.py | 2 + .../evaluations/datasets/t2_ragbench.py | 144 ++++++++++++++++++ evaluations/tests/test_datasets.py | 126 +++++++++++++++ 4 files changed, 276 insertions(+) create mode 100644 evaluations/evaluations/datasets/t2_ragbench.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4490b25e..d1276c01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog ## [Unreleased] +### Added + +- `t2_finqa` evaluation dataset (T²-RAGBench FinQA subset, `G4KMU/t2-ragbench`): 2,789 single-page PDFs / 8,281 numeric QA, ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`. + ### Fixed - Skill tools (`search`/`cite`/`list_documents`/`get_document`) and the analysis sandbox serialize access to the shared LanceDB connection through one lock, so a turn's concurrently executed tool calls no longer trigger `RuntimeError: Already borrowed`. diff --git a/evaluations/evaluations/datasets/__init__.py b/evaluations/evaluations/datasets/__init__.py index c973aa5f..55cc8604 100644 --- a/evaluations/evaluations/datasets/__init__.py +++ b/evaluations/evaluations/datasets/__init__.py @@ -1,6 +1,7 @@ from evaluations.config import DatasetSpec from .open_rag_bench import ORB_MULTIMODAL_SPEC, ORB_TEXT_SPEC +from .t2_ragbench import T2_FINQA_SPEC from .wix import WIX_SPEC DATASETS: dict[str, DatasetSpec] = { @@ -9,6 +10,7 @@ DATASETS: dict[str, DatasetSpec] = { WIX_SPEC, ORB_TEXT_SPEC, ORB_MULTIMODAL_SPEC, + T2_FINQA_SPEC, ) } diff --git a/evaluations/evaluations/datasets/t2_ragbench.py b/evaluations/evaluations/datasets/t2_ragbench.py new file mode 100644 index 00000000..21ee0adf --- /dev/null +++ b/evaluations/evaluations/datasets/t2_ragbench.py @@ -0,0 +1,144 @@ +import json +import shutil +from collections.abc import Mapping +from functools import partial +from pathlib import Path +from typing import Any + +from datasets import Dataset +from huggingface_hub import hf_hub_download +from pydantic_evals import Case + +from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample +from evaluations.evaluators import MAPEvaluator + +REPO_ID = "G4KMU/t2-ragbench" +SPLITS = ("dev", "test", "train") + + +def get_cache_dir() -> Path: + cache_dir = Path.home() / ".cache" / "haiku.rag" / "evaluations" / "t2_pdfs" + cache_dir.mkdir(parents=True, exist_ok=True) + return cache_dir + + +_rows_cache: dict[str, list[dict[str, Any]]] = {} + + +def _load_rows(subset: str) -> list[dict[str, Any]]: + cached = _rows_cache.get(subset) + if cached is not None: + return cached + + rows: list[dict[str, Any]] = [] + for split in SPLITS: + path = hf_hub_download( + REPO_ID, + f"data/{subset}/{split}/metadata.jsonl", + repo_type="dataset", + ) + with open(path) as f: + for line in f: + if not line.strip(): + continue + rows.append({**json.loads(line), "subset": subset}) + + _rows_cache[subset] = rows + return rows + + +def download_t2_pdf(subset: str, split: str, file_name: str) -> Path: + # hf_hub_download may return a content-addressed blob path with no suffix; + # the converter dispatches on extension, so materialize a real ``.pdf`` file. + dest = get_cache_dir() / f"{subset}_{split}_{file_name.replace('/', '_')}" + if dest.exists(): + return dest + + src = hf_hub_download( + REPO_ID, + f"data/{subset}/{split}/{file_name}", + repo_type="dataset", + ) + shutil.copyfile(src, dest) + return dest + + +def load_t2_corpus(subset: str) -> Dataset: + seen: set[str] = set() + docs: list[dict[str, Any]] = [] + for row in _load_rows(subset): + context_id = row["context_id"] + if context_id in seen: + continue + seen.add(context_id) + docs.append(row) + return Dataset.from_list(docs) + + +def load_t2_qa(subset: str) -> Dataset: + return Dataset.from_list(_load_rows(subset)) + + +def map_t2_document(doc: Mapping[str, Any]) -> DocumentPayload | None: + pdf_path = download_t2_pdf(doc["subset"], doc["split"], doc["file_name"]) + + metadata: dict[str, Any] = {"file_name": doc["file_name"]} + for key in ("company_name", "company_symbol", "report_year", "company_sector"): + value = doc.get(key) + if value is not None: + metadata[key] = value + + title_parts = [ + str(doc[key]) for key in ("company_name", "report_year") if doc.get(key) + ] + title = " ".join(title_parts) if title_parts else doc["context_id"] + + return DocumentPayload( + uri=doc["context_id"], + source_path=pdf_path, + title=title, + metadata=metadata, + ) + + +def map_t2_retrieval(doc: Mapping[str, Any]) -> RetrievalSample | None: + return RetrievalSample( + question=doc["question"], + expected_uris=(doc["context_id"],), + ) + + +def build_t2_case(index: int, doc: Mapping[str, Any]) -> Case[str, str, dict[str, str]]: + metadata = { + "case_index": str(index), + "context_id": doc["context_id"], + "id": doc["id"], + } + + return Case( + name=f"{index}_{doc['id']}", + inputs=doc["question"], + expected_output=str(doc["program_answer"]), + metadata=metadata, + ) + + +def _t2_spec(subset: str, key: str, db_filename: str) -> DatasetSpec: + return DatasetSpec( + key=key, + db_filename=db_filename, + document_loader=partial(load_t2_corpus, subset), + document_mapper=map_t2_document, + qa_loader=partial(load_t2_qa, subset), + qa_case_builder=build_t2_case, + retrieval_loader=partial(load_t2_qa, subset), + retrieval_mapper=map_t2_retrieval, + retrieval_evaluator=MAPEvaluator(), + ) + + +T2_FINQA_SPEC = _t2_spec( + subset="FinQA", + key="t2_finqa", + db_filename="t2_ragbench_finqa.lancedb", +) diff --git a/evaluations/tests/test_datasets.py b/evaluations/tests/test_datasets.py index 07a516ee..df1c8344 100644 --- a/evaluations/tests/test_datasets.py +++ b/evaluations/tests/test_datasets.py @@ -7,6 +7,13 @@ from evaluations.datasets.open_rag_bench import ( map_orb_document, map_orb_retrieval, ) +from evaluations.datasets.t2_ragbench import ( + build_t2_case, + download_t2_pdf, + load_t2_corpus, + map_t2_document, + map_t2_retrieval, +) from evaluations.datasets.wix import ( build_wix_case, map_wix_document, @@ -158,3 +165,122 @@ class TestOpenRAGBench: assert is_multimodal_query("image") is True assert is_multimodal_query("image_table") is True assert is_multimodal_query("text") is False + + +class TestT2RAGBench: + def _row(self, **overrides: object) -> dict[str, object]: + row: dict[str, object] = { + "id": "finqa_dev_0", + "context_id": "finqa_dev_ctx_138", + "subset": "FinQA", + "split": "dev", + "file_name": "pdf/V/2008/page_17.pdf", + "question": "What was the average payment volume per transaction?", + "program_answer": "127.4", + "company_name": "Visa Inc.", + "company_symbol": "V", + "report_year": 2008, + "company_sector": "Financials", + } + row.update(overrides) + return row + + def test_map_document(self, tmp_path: Path) -> None: + pdf_path = tmp_path / "page_17.pdf" + pdf_path.write_bytes(b"%PDF-fake") + from unittest.mock import patch + + with patch( + "evaluations.datasets.t2_ragbench.download_t2_pdf", + return_value=pdf_path, + ) as download: + payload = map_t2_document(self._row()) + + download.assert_called_once_with("FinQA", "dev", "pdf/V/2008/page_17.pdf") + assert payload is not None + assert payload.uri == "finqa_dev_ctx_138" + assert payload.source_path == pdf_path + assert payload.content is None + assert payload.title == "Visa Inc. 2008" + assert payload.metadata == { + "file_name": "pdf/V/2008/page_17.pdf", + "company_name": "Visa Inc.", + "company_symbol": "V", + "report_year": 2008, + "company_sector": "Financials", + } + + def test_map_document_title_falls_back_to_context_id(self, tmp_path: Path) -> None: + pdf_path = tmp_path / "page.pdf" + pdf_path.write_bytes(b"%PDF-fake") + from unittest.mock import patch + + row = self._row(company_name=None, report_year=None) + with patch( + "evaluations.datasets.t2_ragbench.download_t2_pdf", + return_value=pdf_path, + ): + payload = map_t2_document(row) + + assert payload is not None + assert payload.title == "finqa_dev_ctx_138" + assert payload.metadata is not None + assert "company_name" not in payload.metadata + assert "report_year" not in payload.metadata + + def test_map_retrieval(self) -> None: + sample = map_t2_retrieval(self._row()) + assert sample is not None + assert sample.question == self._row()["question"] + assert sample.expected_uris == ("finqa_dev_ctx_138",) + + def test_build_case(self) -> None: + case = build_t2_case(3, self._row()) + assert case.name == "3_finqa_dev_0" + assert case.inputs == self._row()["question"] + assert case.expected_output == "127.4" + assert case.metadata is not None + assert case.metadata["case_index"] == "3" + assert case.metadata["context_id"] == "finqa_dev_ctx_138" + + def test_build_case_casts_numeric_answer(self) -> None: + case = build_t2_case(0, self._row(program_answer=127.4)) + assert case.expected_output == "127.4" + + def test_download_pdf_materializes_pdf_suffix(self, tmp_path: Path) -> None: + # hf_hub_download can return a suffix-less blob path; the cached copy + # must carry the .pdf extension the converter dispatches on. + blob = tmp_path / "blobs" / "6aa49306deadbeef" + blob.parent.mkdir() + blob.write_bytes(b"%PDF-fake") + cache = tmp_path / "cache" + cache.mkdir() + from unittest.mock import patch + + with ( + patch("evaluations.datasets.t2_ragbench.get_cache_dir", return_value=cache), + patch( + "evaluations.datasets.t2_ragbench.hf_hub_download", + return_value=str(blob), + ), + ): + out = download_t2_pdf("FinQA", "dev", "pdf/V/2008/page_17.pdf") + + assert out.suffix == ".pdf" + assert out.exists() + assert out.read_bytes() == b"%PDF-fake" + assert out.name == "FinQA_dev_pdf_V_2008_page_17.pdf" + + def test_load_corpus_dedupes_by_context_id(self) -> None: + from unittest.mock import patch + + rows = [ + self._row(id="finqa_dev_0", context_id="ctx_a"), + self._row(id="finqa_dev_1", context_id="ctx_a"), + self._row(id="finqa_dev_2", context_id="ctx_b"), + ] + with patch("evaluations.datasets.t2_ragbench._load_rows", return_value=rows): + corpus = load_t2_corpus("FinQA") + + assert len(corpus) == 2 + assert {r["context_id"] for r in corpus} == {"ctx_a", "ctx_b"} From c2a48b55c0f8000e1c439dbf0b7e932031f32374 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 4 Jun 2026 14:19:44 +0300 Subject: [PATCH 02/13] =?UTF-8?q?Add=20deterministic=20Number-Match=20QA?= =?UTF-8?q?=20scoring=20for=20T=C2=B2-RAGBench?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 +- evaluations/evaluations/benchmark.py | 52 ++++++++++------ evaluations/evaluations/config.py | 1 + .../evaluations/datasets/t2_ragbench.py | 3 +- .../evaluations/evaluators/__init__.py | 2 + .../evaluations/evaluators/number_match.py | 30 +++++++++ evaluations/evaluations/numbers.py | 61 +++++++++++++++++++ evaluations/tests/test_evaluators.py | 39 ++++++++++++ evaluations/tests/test_numbers.py | 38 ++++++++++++ 9 files changed, 208 insertions(+), 20 deletions(-) create mode 100644 evaluations/evaluations/evaluators/number_match.py create mode 100644 evaluations/evaluations/numbers.py create mode 100644 evaluations/tests/test_numbers.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d1276c01..0117b028 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### Added -- `t2_finqa` evaluation dataset (T²-RAGBench FinQA subset, `G4KMU/t2-ragbench`): 2,789 single-page PDFs / 8,281 numeric QA, ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`. +- `t2_finqa` evaluation dataset (T²-RAGBench FinQA subset, `G4KMU/t2-ragbench`): 2,789 single-page PDFs / 8,281 numeric QA, ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`. QA is scored with a deterministic `NumberMatchEvaluator` (relative tolerance 0.01) via the new `DatasetSpec.qa_evaluator`, bypassing the LLM judge. ### Fixed diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 0f58de66..dd7c9d99 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -365,18 +365,23 @@ async def run_qa_benchmark( _attach_relevant_uris(cases, spec, limit) citation_evaluator = _citation_evaluator_for(spec.retrieval_evaluator) - evaluators: list[Evaluator] = [ - LLMJudge( - rubric=ANSWER_EQUIVALENCE_RUBRIC, - include_input=True, - include_expected_output=True, - model=get_model(judge_config, config), - assertion={ - "evaluation_name": "answer_equivalent", - "include_reason": True, - }, - ), - ] + qa_evaluator = spec.qa_evaluator + evaluators: list[Evaluator] + if qa_evaluator is not None: + evaluators = [qa_evaluator] + else: + evaluators = [ + LLMJudge( + rubric=ANSWER_EQUIVALENCE_RUBRIC, + include_input=True, + include_expected_output=True, + model=get_model(judge_config, config), + assertion={ + "evaluation_name": "answer_equivalent", + "include_reason": True, + }, + ), + ] if citation_evaluator is not None: evaluators.append(citation_evaluator) @@ -419,17 +424,28 @@ async def run_qa_benchmark( report = await _evaluate(answer_question) - passing_cases = sum( - 1 - for case in report.cases - if case.assertions.get("answer_equivalent") - and case.assertions["answer_equivalent"].value - ) total_processed = len(report.cases) failures = report.failures + if qa_evaluator is not None: + score_key = qa_evaluator.get_default_evaluation_name() + passing_cases = sum( + 1 + for case in report.cases + if score_key in case.scores and case.scores[score_key].value >= 1.0 + ) + scoring = score_key + else: + passing_cases = sum( + 1 + for case in report.cases + if case.assertions.get("answer_equivalent") + and case.assertions["answer_equivalent"].value + ) + scoring = "answer_equivalent" accuracy = passing_cases / total_processed if total_processed > 0 else 0 console.print("\n=== QA Benchmark Results ===", style="bold cyan") + console.print(f"Scoring: {scoring}") console.print(f"Total questions: {total_processed}") console.print(f"Correct answers: {passing_cases}") console.print(f"QA Accuracy: {accuracy:.4f} ({accuracy * 100:.2f}%)") diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py index 063fb104..c5ecfd17 100644 --- a/evaluations/evaluations/config.py +++ b/evaluations/evaluations/config.py @@ -44,6 +44,7 @@ class DatasetSpec: retrieval_loader: RetrievalLoader | None = None retrieval_mapper: RetrievalMapper | None = None retrieval_evaluator: Evaluator | None = None + qa_evaluator: Evaluator | None = None document_limit: int | None = None def db_path(self, override_path: Path | None = None) -> Path: diff --git a/evaluations/evaluations/datasets/t2_ragbench.py b/evaluations/evaluations/datasets/t2_ragbench.py index 21ee0adf..a4713499 100644 --- a/evaluations/evaluations/datasets/t2_ragbench.py +++ b/evaluations/evaluations/datasets/t2_ragbench.py @@ -10,7 +10,7 @@ from huggingface_hub import hf_hub_download from pydantic_evals import Case from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample -from evaluations.evaluators import MAPEvaluator +from evaluations.evaluators import MAPEvaluator, NumberMatchEvaluator REPO_ID = "G4KMU/t2-ragbench" SPLITS = ("dev", "test", "train") @@ -134,6 +134,7 @@ def _t2_spec(subset: str, key: str, db_filename: str) -> DatasetSpec: retrieval_loader=partial(load_t2_qa, subset), retrieval_mapper=map_t2_retrieval, retrieval_evaluator=MAPEvaluator(), + qa_evaluator=NumberMatchEvaluator(), ) diff --git a/evaluations/evaluations/evaluators/__init__.py b/evaluations/evaluations/evaluators/__init__.py index bd15abc0..d90f7449 100644 --- a/evaluations/evaluations/evaluators/__init__.py +++ b/evaluations/evaluations/evaluators/__init__.py @@ -5,6 +5,7 @@ from evaluations.evaluators.judge import ( LLMJudgeResponseSchema, ) from evaluations.evaluators.map import MAPEvaluator +from evaluations.evaluators.number_match import NumberMatchEvaluator __all__ = [ "ANSWER_EQUIVALENCE_RUBRIC", @@ -12,4 +13,5 @@ __all__ = [ "LLMJudge", "LLMJudgeResponseSchema", "MAPEvaluator", + "NumberMatchEvaluator", ] diff --git a/evaluations/evaluations/evaluators/number_match.py b/evaluations/evaluations/evaluators/number_match.py new file mode 100644 index 00000000..968a5b36 --- /dev/null +++ b/evaluations/evaluations/evaluators/number_match.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass + +from pydantic_evals.evaluators import Evaluator, EvaluatorContext + +from evaluations.numbers import extract_numbers, numbers_close + + +@dataclass +class NumberMatchEvaluator(Evaluator): + """Deterministic numeric scoring for datasets with numeric gold answers. + + Scores 1.0 when any number parsed from the prediction matches the gold + answer within relative tolerance ``eps``, else 0.0. Non-numeric predictions + score 0.0. + """ + + eps: float = 0.01 + + def get_default_evaluation_name(self) -> str: + return "number_match" + + def evaluate(self, ctx: EvaluatorContext) -> float: + gold = extract_numbers(str(ctx.expected_output)) + if not gold: + return 0.0 + target = gold[0] + candidates = extract_numbers(str(ctx.output)) + return ( + 1.0 if any(numbers_close(c, target, self.eps) for c in candidates) else 0.0 + ) diff --git a/evaluations/evaluations/numbers.py b/evaluations/evaluations/numbers.py new file mode 100644 index 00000000..d0778000 --- /dev/null +++ b/evaluations/evaluations/numbers.py @@ -0,0 +1,61 @@ +import re + +_SCALE = { + "thousand": 1e3, + "million": 1e6, + "billion": 1e9, + "trillion": 1e12, +} + +_NUMBER_RE = re.compile(r"[(-]?\$?\d[\d,]*(?:\.\d+)?%?\)?") +_SCALED_RE = re.compile( + r"([(-]?\$?\d[\d,]*(?:\.\d+)?)\s*(thousand|million|billion|trillion)", + re.IGNORECASE, +) + + +def _to_float(token: str) -> float | None: + stripped = token.lstrip() + negative = stripped.startswith("(") or stripped.startswith("-") + cleaned = ( + token.replace("$", "") + .replace(",", "") + .replace("%", "") + .replace("(", "") + .replace(")", "") + .strip() + .lstrip("-") + ) + if not cleaned or cleaned == ".": + return None + try: + value = float(cleaned) + except ValueError: + return None + return -value if negative else value + + +def extract_numbers(text: str) -> list[float]: + """Pull numeric values out of free-form text. + + Handles currency, thousands separators, trailing percent signs, and + parenthesised negatives. Numbers qualified by a scale word ("1.2 million") + contribute both the raw and the scaled value, so either phrasing can match. + """ + numbers: list[float] = [] + for token in _NUMBER_RE.findall(text): + value = _to_float(token) + if value is not None: + numbers.append(value) + for number, scale in _SCALED_RE.findall(text): + value = _to_float(number) + if value is not None: + numbers.append(value * _SCALE[scale.lower()]) + return numbers + + +def numbers_close(value: float, target: float, eps: float = 0.01) -> bool: + """Whether ``value`` matches ``target`` within relative tolerance ``eps``.""" + if target == 0: + return value == 0 + return abs(value - target) / abs(target) <= eps diff --git a/evaluations/tests/test_evaluators.py b/evaluations/tests/test_evaluators.py index 68bc34a6..936d0392 100644 --- a/evaluations/tests/test_evaluators.py +++ b/evaluations/tests/test_evaluators.py @@ -3,6 +3,7 @@ from unittest.mock import MagicMock import pytest from evaluations.evaluators.map import MAPEvaluator +from evaluations.evaluators.number_match import NumberMatchEvaluator class TestMAPEvaluator: @@ -54,3 +55,41 @@ class TestMAPEvaluator: def test_empty_relevant_uris(self) -> None: ctx = self._make_ctx([], ["doc1", "doc2"]) assert self.evaluator.evaluate(ctx) == 0.0 + + +class TestNumberMatchEvaluator: + def setup_method(self) -> None: + self.evaluator = NumberMatchEvaluator() + + def _make_ctx(self, expected: str, output: str) -> MagicMock: + ctx = MagicMock() + ctx.expected_output = expected + ctx.output = output + return ctx + + def test_exact(self) -> None: + assert self.evaluator.evaluate(self._make_ctx("127.4", "127.4")) == 1.0 + + def test_within_tolerance(self) -> None: + ctx = self._make_ctx("127.4", "about $127.40 per transaction") + assert self.evaluator.evaluate(ctx) == 1.0 + + def test_outside_tolerance(self) -> None: + ctx = self._make_ctx("127.4", "the answer is 150") + assert self.evaluator.evaluate(ctx) == 0.0 + + def test_picks_matching_candidate_among_many(self) -> None: + ctx = self._make_ctx("50.3", "In 2008 it grew from 27.0 to 50.3 percent") + assert self.evaluator.evaluate(ctx) == 1.0 + + def test_non_numeric_prediction(self) -> None: + ctx = self._make_ctx("127.4", "I cannot determine the value") + assert self.evaluator.evaluate(ctx) == 0.0 + + def test_non_numeric_gold(self) -> None: + ctx = self._make_ctx("not a number", "127.4") + assert self.evaluator.evaluate(ctx) == 0.0 + + def test_negative_match(self) -> None: + ctx = self._make_ctx("-12.3", "the change was (12.3)") + assert self.evaluator.evaluate(ctx) == 1.0 diff --git a/evaluations/tests/test_numbers.py b/evaluations/tests/test_numbers.py new file mode 100644 index 00000000..2f38a376 --- /dev/null +++ b/evaluations/tests/test_numbers.py @@ -0,0 +1,38 @@ +from evaluations.numbers import extract_numbers, numbers_close + + +class TestExtractNumbers: + def test_plain(self) -> None: + assert extract_numbers("the answer is 127.4 dollars") == [127.4] + + def test_currency_and_thousands(self) -> None: + assert extract_numbers("$1,234.5") == [1234.5] + + def test_percent_stripped(self) -> None: + assert extract_numbers("margin was 50.3%") == [50.3] + + def test_parenthesised_negative(self) -> None: + assert extract_numbers("loss of (123)") == [-123.0] + + def test_scale_word_adds_scaled_and_raw(self) -> None: + numbers = extract_numbers("revenue of 1.2 billion") + assert 1.2 in numbers + assert 1.2e9 in numbers + + def test_no_numbers(self) -> None: + assert extract_numbers("no figures here") == [] + + +class TestNumbersClose: + def test_within_tolerance(self) -> None: + assert numbers_close(127.4, 127.40, 0.01) + + def test_just_inside(self) -> None: + assert numbers_close(100.9, 100.0, 0.01) + + def test_outside_tolerance(self) -> None: + assert not numbers_close(102.0, 100.0, 0.01) + + def test_zero_target_exact(self) -> None: + assert numbers_close(0.0, 0.0, 0.01) + assert not numbers_close(0.1, 0.0, 0.01) From 80594cd38cb265e72159942b0a10b232ddee01d4 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 4 Jun 2026 14:30:35 +0300 Subject: [PATCH 03/13] Handle percent/decimal convention in Number-Match --- evaluations/evaluations/numbers.py | 4 ++++ evaluations/tests/test_evaluators.py | 8 ++++++++ evaluations/tests/test_numbers.py | 4 ++-- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/evaluations/evaluations/numbers.py b/evaluations/evaluations/numbers.py index d0778000..0bf3c8ed 100644 --- a/evaluations/evaluations/numbers.py +++ b/evaluations/evaluations/numbers.py @@ -47,6 +47,10 @@ def extract_numbers(text: str) -> list[float]: value = _to_float(token) if value is not None: numbers.append(value) + if "%" in token: + # Gold answers store ratios as either a percent (24.69) or a + # decimal (0.935); offer both readings of a percent figure. + numbers.append(value / 100) for number, scale in _SCALED_RE.findall(text): value = _to_float(number) if value is not None: diff --git a/evaluations/tests/test_evaluators.py b/evaluations/tests/test_evaluators.py index 936d0392..97dbbb19 100644 --- a/evaluations/tests/test_evaluators.py +++ b/evaluations/tests/test_evaluators.py @@ -82,6 +82,14 @@ class TestNumberMatchEvaluator: ctx = self._make_ctx("50.3", "In 2008 it grew from 27.0 to 50.3 percent") assert self.evaluator.evaluate(ctx) == 1.0 + def test_percent_answer_matches_decimal_gold(self) -> None: + ctx = self._make_ctx("0.935", "the cumulative total return was 93.5%") + assert self.evaluator.evaluate(ctx) == 1.0 + + def test_percent_answer_matches_percent_gold(self) -> None: + ctx = self._make_ctx("24.691358024691358", "approximately 24.69% of production") + assert self.evaluator.evaluate(ctx) == 1.0 + def test_non_numeric_prediction(self) -> None: ctx = self._make_ctx("127.4", "I cannot determine the value") assert self.evaluator.evaluate(ctx) == 0.0 diff --git a/evaluations/tests/test_numbers.py b/evaluations/tests/test_numbers.py index 2f38a376..0cc937c4 100644 --- a/evaluations/tests/test_numbers.py +++ b/evaluations/tests/test_numbers.py @@ -8,8 +8,8 @@ class TestExtractNumbers: def test_currency_and_thousands(self) -> None: assert extract_numbers("$1,234.5") == [1234.5] - def test_percent_stripped(self) -> None: - assert extract_numbers("margin was 50.3%") == [50.3] + def test_percent_yields_both_readings(self) -> None: + assert extract_numbers("margin was 50.3%") == [50.3, 0.503] def test_parenthesised_negative(self) -> None: assert extract_numbers("loss of (123)") == [-123.0] From 390deb42030a425d8dbfeb10e5b955e85e295dea Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 4 Jun 2026 14:46:28 +0300 Subject: [PATCH 04/13] Normalize unicode signs --- evaluations/evaluations/numbers.py | 1 + evaluations/tests/test_numbers.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/evaluations/evaluations/numbers.py b/evaluations/evaluations/numbers.py index 0bf3c8ed..82618ccb 100644 --- a/evaluations/evaluations/numbers.py +++ b/evaluations/evaluations/numbers.py @@ -42,6 +42,7 @@ def extract_numbers(text: str) -> list[float]: parenthesised negatives. Numbers qualified by a scale word ("1.2 million") contribute both the raw and the scaled value, so either phrasing can match. """ + text = text.replace("−", "-") # normalize the typographic minus sign numbers: list[float] = [] for token in _NUMBER_RE.findall(text): value = _to_float(token) diff --git a/evaluations/tests/test_numbers.py b/evaluations/tests/test_numbers.py index 0cc937c4..5dfaac92 100644 --- a/evaluations/tests/test_numbers.py +++ b/evaluations/tests/test_numbers.py @@ -14,6 +14,9 @@ class TestExtractNumbers: def test_parenthesised_negative(self) -> None: assert extract_numbers("loss of (123)") == [-123.0] + def test_unicode_minus(self) -> None: + assert extract_numbers("a change of −1.9 million") == [-1.9, -1.9e6] + def test_scale_word_adds_scaled_and_raw(self) -> None: numbers = extract_numbers("revenue of 1.2 billion") assert 1.2 in numbers From de9731d5f30cdae76b1b1727f027cdc7bf789885 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 4 Jun 2026 15:14:20 +0300 Subject: [PATCH 05/13] =?UTF-8?q?Score=20Number-Match=20on=20the=20declare?= =?UTF-8?q?d=20ANSWER=20line,=20magnitude=20and=20=C3=97100=20scale?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../evaluations/evaluators/number_match.py | 24 +++++++++++--- evaluations/tests/test_evaluators.py | 33 +++++++++++++++++++ 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/evaluations/evaluations/evaluators/number_match.py b/evaluations/evaluations/evaluators/number_match.py index 968a5b36..8ea70304 100644 --- a/evaluations/evaluations/evaluators/number_match.py +++ b/evaluations/evaluations/evaluators/number_match.py @@ -1,9 +1,19 @@ +import re from dataclasses import dataclass from pydantic_evals.evaluators import Evaluator, EvaluatorContext from evaluations.numbers import extract_numbers, numbers_close +_ANSWER_RE = re.compile(r"(?im)^[\s*>#_-]*(?:final\s+)?answer\s*[:=]\s*(.+)$") + + +def _answer_segment(text: str) -> str: + """Restrict to a declared ``ANSWER:`` line when present, so numbers in the + surrounding reasoning don't count. Falls back to the whole text.""" + matches = _ANSWER_RE.findall(text) + return matches[-1] if matches else text + @dataclass class NumberMatchEvaluator(Evaluator): @@ -23,8 +33,14 @@ class NumberMatchEvaluator(Evaluator): gold = extract_numbers(str(ctx.expected_output)) if not gold: return 0.0 - target = gold[0] - candidates = extract_numbers(str(ctx.output)) - return ( - 1.0 if any(numbers_close(c, target, self.eps) for c in candidates) else 0.0 + # Gold mixes conventions: signs for changes are inconsistent (+0.2 vs + # -1.9) and ratios appear as either a percent (37.81) or a decimal + # (0.3781). Compare the declared answer by magnitude, at a ×100 scale + # either way. Safe because we score only the single ANSWER-line number. + target = abs(gold[0]) + candidates = [abs(c) for c in extract_numbers(_answer_segment(str(ctx.output)))] + scales = (1.0, 0.01, 100.0) + matched = any( + numbers_close(c * s, target, self.eps) for c in candidates for s in scales ) + return 1.0 if matched else 0.0 diff --git a/evaluations/tests/test_evaluators.py b/evaluations/tests/test_evaluators.py index 97dbbb19..b70a74b3 100644 --- a/evaluations/tests/test_evaluators.py +++ b/evaluations/tests/test_evaluators.py @@ -101,3 +101,36 @@ class TestNumberMatchEvaluator: def test_negative_match(self) -> None: ctx = self._make_ctx("-12.3", "the change was (12.3)") assert self.evaluator.evaluate(ctx) == 1.0 + + def test_sign_insensitive_against_inconsistent_gold(self) -> None: + # gold stores this decrease as +0.2; model declares the signed -0.2 + ctx = self._make_ctx("0.1999999999999993", "declined 0.2 pp\nANSWER: -0.2") + assert self.evaluator.evaluate(ctx) == 1.0 + + def test_bare_percent_matches_decimal_gold(self) -> None: + # model declares the percentage without a % sign; gold is the decimal + ctx = self._make_ctx("0.3781", "growth was 37.81%\nANSWER: 37.81") + assert self.evaluator.evaluate(ctx) == 1.0 + + def test_scale_mismatch_does_not_flip_genuine_error(self) -> None: + ctx = self._make_ctx("30.443", "ANSWER: 2330.8%") + assert self.evaluator.evaluate(ctx) == 0.0 + + def test_answer_line_ignores_reasoning_distractors(self) -> None: + # gold matches a distractor in the body, but the declared answer is wrong + ctx = self._make_ctx( + "0.728", + "Finished goods were 72.8% of inventory.\nANSWER: 82.8%", + ) + assert self.evaluator.evaluate(ctx) == 0.0 + + def test_answer_line_used_when_correct(self) -> None: + ctx = self._make_ctx( + "0.935", + "The graph shows growth to 193.5.\n\nANSWER: 93.5%", + ) + assert self.evaluator.evaluate(ctx) == 1.0 + + def test_falls_back_to_full_text_without_answer_line(self) -> None: + ctx = self._make_ctx("127.4", "The average works out to $127.40 each.") + assert self.evaluator.evaluate(ctx) == 1.0 From d5b0aafa2be9926f7c2356769c684c48335383f4 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 4 Jun 2026 17:18:06 +0300 Subject: [PATCH 06/13] =?UTF-8?q?Add=20T=C2=B2-RAGBench=20TAT-DQA=20subset?= =?UTF-8?q?;=20generalize=20subset=20layout?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 +- evaluations/evaluations/datasets/__init__.py | 3 +- .../evaluations/datasets/t2_ragbench.py | 39 +++++++++++++------ evaluations/tests/test_datasets.py | 30 ++++++++++++++ 4 files changed, 60 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0117b028..d6c08706 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### Added -- `t2_finqa` evaluation dataset (T²-RAGBench FinQA subset, `G4KMU/t2-ragbench`): 2,789 single-page PDFs / 8,281 numeric QA, ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`. QA is scored with a deterministic `NumberMatchEvaluator` (relative tolerance 0.01) via the new `DatasetSpec.qa_evaluator`, bypassing the LLM judge. +- `t2_finqa` and `t2_tatdqa` evaluation datasets (T²-RAGBench subsets, `G4KMU/t2-ragbench`): financial-report PDFs ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`. QA is scored with a deterministic `NumberMatchEvaluator` (relative tolerance 0.01) via the new `DatasetSpec.qa_evaluator`, bypassing the LLM judge. ### Fixed diff --git a/evaluations/evaluations/datasets/__init__.py b/evaluations/evaluations/datasets/__init__.py index 55cc8604..c2ee4b84 100644 --- a/evaluations/evaluations/datasets/__init__.py +++ b/evaluations/evaluations/datasets/__init__.py @@ -1,7 +1,7 @@ from evaluations.config import DatasetSpec from .open_rag_bench import ORB_MULTIMODAL_SPEC, ORB_TEXT_SPEC -from .t2_ragbench import T2_FINQA_SPEC +from .t2_ragbench import T2_FINQA_SPEC, T2_TATDQA_SPEC from .wix import WIX_SPEC DATASETS: dict[str, DatasetSpec] = { @@ -11,6 +11,7 @@ DATASETS: dict[str, DatasetSpec] = { ORB_TEXT_SPEC, ORB_MULTIMODAL_SPEC, T2_FINQA_SPEC, + T2_TATDQA_SPEC, ) } diff --git a/evaluations/evaluations/datasets/t2_ragbench.py b/evaluations/evaluations/datasets/t2_ragbench.py index a4713499..5f172e9c 100644 --- a/evaluations/evaluations/datasets/t2_ragbench.py +++ b/evaluations/evaluations/datasets/t2_ragbench.py @@ -13,7 +13,23 @@ from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample from evaluations.evaluators import MAPEvaluator, NumberMatchEvaluator REPO_ID = "G4KMU/t2-ragbench" -SPLITS = ("dev", "test", "train") + +# Per-subset layout: which metadata files hold the rows, and the repo prefix the +# PDF lives under ({split} is filled from the row). +_SUBSETS: dict[str, dict[str, Any]] = { + "FinQA": { + "metadata": tuple( + f"data/FinQA/{s}/metadata.jsonl" for s in ("dev", "test", "train") + ), + "pdf_prefix": "data/FinQA/{split}/", + }, + "TAT-DQA": { + "metadata": tuple( + f"data/TAT-DQA/{s}/metadata.jsonl" for s in ("dev", "test", "train") + ), + "pdf_prefix": "data/TAT-DQA/{split}/", + }, +} def get_cache_dir() -> Path: @@ -31,12 +47,8 @@ def _load_rows(subset: str) -> list[dict[str, Any]]: return cached rows: list[dict[str, Any]] = [] - for split in SPLITS: - path = hf_hub_download( - REPO_ID, - f"data/{subset}/{split}/metadata.jsonl", - repo_type="dataset", - ) + for metadata_file in _SUBSETS[subset]["metadata"]: + path = hf_hub_download(REPO_ID, metadata_file, repo_type="dataset") with open(path) as f: for line in f: if not line.strip(): @@ -54,11 +66,8 @@ def download_t2_pdf(subset: str, split: str, file_name: str) -> Path: if dest.exists(): return dest - src = hf_hub_download( - REPO_ID, - f"data/{subset}/{split}/{file_name}", - repo_type="dataset", - ) + repo_path = _SUBSETS[subset]["pdf_prefix"].format(split=split) + file_name + src = hf_hub_download(REPO_ID, repo_path, repo_type="dataset") shutil.copyfile(src, dest) return dest @@ -143,3 +152,9 @@ T2_FINQA_SPEC = _t2_spec( key="t2_finqa", db_filename="t2_ragbench_finqa.lancedb", ) + +T2_TATDQA_SPEC = _t2_spec( + subset="TAT-DQA", + key="t2_tatdqa", + db_filename="t2_ragbench_tatdqa.lancedb", +) diff --git a/evaluations/tests/test_datasets.py b/evaluations/tests/test_datasets.py index df1c8344..eda78bbf 100644 --- a/evaluations/tests/test_datasets.py +++ b/evaluations/tests/test_datasets.py @@ -271,6 +271,36 @@ class TestT2RAGBench: assert out.read_bytes() == b"%PDF-fake" assert out.name == "FinQA_dev_pdf_V_2008_page_17.pdf" + def test_pdf_repo_path_per_subset(self, tmp_path: Path) -> None: + from unittest.mock import patch + + blob = tmp_path / "blob" + blob.write_bytes(b"%PDF-fake") + cache = tmp_path / "cache" + cache.mkdir() + cases = [ + ( + "FinQA", + "dev", + "pdf/V/2008/page_17.pdf", + "data/FinQA/dev/pdf/V/2008/page_17.pdf", + ), + ("TAT-DQA", "dev", "raw/abc123.pdf", "data/TAT-DQA/dev/raw/abc123.pdf"), + ] + for subset, split, file_name, expected_repo_path in cases: + with ( + patch( + "evaluations.datasets.t2_ragbench.get_cache_dir", + return_value=cache, + ), + patch( + "evaluations.datasets.t2_ragbench.hf_hub_download", + return_value=str(blob), + ) as dl, + ): + download_t2_pdf(subset, split, file_name) + assert dl.call_args.args[1] == expected_repo_path + def test_load_corpus_dedupes_by_context_id(self) -> None: from unittest.mock import patch From 1d7d54027019809ebb31613f2a564da3af08eba9 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 5 Jun 2026 11:27:35 +0300 Subject: [PATCH 07/13] Match the numeric scale convention in Number-Match --- evaluations/evaluations/evaluators/number_match.py | 11 ++++++----- evaluations/tests/test_evaluators.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/evaluations/evaluations/evaluators/number_match.py b/evaluations/evaluations/evaluators/number_match.py index 8ea70304..eb657e24 100644 --- a/evaluations/evaluations/evaluators/number_match.py +++ b/evaluations/evaluations/evaluators/number_match.py @@ -33,13 +33,14 @@ class NumberMatchEvaluator(Evaluator): gold = extract_numbers(str(ctx.expected_output)) if not gold: return 0.0 - # Gold mixes conventions: signs for changes are inconsistent (+0.2 vs - # -1.9) and ratios appear as either a percent (37.81) or a decimal - # (0.3781). Compare the declared answer by magnitude, at a ×100 scale - # either way. Safe because we score only the single ANSWER-line number. + # Gold mixes conventions: change signs are inconsistent (+0.2 vs -1.9), + # ratios appear as a percent or a decimal (37.81 vs 0.3781), and figures + # appear in units or thousands (4575515 vs 4575515000). Compare the + # declared answer by magnitude at ×100 and ×1000 scales either way. Safe + # because we score only the single ANSWER-line number. target = abs(gold[0]) candidates = [abs(c) for c in extract_numbers(_answer_segment(str(ctx.output)))] - scales = (1.0, 0.01, 100.0) + scales = (1.0, 0.01, 100.0, 0.001, 1000.0) matched = any( numbers_close(c * s, target, self.eps) for c in candidates for s in scales ) diff --git a/evaluations/tests/test_evaluators.py b/evaluations/tests/test_evaluators.py index b70a74b3..c0adecd3 100644 --- a/evaluations/tests/test_evaluators.py +++ b/evaluations/tests/test_evaluators.py @@ -116,6 +116,17 @@ class TestNumberMatchEvaluator: ctx = self._make_ctx("30.443", "ANSWER: 2330.8%") assert self.evaluator.evaluate(ctx) == 0.0 + def test_thousands_convention(self) -> None: + # gold is in thousands; model gives the full-dollar figure + ctx = self._make_ctx("4575515.0", "...\nANSWER: 4575515000") + assert self.evaluator.evaluate(ctx) == 1.0 + + def test_thousands_convention_reversed(self) -> None: + ctx = self._make_ctx( + "46.30434782608695", "fair value per share\nANSWER: 46304.35" + ) + assert self.evaluator.evaluate(ctx) == 1.0 + def test_answer_line_ignores_reasoning_distractors(self) -> None: # gold matches a distractor in the body, but the declared answer is wrong ctx = self._make_ctx( From cd77bd9889838517c6dc6e0dc880eac098132c67 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 5 Jun 2026 13:12:15 +0300 Subject: [PATCH 08/13] Bound analysis execute_code calls to avoid request-limit nulls --- CHANGELOG.md | 1 + docs/configuration/qa.md | 2 ++ docs/skills/analysis.md | 2 ++ haiku_rag_slim/haiku/rag/config/models.py | 1 + haiku_rag_slim/haiku/rag/skills/_deps.py | 2 ++ haiku_rag_slim/haiku/rag/skills/_tools.py | 8 +++++ haiku_rag_slim/haiku/rag/skills/analysis.py | 1 + tests/skills/test_analysis.py | 36 +++++++++++++++++++++ 8 files changed, 53 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6c08706..78fd6ac2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ### Added +- `analysis.max_executions` (default 15): caps `execute_code` calls per analysis question. Past the cap the tool returns a notice telling the skill to answer from what it has, instead of spiralling into `request_limit` and returning nothing. The analysis skill sets `request_limit` to 30 as a backstop. - `t2_finqa` and `t2_tatdqa` evaluation datasets (T²-RAGBench subsets, `G4KMU/t2-ragbench`): financial-report PDFs ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`. QA is scored with a deterministic `NumberMatchEvaluator` (relative tolerance 0.01) via the new `DatasetSpec.qa_evaluator`, bypassing the LLM judge. ### Fixed diff --git a/docs/configuration/qa.md b/docs/configuration/qa.md index f8f76f7f..ec527a1e 100644 --- a/docs/configuration/qa.md +++ b/docs/configuration/qa.md @@ -52,10 +52,12 @@ analysis: temperature: 0.0 # Default: 0.0 (deterministic for code generation) code_timeout: 60.0 # Max seconds for code execution max_output_chars: 50000 # Truncate output after this many chars + max_executions: 15 # Max execute_code calls per question ``` - **model**: LLM configuration (see [Providers](providers.md#model-settings)). When unset, falls back to `qa.model`. - **code_timeout**: Maximum seconds for each code execution (default: 60) - **max_output_chars**: Truncate code output after this many characters (default: 50000) +- **max_executions**: Maximum `execute_code` calls per question before the skill is told to answer from what it has (default: 15) See [Analysis skill](../skills/analysis.md) for usage details. diff --git a/docs/skills/analysis.md b/docs/skills/analysis.md index 2012207a..740678ac 100644 --- a/docs/skills/analysis.md +++ b/docs/skills/analysis.md @@ -37,6 +37,7 @@ The interpreter is [pydantic-monty](https://github.com/pydantic/monty), isolated - **Limited imports.** Only `json`, `re`, `math`, `pathlib`. - **Execution timeout** (default 60s, configurable via `analysis.code_timeout`). - **Output truncation** (default 50000 chars, configurable via `analysis.max_output_chars`). +- **Execution budget** (default 15 calls, configurable via `analysis.max_executions`). Past the budget, `execute_code` returns a notice telling the skill to answer from what it has instead of running more code. Variables persist between `execute_code` calls within one invocation, so the agent can build state step by step. A fresh sandbox is built per `client.analyze` call. @@ -190,6 +191,7 @@ analysis: name: claude-sonnet-4-20250514 code_timeout: 60.0 # Max seconds per code execution max_output_chars: 50000 # Truncate output after this many chars + max_executions: 15 # Max execute_code calls per question ``` When `analysis.model` is unset, the skill falls back to `qa.model`. diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index 50a9cc1f..5263eca0 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -96,6 +96,7 @@ class AnalysisConfig(BaseModel): model: ModelConfig | None = None code_timeout: float = 60.0 max_output_chars: int = 50_000 + max_executions: int = 15 class PictureDescriptionConfig(BaseModel): diff --git a/haiku_rag_slim/haiku/rag/skills/_deps.py b/haiku_rag_slim/haiku/rag/skills/_deps.py index 530d47b4..f113b6ab 100644 --- a/haiku_rag_slim/haiku/rag/skills/_deps.py +++ b/haiku_rag_slim/haiku/rag/skills/_deps.py @@ -27,6 +27,7 @@ class RAGRunDeps(SkillRunDeps): @dataclass class AnalysisRunDeps(RAGRunDeps): sandbox: "Sandbox | None" = None + execute_count: int = 0 def _reset_invocation_state(state: Any) -> None: @@ -73,6 +74,7 @@ def make_analysis_lifespan(db_path: Path, config: AppConfig): async with HaikuRAG(db_path, config=config, read_only=True) as rag: deps.rag = rag deps.search_count = 0 + deps.execute_count = 0 sandbox = Sandbox( db_path=db_path, config=config, diff --git a/haiku_rag_slim/haiku/rag/skills/_tools.py b/haiku_rag_slim/haiku/rag/skills/_tools.py index 7366d794..71132b54 100644 --- a/haiku_rag_slim/haiku/rag/skills/_tools.py +++ b/haiku_rag_slim/haiku/rag/skills/_tools.py @@ -265,6 +265,7 @@ def create_skill_tools( tools["get_document"] = get_document if "execute_code" in tool_names: + max_executions = config.analysis.max_executions async def execute_code(ctx: RunContext[AnalysisRunDeps], code: str) -> str: """Execute Python code in a sandboxed interpreter. @@ -280,6 +281,13 @@ def create_skill_tools( Args: code: Python code to execute. """ + ctx.deps.execute_count += 1 + if ctx.deps.execute_count > max_executions: + return ( + "Code-execution limit reached. Give your final answer now " + "from what you already have; do not call execute_code again." + ) + assert ctx.deps is not None and ctx.deps.sandbox is not None, ( "AnalysisRunDeps.sandbox is not set — skill lifespan must run before execute_code." ) diff --git a/haiku_rag_slim/haiku/rag/skills/analysis.py b/haiku_rag_slim/haiku/rag/skills/analysis.py index bcdc674d..39aa127c 100644 --- a/haiku_rag_slim/haiku/rag/skills/analysis.py +++ b/haiku_rag_slim/haiku/rag/skills/analysis.py @@ -97,4 +97,5 @@ def create_skill( state_namespace=STATE_NAMESPACE, deps_type=AnalysisRunDeps, lifespan=make_analysis_lifespan(db_path, config), + request_limit=30, ) diff --git a/tests/skills/test_analysis.py b/tests/skills/test_analysis.py index 1c28c959..99dac76d 100644 --- a/tests/skills/test_analysis.py +++ b/tests/skills/test_analysis.py @@ -55,6 +55,14 @@ class TestAnalysisSkillCreation: assert skill.metadata.description assert skill.instructions + def test_create_skill_sets_request_limit_backstop( + self, test_app_config, temp_db_path + ): + from haiku.rag.skills.analysis import create_skill + + skill = create_skill(config=test_app_config, db_path=temp_db_path) + assert skill.request_limit == 30 + def test_create_skill_has_expected_tools(self, test_app_config, temp_db_path): from haiku.rag.skills.analysis import create_skill @@ -148,6 +156,23 @@ class TestExecuteCodeTool: assert "ZeroDivisionError" in result assert state.executions[0].success is False + async def test_execute_code_rate_limited(self, rag_db, sandbox_factory): + from haiku.rag.skills.analysis import create_skill + + config = AppConfig() + config.analysis.max_executions = 2 + skill = create_skill(db_path=rag_db, config=config) + execute_code = _get_tool(skill, "execute_code") + state = AnalysisState() + ctx = _make_ctx(state, sandbox=sandbox_factory()) + + await execute_code(ctx, code="print('first')") + await execute_code(ctx, code="print('second')") + result = await execute_code(ctx, code="print('third')") + assert "limit reached" in result.lower() + assert ctx.deps.execute_count == 3 + assert len(state.executions) == 2 + async def test_execute_code_applies_document_filter(self, rag_db, sandbox_factory): from haiku.rag.skills.analysis import create_skill @@ -295,6 +320,7 @@ class TestAnalysisLifespan: assert deps.rag is not None assert deps.rag.is_read_only assert deps.search_count == 0 + assert deps.execute_count == 0 assert isinstance(deps.sandbox, Sandbox) docs = await deps.rag.list_documents() assert len(docs) == 2 @@ -311,6 +337,16 @@ class TestAnalysisLifespan: assert deps.sandbox is not None assert deps.sandbox._context.filter == "title = 'AI Overview'" + async def test_lifespan_resets_counts_per_invocation(self, rag_db): + from haiku.rag.skills._deps import AnalysisRunDeps, make_analysis_lifespan + + config = AppConfig() + lifespan = make_analysis_lifespan(rag_db, config) + deps = AnalysisRunDeps(search_count=7, execute_count=42) + async with lifespan(deps): + assert deps.search_count == 0 + assert deps.execute_count == 0 + async def test_skill_has_lifespan_and_deps_type( self, test_app_config, temp_db_path ): From d386d7f90071960b5002ef578286db001bd98e53 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 5 Jun 2026 15:11:03 +0300 Subject: [PATCH 09/13] Bump haiku.skills to 0.17.2 --- CHANGELOG.md | 4 ++++ haiku_rag_slim/pyproject.toml | 2 +- uv.lock | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 78fd6ac2..0fe23d6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog ## [Unreleased] +### Changed + +- Bump `haiku.skills>=0.17.2`: skill runs use a retry budget of 3 for tool calls and output validation (was 1). + ### Added - `analysis.max_executions` (default 15): caps `execute_code` calls per analysis question. Past the cap the tool returns a notice telling the skill to answer from what it has, instead of spiralling into `request_limit` and returning nothing. The analysis skill sets `request_limit` to 30 as a backstop. diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index 2c683ca1..1f68c795 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ dependencies = [ "docling-core>=2.75.0", - "haiku.skills>=0.17.1", + "haiku.skills>=0.17.2", "httpx>=0.28.1", "jinja2>=3.1.0", "jsonpatch>=1.33", diff --git a/uv.lock b/uv.lock index 4e749118..07eab071 100644 --- a/uv.lock +++ b/uv.lock @@ -1746,7 +1746,7 @@ requires-dist = [ { name = "docling-core", specifier = ">=2.75.0" }, { name = "fastapi", marker = "extra == 'ingester'", specifier = ">=0.125" }, { name = "haiku-rag-slim", extras = ["s3"], marker = "extra == 'ingester'", editable = "haiku_rag_slim" }, - { name = "haiku-skills", specifier = ">=0.17.1" }, + { name = "haiku-skills", specifier = ">=0.17.2" }, { name = "httpx", specifier = ">=0.28.1" }, { name = "jinja2", specifier = ">=3.1.0" }, { name = "jsonpatch", specifier = ">=1.33" }, @@ -1788,7 +1788,7 @@ provides-extras = ["docling", "s3", "voyageai", "mxbai", "cohere", "zeroentropy" [[package]] name = "haiku-skills" -version = "0.17.1" +version = "0.17.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "ag-ui-protocol" }, @@ -1798,9 +1798,9 @@ dependencies = [ { name = "pyyaml" }, { name = "skills-ref" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/01/90/7eb6c20aada4e861589ecc300bcf5c827335d803c4e225a79b0924dd9793/haiku_skills-0.17.1.tar.gz", hash = "sha256:062385ae67f61e37a9790721da50f7b38ed0b903e9a8efb2bd76b4b9fcf94651", size = 187965, upload-time = "2026-05-21T10:37:45.242Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/7f/b933476724947fdd4989365b8e924f80d0149225307322c02a3fcdc0f327/haiku_skills-0.17.2.tar.gz", hash = "sha256:8694355df2a83e39146a22fa7ce373b2991adb52eb73c30b101bd21509cf787c", size = 188105, upload-time = "2026-06-05T12:05:43.783Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c9/86/ff441d3c5fba2e66d29c135659d536dd47a5e920e4b351c9621a89599510/haiku_skills-0.17.1-py3-none-any.whl", hash = "sha256:7bdcafc5f184bb765eb9c86e0147d535272dc42c1dc8cef4ce1a4d1464376ff0", size = 32965, upload-time = "2026-05-21T10:37:44.192Z" }, + { url = "https://files.pythonhosted.org/packages/52/bd/09d5a30e248bbf818154ca0d9d7d5b0fba65f75e960d981763ff25b35d4c/haiku_skills-0.17.2-py3-none-any.whl", hash = "sha256:89990a392cb610da1d599ea55f342055b747066d45fa55c5d360e6870af7c5e2", size = 32973, upload-time = "2026-06-05T12:05:42.706Z" }, ] [[package]] From a3a73f1331fd6f27eb51bcb82f2c1a285936e235 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 8 Jun 2026 09:36:01 +0300 Subject: [PATCH 10/13] Add --filter-ids to run QA on a case-id subset --- CHANGELOG.md | 1 + docs/benchmarks.md | 14 +++++- evaluations/evaluations/benchmark.py | 30 ++++++++++++ evaluations/tests/test_benchmark.py | 68 ++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fe23d6a..be2157f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - `analysis.max_executions` (default 15): caps `execute_code` calls per analysis question. Past the cap the tool returns a notice telling the skill to answer from what it has, instead of spiralling into `request_limit` and returning nothing. The analysis skill sets `request_limit` to 30 as a backstop. - `t2_finqa` and `t2_tatdqa` evaluation datasets (T²-RAGBench subsets, `G4KMU/t2-ragbench`): financial-report PDFs ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`. QA is scored with a deterministic `NumberMatchEvaluator` (relative tolerance 0.01) via the new `DatasetSpec.qa_evaluator`, bypassing the LLM judge. +- `evaluations run --filter-ids `: run QA on just the case ids listed in a file (failure-subset rerun); retrieval is unaffected. ### Fixed diff --git a/docs/benchmarks.md b/docs/benchmarks.md index d87ec53d..36f8a802 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,6 +1,6 @@ # Benchmarks -We evaluate `haiku.rag` on a small set of datasets that exercise different parts of the pipeline. Wix and OpenRAG Bench (ORB) are the two we currently track. Retrieval, QA accuracy, and citation retrieval are scored end-to-end through the rag and rag-analysis skills. +We evaluate `haiku.rag` on a small set of datasets that exercise different parts of the pipeline. Wix, OpenRAG Bench (ORB), and T²-RAGBench are the datasets we currently track. Retrieval, QA accuracy, and citation retrieval are scored end-to-end through the rag and rag-analysis skills. ## Running Evaluations @@ -151,6 +151,18 @@ Two approaches are benchmarked separately: *Measured on haiku.rag v0.50.0 with `mxbai-rerank-base-v2`, judged by `vllm:Qwen3.6-35B-A3B-NVFP4`. Nemotron covered 2836 / 3045 cases.* +### T²-RAGBench (FinQA) + +[T²-RAGBench](https://huggingface.co/datasets/G4KMU/t2-ragbench) reformulates financial-report QA into context-independent questions with short numeric answers and a 1:1 gold document mapping. The FinQA subset is 2,789 single-page PDFs / 8,281 questions, ingested via docling. Unlike the other datasets, QA is scored deterministically with `NumberMatchEvaluator` (relative tolerance 0.01) instead of an LLM judge, so QA accuracy here is exact numeric match rather than a judged fraction. + +##### QA accuracy + citation retrieval + +| Embedding Model | Reranker | Target | Skill model | Cases | QA accuracy | Mean `cited_map` | +|----------------------|------------------------|------------------|------------------------------|------:|-------------|------------------| +| `qwen3-embedding:4b` | `mxbai-rerank-base-v2` | `analysis-skill` | `vllm:Qwen3.6-35B-A3B-NVFP4` | 7939 | 0.77 | 0.78 | + +*Measured on haiku.rag v0.55.0, deterministic Number-Match scoring (ε=0.01), 2560-dim `qwen3-embedding:4b` (vLLM) with `mxbai-rerank-base-v2`. 341 / 8281 cases excluded as nulls (analysis spirals from the request limit and in-generation loops). Accuracy and `cited_map` are over the 7939 scored cases. Mean 16.0s/case.* + ### Wix [WixQA](https://huggingface.co/datasets/Wix/WixQA) is real customer support questions paired with curated answers. 200 cases. diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index dd7c9d99..54922311 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -334,6 +334,16 @@ def _attach_relevant_uris( case.metadata = metadata +def _filter_qa_corpus(corpus, case_ids: set[str] | None): + """Keep only rows whose ``id`` is in ``case_ids`` (failure-subset reruns). + + Returns the corpus unchanged when ``case_ids`` is None. + """ + if case_ids is None: + return corpus + return corpus.filter(lambda row: row.get("id") in case_ids) + + async def run_qa_benchmark( spec: DatasetSpec, config: AppConfig, @@ -343,8 +353,10 @@ async def run_qa_benchmark( judge_model: ModelConfig | None = None, target: Target = "rag-skill", skill_model: ModelConfig | None = None, + case_ids: set[str] | None = None, ) -> ReportCaseFailure[str, str, dict[str, str]] | None: corpus = spec.qa_loader() + corpus = _filter_qa_corpus(corpus, case_ids) if limit is not None: corpus = corpus.select(range(min(limit, len(corpus)))) @@ -499,6 +511,7 @@ async def evaluate_dataset( judge_model: ModelConfig | None = None, target: Target = "rag-skill", skill_model: ModelConfig | None = None, + case_ids: set[str] | None = None, ) -> None: if not skip_db: console.print(f"Using dataset: {spec.key}", style="bold magenta") @@ -530,6 +543,7 @@ async def evaluate_dataset( judge_model=judge_model, target=target, skill_model=skill_model, + case_ids=case_ids, ) @@ -555,6 +569,13 @@ def _load_config(config_path: Path | None) -> AppConfig: return AppConfig() +def _load_case_ids(path: Path | None) -> set[str] | None: + """Read a newline-delimited case-id file into a set (None when no path).""" + if path is None: + return None + return {line.strip() for line in path.read_text().splitlines() if line.strip()} + + def _resolve_dataset(dataset: str) -> DatasetSpec: """Resolve a dataset key to a DatasetSpec or raise BadParameter.""" spec = DATASETS.get(dataset.lower()) @@ -612,6 +633,14 @@ def run( "analysis.model when --target is analysis-skill) from the config." ), ), + filter_ids: Path | None = typer.Option( + None, + "--filter-ids", + help=( + "Path to a newline-delimited file of QA case ids to run " + "(failure-subset rerun). Filters QA only; retrieval is unaffected." + ), + ), ) -> None: spec = _resolve_dataset(dataset) app_config = _load_config(config) @@ -638,6 +667,7 @@ def run( judge_model=judge_model_config, target=target_value, skill_model=skill_model_config, + case_ids=_load_case_ids(filter_ids), ) ) diff --git a/evaluations/tests/test_benchmark.py b/evaluations/tests/test_benchmark.py index 84b93d32..c7ffde27 100644 --- a/evaluations/tests/test_benchmark.py +++ b/evaluations/tests/test_benchmark.py @@ -406,3 +406,71 @@ class TestAttachRelevantUris: ) _attach_relevant_uris(cases, spec, limit=None) assert cases[0].metadata is None + + +class TestFilterQaCorpus: + def test_keeps_only_matching_ids(self) -> None: + from datasets import Dataset + + from evaluations.benchmark import _filter_qa_corpus + + corpus = Dataset.from_list( + [{"id": "a", "q": 1}, {"id": "b", "q": 2}, {"id": "c", "q": 3}] + ) + out = _filter_qa_corpus(corpus, {"a", "c"}) + assert [r["id"] for r in out] == ["a", "c"] + + def test_none_returns_corpus_unchanged(self) -> None: + from datasets import Dataset + + from evaluations.benchmark import _filter_qa_corpus + + corpus = Dataset.from_list([{"id": "a"}]) + assert _filter_qa_corpus(corpus, None) is corpus + + +class TestLoadCaseIds: + def test_reads_strips_and_drops_blanks(self, tmp_path: Path) -> None: + from evaluations.benchmark import _load_case_ids + + f = tmp_path / "ids.txt" + f.write_text("finqa_dev_16\n finqa_dev_66 \n\n\nfinqa_dev_113\n") + assert _load_case_ids(f) == {"finqa_dev_16", "finqa_dev_66", "finqa_dev_113"} + + def test_none_path_returns_none(self) -> None: + from evaluations.benchmark import _load_case_ids + + assert _load_case_ids(None) is None + + +class TestEvaluateDatasetCaseIds: + def _spec(self) -> DatasetSpec: + return DatasetSpec( + key="test", + db_filename="test.lancedb", + document_loader=lambda: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + document_mapper=lambda doc: None, + qa_loader=lambda: [], # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + qa_case_builder=lambda idx, doc: None, # type: ignore[arg-type] # ty: ignore[invalid-argument-type] + ) + + @pytest.mark.asyncio + async def test_threads_case_ids_to_qa_benchmark(self) -> None: + from evaluations.benchmark import evaluate_dataset + + with patch( + "evaluations.benchmark.run_qa_benchmark", new_callable=AsyncMock + ) as mock_qa: + await evaluate_dataset( + spec=self._spec(), + config=AppConfig(), + skip_db=True, + skip_retrieval=True, + skip_qa=False, + limit=None, + name=None, + db_path=None, + case_ids={"finqa_dev_16", "finqa_dev_66"}, + ) + mock_qa.assert_called_once() + assert mock_qa.call_args[1]["case_ids"] == {"finqa_dev_16", "finqa_dev_66"} From e9bd56467c1cd997e3dccbfdfd37723a771574cb Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 8 Jun 2026 10:46:32 +0300 Subject: [PATCH 11/13] =?UTF-8?q?Add=20T=C2=B2-RAGBench=20leaderboard=20su?= =?UTF-8?q?bmission=20exporter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 +- CHANGELOG.md | 1 + evaluations/evaluations/submission.py | 74 +++++++++++++++ evaluations/scripts/build_t2_submission.py | 103 +++++++++++++++++++++ evaluations/tests/test_submission.py | 72 ++++++++++++++ 5 files changed, 252 insertions(+), 1 deletion(-) create mode 100644 evaluations/evaluations/submission.py create mode 100755 evaluations/scripts/build_t2_submission.py create mode 100644 evaluations/tests/test_submission.py diff --git a/.gitignore b/.gitignore index 291aa0ef..5010d886 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,8 @@ wheels/ # tests .coverage* evaluations/evaluations/data/ -evaluations/scripts/ +evaluations/scripts/* +!evaluations/scripts/build_t2_submission.py tests/data/ .pytest_cache/ .ruff_cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md index be2157f4..5f4694dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - `analysis.max_executions` (default 15): caps `execute_code` calls per analysis question. Past the cap the tool returns a notice telling the skill to answer from what it has, instead of spiralling into `request_limit` and returning nothing. The analysis skill sets `request_limit` to 30 as a backstop. - `t2_finqa` and `t2_tatdqa` evaluation datasets (T²-RAGBench subsets, `G4KMU/t2-ragbench`): financial-report PDFs ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`. QA is scored with a deterministic `NumberMatchEvaluator` (relative tolerance 0.01) via the new `DatasetSpec.qa_evaluator`, bypassing the LLM judge. - `evaluations run --filter-ids `: run QA on just the case ids listed in a file (failure-subset rerun); retrieval is unaffected. +- `evaluations/scripts/build_t2_submission.py` + `evaluations.submission`: build a T²-RAGBench leaderboard submission JSONL (`{id, subset, context_id, prediction}`) by joining QA predictions with retrieval rankings by question. ### Fixed diff --git a/evaluations/evaluations/submission.py b/evaluations/evaluations/submission.py new file mode 100644 index 00000000..d65121f4 --- /dev/null +++ b/evaluations/evaluations/submission.py @@ -0,0 +1,74 @@ +import re +from collections.abc import Iterable, Mapping, Sequence +from typing import Any + +from evaluations.evaluators.number_match import _answer_segment + +# A single numeric literal: optional sign, optional $, digits with thousands +# separators, optional decimal, optional trailing percent. Scale words +# (million/billion) are deliberately NOT expanded — T² gold answers are bare +# numbers, so expanding would mis-scale (e.g. "688 million" must stay 688). +_NUM_RE = re.compile(r"[-−]?\$?\s*\d[\d,]*(?:\.\d+)?\s*%?") + + +def _format_number(value: float) -> str: + """Render without a trailing ``.0`` for integers; plain decimal otherwise.""" + if value == int(value): + return str(int(value)) + return repr(value) + + +def extract_prediction(output: str | None) -> str: + """Pull the primary numeric answer from a skill output, for submission. + + Restricts to a declared ``ANSWER:`` line when present (via ``_answer_segment``) + so reasoning numbers don't leak. Strips ``$`` and thousands separators, + converts a trailing ``%`` to a fraction (T² gold stores percentages as + decimals), and normalizes the unicode minus. Returns ``""`` for empty/no-number + outputs (nulls) — the leaderboard counts those as wrong. + + NOTE: the exact normalization the leaderboard's NM applies is unconfirmed; + validate against their scorer before a final submission. + """ + if not output: + return "" + match = _NUM_RE.search(_answer_segment(output)) + if match is None: + return "" + token = match.group(0).replace("$", "").replace(",", "").replace(" ", "") + token = token.replace("−", "-") + if token.endswith("%"): + return _format_number(float(token[:-1]) / 100) + return _format_number(float(token)) + + +def build_submission_rows( + predictions: Iterable[Mapping[str, Any]], + retrieval_by_question: Mapping[str, Sequence[str]], + subset: str, + topk: int = 3, +) -> list[dict[str, Any]]: + """Assemble T² leaderboard submission rows. + + Args: + predictions: rows with ``id``, ``question`` and ``output`` (the QA run). + retrieval_by_question: question text -> ranked retrieved context ids. + subset: dataset subset name (e.g. ``"FinQA"``). + topk: how many ranked context ids to include. ``context_id`` is a single + string when ``topk == 1``, else a list of up to ``topk`` ids. + + Returns one dict per prediction: ``{id, subset, context_id, prediction}``. + """ + rows: list[dict[str, Any]] = [] + for pred in predictions: + ranked = list(retrieval_by_question.get(pred["question"], []))[:topk] + context_id: Any = (ranked[0] if ranked else None) if topk == 1 else ranked + rows.append( + { + "id": pred["id"], + "subset": subset, + "context_id": context_id, + "prediction": extract_prediction(pred.get("output")), + } + ) + return rows diff --git a/evaluations/scripts/build_t2_submission.py b/evaluations/scripts/build_t2_submission.py new file mode 100755 index 00000000..40f58b09 --- /dev/null +++ b/evaluations/scripts/build_t2_submission.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +"""Build a T²-RAGBench leaderboard submission file (JSONL) for one subset. + +Joins QA predictions with retrieval rankings by question text and emits one +object per question: ``{id, subset, context_id, prediction}`` — the format the +leaderboard expects (NM is scored on ``prediction``, MRR@3 on ``context_id``). +See https://t2ragbench.demo.hcds.uni-hamburg.de/submission.html. + +Predictions come from a QA-run CSV (the merged master export: needs columns +``id``, ``question``, ``output``). Retrieval rankings come from a Logfire +retrieval trace (case spans store the ranked retrieved context ids in +``output`` and the question in ``inputs``) or, with --retrieval-csv, a CSV with +``question`` and a JSON-list ``output``. + +Logfire access needs LOGFIRE_READ_TOKEN in the environment (EU project). + +Example: + LOGFIRE_READ_TOKEN=... uv run python scripts/build_t2_submission.py \ + --predictions ../t2_finqa_qwen3.6_019e982d.csv \ + --retrieval-trace 019e9296e64d38b33f3592beff6660a9 \ + --subset FinQA --topk 3 --out finqa_submission.jsonl +""" + +import argparse +import csv +import datetime +import json +import os +import sys + +from evaluations.submission import build_submission_rows + + +def _load_predictions(path: str) -> list[dict[str, str]]: + with open(path) as f: + return list(csv.DictReader(f)) + + +def _retrieval_from_csv(path: str) -> dict[str, list[str]]: + out: dict[str, list[str]] = {} + with open(path) as f: + for row in csv.DictReader(f): + out[row["question"]] = json.loads(row["output"]) + return out + + +def _retrieval_from_trace(trace_id: str) -> dict[str, list[str]]: + from logfire.experimental.query_client import LogfireQueryClient + + token = os.environ.get("LOGFIRE_READ_TOKEN") + if not token: + sys.exit("LOGFIRE_READ_TOKEN not set (needed to read the retrieval trace).") + client = LogfireQueryClient( + read_token=token, base_url="https://logfire-eu.pydantic.dev" + ) + rows = client.query_json_rows( + sql=( + "SELECT attributes->>'inputs' AS question, attributes->>'output' AS ranked " + f"FROM records WHERE trace_id = '{trace_id}' AND span_name LIKE 'case:%'" + ), + min_timestamp=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + limit=10000, + )["rows"] + return {r["question"]: json.loads(r["ranked"]) for r in rows if r.get("ranked")} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--predictions", required=True, help="QA-run CSV.") + parser.add_argument("--retrieval-trace", help="Logfire retrieval trace id.") + parser.add_argument("--retrieval-csv", help="Retrieval CSV (question, output).") + parser.add_argument("--subset", required=True, help="Subset name, e.g. FinQA.") + parser.add_argument("--topk", type=int, default=3, help="Ranked context ids.") + parser.add_argument("--out", required=True, help="Output .jsonl path.") + args = parser.parse_args() + + if bool(args.retrieval_trace) == bool(args.retrieval_csv): + sys.exit("Pass exactly one of --retrieval-trace or --retrieval-csv.") + + predictions = _load_predictions(args.predictions) + retrieval = ( + _retrieval_from_csv(args.retrieval_csv) + if args.retrieval_csv + else _retrieval_from_trace(args.retrieval_trace) + ) + + rows = build_submission_rows( + predictions, retrieval, subset=args.subset, topk=args.topk + ) + with open(args.out, "w") as f: + for row in rows: + f.write(json.dumps(row) + "\n") + + answered = sum(1 for r in rows if r["prediction"]) + matched = sum(1 for r in rows if r["context_id"]) + print( + f"wrote {args.out}: {len(rows)} rows " + f"({answered} with a prediction, {matched} with a retrieved context_id)" + ) + + +if __name__ == "__main__": + main() diff --git a/evaluations/tests/test_submission.py b/evaluations/tests/test_submission.py new file mode 100644 index 00000000..cbb8eede --- /dev/null +++ b/evaluations/tests/test_submission.py @@ -0,0 +1,72 @@ +from evaluations.submission import build_submission_rows, extract_prediction + + +class TestExtractPrediction: + def test_answer_line_integer(self) -> None: + assert extract_prediction("...\n\nANSWER: 18.6") == "18.6" + + def test_percent_becomes_fraction(self) -> None: + # T² gold stores percentages as decimals. + assert extract_prediction("ANSWER: 93.5%") == "0.935" + + def test_strips_currency_commas_and_scale_word(self) -> None: + # "$688 million" -> 688 (scale word not expanded; gold is the bare number) + assert extract_prediction("ANSWER: $688 million") == "688" + assert extract_prediction("ANSWER: $1,234.5") == "1234.5" + + def test_unicode_minus(self) -> None: + assert extract_prediction("ANSWER: −1.9") == "-1.9" + + def test_uses_answer_line_not_reasoning(self) -> None: + out = "We saw 72.8 in the table but recomputed.\nANSWER: 82.8" + assert extract_prediction(out) == "82.8" + + def test_empty_output_is_blank(self) -> None: + assert extract_prediction("") == "" + assert extract_prediction(None) == "" + + def test_no_number_is_blank(self) -> None: + assert extract_prediction("ANSWER: not reported") == "" + + +class TestBuildSubmissionRows: + def _preds(self) -> list[dict[str, str]]: + return [ + {"id": "finqa_dev_0", "question": "Q1?", "output": "ANSWER: 127.4"}, + {"id": "finqa_dev_1", "question": "Q2?", "output": ""}, # null + ] + + def _retrieval(self) -> dict[str, list[str]]: + return { + "Q1?": ["ctx_a", "ctx_b", "ctx_c", "ctx_d"], + "Q2?": ["ctx_e", "ctx_f"], + } + + def test_topk_list_and_fields(self) -> None: + rows = build_submission_rows( + self._preds(), self._retrieval(), subset="FinQA", topk=3 + ) + assert rows[0] == { + "id": "finqa_dev_0", + "subset": "FinQA", + "context_id": ["ctx_a", "ctx_b", "ctx_c"], + "prediction": "127.4", + } + # null prediction -> blank string (counted wrong); ranking still attached + assert rows[1]["prediction"] == "" + assert rows[1]["context_id"] == ["ctx_e", "ctx_f"] + + def test_topk_one_emits_single_string(self) -> None: + rows = build_submission_rows( + self._preds(), self._retrieval(), subset="FinQA", topk=1 + ) + assert rows[0]["context_id"] == "ctx_a" + + def test_missing_retrieval_is_empty(self) -> None: + rows = build_submission_rows( + [{"id": "x", "question": "unseen?", "output": "ANSWER: 1"}], + self._retrieval(), + subset="FinQA", + topk=3, + ) + assert rows[0]["context_id"] == [] From 295570797c408a39a5d64318b0a680aeebdf8182 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 8 Jun 2026 11:42:16 +0300 Subject: [PATCH 12/13] Disable Logfire scrubbing in evaluations --- CHANGELOG.md | 1 + evaluations/evaluations/benchmark.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f4694dd..9dd0c064 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Changed - Bump `haiku.skills>=0.17.2`: skill runs use a retry budget of 3 for tool calls and output validation (was 1). +- Evaluations disable Logfire scrubbing (`scrubbing=False`) so financial answers containing words like "authorized" aren't redacted from logged outputs. ### Added diff --git a/evaluations/evaluations/benchmark.py b/evaluations/evaluations/benchmark.py index 54922311..a3f06aee 100644 --- a/evaluations/evaluations/benchmark.py +++ b/evaluations/evaluations/benchmark.py @@ -40,7 +40,11 @@ load_dotenv(find_dotenv(usecwd=True)) HF_REPO_ID = "ggozad/haiku-rag-eval-dbs" -logfire.configure(send_to_logfire="if-token-present", service_name="evals") +# Scrubbing off: eval outputs are financial answers with words like "authorized" +# that trip Logfire's secret scrubber and redact the model's answer text. +logfire.configure( + send_to_logfire="if-token-present", service_name="evals", scrubbing=False +) logfire.instrument_pydantic_ai() configure_cli_logging() console = Console() From 9ef25e2e53dd2b64833520072b8d3aced89d1fd8 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 8 Jun 2026 11:44:21 +0300 Subject: [PATCH 13/13] =?UTF-8?q?Order=20benchmarks=20docs=20ORB,=20T?= =?UTF-8?q?=C2=B2,=20Wix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/benchmarks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 36f8a802..70be679f 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,6 +1,6 @@ # Benchmarks -We evaluate `haiku.rag` on a small set of datasets that exercise different parts of the pipeline. Wix, OpenRAG Bench (ORB), and T²-RAGBench are the datasets we currently track. Retrieval, QA accuracy, and citation retrieval are scored end-to-end through the rag and rag-analysis skills. +We evaluate `haiku.rag` on a small set of datasets that exercise different parts of the pipeline. OpenRAG Bench (ORB), T²-RAGBench, and Wix are the datasets we currently track. Retrieval, QA accuracy, and citation retrieval are scored end-to-end through the rag and rag-analysis skills. ## Running Evaluations