diff --git a/.gitignore b/.gitignore index 291aa0ef..5010d886 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,8 @@ wheels/ # tests .coverage* evaluations/evaluations/data/ -evaluations/scripts/ +evaluations/scripts/* +!evaluations/scripts/build_t2_submission.py tests/data/ .pytest_cache/ .ruff_cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md index be2157f4..5f4694dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - `analysis.max_executions` (default 15): caps `execute_code` calls per analysis question. Past the cap the tool returns a notice telling the skill to answer from what it has, instead of spiralling into `request_limit` and returning nothing. The analysis skill sets `request_limit` to 30 as a backstop. - `t2_finqa` and `t2_tatdqa` evaluation datasets (T²-RAGBench subsets, `G4KMU/t2-ragbench`): financial-report PDFs ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`. QA is scored with a deterministic `NumberMatchEvaluator` (relative tolerance 0.01) via the new `DatasetSpec.qa_evaluator`, bypassing the LLM judge. - `evaluations run --filter-ids `: run QA on just the case ids listed in a file (failure-subset rerun); retrieval is unaffected. +- `evaluations/scripts/build_t2_submission.py` + `evaluations.submission`: build a T²-RAGBench leaderboard submission JSONL (`{id, subset, context_id, prediction}`) by joining QA predictions with retrieval rankings by question. ### Fixed diff --git a/evaluations/evaluations/submission.py b/evaluations/evaluations/submission.py new file mode 100644 index 00000000..d65121f4 --- /dev/null +++ b/evaluations/evaluations/submission.py @@ -0,0 +1,74 @@ +import re +from collections.abc import Iterable, Mapping, Sequence +from typing import Any + +from evaluations.evaluators.number_match import _answer_segment + +# A single numeric literal: optional sign, optional $, digits with thousands +# separators, optional decimal, optional trailing percent. Scale words +# (million/billion) are deliberately NOT expanded — T² gold answers are bare +# numbers, so expanding would mis-scale (e.g. "688 million" must stay 688). +_NUM_RE = re.compile(r"[-−]?\$?\s*\d[\d,]*(?:\.\d+)?\s*%?") + + +def _format_number(value: float) -> str: + """Render without a trailing ``.0`` for integers; plain decimal otherwise.""" + if value == int(value): + return str(int(value)) + return repr(value) + + +def extract_prediction(output: str | None) -> str: + """Pull the primary numeric answer from a skill output, for submission. + + Restricts to a declared ``ANSWER:`` line when present (via ``_answer_segment``) + so reasoning numbers don't leak. Strips ``$`` and thousands separators, + converts a trailing ``%`` to a fraction (T² gold stores percentages as + decimals), and normalizes the unicode minus. Returns ``""`` for empty/no-number + outputs (nulls) — the leaderboard counts those as wrong. + + NOTE: the exact normalization the leaderboard's NM applies is unconfirmed; + validate against their scorer before a final submission. + """ + if not output: + return "" + match = _NUM_RE.search(_answer_segment(output)) + if match is None: + return "" + token = match.group(0).replace("$", "").replace(",", "").replace(" ", "") + token = token.replace("−", "-") + if token.endswith("%"): + return _format_number(float(token[:-1]) / 100) + return _format_number(float(token)) + + +def build_submission_rows( + predictions: Iterable[Mapping[str, Any]], + retrieval_by_question: Mapping[str, Sequence[str]], + subset: str, + topk: int = 3, +) -> list[dict[str, Any]]: + """Assemble T² leaderboard submission rows. + + Args: + predictions: rows with ``id``, ``question`` and ``output`` (the QA run). + retrieval_by_question: question text -> ranked retrieved context ids. + subset: dataset subset name (e.g. ``"FinQA"``). + topk: how many ranked context ids to include. ``context_id`` is a single + string when ``topk == 1``, else a list of up to ``topk`` ids. + + Returns one dict per prediction: ``{id, subset, context_id, prediction}``. + """ + rows: list[dict[str, Any]] = [] + for pred in predictions: + ranked = list(retrieval_by_question.get(pred["question"], []))[:topk] + context_id: Any = (ranked[0] if ranked else None) if topk == 1 else ranked + rows.append( + { + "id": pred["id"], + "subset": subset, + "context_id": context_id, + "prediction": extract_prediction(pred.get("output")), + } + ) + return rows diff --git a/evaluations/scripts/build_t2_submission.py b/evaluations/scripts/build_t2_submission.py new file mode 100755 index 00000000..40f58b09 --- /dev/null +++ b/evaluations/scripts/build_t2_submission.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +"""Build a T²-RAGBench leaderboard submission file (JSONL) for one subset. + +Joins QA predictions with retrieval rankings by question text and emits one +object per question: ``{id, subset, context_id, prediction}`` — the format the +leaderboard expects (NM is scored on ``prediction``, MRR@3 on ``context_id``). +See https://t2ragbench.demo.hcds.uni-hamburg.de/submission.html. + +Predictions come from a QA-run CSV (the merged master export: needs columns +``id``, ``question``, ``output``). Retrieval rankings come from a Logfire +retrieval trace (case spans store the ranked retrieved context ids in +``output`` and the question in ``inputs``) or, with --retrieval-csv, a CSV with +``question`` and a JSON-list ``output``. + +Logfire access needs LOGFIRE_READ_TOKEN in the environment (EU project). + +Example: + LOGFIRE_READ_TOKEN=... uv run python scripts/build_t2_submission.py \ + --predictions ../t2_finqa_qwen3.6_019e982d.csv \ + --retrieval-trace 019e9296e64d38b33f3592beff6660a9 \ + --subset FinQA --topk 3 --out finqa_submission.jsonl +""" + +import argparse +import csv +import datetime +import json +import os +import sys + +from evaluations.submission import build_submission_rows + + +def _load_predictions(path: str) -> list[dict[str, str]]: + with open(path) as f: + return list(csv.DictReader(f)) + + +def _retrieval_from_csv(path: str) -> dict[str, list[str]]: + out: dict[str, list[str]] = {} + with open(path) as f: + for row in csv.DictReader(f): + out[row["question"]] = json.loads(row["output"]) + return out + + +def _retrieval_from_trace(trace_id: str) -> dict[str, list[str]]: + from logfire.experimental.query_client import LogfireQueryClient + + token = os.environ.get("LOGFIRE_READ_TOKEN") + if not token: + sys.exit("LOGFIRE_READ_TOKEN not set (needed to read the retrieval trace).") + client = LogfireQueryClient( + read_token=token, base_url="https://logfire-eu.pydantic.dev" + ) + rows = client.query_json_rows( + sql=( + "SELECT attributes->>'inputs' AS question, attributes->>'output' AS ranked " + f"FROM records WHERE trace_id = '{trace_id}' AND span_name LIKE 'case:%'" + ), + min_timestamp=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + limit=10000, + )["rows"] + return {r["question"]: json.loads(r["ranked"]) for r in rows if r.get("ranked")} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--predictions", required=True, help="QA-run CSV.") + parser.add_argument("--retrieval-trace", help="Logfire retrieval trace id.") + parser.add_argument("--retrieval-csv", help="Retrieval CSV (question, output).") + parser.add_argument("--subset", required=True, help="Subset name, e.g. FinQA.") + parser.add_argument("--topk", type=int, default=3, help="Ranked context ids.") + parser.add_argument("--out", required=True, help="Output .jsonl path.") + args = parser.parse_args() + + if bool(args.retrieval_trace) == bool(args.retrieval_csv): + sys.exit("Pass exactly one of --retrieval-trace or --retrieval-csv.") + + predictions = _load_predictions(args.predictions) + retrieval = ( + _retrieval_from_csv(args.retrieval_csv) + if args.retrieval_csv + else _retrieval_from_trace(args.retrieval_trace) + ) + + rows = build_submission_rows( + predictions, retrieval, subset=args.subset, topk=args.topk + ) + with open(args.out, "w") as f: + for row in rows: + f.write(json.dumps(row) + "\n") + + answered = sum(1 for r in rows if r["prediction"]) + matched = sum(1 for r in rows if r["context_id"]) + print( + f"wrote {args.out}: {len(rows)} rows " + f"({answered} with a prediction, {matched} with a retrieved context_id)" + ) + + +if __name__ == "__main__": + main() diff --git a/evaluations/tests/test_submission.py b/evaluations/tests/test_submission.py new file mode 100644 index 00000000..cbb8eede --- /dev/null +++ b/evaluations/tests/test_submission.py @@ -0,0 +1,72 @@ +from evaluations.submission import build_submission_rows, extract_prediction + + +class TestExtractPrediction: + def test_answer_line_integer(self) -> None: + assert extract_prediction("...\n\nANSWER: 18.6") == "18.6" + + def test_percent_becomes_fraction(self) -> None: + # T² gold stores percentages as decimals. + assert extract_prediction("ANSWER: 93.5%") == "0.935" + + def test_strips_currency_commas_and_scale_word(self) -> None: + # "$688 million" -> 688 (scale word not expanded; gold is the bare number) + assert extract_prediction("ANSWER: $688 million") == "688" + assert extract_prediction("ANSWER: $1,234.5") == "1234.5" + + def test_unicode_minus(self) -> None: + assert extract_prediction("ANSWER: −1.9") == "-1.9" + + def test_uses_answer_line_not_reasoning(self) -> None: + out = "We saw 72.8 in the table but recomputed.\nANSWER: 82.8" + assert extract_prediction(out) == "82.8" + + def test_empty_output_is_blank(self) -> None: + assert extract_prediction("") == "" + assert extract_prediction(None) == "" + + def test_no_number_is_blank(self) -> None: + assert extract_prediction("ANSWER: not reported") == "" + + +class TestBuildSubmissionRows: + def _preds(self) -> list[dict[str, str]]: + return [ + {"id": "finqa_dev_0", "question": "Q1?", "output": "ANSWER: 127.4"}, + {"id": "finqa_dev_1", "question": "Q2?", "output": ""}, # null + ] + + def _retrieval(self) -> dict[str, list[str]]: + return { + "Q1?": ["ctx_a", "ctx_b", "ctx_c", "ctx_d"], + "Q2?": ["ctx_e", "ctx_f"], + } + + def test_topk_list_and_fields(self) -> None: + rows = build_submission_rows( + self._preds(), self._retrieval(), subset="FinQA", topk=3 + ) + assert rows[0] == { + "id": "finqa_dev_0", + "subset": "FinQA", + "context_id": ["ctx_a", "ctx_b", "ctx_c"], + "prediction": "127.4", + } + # null prediction -> blank string (counted wrong); ranking still attached + assert rows[1]["prediction"] == "" + assert rows[1]["context_id"] == ["ctx_e", "ctx_f"] + + def test_topk_one_emits_single_string(self) -> None: + rows = build_submission_rows( + self._preds(), self._retrieval(), subset="FinQA", topk=1 + ) + assert rows[0]["context_id"] == "ctx_a" + + def test_missing_retrieval_is_empty(self) -> None: + rows = build_submission_rows( + [{"id": "x", "question": "unseen?", "output": "ANSWER: 1"}], + self._retrieval(), + subset="FinQA", + topk=3, + ) + assert rows[0]["context_id"] == []