Add T²-RAGBench leaderboard submission exporter

.gitignore

@@ -13,7 +13,8 @@ wheels/
 # tests
 .coverage*
 evaluations/evaluations/data/
-evaluations/scripts/
+evaluations/scripts/*
+!evaluations/scripts/build_t2_submission.py
 tests/data/
 .pytest_cache/
 .ruff_cache/

CHANGELOG.md

@@ -10,6 +10,7 @@
 - `analysis.max_executions` (default 15): caps `execute_code` calls per analysis question. Past the cap the tool returns a notice telling the skill to answer from what it has, instead of spiralling into `request_limit` and returning nothing. The analysis skill sets `request_limit` to 30 as a backstop.
 - `t2_finqa` and `t2_tatdqa` evaluation datasets (T²-RAGBench subsets, `G4KMU/t2-ragbench`): financial-report PDFs ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`. QA is scored with a deterministic `NumberMatchEvaluator` (relative tolerance 0.01) via the new `DatasetSpec.qa_evaluator`, bypassing the LLM judge.
 - `evaluations run --filter-ids <file>`: run QA on just the case ids listed in a file (failure-subset rerun); retrieval is unaffected.
+- `evaluations/scripts/build_t2_submission.py` + `evaluations.submission`: build a T²-RAGBench leaderboard submission JSONL (`{id, subset, context_id, prediction}`) by joining QA predictions with retrieval rankings by question.
 
 ### Fixed
 

evaluations/evaluations/submission.py

@@ -0,0 +1,74 @@
+import re
+from collections.abc import Iterable, Mapping, Sequence
+from typing import Any
+
+from evaluations.evaluators.number_match import _answer_segment
+
+# A single numeric literal: optional sign, optional $, digits with thousands
+# separators, optional decimal, optional trailing percent. Scale words
+# (million/billion) are deliberately NOT expanded — T² gold answers are bare
+# numbers, so expanding would mis-scale (e.g. "688 million" must stay 688).
+_NUM_RE = re.compile(r"[-−]?\$?\s*\d[\d,]*(?:\.\d+)?\s*%?")
+
+
+def _format_number(value: float) -> str:
+    """Render without a trailing ``.0`` for integers; plain decimal otherwise."""
+    if value == int(value):
+        return str(int(value))
+    return repr(value)
+
+
+def extract_prediction(output: str | None) -> str:
+    """Pull the primary numeric answer from a skill output, for submission.
+
+    Restricts to a declared ``ANSWER:`` line when present (via ``_answer_segment``)
+    so reasoning numbers don't leak. Strips ``$`` and thousands separators,
+    converts a trailing ``%`` to a fraction (T² gold stores percentages as
+    decimals), and normalizes the unicode minus. Returns ``""`` for empty/no-number
+    outputs (nulls) — the leaderboard counts those as wrong.
+
+    NOTE: the exact normalization the leaderboard's NM applies is unconfirmed;
+    validate against their scorer before a final submission.
+    """
+    if not output:
+        return ""
+    match = _NUM_RE.search(_answer_segment(output))
+    if match is None:
+        return ""
+    token = match.group(0).replace("$", "").replace(",", "").replace(" ", "")
+    token = token.replace("−", "-")
+    if token.endswith("%"):
+        return _format_number(float(token[:-1]) / 100)
+    return _format_number(float(token))
+
+
+def build_submission_rows(
+    predictions: Iterable[Mapping[str, Any]],
+    retrieval_by_question: Mapping[str, Sequence[str]],
+    subset: str,
+    topk: int = 3,
+) -> list[dict[str, Any]]:
+    """Assemble T² leaderboard submission rows.
+
+    Args:
+        predictions: rows with ``id``, ``question`` and ``output`` (the QA run).
+        retrieval_by_question: question text -> ranked retrieved context ids.
+        subset: dataset subset name (e.g. ``"FinQA"``).
+        topk: how many ranked context ids to include. ``context_id`` is a single
+            string when ``topk == 1``, else a list of up to ``topk`` ids.
+
+    Returns one dict per prediction: ``{id, subset, context_id, prediction}``.
+    """
+    rows: list[dict[str, Any]] = []
+    for pred in predictions:
+        ranked = list(retrieval_by_question.get(pred["question"], []))[:topk]
+        context_id: Any = (ranked[0] if ranked else None) if topk == 1 else ranked
+        rows.append(
+            {
+                "id": pred["id"],
+                "subset": subset,
+                "context_id": context_id,
+                "prediction": extract_prediction(pred.get("output")),
+            }
+        )
+    return rows

evaluations/scripts/build_t2_submission.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+"""Build a T²-RAGBench leaderboard submission file (JSONL) for one subset.
+
+Joins QA predictions with retrieval rankings by question text and emits one
+object per question: ``{id, subset, context_id, prediction}`` — the format the
+leaderboard expects (NM is scored on ``prediction``, MRR@3 on ``context_id``).
+See https://t2ragbench.demo.hcds.uni-hamburg.de/submission.html.
+
+Predictions come from a QA-run CSV (the merged master export: needs columns
+``id``, ``question``, ``output``). Retrieval rankings come from a Logfire
+retrieval trace (case spans store the ranked retrieved context ids in
+``output`` and the question in ``inputs``) or, with --retrieval-csv, a CSV with
+``question`` and a JSON-list ``output``.
+
+Logfire access needs LOGFIRE_READ_TOKEN in the environment (EU project).
+
+Example:
+    LOGFIRE_READ_TOKEN=... uv run python scripts/build_t2_submission.py \
+        --predictions ../t2_finqa_qwen3.6_019e982d.csv \
+        --retrieval-trace 019e9296e64d38b33f3592beff6660a9 \
+        --subset FinQA --topk 3 --out finqa_submission.jsonl
+"""
+
+import argparse
+import csv
+import datetime
+import json
+import os
+import sys
+
+from evaluations.submission import build_submission_rows
+
+
+def _load_predictions(path: str) -> list[dict[str, str]]:
+    with open(path) as f:
+        return list(csv.DictReader(f))
+
+
+def _retrieval_from_csv(path: str) -> dict[str, list[str]]:
+    out: dict[str, list[str]] = {}
+    with open(path) as f:
+        for row in csv.DictReader(f):
+            out[row["question"]] = json.loads(row["output"])
+    return out
+
+
+def _retrieval_from_trace(trace_id: str) -> dict[str, list[str]]:
+    from logfire.experimental.query_client import LogfireQueryClient
+
+    token = os.environ.get("LOGFIRE_READ_TOKEN")
+    if not token:
+        sys.exit("LOGFIRE_READ_TOKEN not set (needed to read the retrieval trace).")
+    client = LogfireQueryClient(
+        read_token=token, base_url="https://logfire-eu.pydantic.dev"
+    )
+    rows = client.query_json_rows(
+        sql=(
+            "SELECT attributes->>'inputs' AS question, attributes->>'output' AS ranked "
+            f"FROM records WHERE trace_id = '{trace_id}' AND span_name LIKE 'case:%'"
+        ),
+        min_timestamp=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
+        limit=10000,
+    )["rows"]
+    return {r["question"]: json.loads(r["ranked"]) for r in rows if r.get("ranked")}
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--predictions", required=True, help="QA-run CSV.")
+    parser.add_argument("--retrieval-trace", help="Logfire retrieval trace id.")
+    parser.add_argument("--retrieval-csv", help="Retrieval CSV (question, output).")
+    parser.add_argument("--subset", required=True, help="Subset name, e.g. FinQA.")
+    parser.add_argument("--topk", type=int, default=3, help="Ranked context ids.")
+    parser.add_argument("--out", required=True, help="Output .jsonl path.")
+    args = parser.parse_args()
+
+    if bool(args.retrieval_trace) == bool(args.retrieval_csv):
+        sys.exit("Pass exactly one of --retrieval-trace or --retrieval-csv.")
+
+    predictions = _load_predictions(args.predictions)
+    retrieval = (
+        _retrieval_from_csv(args.retrieval_csv)
+        if args.retrieval_csv
+        else _retrieval_from_trace(args.retrieval_trace)
+    )
+
+    rows = build_submission_rows(
+        predictions, retrieval, subset=args.subset, topk=args.topk
+    )
+    with open(args.out, "w") as f:
+        for row in rows:
+            f.write(json.dumps(row) + "\n")
+
+    answered = sum(1 for r in rows if r["prediction"])
+    matched = sum(1 for r in rows if r["context_id"])
+    print(
+        f"wrote {args.out}: {len(rows)} rows "
+        f"({answered} with a prediction, {matched} with a retrieved context_id)"
+    )
+
+
+if __name__ == "__main__":
+    main()

evaluations/tests/test_submission.py

@@ -0,0 +1,72 @@
+from evaluations.submission import build_submission_rows, extract_prediction
+
+
+class TestExtractPrediction:
+    def test_answer_line_integer(self) -> None:
+        assert extract_prediction("...\n\nANSWER: 18.6") == "18.6"
+
+    def test_percent_becomes_fraction(self) -> None:
+        # T² gold stores percentages as decimals.
+        assert extract_prediction("ANSWER: 93.5%") == "0.935"
+
+    def test_strips_currency_commas_and_scale_word(self) -> None:
+        # "$688 million" -> 688 (scale word not expanded; gold is the bare number)
+        assert extract_prediction("ANSWER: $688 million") == "688"
+        assert extract_prediction("ANSWER: $1,234.5") == "1234.5"
+
+    def test_unicode_minus(self) -> None:
+        assert extract_prediction("ANSWER: −1.9") == "-1.9"
+
+    def test_uses_answer_line_not_reasoning(self) -> None:
+        out = "We saw 72.8 in the table but recomputed.\nANSWER: 82.8"
+        assert extract_prediction(out) == "82.8"
+
+    def test_empty_output_is_blank(self) -> None:
+        assert extract_prediction("") == ""
+        assert extract_prediction(None) == ""
+
+    def test_no_number_is_blank(self) -> None:
+        assert extract_prediction("ANSWER: not reported") == ""
+
+
+class TestBuildSubmissionRows:
+    def _preds(self) -> list[dict[str, str]]:
+        return [
+            {"id": "finqa_dev_0", "question": "Q1?", "output": "ANSWER: 127.4"},
+            {"id": "finqa_dev_1", "question": "Q2?", "output": ""},  # null
+        ]
+
+    def _retrieval(self) -> dict[str, list[str]]:
+        return {
+            "Q1?": ["ctx_a", "ctx_b", "ctx_c", "ctx_d"],
+            "Q2?": ["ctx_e", "ctx_f"],
+        }
+
+    def test_topk_list_and_fields(self) -> None:
+        rows = build_submission_rows(
+            self._preds(), self._retrieval(), subset="FinQA", topk=3
+        )
+        assert rows[0] == {
+            "id": "finqa_dev_0",
+            "subset": "FinQA",
+            "context_id": ["ctx_a", "ctx_b", "ctx_c"],
+            "prediction": "127.4",
+        }
+        # null prediction -> blank string (counted wrong); ranking still attached
+        assert rows[1]["prediction"] == ""
+        assert rows[1]["context_id"] == ["ctx_e", "ctx_f"]
+
+    def test_topk_one_emits_single_string(self) -> None:
+        rows = build_submission_rows(
+            self._preds(), self._retrieval(), subset="FinQA", topk=1
+        )
+        assert rows[0]["context_id"] == "ctx_a"
+
+    def test_missing_retrieval_is_empty(self) -> None:
+        rows = build_submission_rows(
+            [{"id": "x", "question": "unseen?", "output": "ANSWER: 1"}],
+            self._retrieval(),
+            subset="FinQA",
+            topk=3,
+        )
+        assert rows[0]["context_id"] == []