Add deterministic Number-Match QA scoring for T²-RAGBench

CHANGELOG.md

@@ -3,7 +3,7 @@
 
 ### Added
 
-- `t2_finqa` evaluation dataset (T²-RAGBench FinQA subset, `G4KMU/t2-ragbench`): 2,789 single-page PDFs / 8,281 numeric QA, ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`.
+- `t2_finqa` evaluation dataset (T²-RAGBench FinQA subset, `G4KMU/t2-ragbench`): 2,789 single-page PDFs / 8,281 numeric QA, ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`. QA is scored with a deterministic `NumberMatchEvaluator` (relative tolerance 0.01) via the new `DatasetSpec.qa_evaluator`, bypassing the LLM judge.
 
 ### Fixed
 

evaluations/evaluations/benchmark.py

@@ -365,18 +365,23 @@ async def run_qa_benchmark(
     _attach_relevant_uris(cases, spec, limit)
     citation_evaluator = _citation_evaluator_for(spec.retrieval_evaluator)
 
-    evaluators: list[Evaluator] = [
-        LLMJudge(
-            rubric=ANSWER_EQUIVALENCE_RUBRIC,
-            include_input=True,
-            include_expected_output=True,
-            model=get_model(judge_config, config),
-            assertion={
-                "evaluation_name": "answer_equivalent",
-                "include_reason": True,
-            },
-        ),
-    ]
+    qa_evaluator = spec.qa_evaluator
+    evaluators: list[Evaluator]
+    if qa_evaluator is not None:
+        evaluators = [qa_evaluator]
+    else:
+        evaluators = [
+            LLMJudge(
+                rubric=ANSWER_EQUIVALENCE_RUBRIC,
+                include_input=True,
+                include_expected_output=True,
+                model=get_model(judge_config, config),
+                assertion={
+                    "evaluation_name": "answer_equivalent",
+                    "include_reason": True,
+                },
+            ),
+        ]
     if citation_evaluator is not None:
         evaluators.append(citation_evaluator)
 
@@ -419,17 +424,28 @@ async def run_qa_benchmark(
 
     report = await _evaluate(answer_question)
 
-    passing_cases = sum(
-        1
-        for case in report.cases
-        if case.assertions.get("answer_equivalent")
-        and case.assertions["answer_equivalent"].value
-    )
     total_processed = len(report.cases)
     failures = report.failures
+    if qa_evaluator is not None:
+        score_key = qa_evaluator.get_default_evaluation_name()
+        passing_cases = sum(
+            1
+            for case in report.cases
+            if score_key in case.scores and case.scores[score_key].value >= 1.0
+        )
+        scoring = score_key
+    else:
+        passing_cases = sum(
+            1
+            for case in report.cases
+            if case.assertions.get("answer_equivalent")
+            and case.assertions["answer_equivalent"].value
+        )
+        scoring = "answer_equivalent"
     accuracy = passing_cases / total_processed if total_processed > 0 else 0
 
     console.print("\n=== QA Benchmark Results ===", style="bold cyan")
+    console.print(f"Scoring: {scoring}")
     console.print(f"Total questions: {total_processed}")
     console.print(f"Correct answers: {passing_cases}")
     console.print(f"QA Accuracy: {accuracy:.4f} ({accuracy * 100:.2f}%)")

evaluations/evaluations/config.py

@@ -44,6 +44,7 @@ class DatasetSpec:
     retrieval_loader: RetrievalLoader | None = None
     retrieval_mapper: RetrievalMapper | None = None
     retrieval_evaluator: Evaluator | None = None
+    qa_evaluator: Evaluator | None = None
     document_limit: int | None = None
 
     def db_path(self, override_path: Path | None = None) -> Path:

evaluations/evaluations/datasets/t2_ragbench.py

@@ -10,7 +10,7 @@ from huggingface_hub import hf_hub_download
 from pydantic_evals import Case
 
 from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample
-from evaluations.evaluators import MAPEvaluator
+from evaluations.evaluators import MAPEvaluator, NumberMatchEvaluator
 
 REPO_ID = "G4KMU/t2-ragbench"
 SPLITS = ("dev", "test", "train")
@@ -134,6 +134,7 @@ def _t2_spec(subset: str, key: str, db_filename: str) -> DatasetSpec:
         retrieval_loader=partial(load_t2_qa, subset),
         retrieval_mapper=map_t2_retrieval,
         retrieval_evaluator=MAPEvaluator(),
+        qa_evaluator=NumberMatchEvaluator(),
     )
 
 

evaluations/evaluations/evaluators/__init__.py

@@ -5,6 +5,7 @@ from evaluations.evaluators.judge import (
     LLMJudgeResponseSchema,
 )
 from evaluations.evaluators.map import MAPEvaluator
+from evaluations.evaluators.number_match import NumberMatchEvaluator
 
 __all__ = [
     "ANSWER_EQUIVALENCE_RUBRIC",
@@ -12,4 +13,5 @@ __all__ = [
     "LLMJudge",
     "LLMJudgeResponseSchema",
     "MAPEvaluator",
+    "NumberMatchEvaluator",
 ]

evaluations/evaluations/evaluators/number_match.py

@@ -0,0 +1,30 @@
+from dataclasses import dataclass
+
+from pydantic_evals.evaluators import Evaluator, EvaluatorContext
+
+from evaluations.numbers import extract_numbers, numbers_close
+
+
+@dataclass
+class NumberMatchEvaluator(Evaluator):
+    """Deterministic numeric scoring for datasets with numeric gold answers.
+
+    Scores 1.0 when any number parsed from the prediction matches the gold
+    answer within relative tolerance ``eps``, else 0.0. Non-numeric predictions
+    score 0.0.
+    """
+
+    eps: float = 0.01
+
+    def get_default_evaluation_name(self) -> str:
+        return "number_match"
+
+    def evaluate(self, ctx: EvaluatorContext) -> float:
+        gold = extract_numbers(str(ctx.expected_output))
+        if not gold:
+            return 0.0
+        target = gold[0]
+        candidates = extract_numbers(str(ctx.output))
+        return (
+            1.0 if any(numbers_close(c, target, self.eps) for c in candidates) else 0.0
+        )

evaluations/evaluations/numbers.py

@@ -0,0 +1,61 @@
+import re
+
+_SCALE = {
+    "thousand": 1e3,
+    "million": 1e6,
+    "billion": 1e9,
+    "trillion": 1e12,
+}
+
+_NUMBER_RE = re.compile(r"[(-]?\$?\d[\d,]*(?:\.\d+)?%?\)?")
+_SCALED_RE = re.compile(
+    r"([(-]?\$?\d[\d,]*(?:\.\d+)?)\s*(thousand|million|billion|trillion)",
+    re.IGNORECASE,
+)
+
+
+def _to_float(token: str) -> float | None:
+    stripped = token.lstrip()
+    negative = stripped.startswith("(") or stripped.startswith("-")
+    cleaned = (
+        token.replace("$", "")
+        .replace(",", "")
+        .replace("%", "")
+        .replace("(", "")
+        .replace(")", "")
+        .strip()
+        .lstrip("-")
+    )
+    if not cleaned or cleaned == ".":
+        return None
+    try:
+        value = float(cleaned)
+    except ValueError:
+        return None
+    return -value if negative else value
+
+
+def extract_numbers(text: str) -> list[float]:
+    """Pull numeric values out of free-form text.
+
+    Handles currency, thousands separators, trailing percent signs, and
+    parenthesised negatives. Numbers qualified by a scale word ("1.2 million")
+    contribute both the raw and the scaled value, so either phrasing can match.
+    """
+    numbers: list[float] = []
+    for token in _NUMBER_RE.findall(text):
+        value = _to_float(token)
+        if value is not None:
+            numbers.append(value)
+    for number, scale in _SCALED_RE.findall(text):
+        value = _to_float(number)
+        if value is not None:
+            numbers.append(value * _SCALE[scale.lower()])
+    return numbers
+
+
+def numbers_close(value: float, target: float, eps: float = 0.01) -> bool:
+    """Whether ``value`` matches ``target`` within relative tolerance ``eps``."""
+    if target == 0:
+        return value == 0
+    return abs(value - target) / abs(target) <= eps

evaluations/tests/test_evaluators.py

@@ -3,6 +3,7 @@ from unittest.mock import MagicMock
 import pytest
 
 from evaluations.evaluators.map import MAPEvaluator
+from evaluations.evaluators.number_match import NumberMatchEvaluator
 
 
 class TestMAPEvaluator:
@@ -54,3 +55,41 @@ class TestMAPEvaluator:
     def test_empty_relevant_uris(self) -> None:
         ctx = self._make_ctx([], ["doc1", "doc2"])
         assert self.evaluator.evaluate(ctx) == 0.0
+
+
+class TestNumberMatchEvaluator:
+    def setup_method(self) -> None:
+        self.evaluator = NumberMatchEvaluator()
+
+    def _make_ctx(self, expected: str, output: str) -> MagicMock:
+        ctx = MagicMock()
+        ctx.expected_output = expected
+        ctx.output = output
+        return ctx
+
+    def test_exact(self) -> None:
+        assert self.evaluator.evaluate(self._make_ctx("127.4", "127.4")) == 1.0
+
+    def test_within_tolerance(self) -> None:
+        ctx = self._make_ctx("127.4", "about $127.40 per transaction")
+        assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_outside_tolerance(self) -> None:
+        ctx = self._make_ctx("127.4", "the answer is 150")
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_picks_matching_candidate_among_many(self) -> None:
+        ctx = self._make_ctx("50.3", "In 2008 it grew from 27.0 to 50.3 percent")
+        assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_non_numeric_prediction(self) -> None:
+        ctx = self._make_ctx("127.4", "I cannot determine the value")
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_non_numeric_gold(self) -> None:
+        ctx = self._make_ctx("not a number", "127.4")
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_negative_match(self) -> None:
+        ctx = self._make_ctx("-12.3", "the change was (12.3)")
+        assert self.evaluator.evaluate(ctx) == 1.0

evaluations/tests/test_numbers.py

@@ -0,0 +1,38 @@
+from evaluations.numbers import extract_numbers, numbers_close
+
+
+class TestExtractNumbers:
+    def test_plain(self) -> None:
+        assert extract_numbers("the answer is 127.4 dollars") == [127.4]
+
+    def test_currency_and_thousands(self) -> None:
+        assert extract_numbers("$1,234.5") == [1234.5]
+
+    def test_percent_stripped(self) -> None:
+        assert extract_numbers("margin was 50.3%") == [50.3]
+
+    def test_parenthesised_negative(self) -> None:
+        assert extract_numbers("loss of (123)") == [-123.0]
+
+    def test_scale_word_adds_scaled_and_raw(self) -> None:
+        numbers = extract_numbers("revenue of 1.2 billion")
+        assert 1.2 in numbers
+        assert 1.2e9 in numbers
+
+    def test_no_numbers(self) -> None:
+        assert extract_numbers("no figures here") == []
+
+
+class TestNumbersClose:
+    def test_within_tolerance(self) -> None:
+        assert numbers_close(127.4, 127.40, 0.01)
+
+    def test_just_inside(self) -> None:
+        assert numbers_close(100.9, 100.0, 0.01)
+
+    def test_outside_tolerance(self) -> None:
+        assert not numbers_close(102.0, 100.0, 0.01)
+
+    def test_zero_target_exact(self) -> None:
+        assert numbers_close(0.0, 0.0, 0.01)
+        assert not numbers_close(0.1, 0.0, 0.01)