From de9731d5f30cdae76b1b1727f027cdc7bf789885 Mon Sep 17 00:00:00 2001
From: Yiorgis Gozadinos <ggozadinos@gmail.com>
Date: Thu, 4 Jun 2026 15:14:20 +0300
Subject: [PATCH] =?UTF-8?q?Score=20Number-Match=20on=20the=20declared=20AN?=
 =?UTF-8?q?SWER=20line,=20magnitude=20and=20=C3=97100=20scale?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../evaluations/evaluators/number_match.py    | 24 +++++++++++---
 evaluations/tests/test_evaluators.py          | 33 +++++++++++++++++++
 2 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/evaluations/evaluations/evaluators/number_match.py b/evaluations/evaluations/evaluators/number_match.py
index 968a5b36..8ea70304 100644
--- a/evaluations/evaluations/evaluators/number_match.py
+++ b/evaluations/evaluations/evaluators/number_match.py
@@ -1,9 +1,19 @@
+import re
 from dataclasses import dataclass
 
 from pydantic_evals.evaluators import Evaluator, EvaluatorContext
 
 from evaluations.numbers import extract_numbers, numbers_close
 
+_ANSWER_RE = re.compile(r"(?im)^[\s*>#_-]*(?:final\s+)?answer\s*[:=]\s*(.+)$")
+
+
+def _answer_segment(text: str) -> str:
+    """Restrict to a declared ``ANSWER:`` line when present, so numbers in the
+    surrounding reasoning don't count. Falls back to the whole text."""
+    matches = _ANSWER_RE.findall(text)
+    return matches[-1] if matches else text
+
 
 @dataclass
 class NumberMatchEvaluator(Evaluator):
@@ -23,8 +33,14 @@ class NumberMatchEvaluator(Evaluator):
         gold = extract_numbers(str(ctx.expected_output))
         if not gold:
             return 0.0
-        target = gold[0]
-        candidates = extract_numbers(str(ctx.output))
-        return (
-            1.0 if any(numbers_close(c, target, self.eps) for c in candidates) else 0.0
+        # Gold mixes conventions: signs for changes are inconsistent (+0.2 vs
+        # -1.9) and ratios appear as either a percent (37.81) or a decimal
+        # (0.3781). Compare the declared answer by magnitude, at a ×100 scale
+        # either way. Safe because we score only the single ANSWER-line number.
+        target = abs(gold[0])
+        candidates = [abs(c) for c in extract_numbers(_answer_segment(str(ctx.output)))]
+        scales = (1.0, 0.01, 100.0)
+        matched = any(
+            numbers_close(c * s, target, self.eps) for c in candidates for s in scales
         )
+        return 1.0 if matched else 0.0
diff --git a/evaluations/tests/test_evaluators.py b/evaluations/tests/test_evaluators.py
index 97dbbb19..b70a74b3 100644
--- a/evaluations/tests/test_evaluators.py
+++ b/evaluations/tests/test_evaluators.py
@@ -101,3 +101,36 @@ class TestNumberMatchEvaluator:
     def test_negative_match(self) -> None:
         ctx = self._make_ctx("-12.3", "the change was (12.3)")
         assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_sign_insensitive_against_inconsistent_gold(self) -> None:
+        # gold stores this decrease as +0.2; model declares the signed -0.2
+        ctx = self._make_ctx("0.1999999999999993", "declined 0.2 pp\nANSWER: -0.2")
+        assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_bare_percent_matches_decimal_gold(self) -> None:
+        # model declares the percentage without a % sign; gold is the decimal
+        ctx = self._make_ctx("0.3781", "growth was 37.81%\nANSWER: 37.81")
+        assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_scale_mismatch_does_not_flip_genuine_error(self) -> None:
+        ctx = self._make_ctx("30.443", "ANSWER: 2330.8%")
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_answer_line_ignores_reasoning_distractors(self) -> None:
+        # gold matches a distractor in the body, but the declared answer is wrong
+        ctx = self._make_ctx(
+            "0.728",
+            "Finished goods were 72.8% of inventory.\nANSWER: 82.8%",
+        )
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_answer_line_used_when_correct(self) -> None:
+        ctx = self._make_ctx(
+            "0.935",
+            "The graph shows growth to 193.5.\n\nANSWER: 93.5%",
+        )
+        assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_falls_back_to_full_text_without_answer_line(self) -> None:
+        ctx = self._make_ctx("127.4", "The average works out to $127.40 each.")
+        assert self.evaluator.evaluate(ctx) == 1.0