Score Number-Match on the declared ANSWER line, magnitude and ×100 scale

evaluations/evaluations/evaluators/number_match.py

@@ -1,9 +1,19 @@
+import re
 from dataclasses import dataclass
 
 from pydantic_evals.evaluators import Evaluator, EvaluatorContext
 
 from evaluations.numbers import extract_numbers, numbers_close
 
+_ANSWER_RE = re.compile(r"(?im)^[\s*>#_-]*(?:final\s+)?answer\s*[:=]\s*(.+)$")
+
+
+def _answer_segment(text: str) -> str:
+    """Restrict to a declared ``ANSWER:`` line when present, so numbers in the
+    surrounding reasoning don't count. Falls back to the whole text."""
+    matches = _ANSWER_RE.findall(text)
+    return matches[-1] if matches else text
+
 
 @dataclass
 class NumberMatchEvaluator(Evaluator):
@@ -23,8 +33,14 @@ class NumberMatchEvaluator(Evaluator):
         gold = extract_numbers(str(ctx.expected_output))
         if not gold:
             return 0.0
-        target = gold[0]
-        candidates = extract_numbers(str(ctx.output))
-        return (
-            1.0 if any(numbers_close(c, target, self.eps) for c in candidates) else 0.0
+        # Gold mixes conventions: signs for changes are inconsistent (+0.2 vs
+        # -1.9) and ratios appear as either a percent (37.81) or a decimal
+        # (0.3781). Compare the declared answer by magnitude, at a ×100 scale
+        # either way. Safe because we score only the single ANSWER-line number.
+        target = abs(gold[0])
+        candidates = [abs(c) for c in extract_numbers(_answer_segment(str(ctx.output)))]
+        scales = (1.0, 0.01, 100.0)
+        matched = any(
+            numbers_close(c * s, target, self.eps) for c in candidates for s in scales
         )
+        return 1.0 if matched else 0.0

evaluations/tests/test_evaluators.py

@@ -101,3 +101,36 @@ class TestNumberMatchEvaluator:
     def test_negative_match(self) -> None:
         ctx = self._make_ctx("-12.3", "the change was (12.3)")
         assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_sign_insensitive_against_inconsistent_gold(self) -> None:
+        # gold stores this decrease as +0.2; model declares the signed -0.2
+        ctx = self._make_ctx("0.1999999999999993", "declined 0.2 pp\nANSWER: -0.2")
+        assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_bare_percent_matches_decimal_gold(self) -> None:
+        # model declares the percentage without a % sign; gold is the decimal
+        ctx = self._make_ctx("0.3781", "growth was 37.81%\nANSWER: 37.81")
+        assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_scale_mismatch_does_not_flip_genuine_error(self) -> None:
+        ctx = self._make_ctx("30.443", "ANSWER: 2330.8%")
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_answer_line_ignores_reasoning_distractors(self) -> None:
+        # gold matches a distractor in the body, but the declared answer is wrong
+        ctx = self._make_ctx(
+            "0.728",
+            "Finished goods were 72.8% of inventory.\nANSWER: 82.8%",
+        )
+        assert self.evaluator.evaluate(ctx) == 0.0
+
+    def test_answer_line_used_when_correct(self) -> None:
+        ctx = self._make_ctx(
+            "0.935",
+            "The graph shows growth to 193.5.\n\nANSWER: 93.5%",
+        )
+        assert self.evaluator.evaluate(ctx) == 1.0
+
+    def test_falls_back_to_full_text_without_answer_line(self) -> None:
+        ctx = self._make_ctx("127.4", "The average works out to $127.40 each.")
+        assert self.evaluator.evaluate(ctx) == 1.0