diff --git a/evaluations/evaluations/evaluators/number_match.py b/evaluations/evaluations/evaluators/number_match.py index 8ea70304..eb657e24 100644 --- a/evaluations/evaluations/evaluators/number_match.py +++ b/evaluations/evaluations/evaluators/number_match.py @@ -33,13 +33,14 @@ class NumberMatchEvaluator(Evaluator): gold = extract_numbers(str(ctx.expected_output)) if not gold: return 0.0 - # Gold mixes conventions: signs for changes are inconsistent (+0.2 vs - # -1.9) and ratios appear as either a percent (37.81) or a decimal - # (0.3781). Compare the declared answer by magnitude, at a ×100 scale - # either way. Safe because we score only the single ANSWER-line number. + # Gold mixes conventions: change signs are inconsistent (+0.2 vs -1.9), + # ratios appear as a percent or a decimal (37.81 vs 0.3781), and figures + # appear in units or thousands (4575515 vs 4575515000). Compare the + # declared answer by magnitude at ×100 and ×1000 scales either way. Safe + # because we score only the single ANSWER-line number. target = abs(gold[0]) candidates = [abs(c) for c in extract_numbers(_answer_segment(str(ctx.output)))] - scales = (1.0, 0.01, 100.0) + scales = (1.0, 0.01, 100.0, 0.001, 1000.0) matched = any( numbers_close(c * s, target, self.eps) for c in candidates for s in scales ) diff --git a/evaluations/tests/test_evaluators.py b/evaluations/tests/test_evaluators.py index b70a74b3..c0adecd3 100644 --- a/evaluations/tests/test_evaluators.py +++ b/evaluations/tests/test_evaluators.py @@ -116,6 +116,17 @@ class TestNumberMatchEvaluator: ctx = self._make_ctx("30.443", "ANSWER: 2330.8%") assert self.evaluator.evaluate(ctx) == 0.0 + def test_thousands_convention(self) -> None: + # gold is in thousands; model gives the full-dollar figure + ctx = self._make_ctx("4575515.0", "...\nANSWER: 4575515000") + assert self.evaluator.evaluate(ctx) == 1.0 + + def test_thousands_convention_reversed(self) -> None: + ctx = self._make_ctx( + "46.30434782608695", "fair value per share\nANSWER: 46304.35" + ) + assert self.evaluator.evaluate(ctx) == 1.0 + def test_answer_line_ignores_reasoning_distractors(self) -> None: # gold matches a distractor in the body, but the declared answer is wrong ctx = self._make_ctx(