danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Match the numeric scale convention in Number-Match

Browse source
This commit is contained in:
Yiorgis Gozadinos 2026-06-05 11:27:35 +03:00
parent d5b0aafa2b
commit 1d7d540270
No known key found for this signature in database
2 changed files with 17 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
evaluations/evaluations/evaluators/number_match.py
View file

@ -33,13 +33,14 @@ class NumberMatchEvaluator(Evaluator):
gold = extract_numbers(str(ctx.expected_output)) gold = extract_numbers(str(ctx.expected_output))
if not gold: if not gold:
return 0.0 return 0.0
# Gold mixes conventions: signs for changes are inconsistent (+0.2 vs # Gold mixes conventions: change signs are inconsistent (+0.2 vs -1.9),
# -1.9) and ratios appear as either a percent (37.81) or a decimal # ratios appear as a percent or a decimal (37.81 vs 0.3781), and figures
# (0.3781). Compare the declared answer by magnitude, at a ×100 scale # appear in units or thousands (4575515 vs 4575515000). Compare the
# either way. Safe because we score only the single ANSWER-line number. # declared answer by magnitude at ×100 and ×1000 scales either way. Safe
# because we score only the single ANSWER-line number.
target = abs(gold[0]) target = abs(gold[0])
candidates = [abs(c) for c in extract_numbers(_answer_segment(str(ctx.output)))] candidates = [abs(c) for c in extract_numbers(_answer_segment(str(ctx.output)))]
scales = (1.0, 0.01, 100.0) scales = (1.0, 0.01, 100.0, 0.001, 1000.0)
matched = any( matched = any(
numbers_close(c * s, target, self.eps) for c in candidates for s in scales numbers_close(c * s, target, self.eps) for c in candidates for s in scales
) )

11
evaluations/tests/test_evaluators.py
View file

@ -116,6 +116,17 @@ class TestNumberMatchEvaluator:
ctx = self._make_ctx("30.443", "ANSWER: 2330.8%") ctx = self._make_ctx("30.443", "ANSWER: 2330.8%")
assert self.evaluator.evaluate(ctx) == 0.0 assert self.evaluator.evaluate(ctx) == 0.0
def test_thousands_convention(self) -> None:
# gold is in thousands; model gives the full-dollar figure
ctx = self._make_ctx("4575515.0", "...\nANSWER: 4575515000")
assert self.evaluator.evaluate(ctx) == 1.0
def test_thousands_convention_reversed(self) -> None:
ctx = self._make_ctx(
"46.30434782608695", "fair value per share\nANSWER: 46304.35"
)
assert self.evaluator.evaluate(ctx) == 1.0
def test_answer_line_ignores_reasoning_distractors(self) -> None: def test_answer_line_ignores_reasoning_distractors(self) -> None:
# gold matches a distractor in the body, but the declared answer is wrong # gold matches a distractor in the body, but the declared answer is wrong
ctx = self._make_ctx( ctx = self._make_ctx(