danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Normalize unicode signs

Browse source
This commit is contained in:
Yiorgis Gozadinos 2026-06-04 14:46:28 +03:00
parent 80594cd38c
commit 390deb4203
No known key found for this signature in database
2 changed files with 4 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
evaluations/evaluations/numbers.py
View file

@ -42,6 +42,7 @@ def extract_numbers(text: str) -> list[float]:
parenthesised negatives. Numbers qualified by a scale word ("1.2 million") parenthesised negatives. Numbers qualified by a scale word ("1.2 million")
contribute both the raw and the scaled value, so either phrasing can match. contribute both the raw and the scaled value, so either phrasing can match.
""" """
text = text.replace("−", "-") # normalize the typographic minus sign
numbers: list[float] = [] numbers: list[float] = []
for token in _NUMBER_RE.findall(text): for token in _NUMBER_RE.findall(text):
value = _to_float(token) value = _to_float(token)

3
evaluations/tests/test_numbers.py
View file

@ -14,6 +14,9 @@ class TestExtractNumbers:
def test_parenthesised_negative(self) -> None: def test_parenthesised_negative(self) -> None:
assert extract_numbers("loss of (123)") == [-123.0] assert extract_numbers("loss of (123)") == [-123.0]
def test_unicode_minus(self) -> None:
assert extract_numbers("a change of −1.9 million") == [-1.9, -1.9e6]
def test_scale_word_adds_scaled_and_raw(self) -> None: def test_scale_word_adds_scaled_and_raw(self) -> None:
numbers = extract_numbers("revenue of 1.2 billion") numbers = extract_numbers("revenue of 1.2 billion")
assert 1.2 in numbers assert 1.2 in numbers