danvics/haiku.rag
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add stable ids to FRAMES question rows

Browse source
This commit is contained in:
Yiorgis Gozadinos 2026-07-26 12:29:35 +03:00
parent aa49ec6cb3
commit a57a73b6d0
No known key found for this signature in database
2 changed files with 28 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
evaluations/evaluations/datasets/frames.py
View file

@ -52,7 +52,9 @@ def question_is_answerable(doc: Mapping[str, Any]) -> bool:
def load_frames_questions() -> Dataset: def load_frames_questions() -> Dataset:
return load_frames_test().filter(question_is_answerable) """Answerable questions with a stable `id` (the dataset row number)."""
dataset = load_frames_test().filter(question_is_answerable)
return dataset.map(lambda row: {"id": str(row["Unnamed: 0"])})
def parse_wiki_links(raw: str) -> list[str]: def parse_wiki_links(raw: str) -> list[str]:
@ -304,10 +306,11 @@ def build_frames_case(
index: int, doc: Mapping[str, Any] index: int, doc: Mapping[str, Any]
) -> Case[str, str, dict[str, str]]: ) -> Case[str, str, dict[str, str]]:
return Case( return Case(
name=f"{index}", name=f"{index}_{doc['id']}",
inputs=doc["Prompt"], inputs=doc["Prompt"],
expected_output=doc["Answer"], expected_output=doc["Answer"],
metadata={ metadata={
"question_id": str(doc["id"]),
"reasoning_types": str(doc["reasoning_types"]), "reasoning_types": str(doc["reasoning_types"]),
"case_index": str(index), "case_index": str(index),
}, },

24
evaluations/tests/test_datasets.py
View file

@ -500,15 +500,17 @@ class TestFrames:
def test_build_case(self) -> None: def test_build_case(self) -> None:
row = { row = {
"id": "7",
"Prompt": "Who was the 15th president?", "Prompt": "Who was the 15th president?",
"Answer": "James Buchanan", "Answer": "James Buchanan",
"reasoning_types": "Multiple constraints | Temporal reasoning", "reasoning_types": "Multiple constraints | Temporal reasoning",
} }
case = build_frames_case(3, row) case = build_frames_case(3, row)
assert case.name == "3" assert case.name == "3_7"
assert case.inputs == "Who was the 15th president?" assert case.inputs == "Who was the 15th president?"
assert case.expected_output == "James Buchanan" assert case.expected_output == "James Buchanan"
assert case.metadata == { assert case.metadata == {
"question_id": "7",
"reasoning_types": "Multiple constraints | Temporal reasoning", "reasoning_types": "Multiple constraints | Temporal reasoning",
"case_index": "3", "case_index": "3",
} }
@ -698,3 +700,23 @@ class TestFrames:
kept = {"wiki_links": "['https://en.wikipedia.org/wiki/Capybara']"} kept = {"wiki_links": "['https://en.wikipedia.org/wiki/Capybara']"}
assert question_is_answerable(gone) is False assert question_is_answerable(gone) is False
assert question_is_answerable(kept) is True assert question_is_answerable(kept) is True
def test_questions_carry_stable_ids(self, monkeypatch) -> None:
import evaluations.datasets.frames as frames
from datasets import Dataset
rows = Dataset.from_list(
[
{
"Unnamed: 0": 7,
"wiki_links": "['https://en.wikipedia.org/wiki/Capybara']",
},
{
"Unnamed: 0": 8,
"wiki_links": "['https://en.wikipedia.org/wiki/Jack_Vance_(tennis)']",
},
]
)
monkeypatch.setattr(frames, "load_frames_test", lambda: rows)
questions = frames.load_frames_questions()
assert [row["id"] for row in questions] == ["7"]