From 152d57d6e28964a200ebfd418292dcea85dd1442 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 23 Jul 2026 19:18:16 +0300 Subject: [PATCH] Add frames evaluation dataset --- CHANGELOG.md | 1 + evaluations/README.md | 1 + evaluations/configs/frames.yaml | 39 +++ evaluations/evaluations/datasets/__init__.py | 2 + evaluations/evaluations/datasets/frames.py | 287 +++++++++++++++++++ evaluations/pyproject.toml | 1 + evaluations/tests/test_datasets.py | 244 ++++++++++++++++ uv.lock | 2 + 8 files changed, 577 insertions(+) create mode 100644 evaluations/configs/frames.yaml create mode 100644 evaluations/evaluations/datasets/frames.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c968085..84a8f398 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ ### Added +- `frames` evaluation dataset. - `evaluations run --filter/-f CLAUSE`: SQL `WHERE` clause over document columns, applied to the retrieval benchmark's searches and to every capability search during QA. Recorded as `document_filter` in experiment metadata. - `mtrag_clapnq` / `mtrag_clapnq_rewrite` / `mtrag_clapnq_live` / `mtrag_clapnq_live_uncompacted` evaluation datasets: multi-turn QA with gold-prefix and live-session conversation replay, live arms with and without `EvidenceCompactionCapability`, Recall@k/nDCG@k retrieval metrics, eligibility-aware citation scoring, refusal precision/recall, per-turn tool-traffic attributes, and `citation_status` / `turn_citation_status` eval attributes. - Raw chunk metadata is now exposed to search and citation results, through `SearchResult.chunk_meta` and `Citation.chunk_meta`. For context-expanded results, the metadata is that of the anchor chunk. diff --git a/evaluations/README.md b/evaluations/README.md index 341fb3a6..7931826c 100644 --- a/evaluations/README.md +++ b/evaluations/README.md @@ -10,6 +10,7 @@ Contains evaluation scripts for benchmarking RAG retrieval and QA performance. A - HotpotQA (`hotpotqa`) — multi-hop QA over Wikipedia paragraphs (distractor validation split, 7,405 questions, two gold documents per question) - MTRAG ClapNQ (`mtrag_clapnq`, `mtrag_clapnq_rewrite`) — IBM's multi-turn RAG benchmark, ClapNQ (Wikipedia) domain: 183,408 passages, 208 retrieval queries with binary qrels, 224 generation tasks. The base key retrieves with the raw last user turn; the `_rewrite` variant uses the human standalone rewrites (both share one database). Retrieval reports Recall@5/@10, nDCG@5/@10, and MAP against IBM's published setup. QA replays each task's reference conversation prefix as message history and answers the final turn; the judge sees the conversation as a transcript, citation MAP is scored only on turns with gold passages, and refusal precision/recall is reported against the answerability labels. Generation scores are internal (our judge and rubric), not comparable with IBM's published generation numbers. The `mtrag_clapnq_live` key replays whole conversations (one case per conversation, `--limit` counts conversations) through a single capability session, carrying the model's own answers and tool history across turns; it reports the same outcomes per turn plus micro (per-turn) and macro (per-conversation) aggregates. +- FRAMES (`frames`) — multi-hop QA (824 questions, 2-23 gold Wikipedia articles per question). The corpus is the union of the ~2.5k linked articles, fetched from the Wikipedia REST API at current revision (revision id and fetch date recorded in the article cache) with navigation chrome stripped. There is no official FRAMES evaluation setup; numbers here correspond to the paper's multi-step retrieval setting (fixed corpus, agentic retrieval, judged accuracy) and are not comparable to its closed-book, oracle-prompt, or web-search settings. Answers were authored against ~2024 revisions and may have drifted with article content. - OpenRAG Bench, two variants: - `orb_text` — text embedder (`qwen3-embedding:4b`, 2560-dim) with VLM picture descriptions baked into chunk content at ingest. Use for text-only retrieval/QA against figure-rich corpora. - `orb_multimodal` — multimodal embedder (`qwen3-vl-embedding-8b`, 4096-dim) with picture vectors in the same space as text. Use for cross-modal retrieval (text-as-query → figure hits, image-as-query) and vision QA where the figure itself is the answer. diff --git a/evaluations/configs/frames.yaml b/evaluations/configs/frames.yaml new file mode 100644 index 00000000..7c20fad4 --- /dev/null +++ b/evaluations/configs/frames.yaml @@ -0,0 +1,39 @@ +# Reference config for the `frames` evaluation database. +# FRAMES (google/frames-benchmark): 824 multi-hop questions over a corpus of +# the ~2.5k Wikipedia articles linked per question, fetched at current +# revision (revid + fetch date recorded in the article cache). +# Run: evaluations run frames --config configs/frames.yaml +# base_url uses the `vllm` host serving each model over an OpenAI-compatible API. + +environment: development + +storage: + auto_vacuum: false + +embeddings: + model: + provider: openai + name: qwen3-embedding-4b + vector_dim: 2560 + base_url: http://vllm:11431/v1 + +reranking: + model: + provider: vllm + name: Qwen/Qwen3-Reranker-4B + base_url: http://vllm:11433 + +qa: + model: + provider: openai + name: gemma4-26b + base_url: http://vllm:11432/v1 + max_tokens: 49152 + +evaluations: + judge: + provider: openai + name: RedHatAI/Qwen3.6-35B-A3B-NVFP4 + base_url: http://vllm:11430/v1 + temperature: 0.0 + max_tokens: 4096 diff --git a/evaluations/evaluations/datasets/__init__.py b/evaluations/evaluations/datasets/__init__.py index 45c29adb..4a968b5f 100644 --- a/evaluations/evaluations/datasets/__init__.py +++ b/evaluations/evaluations/datasets/__init__.py @@ -1,5 +1,6 @@ from evaluations.config import DatasetSpec +from .frames import FRAMES_SPEC from .hotpotqa import HOTPOTQA_SPEC from .mtrag import ( MTRAG_CLAPNQ_LIVE_SPEC, @@ -17,6 +18,7 @@ from .t2_ragbench import T2_FINQA_SPEC, T2_TATDQA_SPEC DATASETS: dict[str, DatasetSpec] = { spec.key: spec for spec in ( + FRAMES_SPEC, HOTPOTQA_SPEC, MTRAG_CLAPNQ_SPEC, MTRAG_CLAPNQ_REWRITE_SPEC, diff --git a/evaluations/evaluations/datasets/frames.py b/evaluations/evaluations/datasets/frames.py new file mode 100644 index 00000000..f1860bf7 --- /dev/null +++ b/evaluations/evaluations/datasets/frames.py @@ -0,0 +1,287 @@ +"""FRAMES benchmark (google/frames-benchmark). + +824 multi-hop questions, each grounded in two or more Wikipedia articles. The +corpus is the union of the articles linked per question, fetched from the +Wikipedia REST API at current revision and cached locally with the revision id +and fetch date. +""" + +import ast +import json +import logging +import re +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, quote, unquote, urlsplit + +import httpx +from bs4 import BeautifulSoup +from datasets import Dataset, load_dataset +from pydantic_evals import Case + +from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample +from evaluations.evaluators import CitationMAPEvaluator, MAPEvaluator + +logger = logging.getLogger(__name__) + +USER_AGENT = "haiku.rag-evaluations (https://github.com/ggozad/haiku.rag)" + + +def load_frames_test() -> Dataset: + return load_dataset("google/frames-benchmark")["test"] + + +def parse_wiki_links(raw: str) -> list[str]: + """Extract URLs from a `wiki_links` value. + + The value is a Python-list-repr string. A single list element may pack + several comma-separated URLs, and may carry trailing prose annotations; + titles themselves can contain commas, so elements are split only where a + new URL starts. + """ + links: list[str] = [] + for element in ast.literal_eval(raw): + for part in re.split(r",\s*(?=http)", element): + tokens = part.split() + if not tokens: + continue + url = tokens[0].strip(", ") + if url: + links.append(url) + return links + + +def normalize_wiki_url(url: str) -> str | None: + """Canonical article URL, used both as document uri and expected uri. + + Strips fragments, decodes percent-escapes, folds mobile hosts, resolves + `index.php?title=` and `Special:Search` forms, and applies MediaWiki title + canonicalization (underscores, first letter uppercased). Returns None for + strings that don't point to an article. + """ + url = url.strip() + if not url: + return None + if "://" not in url: + url = "https://" + url + parts = urlsplit(url) + host = parts.netloc.replace(".m.wikipedia.org", ".wikipedia.org") + if host == "w.wiki": + return url + if parts.path.startswith("/wiki/"): + title = parts.path[len("/wiki/") :] + elif parts.path.startswith("/w/index.php"): + query = parse_qs(parts.query) + title = query.get("title", [""])[0] + if not title or title.startswith("Special:"): + title = query.get("search", [""])[0] + else: + return None + title = unquote(title).replace(" ", "_").strip("_") + if not title: + return None + return f"https://{host}/wiki/{title[0].upper() + title[1:]}" + + +def parse_revid(etag: str | None) -> str | None: + """Revision id from a Wikipedia REST ETag header (`W/"/"`).""" + if not etag: + return None + match = re.search(r'"([^/"]+)/', etag) + return match.group(1) if match else None + + +def strip_navigation(html: str) -> str: + """Drop navigation chrome (navboxes, succession boxes) from parsoid HTML. + + These render as link-spam tables naming hundreds of related articles, + polluting retrieval. Infoboxes carry no navigation role and are kept. + """ + soup = BeautifulSoup(html, "html.parser") + for element in soup.find_all(attrs={"role": "navigation"}): + element.decompose() + return str(soup) + + +def get_cache_dir() -> Path: + cache_dir = Path.home() / ".cache" / "haiku.rag" / "evaluations" / "frames_articles" + cache_dir.mkdir(parents=True, exist_ok=True) + return cache_dir + + +def _fetch_category_page( + host: str, title: str, client: httpx.Client +) -> tuple[str, str, str | None]: + """Category pages render empty via page/html; synthesize a members list.""" + response = client.get( + f"https://{host}/w/api.php", + params={ + "action": "query", + "list": "categorymembers", + "cmtitle": title, + "cmlimit": "500", + "format": "json", + }, + ) + response.raise_for_status() + members = [m["title"] for m in response.json()["query"]["categorymembers"]] + display = title.replace("_", " ") + content = f"# {display}\n\nPages in this category:\n" + content += "\n".join(f"- {member}" for member in members) + "\n" + return content, "md", None + + +def _fetch_article_page( + uri: str, client: httpx.Client +) -> tuple[str, str, str | None, str]: + """Fetch parsoid HTML for an article; returns (content, format, revid, title).""" + parts = urlsplit(uri) + host = parts.netloc + if host == "w.wiki": + resolved = urlsplit(str(client.get(uri).url)) + host = resolved.netloc + title = unquote(resolved.path[len("/wiki/") :]) + else: + title = unquote(parts.path[len("/wiki/") :]) + response = client.get( + f"https://{host}/api/rest_v1/page/html/{quote(title, safe='')}" + ) + response.raise_for_status() + revid = parse_revid(response.headers.get("etag")) + return response.text, "html", revid, title + + +def fetch_article( + uri: str, cache_dir: Path, client: httpx.Client | None +) -> dict[str, Any] | None: + """Return a corpus row for `uri`, fetching and caching it if needed. + + The cache holds the raw page plus a JSON sidecar with title, format, + revision id, and fetch date; a present sidecar marks a complete entry and + is served without network access. + """ + base = quote(uri, safe="") + meta_path = cache_dir / f"{base}.json" + if meta_path.exists(): + row = json.loads(meta_path.read_text()) + row["path"] = str(cache_dir / f"{base}.{row['format']}") + return row + + assert client is not None + try: + title = unquote(urlsplit(uri).path[len("/wiki/") :]) + if title.startswith("Category:"): + content, format, revid = _fetch_category_page( + urlsplit(uri).netloc, title, client + ) + else: + content, format, revid, title = _fetch_article_page(uri, client) + except Exception as e: + logger.warning(f"Failed to fetch {uri}: {e}") + return None + + row: dict[str, Any] = { + "uri": uri, + "title": title.replace("_", " "), + "format": format, + "revid": revid, + "fetched_at": datetime.now(UTC).date().isoformat(), + } + content_path = cache_dir / f"{base}.{format}" + content_path.write_text(content) + meta_path.write_text(json.dumps(row)) + row["path"] = str(content_path) + return row + + +def question_expected_uris(doc: Mapping[str, Any]) -> tuple[str, ...]: + uris: list[str] = [] + for link in parse_wiki_links(doc["wiki_links"]): + normalized = normalize_wiki_url(link) + if normalized is not None and normalized not in uris: + uris.append(normalized) + return tuple(uris) + + +_cached_corpus: list[dict[str, Any]] | None = None + + +def load_frames_corpus() -> list[dict[str, Any]]: + """Fetch (or read from cache) every article linked by any question.""" + global _cached_corpus + if _cached_corpus is None: + uris: dict[str, None] = {} + for doc in load_frames_test(): + for uri in question_expected_uris(doc): + uris.setdefault(uri) + cache_dir = get_cache_dir() + rows: list[dict[str, Any]] = [] + with httpx.Client( + headers={"User-Agent": USER_AGENT}, follow_redirects=True, timeout=60.0 + ) as client: + for index, uri in enumerate(uris, start=1): + row = fetch_article(uri, cache_dir, client) + if row is not None: + rows.append(row) + if index % 100 == 0: + logger.info(f"Fetched {index}/{len(uris)} articles") + logger.info(f"Fetched {len(rows)}/{len(uris)} articles") + _cached_corpus = rows + return _cached_corpus + + +def document_loader() -> Dataset: + return Dataset.from_list(load_frames_corpus()) + + +def map_frames_document(doc: Mapping[str, Any]) -> DocumentPayload: + content = Path(doc["path"]).read_text() + if doc["format"] == "html": + content = strip_navigation(content) + metadata: dict[str, str] = {"fetched_at": doc["fetched_at"]} + if doc.get("revid"): + metadata["revid"] = doc["revid"] + return DocumentPayload( + uri=doc["uri"], + content=content, + title=doc["title"], + metadata=metadata, + format=doc["format"], + ) + + +def map_frames_retrieval(doc: Mapping[str, Any]) -> RetrievalSample | None: + uris = question_expected_uris(doc) + if not uris: + return None + return RetrievalSample(question=doc["Prompt"], expected_uris=uris) + + +def build_frames_case( + index: int, doc: Mapping[str, Any] +) -> Case[str, str, dict[str, str]]: + return Case( + name=f"{index}", + inputs=doc["Prompt"], + expected_output=doc["Answer"], + metadata={ + "reasoning_types": str(doc["reasoning_types"]), + "case_index": str(index), + }, + ) + + +FRAMES_SPEC = DatasetSpec( + key="frames", + db_filename="frames.lancedb", + document_loader=document_loader, + document_mapper=map_frames_document, + qa_loader=load_frames_test, + qa_case_builder=build_frames_case, + retrieval_loader=load_frames_test, + retrieval_mapper=map_frames_retrieval, + retrieval_evaluators=[MAPEvaluator()], + citation_evaluator=CitationMAPEvaluator(), +) diff --git a/evaluations/pyproject.toml b/evaluations/pyproject.toml index 54d58c94..569ed6f3 100644 --- a/evaluations/pyproject.toml +++ b/evaluations/pyproject.toml @@ -10,6 +10,7 @@ requires-python = ">=3.12" dependencies = [ "haiku.rag-slim", "pydantic-ai-slim[evals,logfire]>=1.81.0", + "beautifulsoup4>=4.12.0", "datasets>=4.6.1", "huggingface_hub>=0.20.0", "typer>=0.21.0,<0.22.0", diff --git a/evaluations/tests/test_datasets.py b/evaluations/tests/test_datasets.py index 262035f9..40805377 100644 --- a/evaluations/tests/test_datasets.py +++ b/evaluations/tests/test_datasets.py @@ -1,5 +1,15 @@ from pathlib import Path +from evaluations.datasets.frames import ( + build_frames_case, + fetch_article, + map_frames_document, + map_frames_retrieval, + normalize_wiki_url, + parse_revid, + parse_wiki_links, + strip_navigation, +) from evaluations.datasets.hotpotqa import ( build_hotpotqa_case, extract_unique_documents, @@ -320,3 +330,237 @@ class TestT2RAGBench: assert len(corpus) == 2 assert {r["context_id"] for r in corpus} == {"ctx_a", "ctx_b"} + + +class TestFrames: + def test_parse_wiki_links_plain(self) -> None: + raw = ( + "['https://en.wikipedia.org/wiki/James_Buchanan', " + "'https://en.wikipedia.org/wiki/Harriet_Lane']" + ) + assert parse_wiki_links(raw) == [ + "https://en.wikipedia.org/wiki/James_Buchanan", + "https://en.wikipedia.org/wiki/Harriet_Lane", + ] + + def test_parse_wiki_links_splits_comma_joined_urls(self) -> None: + raw = ( + "['https://en.wikipedia.org/wiki/Tim_Salmon, " + "https://en.wikipedia.org/wiki/Troy_Glaus, ']" + ) + assert parse_wiki_links(raw) == [ + "https://en.wikipedia.org/wiki/Tim_Salmon", + "https://en.wikipedia.org/wiki/Troy_Glaus", + ] + + def test_parse_wiki_links_keeps_commas_inside_titles(self) -> None: + raw = ( + "['https://en.wikipedia.org/wiki/Lincoln,_Nebraska', " + "'https://en.wikipedia.org/wiki/Key_West#:~:text=The%20southernmost," + "apart%20at%20their%20closest%20points.']" + ) + assert parse_wiki_links(raw) == [ + "https://en.wikipedia.org/wiki/Lincoln,_Nebraska", + "https://en.wikipedia.org/wiki/Key_West#:~:text=The%20southernmost," + "apart%20at%20their%20closest%20points.", + ] + + def test_parse_wiki_links_strips_trailing_annotation(self) -> None: + raw = "['https://en.wikipedia.org/wiki/Pok%C3%A9mon (NOT REQUIRED, BUT HELPFUL) ']" + assert parse_wiki_links(raw) == ["https://en.wikipedia.org/wiki/Pok%C3%A9mon"] + + def test_normalize_strips_fragment_and_mobile_host(self) -> None: + assert ( + normalize_wiki_url("https://en.m.wikipedia.org/wiki/World_War_I#Aftermath") + == "https://en.wikipedia.org/wiki/World_War_I" + ) + + def test_normalize_decodes_and_canonicalizes_title(self) -> None: + assert ( + normalize_wiki_url("https://en.wikipedia.org/wiki/pain %26 Gain") + == "https://en.wikipedia.org/wiki/Pain_&_Gain" + ) + + def test_normalize_schemeless(self) -> None: + assert ( + normalize_wiki_url("en.wikipedia.org/wiki/Grazia_Deledda") + == "https://en.wikipedia.org/wiki/Grazia_Deledda" + ) + + def test_normalize_index_php_title(self) -> None: + assert ( + normalize_wiki_url( + "https://en.wikipedia.org/w/index.php?title=Bronco&redirect=no" + ) + == "https://en.wikipedia.org/wiki/Bronco" + ) + + def test_normalize_search_url(self) -> None: + url = ( + "https://en.wikipedia.org/w/index.php?search=Polytrichum+piliferum" + "&title=Special:Search&profile=advanced&fulltext=1&ns0=1" + ) + assert ( + normalize_wiki_url(url) + == "https://en.wikipedia.org/wiki/Polytrichum_piliferum" + ) + + def test_normalize_shortlink_passthrough(self) -> None: + assert normalize_wiki_url("https://w.wiki/ASFv") == "https://w.wiki/ASFv" + + def test_normalize_rejects_non_article(self) -> None: + assert normalize_wiki_url("") is None + assert normalize_wiki_url("https://en.wikipedia.org/foo") is None + + def test_parse_revid(self) -> None: + assert parse_revid('W/"1364811104/52cd04f4-864c-11f1"') == "1364811104" + assert parse_revid('"1234/abc"') == "1234" + assert parse_revid(None) is None + assert parse_revid("") is None + + def test_strip_navigation_removes_navboxes_keeps_infobox(self) -> None: + html = ( + "" + '
Born April 23, 1791
' + "

Some prose.

" + '
v t e Presidents
' + "" + ) + stripped = strip_navigation(html) + assert "Born April 23, 1791" in stripped + assert "Some prose." in stripped + assert "v t e Presidents" not in stripped + + def test_map_retrieval_normalizes_and_dedupes(self) -> None: + row = { + "Prompt": "Who was the 15th president?", + "wiki_links": ( + "['https://en.wikipedia.org/wiki/James_Buchanan#Presidency', " + "'https://en.m.wikipedia.org/wiki/James_Buchanan', " + "'https://en.wikipedia.org/wiki/Harriet_Lane']" + ), + } + sample = map_frames_retrieval(row) + assert sample is not None + assert sample.question == "Who was the 15th president?" + assert sample.expected_uris == ( + "https://en.wikipedia.org/wiki/James_Buchanan", + "https://en.wikipedia.org/wiki/Harriet_Lane", + ) + + def test_map_retrieval_empty_links(self) -> None: + assert map_frames_retrieval({"Prompt": "Q", "wiki_links": "[]"}) is None + + def test_map_document_html_strips_navigation(self, tmp_path: Path) -> None: + page = tmp_path / "article.html" + page.write_text( + "

Buchanan was a president.

" + '
v t e spam
' + ) + row = { + "uri": "https://en.wikipedia.org/wiki/James_Buchanan", + "title": "James Buchanan", + "path": str(page), + "format": "html", + "revid": "1364811104", + "fetched_at": "2026-07-23", + } + payload = map_frames_document(row) + assert payload.uri == "https://en.wikipedia.org/wiki/James_Buchanan" + assert payload.title == "James Buchanan" + assert payload.format == "html" + assert "Buchanan was a president." in (payload.content or "") + assert "v t e spam" not in (payload.content or "") + assert payload.metadata == { + "revid": "1364811104", + "fetched_at": "2026-07-23", + } + + def test_map_document_markdown_passthrough(self, tmp_path: Path) -> None: + page = tmp_path / "category.md" + page.write_text( + "Pages in Category:Summer Olympics in London:\n- 1908 Summer Olympics\n" + ) + row = { + "uri": "https://en.wikipedia.org/wiki/Category:Summer_Olympics_in_London", + "title": "Category:Summer Olympics in London", + "path": str(page), + "format": "md", + "revid": None, + "fetched_at": "2026-07-23", + } + payload = map_frames_document(row) + assert payload.format == "md" + assert "1908 Summer Olympics" in (payload.content or "") + assert payload.metadata == {"fetched_at": "2026-07-23"} + + def test_build_case(self) -> None: + row = { + "Prompt": "Who was the 15th president?", + "Answer": "James Buchanan", + "reasoning_types": "Multiple constraints | Temporal reasoning", + } + case = build_frames_case(3, row) + assert case.name == "3" + assert case.inputs == "Who was the 15th president?" + assert case.expected_output == "James Buchanan" + assert case.metadata == { + "reasoning_types": "Multiple constraints | Temporal reasoning", + "case_index": "3", + } + + def test_fetch_article_cache_hit_needs_no_network(self, tmp_path: Path) -> None: + uri = "https://en.wikipedia.org/wiki/James_Buchanan" + from urllib.parse import quote + + base = quote(uri, safe="") + (tmp_path / f"{base}.html").write_text("cached") + (tmp_path / f"{base}.json").write_text( + '{"uri": "https://en.wikipedia.org/wiki/James_Buchanan",' + ' "title": "James Buchanan", "format": "html",' + ' "revid": "123", "fetched_at": "2026-07-23"}' + ) + row = fetch_article(uri, tmp_path, client=None) + assert row is not None + assert row["uri"] == uri + assert row["revid"] == "123" + assert row["format"] == "html" + assert Path(row["path"]).read_text().startswith("") + + def test_fetch_article_category_synthesizes_members(self, tmp_path: Path) -> None: + class StubResponse: + def __init__(self, payload: dict) -> None: + self._payload = payload + + def raise_for_status(self) -> None: + pass + + def json(self) -> dict: + return self._payload + + class StubClient: + def get(self, url: str, params: dict | None = None) -> StubResponse: + assert params is not None + assert params["list"] == "categorymembers" + return StubResponse( + { + "query": { + "categorymembers": [ + {"title": "1908 Summer Olympics"}, + {"title": "2012 Summer Olympics"}, + ] + } + } + ) + + uri = "https://en.wikipedia.org/wiki/Category:Summer_Olympics_in_London" + row = fetch_article( + uri, + tmp_path, + client=StubClient(), # ty: ignore[invalid-argument-type] + ) + assert row is not None + assert row["format"] == "md" + content = Path(row["path"]).read_text() + assert "1908 Summer Olympics" in content + assert "2012 Summer Olympics" in content diff --git a/uv.lock b/uv.lock index c7e6d65f..20708676 100644 --- a/uv.lock +++ b/uv.lock @@ -1647,6 +1647,7 @@ name = "haiku-rag-evals" version = "0.77.0" source = { editable = "evaluations" } dependencies = [ + { name = "beautifulsoup4" }, { name = "datasets" }, { name = "haiku-rag-slim" }, { name = "huggingface-hub" }, @@ -1657,6 +1658,7 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "beautifulsoup4", specifier = ">=4.12.0" }, { name = "datasets", specifier = ">=4.6.1" }, { name = "haiku-rag-slim", editable = "haiku_rag_slim" }, { name = "huggingface-hub", specifier = ">=0.20.0" },