From 152d57d6e28964a200ebfd418292dcea85dd1442 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 23 Jul 2026 19:18:16 +0300 Subject: [PATCH 1/8] Add frames evaluation dataset --- CHANGELOG.md | 1 + evaluations/README.md | 1 + evaluations/configs/frames.yaml | 39 +++ evaluations/evaluations/datasets/__init__.py | 2 + evaluations/evaluations/datasets/frames.py | 287 +++++++++++++++++++ evaluations/pyproject.toml | 1 + evaluations/tests/test_datasets.py | 244 ++++++++++++++++ uv.lock | 2 + 8 files changed, 577 insertions(+) create mode 100644 evaluations/configs/frames.yaml create mode 100644 evaluations/evaluations/datasets/frames.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c968085..84a8f398 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ ### Added +- `frames` evaluation dataset. - `evaluations run --filter/-f CLAUSE`: SQL `WHERE` clause over document columns, applied to the retrieval benchmark's searches and to every capability search during QA. Recorded as `document_filter` in experiment metadata. - `mtrag_clapnq` / `mtrag_clapnq_rewrite` / `mtrag_clapnq_live` / `mtrag_clapnq_live_uncompacted` evaluation datasets: multi-turn QA with gold-prefix and live-session conversation replay, live arms with and without `EvidenceCompactionCapability`, Recall@k/nDCG@k retrieval metrics, eligibility-aware citation scoring, refusal precision/recall, per-turn tool-traffic attributes, and `citation_status` / `turn_citation_status` eval attributes. - Raw chunk metadata is now exposed to search and citation results, through `SearchResult.chunk_meta` and `Citation.chunk_meta`. For context-expanded results, the metadata is that of the anchor chunk. diff --git a/evaluations/README.md b/evaluations/README.md index 341fb3a6..7931826c 100644 --- a/evaluations/README.md +++ b/evaluations/README.md @@ -10,6 +10,7 @@ Contains evaluation scripts for benchmarking RAG retrieval and QA performance. A - HotpotQA (`hotpotqa`) — multi-hop QA over Wikipedia paragraphs (distractor validation split, 7,405 questions, two gold documents per question) - MTRAG ClapNQ (`mtrag_clapnq`, `mtrag_clapnq_rewrite`) — IBM's multi-turn RAG benchmark, ClapNQ (Wikipedia) domain: 183,408 passages, 208 retrieval queries with binary qrels, 224 generation tasks. The base key retrieves with the raw last user turn; the `_rewrite` variant uses the human standalone rewrites (both share one database). Retrieval reports Recall@5/@10, nDCG@5/@10, and MAP against IBM's published setup. QA replays each task's reference conversation prefix as message history and answers the final turn; the judge sees the conversation as a transcript, citation MAP is scored only on turns with gold passages, and refusal precision/recall is reported against the answerability labels. Generation scores are internal (our judge and rubric), not comparable with IBM's published generation numbers. The `mtrag_clapnq_live` key replays whole conversations (one case per conversation, `--limit` counts conversations) through a single capability session, carrying the model's own answers and tool history across turns; it reports the same outcomes per turn plus micro (per-turn) and macro (per-conversation) aggregates. +- FRAMES (`frames`) — multi-hop QA (824 questions, 2-23 gold Wikipedia articles per question). The corpus is the union of the ~2.5k linked articles, fetched from the Wikipedia REST API at current revision (revision id and fetch date recorded in the article cache) with navigation chrome stripped. There is no official FRAMES evaluation setup; numbers here correspond to the paper's multi-step retrieval setting (fixed corpus, agentic retrieval, judged accuracy) and are not comparable to its closed-book, oracle-prompt, or web-search settings. Answers were authored against ~2024 revisions and may have drifted with article content. - OpenRAG Bench, two variants: - `orb_text` — text embedder (`qwen3-embedding:4b`, 2560-dim) with VLM picture descriptions baked into chunk content at ingest. Use for text-only retrieval/QA against figure-rich corpora. - `orb_multimodal` — multimodal embedder (`qwen3-vl-embedding-8b`, 4096-dim) with picture vectors in the same space as text. Use for cross-modal retrieval (text-as-query → figure hits, image-as-query) and vision QA where the figure itself is the answer. diff --git a/evaluations/configs/frames.yaml b/evaluations/configs/frames.yaml new file mode 100644 index 00000000..7c20fad4 --- /dev/null +++ b/evaluations/configs/frames.yaml @@ -0,0 +1,39 @@ +# Reference config for the `frames` evaluation database. +# FRAMES (google/frames-benchmark): 824 multi-hop questions over a corpus of +# the ~2.5k Wikipedia articles linked per question, fetched at current +# revision (revid + fetch date recorded in the article cache). +# Run: evaluations run frames --config configs/frames.yaml +# base_url uses the `vllm` host serving each model over an OpenAI-compatible API. + +environment: development + +storage: + auto_vacuum: false + +embeddings: + model: + provider: openai + name: qwen3-embedding-4b + vector_dim: 2560 + base_url: http://vllm:11431/v1 + +reranking: + model: + provider: vllm + name: Qwen/Qwen3-Reranker-4B + base_url: http://vllm:11433 + +qa: + model: + provider: openai + name: gemma4-26b + base_url: http://vllm:11432/v1 + max_tokens: 49152 + +evaluations: + judge: + provider: openai + name: RedHatAI/Qwen3.6-35B-A3B-NVFP4 + base_url: http://vllm:11430/v1 + temperature: 0.0 + max_tokens: 4096 diff --git a/evaluations/evaluations/datasets/__init__.py b/evaluations/evaluations/datasets/__init__.py index 45c29adb..4a968b5f 100644 --- a/evaluations/evaluations/datasets/__init__.py +++ b/evaluations/evaluations/datasets/__init__.py @@ -1,5 +1,6 @@ from evaluations.config import DatasetSpec +from .frames import FRAMES_SPEC from .hotpotqa import HOTPOTQA_SPEC from .mtrag import ( MTRAG_CLAPNQ_LIVE_SPEC, @@ -17,6 +18,7 @@ from .t2_ragbench import T2_FINQA_SPEC, T2_TATDQA_SPEC DATASETS: dict[str, DatasetSpec] = { spec.key: spec for spec in ( + FRAMES_SPEC, HOTPOTQA_SPEC, MTRAG_CLAPNQ_SPEC, MTRAG_CLAPNQ_REWRITE_SPEC, diff --git a/evaluations/evaluations/datasets/frames.py b/evaluations/evaluations/datasets/frames.py new file mode 100644 index 00000000..f1860bf7 --- /dev/null +++ b/evaluations/evaluations/datasets/frames.py @@ -0,0 +1,287 @@ +"""FRAMES benchmark (google/frames-benchmark). + +824 multi-hop questions, each grounded in two or more Wikipedia articles. The +corpus is the union of the articles linked per question, fetched from the +Wikipedia REST API at current revision and cached locally with the revision id +and fetch date. +""" + +import ast +import json +import logging +import re +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, quote, unquote, urlsplit + +import httpx +from bs4 import BeautifulSoup +from datasets import Dataset, load_dataset +from pydantic_evals import Case + +from evaluations.config import DatasetSpec, DocumentPayload, RetrievalSample +from evaluations.evaluators import CitationMAPEvaluator, MAPEvaluator + +logger = logging.getLogger(__name__) + +USER_AGENT = "haiku.rag-evaluations (https://github.com/ggozad/haiku.rag)" + + +def load_frames_test() -> Dataset: + return load_dataset("google/frames-benchmark")["test"] + + +def parse_wiki_links(raw: str) -> list[str]: + """Extract URLs from a `wiki_links` value. + + The value is a Python-list-repr string. A single list element may pack + several comma-separated URLs, and may carry trailing prose annotations; + titles themselves can contain commas, so elements are split only where a + new URL starts. + """ + links: list[str] = [] + for element in ast.literal_eval(raw): + for part in re.split(r",\s*(?=http)", element): + tokens = part.split() + if not tokens: + continue + url = tokens[0].strip(", ") + if url: + links.append(url) + return links + + +def normalize_wiki_url(url: str) -> str | None: + """Canonical article URL, used both as document uri and expected uri. + + Strips fragments, decodes percent-escapes, folds mobile hosts, resolves + `index.php?title=` and `Special:Search` forms, and applies MediaWiki title + canonicalization (underscores, first letter uppercased). Returns None for + strings that don't point to an article. + """ + url = url.strip() + if not url: + return None + if "://" not in url: + url = "https://" + url + parts = urlsplit(url) + host = parts.netloc.replace(".m.wikipedia.org", ".wikipedia.org") + if host == "w.wiki": + return url + if parts.path.startswith("/wiki/"): + title = parts.path[len("/wiki/") :] + elif parts.path.startswith("/w/index.php"): + query = parse_qs(parts.query) + title = query.get("title", [""])[0] + if not title or title.startswith("Special:"): + title = query.get("search", [""])[0] + else: + return None + title = unquote(title).replace(" ", "_").strip("_") + if not title: + return None + return f"https://{host}/wiki/{title[0].upper() + title[1:]}" + + +def parse_revid(etag: str | None) -> str | None: + """Revision id from a Wikipedia REST ETag header (`W/"/"`).""" + if not etag: + return None + match = re.search(r'"([^/"]+)/', etag) + return match.group(1) if match else None + + +def strip_navigation(html: str) -> str: + """Drop navigation chrome (navboxes, succession boxes) from parsoid HTML. + + These render as link-spam tables naming hundreds of related articles, + polluting retrieval. Infoboxes carry no navigation role and are kept. + """ + soup = BeautifulSoup(html, "html.parser") + for element in soup.find_all(attrs={"role": "navigation"}): + element.decompose() + return str(soup) + + +def get_cache_dir() -> Path: + cache_dir = Path.home() / ".cache" / "haiku.rag" / "evaluations" / "frames_articles" + cache_dir.mkdir(parents=True, exist_ok=True) + return cache_dir + + +def _fetch_category_page( + host: str, title: str, client: httpx.Client +) -> tuple[str, str, str | None]: + """Category pages render empty via page/html; synthesize a members list.""" + response = client.get( + f"https://{host}/w/api.php", + params={ + "action": "query", + "list": "categorymembers", + "cmtitle": title, + "cmlimit": "500", + "format": "json", + }, + ) + response.raise_for_status() + members = [m["title"] for m in response.json()["query"]["categorymembers"]] + display = title.replace("_", " ") + content = f"# {display}\n\nPages in this category:\n" + content += "\n".join(f"- {member}" for member in members) + "\n" + return content, "md", None + + +def _fetch_article_page( + uri: str, client: httpx.Client +) -> tuple[str, str, str | None, str]: + """Fetch parsoid HTML for an article; returns (content, format, revid, title).""" + parts = urlsplit(uri) + host = parts.netloc + if host == "w.wiki": + resolved = urlsplit(str(client.get(uri).url)) + host = resolved.netloc + title = unquote(resolved.path[len("/wiki/") :]) + else: + title = unquote(parts.path[len("/wiki/") :]) + response = client.get( + f"https://{host}/api/rest_v1/page/html/{quote(title, safe='')}" + ) + response.raise_for_status() + revid = parse_revid(response.headers.get("etag")) + return response.text, "html", revid, title + + +def fetch_article( + uri: str, cache_dir: Path, client: httpx.Client | None +) -> dict[str, Any] | None: + """Return a corpus row for `uri`, fetching and caching it if needed. + + The cache holds the raw page plus a JSON sidecar with title, format, + revision id, and fetch date; a present sidecar marks a complete entry and + is served without network access. + """ + base = quote(uri, safe="") + meta_path = cache_dir / f"{base}.json" + if meta_path.exists(): + row = json.loads(meta_path.read_text()) + row["path"] = str(cache_dir / f"{base}.{row['format']}") + return row + + assert client is not None + try: + title = unquote(urlsplit(uri).path[len("/wiki/") :]) + if title.startswith("Category:"): + content, format, revid = _fetch_category_page( + urlsplit(uri).netloc, title, client + ) + else: + content, format, revid, title = _fetch_article_page(uri, client) + except Exception as e: + logger.warning(f"Failed to fetch {uri}: {e}") + return None + + row: dict[str, Any] = { + "uri": uri, + "title": title.replace("_", " "), + "format": format, + "revid": revid, + "fetched_at": datetime.now(UTC).date().isoformat(), + } + content_path = cache_dir / f"{base}.{format}" + content_path.write_text(content) + meta_path.write_text(json.dumps(row)) + row["path"] = str(content_path) + return row + + +def question_expected_uris(doc: Mapping[str, Any]) -> tuple[str, ...]: + uris: list[str] = [] + for link in parse_wiki_links(doc["wiki_links"]): + normalized = normalize_wiki_url(link) + if normalized is not None and normalized not in uris: + uris.append(normalized) + return tuple(uris) + + +_cached_corpus: list[dict[str, Any]] | None = None + + +def load_frames_corpus() -> list[dict[str, Any]]: + """Fetch (or read from cache) every article linked by any question.""" + global _cached_corpus + if _cached_corpus is None: + uris: dict[str, None] = {} + for doc in load_frames_test(): + for uri in question_expected_uris(doc): + uris.setdefault(uri) + cache_dir = get_cache_dir() + rows: list[dict[str, Any]] = [] + with httpx.Client( + headers={"User-Agent": USER_AGENT}, follow_redirects=True, timeout=60.0 + ) as client: + for index, uri in enumerate(uris, start=1): + row = fetch_article(uri, cache_dir, client) + if row is not None: + rows.append(row) + if index % 100 == 0: + logger.info(f"Fetched {index}/{len(uris)} articles") + logger.info(f"Fetched {len(rows)}/{len(uris)} articles") + _cached_corpus = rows + return _cached_corpus + + +def document_loader() -> Dataset: + return Dataset.from_list(load_frames_corpus()) + + +def map_frames_document(doc: Mapping[str, Any]) -> DocumentPayload: + content = Path(doc["path"]).read_text() + if doc["format"] == "html": + content = strip_navigation(content) + metadata: dict[str, str] = {"fetched_at": doc["fetched_at"]} + if doc.get("revid"): + metadata["revid"] = doc["revid"] + return DocumentPayload( + uri=doc["uri"], + content=content, + title=doc["title"], + metadata=metadata, + format=doc["format"], + ) + + +def map_frames_retrieval(doc: Mapping[str, Any]) -> RetrievalSample | None: + uris = question_expected_uris(doc) + if not uris: + return None + return RetrievalSample(question=doc["Prompt"], expected_uris=uris) + + +def build_frames_case( + index: int, doc: Mapping[str, Any] +) -> Case[str, str, dict[str, str]]: + return Case( + name=f"{index}", + inputs=doc["Prompt"], + expected_output=doc["Answer"], + metadata={ + "reasoning_types": str(doc["reasoning_types"]), + "case_index": str(index), + }, + ) + + +FRAMES_SPEC = DatasetSpec( + key="frames", + db_filename="frames.lancedb", + document_loader=document_loader, + document_mapper=map_frames_document, + qa_loader=load_frames_test, + qa_case_builder=build_frames_case, + retrieval_loader=load_frames_test, + retrieval_mapper=map_frames_retrieval, + retrieval_evaluators=[MAPEvaluator()], + citation_evaluator=CitationMAPEvaluator(), +) diff --git a/evaluations/pyproject.toml b/evaluations/pyproject.toml index 54d58c94..569ed6f3 100644 --- a/evaluations/pyproject.toml +++ b/evaluations/pyproject.toml @@ -10,6 +10,7 @@ requires-python = ">=3.12" dependencies = [ "haiku.rag-slim", "pydantic-ai-slim[evals,logfire]>=1.81.0", + "beautifulsoup4>=4.12.0", "datasets>=4.6.1", "huggingface_hub>=0.20.0", "typer>=0.21.0,<0.22.0", diff --git a/evaluations/tests/test_datasets.py b/evaluations/tests/test_datasets.py index 262035f9..40805377 100644 --- a/evaluations/tests/test_datasets.py +++ b/evaluations/tests/test_datasets.py @@ -1,5 +1,15 @@ from pathlib import Path +from evaluations.datasets.frames import ( + build_frames_case, + fetch_article, + map_frames_document, + map_frames_retrieval, + normalize_wiki_url, + parse_revid, + parse_wiki_links, + strip_navigation, +) from evaluations.datasets.hotpotqa import ( build_hotpotqa_case, extract_unique_documents, @@ -320,3 +330,237 @@ class TestT2RAGBench: assert len(corpus) == 2 assert {r["context_id"] for r in corpus} == {"ctx_a", "ctx_b"} + + +class TestFrames: + def test_parse_wiki_links_plain(self) -> None: + raw = ( + "['https://en.wikipedia.org/wiki/James_Buchanan', " + "'https://en.wikipedia.org/wiki/Harriet_Lane']" + ) + assert parse_wiki_links(raw) == [ + "https://en.wikipedia.org/wiki/James_Buchanan", + "https://en.wikipedia.org/wiki/Harriet_Lane", + ] + + def test_parse_wiki_links_splits_comma_joined_urls(self) -> None: + raw = ( + "['https://en.wikipedia.org/wiki/Tim_Salmon, " + "https://en.wikipedia.org/wiki/Troy_Glaus, ']" + ) + assert parse_wiki_links(raw) == [ + "https://en.wikipedia.org/wiki/Tim_Salmon", + "https://en.wikipedia.org/wiki/Troy_Glaus", + ] + + def test_parse_wiki_links_keeps_commas_inside_titles(self) -> None: + raw = ( + "['https://en.wikipedia.org/wiki/Lincoln,_Nebraska', " + "'https://en.wikipedia.org/wiki/Key_West#:~:text=The%20southernmost," + "apart%20at%20their%20closest%20points.']" + ) + assert parse_wiki_links(raw) == [ + "https://en.wikipedia.org/wiki/Lincoln,_Nebraska", + "https://en.wikipedia.org/wiki/Key_West#:~:text=The%20southernmost," + "apart%20at%20their%20closest%20points.", + ] + + def test_parse_wiki_links_strips_trailing_annotation(self) -> None: + raw = "['https://en.wikipedia.org/wiki/Pok%C3%A9mon (NOT REQUIRED, BUT HELPFUL) ']" + assert parse_wiki_links(raw) == ["https://en.wikipedia.org/wiki/Pok%C3%A9mon"] + + def test_normalize_strips_fragment_and_mobile_host(self) -> None: + assert ( + normalize_wiki_url("https://en.m.wikipedia.org/wiki/World_War_I#Aftermath") + == "https://en.wikipedia.org/wiki/World_War_I" + ) + + def test_normalize_decodes_and_canonicalizes_title(self) -> None: + assert ( + normalize_wiki_url("https://en.wikipedia.org/wiki/pain %26 Gain") + == "https://en.wikipedia.org/wiki/Pain_&_Gain" + ) + + def test_normalize_schemeless(self) -> None: + assert ( + normalize_wiki_url("en.wikipedia.org/wiki/Grazia_Deledda") + == "https://en.wikipedia.org/wiki/Grazia_Deledda" + ) + + def test_normalize_index_php_title(self) -> None: + assert ( + normalize_wiki_url( + "https://en.wikipedia.org/w/index.php?title=Bronco&redirect=no" + ) + == "https://en.wikipedia.org/wiki/Bronco" + ) + + def test_normalize_search_url(self) -> None: + url = ( + "https://en.wikipedia.org/w/index.php?search=Polytrichum+piliferum" + "&title=Special:Search&profile=advanced&fulltext=1&ns0=1" + ) + assert ( + normalize_wiki_url(url) + == "https://en.wikipedia.org/wiki/Polytrichum_piliferum" + ) + + def test_normalize_shortlink_passthrough(self) -> None: + assert normalize_wiki_url("https://w.wiki/ASFv") == "https://w.wiki/ASFv" + + def test_normalize_rejects_non_article(self) -> None: + assert normalize_wiki_url("") is None + assert normalize_wiki_url("https://en.wikipedia.org/foo") is None + + def test_parse_revid(self) -> None: + assert parse_revid('W/"1364811104/52cd04f4-864c-11f1"') == "1364811104" + assert parse_revid('"1234/abc"') == "1234" + assert parse_revid(None) is None + assert parse_revid("") is None + + def test_strip_navigation_removes_navboxes_keeps_infobox(self) -> None: + html = ( + "" + '
Born April 23, 1791
' + "

Some prose.

" + '
v t e Presidents
' + "" + ) + stripped = strip_navigation(html) + assert "Born April 23, 1791" in stripped + assert "Some prose." in stripped + assert "v t e Presidents" not in stripped + + def test_map_retrieval_normalizes_and_dedupes(self) -> None: + row = { + "Prompt": "Who was the 15th president?", + "wiki_links": ( + "['https://en.wikipedia.org/wiki/James_Buchanan#Presidency', " + "'https://en.m.wikipedia.org/wiki/James_Buchanan', " + "'https://en.wikipedia.org/wiki/Harriet_Lane']" + ), + } + sample = map_frames_retrieval(row) + assert sample is not None + assert sample.question == "Who was the 15th president?" + assert sample.expected_uris == ( + "https://en.wikipedia.org/wiki/James_Buchanan", + "https://en.wikipedia.org/wiki/Harriet_Lane", + ) + + def test_map_retrieval_empty_links(self) -> None: + assert map_frames_retrieval({"Prompt": "Q", "wiki_links": "[]"}) is None + + def test_map_document_html_strips_navigation(self, tmp_path: Path) -> None: + page = tmp_path / "article.html" + page.write_text( + "

Buchanan was a president.

" + '
v t e spam
' + ) + row = { + "uri": "https://en.wikipedia.org/wiki/James_Buchanan", + "title": "James Buchanan", + "path": str(page), + "format": "html", + "revid": "1364811104", + "fetched_at": "2026-07-23", + } + payload = map_frames_document(row) + assert payload.uri == "https://en.wikipedia.org/wiki/James_Buchanan" + assert payload.title == "James Buchanan" + assert payload.format == "html" + assert "Buchanan was a president." in (payload.content or "") + assert "v t e spam" not in (payload.content or "") + assert payload.metadata == { + "revid": "1364811104", + "fetched_at": "2026-07-23", + } + + def test_map_document_markdown_passthrough(self, tmp_path: Path) -> None: + page = tmp_path / "category.md" + page.write_text( + "Pages in Category:Summer Olympics in London:\n- 1908 Summer Olympics\n" + ) + row = { + "uri": "https://en.wikipedia.org/wiki/Category:Summer_Olympics_in_London", + "title": "Category:Summer Olympics in London", + "path": str(page), + "format": "md", + "revid": None, + "fetched_at": "2026-07-23", + } + payload = map_frames_document(row) + assert payload.format == "md" + assert "1908 Summer Olympics" in (payload.content or "") + assert payload.metadata == {"fetched_at": "2026-07-23"} + + def test_build_case(self) -> None: + row = { + "Prompt": "Who was the 15th president?", + "Answer": "James Buchanan", + "reasoning_types": "Multiple constraints | Temporal reasoning", + } + case = build_frames_case(3, row) + assert case.name == "3" + assert case.inputs == "Who was the 15th president?" + assert case.expected_output == "James Buchanan" + assert case.metadata == { + "reasoning_types": "Multiple constraints | Temporal reasoning", + "case_index": "3", + } + + def test_fetch_article_cache_hit_needs_no_network(self, tmp_path: Path) -> None: + uri = "https://en.wikipedia.org/wiki/James_Buchanan" + from urllib.parse import quote + + base = quote(uri, safe="") + (tmp_path / f"{base}.html").write_text("cached") + (tmp_path / f"{base}.json").write_text( + '{"uri": "https://en.wikipedia.org/wiki/James_Buchanan",' + ' "title": "James Buchanan", "format": "html",' + ' "revid": "123", "fetched_at": "2026-07-23"}' + ) + row = fetch_article(uri, tmp_path, client=None) + assert row is not None + assert row["uri"] == uri + assert row["revid"] == "123" + assert row["format"] == "html" + assert Path(row["path"]).read_text().startswith("") + + def test_fetch_article_category_synthesizes_members(self, tmp_path: Path) -> None: + class StubResponse: + def __init__(self, payload: dict) -> None: + self._payload = payload + + def raise_for_status(self) -> None: + pass + + def json(self) -> dict: + return self._payload + + class StubClient: + def get(self, url: str, params: dict | None = None) -> StubResponse: + assert params is not None + assert params["list"] == "categorymembers" + return StubResponse( + { + "query": { + "categorymembers": [ + {"title": "1908 Summer Olympics"}, + {"title": "2012 Summer Olympics"}, + ] + } + } + ) + + uri = "https://en.wikipedia.org/wiki/Category:Summer_Olympics_in_London" + row = fetch_article( + uri, + tmp_path, + client=StubClient(), # ty: ignore[invalid-argument-type] + ) + assert row is not None + assert row["format"] == "md" + content = Path(row["path"]).read_text() + assert "1908 Summer Olympics" in content + assert "2012 Summer Olympics" in content diff --git a/uv.lock b/uv.lock index c7e6d65f..20708676 100644 --- a/uv.lock +++ b/uv.lock @@ -1647,6 +1647,7 @@ name = "haiku-rag-evals" version = "0.77.0" source = { editable = "evaluations" } dependencies = [ + { name = "beautifulsoup4" }, { name = "datasets" }, { name = "haiku-rag-slim" }, { name = "huggingface-hub" }, @@ -1657,6 +1658,7 @@ dependencies = [ [package.metadata] requires-dist = [ + { name = "beautifulsoup4", specifier = ">=4.12.0" }, { name = "datasets", specifier = ">=4.6.1" }, { name = "haiku-rag-slim", editable = "haiku_rag_slim" }, { name = "huggingface-hub", specifier = ">=0.20.0" }, From c00ccb73226d6691c1dc68be318c8d05376f3642 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 24 Jul 2026 12:48:01 +0300 Subject: [PATCH 2/8] Harden FRAMES corpus fetching against Wikimedia rate limits --- evaluations/evaluations/datasets/frames.py | 45 ++++++-- evaluations/tests/test_datasets.py | 124 +++++++++++++++++++++ 2 files changed, 158 insertions(+), 11 deletions(-) diff --git a/evaluations/evaluations/datasets/frames.py b/evaluations/evaluations/datasets/frames.py index f1860bf7..64371b26 100644 --- a/evaluations/evaluations/datasets/frames.py +++ b/evaluations/evaluations/datasets/frames.py @@ -10,6 +10,7 @@ import ast import json import logging import re +import time from collections.abc import Mapping from datetime import UTC, datetime from pathlib import Path @@ -27,6 +28,9 @@ from evaluations.evaluators import CitationMAPEvaluator, MAPEvaluator logger = logging.getLogger(__name__) USER_AGENT = "haiku.rag-evaluations (https://github.com/ggozad/haiku.rag)" +FETCH_ATTEMPTS = 3 +THROTTLE_SECONDS = 1.0 +RATE_LIMIT_BACKOFF_SECONDS = 60.0 def load_frames_test() -> Dataset: @@ -153,6 +157,13 @@ def _fetch_article_page( return response.text, "html", revid, title +def _backoff_seconds(error: Exception, attempt: int) -> float: + if isinstance(error, httpx.HTTPStatusError) and error.response.status_code == 429: + retry_after = error.response.headers.get("retry-after") + return float(retry_after) if retry_after else RATE_LIMIT_BACKOFF_SECONDS + return 5.0 * attempt + + def fetch_article( uri: str, cache_dir: Path, client: httpx.Client | None ) -> dict[str, Any] | None: @@ -170,17 +181,24 @@ def fetch_article( return row assert client is not None - try: - title = unquote(urlsplit(uri).path[len("/wiki/") :]) - if title.startswith("Category:"): - content, format, revid = _fetch_category_page( - urlsplit(uri).netloc, title, client - ) - else: - content, format, revid, title = _fetch_article_page(uri, client) - except Exception as e: - logger.warning(f"Failed to fetch {uri}: {e}") - return None + title = unquote(urlsplit(uri).path[len("/wiki/") :]) + # Wikimedia throttles sustained bot traffic; pace uncached fetches. + time.sleep(THROTTLE_SECONDS) + for attempt in range(1, FETCH_ATTEMPTS + 1): + try: + if title.startswith("Category:"): + content, format, revid = _fetch_category_page( + urlsplit(uri).netloc, title, client + ) + else: + content, format, revid, title = _fetch_article_page(uri, client) + break + except Exception as e: + if attempt == FETCH_ATTEMPTS: + logger.warning(f"Failed to fetch {uri}: {e}") + return None + logger.info(f"Retrying {uri} after error: {e}") + time.sleep(_backoff_seconds(e, attempt)) row: dict[str, Any] = { "uri": uri, @@ -228,6 +246,11 @@ def load_frames_corpus() -> list[dict[str, Any]]: if index % 100 == 0: logger.info(f"Fetched {index}/{len(uris)} articles") logger.info(f"Fetched {len(rows)}/{len(uris)} articles") + if len(rows) < len(uris): + raise RuntimeError( + f"Fetched only {len(rows)}/{len(uris)} FRAMES articles; " + "refusing to build a partial corpus. Re-run to resume from cache." + ) _cached_corpus = rows return _cached_corpus diff --git a/evaluations/tests/test_datasets.py b/evaluations/tests/test_datasets.py index 40805377..61f96bc3 100644 --- a/evaluations/tests/test_datasets.py +++ b/evaluations/tests/test_datasets.py @@ -1,6 +1,9 @@ from pathlib import Path +import pytest + from evaluations.datasets.frames import ( + FETCH_ATTEMPTS, build_frames_case, fetch_article, map_frames_document, @@ -564,3 +567,124 @@ class TestFrames: content = Path(row["path"]).read_text() assert "1908 Summer Olympics" in content assert "2012 Summer Olympics" in content + + def test_fetch_article_retries_transient_failures( + self, tmp_path: Path, monkeypatch + ) -> None: + sleeps: list[float] = [] + monkeypatch.setattr( + "evaluations.datasets.frames.time.sleep", lambda s: sleeps.append(s) + ) + + class FlakyResponse: + text = "ok" + headers = {"etag": 'W/"42/uuid"'} + + def raise_for_status(self) -> None: + pass + + class FlakyClient: + def __init__(self) -> None: + self.calls = 0 + + def get(self, url: str, params: dict | None = None) -> FlakyResponse: + self.calls += 1 + if self.calls < 3: + raise OSError("connection reset") + return FlakyResponse() + + client = FlakyClient() + row = fetch_article( + "https://en.wikipedia.org/wiki/Capybara", + tmp_path, + client=client, # ty: ignore[invalid-argument-type] + ) + assert row is not None + assert row["revid"] == "42" + assert client.calls == 3 + # One throttle sleep before fetching plus one backoff per failure. + assert len(sleeps) == 3 + + def test_fetch_article_honors_retry_after_on_rate_limit( + self, tmp_path: Path, monkeypatch + ) -> None: + import httpx + + sleeps: list[float] = [] + monkeypatch.setattr( + "evaluations.datasets.frames.time.sleep", lambda s: sleeps.append(s) + ) + request = httpx.Request("GET", "https://en.wikipedia.org/x") + + class OkResponse: + text = "ok" + headers = {"etag": 'W/"42/uuid"'} + + def raise_for_status(self) -> None: + pass + + class RateLimitedClient: + def __init__(self) -> None: + self.calls = 0 + + def get(self, url: str, params: dict | None = None) -> OkResponse: + self.calls += 1 + if self.calls == 1: + raise httpx.HTTPStatusError( + "429 too many requests", + request=request, + response=httpx.Response( + 429, headers={"retry-after": "13"}, request=request + ), + ) + return OkResponse() + + row = fetch_article( + "https://en.wikipedia.org/wiki/Capybara", + tmp_path, + client=RateLimitedClient(), # ty: ignore[invalid-argument-type] + ) + assert row is not None + assert 13.0 in sleeps + + def test_fetch_article_gives_up_after_max_attempts( + self, tmp_path: Path, monkeypatch + ) -> None: + monkeypatch.setattr("evaluations.datasets.frames.time.sleep", lambda s: None) + + class DeadClient: + def __init__(self) -> None: + self.calls = 0 + + def get(self, url: str, params: dict | None = None): + self.calls += 1 + raise OSError("connection reset") + + client = DeadClient() + row = fetch_article( + "https://en.wikipedia.org/wiki/Capybara", + tmp_path, + client=client, # ty: ignore[invalid-argument-type] + ) + assert row is None + assert client.calls == FETCH_ATTEMPTS + + def test_load_corpus_raises_on_partial_fetch(self, monkeypatch) -> None: + import evaluations.datasets.frames as frames + + monkeypatch.setattr(frames, "_cached_corpus", None) + monkeypatch.setattr( + frames, + "load_frames_test", + lambda: [ + { + "wiki_links": "['https://en.wikipedia.org/wiki/A', " + "'https://en.wikipedia.org/wiki/B']" + } + ], + ) + monkeypatch.setattr( + frames, "fetch_article", lambda uri, cache_dir, client: None + ) + with pytest.raises(RuntimeError, match="0/2"): + frames.load_frames_corpus() From aa49ec6cb3e79cdf96a222b14b73e8e0cf3ce69c Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 24 Jul 2026 15:22:53 +0300 Subject: [PATCH 3/8] Exclude FRAMES questions whose articles were deleted from Wikipedia --- evaluations/README.md | 2 +- evaluations/evaluations/datasets/frames.py | 24 +++++++++++++++++++--- evaluations/tests/test_datasets.py | 12 ++++++++++- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/evaluations/README.md b/evaluations/README.md index 7931826c..06acebd5 100644 --- a/evaluations/README.md +++ b/evaluations/README.md @@ -10,7 +10,7 @@ Contains evaluation scripts for benchmarking RAG retrieval and QA performance. A - HotpotQA (`hotpotqa`) — multi-hop QA over Wikipedia paragraphs (distractor validation split, 7,405 questions, two gold documents per question) - MTRAG ClapNQ (`mtrag_clapnq`, `mtrag_clapnq_rewrite`) — IBM's multi-turn RAG benchmark, ClapNQ (Wikipedia) domain: 183,408 passages, 208 retrieval queries with binary qrels, 224 generation tasks. The base key retrieves with the raw last user turn; the `_rewrite` variant uses the human standalone rewrites (both share one database). Retrieval reports Recall@5/@10, nDCG@5/@10, and MAP against IBM's published setup. QA replays each task's reference conversation prefix as message history and answers the final turn; the judge sees the conversation as a transcript, citation MAP is scored only on turns with gold passages, and refusal precision/recall is reported against the answerability labels. Generation scores are internal (our judge and rubric), not comparable with IBM's published generation numbers. The `mtrag_clapnq_live` key replays whole conversations (one case per conversation, `--limit` counts conversations) through a single capability session, carrying the model's own answers and tool history across turns; it reports the same outcomes per turn plus micro (per-turn) and macro (per-conversation) aggregates. -- FRAMES (`frames`) — multi-hop QA (824 questions, 2-23 gold Wikipedia articles per question). The corpus is the union of the ~2.5k linked articles, fetched from the Wikipedia REST API at current revision (revision id and fetch date recorded in the article cache) with navigation chrome stripped. There is no official FRAMES evaluation setup; numbers here correspond to the paper's multi-step retrieval setting (fixed corpus, agentic retrieval, judged accuracy) and are not comparable to its closed-book, oracle-prompt, or web-search settings. Answers were authored against ~2024 revisions and may have drifted with article content. +- FRAMES (`frames`) — multi-hop QA (822 questions, 2-23 gold Wikipedia articles per question; 2 of the original 824 questions are excluded because a linked article has been deleted from Wikipedia). The corpus is the union of the 2,521 linked articles, fetched from the Wikipedia REST API at current revision (revision id and fetch date recorded in the article cache) with navigation chrome stripped. There is no official FRAMES evaluation setup; numbers here correspond to the paper's multi-step retrieval setting (fixed corpus, agentic retrieval, judged accuracy) and are not comparable to its closed-book, oracle-prompt, or web-search settings. Answers were authored against ~2024 revisions and may have drifted with article content. - OpenRAG Bench, two variants: - `orb_text` — text embedder (`qwen3-embedding:4b`, 2560-dim) with VLM picture descriptions baked into chunk content at ingest. Use for text-only retrieval/QA against figure-rich corpora. - `orb_multimodal` — multimodal embedder (`qwen3-vl-embedding-8b`, 4096-dim) with picture vectors in the same space as text. Use for cross-modal retrieval (text-as-query → figure hits, image-as-query) and vision QA where the figure itself is the answer. diff --git a/evaluations/evaluations/datasets/frames.py b/evaluations/evaluations/datasets/frames.py index 64371b26..7f019acc 100644 --- a/evaluations/evaluations/datasets/frames.py +++ b/evaluations/evaluations/datasets/frames.py @@ -33,10 +33,28 @@ THROTTLE_SECONDS = 1.0 RATE_LIMIT_BACKOFF_SECONDS = 60.0 +# Articles deleted from Wikipedia since FRAMES was authored; the questions +# linking them have lost their evidence and are excluded from the benchmark. +_DELETED_ARTICLES = frozenset( + { + "https://en.wikipedia.org/wiki/Nemanja_Marković", + "https://en.wikipedia.org/wiki/Jack_Vance_(tennis)", + } +) + + def load_frames_test() -> Dataset: return load_dataset("google/frames-benchmark")["test"] +def question_is_answerable(doc: Mapping[str, Any]) -> bool: + return not _DELETED_ARTICLES & set(question_expected_uris(doc)) + + +def load_frames_questions() -> Dataset: + return load_frames_test().filter(question_is_answerable) + + def parse_wiki_links(raw: str) -> list[str]: """Extract URLs from a `wiki_links` value. @@ -231,7 +249,7 @@ def load_frames_corpus() -> list[dict[str, Any]]: global _cached_corpus if _cached_corpus is None: uris: dict[str, None] = {} - for doc in load_frames_test(): + for doc in load_frames_questions(): for uri in question_expected_uris(doc): uris.setdefault(uri) cache_dir = get_cache_dir() @@ -301,9 +319,9 @@ FRAMES_SPEC = DatasetSpec( db_filename="frames.lancedb", document_loader=document_loader, document_mapper=map_frames_document, - qa_loader=load_frames_test, + qa_loader=load_frames_questions, qa_case_builder=build_frames_case, - retrieval_loader=load_frames_test, + retrieval_loader=load_frames_questions, retrieval_mapper=map_frames_retrieval, retrieval_evaluators=[MAPEvaluator()], citation_evaluator=CitationMAPEvaluator(), diff --git a/evaluations/tests/test_datasets.py b/evaluations/tests/test_datasets.py index 61f96bc3..947ff42f 100644 --- a/evaluations/tests/test_datasets.py +++ b/evaluations/tests/test_datasets.py @@ -11,6 +11,7 @@ from evaluations.datasets.frames import ( normalize_wiki_url, parse_revid, parse_wiki_links, + question_is_answerable, strip_navigation, ) from evaluations.datasets.hotpotqa import ( @@ -675,7 +676,7 @@ class TestFrames: monkeypatch.setattr(frames, "_cached_corpus", None) monkeypatch.setattr( frames, - "load_frames_test", + "load_frames_questions", lambda: [ { "wiki_links": "['https://en.wikipedia.org/wiki/A', " @@ -688,3 +689,12 @@ class TestFrames: ) with pytest.raises(RuntimeError, match="0/2"): frames.load_frames_corpus() + + def test_question_with_deleted_article_is_excluded(self) -> None: + gone = { + "wiki_links": "['https://en.wikipedia.org/wiki/Jack_Vance_(tennis)', " + "'https://en.wikipedia.org/wiki/Capybara']" + } + kept = {"wiki_links": "['https://en.wikipedia.org/wiki/Capybara']"} + assert question_is_answerable(gone) is False + assert question_is_answerable(kept) is True From a57a73b6d0ee578e20718aae6d1d1ebaba2b069a Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Sun, 26 Jul 2026 12:29:35 +0300 Subject: [PATCH 4/8] Add stable ids to FRAMES question rows --- evaluations/evaluations/datasets/frames.py | 7 +++++-- evaluations/tests/test_datasets.py | 24 +++++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/evaluations/evaluations/datasets/frames.py b/evaluations/evaluations/datasets/frames.py index 7f019acc..4b54874e 100644 --- a/evaluations/evaluations/datasets/frames.py +++ b/evaluations/evaluations/datasets/frames.py @@ -52,7 +52,9 @@ def question_is_answerable(doc: Mapping[str, Any]) -> bool: def load_frames_questions() -> Dataset: - return load_frames_test().filter(question_is_answerable) + """Answerable questions with a stable `id` (the dataset row number).""" + dataset = load_frames_test().filter(question_is_answerable) + return dataset.map(lambda row: {"id": str(row["Unnamed: 0"])}) def parse_wiki_links(raw: str) -> list[str]: @@ -304,10 +306,11 @@ def build_frames_case( index: int, doc: Mapping[str, Any] ) -> Case[str, str, dict[str, str]]: return Case( - name=f"{index}", + name=f"{index}_{doc['id']}", inputs=doc["Prompt"], expected_output=doc["Answer"], metadata={ + "question_id": str(doc["id"]), "reasoning_types": str(doc["reasoning_types"]), "case_index": str(index), }, diff --git a/evaluations/tests/test_datasets.py b/evaluations/tests/test_datasets.py index 947ff42f..fbaac78c 100644 --- a/evaluations/tests/test_datasets.py +++ b/evaluations/tests/test_datasets.py @@ -500,15 +500,17 @@ class TestFrames: def test_build_case(self) -> None: row = { + "id": "7", "Prompt": "Who was the 15th president?", "Answer": "James Buchanan", "reasoning_types": "Multiple constraints | Temporal reasoning", } case = build_frames_case(3, row) - assert case.name == "3" + assert case.name == "3_7" assert case.inputs == "Who was the 15th president?" assert case.expected_output == "James Buchanan" assert case.metadata == { + "question_id": "7", "reasoning_types": "Multiple constraints | Temporal reasoning", "case_index": "3", } @@ -698,3 +700,23 @@ class TestFrames: kept = {"wiki_links": "['https://en.wikipedia.org/wiki/Capybara']"} assert question_is_answerable(gone) is False assert question_is_answerable(kept) is True + + def test_questions_carry_stable_ids(self, monkeypatch) -> None: + import evaluations.datasets.frames as frames + from datasets import Dataset + + rows = Dataset.from_list( + [ + { + "Unnamed: 0": 7, + "wiki_links": "['https://en.wikipedia.org/wiki/Capybara']", + }, + { + "Unnamed: 0": 8, + "wiki_links": "['https://en.wikipedia.org/wiki/Jack_Vance_(tennis)']", + }, + ] + ) + monkeypatch.setattr(frames, "load_frames_test", lambda: rows) + questions = frames.load_frames_questions() + assert [row["id"] for row in questions] == ["7"] From 5e10846292d29804847d872a7a2044e6f10d7c4d Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Sun, 26 Jul 2026 18:56:00 +0300 Subject: [PATCH 5/8] Raise FRAMES input budget: qa max_tokens 8192, judge 16384 --- evaluations/configs/frames.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/evaluations/configs/frames.yaml b/evaluations/configs/frames.yaml index 7c20fad4..f9339aa9 100644 --- a/evaluations/configs/frames.yaml +++ b/evaluations/configs/frames.yaml @@ -28,7 +28,9 @@ qa: provider: openai name: gemma4-26b base_url: http://vllm:11432/v1 - max_tokens: 49152 + # vLLM reserves max_tokens out of max_model_len; a large value starves + # the input budget and 400s long agentic contexts. + max_tokens: 8192 evaluations: judge: @@ -36,4 +38,4 @@ evaluations: name: RedHatAI/Qwen3.6-35B-A3B-NVFP4 base_url: http://vllm:11430/v1 temperature: 0.0 - max_tokens: 4096 + max_tokens: 16384 From bc6c23cbf3586fa62d1ad4c84df36742b8e63f9a Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 27 Jul 2026 09:51:42 +0300 Subject: [PATCH 6/8] Bound FRAMES analysis sandbox output at 20k chars --- evaluations/configs/frames.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/evaluations/configs/frames.yaml b/evaluations/configs/frames.yaml index f9339aa9..946eeb5a 100644 --- a/evaluations/configs/frames.yaml +++ b/evaluations/configs/frames.yaml @@ -23,6 +23,11 @@ reranking: name: Qwen/Qwen3-Reranker-4B base_url: http://vllm:11433 +analysis: + # Bounds per-execution sandbox output so accumulated code returns cannot + # outgrow the model's input budget. + max_output_chars: 20000 + qa: model: provider: openai From 4df31ff16e8c15037d76fcf662ec2a0cb76146d9 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 30 Jul 2026 09:34:36 +0300 Subject: [PATCH 7/8] Point FRAMES at the live reranker and give the judge room to think Port 11433 serves nothing; Qwen3-Reranker-4B is on 11455. Qwen3.6 spends its budget reasoning before it answers, and its 131072 window leaves ample input space at 32768. --- evaluations/configs/frames.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/evaluations/configs/frames.yaml b/evaluations/configs/frames.yaml index 946eeb5a..51f0b647 100644 --- a/evaluations/configs/frames.yaml +++ b/evaluations/configs/frames.yaml @@ -21,7 +21,7 @@ reranking: model: provider: vllm name: Qwen/Qwen3-Reranker-4B - base_url: http://vllm:11433 + base_url: http://vllm:11455 analysis: # Bounds per-execution sandbox output so accumulated code returns cannot @@ -40,7 +40,13 @@ qa: evaluations: judge: provider: openai - name: RedHatAI/Qwen3.6-35B-A3B-NVFP4 - base_url: http://vllm:11430/v1 - temperature: 0.0 + name: Inferact/Qwen3.8-27B-NVFP4 + base_url: http://vllm:11439/v1 + temperature: 0.6 max_tokens: 16384 + extra_body: + top_p: 0.95 + top_k: 20 + min_p: 0 + chat_template_kwargs: + reasoning_effort: low From ab910eb776a8d2bfdfa4fc0ae788c1b37605b51d Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 24 Aug 2026 08:43:39 +0300 Subject: [PATCH 8/8] Add FRAMES to the benchmarks page --- docs/benchmarks.md | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/docs/benchmarks.md b/docs/benchmarks.md index fec6ecb7..fc69b6aa 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -1,6 +1,6 @@ # Benchmarks -We evaluate `haiku.rag` on a small set of datasets that exercise different parts of the pipeline. OpenRAG Bench (ORB), T²-RAGBench, HotpotQA, and MTRAG are the datasets we currently track. Retrieval, QA accuracy, and citation retrieval are scored end-to-end through the RAG and analysis capabilities. +We evaluate `haiku.rag` on a small set of datasets that exercise different parts of the pipeline. OpenRAG Bench (ORB), T²-RAGBench, HotpotQA, FRAMES, and MTRAG are the datasets we currently track. Retrieval, QA accuracy, and citation retrieval are scored end-to-end through the RAG and analysis capabilities. ## Current results @@ -100,6 +100,27 @@ The reranker's contribution is larger here than on the single-doc datasets: hybr *Measured on haiku.rag v0.66.0 with `qwen3-embedding:4b` (vLLM, dim 2560), judged by `vllm:Qwen3.6-35B-A3B-NVFP4`, 7,405 cases. The reranker lifts QA accuracy +2.7pts and `cited_map` +4.6pts. Without a reranker, `cited_map` (0.75) still exceeds the no-reranker retrieval MAP (0.70): the skill reformulates queries across search calls, partially recovering second-hop documents that a single query misses.* +### FRAMES + +[FRAMES](https://huggingface.co/datasets/google/frames-benchmark) is Google's multi-hop QA benchmark: 824 questions, each grounded in 2–23 Wikipedia articles, exercising temporal, numerical, and tabular reasoning across documents. We evaluate 822 questions (2 excluded: a linked article was deleted from Wikipedia) over a fixed corpus of the 2,521 linked articles fetched at current revision. There is no official FRAMES evaluation setup; our protocol — fixed corpus, agentic retrieval, judged accuracy — corresponds to the paper's *multi-step retrieval* setting, where [the paper](https://arxiv.org/abs/2409.12941) reports 0.66 with Gemini-Pro-1.5 (0.729 in its oracle setting, with gold articles provided). Answers were authored against ~2024 revisions and may have drifted with article content. + +##### Retrieval (MAP) + +| Embedding Model | Reranker | Cases | MAP | +|----------------------|---------------------|------:|-------:| +| `qwen3-embedding:4b` | `Qwen3-Reranker-4B` | 822 | 0.5631 | + +*Single-query retrieval is capped by FRAMES' indirection: in the zero-MAP queries the gold article's subject is never named in the question ("the year the Titanic sank" → `1912_Summer_Olympics`). The agentic targets recover these through iterative search, passing 55% of the very cases single-shot retrieval scores zero on.* + +##### QA accuracy + citation retrieval + +| Capability model | Target | QA accuracy | Mean `cited_map` | +|------------------|--------|-------------|------------------| +| `vllm:Muse-Glimmer-30B-NVFP4` | `analysis-capability` | 0.7506 | 0.5852 | +| `vllm:Qwen3.8-27B-NVFP4` | `analysis-capability` | 0.8095 | 0.6847 | + +*Both rows use `qwen3-embedding:4b` (vLLM, dim 2560), `Qwen3-Reranker-4B`, and are judged by `Qwen3.8-27B`. QA accuracy is over judged cases; counting unanswered cases as failures, the floors are 0.7397 (`Muse-Glimmer`, 1.5% lost to provider errors) and 0.7701 (`Qwen3.8`, 4.87% lost to answers truncated at `max_tokens: 16384`). The `Qwen3.8` row is self-judged — a 100-case paired cross-judge (Glimmer as judge, difference-in-differences) measured the self-preference at +1.0pp with 99% judge agreement. Cite rates are 90.1% (`Muse-Glimmer`) and 99.3% (`Qwen3.8`); `cited_map` is structurally capped below 1 on FRAMES because gold sets span 2–23 articles while answering typically uses a subset. `Qwen3.8` reaches its score on substantially less tool traffic than `Muse-Glimmer` (4.0 searches and 3.9 code executions per case vs 7.0 and 9.5, measured identically from tool spans).* + ### MTRAG (ClapNQ) [MTRAG](https://github.com/IBM/mt-rag-benchmark) is IBM's multi-turn RAG benchmark (TACL 2025, SemEval-2026 Task 8): human-authored conversations with per-turn answerability labels and binary relevance judgments. We evaluate the ClapNQ (Wikipedia) domain: 183,408 passages, 29 conversations, 224 turns, 208 retrieval queries.