Add --filter-ids to run QA on a case-id subset

CHANGELOG.md

@@ -9,6 +9,7 @@
 
 - `analysis.max_executions` (default 15): caps `execute_code` calls per analysis question. Past the cap the tool returns a notice telling the skill to answer from what it has, instead of spiralling into `request_limit` and returning nothing. The analysis skill sets `request_limit` to 30 as a backstop.
 - `t2_finqa` and `t2_tatdqa` evaluation datasets (T²-RAGBench subsets, `G4KMU/t2-ragbench`): financial-report PDFs ingested via docling with `uri = context_id` and gold retrieval keyed on `context_id`. QA is scored with a deterministic `NumberMatchEvaluator` (relative tolerance 0.01) via the new `DatasetSpec.qa_evaluator`, bypassing the LLM judge.
+- `evaluations run --filter-ids <file>`: run QA on just the case ids listed in a file (failure-subset rerun); retrieval is unaffected.
 
 ### Fixed
 

docs/benchmarks.md

@@ -1,6 +1,6 @@
 # Benchmarks
 
-We evaluate `haiku.rag` on a small set of datasets that exercise different parts of the pipeline. Wix and OpenRAG Bench (ORB) are the two we currently track. Retrieval, QA accuracy, and citation retrieval are scored end-to-end through the rag and rag-analysis skills.
+We evaluate `haiku.rag` on a small set of datasets that exercise different parts of the pipeline. Wix, OpenRAG Bench (ORB), and T²-RAGBench are the datasets we currently track. Retrieval, QA accuracy, and citation retrieval are scored end-to-end through the rag and rag-analysis skills.
 
 ## Running Evaluations
 
@@ -151,6 +151,18 @@ Two approaches are benchmarked separately:
 
 *Measured on haiku.rag v0.50.0 with `mxbai-rerank-base-v2`, judged by `vllm:Qwen3.6-35B-A3B-NVFP4`. Nemotron covered 2836 / 3045 cases.*
 
+### T²-RAGBench (FinQA)
+
+[T²-RAGBench](https://huggingface.co/datasets/G4KMU/t2-ragbench) reformulates financial-report QA into context-independent questions with short numeric answers and a 1:1 gold document mapping. The FinQA subset is 2,789 single-page PDFs / 8,281 questions, ingested via docling. Unlike the other datasets, QA is scored deterministically with `NumberMatchEvaluator` (relative tolerance 0.01) instead of an LLM judge, so QA accuracy here is exact numeric match rather than a judged fraction.
+
+##### QA accuracy + citation retrieval
+
+| Embedding Model      | Reranker               | Target           | Skill model                  | Cases | QA accuracy | Mean `cited_map` |
+|----------------------|------------------------|------------------|------------------------------|------:|-------------|------------------|
+| `qwen3-embedding:4b` | `mxbai-rerank-base-v2` | `analysis-skill` | `vllm:Qwen3.6-35B-A3B-NVFP4` |  7939 | 0.77        | 0.78             |
+
+*Measured on haiku.rag v0.55.0, deterministic Number-Match scoring (ε=0.01), 2560-dim `qwen3-embedding:4b` (vLLM) with `mxbai-rerank-base-v2`. 341 / 8281 cases excluded as nulls (analysis spirals from the request limit and in-generation loops). Accuracy and `cited_map` are over the 7939 scored cases. Mean 16.0s/case.*
+
 ### Wix
 
 [WixQA](https://huggingface.co/datasets/Wix/WixQA) is real customer support questions paired with curated answers. 200 cases.

evaluations/evaluations/benchmark.py

@@ -334,6 +334,16 @@ def _attach_relevant_uris(
         case.metadata = metadata
 
 
+def _filter_qa_corpus(corpus, case_ids: set[str] | None):
+    """Keep only rows whose ``id`` is in ``case_ids`` (failure-subset reruns).
+
+    Returns the corpus unchanged when ``case_ids`` is None.
+    """
+    if case_ids is None:
+        return corpus
+    return corpus.filter(lambda row: row.get("id") in case_ids)
+
+
 async def run_qa_benchmark(
     spec: DatasetSpec,
     config: AppConfig,
@@ -343,8 +353,10 @@ async def run_qa_benchmark(
     judge_model: ModelConfig | None = None,
     target: Target = "rag-skill",
     skill_model: ModelConfig | None = None,
+    case_ids: set[str] | None = None,
 ) -> ReportCaseFailure[str, str, dict[str, str]] | None:
     corpus = spec.qa_loader()
+    corpus = _filter_qa_corpus(corpus, case_ids)
     if limit is not None:
         corpus = corpus.select(range(min(limit, len(corpus))))
 
@@ -499,6 +511,7 @@ async def evaluate_dataset(
     judge_model: ModelConfig | None = None,
     target: Target = "rag-skill",
     skill_model: ModelConfig | None = None,
+    case_ids: set[str] | None = None,
 ) -> None:
     if not skip_db:
         console.print(f"Using dataset: {spec.key}", style="bold magenta")
@@ -530,6 +543,7 @@ async def evaluate_dataset(
             judge_model=judge_model,
             target=target,
             skill_model=skill_model,
+            case_ids=case_ids,
         )
 
 
@@ -555,6 +569,13 @@ def _load_config(config_path: Path | None) -> AppConfig:
     return AppConfig()
 
 
+def _load_case_ids(path: Path | None) -> set[str] | None:
+    """Read a newline-delimited case-id file into a set (None when no path)."""
+    if path is None:
+        return None
+    return {line.strip() for line in path.read_text().splitlines() if line.strip()}
+
+
 def _resolve_dataset(dataset: str) -> DatasetSpec:
     """Resolve a dataset key to a DatasetSpec or raise BadParameter."""
     spec = DATASETS.get(dataset.lower())
@@ -612,6 +633,14 @@ def run(
             "analysis.model when --target is analysis-skill) from the config."
         ),
     ),
+    filter_ids: Path | None = typer.Option(
+        None,
+        "--filter-ids",
+        help=(
+            "Path to a newline-delimited file of QA case ids to run "
+            "(failure-subset rerun). Filters QA only; retrieval is unaffected."
+        ),
+    ),
 ) -> None:
     spec = _resolve_dataset(dataset)
     app_config = _load_config(config)
@@ -638,6 +667,7 @@ def run(
             judge_model=judge_model_config,
             target=target_value,
             skill_model=skill_model_config,
+            case_ids=_load_case_ids(filter_ids),
         )
     )
 

evaluations/tests/test_benchmark.py

@@ -406,3 +406,71 @@ class TestAttachRelevantUris:
         )
         _attach_relevant_uris(cases, spec, limit=None)
         assert cases[0].metadata is None
+
+
+class TestFilterQaCorpus:
+    def test_keeps_only_matching_ids(self) -> None:
+        from datasets import Dataset
+
+        from evaluations.benchmark import _filter_qa_corpus
+
+        corpus = Dataset.from_list(
+            [{"id": "a", "q": 1}, {"id": "b", "q": 2}, {"id": "c", "q": 3}]
+        )
+        out = _filter_qa_corpus(corpus, {"a", "c"})
+        assert [r["id"] for r in out] == ["a", "c"]
+
+    def test_none_returns_corpus_unchanged(self) -> None:
+        from datasets import Dataset
+
+        from evaluations.benchmark import _filter_qa_corpus
+
+        corpus = Dataset.from_list([{"id": "a"}])
+        assert _filter_qa_corpus(corpus, None) is corpus
+
+
+class TestLoadCaseIds:
+    def test_reads_strips_and_drops_blanks(self, tmp_path: Path) -> None:
+        from evaluations.benchmark import _load_case_ids
+
+        f = tmp_path / "ids.txt"
+        f.write_text("finqa_dev_16\n  finqa_dev_66 \n\n\nfinqa_dev_113\n")
+        assert _load_case_ids(f) == {"finqa_dev_16", "finqa_dev_66", "finqa_dev_113"}
+
+    def test_none_path_returns_none(self) -> None:
+        from evaluations.benchmark import _load_case_ids
+
+        assert _load_case_ids(None) is None
+
+
+class TestEvaluateDatasetCaseIds:
+    def _spec(self) -> DatasetSpec:
+        return DatasetSpec(
+            key="test",
+            db_filename="test.lancedb",
+            document_loader=lambda: None,  # type: ignore[arg-type]  # ty: ignore[invalid-argument-type]
+            document_mapper=lambda doc: None,
+            qa_loader=lambda: [],  # type: ignore[arg-type]  # ty: ignore[invalid-argument-type]
+            qa_case_builder=lambda idx, doc: None,  # type: ignore[arg-type]  # ty: ignore[invalid-argument-type]
+        )
+
+    @pytest.mark.asyncio
+    async def test_threads_case_ids_to_qa_benchmark(self) -> None:
+        from evaluations.benchmark import evaluate_dataset
+
+        with patch(
+            "evaluations.benchmark.run_qa_benchmark", new_callable=AsyncMock
+        ) as mock_qa:
+            await evaluate_dataset(
+                spec=self._spec(),
+                config=AppConfig(),
+                skip_db=True,
+                skip_retrieval=True,
+                skip_qa=False,
+                limit=None,
+                name=None,
+                db_path=None,
+                case_ids={"finqa_dev_16", "finqa_dev_66"},
+            )
+        mock_qa.assert_called_once()
+        assert mock_qa.call_args[1]["case_ids"] == {"finqa_dev_16", "finqa_dev_66"}