Vacuum each built collection and assert FTS coverage

The chunks FTS index is created once when the table is created, over zero
rows, and nothing folds later rows into it but an optimize. build_databases
bypasses populate_db, and with it the closing vacuum, so every database it
built had an index covering nothing.

Full-text search then returns near-arbitrary rows while still returning
results, so nothing looks wrong: measured FTS recall@5 of 0.0000 across 208
queries on a 40k pool, with hybrid losing 9.9pp to pure vector because the
dead branch still consumed half the fused slots.

Reachable outside the eval too, on any storage.auto_vacuum: false, which
every reference config sets. Reproduced through create_document alone:
five documents, five chunks, zero indexed rows.

Claude-Session: https://claude.ai/code/session_01WhudUtZm6qqiuv8Y1sbwSc

CHANGELOG.md

@@ -18,6 +18,10 @@
 
 - `dot` removed from `search.vector_index_metric`; switch to `cosine` or `l2`
   and rerun `create-index`.
+### Fixed
+
+- `mtrag_federated` builds vacuum each collection after ingest and assert the chunks FTS index covers every row. Without the vacuum the index stays at zero rows, and full-text search returns near-arbitrary rows while still returning results.
+
 ### Added
 
 - `evaluations run --retrieval-limit N`: candidates each database fetches during the retrieval benchmark, overriding the dataset's `retrieval_limit`.

evaluations/evaluations/datasets/mtrag_federated.py

@@ -37,6 +37,7 @@ DEFAULT_SEED = 20260831
 GOLD_TITLE_FLOOR = 10_723
 DEFAULT_BUDGET = 40_000
 INGEST_BATCH_SIZE = 512
+FTS_INDEX_NAME = "content_fts_idx"
 
 
 def collection_of(title: str, n: int, seed: int = DEFAULT_SEED) -> int:
@@ -172,6 +173,25 @@ def emitted_config(reference: Path, n: int, seed: int = DEFAULT_SEED) -> dict[st
     return settings
 
 
+class FTSIndexNotCoveringRows(AssertionError):
+    """The chunks FTS index does not cover every row, so full-text search is
+    dead while still returning results."""
+
+
+async def assert_fts_covers_rows(table: Any, name: str) -> None:
+    rows = await table.count_rows()
+    indices = {index.name for index in await table.list_indices()}
+    if FTS_INDEX_NAME not in indices:
+        raise FTSIndexNotCoveringRows(f"{name}: no {FTS_INDEX_NAME} on {rows} rows")
+    stats = await table.index_stats(FTS_INDEX_NAME)
+    indexed = getattr(stats, "num_indexed_rows", 0) or 0
+    if indexed < rows:
+        raise FTSIndexNotCoveringRows(
+            f"{name}: {FTS_INDEX_NAME} covers {indexed} of {rows} rows; "
+            "full-text search would return near-arbitrary rows"
+        )
+
+
 async def build_databases(
     config: AppConfig,
     n: int,
@@ -213,6 +233,13 @@ async def build_databases(
             await _ingest_batched(
                 client, MTRAG_FEDERATED_SPEC, grouped[name], INGEST_BATCH_SIZE
             )
+            # The chunks FTS index is built once when the table is created, over
+            # zero rows, and nothing folds later rows into it but an optimize.
+            # `auto_vacuum` is false here, as in every reference config, so
+            # without this the index covers nothing and full-text search returns
+            # near-arbitrary rows while still looking like it works.
+            await client.store.vacuum(retention_seconds=0)
+            await assert_fts_covers_rows(client.store.chunks_table, name)
         written[name] = len(grouped[name])
     return written
 

evaluations/tests/test_mtrag_federated.py

@@ -7,8 +7,10 @@ import pytest
 from evaluations.datasets import DATASETS
 from evaluations.datasets.mtrag_federated import (
     DEFAULT_BUDGET,
+    FTSIndexNotCoveringRows,
     GOLD_TITLE_FLOOR,
     MTRAG_FEDERATED_SPEC,
+    assert_fts_covers_rows,
     collection_names,
     collection_of,
     partition_records,
@@ -248,3 +250,49 @@ class TestRetrievalLimitOverride:
 
         await run_retrieval_benchmark(MTRAG_FEDERATED_SPEC, _smoke_config(), limit=1)
         assert seen and set(seen) == {MTRAG_FEDERATED_SPEC.retrieval_limit}
+
+
+class TestFTSCoverageAssertion:
+    """The chunks FTS index is built once over zero rows and only an optimize
+    folds later rows in, so a build that skips it ships dead full-text search
+    that still returns results."""
+
+    class _Index:
+        def __init__(self, name: str) -> None:
+            self.name = name
+
+    class _Stats:
+        def __init__(self, indexed: int) -> None:
+            self.num_indexed_rows = indexed
+
+    class _Table:
+        def __init__(self, rows: int, indexed: int | None) -> None:
+            self._rows = rows
+            self._indexed = indexed
+
+        async def count_rows(self) -> int:
+            return self._rows
+
+        async def list_indices(self):
+            if self._indexed is None:
+                return []
+            return [TestFTSCoverageAssertion._Index("content_fts_idx")]
+
+        async def index_stats(self, name: str):
+            assert name == "content_fts_idx"
+            return TestFTSCoverageAssertion._Stats(self._indexed or 0)
+
+    async def test_passes_when_the_index_covers_every_row(self) -> None:
+        await assert_fts_covers_rows(self._Table(100, 100), "clapnq_0")
+
+    async def test_rejects_a_zero_row_index(self) -> None:
+        with pytest.raises(FTSIndexNotCoveringRows, match="covers 0 of 100"):
+            await assert_fts_covers_rows(self._Table(100, 0), "clapnq_0")
+
+    async def test_rejects_a_partially_covering_index(self) -> None:
+        with pytest.raises(FTSIndexNotCoveringRows, match="covers 60 of 100"):
+            await assert_fts_covers_rows(self._Table(100, 60), "clapnq_0")
+
+    async def test_rejects_a_missing_index(self) -> None:
+        with pytest.raises(FTSIndexNotCoveringRows, match="no content_fts_idx"):
+            await assert_fts_covers_rows(self._Table(100, None), "clapnq_0")