From 3183cc0ad71f763f1bdf50e9dd3cf49e69eb272c Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 31 Aug 2026 16:05:15 +0300 Subject: [PATCH] Vacuum each built collection and assert FTS coverage The chunks FTS index is created once when the table is created, over zero rows, and nothing folds later rows into it but an optimize. build_databases bypasses populate_db, and with it the closing vacuum, so every database it built had an index covering nothing. Full-text search then returns near-arbitrary rows while still returning results, so nothing looks wrong: measured FTS recall@5 of 0.0000 across 208 queries on a 40k pool, with hybrid losing 9.9pp to pure vector because the dead branch still consumed half the fused slots. Reachable outside the eval too, on any storage.auto_vacuum: false, which every reference config sets. Reproduced through create_document alone: five documents, five chunks, zero indexed rows. Claude-Session: https://claude.ai/code/session_01WhudUtZm6qqiuv8Y1sbwSc --- CHANGELOG.md | 4 ++ .../evaluations/datasets/mtrag_federated.py | 27 +++++++++++ evaluations/tests/test_mtrag_federated.py | 48 +++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbefbdb9..6cba3c29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Fixed + +- `mtrag_federated` builds vacuum each collection after ingest and assert the chunks FTS index covers every row. Without the vacuum the index stays at zero rows, and full-text search returns near-arbitrary rows while still returning results. + ### Added - `evaluations run --retrieval-limit N`: candidates each database fetches during the retrieval benchmark, overriding the dataset's `retrieval_limit`. diff --git a/evaluations/evaluations/datasets/mtrag_federated.py b/evaluations/evaluations/datasets/mtrag_federated.py index c1e77dc7..c75a0c05 100644 --- a/evaluations/evaluations/datasets/mtrag_federated.py +++ b/evaluations/evaluations/datasets/mtrag_federated.py @@ -37,6 +37,7 @@ DEFAULT_SEED = 20260831 GOLD_TITLE_FLOOR = 10_723 DEFAULT_BUDGET = 40_000 INGEST_BATCH_SIZE = 512 +FTS_INDEX_NAME = "content_fts_idx" def collection_of(title: str, n: int, seed: int = DEFAULT_SEED) -> int: @@ -172,6 +173,25 @@ def emitted_config(reference: Path, n: int, seed: int = DEFAULT_SEED) -> dict[st return settings +class FTSIndexNotCoveringRows(AssertionError): + """The chunks FTS index does not cover every row, so full-text search is + dead while still returning results.""" + + +async def assert_fts_covers_rows(table: Any, name: str) -> None: + rows = await table.count_rows() + indices = {index.name for index in await table.list_indices()} + if FTS_INDEX_NAME not in indices: + raise FTSIndexNotCoveringRows(f"{name}: no {FTS_INDEX_NAME} on {rows} rows") + stats = await table.index_stats(FTS_INDEX_NAME) + indexed = getattr(stats, "num_indexed_rows", 0) or 0 + if indexed < rows: + raise FTSIndexNotCoveringRows( + f"{name}: {FTS_INDEX_NAME} covers {indexed} of {rows} rows; " + "full-text search would return near-arbitrary rows" + ) + + async def build_databases( config: AppConfig, n: int, @@ -213,6 +233,13 @@ async def build_databases( await _ingest_batched( client, MTRAG_FEDERATED_SPEC, grouped[name], INGEST_BATCH_SIZE ) + # The chunks FTS index is built once when the table is created, over + # zero rows, and nothing folds later rows into it but an optimize. + # `auto_vacuum` is false here, as in every reference config, so + # without this the index covers nothing and full-text search returns + # near-arbitrary rows while still looking like it works. + await client.store.vacuum(retention_seconds=0) + await assert_fts_covers_rows(client.store.chunks_table, name) written[name] = len(grouped[name]) return written diff --git a/evaluations/tests/test_mtrag_federated.py b/evaluations/tests/test_mtrag_federated.py index 2368ada5..593ec390 100644 --- a/evaluations/tests/test_mtrag_federated.py +++ b/evaluations/tests/test_mtrag_federated.py @@ -7,8 +7,10 @@ import pytest from evaluations.datasets import DATASETS from evaluations.datasets.mtrag_federated import ( DEFAULT_BUDGET, + FTSIndexNotCoveringRows, GOLD_TITLE_FLOOR, MTRAG_FEDERATED_SPEC, + assert_fts_covers_rows, collection_names, collection_of, partition_records, @@ -248,3 +250,49 @@ class TestRetrievalLimitOverride: await run_retrieval_benchmark(MTRAG_FEDERATED_SPEC, _smoke_config(), limit=1) assert seen and set(seen) == {MTRAG_FEDERATED_SPEC.retrieval_limit} + + +class TestFTSCoverageAssertion: + """The chunks FTS index is built once over zero rows and only an optimize + folds later rows in, so a build that skips it ships dead full-text search + that still returns results.""" + + class _Index: + def __init__(self, name: str) -> None: + self.name = name + + class _Stats: + def __init__(self, indexed: int) -> None: + self.num_indexed_rows = indexed + + class _Table: + def __init__(self, rows: int, indexed: int | None) -> None: + self._rows = rows + self._indexed = indexed + + async def count_rows(self) -> int: + return self._rows + + async def list_indices(self): + if self._indexed is None: + return [] + return [TestFTSCoverageAssertion._Index("content_fts_idx")] + + async def index_stats(self, name: str): + assert name == "content_fts_idx" + return TestFTSCoverageAssertion._Stats(self._indexed or 0) + + async def test_passes_when_the_index_covers_every_row(self) -> None: + await assert_fts_covers_rows(self._Table(100, 100), "clapnq_0") + + async def test_rejects_a_zero_row_index(self) -> None: + with pytest.raises(FTSIndexNotCoveringRows, match="covers 0 of 100"): + await assert_fts_covers_rows(self._Table(100, 0), "clapnq_0") + + async def test_rejects_a_partially_covering_index(self) -> None: + with pytest.raises(FTSIndexNotCoveringRows, match="covers 60 of 100"): + await assert_fts_covers_rows(self._Table(100, 60), "clapnq_0") + + async def test_rejects_a_missing_index(self) -> None: + with pytest.raises(FTSIndexNotCoveringRows, match="no content_fts_idx"): + await assert_fts_covers_rows(self._Table(100, None), "clapnq_0")