diff --git a/CHANGELOG.md b/CHANGELOG.md index bd166272..a4eba9af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ - `--full-citations` on `haiku-rag ask` and `haiku-rag analyze` renders citation text untruncated. `format_citations_rich` takes a `full` argument. +- `doctor` fails when the chunks FTS index covers no rows. +- FTS and hybrid searches log a warning when the FTS index covers no rows. ### Removed diff --git a/docs/cli.md b/docs/cli.md index e300a8ed..cd1fba8e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -329,6 +329,7 @@ Checks include: - the configured embedding identity matches the stored settings - no database migrations are pending - the vector index covers all chunks +- the full-text index covers the chunks it searches - near-identical documents (by embedding-centroid similarity) are grouped and reported, with the largest member flagged as the likely one to keep (advisory only, never deleted, tuned via `doctor.duplicates` in config) - API keys are set for configured providers @@ -340,7 +341,7 @@ It also probes the external endpoints the config uses and reports them under a P SaaS providers (OpenAI, Anthropic, Cohere, Jina, ZeroEntropy, Voyage) are covered by the API-key check rather than a network probe. In-process local models (sentence-transformers, cross-encoder, jina-local) have no endpoint and are reported as such. -Each failure prints the command that fixes it (`rebuild`, `create-index`, `migrate`, `rebuild --set-embedder`). `doctor` makes no changes. It exits with status 1 when any check fails, so it can gate CI or monitoring. +Each failure prints the command that fixes it (`rebuild`, `create-index`, `vacuum`, `migrate`, `rebuild --set-embedder`). `doctor` makes no changes. It exits with status 1 when any check fails, so it can gate CI or monitoring. ### Migrate Database diff --git a/haiku_rag_slim/haiku/rag/doctor.py b/haiku_rag_slim/haiku/rag/doctor.py index e6708456..3b004395 100644 --- a/haiku_rag_slim/haiku/rag/doctor.py +++ b/haiku_rag_slim/haiku/rag/doctor.py @@ -17,7 +17,7 @@ from haiku.rag.config.models import ( ) from haiku.rag.store.engine import Store, connect_lancedb from haiku.rag.store.info import get_database_stats -from haiku.rag.store.schema import REQUIRED_TABLES +from haiku.rag.store.schema import REQUIRED_TABLES, index_specs from haiku.rag.store.upgrades import get_pending_upgrades # Cap how many offending ids we collect per check; doctor is a summary, not a dump. @@ -225,6 +225,56 @@ def _classify_unchunked( return results +async def _check_fts_coverage(store: Store) -> CheckResult: + """An FTS index that covers no rows, and a populated table with no FTS + index at all, both make lance serve results unsorted by score with + matching rows dropped. optimize indexes the rows of an index that + exists; it never creates one that is absent.""" + from lancedb.index import FTS + + uncovered: list[str] = [] + missing: list[str] = [] + for table_name, table in store._tables().items(): + declared = [c for c, cfg in index_specs(table_name) if isinstance(cfg, FTS)] + if not declared: + continue + rows = await table.count_rows() + if not rows: + continue + indices = await table.list_indices() + for column in declared: + index = next( + (i for i in indices if column in i.columns and i.index_type == "FTS"), + None, + ) + if index is None: + missing.append(f"{table_name}.{column}: no index over {rows} rows") + continue + stats = await table.index_stats(index.name) + if stats is None or stats.num_indexed_rows == 0: + uncovered.append(f"{table_name}.{column}: 0 of {rows} rows indexed") + if missing or uncovered: + return CheckResult( + name="fts_index_coverage", + severity=Severity.FAIL, + message=( + "Full-text search index does not cover its rows; FTS and " + "hybrid results are unsorted and incomplete." + ), + remediation=( + "Run 'haiku-rag rebuild --embed-only' to build the index." + if missing + else "Run 'haiku-rag vacuum' to index the rows." + ), + details=missing + uncovered, + ) + return CheckResult( + name="fts_index_coverage", + severity=Severity.OK, + message="Full-text search indexes cover their tables.", + ) + + async def _column_values(table, column: str) -> list: rows = await table.query().select([column]).to_list() return [row[column] for row in rows] @@ -702,6 +752,9 @@ async def run_db_checks( self_refs_by_doc.setdefault(row["document_id"], set()).add(row["self_ref"]) labels_by_doc.setdefault(row["document_id"], set()).add(row["label"]) + notify("Checking index coverage") + results.append(await _check_fts_coverage(store)) + notify("Checking referential integrity") results.append(_check_document_meta_parity(doc_ids, meta_doc_ids)) results.append(_check_orphaned_chunks(chunk_doc_ids, doc_ids)) diff --git a/haiku_rag_slim/haiku/rag/store/repositories/chunk.py b/haiku_rag_slim/haiku/rag/store/repositories/chunk.py index 5ab1ba9c..d3a63a57 100644 --- a/haiku_rag_slim/haiku/rag/store/repositories/chunk.py +++ b/haiku_rag_slim/haiku/rag/store/repositories/chunk.py @@ -6,7 +6,6 @@ from uuid import uuid4 if TYPE_CHECKING: from lancedb.query import AsyncQueryBase -from lancedb.index import FTS from lancedb.rerankers import RRFReranker from haiku.rag.store.engine import Store @@ -23,17 +22,49 @@ class ChunkRepository: def __init__(self, store: Store) -> None: self.store = store self.embedder = store.embedder + self._fts_coverage_checked = False - async def _ensure_fts_index(self) -> None: - """Ensure FTS index exists on the content_fts column.""" + async def _warn_if_fts_uncovered(self) -> None: + """An FTS index covering no rows makes lance serve a broken scan path: + results unsorted by score, with matching documents dropped. Checked + on the first search that uses the index, once per repository after + the table holds rows.""" + if self._fts_coverage_checked: + return try: - await self.store.chunks_table.create_index( - "content_fts", - config=FTS(with_position=True, remove_stop_words=False), - replace=True, + indices = await self.store.chunks_table.list_indices() + index = next( + ( + i + for i in indices + if "content_fts" in i.columns and i.index_type == "FTS" + ), + None, ) - except Exception as e: - logger.warning(f"FTS index build failed; full-text search degraded: {e}") + stats = ( + await self.store.chunks_table.index_stats(index.name) if index else None + ) + if stats is not None and stats.num_indexed_rows > 0: + self._fts_coverage_checked = True + return + # An empty table proves nothing; check again once it has rows. + if not await self.store.chunks_table.count_rows(): + return + except Exception: + self._fts_coverage_checked = True + logger.debug("FTS coverage check failed", exc_info=True) + return + self._fts_coverage_checked = True + if index is None: + logger.warning( + "No full-text search index; FTS and hybrid results are " + "degraded. Run 'haiku-rag rebuild --embed-only'." + ) + return + logger.warning( + "Full-text search index covers 0 rows; FTS and hybrid results " + "are degraded. Run 'haiku-rag vacuum'." + ) def _contextualize_content(self, chunk: Chunk) -> str: """Generate contextualized content for FTS by prepending headings.""" @@ -240,6 +271,9 @@ class ChunkRepository: id_list = ", ".join(f"'{d}'" for d in docs_df["id"]) chunk_filter = f"document_id IN ({id_list})" + if search_type != "vector" and query.strip(): + await self._warn_if_fts_uncovered() + if search_type == "fts": results = self.store.chunks_table.query().nearest_to_text( query, columns="content_fts" diff --git a/tests/test_chunk.py b/tests/test_chunk.py index 242ba26b..ce091e5c 100644 --- a/tests/test_chunk.py +++ b/tests/test_chunk.py @@ -1,3 +1,5 @@ +from unittest.mock import patch + import pytest from haiku.rag.client import HaikuRAG @@ -477,8 +479,28 @@ async def test_chunk_content_fts(temp_db_path, metadata, content, expected_conte assert record["content_fts"] == expected_content_fts -async def test_ensure_fts_index_warns_on_failure(temp_db_path): - """A failed FTS index build is surfaced at WARNING, not swallowed silently.""" +async def _import_one(client) -> None: + from docling_core.types.doc.document import DoclingDocument + from docling_core.types.doc.labels import DocItemLabel + + doc = DoclingDocument(name="one") + doc.add_text(label=DocItemLabel.TEXT, text="a document about gardens") + await client.import_document( + doc, + [ + Chunk( + content="a document about gardens", + embedding=[0.1] * get_config().embeddings.model.vector_dim, + order=0, + ) + ], + uri="test://one", + ) + + +async def test_fts_search_warns_when_index_covers_no_rows(temp_db_path): + """A database whose FTS index predates its rows covers none of them; + searching that state warns, once.""" import logging from haiku.rag.store.repositories import chunk as chunk_module @@ -486,18 +508,119 @@ async def test_ensure_fts_index_warns_on_failure(temp_db_path): async with HaikuRAG( db_path=temp_db_path, config=get_config(), create=True ) as client: - repo = client.chunk_repository - - async def _boom(*_args, **_kwargs): - raise RuntimeError("index build failed") - - repo.store.chunks_table.create_index = _boom + await _import_one(client) with capture_logs(chunk_module.logger, logging.WARNING) as records: - await repo._ensure_fts_index() + await client.chunk_repository.search("gardens", search_type="fts") + await client.chunk_repository.search("gardens", search_type="fts") - assert [r for r in records if r.levelno == logging.WARNING] - assert any("index build failed" in r.getMessage() for r in records) + warned = [r for r in records if "covers 0 rows" in r.getMessage()] + assert len(warned) == 1 + + +async def test_fts_search_warns_when_index_is_missing(temp_db_path, monkeypatch): + import logging + + from lancedb.table import AsyncTable + + from haiku.rag.store.repositories import chunk as chunk_module + + original = AsyncTable.list_indices + + async def no_chunk_indices(self): + if self.name == "chunks": + return [] + return await original(self) + + async with HaikuRAG( + db_path=temp_db_path, config=get_config(), create=True + ) as client: + await _import_one(client) + monkeypatch.setattr(AsyncTable, "list_indices", no_chunk_indices) + + with capture_logs(chunk_module.logger, logging.WARNING) as records: + await client.chunk_repository.search("gardens", search_type="fts") + + warned = [ + r.getMessage() + for r in records + if "No full-text search index" in r.getMessage() + ] + assert len(warned) == 1 + assert "rebuild --embed-only" in warned[0] + assert "vacuum" not in warned[0] + + +async def test_fts_search_does_not_warn_when_index_covers_rows(temp_db_path): + import logging + + from haiku.rag.store.repositories import chunk as chunk_module + + async with HaikuRAG( + db_path=temp_db_path, config=get_config(), create=True + ) as client: + await _import_one(client) + await client.store.vacuum(retention_seconds=0) + + with capture_logs(chunk_module.logger, logging.WARNING) as records: + results = await client.chunk_repository.search("gardens", search_type="fts") + + assert results + assert not records + + +async def test_fts_coverage_check_failure_does_not_break_search(temp_db_path): + """The coverage check is a diagnostic: a metadata failure must not take + the search down with it.""" + from lancedb.table import AsyncTable + + async def boom(self): + raise RuntimeError("metadata unavailable") + + async with HaikuRAG( + db_path=temp_db_path, config=get_config(), create=True + ) as client: + await _import_one(client) + + with patch.object(AsyncTable, "list_indices", boom): + results = await client.chunk_repository.search("gardens", search_type="fts") + + assert results + + +async def test_fts_search_on_an_empty_table_does_not_suppress_later_warnings( + temp_db_path, +): + """An empty table proves nothing about coverage, so searching it must not + spend the once-per-repository check.""" + import logging + + from haiku.rag.store.repositories import chunk as chunk_module + + async with HaikuRAG( + db_path=temp_db_path, config=get_config(), create=True + ) as client: + await client.chunk_repository.search("gardens", search_type="fts") + await _import_one(client) + + with capture_logs(chunk_module.logger, logging.WARNING) as records: + await client.chunk_repository.search("gardens", search_type="fts") + + assert any("covers 0 rows" in r.getMessage() for r in records) + + +async def test_fts_search_does_not_warn_on_an_empty_table(temp_db_path): + import logging + + from haiku.rag.store.repositories import chunk as chunk_module + + async with HaikuRAG( + db_path=temp_db_path, config=get_config(), create=True + ) as client: + with capture_logs(chunk_module.logger, logging.WARNING) as records: + await client.chunk_repository.search("gardens", search_type="fts") + + assert not records @pytest.mark.vcr() diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 90282b38..eaa59ccb 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -81,18 +81,25 @@ async def _build_db( name: str = "test", vector_dim: int = VECTOR_DIM, stored_vector_dim: int | None = None, + fts_index: str | None = "covering", ): """Create a consistent single-document database without touching an embedder. ``stored_vector_dim`` records a different dimension in settings than the chunks table actually uses, to exercise the vector-dimension check. + ``fts_index`` places the chunks FTS index: "covering" builds it over the + rows, "empty" builds it before them, None never builds it. """ + from lancedb.index import FTS + db = await lancedb.connect_async(path) settings_tbl = await db.create_table("settings", schema=SettingsRecord) docs_tbl = await db.create_table("documents", schema=DocumentRecord) meta_tbl = await db.create_table("document_meta", schema=DocumentMetaRecord) chunks_tbl = await db.create_table("chunks", schema=create_chunk_model(vector_dim)) items_tbl = await db.create_table("document_items", schema=DocumentItemRecord) + if fts_index == "empty": + await chunks_tbl.create_index("content_fts", config=FTS()) await settings_tbl.add( [ @@ -134,6 +141,8 @@ async def _build_db( ) ] ) + if fts_index == "covering": + await chunks_tbl.create_index("content_fts", config=FTS()) return db @@ -201,6 +210,41 @@ async def test_missing_table_fails_without_opening_store(temp_db_path): assert "documents" in tables.details +@pytest.mark.asyncio +async def test_missing_fts_index_fails(temp_db_path): + """optimize indexes the rows of an index that exists; it never creates a + missing one, so the remediation has to build it.""" + await _build_db(temp_db_path, fts_index=None) + report = await run_doctor(_config(), temp_db_path, {}) + result = _result(report, "fts_index_coverage") + assert result.severity is Severity.FAIL + assert "chunks.content_fts" in result.details[0] + assert "no index over 1 rows" in result.details[0] + assert "rebuild" in (result.remediation or "") + assert "vacuum" not in (result.remediation or "") + assert report.failed + + +@pytest.mark.asyncio +async def test_fts_index_covering_no_rows_fails(temp_db_path): + """The state a bulk write without a closing vacuum leaves behind.""" + await _build_db(temp_db_path, fts_index="empty") + report = await run_doctor(_config(), temp_db_path, {}) + result = _result(report, "fts_index_coverage") + assert result.severity is Severity.FAIL + assert "0 of 1 rows indexed" in result.details[0] + assert "vacuum" in (result.remediation or "") + + +@pytest.mark.asyncio +async def test_fts_coverage_passes_an_empty_table(temp_db_path): + db = await _build_db(temp_db_path) + chunks_tbl = await db.open_table("chunks") + await chunks_tbl.delete("id = 'c1'") + report = await run_doctor(_config(), temp_db_path, {}) + assert _result(report, "fts_index_coverage").severity is Severity.OK + + @pytest.mark.asyncio async def test_orphaned_chunk_fails(temp_db_path): db = await _build_db(temp_db_path) diff --git a/tests/test_mcp.py b/tests/test_mcp.py index 3ee9e237..16177122 100644 --- a/tests/test_mcp.py +++ b/tests/test_mcp.py @@ -91,7 +91,7 @@ class TestMCPReadTools: embedding=embedding, ) ) - await rag.chunk_repository._ensure_fts_index() + await rag.store.chunks_table.optimize() mcp = create_mcp_server(mcp_db, read_only=True) async with Client(mcp) as client: