Guard against an FTS index that covers no rows
lance serves unsorted results with matching rows dropped when a declared FTS index has indexed nothing, or when a populated table has no FTS index at all. doctor fails on both: vacuum remediation for an existing index, rebuild --embed-only for a missing one, since optimize never creates an index. The chunk repository warns once per repository on the first FTS or hybrid search against either state; a failing coverage check is logged and never fails the search. Removes _ensure_fts_index, which had no callers.
This commit is contained in:
Yiorgis Gozadinos
parent
2eaf2c6aea
commit
61df6b65a6
7 changed files with 280 additions and 23 deletions
|
|
@ -6,6 +6,8 @@
|
|||
|
||||
- `--full-citations` on `haiku-rag ask` and `haiku-rag analyze` renders citation
|
||||
text untruncated. `format_citations_rich` takes a `full` argument.
|
||||
- `doctor` fails when the chunks FTS index covers no rows.
|
||||
- FTS and hybrid searches log a warning when the FTS index covers no rows.
|
||||
|
||||
### Removed
|
||||
|
||||
|
|
|
|||
|
|
@ -329,6 +329,7 @@ Checks include:
|
|||
- the configured embedding identity matches the stored settings
|
||||
- no database migrations are pending
|
||||
- the vector index covers all chunks
|
||||
- the full-text index covers the chunks it searches
|
||||
- near-identical documents (by embedding-centroid similarity) are grouped and reported, with the largest member flagged as the likely one to keep (advisory only, never deleted, tuned via `doctor.duplicates` in config)
|
||||
- API keys are set for configured providers
|
||||
|
||||
|
|
@ -340,7 +341,7 @@ It also probes the external endpoints the config uses and reports them under a P
|
|||
|
||||
SaaS providers (OpenAI, Anthropic, Cohere, Jina, ZeroEntropy, Voyage) are covered by the API-key check rather than a network probe. In-process local models (sentence-transformers, cross-encoder, jina-local) have no endpoint and are reported as such.
|
||||
|
||||
Each failure prints the command that fixes it (`rebuild`, `create-index`, `migrate`, `rebuild --set-embedder`). `doctor` makes no changes. It exits with status 1 when any check fails, so it can gate CI or monitoring.
|
||||
Each failure prints the command that fixes it (`rebuild`, `create-index`, `vacuum`, `migrate`, `rebuild --set-embedder`). `doctor` makes no changes. It exits with status 1 when any check fails, so it can gate CI or monitoring.
|
||||
|
||||
### Migrate Database
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from haiku.rag.config.models import (
|
|||
)
|
||||
from haiku.rag.store.engine import Store, connect_lancedb
|
||||
from haiku.rag.store.info import get_database_stats
|
||||
from haiku.rag.store.schema import REQUIRED_TABLES
|
||||
from haiku.rag.store.schema import REQUIRED_TABLES, index_specs
|
||||
from haiku.rag.store.upgrades import get_pending_upgrades
|
||||
|
||||
# Cap how many offending ids we collect per check; doctor is a summary, not a dump.
|
||||
|
|
@ -225,6 +225,56 @@ def _classify_unchunked(
|
|||
return results
|
||||
|
||||
|
||||
async def _check_fts_coverage(store: Store) -> CheckResult:
|
||||
"""An FTS index that covers no rows, and a populated table with no FTS
|
||||
index at all, both make lance serve results unsorted by score with
|
||||
matching rows dropped. optimize indexes the rows of an index that
|
||||
exists; it never creates one that is absent."""
|
||||
from lancedb.index import FTS
|
||||
|
||||
uncovered: list[str] = []
|
||||
missing: list[str] = []
|
||||
for table_name, table in store._tables().items():
|
||||
declared = [c for c, cfg in index_specs(table_name) if isinstance(cfg, FTS)]
|
||||
if not declared:
|
||||
continue
|
||||
rows = await table.count_rows()
|
||||
if not rows:
|
||||
continue
|
||||
indices = await table.list_indices()
|
||||
for column in declared:
|
||||
index = next(
|
||||
(i for i in indices if column in i.columns and i.index_type == "FTS"),
|
||||
None,
|
||||
)
|
||||
if index is None:
|
||||
missing.append(f"{table_name}.{column}: no index over {rows} rows")
|
||||
continue
|
||||
stats = await table.index_stats(index.name)
|
||||
if stats is None or stats.num_indexed_rows == 0:
|
||||
uncovered.append(f"{table_name}.{column}: 0 of {rows} rows indexed")
|
||||
if missing or uncovered:
|
||||
return CheckResult(
|
||||
name="fts_index_coverage",
|
||||
severity=Severity.FAIL,
|
||||
message=(
|
||||
"Full-text search index does not cover its rows; FTS and "
|
||||
"hybrid results are unsorted and incomplete."
|
||||
),
|
||||
remediation=(
|
||||
"Run 'haiku-rag rebuild --embed-only' to build the index."
|
||||
if missing
|
||||
else "Run 'haiku-rag vacuum' to index the rows."
|
||||
),
|
||||
details=missing + uncovered,
|
||||
)
|
||||
return CheckResult(
|
||||
name="fts_index_coverage",
|
||||
severity=Severity.OK,
|
||||
message="Full-text search indexes cover their tables.",
|
||||
)
|
||||
|
||||
|
||||
async def _column_values(table, column: str) -> list:
|
||||
rows = await table.query().select([column]).to_list()
|
||||
return [row[column] for row in rows]
|
||||
|
|
@ -702,6 +752,9 @@ async def run_db_checks(
|
|||
self_refs_by_doc.setdefault(row["document_id"], set()).add(row["self_ref"])
|
||||
labels_by_doc.setdefault(row["document_id"], set()).add(row["label"])
|
||||
|
||||
notify("Checking index coverage")
|
||||
results.append(await _check_fts_coverage(store))
|
||||
|
||||
notify("Checking referential integrity")
|
||||
results.append(_check_document_meta_parity(doc_ids, meta_doc_ids))
|
||||
results.append(_check_orphaned_chunks(chunk_doc_ids, doc_ids))
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ from uuid import uuid4
|
|||
if TYPE_CHECKING:
|
||||
from lancedb.query import AsyncQueryBase
|
||||
|
||||
from lancedb.index import FTS
|
||||
from lancedb.rerankers import RRFReranker
|
||||
|
||||
from haiku.rag.store.engine import Store
|
||||
|
|
@ -23,17 +22,49 @@ class ChunkRepository:
|
|||
def __init__(self, store: Store) -> None:
|
||||
self.store = store
|
||||
self.embedder = store.embedder
|
||||
self._fts_coverage_checked = False
|
||||
|
||||
async def _ensure_fts_index(self) -> None:
|
||||
"""Ensure FTS index exists on the content_fts column."""
|
||||
async def _warn_if_fts_uncovered(self) -> None:
|
||||
"""An FTS index covering no rows makes lance serve a broken scan path:
|
||||
results unsorted by score, with matching documents dropped. Checked
|
||||
on the first search that uses the index, once per repository after
|
||||
the table holds rows."""
|
||||
if self._fts_coverage_checked:
|
||||
return
|
||||
try:
|
||||
await self.store.chunks_table.create_index(
|
||||
"content_fts",
|
||||
config=FTS(with_position=True, remove_stop_words=False),
|
||||
replace=True,
|
||||
indices = await self.store.chunks_table.list_indices()
|
||||
index = next(
|
||||
(
|
||||
i
|
||||
for i in indices
|
||||
if "content_fts" in i.columns and i.index_type == "FTS"
|
||||
),
|
||||
None,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"FTS index build failed; full-text search degraded: {e}")
|
||||
stats = (
|
||||
await self.store.chunks_table.index_stats(index.name) if index else None
|
||||
)
|
||||
if stats is not None and stats.num_indexed_rows > 0:
|
||||
self._fts_coverage_checked = True
|
||||
return
|
||||
# An empty table proves nothing; check again once it has rows.
|
||||
if not await self.store.chunks_table.count_rows():
|
||||
return
|
||||
except Exception:
|
||||
self._fts_coverage_checked = True
|
||||
logger.debug("FTS coverage check failed", exc_info=True)
|
||||
return
|
||||
self._fts_coverage_checked = True
|
||||
if index is None:
|
||||
logger.warning(
|
||||
"No full-text search index; FTS and hybrid results are "
|
||||
"degraded. Run 'haiku-rag rebuild --embed-only'."
|
||||
)
|
||||
return
|
||||
logger.warning(
|
||||
"Full-text search index covers 0 rows; FTS and hybrid results "
|
||||
"are degraded. Run 'haiku-rag vacuum'."
|
||||
)
|
||||
|
||||
def _contextualize_content(self, chunk: Chunk) -> str:
|
||||
"""Generate contextualized content for FTS by prepending headings."""
|
||||
|
|
@ -240,6 +271,9 @@ class ChunkRepository:
|
|||
id_list = ", ".join(f"'{d}'" for d in docs_df["id"])
|
||||
chunk_filter = f"document_id IN ({id_list})"
|
||||
|
||||
if search_type != "vector" and query.strip():
|
||||
await self._warn_if_fts_uncovered()
|
||||
|
||||
if search_type == "fts":
|
||||
results = self.store.chunks_table.query().nearest_to_text(
|
||||
query, columns="content_fts"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from haiku.rag.client import HaikuRAG
|
||||
|
|
@ -477,8 +479,28 @@ async def test_chunk_content_fts(temp_db_path, metadata, content, expected_conte
|
|||
assert record["content_fts"] == expected_content_fts
|
||||
|
||||
|
||||
async def test_ensure_fts_index_warns_on_failure(temp_db_path):
|
||||
"""A failed FTS index build is surfaced at WARNING, not swallowed silently."""
|
||||
async def _import_one(client) -> None:
|
||||
from docling_core.types.doc.document import DoclingDocument
|
||||
from docling_core.types.doc.labels import DocItemLabel
|
||||
|
||||
doc = DoclingDocument(name="one")
|
||||
doc.add_text(label=DocItemLabel.TEXT, text="a document about gardens")
|
||||
await client.import_document(
|
||||
doc,
|
||||
[
|
||||
Chunk(
|
||||
content="a document about gardens",
|
||||
embedding=[0.1] * get_config().embeddings.model.vector_dim,
|
||||
order=0,
|
||||
)
|
||||
],
|
||||
uri="test://one",
|
||||
)
|
||||
|
||||
|
||||
async def test_fts_search_warns_when_index_covers_no_rows(temp_db_path):
|
||||
"""A database whose FTS index predates its rows covers none of them;
|
||||
searching that state warns, once."""
|
||||
import logging
|
||||
|
||||
from haiku.rag.store.repositories import chunk as chunk_module
|
||||
|
|
@ -486,18 +508,119 @@ async def test_ensure_fts_index_warns_on_failure(temp_db_path):
|
|||
async with HaikuRAG(
|
||||
db_path=temp_db_path, config=get_config(), create=True
|
||||
) as client:
|
||||
repo = client.chunk_repository
|
||||
|
||||
async def _boom(*_args, **_kwargs):
|
||||
raise RuntimeError("index build failed")
|
||||
|
||||
repo.store.chunks_table.create_index = _boom
|
||||
await _import_one(client)
|
||||
|
||||
with capture_logs(chunk_module.logger, logging.WARNING) as records:
|
||||
await repo._ensure_fts_index()
|
||||
await client.chunk_repository.search("gardens", search_type="fts")
|
||||
await client.chunk_repository.search("gardens", search_type="fts")
|
||||
|
||||
assert [r for r in records if r.levelno == logging.WARNING]
|
||||
assert any("index build failed" in r.getMessage() for r in records)
|
||||
warned = [r for r in records if "covers 0 rows" in r.getMessage()]
|
||||
assert len(warned) == 1
|
||||
|
||||
|
||||
async def test_fts_search_warns_when_index_is_missing(temp_db_path, monkeypatch):
|
||||
import logging
|
||||
|
||||
from lancedb.table import AsyncTable
|
||||
|
||||
from haiku.rag.store.repositories import chunk as chunk_module
|
||||
|
||||
original = AsyncTable.list_indices
|
||||
|
||||
async def no_chunk_indices(self):
|
||||
if self.name == "chunks":
|
||||
return []
|
||||
return await original(self)
|
||||
|
||||
async with HaikuRAG(
|
||||
db_path=temp_db_path, config=get_config(), create=True
|
||||
) as client:
|
||||
await _import_one(client)
|
||||
monkeypatch.setattr(AsyncTable, "list_indices", no_chunk_indices)
|
||||
|
||||
with capture_logs(chunk_module.logger, logging.WARNING) as records:
|
||||
await client.chunk_repository.search("gardens", search_type="fts")
|
||||
|
||||
warned = [
|
||||
r.getMessage()
|
||||
for r in records
|
||||
if "No full-text search index" in r.getMessage()
|
||||
]
|
||||
assert len(warned) == 1
|
||||
assert "rebuild --embed-only" in warned[0]
|
||||
assert "vacuum" not in warned[0]
|
||||
|
||||
|
||||
async def test_fts_search_does_not_warn_when_index_covers_rows(temp_db_path):
|
||||
import logging
|
||||
|
||||
from haiku.rag.store.repositories import chunk as chunk_module
|
||||
|
||||
async with HaikuRAG(
|
||||
db_path=temp_db_path, config=get_config(), create=True
|
||||
) as client:
|
||||
await _import_one(client)
|
||||
await client.store.vacuum(retention_seconds=0)
|
||||
|
||||
with capture_logs(chunk_module.logger, logging.WARNING) as records:
|
||||
results = await client.chunk_repository.search("gardens", search_type="fts")
|
||||
|
||||
assert results
|
||||
assert not records
|
||||
|
||||
|
||||
async def test_fts_coverage_check_failure_does_not_break_search(temp_db_path):
|
||||
"""The coverage check is a diagnostic: a metadata failure must not take
|
||||
the search down with it."""
|
||||
from lancedb.table import AsyncTable
|
||||
|
||||
async def boom(self):
|
||||
raise RuntimeError("metadata unavailable")
|
||||
|
||||
async with HaikuRAG(
|
||||
db_path=temp_db_path, config=get_config(), create=True
|
||||
) as client:
|
||||
await _import_one(client)
|
||||
|
||||
with patch.object(AsyncTable, "list_indices", boom):
|
||||
results = await client.chunk_repository.search("gardens", search_type="fts")
|
||||
|
||||
assert results
|
||||
|
||||
|
||||
async def test_fts_search_on_an_empty_table_does_not_suppress_later_warnings(
|
||||
temp_db_path,
|
||||
):
|
||||
"""An empty table proves nothing about coverage, so searching it must not
|
||||
spend the once-per-repository check."""
|
||||
import logging
|
||||
|
||||
from haiku.rag.store.repositories import chunk as chunk_module
|
||||
|
||||
async with HaikuRAG(
|
||||
db_path=temp_db_path, config=get_config(), create=True
|
||||
) as client:
|
||||
await client.chunk_repository.search("gardens", search_type="fts")
|
||||
await _import_one(client)
|
||||
|
||||
with capture_logs(chunk_module.logger, logging.WARNING) as records:
|
||||
await client.chunk_repository.search("gardens", search_type="fts")
|
||||
|
||||
assert any("covers 0 rows" in r.getMessage() for r in records)
|
||||
|
||||
|
||||
async def test_fts_search_does_not_warn_on_an_empty_table(temp_db_path):
|
||||
import logging
|
||||
|
||||
from haiku.rag.store.repositories import chunk as chunk_module
|
||||
|
||||
async with HaikuRAG(
|
||||
db_path=temp_db_path, config=get_config(), create=True
|
||||
) as client:
|
||||
with capture_logs(chunk_module.logger, logging.WARNING) as records:
|
||||
await client.chunk_repository.search("gardens", search_type="fts")
|
||||
|
||||
assert not records
|
||||
|
||||
|
||||
@pytest.mark.vcr()
|
||||
|
|
|
|||
|
|
@ -81,18 +81,25 @@ async def _build_db(
|
|||
name: str = "test",
|
||||
vector_dim: int = VECTOR_DIM,
|
||||
stored_vector_dim: int | None = None,
|
||||
fts_index: str | None = "covering",
|
||||
):
|
||||
"""Create a consistent single-document database without touching an embedder.
|
||||
|
||||
``stored_vector_dim`` records a different dimension in settings than the
|
||||
chunks table actually uses, to exercise the vector-dimension check.
|
||||
``fts_index`` places the chunks FTS index: "covering" builds it over the
|
||||
rows, "empty" builds it before them, None never builds it.
|
||||
"""
|
||||
from lancedb.index import FTS
|
||||
|
||||
db = await lancedb.connect_async(path)
|
||||
settings_tbl = await db.create_table("settings", schema=SettingsRecord)
|
||||
docs_tbl = await db.create_table("documents", schema=DocumentRecord)
|
||||
meta_tbl = await db.create_table("document_meta", schema=DocumentMetaRecord)
|
||||
chunks_tbl = await db.create_table("chunks", schema=create_chunk_model(vector_dim))
|
||||
items_tbl = await db.create_table("document_items", schema=DocumentItemRecord)
|
||||
if fts_index == "empty":
|
||||
await chunks_tbl.create_index("content_fts", config=FTS())
|
||||
|
||||
await settings_tbl.add(
|
||||
[
|
||||
|
|
@ -134,6 +141,8 @@ async def _build_db(
|
|||
)
|
||||
]
|
||||
)
|
||||
if fts_index == "covering":
|
||||
await chunks_tbl.create_index("content_fts", config=FTS())
|
||||
return db
|
||||
|
||||
|
||||
|
|
@ -201,6 +210,41 @@ async def test_missing_table_fails_without_opening_store(temp_db_path):
|
|||
assert "documents" in tables.details
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_fts_index_fails(temp_db_path):
|
||||
"""optimize indexes the rows of an index that exists; it never creates a
|
||||
missing one, so the remediation has to build it."""
|
||||
await _build_db(temp_db_path, fts_index=None)
|
||||
report = await run_doctor(_config(), temp_db_path, {})
|
||||
result = _result(report, "fts_index_coverage")
|
||||
assert result.severity is Severity.FAIL
|
||||
assert "chunks.content_fts" in result.details[0]
|
||||
assert "no index over 1 rows" in result.details[0]
|
||||
assert "rebuild" in (result.remediation or "")
|
||||
assert "vacuum" not in (result.remediation or "")
|
||||
assert report.failed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fts_index_covering_no_rows_fails(temp_db_path):
|
||||
"""The state a bulk write without a closing vacuum leaves behind."""
|
||||
await _build_db(temp_db_path, fts_index="empty")
|
||||
report = await run_doctor(_config(), temp_db_path, {})
|
||||
result = _result(report, "fts_index_coverage")
|
||||
assert result.severity is Severity.FAIL
|
||||
assert "0 of 1 rows indexed" in result.details[0]
|
||||
assert "vacuum" in (result.remediation or "")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fts_coverage_passes_an_empty_table(temp_db_path):
|
||||
db = await _build_db(temp_db_path)
|
||||
chunks_tbl = await db.open_table("chunks")
|
||||
await chunks_tbl.delete("id = 'c1'")
|
||||
report = await run_doctor(_config(), temp_db_path, {})
|
||||
assert _result(report, "fts_index_coverage").severity is Severity.OK
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_orphaned_chunk_fails(temp_db_path):
|
||||
db = await _build_db(temp_db_path)
|
||||
|
|
|
|||
|
|
@ -91,7 +91,7 @@ class TestMCPReadTools:
|
|||
embedding=embedding,
|
||||
)
|
||||
)
|
||||
await rag.chunk_repository._ensure_fts_index()
|
||||
await rag.store.chunks_table.optimize()
|
||||
|
||||
mcp = create_mcp_server(mcp_db, read_only=True)
|
||||
async with Client(mcp) as client:
|
||||
|
|
|
|||
Loading…
Reference in a new issue