diff --git a/CHANGELOG.md b/CHANGELOG.md index 496e5656..589d6d3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog ## [Unreleased] +### Added + +- `haiku-rag doctor` checks a database for consistency (orphaned chunks/items, documents without chunks, dangling `doc_item_refs`, vector-dimension mismatch, unembedded chunks, missing picture data, settings/embedding drift, pending migrations, vector-index coverage, provider API keys) and exits 1 when any check fails. + ## [0.60.0] - 2026-06-22 ### Added diff --git a/docs/cli.md b/docs/cli.md index 037d544b..0a55151a 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -284,6 +284,32 @@ At the end, a separate "Versions" section lists runtime package versions: - lancedb - docling +### Doctor + +Check the database for consistency problems and print a pass/warn/fail report: + +```bash +haiku-rag doctor [--db /path/to/your.lancedb] +``` + +Checks include: + +- required tables are present +- `documents` and `document_meta` are in 1:1 correspondence +- chunks and document items reference documents that exist +- every document produced chunks and document items +- chunk `doc_item_refs` resolve to existing document items +- chunk vector size matches the stored embedding dimension +- chunks are embedded (no all-zero vectors) +- picture items carry their image data +- exactly one settings row is present +- the configured embedding identity matches the stored settings +- no database migrations are pending +- the vector index covers all chunks +- API keys are set for configured providers + +Each failure prints the command that fixes it (`rebuild`, `create-index`, `migrate`, `rebuild --set-embedder`). `doctor` makes no changes. It exits with status 1 when any check fails, so it can gate CI or monitoring. + ### Migrate Database Apply pending database migrations: diff --git a/haiku_rag_slim/haiku/rag/app.py b/haiku_rag_slim/haiku/rag/app.py index 6d39fbb1..55a37cc7 100644 --- a/haiku_rag_slim/haiku/rag/app.py +++ b/haiku_rag_slim/haiku/rag/app.py @@ -198,6 +198,44 @@ class HaikuRAGApp: # pragma: no cover f" [repr.attrib_name]docling-document schema[/repr.attrib_name]: {info.packages['docling_document_schema']}" ) + async def doctor(self) -> bool: + """Run health checks and print a report. Returns True if any check failed.""" + import os + + from haiku.rag.doctor import Severity, run_doctor + + self.console.print("[bold]haiku.rag doctor[/bold]") + self.console.print( + f" [repr.attrib_name]path[/repr.attrib_name]: {self._display_path}" + ) + + if self._is_local and not self.db_path.exists(): + self.console.print("[red]Database path does not exist.[/red]") + return True + + report = await run_doctor(self.config, self.db_path, dict(os.environ)) + + glyphs = { + Severity.OK: "[green]✓[/green]", + Severity.WARN: "[yellow]![/yellow]", + Severity.FAIL: "[red]✗[/red]", + } + self.console.rule() + for result in report.results: + self.console.print(f"{glyphs[result.severity]} {result.message}") + for detail in result.details: + self.console.print(f" [dim]{detail}[/dim]") + if result.remediation: + self.console.print(f" [dim]→ {result.remediation}[/dim]") + + self.console.rule() + self.console.print( + f"[green]{report.count(Severity.OK)} ok[/green], " + f"[yellow]{report.count(Severity.WARN)} warning(s)[/yellow], " + f"[red]{report.count(Severity.FAIL)} failure(s)[/red]" + ) + return report.failed + async def history(self, table: str | None = None, limit: int | None = None): """Display version history for database tables. diff --git a/haiku_rag_slim/haiku/rag/cli.py b/haiku_rag_slim/haiku/rag/cli.py index 3c02ddc4..99685812 100644 --- a/haiku_rag_slim/haiku/rag/cli.py +++ b/haiku_rag_slim/haiku/rag/cli.py @@ -586,6 +586,19 @@ def info( # pragma: no cover asyncio.run(app.info()) +@_cli.command("doctor", help="Check database and provider health") +def doctor( # pragma: no cover + db: Path | None = typer.Option( + None, + "--db", + help="Path to the LanceDB database file", + ), +): + app = create_app(db) + if asyncio.run(app.doctor()): + raise typer.Exit(code=1) + + @_cli.command("history", help="Show version history for database tables") def history( # pragma: no cover db: Path | None = typer.Option( diff --git a/haiku_rag_slim/haiku/rag/doctor.py b/haiku_rag_slim/haiku/rag/doctor.py new file mode 100644 index 00000000..e4ac7721 --- /dev/null +++ b/haiku_rag_slim/haiku/rag/doctor.py @@ -0,0 +1,487 @@ +import json +from enum import StrEnum +from pathlib import Path + +import numpy as np +from pydantic import BaseModel, Field + +from haiku.rag.config import AppConfig +from haiku.rag.store.engine import ( + REQUIRED_TABLES, + Store, + connect_lancedb, + get_database_stats, +) +from haiku.rag.store.repositories.settings import SettingsRepository +from haiku.rag.store.upgrades import get_pending_upgrades + +# Cap how many offending ids we collect per check; doctor is a summary, not a dump. +_SAMPLE_LIMIT = 5 + +# API providers and the environment variable that carries their key. +_PROVIDER_ENV_VARS: dict[str, str] = { + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "cohere": "CO_API_KEY", + "voyageai": "VOYAGE_API_KEY", + "jina": "JINA_API_KEY", + "zeroentropy": "ZEROENTROPY_API_KEY", +} + + +class Severity(StrEnum): + OK = "ok" + WARN = "warn" + FAIL = "fail" + + +class CheckResult(BaseModel): + name: str + severity: Severity + message: str + remediation: str | None = None + details: list[str] = Field(default_factory=list) + + +class DoctorReport(BaseModel): + results: list[CheckResult] = Field(default_factory=list) + + @property + def failed(self) -> bool: + return any(r.severity is Severity.FAIL for r in self.results) + + def count(self, severity: Severity) -> int: + return sum(1 for r in self.results if r.severity is severity) + + +def _sample(ids: list[str]) -> list[str]: + """Cap a list of offending ids for display, noting how many were elided.""" + if len(ids) <= _SAMPLE_LIMIT: + return list(ids) + extra = len(ids) - _SAMPLE_LIMIT + return [*ids[:_SAMPLE_LIMIT], f"... (+{extra} more)"] + + +def _configured_providers(config: AppConfig) -> set[str]: + """Providers referenced by the current config across every model role.""" + providers = {config.embeddings.model.provider} + for model in ( + config.reranking.model, + config.qa.model, + config.analysis.model, + ): + if model is not None: + providers.add(model.provider) + return providers + + +def _check_api_keys(config: AppConfig, environ: dict[str, str]) -> CheckResult: + missing: list[str] = [] + for provider in sorted(_configured_providers(config)): + env_var = _PROVIDER_ENV_VARS.get(provider) + if env_var and not environ.get(env_var): + missing.append(f"{provider} ({env_var})") + if missing: + return CheckResult( + name="api_keys", + severity=Severity.FAIL, + message="Configured providers are missing their API key.", + remediation="Set the listed environment variables.", + details=missing, + ) + return CheckResult( + name="api_keys", + severity=Severity.OK, + message="API keys present for all configured providers.", + ) + + +def _check_tables_present(stats: dict) -> CheckResult: + missing = [name for name in REQUIRED_TABLES if not stats[name]["exists"]] + if missing: + return CheckResult( + name="tables_present", + severity=Severity.FAIL, + message="Required tables are missing.", + remediation="Run 'haiku-rag init' for a new database or 'haiku-rag migrate'.", + details=missing, + ) + return CheckResult( + name="tables_present", + severity=Severity.OK, + message="All required tables are present.", + ) + + +async def _column_values(table, column: str) -> list: + rows = await table.query().select([column]).to_list() + return [row[column] for row in rows] + + +async def run_db_checks( + store: Store, config: AppConfig, stats: dict +) -> list[CheckResult]: + """Referential and content-integrity checks against an open read-only Store. + + Assumes all required tables exist (the caller short-circuits otherwise). + """ + results: list[CheckResult] = [] + + doc_ids = set(await _column_values(store.documents_table, "id")) + meta_doc_ids = set(await _column_values(store.document_meta_table, "document_id")) + + chunk_rows = ( + await store.chunks_table.query() + .select(["id", "document_id", "metadata"]) + .to_list() + ) + chunk_doc_ids = {row["document_id"] for row in chunk_rows} + + item_rows = ( + await store.document_items_table.query() + .select(["document_id", "self_ref"]) + .to_list() + ) + item_doc_ids = {row["document_id"] for row in item_rows} + self_refs_by_doc: dict[str, set[str]] = {} + for row in item_rows: + self_refs_by_doc.setdefault(row["document_id"], set()).add(row["self_ref"]) + + # documents <-> document_meta must be 1:1. + orphan_docs = doc_ids - meta_doc_ids + orphan_meta = meta_doc_ids - doc_ids + if orphan_docs or orphan_meta: + details = [f"document with no meta: {d}" for d in _sample(sorted(orphan_docs))] + details += [f"meta with no document: {d}" for d in _sample(sorted(orphan_meta))] + results.append( + CheckResult( + name="document_meta_parity", + severity=Severity.FAIL, + message="documents and document_meta are out of sync.", + remediation="haiku-rag rebuild", + details=details, + ) + ) + else: + results.append( + CheckResult( + name="document_meta_parity", + severity=Severity.OK, + message="documents and document_meta are consistent.", + ) + ) + + # Orphaned chunks / items reference a document that no longer exists. + orphan_chunk_docs = chunk_doc_ids - doc_ids + results.append( + CheckResult( + name="orphaned_chunks", + severity=Severity.FAIL if orphan_chunk_docs else Severity.OK, + message=( + "Chunks reference missing documents." + if orphan_chunk_docs + else "No orphaned chunks." + ), + remediation="haiku-rag rebuild" if orphan_chunk_docs else None, + details=_sample(sorted(orphan_chunk_docs)), + ) + ) + + orphan_item_docs = item_doc_ids - doc_ids + results.append( + CheckResult( + name="orphaned_document_items", + severity=Severity.FAIL if orphan_item_docs else Severity.OK, + message=( + "Document items reference missing documents." + if orphan_item_docs + else "No orphaned document items." + ), + remediation="haiku-rag rebuild" if orphan_item_docs else None, + details=_sample(sorted(orphan_item_docs)), + ) + ) + + # Documents that never produced chunks / items. + docs_without_chunks = doc_ids - chunk_doc_ids + results.append( + CheckResult( + name="documents_without_chunks", + severity=Severity.WARN if docs_without_chunks else Severity.OK, + message=( + f"{len(docs_without_chunks)} document(s) have no chunks." + if docs_without_chunks + else "Every document has chunks." + ), + remediation="haiku-rag rebuild" if docs_without_chunks else None, + details=_sample(sorted(docs_without_chunks)), + ) + ) + + docs_without_items = doc_ids - item_doc_ids + results.append( + CheckResult( + name="documents_without_items", + severity=Severity.WARN if docs_without_items else Severity.OK, + message=( + f"{len(docs_without_items)} document(s) have no document items." + if docs_without_items + else "Every document has document items." + ), + remediation="haiku-rag rebuild" if docs_without_items else None, + details=_sample(sorted(docs_without_items)), + ) + ) + + # Chunk metadata may reference self_refs that do not exist for that document. + dangling: list[str] = [] + for row in chunk_rows: + refs = json.loads(row.get("metadata") or "{}").get("doc_item_refs") or [] + known = self_refs_by_doc.get(row["document_id"], set()) + if any(ref not in known for ref in refs): + dangling.append(row["id"]) + results.append( + CheckResult( + name="dangling_doc_item_refs", + severity=Severity.FAIL if dangling else Severity.OK, + message=( + f"{len(dangling)} chunk(s) reference missing document items." + if dangling + else "All chunk doc_item_refs resolve." + ), + remediation="haiku-rag rebuild" if dangling else None, + details=_sample(dangling), + ) + ) + + # Vector dimension consistency and unembedded (all-zero) vectors share one + # scan of the vector column — the heaviest check on large corpora. + arrow = await store.chunks_table.query().select(["id", "vector"]).to_arrow() + stored = await SettingsRepository(store).get_current_settings() + stored_dim = stored.get("embeddings", {}).get("model", {}).get("vector_dim") + actual_dim = arrow.schema.field("vector").type.list_size + if stored_dim and stored_dim != actual_dim: + results.append( + CheckResult( + name="vector_dimension", + severity=Severity.FAIL, + message=( + f"Chunk vector size {actual_dim} does not match stored " + f"vector_dim {stored_dim}." + ), + remediation="haiku-rag rebuild", + ) + ) + else: + results.append( + CheckResult( + name="vector_dimension", + severity=Severity.OK, + message=f"Chunk vectors are {actual_dim}-dimensional.", + ) + ) + + ids = arrow.column("id").to_pylist() + vectors = np.asarray(arrow.column("vector").to_pylist(), dtype=float) + zero_ids: list[str] = [] + if vectors.size: + zero_ids = [ids[i] for i in np.nonzero(~vectors.any(axis=1))[0]] + results.append( + CheckResult( + name="unembedded_chunks", + severity=Severity.WARN if zero_ids else Severity.OK, + message=( + f"{len(zero_ids)} chunk(s) have an all-zero (unembedded) vector." + if zero_ids + else "All chunks are embedded." + ), + remediation="haiku-rag rebuild --embed-only" if zero_ids else None, + details=_sample(zero_ids), + ) + ) + + # Pictures should carry their raster bytes after extraction. + total_pictures = await store.document_items_table.count_rows("label = 'picture'") + missing_pictures = len( + await store.document_items_table.query() + .select(["self_ref"]) + .where("label = 'picture' AND picture_data IS NULL") + .to_list() + ) + results.append( + CheckResult( + name="picture_data", + severity=Severity.WARN if missing_pictures else Severity.OK, + message=( + f"{missing_pictures} of {total_pictures} picture item(s) " + "have no image data." + if missing_pictures + else f"All {total_pictures} picture item(s) have image data." + ), + remediation="haiku-rag rebuild" if missing_pictures else None, + ) + ) + + # Settings must hold exactly one canonical row. + total_settings = await store.settings_table.count_rows() + canonical = len( + await store.settings_table.query().where("id = 'settings'").to_list() + ) + if total_settings == 0 or canonical != 1: + results.append( + CheckResult( + name="settings_row", + severity=Severity.FAIL, + message=( + f"Expected exactly one 'settings' row, found {canonical} " + f"(of {total_settings} total)." + ), + remediation="haiku-rag migrate", + ) + ) + else: + results.append( + CheckResult( + name="settings_row", + severity=Severity.OK, + message="Settings row is present.", + ) + ) + + results.append(_check_embedding_drift(stored, config)) + + stored_version = str(stored.get("version", "unknown")) + pending = ( + get_pending_upgrades(stored_version) if stored_version != "unknown" else [] + ) + results.append( + CheckResult( + name="pending_migrations", + severity=Severity.WARN if pending else Severity.OK, + message=( + f"{len(pending)} migration(s) pending (db version {stored_version})." + if pending + else f"Database is up to date (version {stored_version})." + ), + remediation="haiku-rag migrate" if pending else None, + details=[f"{step.version}: {step.description or ''}" for step in pending], + ) + ) + + results.append(_check_vector_index(stats)) + + return results + + +def _check_embedding_drift(stored: dict, config: AppConfig) -> CheckResult: + stored_model = stored.get("embeddings", {}).get("model", {}) + current_model = config.embeddings.model + if not stored_model: + return CheckResult( + name="embedding_drift", + severity=Severity.OK, + message="No stored embedding identity to compare.", + ) + + stored_dim = stored_model.get("vector_dim") + if stored_dim and stored_dim != current_model.vector_dim: + return CheckResult( + name="embedding_drift", + severity=Severity.FAIL, + message=( + f"Embedding vector_dim differs: stored {stored_dim} -> " + f"config {current_model.vector_dim}." + ), + remediation="haiku-rag rebuild", + ) + + drift: list[str] = [] + if stored_model.get("provider") not in (None, current_model.provider): + drift.append( + f"provider: {stored_model['provider']} -> {current_model.provider}" + ) + if stored_model.get("name") not in (None, current_model.name): + drift.append(f"name: {stored_model['name']} -> {current_model.name}") + if drift: + return CheckResult( + name="embedding_drift", + severity=Severity.WARN, + message="Embedding identity differs from config (vector_dim matches).", + remediation="haiku-rag rebuild --set-embedder", + details=drift, + ) + return CheckResult( + name="embedding_drift", + severity=Severity.OK, + message="Embedding identity matches the stored settings.", + ) + + +def _check_vector_index(stats: dict) -> CheckResult: + chunks = stats["chunks"] + num_chunks = chunks.get("num_rows", 0) + if not chunks.get("has_vector_index"): + if num_chunks >= 256: + return CheckResult( + name="vector_index", + severity=Severity.WARN, + message="No vector index; similarity search falls back to a scan.", + remediation="haiku-rag create-index", + ) + return CheckResult( + name="vector_index", + severity=Severity.OK, + message=f"No vector index yet (need {256 - num_chunks} more chunks).", + ) + unindexed = chunks.get("num_unindexed_rows", 0) + if unindexed > 0: + return CheckResult( + name="vector_index", + severity=Severity.WARN, + message=f"{unindexed} chunk(s) are not in the vector index.", + remediation="haiku-rag create-index", + ) + return CheckResult( + name="vector_index", + severity=Severity.OK, + message="Vector index covers all chunks.", + ) + + +async def run_doctor( + config: AppConfig, db_path: Path, environ: dict[str, str] +) -> DoctorReport: + """Open the database read-only and run every diagnostic check. + + Opens with validation and migration checks skipped so a drifted or + pre-migration database can still be diagnosed rather than refusing to open. + """ + db = await connect_lancedb(config, db_path) + stats = await get_database_stats(db) + + if not any(entry["exists"] for entry in stats.values()): + return DoctorReport( + results=[ + CheckResult( + name="tables_present", + severity=Severity.FAIL, + message="Database is empty.", + remediation="haiku-rag init", + ) + ] + ) + + results = [_check_tables_present(stats)] + missing = [name for name in REQUIRED_TABLES if not stats[name]["exists"]] + if not missing: + async with Store( + db_path, + config=config, + skip_validation=True, + read_only=True, + skip_migration_check=True, + ) as store: + results += await run_db_checks(store, config, stats) + + results.append(_check_api_keys(config, environ)) + return DoctorReport(results=results) diff --git a/tests/test_doctor.py b/tests/test_doctor.py new file mode 100644 index 00000000..35546ac5 --- /dev/null +++ b/tests/test_doctor.py @@ -0,0 +1,425 @@ +import json +from importlib import metadata +from unittest.mock import AsyncMock, MagicMock + +import lancedb +import pytest +from typer.testing import CliRunner + +from haiku.rag.cli import _cli as cli +from haiku.rag.config.models import AppConfig, EmbeddingModelConfig, EmbeddingsConfig +from haiku.rag.doctor import ( + CheckResult, + DoctorReport, + Severity, + _check_embedding_drift, + _check_vector_index, + _sample, + run_doctor, +) +from haiku.rag.store.engine import ( + DocumentItemRecord, + DocumentMetaRecord, + DocumentRecord, + SettingsRecord, + create_chunk_model, +) + +runner = CliRunner() + +CURRENT_VERSION = metadata.version("haiku.rag-slim") +VECTOR_DIM = 4 +ChunkRecord = create_chunk_model(VECTOR_DIM) + + +def _config(provider: str = "ollama", name: str = "test", vector_dim: int = VECTOR_DIM): + return AppConfig( + embeddings=EmbeddingsConfig( + model=EmbeddingModelConfig( + provider=provider, name=name, vector_dim=vector_dim + ) + ) + ) + + +async def _build_db( + path, + *, + version: str = CURRENT_VERSION, + provider: str = "ollama", + name: str = "test", + vector_dim: int = VECTOR_DIM, + stored_vector_dim: int | None = None, +): + """Create a consistent single-document database without touching an embedder. + + ``stored_vector_dim`` records a different dimension in settings than the + chunks table actually uses, to exercise the vector-dimension check. + """ + db = await lancedb.connect_async(path) + settings_tbl = await db.create_table("settings", schema=SettingsRecord) + docs_tbl = await db.create_table("documents", schema=DocumentRecord) + meta_tbl = await db.create_table("document_meta", schema=DocumentMetaRecord) + chunks_tbl = await db.create_table("chunks", schema=create_chunk_model(vector_dim)) + items_tbl = await db.create_table("document_items", schema=DocumentItemRecord) + + await settings_tbl.add( + [ + SettingsRecord( + id="settings", + settings=json.dumps( + { + "version": version, + "embeddings": { + "model": { + "provider": provider, + "name": name, + "vector_dim": stored_vector_dim or vector_dim, + } + }, + } + ), + ) + ] + ) + await docs_tbl.add([DocumentRecord(id="d1", content="hello")]) + await meta_tbl.add([DocumentMetaRecord(document_id="d1", uri="test://d1")]) + await items_tbl.add( + [ + DocumentItemRecord( + document_id="d1", position=0, self_ref="#/texts/0", text="x" + ) + ] + ) + chunk_model = create_chunk_model(vector_dim) + await chunks_tbl.add( + [ + chunk_model( + id="c1", + document_id="d1", + content="hello", + metadata=json.dumps({"doc_item_refs": ["#/texts/0"]}), + vector=[0.1] * vector_dim, + ) + ] + ) + return db + + +def _result(report: DoctorReport, name: str) -> CheckResult: + return next(r for r in report.results if r.name == name) + + +@pytest.mark.asyncio +async def test_healthy_db_all_ok(temp_db_path): + await _build_db(temp_db_path) + report = await run_doctor(_config(), temp_db_path, {}) + assert not report.failed + assert report.count(Severity.WARN) == 0 + assert all(r.severity is Severity.OK for r in report.results) + + +@pytest.mark.asyncio +async def test_empty_db_fails(temp_db_path): + report = await run_doctor(_config(), temp_db_path, {}) + assert report.failed + assert _result(report, "tables_present").message == "Database is empty." + + +@pytest.mark.asyncio +async def test_missing_table_fails_without_opening_store(temp_db_path): + db = await lancedb.connect_async(temp_db_path) + await db.create_table("settings", schema=SettingsRecord) + report = await run_doctor(_config(), temp_db_path, {}) + assert report.failed + tables = _result(report, "tables_present") + assert tables.severity is Severity.FAIL + assert "documents" in tables.details + + +@pytest.mark.asyncio +async def test_orphaned_chunk_fails(temp_db_path): + db = await _build_db(temp_db_path) + chunks_tbl = await db.open_table("chunks") + await chunks_tbl.add( + [ + ChunkRecord( + id="orphan", + document_id="ghost", + content="x", + vector=[0.2] * VECTOR_DIM, + ) + ] + ) + report = await run_doctor(_config(), temp_db_path, {}) + result = _result(report, "orphaned_chunks") + assert result.severity is Severity.FAIL + assert "ghost" in result.details + assert report.failed + + +@pytest.mark.asyncio +async def test_orphaned_document_item_fails(temp_db_path): + db = await _build_db(temp_db_path) + items_tbl = await db.open_table("document_items") + await items_tbl.add( + [DocumentItemRecord(document_id="ghost", position=0, self_ref="#/texts/0")] + ) + report = await run_doctor(_config(), temp_db_path, {}) + assert _result(report, "orphaned_document_items").severity is Severity.FAIL + + +@pytest.mark.asyncio +async def test_document_without_chunks_warns(temp_db_path): + db = await _build_db(temp_db_path) + docs_tbl = await db.open_table("documents") + meta_tbl = await db.open_table("document_meta") + await docs_tbl.add([DocumentRecord(id="d2", content="no chunks")]) + await meta_tbl.add([DocumentMetaRecord(document_id="d2", uri="test://d2")]) + report = await run_doctor(_config(), temp_db_path, {}) + assert _result(report, "documents_without_chunks").severity is Severity.WARN + assert _result(report, "documents_without_items").severity is Severity.WARN + assert not report.failed + + +@pytest.mark.asyncio +async def test_document_meta_parity_fails(temp_db_path): + db = await _build_db(temp_db_path) + docs_tbl = await db.open_table("documents") + await docs_tbl.add([DocumentRecord(id="d2", content="no meta")]) + report = await run_doctor(_config(), temp_db_path, {}) + result = _result(report, "document_meta_parity") + assert result.severity is Severity.FAIL + assert any("d2" in d for d in result.details) + + +@pytest.mark.asyncio +async def test_dangling_doc_item_ref_fails(temp_db_path): + db = await _build_db(temp_db_path) + chunks_tbl = await db.open_table("chunks") + await chunks_tbl.add( + [ + ChunkRecord( + id="c2", + document_id="d1", + content="x", + metadata=json.dumps({"doc_item_refs": ["#/texts/999"]}), + vector=[0.3] * VECTOR_DIM, + ) + ] + ) + report = await run_doctor(_config(), temp_db_path, {}) + result = _result(report, "dangling_doc_item_refs") + assert result.severity is Severity.FAIL + assert "c2" in result.details + + +@pytest.mark.asyncio +async def test_unembedded_chunk_warns(temp_db_path): + db = await _build_db(temp_db_path) + chunks_tbl = await db.open_table("chunks") + await chunks_tbl.add( + [ + ChunkRecord( + id="zero", + document_id="d1", + content="x", + metadata=json.dumps({"doc_item_refs": ["#/texts/0"]}), + vector=[0.0] * VECTOR_DIM, + ) + ] + ) + report = await run_doctor(_config(), temp_db_path, {}) + result = _result(report, "unembedded_chunks") + assert result.severity is Severity.WARN + assert "zero" in result.details + assert not report.failed + + +@pytest.mark.asyncio +async def test_missing_picture_data_warns(temp_db_path): + db = await _build_db(temp_db_path) + items_tbl = await db.open_table("document_items") + await items_tbl.add( + [ + DocumentItemRecord( + document_id="d1", + position=1, + self_ref="#/pictures/0", + label="picture", + picture_data=None, + ) + ] + ) + report = await run_doctor(_config(), temp_db_path, {}) + assert _result(report, "picture_data").severity is Severity.WARN + assert not report.failed + + +@pytest.mark.asyncio +async def test_picture_with_data_ok(temp_db_path): + db = await _build_db(temp_db_path) + items_tbl = await db.open_table("document_items") + await items_tbl.add( + [ + DocumentItemRecord( + document_id="d1", + position=1, + self_ref="#/pictures/0", + label="picture", + picture_data=b"\x89PNG", + ) + ] + ) + report = await run_doctor(_config(), temp_db_path, {}) + assert _result(report, "picture_data").severity is Severity.OK + + +@pytest.mark.asyncio +async def test_embedding_name_drift_warns(temp_db_path): + await _build_db(temp_db_path, name="test") + report = await run_doctor(_config(name="different"), temp_db_path, {}) + result = _result(report, "embedding_drift") + assert result.severity is Severity.WARN + assert not report.failed + + +@pytest.mark.asyncio +async def test_embedding_dim_drift_fails(temp_db_path): + await _build_db(temp_db_path, vector_dim=VECTOR_DIM) + report = await run_doctor(_config(vector_dim=VECTOR_DIM + 1), temp_db_path, {}) + assert _result(report, "embedding_drift").severity is Severity.FAIL + assert report.failed + + +@pytest.mark.asyncio +async def test_embedding_provider_drift_warns(temp_db_path): + await _build_db(temp_db_path, provider="ollama") + report = await run_doctor(_config(provider="vllm"), temp_db_path, {}) + result = _result(report, "embedding_drift") + assert result.severity is Severity.WARN + assert any("provider" in d for d in result.details) + + +@pytest.mark.asyncio +async def test_vector_dimension_mismatch_fails(temp_db_path): + await _build_db( + temp_db_path, vector_dim=VECTOR_DIM, stored_vector_dim=VECTOR_DIM + 1 + ) + report = await run_doctor(_config(vector_dim=VECTOR_DIM + 1), temp_db_path, {}) + result = _result(report, "vector_dimension") + assert result.severity is Severity.FAIL + assert report.failed + + +@pytest.mark.asyncio +async def test_pending_migration_warns(temp_db_path): + await _build_db(temp_db_path, version="0.40.0") + report = await run_doctor(_config(), temp_db_path, {}) + assert _result(report, "pending_migrations").severity is Severity.WARN + assert not report.failed + + +@pytest.mark.asyncio +async def test_missing_api_key_fails(temp_db_path): + await _build_db(temp_db_path, provider="openai", name="text-embedding-3-small") + config = _config(provider="openai", name="text-embedding-3-small") + report = await run_doctor(config, temp_db_path, environ={}) + result = _result(report, "api_keys") + assert result.severity is Severity.FAIL + assert any("OPENAI_API_KEY" in d for d in result.details) + + +@pytest.mark.asyncio +async def test_present_api_key_ok(temp_db_path): + await _build_db(temp_db_path, provider="openai", name="text-embedding-3-small") + config = _config(provider="openai", name="text-embedding-3-small") + report = await run_doctor(config, temp_db_path, environ={"OPENAI_API_KEY": "sk-x"}) + assert _result(report, "api_keys").severity is Severity.OK + + +@pytest.mark.asyncio +async def test_settings_row_missing_fails(temp_db_path): + db = await _build_db(temp_db_path) + settings_tbl = await db.open_table("settings") + await settings_tbl.delete("id = 'settings'") + report = await run_doctor(_config(), temp_db_path, {}) + assert _result(report, "settings_row").severity is Severity.FAIL + assert report.failed + + +@pytest.mark.asyncio +async def test_many_orphans_are_sampled(temp_db_path): + db = await _build_db(temp_db_path) + chunks_tbl = await db.open_table("chunks") + await chunks_tbl.add( + [ + ChunkRecord( + id=f"o{i}", + document_id=f"ghost{i}", + content="x", + vector=[0.2] * VECTOR_DIM, + ) + for i in range(8) + ] + ) + report = await run_doctor(_config(), temp_db_path, {}) + details = _result(report, "orphaned_chunks").details + assert len(details) == 6 + assert details[-1] == "... (+3 more)" + + +def test_sample_returns_all_within_limit(): + assert _sample(["a", "b"]) == ["a", "b"] + + +def test_embedding_drift_ok_without_stored_identity(): + assert _check_embedding_drift({}, _config()).severity is Severity.OK + + +def test_vector_index_ok_below_threshold(): + stats = {"chunks": {"num_rows": 10, "has_vector_index": False}} + assert _check_vector_index(stats).severity is Severity.OK + + +def test_vector_index_warns_when_missing_above_threshold(): + stats = {"chunks": {"num_rows": 300, "has_vector_index": False}} + result = _check_vector_index(stats) + assert result.severity is Severity.WARN + assert result.remediation == "haiku-rag create-index" + + +def test_vector_index_warns_on_unindexed_backlog(): + stats = { + "chunks": {"num_rows": 300, "has_vector_index": True, "num_unindexed_rows": 5} + } + assert _check_vector_index(stats).severity is Severity.WARN + + +def test_vector_index_ok_when_fully_indexed(): + stats = { + "chunks": {"num_rows": 300, "has_vector_index": True, "num_unindexed_rows": 0} + } + assert _check_vector_index(stats).severity is Severity.OK + + +def test_cli_doctor_nonexistent_db_exits_1(tmp_path): + result = runner.invoke(cli, ["doctor", "--db", str(tmp_path / "nope.lancedb")]) + assert result.exit_code == 1 + assert "does not exist" in result.output + + +def test_cli_doctor_exits_0_when_healthy(monkeypatch): + app = MagicMock() + app.doctor = AsyncMock(return_value=False) + monkeypatch.setattr("haiku.rag.cli.create_app", lambda *_a, **_k: app) + result = runner.invoke(cli, ["doctor", "--db", "/tmp/whatever.lancedb"]) + assert result.exit_code == 0 + + +def test_cli_doctor_exits_1_on_failure(monkeypatch): + app = MagicMock() + app.doctor = AsyncMock(return_value=True) + monkeypatch.setattr("haiku.rag.cli.create_app", lambda *_a, **_k: app) + result = runner.invoke(cli, ["doctor", "--db", "/tmp/whatever.lancedb"]) + assert result.exit_code == 1