From 4f8cc4eb308821450b91c6db464a813c3be69b6c Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Sun, 28 Jun 2026 10:44:30 +0300 Subject: [PATCH] Show a progress spinner while doctor runs --- CHANGELOG.md | 4 ++++ docs/cli.md | 2 ++ haiku_rag_slim/haiku/rag/app.py | 19 +++++++++++++++++-- haiku_rag_slim/haiku/rag/doctor.py | 30 +++++++++++++++++++++++++++--- tests/test_doctor.py | 11 +++++++++++ 5 files changed, 61 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b7498fc..d8aa55c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,10 @@ # Changelog ## [Unreleased] +### Added + +- `doctor` shows a spinner naming the check currently in progress while it runs. + ### Changed - `doctor` near-duplicate detection compares whole-document embedding centroids instead of per-chunk overlap, and no longer flags a small document contained in a larger one. The per-chunk check could take hours and tens of GB of memory on corpora with large documents. The centroid check is independent of document size and reduces each document to a centroid during the vector scan without a second copy of the matrix. diff --git a/docs/cli.md b/docs/cli.md index 58ce3360..92e25855 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -292,6 +292,8 @@ Check the database for consistency problems and print a pass/warn/fail report: haiku-rag doctor [--db /path/to/your.lancedb] [--duplicates-out groups.yaml] ``` +While it runs, doctor shows a spinner naming the check currently in progress. + `--duplicates-out PATH` additionally writes the near-duplicate document groups to a YAML file (one block per group with `keep` and a list of `documents`, each carrying `document_id`, `document`, `chunks`, `similarity`, and `keep_suggested`) for offline review. Checks include: diff --git a/haiku_rag_slim/haiku/rag/app.py b/haiku_rag_slim/haiku/rag/app.py index 07c9c691..e6a68fd0 100644 --- a/haiku_rag_slim/haiku/rag/app.py +++ b/haiku_rag_slim/haiku/rag/app.py @@ -201,6 +201,7 @@ class HaikuRAGApp: # pragma: no cover async def doctor(self, duplicates_out: Path | None = None) -> bool: """Run health checks and print a report. Returns True if any check failed.""" import os + from contextlib import nullcontext from haiku.rag.doctor import Severity, run_doctor @@ -213,10 +214,24 @@ class HaikuRAGApp: # pragma: no cover self.console.print("[red]Database path does not exist.[/red]") return True - report = await run_doctor( - self.config, self.db_path, dict(os.environ), duplicates_out=duplicates_out + status = ( + self.console.status("Running checks") if self.console.is_terminal else None ) + def on_progress(label: str) -> None: + if status is not None: + status.update(f"{label}...") + + cm = status if status is not None else nullcontext() + with cm: + report = await run_doctor( + self.config, + self.db_path, + dict(os.environ), + duplicates_out=duplicates_out, + on_progress=on_progress, + ) + glyphs = { Severity.OK: "[green]✓[/green]", Severity.WARN: "[yellow]![/yellow]", diff --git a/haiku_rag_slim/haiku/rag/doctor.py b/haiku_rag_slim/haiku/rag/doctor.py index 732cac63..8b7c3f06 100644 --- a/haiku_rag_slim/haiku/rag/doctor.py +++ b/haiku_rag_slim/haiku/rag/doctor.py @@ -441,13 +441,16 @@ async def run_db_checks( config: AppConfig, stats: dict, duplicates_out: Path | None = None, + on_progress: Callable[[str], None] | None = None, ) -> list[CheckResult]: """Referential and content-integrity checks against an open read-only Store. Assumes all required tables exist (the caller short-circuits otherwise). """ + notify = on_progress or (lambda _label: None) results: list[CheckResult] = [] + notify("Reading document records") doc_ids = set(await _column_values(store.documents_table, "id")) meta_rows = ( await store.document_meta_table.query() @@ -464,6 +467,7 @@ async def run_db_checks( uri_by_doc = {row["document_id"]: row.get("uri") for row in meta_rows} title_by_doc = {row["document_id"]: row.get("title") for row in meta_rows} + notify("Reading chunks") chunk_rows = ( await store.chunks_table.query() .select(["id", "document_id", "metadata"]) @@ -471,6 +475,7 @@ async def run_db_checks( ) chunk_doc_ids = {row["document_id"] for row in chunk_rows} + notify("Reading document items") item_rows = ( await store.document_items_table.query() .select(["document_id", "self_ref", "label"]) @@ -483,6 +488,7 @@ async def run_db_checks( self_refs_by_doc.setdefault(row["document_id"], set()).add(row["self_ref"]) labels_by_doc.setdefault(row["document_id"], set()).add(row["label"]) + notify("Checking referential integrity") # documents <-> document_meta must be 1:1. orphan_docs = doc_ids - meta_doc_ids orphan_meta = meta_doc_ids - doc_ids @@ -538,6 +544,7 @@ async def run_db_checks( ) ) + notify("Checking document chunking") # Documents with no chunks, classified by what they contain and whether the # embedder can index images. results += _classify_unchunked( @@ -561,6 +568,7 @@ async def run_db_checks( ) ) + notify("Checking chunk references") # Chunk metadata may reference self_refs that do not exist for that document. dangling: list[str] = [] for row in chunk_rows: @@ -582,6 +590,7 @@ async def run_db_checks( ) ) + notify("Scanning chunk vectors") # Vector dimension consistency and unembedded (all-zero) vectors share one # scan of the vector column — the heaviest check on large corpora. arrow = ( @@ -643,6 +652,7 @@ async def run_db_checks( ) ) + notify("Detecting near-duplicate documents") # Near-identical documents (centroid cosine). Reduce each document's chunk # vectors to one summed centroid during the scan: dictionary-encode the # document ids into integer codes, then sum each document's embedded rows in @@ -673,6 +683,7 @@ async def run_db_checks( ) ) + notify("Checking picture data") # Pictures from image/PDF sources should carry raster bytes. Pictures that # are external image references in a text document (markdown, HTML) have no # embedded bytes by nature, so a missing raster there is expected. @@ -703,6 +714,7 @@ async def run_db_checks( ) ) + notify("Checking settings and indexes") # Settings must hold exactly one canonical row. total_settings = await store.settings_table.count_rows() canonical = len( @@ -955,12 +967,16 @@ def _endpoint_result( ) -async def run_provider_checks(config: AppConfig) -> list[CheckResult]: +async def run_provider_checks( + config: AppConfig, on_progress: Callable[[str], None] | None = None +) -> list[CheckResult]: """Probe the external endpoints the current config actually uses.""" targets, local = _provider_targets(config) results: list[CheckResult] = [] if targets: + if on_progress is not None: + on_progress("Probing provider endpoints") async with httpx.AsyncClient(timeout=_PROBE_TIMEOUT_S) as client: probes = await asyncio.gather( *(_probe_endpoint(client, url) for url in targets) @@ -984,12 +1000,15 @@ async def run_doctor( db_path: Path, environ: dict[str, str], duplicates_out: Path | None = None, + on_progress: Callable[[str], None] | None = None, ) -> DoctorReport: """Open the database read-only and run every diagnostic check. Opens with validation and migration checks skipped so a drifted or pre-migration database can still be diagnosed rather than refusing to open. """ + notify = on_progress or (lambda _label: None) + notify("Inspecting tables") db = await connect_lancedb(config, db_path) stats = await get_database_stats(db) @@ -1015,9 +1034,14 @@ async def run_doctor( skip_migration_check=True, ) as store: results += await run_db_checks( - store, config, stats, duplicates_out=duplicates_out + store, + config, + stats, + duplicates_out=duplicates_out, + on_progress=on_progress, ) + notify("Checking API keys") results.append(_check_api_keys(config, environ)) - results += await run_provider_checks(config) + results += await run_provider_checks(config, on_progress=on_progress) return DoctorReport(results=results) diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 619a0b6d..0f6ddc2f 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -172,6 +172,17 @@ async def test_healthy_db_all_ok(temp_db_path): assert all(r.severity is Severity.OK for r in report.results) +@pytest.mark.asyncio +async def test_doctor_reports_progress(temp_db_path): + await _build_db(temp_db_path) + labels: list[str] = [] + await run_doctor(_config(), temp_db_path, {}, on_progress=labels.append) + assert "Inspecting tables" in labels + assert "Scanning chunk vectors" in labels + assert "Detecting near-duplicate documents" in labels + assert "Probing provider endpoints" in labels + + @pytest.mark.asyncio async def test_empty_db_fails(temp_db_path): report = await run_doctor(_config(), temp_db_path, {})