From fdcf9a1a1273d11c5d06dbc7b31afba3355bb4b7 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Thu, 20 Nov 2025 14:58:33 +0200 Subject: [PATCH] Show index stats in cli info command --- CHANGELOG.md | 4 ++ haiku_rag_slim/haiku/rag/app.py | 58 ++++++++++++++++++++++-- haiku_rag_slim/haiku/rag/store/engine.py | 45 ++++++++++++++++++ haiku_rag_slim/haiku/rag/utils.py | 10 ++++ 4 files changed, 112 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f47ae492..4e01df74 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ - New `search.vector_refine_factor` config option (default: 10) for accuracy/speed tradeoff - Indexes not created automatically during ingestion to avoid performance degradation - Manual rebuilding required after adding significant new data +- **Enhanced Info Command**: `haiku-rag info` now shows storage sizes and vector index statistics + - Displays storage size for documents and chunks tables in human-readable format + - Shows vector index status (exists/not created) + - Shows indexed and unindexed chunk counts for monitoring index staleness ### Changed diff --git a/haiku_rag_slim/haiku/rag/app.py b/haiku_rag_slim/haiku/rag/app.py index af2d15d6..1a1841c5 100644 --- a/haiku_rag_slim/haiku/rag/app.py +++ b/haiku_rag_slim/haiku/rag/app.py @@ -18,6 +18,7 @@ from haiku.rag.mcp import create_mcp_server from haiku.rag.monitor import FileWatcher from haiku.rag.store.models.chunk import Chunk from haiku.rag.store.models.document import Document +from haiku.rag.utils import format_bytes logger = logging.getLogger(__name__) @@ -83,10 +84,22 @@ class HaikuRAGApp: embed_model = embeddings.get("model") vector_dim = embeddings.get("vector_dim") - num_docs = 0 - if "documents" in table_names: - docs_tbl = db.open_table("documents") - num_docs = int(docs_tbl.count_rows()) # type: ignore[attr-defined] + # Get comprehensive table statistics + from haiku.rag.store.engine import Store + + store = Store(self.db_path, config=self.config) + table_stats = store.get_stats() + store.close() + + num_docs = table_stats["documents"].get("num_rows", 0) + doc_bytes = table_stats["documents"].get("total_bytes", 0) + + num_chunks = table_stats["chunks"].get("num_rows", 0) + chunk_bytes = table_stats["chunks"].get("total_bytes", 0) + + has_vector_index = table_stats["chunks"].get("has_vector_index", False) + num_indexed_rows = table_stats["chunks"].get("num_indexed_rows", 0) + num_unindexed_rows = table_stats["chunks"].get("num_unindexed_rows", 0) # Table versions per table (direct API) doc_versions = ( @@ -116,8 +129,43 @@ class HaikuRAGApp: " [repr.attrib_name]embeddings[/repr.attrib_name]: unknown" ) self.console.print( - f" [repr.attrib_name]documents[/repr.attrib_name]: {num_docs}" + f" [repr.attrib_name]documents[/repr.attrib_name]: {num_docs} " + f"({format_bytes(doc_bytes)})" ) + self.console.print( + f" [repr.attrib_name]chunks[/repr.attrib_name]: {num_chunks} " + f"({format_bytes(chunk_bytes)})" + ) + + # Vector index information + if has_vector_index: + self.console.print( + " [repr.attrib_name]vector index[/repr.attrib_name]: ✓ exists" + ) + self.console.print( + f" [repr.attrib_name]indexed chunks[/repr.attrib_name]: {num_indexed_rows}" + ) + if num_unindexed_rows > 0: + self.console.print( + f" [repr.attrib_name]unindexed chunks[/repr.attrib_name]: [yellow]{num_unindexed_rows}[/yellow] " + "(consider running: haiku-rag create-index)" + ) + else: + self.console.print( + f" [repr.attrib_name]unindexed chunks[/repr.attrib_name]: {num_unindexed_rows}" + ) + else: + if num_chunks >= 256: + self.console.print( + " [repr.attrib_name]vector index[/repr.attrib_name]: [yellow]✗ not created[/yellow] " + "(run: haiku-rag create-index)" + ) + else: + self.console.print( + f" [repr.attrib_name]vector index[/repr.attrib_name]: ✗ not created " + f"(need {256 - num_chunks} more chunks)" + ) + self.console.print( f" [repr.attrib_name]versions (documents)[/repr.attrib_name]: {doc_versions}" ) diff --git a/haiku_rag_slim/haiku/rag/store/engine.py b/haiku_rag_slim/haiku/rag/store/engine.py index 820b25ee..785952d1 100644 --- a/haiku_rag_slim/haiku/rag/store/engine.py +++ b/haiku_rag_slim/haiku/rag/store/engine.py @@ -145,6 +145,51 @@ class Store: and self._config.lancedb.region ) + def get_stats(self) -> dict: + """Get comprehensive table statistics. + + Returns: + Dictionary with statistics for documents and chunks tables including: + - Row counts + - Storage sizes + - Vector index status and statistics + """ + stats_dict: dict = { + "documents": {"exists": False}, + "chunks": {"exists": False}, + } + + # Documents table stats + doc_stats: dict = self.documents_table.stats() # type: ignore[assignment] + stats_dict["documents"] = { + "exists": True, + "num_rows": doc_stats.get("num_rows", 0), + "total_bytes": doc_stats.get("total_bytes", 0), + } + + # Chunks table stats + chunk_stats: dict = self.chunks_table.stats() # type: ignore[assignment] + stats_dict["chunks"] = { + "exists": True, + "num_rows": chunk_stats.get("num_rows", 0), + "total_bytes": chunk_stats.get("total_bytes", 0), + } + + # Vector index stats + indices = self.chunks_table.list_indices() + has_vector_index = any("vector" in str(idx).lower() for idx in indices) + stats_dict["chunks"]["has_vector_index"] = has_vector_index + + if has_vector_index: + index_stats = self.chunks_table.index_stats("vector_idx") + if index_stats is not None: + stats_dict["chunks"]["num_indexed_rows"] = index_stats.num_indexed_rows + stats_dict["chunks"]["num_unindexed_rows"] = ( + index_stats.num_unindexed_rows + ) + + return stats_dict + def _ensure_vector_index(self) -> None: """Create or rebuild vector index on chunks table. diff --git a/haiku_rag_slim/haiku/rag/utils.py b/haiku_rag_slim/haiku/rag/utils.py index c08750b7..ae9d6e04 100644 --- a/haiku_rag_slim/haiku/rag/utils.py +++ b/haiku_rag_slim/haiku/rag/utils.py @@ -8,6 +8,16 @@ from types import ModuleType from packaging.version import Version, parse +def format_bytes(num_bytes: int) -> str: + """Format bytes as human-readable string.""" + size = float(num_bytes) + for unit in ["B", "KB", "MB", "GB", "TB"]: + if size < 1024.0: + return f"{size:.1f} {unit}" + size /= 1024.0 + return f"{size:.1f} PB" + + def get_default_data_dir() -> Path: """Get the user data directory for the current system platform.