Show index stats in cli info command

This commit is contained in:
Yiorgis Gozadinos 2025-11-20 14:58:33 +02:00
parent 6a2f33b464
commit fdcf9a1a12
No known key found for this signature in database
4 changed files with 112 additions and 5 deletions

View file

@ -10,6 +10,10 @@
- New `search.vector_refine_factor` config option (default: 10) for accuracy/speed tradeoff
- Indexes not created automatically during ingestion to avoid performance degradation
- Manual rebuilding required after adding significant new data
- **Enhanced Info Command**: `haiku-rag info` now shows storage sizes and vector index statistics
- Displays storage size for documents and chunks tables in human-readable format
- Shows vector index status (exists/not created)
- Shows indexed and unindexed chunk counts for monitoring index staleness
### Changed

View file

@ -18,6 +18,7 @@ from haiku.rag.mcp import create_mcp_server
from haiku.rag.monitor import FileWatcher
from haiku.rag.store.models.chunk import Chunk
from haiku.rag.store.models.document import Document
from haiku.rag.utils import format_bytes
logger = logging.getLogger(__name__)
@ -83,10 +84,22 @@ class HaikuRAGApp:
embed_model = embeddings.get("model")
vector_dim = embeddings.get("vector_dim")
num_docs = 0
if "documents" in table_names:
docs_tbl = db.open_table("documents")
num_docs = int(docs_tbl.count_rows()) # type: ignore[attr-defined]
# Get comprehensive table statistics
from haiku.rag.store.engine import Store
store = Store(self.db_path, config=self.config)
table_stats = store.get_stats()
store.close()
num_docs = table_stats["documents"].get("num_rows", 0)
doc_bytes = table_stats["documents"].get("total_bytes", 0)
num_chunks = table_stats["chunks"].get("num_rows", 0)
chunk_bytes = table_stats["chunks"].get("total_bytes", 0)
has_vector_index = table_stats["chunks"].get("has_vector_index", False)
num_indexed_rows = table_stats["chunks"].get("num_indexed_rows", 0)
num_unindexed_rows = table_stats["chunks"].get("num_unindexed_rows", 0)
# Table versions per table (direct API)
doc_versions = (
@ -116,8 +129,43 @@ class HaikuRAGApp:
" [repr.attrib_name]embeddings[/repr.attrib_name]: unknown"
)
self.console.print(
f" [repr.attrib_name]documents[/repr.attrib_name]: {num_docs}"
f" [repr.attrib_name]documents[/repr.attrib_name]: {num_docs} "
f"({format_bytes(doc_bytes)})"
)
self.console.print(
f" [repr.attrib_name]chunks[/repr.attrib_name]: {num_chunks} "
f"({format_bytes(chunk_bytes)})"
)
# Vector index information
if has_vector_index:
self.console.print(
" [repr.attrib_name]vector index[/repr.attrib_name]: ✓ exists"
)
self.console.print(
f" [repr.attrib_name]indexed chunks[/repr.attrib_name]: {num_indexed_rows}"
)
if num_unindexed_rows > 0:
self.console.print(
f" [repr.attrib_name]unindexed chunks[/repr.attrib_name]: [yellow]{num_unindexed_rows}[/yellow] "
"(consider running: haiku-rag create-index)"
)
else:
self.console.print(
f" [repr.attrib_name]unindexed chunks[/repr.attrib_name]: {num_unindexed_rows}"
)
else:
if num_chunks >= 256:
self.console.print(
" [repr.attrib_name]vector index[/repr.attrib_name]: [yellow]✗ not created[/yellow] "
"(run: haiku-rag create-index)"
)
else:
self.console.print(
f" [repr.attrib_name]vector index[/repr.attrib_name]: ✗ not created "
f"(need {256 - num_chunks} more chunks)"
)
self.console.print(
f" [repr.attrib_name]versions (documents)[/repr.attrib_name]: {doc_versions}"
)

View file

@ -145,6 +145,51 @@ class Store:
and self._config.lancedb.region
)
def get_stats(self) -> dict:
"""Get comprehensive table statistics.
Returns:
Dictionary with statistics for documents and chunks tables including:
- Row counts
- Storage sizes
- Vector index status and statistics
"""
stats_dict: dict = {
"documents": {"exists": False},
"chunks": {"exists": False},
}
# Documents table stats
doc_stats: dict = self.documents_table.stats() # type: ignore[assignment]
stats_dict["documents"] = {
"exists": True,
"num_rows": doc_stats.get("num_rows", 0),
"total_bytes": doc_stats.get("total_bytes", 0),
}
# Chunks table stats
chunk_stats: dict = self.chunks_table.stats() # type: ignore[assignment]
stats_dict["chunks"] = {
"exists": True,
"num_rows": chunk_stats.get("num_rows", 0),
"total_bytes": chunk_stats.get("total_bytes", 0),
}
# Vector index stats
indices = self.chunks_table.list_indices()
has_vector_index = any("vector" in str(idx).lower() for idx in indices)
stats_dict["chunks"]["has_vector_index"] = has_vector_index
if has_vector_index:
index_stats = self.chunks_table.index_stats("vector_idx")
if index_stats is not None:
stats_dict["chunks"]["num_indexed_rows"] = index_stats.num_indexed_rows
stats_dict["chunks"]["num_unindexed_rows"] = (
index_stats.num_unindexed_rows
)
return stats_dict
def _ensure_vector_index(self) -> None:
"""Create or rebuild vector index on chunks table.

View file

@ -8,6 +8,16 @@ from types import ModuleType
from packaging.version import Version, parse
def format_bytes(num_bytes: int) -> str:
"""Format bytes as human-readable string."""
size = float(num_bytes)
for unit in ["B", "KB", "MB", "GB", "TB"]:
if size < 1024.0:
return f"{size:.1f} {unit}"
size /= 1024.0
return f"{size:.1f} PB"
def get_default_data_dir() -> Path:
"""Get the user data directory for the current system platform.