Show index stats in cli info command
This commit is contained in:
parent
6a2f33b464
commit
fdcf9a1a12
4 changed files with 112 additions and 5 deletions
|
|
@ -10,6 +10,10 @@
|
|||
- New `search.vector_refine_factor` config option (default: 10) for accuracy/speed tradeoff
|
||||
- Indexes not created automatically during ingestion to avoid performance degradation
|
||||
- Manual rebuilding required after adding significant new data
|
||||
- **Enhanced Info Command**: `haiku-rag info` now shows storage sizes and vector index statistics
|
||||
- Displays storage size for documents and chunks tables in human-readable format
|
||||
- Shows vector index status (exists/not created)
|
||||
- Shows indexed and unindexed chunk counts for monitoring index staleness
|
||||
|
||||
### Changed
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from haiku.rag.mcp import create_mcp_server
|
|||
from haiku.rag.monitor import FileWatcher
|
||||
from haiku.rag.store.models.chunk import Chunk
|
||||
from haiku.rag.store.models.document import Document
|
||||
from haiku.rag.utils import format_bytes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -83,10 +84,22 @@ class HaikuRAGApp:
|
|||
embed_model = embeddings.get("model")
|
||||
vector_dim = embeddings.get("vector_dim")
|
||||
|
||||
num_docs = 0
|
||||
if "documents" in table_names:
|
||||
docs_tbl = db.open_table("documents")
|
||||
num_docs = int(docs_tbl.count_rows()) # type: ignore[attr-defined]
|
||||
# Get comprehensive table statistics
|
||||
from haiku.rag.store.engine import Store
|
||||
|
||||
store = Store(self.db_path, config=self.config)
|
||||
table_stats = store.get_stats()
|
||||
store.close()
|
||||
|
||||
num_docs = table_stats["documents"].get("num_rows", 0)
|
||||
doc_bytes = table_stats["documents"].get("total_bytes", 0)
|
||||
|
||||
num_chunks = table_stats["chunks"].get("num_rows", 0)
|
||||
chunk_bytes = table_stats["chunks"].get("total_bytes", 0)
|
||||
|
||||
has_vector_index = table_stats["chunks"].get("has_vector_index", False)
|
||||
num_indexed_rows = table_stats["chunks"].get("num_indexed_rows", 0)
|
||||
num_unindexed_rows = table_stats["chunks"].get("num_unindexed_rows", 0)
|
||||
|
||||
# Table versions per table (direct API)
|
||||
doc_versions = (
|
||||
|
|
@ -116,8 +129,43 @@ class HaikuRAGApp:
|
|||
" [repr.attrib_name]embeddings[/repr.attrib_name]: unknown"
|
||||
)
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]documents[/repr.attrib_name]: {num_docs}"
|
||||
f" [repr.attrib_name]documents[/repr.attrib_name]: {num_docs} "
|
||||
f"({format_bytes(doc_bytes)})"
|
||||
)
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]chunks[/repr.attrib_name]: {num_chunks} "
|
||||
f"({format_bytes(chunk_bytes)})"
|
||||
)
|
||||
|
||||
# Vector index information
|
||||
if has_vector_index:
|
||||
self.console.print(
|
||||
" [repr.attrib_name]vector index[/repr.attrib_name]: ✓ exists"
|
||||
)
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]indexed chunks[/repr.attrib_name]: {num_indexed_rows}"
|
||||
)
|
||||
if num_unindexed_rows > 0:
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]unindexed chunks[/repr.attrib_name]: [yellow]{num_unindexed_rows}[/yellow] "
|
||||
"(consider running: haiku-rag create-index)"
|
||||
)
|
||||
else:
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]unindexed chunks[/repr.attrib_name]: {num_unindexed_rows}"
|
||||
)
|
||||
else:
|
||||
if num_chunks >= 256:
|
||||
self.console.print(
|
||||
" [repr.attrib_name]vector index[/repr.attrib_name]: [yellow]✗ not created[/yellow] "
|
||||
"(run: haiku-rag create-index)"
|
||||
)
|
||||
else:
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]vector index[/repr.attrib_name]: ✗ not created "
|
||||
f"(need {256 - num_chunks} more chunks)"
|
||||
)
|
||||
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]versions (documents)[/repr.attrib_name]: {doc_versions}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -145,6 +145,51 @@ class Store:
|
|||
and self._config.lancedb.region
|
||||
)
|
||||
|
||||
def get_stats(self) -> dict:
|
||||
"""Get comprehensive table statistics.
|
||||
|
||||
Returns:
|
||||
Dictionary with statistics for documents and chunks tables including:
|
||||
- Row counts
|
||||
- Storage sizes
|
||||
- Vector index status and statistics
|
||||
"""
|
||||
stats_dict: dict = {
|
||||
"documents": {"exists": False},
|
||||
"chunks": {"exists": False},
|
||||
}
|
||||
|
||||
# Documents table stats
|
||||
doc_stats: dict = self.documents_table.stats() # type: ignore[assignment]
|
||||
stats_dict["documents"] = {
|
||||
"exists": True,
|
||||
"num_rows": doc_stats.get("num_rows", 0),
|
||||
"total_bytes": doc_stats.get("total_bytes", 0),
|
||||
}
|
||||
|
||||
# Chunks table stats
|
||||
chunk_stats: dict = self.chunks_table.stats() # type: ignore[assignment]
|
||||
stats_dict["chunks"] = {
|
||||
"exists": True,
|
||||
"num_rows": chunk_stats.get("num_rows", 0),
|
||||
"total_bytes": chunk_stats.get("total_bytes", 0),
|
||||
}
|
||||
|
||||
# Vector index stats
|
||||
indices = self.chunks_table.list_indices()
|
||||
has_vector_index = any("vector" in str(idx).lower() for idx in indices)
|
||||
stats_dict["chunks"]["has_vector_index"] = has_vector_index
|
||||
|
||||
if has_vector_index:
|
||||
index_stats = self.chunks_table.index_stats("vector_idx")
|
||||
if index_stats is not None:
|
||||
stats_dict["chunks"]["num_indexed_rows"] = index_stats.num_indexed_rows
|
||||
stats_dict["chunks"]["num_unindexed_rows"] = (
|
||||
index_stats.num_unindexed_rows
|
||||
)
|
||||
|
||||
return stats_dict
|
||||
|
||||
def _ensure_vector_index(self) -> None:
|
||||
"""Create or rebuild vector index on chunks table.
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,16 @@ from types import ModuleType
|
|||
from packaging.version import Version, parse
|
||||
|
||||
|
||||
def format_bytes(num_bytes: int) -> str:
|
||||
"""Format bytes as human-readable string."""
|
||||
size = float(num_bytes)
|
||||
for unit in ["B", "KB", "MB", "GB", "TB"]:
|
||||
if size < 1024.0:
|
||||
return f"{size:.1f} {unit}"
|
||||
size /= 1024.0
|
||||
return f"{size:.1f} PB"
|
||||
|
||||
|
||||
def get_default_data_dir() -> Path:
|
||||
"""Get the user data directory for the current system platform.
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue