info: report partial stats and pending migrations on pre-migration DBs
This commit is contained in:
Yiorgis Gozadinos
parent
4f16714430
commit
a0a9a3410b
3 changed files with 246 additions and 108 deletions
|
|
@ -1,6 +1,10 @@
|
|||
# Changelog
|
||||
## [Unreleased]
|
||||
|
||||
### Fixed
|
||||
|
||||
- **`haiku-rag info` on pre-migration databases**: `info` no longer fails with a misleading `Cannot create tables in read-only mode` error when a required table added by a later version (e.g. `document_items` in 0.40.0) is absent. It now reports stats for the tables that do exist, marks the missing ones as `absent`, and shows a dedicated section listing any pending migrations with the `haiku-rag migrate` hint ([#346](https://github.com/ggozad/haiku.rag/issues/346))
|
||||
|
||||
## [0.39.0] - 2026-04-16
|
||||
|
||||
### Added
|
||||
|
|
|
|||
|
|
@ -68,7 +68,13 @@ class HaikuRAGApp: # pragma: no cover
|
|||
async def info(self):
|
||||
"""Display read-only information about the database without modifying it."""
|
||||
|
||||
from haiku.rag.store.engine import Store, connect_lancedb
|
||||
from haiku.rag.store.engine import connect_lancedb
|
||||
from haiku.rag.store.upgrades import get_pending_upgrades
|
||||
|
||||
if self.before is not None:
|
||||
self.console.print(
|
||||
"[yellow]Note: --before is not supported by info; showing current state.[/yellow]"
|
||||
)
|
||||
|
||||
# Basic: show path/URI
|
||||
self.console.print("[bold]haiku.rag database info[/bold]")
|
||||
|
|
@ -80,10 +86,12 @@ class HaikuRAGApp: # pragma: no cover
|
|||
self.console.print("[red]Database path does not exist.[/red]")
|
||||
return
|
||||
|
||||
# Connect without going through Store to avoid upgrades/validation writes
|
||||
# Connect directly. Don't go through Store so a database that is
|
||||
# missing tables (e.g. pre-migration) still reports what it can.
|
||||
db = connect_lancedb(self.config, self.db_path)
|
||||
existing_tables = set(db.list_tables().tables)
|
||||
|
||||
if not db.list_tables().tables:
|
||||
if not existing_tables:
|
||||
self.console.print(
|
||||
"[red]Database is empty. Use 'haiku-rag init' to initialize.[/red]"
|
||||
)
|
||||
|
|
@ -91,44 +99,28 @@ class HaikuRAGApp: # pragma: no cover
|
|||
|
||||
versions = get_package_versions()
|
||||
|
||||
store = Store(
|
||||
self.db_path,
|
||||
config=self.config,
|
||||
skip_validation=True,
|
||||
read_only=True,
|
||||
skip_migration_check=True,
|
||||
before=self.before,
|
||||
)
|
||||
table_stats = store.get_stats()
|
||||
|
||||
# Read settings after Store init (migrations have run)
|
||||
settings_tbl = db.open_table("settings")
|
||||
arrow = settings_tbl.search().where("id = 'settings'").limit(1).to_arrow()
|
||||
rows = arrow.to_pylist()
|
||||
raw = rows[0].get("settings") or "{}"
|
||||
data = json.loads(raw) if isinstance(raw, str) else (raw or {})
|
||||
stored_version = str(data.get("version", "unknown"))
|
||||
embeddings = data.get("embeddings", {})
|
||||
embed_model_obj = embeddings.get("model", {})
|
||||
embed_provider = embed_model_obj.get("provider", "unknown")
|
||||
embed_model = embed_model_obj.get("name", "unknown")
|
||||
vector_dim = embed_model_obj.get("vector_dim")
|
||||
|
||||
store.close()
|
||||
|
||||
num_docs = table_stats["documents"].get("num_rows", 0)
|
||||
doc_bytes = table_stats["documents"].get("total_bytes", 0)
|
||||
|
||||
num_chunks = table_stats["chunks"].get("num_rows", 0)
|
||||
chunk_bytes = table_stats["chunks"].get("total_bytes", 0)
|
||||
|
||||
has_vector_index = table_stats["chunks"].get("has_vector_index", False)
|
||||
num_indexed_rows = table_stats["chunks"].get("num_indexed_rows", 0)
|
||||
num_unindexed_rows = table_stats["chunks"].get("num_unindexed_rows", 0)
|
||||
|
||||
# Table versions per table (direct API)
|
||||
doc_versions = len(list(db.open_table("documents").list_versions()))
|
||||
chunk_versions = len(list(db.open_table("chunks").list_versions()))
|
||||
stored_version = "unknown"
|
||||
embed_provider = "unknown"
|
||||
embed_model = "unknown"
|
||||
vector_dim = None
|
||||
if "settings" in existing_tables:
|
||||
settings_tbl = db.open_table("settings")
|
||||
rows = (
|
||||
settings_tbl.search()
|
||||
.where("id = 'settings'")
|
||||
.limit(1)
|
||||
.to_arrow()
|
||||
.to_pylist()
|
||||
)
|
||||
if rows:
|
||||
raw = rows[0].get("settings") or "{}"
|
||||
data = json.loads(raw) if isinstance(raw, str) else (raw or {})
|
||||
stored_version = str(data.get("version", "unknown"))
|
||||
embeddings = data.get("embeddings", {})
|
||||
embed_model_obj = embeddings.get("model", {})
|
||||
embed_provider = embed_model_obj.get("provider", "unknown")
|
||||
embed_model = embed_model_obj.get("name", "unknown")
|
||||
vector_dim = embed_model_obj.get("vector_dim")
|
||||
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]haiku.rag version (db)[/repr.attrib_name]: {stored_version}"
|
||||
|
|
@ -138,50 +130,92 @@ class HaikuRAGApp: # pragma: no cover
|
|||
" [repr.attrib_name]embeddings[/repr.attrib_name]: "
|
||||
f"{embed_provider}/{embed_model} (dim: {dim_part})"
|
||||
)
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]documents[/repr.attrib_name]: {num_docs} "
|
||||
f"({format_bytes(doc_bytes)})"
|
||||
)
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]chunks[/repr.attrib_name]: {num_chunks} "
|
||||
f"({format_bytes(chunk_bytes)})"
|
||||
)
|
||||
|
||||
# Per-table row counts and sizes. Missing required tables are shown
|
||||
# as "absent" rather than raising.
|
||||
for name in ("documents", "chunks", "document_items"):
|
||||
if name in existing_tables:
|
||||
stats = db.open_table(name).stats()
|
||||
num_rows = stats.get("num_rows", 0)
|
||||
total_bytes = stats.get("total_bytes", 0)
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]{name}[/repr.attrib_name]: {num_rows} "
|
||||
f"({format_bytes(total_bytes)})"
|
||||
)
|
||||
else:
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]{name}[/repr.attrib_name]: [yellow]absent[/yellow]"
|
||||
)
|
||||
|
||||
# Vector index information
|
||||
if has_vector_index:
|
||||
self.console.print(
|
||||
" [repr.attrib_name]vector index[/repr.attrib_name]: ✓ exists"
|
||||
)
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]indexed chunks[/repr.attrib_name]: {num_indexed_rows}"
|
||||
)
|
||||
if num_unindexed_rows > 0:
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]unindexed chunks[/repr.attrib_name]: [yellow]{num_unindexed_rows}[/yellow] "
|
||||
"(consider running: haiku-rag create-index)"
|
||||
)
|
||||
else:
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]unindexed chunks[/repr.attrib_name]: {num_unindexed_rows}"
|
||||
)
|
||||
else:
|
||||
if num_chunks >= 256:
|
||||
self.console.print(
|
||||
" [repr.attrib_name]vector index[/repr.attrib_name]: [yellow]✗ not created[/yellow] "
|
||||
"(run: haiku-rag create-index)"
|
||||
)
|
||||
else:
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]vector index[/repr.attrib_name]: ✗ not created "
|
||||
f"(need {256 - num_chunks} more chunks)"
|
||||
)
|
||||
if "chunks" in existing_tables:
|
||||
chunks_tbl = db.open_table("chunks")
|
||||
num_chunks = chunks_tbl.stats().get("num_rows", 0)
|
||||
indices = chunks_tbl.list_indices()
|
||||
has_vector_index = any("vector" in str(idx).lower() for idx in indices)
|
||||
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]versions (documents)[/repr.attrib_name]: {doc_versions}"
|
||||
)
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]versions (chunks)[/repr.attrib_name]: {chunk_versions}"
|
||||
if has_vector_index:
|
||||
index_stats = chunks_tbl.index_stats("vector_idx")
|
||||
num_indexed_rows = (
|
||||
index_stats.num_indexed_rows if index_stats is not None else 0
|
||||
)
|
||||
num_unindexed_rows = (
|
||||
index_stats.num_unindexed_rows if index_stats is not None else 0
|
||||
)
|
||||
self.console.print(
|
||||
" [repr.attrib_name]vector index[/repr.attrib_name]: ✓ exists"
|
||||
)
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]indexed chunks[/repr.attrib_name]: {num_indexed_rows}"
|
||||
)
|
||||
if num_unindexed_rows > 0:
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]unindexed chunks[/repr.attrib_name]: [yellow]{num_unindexed_rows}[/yellow] "
|
||||
"(consider running: haiku-rag create-index)"
|
||||
)
|
||||
else:
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]unindexed chunks[/repr.attrib_name]: {num_unindexed_rows}"
|
||||
)
|
||||
else:
|
||||
if num_chunks >= 256:
|
||||
self.console.print(
|
||||
" [repr.attrib_name]vector index[/repr.attrib_name]: [yellow]✗ not created[/yellow] "
|
||||
"(run: haiku-rag create-index)"
|
||||
)
|
||||
else:
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]vector index[/repr.attrib_name]: ✗ not created "
|
||||
f"(need {256 - num_chunks} more chunks)"
|
||||
)
|
||||
|
||||
if "documents" in existing_tables:
|
||||
doc_versions = len(list(db.open_table("documents").list_versions()))
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]versions (documents)[/repr.attrib_name]: {doc_versions}"
|
||||
)
|
||||
if "chunks" in existing_tables:
|
||||
chunk_versions = len(list(db.open_table("chunks").list_versions()))
|
||||
self.console.print(
|
||||
f" [repr.attrib_name]versions (chunks)[/repr.attrib_name]: {chunk_versions}"
|
||||
)
|
||||
|
||||
# Migration status
|
||||
pending = (
|
||||
get_pending_upgrades(stored_version) if stored_version != "unknown" else []
|
||||
)
|
||||
self.console.rule()
|
||||
if pending:
|
||||
self.console.print(
|
||||
f"[bold yellow]Pending migrations: {len(pending)}[/bold yellow] "
|
||||
"(run [cyan]haiku-rag migrate[/cyan] to upgrade)"
|
||||
)
|
||||
for step in pending:
|
||||
desc = step.description or ""
|
||||
self.console.print(f" [yellow]→[/yellow] {step.version}: {desc}")
|
||||
else:
|
||||
self.console.print("[green]Database is up to date.[/green]")
|
||||
|
||||
self.console.rule()
|
||||
self.console.print("[bold]Versions[/bold]")
|
||||
self.console.print(
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ async def test_app_info_with_vector_index(temp_db_path, capsys):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_app_info_uses_connect_lancedb_for_remote(tmp_path, capsys):
|
||||
async def test_app_info_uses_connect_lancedb_for_remote(tmp_path):
|
||||
"""info() should use connect_lancedb() instead of direct lancedb.connect() for remote URIs."""
|
||||
nonexistent = tmp_path / "does_not_exist" / "db.lancedb"
|
||||
config = AppConfig(
|
||||
|
|
@ -173,42 +173,142 @@ async def test_app_info_uses_connect_lancedb_for_remote(tmp_path, capsys):
|
|||
app = HaikuRAGApp(db_path=nonexistent, config=config)
|
||||
|
||||
with patch("haiku.rag.store.engine.connect_lancedb") as mock_connect:
|
||||
# Make the mock return something that lets info() proceed minimally
|
||||
mock_db = mock_connect.return_value
|
||||
mock_table = mock_db.open_table.return_value
|
||||
mock_table.search.return_value.where.return_value.limit.return_value.to_arrow.return_value.to_pylist.return_value = [
|
||||
{
|
||||
"settings": json.dumps(
|
||||
# Empty DB triggers the early-return path - enough to prove connect_lancedb was used
|
||||
mock_connect.return_value.list_tables.return_value.tables = []
|
||||
await app.info()
|
||||
|
||||
mock_connect.assert_called_once_with(config, nonexistent)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_app_info_with_missing_document_items_table(temp_db_path, capsys):
|
||||
"""info() should still output database info and report pending migrations
|
||||
when a required table is absent (as for a DB created before 0.40.0)."""
|
||||
import lancedb
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from pydantic import Field
|
||||
|
||||
db = lancedb.connect(temp_db_path)
|
||||
|
||||
class SettingsRecord(LanceModel):
|
||||
id: str = Field(default="settings")
|
||||
settings: str = Field(default="{}")
|
||||
|
||||
class DocumentRecord(LanceModel):
|
||||
id: str
|
||||
content: str
|
||||
|
||||
class ChunkRecord(LanceModel):
|
||||
id: str
|
||||
document_id: str
|
||||
content: str
|
||||
vector: Vector(3) # type: ignore
|
||||
|
||||
settings_tbl = db.create_table("settings", schema=SettingsRecord)
|
||||
docs_tbl = db.create_table("documents", schema=DocumentRecord)
|
||||
chunks_tbl = db.create_table("chunks", schema=ChunkRecord)
|
||||
# Intentionally omit document_items (added in 0.40.0)
|
||||
|
||||
settings_tbl.add(
|
||||
[
|
||||
SettingsRecord(
|
||||
id="settings",
|
||||
settings=json.dumps(
|
||||
{
|
||||
"version": "1.0.0",
|
||||
"version": "0.39.0",
|
||||
"embeddings": {
|
||||
"model": {
|
||||
"provider": "test",
|
||||
"name": "test",
|
||||
"provider": "openai",
|
||||
"name": "text-embedding-3-small",
|
||||
"vector_dim": 3,
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
}
|
||||
),
|
||||
)
|
||||
]
|
||||
)
|
||||
docs_tbl.add([DocumentRecord(id="doc-1", content="hello")])
|
||||
chunks_tbl.add(
|
||||
[ChunkRecord(id="c1", document_id="doc-1", content="c", vector=[0.1, 0.2, 0.3])]
|
||||
)
|
||||
|
||||
with patch("haiku.rag.store.engine.Store") as mock_store_cls:
|
||||
mock_store = mock_store_cls.return_value
|
||||
mock_store.get_stats.return_value = {
|
||||
"documents": {"exists": True, "num_rows": 0, "total_bytes": 0},
|
||||
"chunks": {
|
||||
"exists": True,
|
||||
"num_rows": 0,
|
||||
"total_bytes": 0,
|
||||
"has_vector_index": False,
|
||||
"num_indexed_rows": 0,
|
||||
"num_unindexed_rows": 0,
|
||||
},
|
||||
}
|
||||
await app.info()
|
||||
app = HaikuRAGApp(db_path=temp_db_path)
|
||||
await app.info()
|
||||
|
||||
mock_connect.assert_called_once_with(config, nonexistent)
|
||||
out = capsys.readouterr().out
|
||||
|
||||
# Core stats should still be reported
|
||||
assert "haiku.rag version (db): 0.39.0" in out
|
||||
assert "documents: 1" in out
|
||||
assert "chunks: 1" in out
|
||||
|
||||
# Missing table should be flagged, not cause a crash
|
||||
assert "document_items: absent" in out
|
||||
|
||||
# Migration status should be surfaced
|
||||
assert "Pending migrations:" in out
|
||||
assert "haiku-rag migrate" in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_app_info_reports_up_to_date(temp_db_path, capsys):
|
||||
"""info() should report the database is up to date when no migrations
|
||||
are pending."""
|
||||
from importlib import metadata
|
||||
|
||||
import lancedb
|
||||
from lancedb.pydantic import LanceModel, Vector
|
||||
from pydantic import Field
|
||||
|
||||
db = lancedb.connect(temp_db_path)
|
||||
|
||||
class SettingsRecord(LanceModel):
|
||||
id: str = Field(default="settings")
|
||||
settings: str = Field(default="{}")
|
||||
|
||||
class DocumentRecord(LanceModel):
|
||||
id: str
|
||||
content: str
|
||||
|
||||
class ChunkRecord(LanceModel):
|
||||
id: str
|
||||
document_id: str
|
||||
content: str
|
||||
vector: Vector(3) # type: ignore
|
||||
|
||||
settings_tbl = db.create_table("settings", schema=SettingsRecord)
|
||||
db.create_table("documents", schema=DocumentRecord)
|
||||
db.create_table("chunks", schema=ChunkRecord)
|
||||
db.create_table("document_items", schema=DocumentItemRecord)
|
||||
|
||||
current_version = metadata.version("haiku.rag-slim")
|
||||
settings_tbl.add(
|
||||
[
|
||||
SettingsRecord(
|
||||
id="settings",
|
||||
settings=json.dumps(
|
||||
{
|
||||
"version": current_version,
|
||||
"embeddings": {
|
||||
"model": {
|
||||
"provider": "openai",
|
||||
"name": "text-embedding-3-small",
|
||||
"vector_dim": 3,
|
||||
}
|
||||
},
|
||||
}
|
||||
),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
app = HaikuRAGApp(db_path=temp_db_path)
|
||||
await app.info()
|
||||
|
||||
out = capsys.readouterr().out
|
||||
assert "Database is up to date." in out
|
||||
assert "Pending migrations" not in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Reference in a new issue