Merge pull request #421 from ggozad/fix/read-only-update
Read operations no longer modify the database
This commit is contained in:
commit
a2def043cc
12 changed files with 341 additions and 71 deletions
|
|
@ -1,6 +1,15 @@
|
||||||
# Changelog
|
# Changelog
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- `haiku-rag rebuild --set-embedder`: adopt the current embedder identity (provider/name) without re-embedding, when the vector dimension is unchanged.
|
||||||
|
|
||||||
|
### Fixed
|
||||||
|
|
||||||
|
- Opening a database no longer writes to it: reads no longer rewrite the stored embedding settings or change the stored version, and the version is never downgraded. Embedding provider/name drift (matching `vector_dim`) warns on read-only opens and raises `ConfigMismatchError` on writable opens; reconcile with `rebuild --set-embedder`.
|
||||||
|
- Read CLI verbs (`list`, `get`, `search`, `visualize`, `ask`, `analyze`, `inspect`, `chat`, `info`, `history`) open the database read-only.
|
||||||
|
|
||||||
## [0.54.0] - 2026-06-04
|
## [0.54.0] - 2026-06-04
|
||||||
|
|
||||||
### Added
|
### Added
|
||||||
|
|
|
||||||
|
|
@ -373,6 +373,9 @@ haiku-rag rebuild --title-only
|
||||||
# Run the VLM over already-stored picture bytes and patch descriptions
|
# Run the VLM over already-stored picture bytes and patch descriptions
|
||||||
# into the docling blob. Skips the docling parse entirely.
|
# into the docling blob. Skips the docling parse entirely.
|
||||||
haiku-rag rebuild --descriptions
|
haiku-rag rebuild --descriptions
|
||||||
|
|
||||||
|
# Adopt the current embedder identity without re-embedding (same vector dimension)
|
||||||
|
haiku-rag rebuild --set-embedder
|
||||||
```
|
```
|
||||||
|
|
||||||
**Rebuild modes:**
|
**Rebuild modes:**
|
||||||
|
|
@ -384,6 +387,9 @@ haiku-rag rebuild --descriptions
|
||||||
| Embed only | `--embed-only` | Changed embedding model or vector dimensions |
|
| Embed only | `--embed-only` | Changed embedding model or vector dimensions |
|
||||||
| Title only | `--title-only` | Generate titles for documents without one |
|
| Title only | `--title-only` | Generate titles for documents without one |
|
||||||
| Descriptions | `--descriptions` | Add VLM picture descriptions to an existing database |
|
| Descriptions | `--descriptions` | Add VLM picture descriptions to an existing database |
|
||||||
|
| Set embedder | `--set-embedder` | Same model, different serving stack (e.g. Ollama to vLLM); vector dimension unchanged |
|
||||||
|
|
||||||
|
**`--set-embedder` mode** updates the stored embedding provider/name to match the current config without re-embedding, valid only when the vector dimension is unchanged. Use it when the same model is served by a different stack so the recorded identity stops drifting from the config. A changed vector dimension is rejected; regenerate embeddings with `--embed-only` or a full rebuild instead.
|
||||||
|
|
||||||
**`--descriptions` mode** runs the configured VLM (`processing.conversion_options.picture_description.model`) over the picture bytes already stored in `document_items.picture_data`, patches each description into the stored docling blob's `pictures[i].meta.description.text`, and re-chunks + re-embeds so chunk text reflects the new descriptions. Requires `processing.pictures: description` in the config. Idempotent: pictures that already carry a description are skipped, so the operation is safe to re-run after a partial failure. The docling parse is skipped entirely. Only the VLM time is paid.
|
**`--descriptions` mode** runs the configured VLM (`processing.conversion_options.picture_description.model`) over the picture bytes already stored in `document_items.picture_data`, patches each description into the stored docling blob's `pictures[i].meta.description.text`, and re-chunks + re-embeds so chunk text reflects the new descriptions. Requires `processing.pictures: description` in the config. Idempotent: pictures that already carry a description are skipped, so the operation is safe to re-run after a partial failure. The docling parse is skipped entirely. Only the VLM time is paid.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -276,7 +276,7 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
async with HaikuRAG(
|
async with HaikuRAG(
|
||||||
db_path=self.db_path,
|
db_path=self.db_path,
|
||||||
config=self.config,
|
config=self.config,
|
||||||
read_only=self.read_only,
|
read_only=True,
|
||||||
before=self.before,
|
before=self.before,
|
||||||
) as self.client:
|
) as self.client:
|
||||||
documents = await self.client.list_documents(filter=filter)
|
documents = await self.client.list_documents(filter=filter)
|
||||||
|
|
@ -328,7 +328,7 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
async with HaikuRAG(
|
async with HaikuRAG(
|
||||||
db_path=self.db_path,
|
db_path=self.db_path,
|
||||||
config=self.config,
|
config=self.config,
|
||||||
read_only=self.read_only,
|
read_only=True,
|
||||||
before=self.before,
|
before=self.before,
|
||||||
) as self.client:
|
) as self.client:
|
||||||
doc = await self.client.get_document_by_id(doc_id)
|
doc = await self.client.get_document_by_id(doc_id)
|
||||||
|
|
@ -385,7 +385,7 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
async with HaikuRAG(
|
async with HaikuRAG(
|
||||||
db_path=self.db_path,
|
db_path=self.db_path,
|
||||||
config=self.config,
|
config=self.config,
|
||||||
read_only=self.read_only,
|
read_only=True,
|
||||||
before=self.before,
|
before=self.before,
|
||||||
) as self.client:
|
) as self.client:
|
||||||
results = await self.client.search(
|
results = await self.client.search(
|
||||||
|
|
@ -407,7 +407,7 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
async with HaikuRAG(
|
async with HaikuRAG(
|
||||||
db_path=self.db_path,
|
db_path=self.db_path,
|
||||||
config=self.config,
|
config=self.config,
|
||||||
read_only=self.read_only,
|
read_only=True,
|
||||||
before=self.before,
|
before=self.before,
|
||||||
) as self.client:
|
) as self.client:
|
||||||
chunk = await self.client.get_chunk_by_id(chunk_id)
|
chunk = await self.client.get_chunk_by_id(chunk_id)
|
||||||
|
|
@ -451,7 +451,7 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
async with HaikuRAG(
|
async with HaikuRAG(
|
||||||
db_path=self.db_path,
|
db_path=self.db_path,
|
||||||
config=self.config,
|
config=self.config,
|
||||||
read_only=self.read_only,
|
read_only=True,
|
||||||
before=self.before,
|
before=self.before,
|
||||||
) as self.client:
|
) as self.client:
|
||||||
answer, citations = await self.client.ask(question, filter=filter)
|
answer, citations = await self.client.ask(question, filter=filter)
|
||||||
|
|
@ -479,7 +479,7 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
async with HaikuRAG(
|
async with HaikuRAG(
|
||||||
db_path=self.db_path,
|
db_path=self.db_path,
|
||||||
config=self.config,
|
config=self.config,
|
||||||
read_only=self.read_only,
|
read_only=True,
|
||||||
before=self.before,
|
before=self.before,
|
||||||
) as self.client:
|
) as self.client:
|
||||||
self.console.print(f"[bold blue]Question:[/bold blue] {question}")
|
self.console.print(f"[bold blue]Question:[/bold blue] {question}")
|
||||||
|
|
@ -506,6 +506,14 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
read_only=self.read_only,
|
read_only=self.read_only,
|
||||||
before=self.before,
|
before=self.before,
|
||||||
) as client:
|
) as client:
|
||||||
|
if mode == RebuildMode.SET_EMBEDDER:
|
||||||
|
async for _ in client.rebuild_database(mode=mode):
|
||||||
|
pass
|
||||||
|
self.console.print(
|
||||||
|
"[bold green]Stored embedder settings updated.[/bold green]"
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
documents = await client.list_documents()
|
documents = await client.list_documents()
|
||||||
total_docs = len(documents)
|
total_docs = len(documents)
|
||||||
|
|
||||||
|
|
@ -558,6 +566,7 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
config=self.config,
|
config=self.config,
|
||||||
skip_validation=True,
|
skip_validation=True,
|
||||||
skip_migration_check=True,
|
skip_migration_check=True,
|
||||||
|
read_only=self.read_only,
|
||||||
) as store:
|
) as store:
|
||||||
return await store.migrate()
|
return await store.migrate()
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -406,7 +406,7 @@ def analyze( # pragma: no cover
|
||||||
@_cli.command("settings", help="Display current configuration settings")
|
@_cli.command("settings", help="Display current configuration settings")
|
||||||
def settings(): # pragma: no cover
|
def settings(): # pragma: no cover
|
||||||
config = get_config()
|
config = get_config()
|
||||||
app = HaikuRAGApp(db_path=Path(), config=config)
|
app = HaikuRAGApp(db_path=Path(), config=config, read_only=True)
|
||||||
app.show_settings()
|
app.show_settings()
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -476,14 +476,23 @@ def rebuild(
|
||||||
"parse entirely. Requires processing.pictures='description'."
|
"parse entirely. Requires processing.pictures='description'."
|
||||||
),
|
),
|
||||||
),
|
),
|
||||||
|
set_embedder: bool = typer.Option(
|
||||||
|
False,
|
||||||
|
"--set-embedder",
|
||||||
|
help=(
|
||||||
|
"Adopt the current embedder identity without re-embedding, when the "
|
||||||
|
"vector dimension is unchanged. Use after swapping the serving stack "
|
||||||
|
"for the same model (e.g. Ollama to vLLM)."
|
||||||
|
),
|
||||||
|
),
|
||||||
):
|
):
|
||||||
from haiku.rag.client import RebuildMode
|
from haiku.rag.client import RebuildMode
|
||||||
|
|
||||||
exclusive = sum([embed_only, rechunk, title_only, descriptions])
|
exclusive = sum([embed_only, rechunk, title_only, descriptions, set_embedder])
|
||||||
if exclusive > 1:
|
if exclusive > 1:
|
||||||
typer.echo(
|
typer.echo(
|
||||||
"Error: --embed-only, --rechunk, --title-only, and --descriptions "
|
"Error: --embed-only, --rechunk, --title-only, --descriptions, and "
|
||||||
"are mutually exclusive"
|
"--set-embedder are mutually exclusive"
|
||||||
)
|
)
|
||||||
raise typer.Exit(1)
|
raise typer.Exit(1)
|
||||||
|
|
||||||
|
|
@ -495,6 +504,8 @@ def rebuild(
|
||||||
mode = RebuildMode.TITLE_ONLY
|
mode = RebuildMode.TITLE_ONLY
|
||||||
elif descriptions: # pragma: no cover
|
elif descriptions: # pragma: no cover
|
||||||
mode = RebuildMode.DESCRIPTIONS
|
mode = RebuildMode.DESCRIPTIONS
|
||||||
|
elif set_embedder: # pragma: no cover
|
||||||
|
mode = RebuildMode.SET_EMBEDDER
|
||||||
else: # pragma: no cover
|
else: # pragma: no cover
|
||||||
mode = RebuildMode.FULL
|
mode = RebuildMode.FULL
|
||||||
|
|
||||||
|
|
@ -601,7 +612,7 @@ def history( # pragma: no cover
|
||||||
|
|
||||||
@_cli.command("download-models", help="Download Docling and Ollama models per config")
|
@_cli.command("download-models", help="Download Docling and Ollama models per config")
|
||||||
def download_models_cmd(): # pragma: no cover
|
def download_models_cmd(): # pragma: no cover
|
||||||
app = HaikuRAGApp(db_path=Path(), config=get_config())
|
app = HaikuRAGApp(db_path=Path(), config=get_config(), read_only=True)
|
||||||
try:
|
try:
|
||||||
asyncio.run(app.download_models())
|
asyncio.run(app.download_models())
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
@ -625,7 +636,7 @@ def inspect( # pragma: no cover
|
||||||
raise typer.Exit(1) from e
|
raise typer.Exit(1) from e
|
||||||
|
|
||||||
db_path = db if db else get_config().storage.data_dir / "haiku.rag.lancedb"
|
db_path = db if db else get_config().storage.data_dir / "haiku.rag.lancedb"
|
||||||
run_inspector(db_path, read_only=_read_only, before=_before)
|
run_inspector(db_path, read_only=True, before=_before)
|
||||||
|
|
||||||
|
|
||||||
@_cli.command("chat", help="Launch interactive chat TUI for conversational RAG")
|
@_cli.command("chat", help="Launch interactive chat TUI for conversational RAG")
|
||||||
|
|
@ -655,7 +666,7 @@ def chat( # pragma: no cover
|
||||||
|
|
||||||
run_chat(
|
run_chat(
|
||||||
db_path,
|
db_path,
|
||||||
read_only=_read_only,
|
read_only=True,
|
||||||
before=_before,
|
before=_before,
|
||||||
model=model,
|
model=model,
|
||||||
skills=skills,
|
skills=skills,
|
||||||
|
|
|
||||||
|
|
@ -49,6 +49,8 @@ class RebuildMode(Enum):
|
||||||
TITLE_ONLY = "title_only" # Only generate titles for untitled documents
|
TITLE_ONLY = "title_only" # Only generate titles for untitled documents
|
||||||
DESCRIPTIONS = "descriptions" # Run the VLM over already-stored picture
|
DESCRIPTIONS = "descriptions" # Run the VLM over already-stored picture
|
||||||
# bytes, patch descriptions into the docling blob, then re-chunk + re-embed.
|
# bytes, patch descriptions into the docling blob, then re-chunk + re-embed.
|
||||||
|
SET_EMBEDDER = "set_embedder" # Adopt the current embedder identity without
|
||||||
|
# re-embedding, when the vector dimension is unchanged.
|
||||||
|
|
||||||
|
|
||||||
class HaikuRAG:
|
class HaikuRAG:
|
||||||
|
|
|
||||||
|
|
@ -72,6 +72,10 @@ async def rebuild_database(
|
||||||
if mode is None:
|
if mode is None:
|
||||||
mode = RebuildMode.FULL
|
mode = RebuildMode.FULL
|
||||||
|
|
||||||
|
if mode == RebuildMode.SET_EMBEDDER:
|
||||||
|
await _set_embedder(client)
|
||||||
|
return
|
||||||
|
|
||||||
# Resolve any leftover staging/marker tables from a previously
|
# Resolve any leftover staging/marker tables from a previously
|
||||||
# interrupted rebuild. Returns True only when phase 1 was already
|
# interrupted rebuild. Returns True only when phase 1 was already
|
||||||
# complete and the current mode is EMBED_ONLY, in which case we resume
|
# complete and the current mode is EMBED_ONLY, in which case we resume
|
||||||
|
|
@ -131,6 +135,29 @@ async def rebuild_database(
|
||||||
logger.warning("Post-rebuild vacuum failed", exc_info=True)
|
logger.warning("Post-rebuild vacuum failed", exc_info=True)
|
||||||
|
|
||||||
|
|
||||||
|
async def _set_embedder(client: "HaikuRAG") -> None:
|
||||||
|
"""Adopt the current embedder identity without re-embedding.
|
||||||
|
|
||||||
|
Only valid when the vector dimension is unchanged — the stored vectors stay
|
||||||
|
usable, so just the recorded provider/name are updated. A changed dimension
|
||||||
|
requires regenerating every embedding via a full rebuild.
|
||||||
|
"""
|
||||||
|
from haiku.rag.store.repositories.settings import ConfigMismatchError
|
||||||
|
|
||||||
|
settings_repo = SettingsRepository(client.store)
|
||||||
|
stored = await settings_repo.get_current_settings()
|
||||||
|
stored_dim = stored.get("embeddings", {}).get("model", {}).get("vector_dim")
|
||||||
|
current_dim = client._config.embeddings.model.vector_dim
|
||||||
|
|
||||||
|
if stored_dim is not None and current_dim != stored_dim:
|
||||||
|
raise ConfigMismatchError(
|
||||||
|
f"Stored vector dimension {stored_dim} differs from current "
|
||||||
|
f"{current_dim}; embeddings must be regenerated. Run 'haiku-rag rebuild'."
|
||||||
|
)
|
||||||
|
|
||||||
|
await settings_repo.save_current_settings()
|
||||||
|
|
||||||
|
|
||||||
async def _hydrate(
|
async def _hydrate(
|
||||||
client: "HaikuRAG", light_docs: list[Document]
|
client: "HaikuRAG", light_docs: list[Document]
|
||||||
) -> AsyncGenerator[Document, None]:
|
) -> AsyncGenerator[Document, None]:
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@ import lancedb
|
||||||
import pyarrow as pa
|
import pyarrow as pa
|
||||||
from lancedb.index import FTS, BTree, IvfPq
|
from lancedb.index import FTS, BTree, IvfPq
|
||||||
from lancedb.pydantic import LanceModel, Vector
|
from lancedb.pydantic import LanceModel, Vector
|
||||||
|
from packaging.version import parse
|
||||||
from pydantic import Field
|
from pydantic import Field
|
||||||
|
|
||||||
from haiku.rag.config import AppConfig, Config
|
from haiku.rag.config import AppConfig, Config
|
||||||
|
|
@ -509,7 +510,7 @@ class Store:
|
||||||
await self.set_haiku_version(metadata.version("haiku.rag-slim"))
|
await self.set_haiku_version(metadata.version("haiku.rag-slim"))
|
||||||
|
|
||||||
async def _check_migrations(self) -> None:
|
async def _check_migrations(self) -> None:
|
||||||
"""Check if migrations are pending and error or update version accordingly.
|
"""Raise if migrations are pending. Opening never writes the version.
|
||||||
|
|
||||||
Raises:
|
Raises:
|
||||||
MigrationRequiredError: If migrations are pending.
|
MigrationRequiredError: If migrations are pending.
|
||||||
|
|
@ -529,10 +530,6 @@ class Store:
|
||||||
"Run 'haiku-rag migrate' to upgrade."
|
"Run 'haiku-rag migrate' to upgrade."
|
||||||
)
|
)
|
||||||
|
|
||||||
# No pending migrations - update version silently if needed (writable only)
|
|
||||||
if not self._read_only and db_version != current_version:
|
|
||||||
await self.set_haiku_version(current_version)
|
|
||||||
|
|
||||||
async def migrate(self) -> list[str]:
|
async def migrate(self) -> list[str]:
|
||||||
"""Run pending database migrations.
|
"""Run pending database migrations.
|
||||||
|
|
||||||
|
|
@ -551,8 +548,9 @@ class Store:
|
||||||
|
|
||||||
applied = await run_pending_upgrades(self, db_version)
|
applied = await run_pending_upgrades(self, db_version)
|
||||||
|
|
||||||
# Update version after successful migration
|
# Advance the schema marker only forward — never downgrade a database
|
||||||
if applied or db_version != current_version:
|
# opened with an older build than last stamped it.
|
||||||
|
if parse(current_version) > parse(db_version):
|
||||||
await self.set_haiku_version(current_version)
|
await self.set_haiku_version(current_version)
|
||||||
|
|
||||||
return applied
|
return applied
|
||||||
|
|
|
||||||
|
|
@ -102,20 +102,21 @@ class SettingsRepository:
|
||||||
await self.store.settings_table.add([settings_record])
|
await self.store.settings_table.add([settings_record])
|
||||||
|
|
||||||
async def validate_config_compatibility(self) -> None:
|
async def validate_config_compatibility(self) -> None:
|
||||||
"""Validate that the current configuration is compatible with stored settings.
|
"""Validate the current configuration against stored settings without writing.
|
||||||
|
|
||||||
``vector_dim`` mismatches raise — corpus and query vectors must live in
|
Opening a database never modifies it. ``vector_dim`` mismatches raise —
|
||||||
the same dimensional space. ``provider`` and ``name`` mismatches are
|
corpus and query vectors must live in the same dimensional space.
|
||||||
treated as soft drift: legitimate when the same model is served by a
|
``provider`` and ``name`` drift (with matching ``vector_dim``) is soft:
|
||||||
different stack (Ollama vs vLLM-via-openai, etc.). Surface the change
|
legitimate when the same model is served by a different stack (Ollama vs
|
||||||
once via a warning and overwrite stored settings so the warning does
|
vLLM-via-openai, etc.). Drift is surfaced via a warning; a writable open
|
||||||
not fire on every subsequent open.
|
then raises so a write cannot mix embedding identities in the corpus,
|
||||||
|
while a read-only open continues. Stored settings are reconciled
|
||||||
|
explicitly via ``haiku-rag rebuild --set-embedder``, never on open.
|
||||||
"""
|
"""
|
||||||
stored_settings = await self.get_current_settings()
|
stored_settings = await self.get_current_settings()
|
||||||
|
|
||||||
# If no stored settings, this is a new database - save current config and return
|
# Nothing stored to validate against — never write on open.
|
||||||
if not stored_settings:
|
if not stored_settings:
|
||||||
await self.save_current_settings()
|
|
||||||
return
|
return
|
||||||
|
|
||||||
current_config = self.store._config.model_dump(mode="json")
|
current_config = self.store._config.model_dump(mode="json")
|
||||||
|
|
@ -155,9 +156,15 @@ class SettingsRepository:
|
||||||
|
|
||||||
if soft_changes:
|
if soft_changes:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
"Embedding identity changed (vector_dim matches, stored settings "
|
"Embedding identity changed (vector_dim matches): %s. If this is "
|
||||||
"will be updated to match current config): %s. If this is not "
|
"intentional, run 'haiku-rag rebuild --set-embedder' to update the "
|
||||||
"intentional, revert your config to match the stored settings.",
|
"stored settings; otherwise revert your config to match the database.",
|
||||||
"; ".join(soft_changes),
|
"; ".join(soft_changes),
|
||||||
)
|
)
|
||||||
await self.save_current_settings()
|
if not self.store.is_read_only:
|
||||||
|
raise ConfigMismatchError(
|
||||||
|
"Database embedding identity differs from current config "
|
||||||
|
f"(vector_dim matches): {'; '.join(soft_changes)}. "
|
||||||
|
"Run 'haiku-rag rebuild --set-embedder' to adopt the current "
|
||||||
|
"embedder, or revert your config to match the database."
|
||||||
|
)
|
||||||
|
|
|
||||||
|
|
@ -38,21 +38,19 @@ class TestMigrationCheck:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_version_bump_without_pending_migrations_updates_silently(
|
async def test_open_does_not_change_version(self, temp_db_path):
|
||||||
self, temp_db_path
|
"""Opening a database never writes the version, even when it differs.
|
||||||
):
|
|
||||||
"""When version is outdated but no migrations pending, update version silently."""
|
A version newer than all upgrade steps has no pending migrations, so the
|
||||||
|
open succeeds; the stored version must be left untouched (no downgrade,
|
||||||
|
no bump) because opening is a pure read.
|
||||||
|
"""
|
||||||
async with Store(temp_db_path, create=True) as store:
|
async with Store(temp_db_path, create=True) as store:
|
||||||
# Set an older version that has no pending migrations
|
|
||||||
# (newer than all current upgrade steps)
|
|
||||||
await store.set_haiku_version("100.0.0")
|
await store.set_haiku_version("100.0.0")
|
||||||
|
|
||||||
# Re-open - should update version silently, no error
|
# Re-open writable - no error, version unchanged
|
||||||
async with Store(temp_db_path) as store:
|
async with Store(temp_db_path) as store:
|
||||||
# Version should now be current
|
assert await store.get_haiku_version() == "100.0.0"
|
||||||
version = await store.get_haiku_version()
|
|
||||||
expected = metadata.version("haiku.rag-slim")
|
|
||||||
assert version == expected
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_pending_migrations_raises_error(self, temp_db_path):
|
async def test_pending_migrations_raises_error(self, temp_db_path):
|
||||||
|
|
@ -160,6 +158,21 @@ class TestMigrateMethod:
|
||||||
applied = await store.migrate()
|
applied = await store.migrate()
|
||||||
assert applied == []
|
assert applied == []
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_migrate_does_not_downgrade_future_version(self, temp_db_path):
|
||||||
|
"""migrate() must not move the version backwards.
|
||||||
|
|
||||||
|
Running an older build's migrate against a DB stamped with a newer
|
||||||
|
version (no pending upgrades) leaves the stored version untouched.
|
||||||
|
"""
|
||||||
|
async with Store(temp_db_path, create=True) as store:
|
||||||
|
await store.set_haiku_version("100.0.0")
|
||||||
|
|
||||||
|
async with Store(temp_db_path, skip_migration_check=True) as store:
|
||||||
|
applied = await store.migrate()
|
||||||
|
assert applied == []
|
||||||
|
assert await store.get_haiku_version() == "100.0.0"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_migrate_raises_read_only_error(self, temp_db_path):
|
async def test_migrate_raises_read_only_error(self, temp_db_path):
|
||||||
"""Store.migrate() should raise ReadOnlyError in read-only mode."""
|
"""Store.migrate() should raise ReadOnlyError in read-only mode."""
|
||||||
|
|
|
||||||
|
|
@ -298,3 +298,61 @@ class TestClientReadOnly:
|
||||||
async with HaikuRAG(temp_db_path, read_only=True) as client:
|
async with HaikuRAG(temp_db_path, read_only=True) as client:
|
||||||
docs = await client.list_documents()
|
docs = await client.list_documents()
|
||||||
assert len(docs) == 1
|
assert len(docs) == 1
|
||||||
|
|
||||||
|
|
||||||
|
class TestAppReadVerbsDoNotWrite:
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_read_verb_leaves_settings_and_version_unchanged_on_drift(
|
||||||
|
self, temp_db_path
|
||||||
|
):
|
||||||
|
"""A read CLI verb opens read-only: drift warns but never writes."""
|
||||||
|
from haiku.rag.app import HaikuRAGApp
|
||||||
|
from haiku.rag.config import AppConfig
|
||||||
|
|
||||||
|
async with Store(temp_db_path, create=True) as store:
|
||||||
|
stored_name_before = (
|
||||||
|
await SettingsRepository(store).get_current_settings()
|
||||||
|
)["embeddings"]["model"]["name"]
|
||||||
|
version_before = await store.get_haiku_version()
|
||||||
|
|
||||||
|
drift = AppConfig()
|
||||||
|
drift.embeddings.model.name = "different-model"
|
||||||
|
|
||||||
|
# list is a read verb — must open read-only and not raise on drift
|
||||||
|
app = HaikuRAGApp(db_path=temp_db_path, config=drift)
|
||||||
|
await app.list_documents()
|
||||||
|
|
||||||
|
async with Store(temp_db_path, skip_validation=True, read_only=True) as store:
|
||||||
|
stored_name_after = (
|
||||||
|
await SettingsRepository(store).get_current_settings()
|
||||||
|
)["embeddings"]["model"]["name"]
|
||||||
|
version_after = await store.get_haiku_version()
|
||||||
|
|
||||||
|
assert stored_name_after == stored_name_before
|
||||||
|
assert version_after == version_before
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_write_verb_raises_on_drift_without_writing(self, temp_db_path):
|
||||||
|
"""A write CLI verb opens writable: drift raises before any write."""
|
||||||
|
from haiku.rag.app import HaikuRAGApp
|
||||||
|
from haiku.rag.config import AppConfig
|
||||||
|
from haiku.rag.store.repositories.settings import ConfigMismatchError
|
||||||
|
|
||||||
|
async with Store(temp_db_path, create=True) as store:
|
||||||
|
stored_name_before = (
|
||||||
|
await SettingsRepository(store).get_current_settings()
|
||||||
|
)["embeddings"]["model"]["name"]
|
||||||
|
|
||||||
|
drift = AppConfig()
|
||||||
|
drift.embeddings.model.name = "different-model"
|
||||||
|
|
||||||
|
app = HaikuRAGApp(db_path=temp_db_path, config=drift)
|
||||||
|
with pytest.raises(ConfigMismatchError):
|
||||||
|
await app.add_document_from_text("hello")
|
||||||
|
|
||||||
|
async with Store(temp_db_path, skip_validation=True, read_only=True) as store:
|
||||||
|
stored_name_after = (
|
||||||
|
await SettingsRepository(store).get_current_settings()
|
||||||
|
)["embeddings"]["model"]["name"]
|
||||||
|
|
||||||
|
assert stored_name_after == stored_name_before
|
||||||
|
|
|
||||||
|
|
@ -960,3 +960,84 @@ async def test_rebuild_descriptions_raises_when_blob_is_missing(
|
||||||
with pytest.raises(ValueError, match="rebuild --descriptions requires"):
|
with pytest.raises(ValueError, match="rebuild --descriptions requires"):
|
||||||
async for _ in rag.rebuild_database(mode=RebuildMode.DESCRIPTIONS):
|
async for _ in rag.rebuild_database(mode=RebuildMode.DESCRIPTIONS):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
async def _add_chunk(client: HaikuRAG, vector: list[float]) -> str:
|
||||||
|
"""Insert a chunk row directly, bypassing the embedder."""
|
||||||
|
record = client.store.ChunkRecord(
|
||||||
|
id="chunk-1",
|
||||||
|
document_id="doc-1",
|
||||||
|
content="hello",
|
||||||
|
content_fts="hello",
|
||||||
|
metadata="{}",
|
||||||
|
order=0,
|
||||||
|
vector=vector,
|
||||||
|
)
|
||||||
|
await client.store.chunks_table.add([record])
|
||||||
|
return record.id
|
||||||
|
|
||||||
|
|
||||||
|
async def _stored_embedding_name(client: HaikuRAG) -> str:
|
||||||
|
from haiku.rag.store.repositories.settings import SettingsRepository
|
||||||
|
|
||||||
|
settings = await SettingsRepository(client.store).get_current_settings()
|
||||||
|
return settings["embeddings"]["model"]["name"]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_rebuild_set_embedder_adopts_identity_without_reembedding(temp_db_path):
|
||||||
|
"""SET_EMBEDDER updates stored embedder identity and leaves vectors untouched."""
|
||||||
|
from haiku.rag.config import AppConfig
|
||||||
|
|
||||||
|
dim = AppConfig().embeddings.model.vector_dim
|
||||||
|
sentinel = [0.5] * dim
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||||
|
await _add_chunk(client, sentinel)
|
||||||
|
|
||||||
|
drift = AppConfig()
|
||||||
|
drift.embeddings.model.name = "different-model"
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, config=drift, skip_validation=True) as client:
|
||||||
|
async for _ in client.rebuild_database(mode=RebuildMode.SET_EMBEDDER):
|
||||||
|
pass
|
||||||
|
|
||||||
|
assert await _stored_embedding_name(client) == "different-model"
|
||||||
|
|
||||||
|
rows = (await client.store.chunks_table.query().to_arrow()).to_pylist()
|
||||||
|
assert len(rows) == 1
|
||||||
|
assert rows[0]["vector"] == pytest.approx(sentinel)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_rebuild_set_embedder_works_on_empty_database(temp_db_path):
|
||||||
|
"""SET_EMBEDDER reconciles even with no documents (preflight must be bypassed)."""
|
||||||
|
from haiku.rag.app import HaikuRAGApp
|
||||||
|
from haiku.rag.config import AppConfig
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True):
|
||||||
|
pass
|
||||||
|
|
||||||
|
drift = AppConfig()
|
||||||
|
drift.embeddings.model.name = "different-model"
|
||||||
|
|
||||||
|
app = HaikuRAGApp(db_path=temp_db_path, config=drift)
|
||||||
|
await app.rebuild(mode=RebuildMode.SET_EMBEDDER)
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, config=drift, skip_validation=True) as client:
|
||||||
|
assert await _stored_embedding_name(client) == "different-model"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_rebuild_set_embedder_raises_on_vector_dim_mismatch(temp_db_path):
|
||||||
|
"""SET_EMBEDDER refuses when the vector dimension changed — a full rebuild is needed."""
|
||||||
|
from haiku.rag.config import AppConfig
|
||||||
|
from haiku.rag.store.repositories.settings import ConfigMismatchError
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True):
|
||||||
|
pass
|
||||||
|
|
||||||
|
drift = AppConfig()
|
||||||
|
drift.embeddings.model.vector_dim = 9999
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, config=drift, skip_validation=True) as client:
|
||||||
|
with pytest.raises(ConfigMismatchError):
|
||||||
|
async for _ in client.rebuild_database(mode=RebuildMode.SET_EMBEDDER):
|
||||||
|
pass
|
||||||
|
|
|
||||||
|
|
@ -48,8 +48,8 @@ class TestValidateConfigCompatibility:
|
||||||
"""Tests for validate_config_compatibility method."""
|
"""Tests for validate_config_compatibility method."""
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_empty_settings_saves_config(self, temp_db_path):
|
async def test_empty_settings_does_not_write(self, temp_db_path):
|
||||||
"""When settings row is missing, validation saves current config."""
|
"""Validation never writes on open, even when the settings row is missing."""
|
||||||
from haiku.rag.store.engine import Store
|
from haiku.rag.store.engine import Store
|
||||||
from haiku.rag.store.repositories.settings import SettingsRepository
|
from haiku.rag.store.repositories.settings import SettingsRepository
|
||||||
|
|
||||||
|
|
@ -60,14 +60,10 @@ class TestValidateConfigCompatibility:
|
||||||
await store.settings_table.delete("id = 'settings'")
|
await store.settings_table.delete("id = 'settings'")
|
||||||
assert await settings_repo.get_current_settings() == {}
|
assert await settings_repo.get_current_settings() == {}
|
||||||
|
|
||||||
# Validation should save settings
|
# Validation must not write — nothing to validate against
|
||||||
await settings_repo.validate_config_compatibility()
|
await settings_repo.validate_config_compatibility()
|
||||||
|
|
||||||
# Now settings should exist
|
assert await settings_repo.get_current_settings() == {}
|
||||||
saved = await settings_repo.get_current_settings()
|
|
||||||
assert (
|
|
||||||
saved.get("embeddings", {}).get("model", {}).get("provider") is not None
|
|
||||||
)
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_compatible_config_no_error(self, temp_db_path):
|
async def test_compatible_config_no_error(self, temp_db_path):
|
||||||
|
|
@ -82,15 +78,14 @@ class TestValidateConfigCompatibility:
|
||||||
await settings_repo.validate_config_compatibility()
|
await settings_repo.validate_config_compatibility()
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_provider_mismatch_warns_and_syncs(
|
async def test_provider_drift_read_only_warns_without_writing(
|
||||||
self, temp_db_path, caplog, monkeypatch
|
self, temp_db_path, caplog, monkeypatch
|
||||||
):
|
):
|
||||||
"""Provider mismatch with matching vector_dim warns and overwrites stored.
|
"""Provider drift (vector_dim matches) on a read-only store warns and continues.
|
||||||
|
|
||||||
Same model served by a different stack (Ollama vs vLLM via openai-compat)
|
Same model served by a different stack (Ollama vs vLLM via openai-compat)
|
||||||
legitimately differs in `provider`. Validation should surface the change
|
legitimately differs in `provider`. A read-only open surfaces the change
|
||||||
once, then trust the user's config as the new source of truth so the
|
but must never modify the stored settings.
|
||||||
warning doesn't fire on every subsequent open.
|
|
||||||
"""
|
"""
|
||||||
from haiku.rag.store.engine import Store
|
from haiku.rag.store.engine import Store
|
||||||
from haiku.rag.store.repositories.settings import SettingsRepository
|
from haiku.rag.store.repositories.settings import SettingsRepository
|
||||||
|
|
@ -107,7 +102,7 @@ class TestValidateConfigCompatibility:
|
||||||
new_config.embeddings.model.provider = "openai"
|
new_config.embeddings.model.provider = "openai"
|
||||||
|
|
||||||
async with Store(
|
async with Store(
|
||||||
temp_db_path, config=new_config, skip_validation=True
|
temp_db_path, config=new_config, skip_validation=True, read_only=True
|
||||||
) as store2:
|
) as store2:
|
||||||
settings_repo = SettingsRepository(store2)
|
settings_repo = SettingsRepository(store2)
|
||||||
|
|
||||||
|
|
@ -122,25 +117,57 @@ class TestValidateConfigCompatibility:
|
||||||
for r in caplog.records
|
for r in caplog.records
|
||||||
)
|
)
|
||||||
|
|
||||||
# Stored settings now match current
|
# Stored settings are untouched
|
||||||
saved = await settings_repo.get_current_settings()
|
saved = await settings_repo.get_current_settings()
|
||||||
assert saved["embeddings"]["model"]["provider"] == "openai"
|
assert saved["embeddings"]["model"]["provider"] == "ollama"
|
||||||
|
|
||||||
# Second open is silent — stored matches current.
|
|
||||||
caplog.clear()
|
|
||||||
with caplog.at_level(logging.WARNING):
|
|
||||||
await settings_repo.validate_config_compatibility()
|
|
||||||
assert not any("provider" in r.getMessage() for r in caplog.records)
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_model_mismatch_warns_and_syncs(
|
async def test_provider_drift_writable_raises_without_writing(
|
||||||
self, temp_db_path, caplog, monkeypatch
|
self, temp_db_path, caplog, monkeypatch
|
||||||
):
|
):
|
||||||
"""Model name mismatch with matching vector_dim warns and overwrites stored."""
|
"""Provider drift on a writable store warns and raises, without writing."""
|
||||||
|
from haiku.rag.store.engine import Store
|
||||||
|
from haiku.rag.store.repositories.settings import (
|
||||||
|
ConfigMismatchError,
|
||||||
|
SettingsRepository,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(logging.getLogger("haiku.rag"), "propagate", True)
|
||||||
|
|
||||||
|
async with Store(temp_db_path, create=True):
|
||||||
|
pass
|
||||||
|
|
||||||
|
new_config = AppConfig()
|
||||||
|
new_config.embeddings.model.provider = "openai"
|
||||||
|
|
||||||
|
async with Store(
|
||||||
|
temp_db_path, config=new_config, skip_validation=True
|
||||||
|
) as store2:
|
||||||
|
settings_repo = SettingsRepository(store2)
|
||||||
|
|
||||||
|
with caplog.at_level(logging.WARNING):
|
||||||
|
with pytest.raises(ConfigMismatchError):
|
||||||
|
await settings_repo.validate_config_compatibility()
|
||||||
|
|
||||||
|
assert any(
|
||||||
|
"provider" in r.getMessage()
|
||||||
|
and "ollama" in r.getMessage()
|
||||||
|
and "openai" in r.getMessage()
|
||||||
|
for r in caplog.records
|
||||||
|
)
|
||||||
|
|
||||||
|
# Stored settings are untouched despite the writable open
|
||||||
|
saved = await settings_repo.get_current_settings()
|
||||||
|
assert saved["embeddings"]["model"]["provider"] == "ollama"
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_model_drift_read_only_warns_without_writing(
|
||||||
|
self, temp_db_path, caplog, monkeypatch
|
||||||
|
):
|
||||||
|
"""Model name drift (vector_dim matches) on a read-only store warns, no write."""
|
||||||
from haiku.rag.store.engine import Store
|
from haiku.rag.store.engine import Store
|
||||||
from haiku.rag.store.repositories.settings import SettingsRepository
|
from haiku.rag.store.repositories.settings import SettingsRepository
|
||||||
|
|
||||||
# See test_provider_mismatch_warns_and_syncs for the propagate=True rationale.
|
|
||||||
monkeypatch.setattr(logging.getLogger("haiku.rag"), "propagate", True)
|
monkeypatch.setattr(logging.getLogger("haiku.rag"), "propagate", True)
|
||||||
|
|
||||||
async with Store(temp_db_path, create=True):
|
async with Store(temp_db_path, create=True):
|
||||||
|
|
@ -150,7 +177,7 @@ class TestValidateConfigCompatibility:
|
||||||
new_config.embeddings.model.name = "different-model"
|
new_config.embeddings.model.name = "different-model"
|
||||||
|
|
||||||
async with Store(
|
async with Store(
|
||||||
temp_db_path, config=new_config, skip_validation=True
|
temp_db_path, config=new_config, skip_validation=True, read_only=True
|
||||||
) as store2:
|
) as store2:
|
||||||
settings_repo = SettingsRepository(store2)
|
settings_repo = SettingsRepository(store2)
|
||||||
|
|
||||||
|
|
@ -163,7 +190,7 @@ class TestValidateConfigCompatibility:
|
||||||
)
|
)
|
||||||
|
|
||||||
saved = await settings_repo.get_current_settings()
|
saved = await settings_repo.get_current_settings()
|
||||||
assert saved["embeddings"]["model"]["name"] == "different-model"
|
assert saved["embeddings"]["model"]["name"] != "different-model"
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_vector_dim_mismatch_raises_error(self, temp_db_path):
|
async def test_vector_dim_mismatch_raises_error(self, temp_db_path):
|
||||||
|
|
@ -189,3 +216,25 @@ class TestValidateConfigCompatibility:
|
||||||
|
|
||||||
assert "vector dimension" in str(exc_info.value)
|
assert "vector dimension" in str(exc_info.value)
|
||||||
assert "9999" in str(exc_info.value)
|
assert "9999" in str(exc_info.value)
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_vector_dim_mismatch_raises_error_read_only(self, temp_db_path):
|
||||||
|
"""vector_dim mismatch raises even read-only — search cannot work."""
|
||||||
|
from haiku.rag.store.engine import Store
|
||||||
|
from haiku.rag.store.repositories.settings import SettingsRepository
|
||||||
|
|
||||||
|
async with Store(temp_db_path, create=True):
|
||||||
|
pass
|
||||||
|
|
||||||
|
new_config = AppConfig()
|
||||||
|
new_config.embeddings.model.vector_dim = 9999
|
||||||
|
|
||||||
|
async with Store(
|
||||||
|
temp_db_path, config=new_config, skip_validation=True, read_only=True
|
||||||
|
) as store2:
|
||||||
|
settings_repo = SettingsRepository(store2)
|
||||||
|
|
||||||
|
with pytest.raises(ConfigMismatchError) as exc_info:
|
||||||
|
await settings_repo.validate_config_compatibility()
|
||||||
|
|
||||||
|
assert "9999" in str(exc_info.value)
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue