diff --git a/CHANGELOG.md b/CHANGELOG.md index 6fa78a95..5712c704 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ ### Added - `api_key` on model and embedding-model config, overriding the provider's environment variable. Honored on the `openai` and `ollama` providers, `vllm` embedders and rerankers, the picture-description VLM endpoint, and `doctor`'s endpoint probes; other providers raise. -- `lancedb.databases`: a name-to-location mapping for searching several databases at once, mutually exclusive with `lancedb.uri`. `client.search(..., sources=[...])` selects which to search, `sources=None` searches all of them, and `SearchResult.source` carries the configured name a result came from. `Document.source` names it on a document from a listing or a lookup. Candidates are fused by the configured reranker over the union, or by reciprocal rank fusion when none is configured. `SearchResult.format_for_agent` names the database, so the model can attribute evidence to one while it answers. `haiku-rag search`, `ask`, `analyze` and `chat` cover the configured set and label each result and citation with its database; every other command works on one, named with `--database NAME` or `--db PATH`. +- `lancedb.databases`: a name-to-location mapping for searching several databases at once, mutually exclusive with `lancedb.uri`. `client.search(..., sources=[...])` selects which to search, `sources=None` searches all of them, and `SearchResult.source` carries the configured name a result came from. `Document.source` names it on a document from a listing or a lookup. Candidates are fused by the configured reranker over the union, or by reciprocal rank fusion when none is configured. Databases searched together must have been written with the same embedder; two that disagree raise `ConfigMismatchError`. `SearchResult.format_for_agent` names the database, so the model can attribute evidence to one while it answers. `haiku-rag search`, `ask`, `analyze` and `chat` cover the configured set and label each result and citation with its database; every other command works on one, named with `--database NAME` or `--db PATH`. - `client.ask(..., sources=[...])` asks across the selected databases, and `Citation.source` names the one a cited chunk came from. The cite fallback for an id absent from the run's results looks only in the selected databases, so a question scoped to some cannot cite another. - `client.analyze(..., sources=[...])` analyzes across the selected databases: the sandbox mounts their documents under one flat `/documents/{id}/` namespace, resolving each id to the database holding it, and in-code `search()` covers the same selection. diff --git a/docs/configuration/storage.md b/docs/configuration/storage.md index 3b7ce5eb..bf17750e 100644 --- a/docs/configuration/storage.md +++ b/docs/configuration/storage.md @@ -194,9 +194,16 @@ The name is the only identity that leaves the configuration. Results, citations and error messages carry it, so a path or a bucket never reaches a log, a trace or a model. -Every database in the set is opened with the same embedding configuration, so -they have to agree on it. One whose stored settings differ raises -`ConfigMismatchError` when it is opened. +Every database in the set is opened with the same embedding configuration. A +different `vector_dim` raises `ConfigMismatchError` on open. A different provider +or model name at the same dimension is a warning on a read-only open, since the +same model served by another stack is spelled differently, and raises on a +writable one. + +Searching embeds the query once for the whole selection, so the databases +searched together must have been written with the same embedder. Two that +disagree with each other raise `ConfigMismatchError` naming both, whatever the +configuration says. ### Searching a set diff --git a/haiku_rag_slim/haiku/rag/client/__init__.py b/haiku_rag_slim/haiku/rag/client/__init__.py index c69240c7..d4a6a467 100644 --- a/haiku_rag_slim/haiku/rag/client/__init__.py +++ b/haiku_rag_slim/haiku/rag/client/__init__.py @@ -60,6 +60,12 @@ _VACUUM_MIN_INTERVAL_S = 300.0 _NAMEABLE_FAILURES = (MigrationRequiredError, ConfigMismatchError, ReadOnlyError) +def _spell(embedding: tuple[str | None, str | None, int | None]) -> str: + """An embedder identity, for an error message.""" + provider, name, vector_dim = embedding + return f"{provider}/{name} at {vector_dim} dimensions" + + def _without_repeats(names: list[str]) -> list[str]: """`names` in order, without repeats. @@ -275,6 +281,35 @@ class HaikuRAG: raise failure return [self._clients[n] for n in names] + def _require_one_embedder(self, clients: "list[HaikuRAG]") -> None: + """Fail when two of these databases were written with different embedders. + + Searching a set embeds the query once, so a database written with another + model answers from a different vector space: its candidates are noise, and + rank fusion gives them slots anyway. Only databases searched together have + to agree, so this is a property of the selection rather than of the set. + + Drift between a database and the *config* is a separate, softer matter — + the same model served by another stack is spelled differently — which + `SettingsRepository` reports on open. + """ + recorded = [ + (client._source, client.store.stored_embedding) + for client in clients + if client.store.stored_embedding is not None + ] + if len(recorded) < 2: + return + (first_name, first), *rest = recorded + for name, embedding in rest: + if embedding != first: + raise ConfigMismatchError( + f"databases '{first_name}' and '{name}' were written with " + f"different embedders ({_spell(first)} and " + f"{_spell(embedding)}); searching them together embeds the " + "query once, so their vectors are not comparable" + ) + async def _open_client(self, name: str, location: str) -> "HaikuRAG": uri, db_path = locate_database(location) config = self._config.model_copy(deep=True) diff --git a/haiku_rag_slim/haiku/rag/client/search.py b/haiku_rag_slim/haiku/rag/client/search.py index a8c5ad25..a8d177c9 100644 --- a/haiku_rag_slim/haiku/rag/client/search.py +++ b/haiku_rag_slim/haiku/rag/client/search.py @@ -75,6 +75,7 @@ async def search_sources( if not names: return [] selected = await client.clients_for(names) + client._require_one_embedder(selected) # One over-fetch decision, and one reranker, for the whole set. fetch_limit = _fetch_limit(client, query, limit) diff --git a/haiku_rag_slim/haiku/rag/store/engine.py b/haiku_rag_slim/haiku/rag/store/engine.py index 992bfc8d..e2f09c04 100644 --- a/haiku_rag_slim/haiku/rag/store/engine.py +++ b/haiku_rag_slim/haiku/rag/store/engine.py @@ -104,6 +104,16 @@ def _stored_vector_dim(settings: dict) -> int | None: return settings.get("embeddings", {}).get("model", {}).get("vector_dim") +def _stored_embedding( + settings: dict, +) -> tuple[str | None, str | None, int | None] | None: + """The embedder a database's chunks were written with, or None if unrecorded.""" + model = settings.get("embeddings", {}).get("model", {}) + if not model: + return None + return model.get("provider"), model.get("name"), model.get("vector_dim") + + # Keeps the vacuum cleanup cutoff safely older than the oldest tagged # version; guards against timestamp precision at the boundary. TAG_RETENTION_MARGIN = timedelta(seconds=1) @@ -204,6 +214,7 @@ class Store: # Create embedder (sync — no LanceDB needed) self.embedder = get_embedder(config=self._config) + self.stored_embedding: tuple[str | None, str | None, int | None] | None = None async def _initialize(self): """Perform async initialization: connect to LanceDB, init tables, validate.""" @@ -225,6 +236,7 @@ class Store: # An existing database's chunks can only be read with the dimension they # were written at. stored_vector_dim = _stored_vector_dim(stored_settings) + self.stored_embedding = _stored_embedding(stored_settings) chunk_vector_dim = stored_vector_dim or self.embedder._vector_dim self.ChunkRecord: type[ChunkRecordBase] = create_chunk_model(chunk_vector_dim) diff --git a/tests/test_multi_db.py b/tests/test_multi_db.py index 86b5dda9..3b1c24fe 100644 --- a/tests/test_multi_db.py +++ b/tests/test_multi_db.py @@ -8,7 +8,7 @@ from pydantic import ValidationError from haiku.rag.client import HaikuRAG from haiku.rag.config import get_config from haiku.rag.config.models import AppConfig, LanceDBConfig -from haiku.rag.store.exceptions import SourceUnavailableError +from haiku.rag.store.exceptions import ConfigMismatchError, SourceUnavailableError from haiku.rag.store.models import Chunk, DocumentItem from haiku.rag.utils import locate_database @@ -68,6 +68,48 @@ async def _seed(config, name, contents): ) +@pytest.fixture +def query_embedding(monkeypatch): + """Vector search with no embedder behind it, recording the queries embedded. + + These tests are about which databases are asked and how often, not about + retrieval quality, and CI has no embedding endpoint. + """ + from haiku.rag.embeddings import EmbedderWrapper + + embedded: list[str] = [] + + async def embed_query(self, text): + embedded.append(text) + return [0.1] * get_config().embeddings.model.vector_dim + + monkeypatch.setattr(EmbedderWrapper, "embed_query", embed_query) + return embedded + + +async def _restore_embedder(config, name, *, provider=None, model_name=None): + """Rewrite what one database records about the embedder that wrote it, + standing in for a database built elsewhere with another model.""" + import json + + import lancedb + + _, db_path = locate_database(config.lancedb.databases[name]) + assert db_path is not None + db = await lancedb.connect_async(str(db_path.resolve())) + table = await db.open_table("settings") + rows = ( + await table.query().where("id = 'settings'").limit(1).to_arrow() + ).to_pylist() + stored = json.loads(rows[0]["settings"]) + model = stored["embeddings"]["model"] + if provider is not None: + model["provider"] = provider + if model_name is not None: + model["name"] = model_name + await table.update({"settings": json.dumps(stored)}, where="id = 'settings'") + + class TestNamingADatabaseDirectly: @pytest.mark.asyncio async def test_an_explicit_db_path_wins_over_the_configured_set( @@ -296,6 +338,49 @@ class TestLookupByIdentifier: assert await rag.get_document_by_uri("test://nowhere") is None +class TestOneEmbedderAcrossTheSet: + """A set is searched with one query vector, so a database written with + another model would answer from a different space.""" + + @pytest.mark.asyncio + async def test_disagreeing_databases_cannot_be_searched_together(self, tmp_path): + config = _config(tmp_path, ["alpha", "beta"]) + await _seed(config, "alpha", ["alpha one"]) + await _seed(config, "beta", ["beta one"]) + await _restore_embedder(config, "beta", model_name="some-other-model") + + async with HaikuRAG(config=config, read_only=True) as rag: + with pytest.raises(ConfigMismatchError, match="different embedders"): + await rag.search("one") + + @pytest.mark.asyncio + async def test_a_database_asked_for_alone_is_never_compared( + self, tmp_path, query_embedding + ): + """Only databases searched together have to agree.""" + config = _config(tmp_path, ["alpha", "beta"]) + await _seed(config, "alpha", ["alpha one"]) + await _seed(config, "beta", ["beta one"]) + await _restore_embedder(config, "beta", model_name="some-other-model") + + async with HaikuRAG(config=config, read_only=True) as rag: + assert await rag.search("one", sources=["alpha"]) is not None + assert await rag.count_documents(filter=None) is not None + + @pytest.mark.asyncio + async def test_agreeing_databases_search_together(self, tmp_path, query_embedding): + """The databases agree with each other; that they were written by a + differently-spelled provider than the config is the soft case.""" + config = _config(tmp_path, ["alpha", "beta"]) + await _seed(config, "alpha", ["alpha one"]) + await _seed(config, "beta", ["beta one"]) + await _restore_embedder(config, "alpha", provider="openai") + await _restore_embedder(config, "beta", provider="openai") + + async with HaikuRAG(config=config, read_only=True) as rag: + assert len(await rag.search("one")) > 0 + + class TestReadOnlyMode: @pytest.mark.asyncio async def test_a_client_covering_a_set_reports_its_mode(self, tmp_path):