Hand storage the database location, not the configuration that placed it

Store, connect_lancedb, gather_database_info and run_doctor take a
location, a path or a URI, and classify it with ConnectionMode.of.
SingleDatabaseSession owns the resolved DatabaseRef and passes its
location down. This removes DatabaseRef.connection(), default_db_path,
the placeholder path for URI-backed databases and the per-database
config copies, so the configuration a client holds is the one the caller
gave it. The chat hands its capabilities the scope it opened along with
the client it lends, and the v0.58.0 migration no longer checks local
free disk for a database behind a URI.
This commit is contained in:
Yiorgis Gozadinos 2026-09-03 12:48:45 +03:00
parent 4f7f69aaf0
commit 9baa213b34
No known key found for this signature in database
23 changed files with 424 additions and 386 deletions

View file

@ -10,6 +10,11 @@
- Searches in one model response deduplicate their results: evidence a sibling - Searches in one model response deduplicate their results: evidence a sibling
search already showed collapses to a reference line, and a picture attaches search already showed collapses to a reference line, and a picture attaches
once per response. once per response.
- `Store(location, config)`, `connect_lancedb(location, config)`,
`gather_database_info(location, config)` and `run_doctor(config, location, ...)`
take the database location, a path or a URI. `ConnectionMode.of(location)`
replaces `ConnectionMode.from_config`. `DatabaseRef.connection()` and
`default_db_path` removed.
## [0.81.0] - 2026-09-01 ## [0.81.0] - 2026-09-01

View file

@ -1,5 +1,4 @@
import logging import logging
from functools import cached_property
from pathlib import Path from pathlib import Path
from typing import TYPE_CHECKING from typing import TYPE_CHECKING
@ -64,21 +63,10 @@ class HaikuRAGApp:
[ref] = self.scope.databases [ref] = self.scope.databases
return ref return ref
@cached_property
def _connection(self) -> "tuple[AppConfig, Path]":
"""How to open the one database this command works on, directly.
Derived per database from its configured location.
"""
from haiku.rag.client.session import default_db_path
config, db_path = self._one.connection(self.config)
return config, db_path or default_db_path(config)
@property @property
def _store_config(self) -> AppConfig: def _location(self) -> "Path | str":
"""The configuration for opening the one database directly.""" """Where the one database this command works on is."""
return self._connection[0] return self._one.location
@property @property
def _is_local(self) -> bool: def _is_local(self) -> bool:
@ -91,12 +79,9 @@ class HaikuRAGApp:
@property @property
def _path(self) -> Path: def _path(self) -> Path:
"""The path of the one database this command works on. """The path of the one local database this command works on."""
assert self._one.db_path is not None
A database behind a URI has none of its own, and the default stands in: return self._one.db_path
the URI in `_store_config` is what decides where it connects.
"""
return self._connection[1]
@property @property
def display_path(self) -> "Path | str": def display_path(self) -> "Path | str":
@ -140,7 +125,7 @@ class HaikuRAGApp:
self.console.print("[red]Database path does not exist.[/red]") self.console.print("[red]Database path does not exist.[/red]")
return return
info = await gather_database_info(self._store_config, self._path) info = await gather_database_info(self._location, self.config)
if not info.exists: if not info.exists:
self.console.print( self.console.print(
@ -282,8 +267,8 @@ class HaikuRAGApp:
cm = status if status is not None else nullcontext() cm = status if status is not None else nullcontext()
with cm: with cm:
report = await run_doctor( report = await run_doctor(
self._store_config, self.config,
self._path, self._location,
dict(os.environ), dict(os.environ),
duplicates_out=duplicates_out, duplicates_out=duplicates_out,
on_progress=on_progress, on_progress=on_progress,
@ -340,8 +325,8 @@ class HaikuRAGApp:
return return
async with Store( async with Store(
self._path, self._location,
config=self._store_config, config=self.config,
skip_validation=True, skip_validation=True,
read_only=True, read_only=True,
skip_migration_check=True, skip_migration_check=True,
@ -415,15 +400,15 @@ class HaikuRAGApp:
""" """
from haiku.rag.store.engine import Store from haiku.rag.store.engine import Store
return Store(self._path, config=self._store_config, read_only=self.read_only) return Store(self._location, config=self.config, read_only=self.read_only)
def _tag_read_store(self) -> "Store": def _tag_read_store(self) -> "Store":
"""Read-only store for tag inspection; works on old or drifted DBs.""" """Read-only store for tag inspection; works on old or drifted DBs."""
from haiku.rag.store.engine import Store from haiku.rag.store.engine import Store
return Store( return Store(
self._path, self._location,
config=self._store_config, config=self.config,
skip_validation=True, skip_validation=True,
skip_migration_check=True, skip_migration_check=True,
read_only=True, read_only=True,
@ -760,8 +745,8 @@ class HaikuRAGApp:
from haiku.rag.store.engine import Store from haiku.rag.store.engine import Store
async with Store( async with Store(
self._path, self._location,
config=self._store_config, config=self.config,
skip_validation=True, skip_validation=True,
skip_migration_check=True, skip_migration_check=True,
read_only=self.read_only, read_only=self.read_only,

View file

@ -42,14 +42,8 @@ def run_chat(
config.qa.model = model_config config.qa.model = model_config
config.analysis.model = model_config config.analysis.model = model_config
# The capabilities read the databases the scope covers, not what the # The app opens the scope and lends that client to the capabilities, which
# configuration names: a `--db PATH` selection is outside the # read what `--db PATH` or `--db-name NAME` selected.
# configuration, and a `--db-name NAME` selection is narrower than it.
if scope.covers_multiple:
capability_config, capability_db_path = config, None
else:
capability_config, capability_db_path = scope.databases[0].connection(config)
enabled = capabilities or ["rag"] enabled = capabilities or ["rag"]
capability_list = [] capability_list = []
defer_loading = len(enabled) > 1 defer_loading = len(enabled) > 1
@ -68,8 +62,7 @@ def run_chat(
capability_list.append( capability_list.append(
create_capability( create_capability(
db_path=capability_db_path, config=config,
config=capability_config,
defer_loading=defer_loading, defer_loading=defer_loading,
vision=driving_model.vision, vision=driving_model.vision,
) )
@ -80,8 +73,7 @@ def run_chat(
capability_list.append( capability_list.append(
create_capability( create_capability(
db_path=capability_db_path, config=config,
config=capability_config,
defer_loading=defer_loading, defer_loading=defer_loading,
vision=driving_model.vision, vision=driving_model.vision,
) )

View file

@ -148,10 +148,12 @@ class ChatApp(App):
# a client whose __aenter__ failed. # a client whose __aenter__ failed.
await client.__aenter__() await client.__aenter__()
self.client = client self.client = client
# Lent to the capabilities: already the databases they were built for, # Lent to the capabilities, with the scope it covers: one connection
# and one connection per database however many capabilities read it. # per database however many capabilities read it, and the analysis
# sandbox is built over the same selection.
for capability in self._capabilities: for capability in self._capabilities:
capability.borrowed_rag = client capability.borrowed_rag = client
capability.scope = self.scope
self._agent = Agent( self._agent = Agent(
self._model, self._model,

View file

@ -21,7 +21,6 @@ from haiku.rag.client.session import (
FederatedSession, FederatedSession,
SingleDatabaseSession, SingleDatabaseSession,
aclose_quietly, aclose_quietly,
default_db_path,
) )
from haiku.rag.config import AppConfig, get_config from haiku.rag.config import AppConfig, get_config
from haiku.rag.converters import get_converter from haiku.rag.converters import get_converter
@ -145,9 +144,6 @@ class HaikuRAG:
of nothing to search. of nothing to search.
""" """
self._configured = config if config is not None else get_config() self._configured = config if config is not None else get_config()
# What the caller configured, kept intact: entering derives a
# single-database configuration from it, and every re-entry derives
# from the configured set.
self._config = self._configured self._config = self._configured
self._requested_db_path = Path(db_path) if db_path is not None else None self._requested_db_path = Path(db_path) if db_path is not None else None
if self._requested_db_path is not None and sources is not None: if self._requested_db_path is not None and sources is not None:
@ -339,15 +335,12 @@ class HaikuRAG:
return self return self
[ref] = scope.databases [ref] = scope.databases
self._config, db_path = ref.connection(self._configured)
self._session = await SingleDatabaseSession( self._session = await SingleDatabaseSession(
db_path if db_path is not None else default_db_path(self._config), ref,
self._config, self._config,
skip_validation=self._skip_validation, skip_validation=self._skip_validation,
create=self._create, create=self._create,
read_only=self._read_only, read_only=self._read_only,
source=ref.name,
).open() ).open()
return self return self

View file

@ -43,15 +43,10 @@ class DatabaseRef:
uri, db_path = locate_database(location) uri, db_path = locate_database(location)
return cls(name=name, uri=uri, db_path=db_path) return cls(name=name, uri=uri, db_path=db_path)
def connection(self, config: AppConfig) -> tuple[AppConfig, Path | None]: @property
"""The configuration and path to open this one database with. def location(self) -> Path | str:
"""Where the database is: its path, or its URI."""
A copy: the caller's configuration still names whatever set it named. return self.db_path if self.db_path is not None else self.uri
"""
one = config.model_copy(deep=True)
one.lancedb.databases = {}
one.lancedb.uri = self.uri
return one, self.db_path
@dataclass(frozen=True) @dataclass(frozen=True)

View file

@ -43,36 +43,30 @@ async def aclose_quietly(closeable: Any, what: str) -> None:
logger.debug("Closing the %s failed on teardown", what, exc_info=True) logger.debug("Closing the %s failed on teardown", what, exc_info=True)
def default_db_path(config: AppConfig) -> Path:
"""Where a database lives when its location names no path."""
return config.storage.data_dir / "haiku.rag.lancedb"
class SingleDatabaseSession: class SingleDatabaseSession:
"""One database: its store, its repositories, and their lifecycle. """One database: its store, its repositories, and their lifecycle.
Everything that needs a store lives here, so nothing above has to ask whether Everything that needs a store lives here, so nothing above has to ask whether
it has one. ``source`` is the configured name this database answers to, or it has one. Built from the resolved reference: ``source`` is the configured
None where nothing names it. name it answers to, or None where nothing names it, and the store receives
its location.
``db_path``, ``config``, ``read_only`` and ``source`` are readable: a client ``ref``, ``config``, ``read_only`` and ``source`` are readable: a client
borrowing this session reports them as its own. borrowing this session reports them as its own.
""" """
def __init__( def __init__(
self, self,
db_path: Path | str, ref: DatabaseRef,
config: AppConfig, config: AppConfig,
*, *,
skip_validation: bool = False, skip_validation: bool = False,
create: bool = False, create: bool = False,
read_only: bool = False, read_only: bool = False,
source: str | None = None,
) -> None: ) -> None:
self.db_path = db_path self.ref = ref
self.config = config self.config = config
self.read_only = read_only self.read_only = read_only
self.source = source
self._skip_validation = skip_validation self._skip_validation = skip_validation
self._create = create self._create = create
self._vacuum_tasks: set[asyncio.Task] = set() self._vacuum_tasks: set[asyncio.Task] = set()
@ -80,19 +74,25 @@ class SingleDatabaseSession:
self._vacuum_dirty = False self._vacuum_dirty = False
@property @property
def location(self) -> Path | str: def source(self) -> str | None:
"""Configured URI or local path for this database. return self.ref.name
Not `db_path`, which is a placeholder where a URI holds the database. @property
""" def location(self) -> Path | str:
return self.config.lancedb.uri or self.db_path """Where this database is: its path, or its URI."""
return self.ref.location
@property
def db_path(self) -> Path | None:
"""The local path, or None for a database behind a URI."""
return self.ref.db_path
async def open(self) -> "SingleDatabaseSession": async def open(self) -> "SingleDatabaseSession":
"""Connect, validate, and build the repositories.""" """Connect, validate, and build the repositories."""
failure: str | None = None failure: str | None = None
try: try:
self.store = Store( self.store = Store(
self.db_path, self.location,
config=self.config, config=self.config,
skip_validation=self._skip_validation, skip_validation=self._skip_validation,
create=self._create, create=self._create,
@ -309,14 +309,11 @@ class FederatedSession:
Registered here because a cancelled `gather` discards its results. Registered here because a cancelled `gather` discards its results.
""" """
ref = self._refs[name]
one, db_path = ref.connection(self._config)
self._sessions[name] = await SingleDatabaseSession( self._sessions[name] = await SingleDatabaseSession(
db_path if db_path is not None else default_db_path(one), self._refs[name],
one, self._config,
skip_validation=self._skip_validation, skip_validation=self._skip_validation,
read_only=self._read_only, read_only=self._read_only,
source=ref.name,
).open() ).open()
async def aclose(self) -> None: async def aclose(self) -> None:

View file

@ -1080,7 +1080,7 @@ async def run_provider_checks(
async def run_doctor( async def run_doctor(
config: AppConfig, config: AppConfig,
db_path: Path, location: Path | str,
environ: dict[str, str], environ: dict[str, str],
duplicates_out: Path | None = None, duplicates_out: Path | None = None,
on_progress: Callable[[str], None] | None = None, on_progress: Callable[[str], None] | None = None,
@ -1092,7 +1092,7 @@ async def run_doctor(
""" """
notify = on_progress or (lambda _label: None) notify = on_progress or (lambda _label: None)
notify("Inspecting tables") notify("Inspecting tables")
db = await connect_lancedb(config, db_path) db = await connect_lancedb(location, config)
stats = await get_database_stats(db) stats = await get_database_stats(db)
results: list[CheckResult] = [] results: list[CheckResult] = []
@ -1110,7 +1110,7 @@ async def run_doctor(
missing = [name for name in REQUIRED_TABLES if not stats[name]["exists"]] missing = [name for name in REQUIRED_TABLES if not stats[name]["exists"]]
if not missing: if not missing:
async with Store( async with Store(
db_path, location,
config=config, config=config,
skip_validation=True, skip_validation=True,
read_only=True, read_only=True,

View file

@ -1,6 +1,5 @@
from fastapi import APIRouter, Depends, HTTPException, status from fastapi import APIRouter, Depends, HTTPException, status
from haiku.rag.client.session import default_db_path
from haiku.rag.ingester.api.server import APIState, get_state from haiku.rag.ingester.api.server import APIState, get_state
from haiku.rag.store.info import DatabaseInfo, gather_database_info from haiku.rag.store.info import DatabaseInfo, gather_database_info
@ -21,5 +20,4 @@ async def database(state: APIState = Depends(get_state)) -> DatabaseInfo:
detail="database not configured", detail="database not configured",
) )
[ref] = state.scope.databases [ref] = state.scope.databases
one, db_path = ref.connection(state.config) return await gather_database_info(ref.location, state.config)
return await gather_database_info(one, db_path or default_db_path(one))

View file

@ -20,13 +20,12 @@ async def database_lines(client: "HaikuRAG") -> list[str]:
Reported through the connection the client already holds. A failure becomes Reported through the connection the client already holds. A failure becomes
a line of the report, and the other databases still report. a line of the report, and the other databases still report.
""" """
from haiku.rag.store.engine import ConnectionMode
from haiku.rag.store.info import get_database_stats from haiku.rag.store.info import get_database_stats
lines: list[str] = [] lines: list[str] = []
db_path = client.store.db_path db_path = client.store.db_path
if client.store._connection_mode == ConnectionMode.LOCAL and not db_path.exists(): if db_path is not None and not db_path.exists():
return ["[red]Database path does not exist.[/red]"] return ["[red]Database path does not exist.[/red]"]
try: try:

View file

@ -38,11 +38,12 @@ class ConnectionMode(Enum):
OBJECT_STORAGE = "object_storage" OBJECT_STORAGE = "object_storage"
@staticmethod @staticmethod
def from_config(config: AppConfig) -> "ConnectionMode": def of(location: Path | str) -> "ConnectionMode":
uri = config.lancedb.uri """How a location is connected to: a path is local, `db://` is LanceDB
if not uri: Cloud, any other scheme is object storage."""
if isinstance(location, Path) or "://" not in location:
return ConnectionMode.LOCAL return ConnectionMode.LOCAL
if uri.startswith("db://"): if location.startswith("db://"):
return ConnectionMode.CLOUD return ConnectionMode.CLOUD
return ConnectionMode.OBJECT_STORAGE return ConnectionMode.OBJECT_STORAGE
@ -72,8 +73,10 @@ def _session(config: AppConfig) -> lancedb.Session:
async def connect_lancedb( async def connect_lancedb(
config: AppConfig, db_path: Path | None = None location: Path | str, config: AppConfig
) -> lancedb.AsyncConnection: ) -> lancedb.AsyncConnection:
"""Connect to the database at `location`, with the connection settings
(credentials, storage options, caches, consistency) from `config`."""
interval = config.lancedb.read_consistency_interval_seconds interval = config.lancedb.read_consistency_interval_seconds
kwargs: dict[str, Any] = { kwargs: dict[str, Any] = {
"session": _session(config), "session": _session(config),
@ -81,22 +84,19 @@ async def connect_lancedb(
timedelta(seconds=interval) if interval is not None else None timedelta(seconds=interval) if interval is not None else None
), ),
} }
mode = ConnectionMode.from_config(config) mode = ConnectionMode.of(location)
if mode == ConnectionMode.CLOUD: if mode == ConnectionMode.CLOUD:
return await lancedb.connect_async( return await lancedb.connect_async(
uri=config.lancedb.uri, uri=str(location),
api_key=config.lancedb.api_key, api_key=config.lancedb.api_key,
region=config.lancedb.region, region=config.lancedb.region,
**kwargs, **kwargs,
) )
elif mode == ConnectionMode.OBJECT_STORAGE: if mode == ConnectionMode.OBJECT_STORAGE:
if config.lancedb.storage_options: if config.lancedb.storage_options:
kwargs["storage_options"] = config.lancedb.storage_options kwargs["storage_options"] = config.lancedb.storage_options
return await lancedb.connect_async(uri=config.lancedb.uri, **kwargs) return await lancedb.connect_async(uri=str(location), **kwargs)
else: return await lancedb.connect_async(Path(location).absolute(), **kwargs)
if db_path is None:
raise ValueError("No lancedb.uri configured and no db_path provided")
return await lancedb.connect_async(db_path.absolute(), **kwargs)
def _stored_vector_dim(settings: dict) -> int | None: def _stored_vector_dim(settings: dict) -> int | None:
@ -180,14 +180,24 @@ class TagInfo:
class Store: class Store:
def __init__( def __init__(
self, self,
db_path: Path | str, location: Path | str,
config: AppConfig | None = None, config: AppConfig | None = None,
skip_validation: bool = False, skip_validation: bool = False,
create: bool = False, create: bool = False,
read_only: bool = False, read_only: bool = False,
skip_migration_check: bool = False, skip_migration_check: bool = False,
): ):
self.db_path: Path = Path(db_path) """A store over the database at `location`, a local path or a URI.
`config` supplies connection settings; where the database is comes
from `location` alone.
"""
self._location: Path | str = location
self.db_path: Path | None = (
Path(location)
if ConnectionMode.of(location) == ConnectionMode.LOCAL
else None
)
self._config = config if config is not None else get_config() self._config = config if config is not None else get_config()
self._read_only = read_only self._read_only = read_only
self._create = create self._create = create
@ -200,7 +210,7 @@ class Store:
self._rebuild_lock = asyncio.Lock() self._rebuild_lock = asyncio.Lock()
self._is_new_db = False self._is_new_db = False
if self._connection_mode == ConnectionMode.LOCAL: if self.db_path is not None:
if not self.db_path.exists(): if not self.db_path.exists():
if not create: if not create:
raise FileNotFoundError( raise FileNotFoundError(
@ -231,7 +241,7 @@ class Store:
async def _initialize(self): async def _initialize(self):
"""Perform async initialization: connect to LanceDB, init tables, validate.""" """Perform async initialization: connect to LanceDB, init tables, validate."""
self.db: lancedb.AsyncConnection = await connect_lancedb( self.db: lancedb.AsyncConnection = await connect_lancedb(
self._config, self.db_path self.location, self._config
) )
# Read once and thread onward: on object storage each of these is a # Read once and thread onward: on object storage each of these is a
@ -392,9 +402,14 @@ class Store:
needed = datetime.now() - oldest + TAG_RETENTION_MARGIN needed = datetime.now() - oldest + TAG_RETENTION_MARGIN
return max(retention, needed) return max(retention, needed)
@property
def location(self) -> Path | str:
"""Where this store connected: a local path, or a URI."""
return self._location
@property @property
def _connection_mode(self) -> ConnectionMode: def _connection_mode(self) -> ConnectionMode:
return ConnectionMode.from_config(self._config) return ConnectionMode.of(self._location)
async def _ensure_vector_index(self) -> None: async def _ensure_vector_index(self) -> None:
"""Create or rebuild vector index on chunks table. """Create or rebuild vector index on chunks table.

View file

@ -96,15 +96,15 @@ class DatabaseInfo(BaseModel):
packages: dict[str, str] = Field(default_factory=dict) packages: dict[str, str] = Field(default_factory=dict)
async def gather_database_info(config: AppConfig, db_path: Path) -> DatabaseInfo: async def gather_database_info(location: Path | str, config: AppConfig) -> DatabaseInfo:
"""Collect read-only database state without going through Store, so a """Collect read-only database state without going through Store, so a
database missing tables (e.g. pre-migration) still reports what it can.""" database missing tables (e.g. pre-migration) still reports what it can."""
from haiku.rag.store.upgrades import get_pending_upgrades from haiku.rag.store.upgrades import get_pending_upgrades
from haiku.rag.utils import get_package_versions from haiku.rag.utils import get_package_versions
display_path = config.lancedb.uri or str(db_path) display_path = str(location)
db = await connect_lancedb(config, db_path) db = await connect_lancedb(location, config)
stats = await get_database_stats(db) stats = await get_database_stats(db)
if not any(entry["exists"] for entry in stats.values()): if not any(entry["exists"] for entry in stats.values()):

View file

@ -77,8 +77,11 @@ async def _apply_split_document_meta(store: Store) -> None:
Exception Exception
): # pragma: no cover - defensive; stats() failure shouldn't block the split ): # pragma: no cover - defensive; stats() failure shouldn't block the split
live_bytes = 0 live_bytes = 0
free_bytes = shutil.disk_usage(store.db_path).free # A database behind a URI has no local disk to run out of.
if live_bytes and free_bytes < live_bytes: free_bytes = (
shutil.disk_usage(store.db_path).free if store.db_path is not None else None
)
if live_bytes and free_bytes is not None and free_bytes < live_bytes:
logger.warning( logger.warning(
"Skipping post-migration vacuum: need ~%.2f GB free to compact the " "Skipping post-migration vacuum: need ~%.2f GB free to compact the "
"documents table, have %.2f GB. Run `haiku-rag vacuum` once you have " "documents table, have %.2f GB. Run `haiku-rag vacuum` once you have "

View file

@ -84,17 +84,18 @@ def test_chat_capabilities_read_the_named_database(tmp_path, monkeypatch):
from haiku.rag.chat import run_chat from haiku.rag.chat import run_chat
run_chat(scope=DatabaseScope.resolve(config, database_name="b")) run_chat(scope=DatabaseScope.resolve(config, database_name="b"))
named_scope = chat_app.call_args.kwargs["scope"]
[named] = chat_app.call_args.kwargs["capabilities"] [named] = chat_app.call_args.kwargs["capabilities"]
run_chat(scope=DatabaseScope.resolve(config)) run_chat(scope=DatabaseScope.resolve(config))
covering_scope = chat_app.call_args.kwargs["scope"]
[covering] = chat_app.call_args.kwargs["capabilities"] [covering] = chat_app.call_args.kwargs["capabilities"]
# The chat lends its own client, so this scope is the fallback: it places # The app opens the scope it is handed and lends that client to the
# the named database alone. # capabilities, which keep the configuration as the caller named it.
[placed] = named.scope.databases assert named_scope.names == ("b",)
assert placed.db_path == tmp_path / "b.lancedb" assert covering_scope.names == ("a", "b")
assert named.config.lancedb.databases == {} assert set(named.config.lancedb.databases) == {"a", "b"}
assert covering.scope.names == ("a", "b")
assert set(covering.config.lancedb.databases) == {"a", "b"} assert set(covering.config.lancedb.databases) == {"a", "b"}
@ -684,6 +685,37 @@ class TestLendingTheClient:
assert borrowed == [client] * len(app._capabilities) assert borrowed == [client] * len(app._capabilities)
assert borrowed assert borrowed
@pytest.mark.asyncio
async def test_mounting_gives_every_capability_the_apps_scope(self, tmp_path):
"""A capability built over the configured set covers what the chat
selected once mounted: the analysis sandbox is built over that scope."""
from haiku.rag.chat.app import ChatApp
from haiku.rag.client.scope import DatabaseScope
from haiku.rag.config.models import AppConfig, LanceDBConfig
config = AppConfig(
lancedb=LanceDBConfig(
databases={
"a": str(tmp_path / "a.lancedb"),
"b": str(tmp_path / "b.lancedb"),
}
)
)
selected = DatabaseScope.resolve(config, database_name="b")
capability = create_capability(config=config)
assert capability.scope.covers_multiple
client = _make_mock_client()
app = ChatApp(scope=selected, capabilities=[capability], read_only=True)
with (
patch("haiku.rag.chat.app.HaikuRAG") as stub_rag,
_covering_returns(stub_rag, client),
):
async with app.run_test():
pass
assert capability.scope == selected
class TestDocumentSelectionIdentity: class TestDocumentSelectionIdentity:
"""Two documents can share a title, within a corpus and across databases, so """Two documents can share a title, within a corpus and across databases, so

View file

@ -7,7 +7,6 @@ import pytest
from haiku.rag.capabilities._tools import search_corpus from haiku.rag.capabilities._tools import search_corpus
from haiku.rag.capabilities.rag import RAGState, create_capability from haiku.rag.capabilities.rag import RAGState, create_capability
from haiku.rag.client import HaikuRAG from haiku.rag.client import HaikuRAG
from haiku.rag.client.scope import DatabaseScope
from haiku.rag.client.session import FederatedSession from haiku.rag.client.session import FederatedSession
from haiku.rag.sandbox import AnalysisContext, Sandbox from haiku.rag.sandbox import AnalysisContext, Sandbox
from haiku.rag.store.exceptions import UnknownDatabaseError from haiku.rag.store.exceptions import UnknownDatabaseError
@ -256,12 +255,9 @@ class TestLendingANamedClient:
await _seed(config, "alpha", ["alpha document about cats"]) await _seed(config, "alpha", ["alpha document about cats"])
await _seed(config, "beta", ["beta document about cats"]) await _seed(config, "beta", ["beta document about cats"])
# `run_chat` derives these for a single-database scope. # What `run_chat` builds: the capability's own scope is the set, and
scope = DatabaseScope.resolve(config, database_name="alpha") # the lent client is what narrows it.
one_config, one_path = scope.databases[0].connection(config) capability = create_capability(config=config, defer_loading=False)
capability = create_capability(
db_path=one_path, config=one_config, defer_loading=False
)
async with HaikuRAG(config=config, sources=["alpha"]) as client: async with HaikuRAG(config=config, sources=["alpha"]) as client:
# What `ChatApp.on_mount` does. # What `ChatApp.on_mount` does.

View file

@ -125,10 +125,44 @@ class TestOneConfiguredLocation:
config = self._config("s3://bucket/one.lancedb") config = self._config("s3://bucket/one.lancedb")
[ref] = DatabaseScope.resolve(config).databases [ref] = DatabaseScope.resolve(config).databases
one, db_path = ref.connection(config)
assert db_path is None assert ref.location == "s3://bucket/one.lancedb"
assert ConnectionMode.from_config(one) == ConnectionMode.OBJECT_STORAGE assert ConnectionMode.of(ref.location) == ConnectionMode.OBJECT_STORAGE
class TestSessionsOwnTheRef:
"""A session is built from the resolved reference and hands storage only
its location; the configuration it keeps is the one the caller named."""
@pytest.mark.asyncio
async def test_a_session_opens_the_location_with_the_undivided_config(
self, tmp_path
):
from haiku.rag.client.session import SingleDatabaseSession
config = _config(tmp_path, ["alpha", "beta"])
await _seed(config, "alpha", ["alpha document about cats"])
[ref] = DatabaseScope.resolve(config, database_name="alpha").databases
session = await SingleDatabaseSession(ref, config, read_only=True).open()
try:
assert session.source == "alpha"
assert session.location == ref.location
assert session.db_path == ref.location
assert session.store.location == ref.location
assert session.store._config is config
finally:
await session.aclose()
@pytest.mark.asyncio
async def test_a_client_keeps_the_configuration_it_was_given(self, tmp_path):
config = _config(tmp_path, ["alpha", "beta"])
await _seed(config, "alpha", ["alpha document about cats"])
async with HaikuRAG(config=config, sources=["alpha"]) as rag:
assert rag._config is config
assert set(rag._config.lancedb.databases) == {"alpha", "beta"}
assert rag.store.location == tmp_path / "alpha.lancedb"
class TestLocate: class TestLocate:

View file

@ -69,7 +69,7 @@ async def _seed(temp_db_path, *, version: str, with_items: bool = True):
async def test_gather_database_info_reports_tables_and_settings(temp_db_path): async def test_gather_database_info_reports_tables_and_settings(temp_db_path):
await _seed(temp_db_path, version="1.2.3") await _seed(temp_db_path, version="1.2.3")
info = await gather_database_info(AppConfig(), temp_db_path) info = await gather_database_info(temp_db_path, AppConfig())
assert info.exists is True assert info.exists is True
assert info.path == str(temp_db_path) assert info.path == str(temp_db_path)
@ -98,7 +98,7 @@ async def test_gather_database_info_flags_missing_table_and_pending_migrations(
): ):
await _seed(temp_db_path, version="0.39.0", with_items=False) await _seed(temp_db_path, version="0.39.0", with_items=False)
info = await gather_database_info(AppConfig(), temp_db_path) info = await gather_database_info(temp_db_path, AppConfig())
tables = {t.name: t for t in info.tables} tables = {t.name: t for t in info.tables}
assert tables["document_items"].exists is False assert tables["document_items"].exists is False
@ -112,7 +112,30 @@ async def test_gather_database_info_empty_database(temp_db_path):
await lancedb.connect_async(temp_db_path) # creates the dir, no tables await lancedb.connect_async(temp_db_path) # creates the dir, no tables
info = await gather_database_info(AppConfig(), temp_db_path) info = await gather_database_info(temp_db_path, AppConfig())
assert info.exists is False assert info.exists is False
assert info.path == str(temp_db_path) assert info.path == str(temp_db_path)
@pytest.mark.asyncio
async def test_gather_database_info_connects_to_the_location_it_is_given():
"""A remote location is passed to the connection as is and reported back
as the path; the configuration's own `uri` plays no part."""
from unittest.mock import AsyncMock, MagicMock, patch
from haiku.rag.config.models import LanceDBConfig
config = AppConfig(lancedb=LanceDBConfig(uri="s3://elsewhere/other.lancedb"))
with patch(
"haiku.rag.store.info.connect_lancedb", new_callable=AsyncMock
) as mock_connect:
listing = MagicMock()
listing.tables = []
mock_connect.return_value.list_tables = AsyncMock(return_value=listing)
info = await gather_database_info("s3://bucket/papers.lancedb", config)
assert mock_connect.call_args.args[0] == "s3://bucket/papers.lancedb"
assert info.path == "s3://bucket/papers.lancedb"
assert info.exists is False

View file

@ -126,6 +126,43 @@ class TestV0_58_0MigrationEdgeCases:
assert set(by_id) == {"a", "b"} # exactly one row each, no duplicates assert set(by_id) == {"a", "b"} # exactly one row each, no duplicates
assert len(rows) == 2 assert len(rows) == 2
async def test_a_remote_store_has_no_disk_to_check(self, temp_db_path, monkeypatch):
"""A store behind a URI has no local path: the reclaim vacuum runs
without a free-disk check."""
from haiku.rag.store.upgrades import v0_58_0
async with Store(temp_db_path, create=True, skip_migration_check=True) as store:
await seed_legacy_documents(
store,
[LegacyDocumentRecord(id="a", content="x", uri="u", metadata="{}")],
)
await store.set_haiku_version("0.57.0")
def no_disk(_path):
raise AssertionError("disk_usage consulted for a remote store")
monkeypatch.setattr(v0_58_0.shutil, "disk_usage", no_disk)
vacuum_calls: list[int] = []
async with Store(temp_db_path, skip_migration_check=True) as store:
store.db_path = None
async def fake_stats():
return {"total_bytes": 10_000_000}
monkeypatch.setattr(store.documents_table, "stats", fake_stats)
orig_vacuum = store.vacuum
async def tracking_vacuum(*args, **kwargs):
vacuum_calls.append(1)
return await orig_vacuum(*args, **kwargs)
monkeypatch.setattr(store, "vacuum", tracking_vacuum)
await store.migrate()
assert vacuum_calls == [1]
async def test_skips_vacuum_when_disk_is_tight(self, temp_db_path, monkeypatch): async def test_skips_vacuum_when_disk_is_tight(self, temp_db_path, monkeypatch):
"""When free disk can't cover one compacted copy, the split still """When free disk can't cover one compacted copy, the split still
completes but the reclaim vacuum is skipped.""" completes but the reclaim vacuum is skipped."""

View file

@ -3,7 +3,6 @@ from pathlib import Path
import pytest import pytest
from haiku.rag.client.scope import DatabaseRef, DatabaseScope from haiku.rag.client.scope import DatabaseRef, DatabaseScope
from haiku.rag.client.session import default_db_path
from haiku.rag.config.models import AppConfig, LanceDBConfig, StorageConfig from haiku.rag.config.models import AppConfig, LanceDBConfig, StorageConfig
from haiku.rag.store.exceptions import ( from haiku.rag.store.exceptions import (
AmbiguousDatabaseError, AmbiguousDatabaseError,
@ -89,16 +88,13 @@ class TestResolution:
assert ref.uri == "" assert ref.uri == ""
def test_a_path_selects_the_database_over_a_configured_uri(self): def test_a_path_selects_the_database_over_a_configured_uri(self):
"""`--db` exists to override what is configured, and the configuration """`--db` exists to override what is configured."""
derived from the ref is what makes the connection follow it."""
config = _config(uri="s3://bucket/one.lancedb") config = _config(uri="s3://bucket/one.lancedb")
scope = DatabaseScope.resolve(config, database_path=Path("/data/local")) scope = DatabaseScope.resolve(config, database_path=Path("/data/local"))
[ref] = scope.databases [ref] = scope.databases
assert ref.db_path == Path("/data/local") assert ref.location == Path("/data/local")
one, _ = ref.connection(config)
assert one.lancedb.uri == ""
def test_nothing_configured_falls_back_to_the_data_directory(self, tmp_path): def test_nothing_configured_falls_back_to_the_data_directory(self, tmp_path):
config = AppConfig(storage=StorageConfig(data_dir=tmp_path)) config = AppConfig(storage=StorageConfig(data_dir=tmp_path))
@ -158,61 +154,23 @@ class TestResolution:
DatabaseScope(()) DatabaseScope(())
class TestConnectionDerivation: class TestLocation:
"""Opening one of a set must not disturb the configuration it came from.""" """One value says where a database is: a path for a local one, a URI string
for a remote one. Storage connects to it as given."""
def test_a_local_location_becomes_a_path(self): def test_a_local_location_is_a_path(self):
config = _config(databases={"alpha": "/data/alpha.lancedb"}) config = _config(databases={"alpha": "/data/alpha.lancedb"})
[ref] = DatabaseScope.resolve(config).databases [ref] = DatabaseScope.resolve(config).databases
one, db_path = ref.connection(config) assert ref.location == Path("/data/alpha.lancedb")
assert db_path == Path("/data/alpha.lancedb") def test_a_uri_location_is_the_uri(self):
assert one.lancedb.uri == ""
assert one.lancedb.databases == {}
def test_a_uri_location_stays_a_uri(self):
config = _config(databases={"alpha": "s3://bucket/alpha.lancedb"}) config = _config(databases={"alpha": "s3://bucket/alpha.lancedb"})
[ref] = DatabaseScope.resolve(config).databases [ref] = DatabaseScope.resolve(config).databases
one, db_path = ref.connection(config) assert ref.location == "s3://bucket/alpha.lancedb"
assert db_path is None def test_a_path_the_caller_gave_is_its_location(self):
assert one.lancedb.uri == "s3://bucket/alpha.lancedb" assert DatabaseRef.at("/data/other.lancedb").location == Path(
"/data/other.lancedb"
def test_the_original_configuration_is_untouched(self): )
"""Rewriting it in place is what left downstream code unable to tell a set
had been named."""
config = _config(databases={"alpha": "/a.lancedb", "beta": "/b.lancedb"})
for ref in DatabaseScope.resolve(config).databases:
ref.connection(config)
assert config.lancedb.databases == {"alpha": "/a.lancedb", "beta": "/b.lancedb"}
assert config.lancedb.uri == ""
def test_each_derived_configuration_is_its_own_copy(self):
config = _config(databases={"alpha": "/a.lancedb", "beta": "s3://b/b.lancedb"})
alpha, beta = DatabaseScope.resolve(config).databases
one, _ = alpha.connection(config)
other, _ = beta.connection(config)
assert one is not other
assert one.lancedb.uri == ""
assert other.lancedb.uri == "s3://b/b.lancedb"
def test_a_database_behind_a_uri_has_no_path_of_its_own(tmp_path):
"""`connection` hands back no path for a URI, and the store still needs one:
the default stands in, and the URI is what decides where it connects."""
config = AppConfig(
storage=StorageConfig(data_dir=tmp_path),
lancedb=LanceDBConfig(databases={"alpha": "s3://bucket/alpha.lancedb"}),
)
[ref] = DatabaseScope.resolve(config).databases
one, db_path = ref.connection(config)
assert db_path is None
assert default_db_path(one) == tmp_path / "haiku.rag.lancedb"

View file

@ -177,8 +177,7 @@ async def test_app_info_opens_a_named_remote_database(tmp_path):
app = HaikuRAGApp(scope=scope, config=config) app = HaikuRAGApp(scope=scope, config=config)
assert app._is_local is False assert app._is_local is False
assert app._store_config.lancedb.uri == "s3://bucket/papers.lancedb" assert app._location == "s3://bucket/papers.lancedb"
assert app._store_config.lancedb.databases == {}
with patch( with patch(
"haiku.rag.store.info.connect_lancedb", new_callable=AsyncMock "haiku.rag.store.info.connect_lancedb", new_callable=AsyncMock
@ -189,13 +188,12 @@ async def test_app_info_opens_a_named_remote_database(tmp_path):
mock_db.list_tables = AsyncMock(return_value=mock_list_result) mock_db.list_tables = AsyncMock(return_value=mock_list_result)
await app.info() await app.info()
opened = mock_connect.call_args.args[0] assert mock_connect.call_args.args[0] == "s3://bucket/papers.lancedb"
assert opened.lancedb.uri == "s3://bucket/papers.lancedb"
async def test_app_doctor_opens_a_named_remote_database(): async def test_app_doctor_opens_a_named_remote_database():
"""`run_doctor` connects with the configuration it is handed: the one """`run_doctor` is handed the database's location, not the configuration
derived for the database, not the one naming the set.""" naming the set."""
from haiku.rag.client.scope import DatabaseScope from haiku.rag.client.scope import DatabaseScope
config = AppConfig( config = AppConfig(
@ -207,7 +205,7 @@ async def test_app_doctor_opens_a_named_remote_database():
run.return_value = MagicMock(checks=[], ok=True, duplicates=None) run.return_value = MagicMock(checks=[], ok=True, duplicates=None)
await app.doctor() await app.doctor()
assert run.call_args.args[0].lancedb.uri == "s3://bucket/papers.lancedb" assert run.call_args.args[1] == "s3://bucket/papers.lancedb"
async def test_app_info_uses_connect_lancedb_for_remote(tmp_path): async def test_app_info_uses_connect_lancedb_for_remote(tmp_path):
@ -230,9 +228,8 @@ async def test_app_info_uses_connect_lancedb_for_remote(tmp_path):
mock_db.list_tables = AsyncMock(return_value=mock_list_result) mock_db.list_tables = AsyncMock(return_value=mock_list_result)
await app.info() await app.info()
# The uri decides where it connects; the path argument is not read.
mock_connect.assert_called_once() mock_connect.assert_called_once()
assert mock_connect.call_args.args[0].lancedb.uri == "s3://bucket/path" assert mock_connect.call_args.args[0] == "s3://bucket/path"
@pytest.mark.asyncio @pytest.mark.asyncio

View file

@ -340,23 +340,19 @@ class TestReportedLocation:
@staticmethod @staticmethod
def _session(location: str): def _session(location: str):
from haiku.rag.client.scope import DatabaseScope from haiku.rag.client.scope import DatabaseScope
from haiku.rag.client.session import SingleDatabaseSession, default_db_path from haiku.rag.client.session import SingleDatabaseSession
from haiku.rag.config.models import AppConfig, LanceDBConfig from haiku.rag.config.models import AppConfig, LanceDBConfig
config = AppConfig(lancedb=LanceDBConfig(databases={"alpha": location})) config = AppConfig(lancedb=LanceDBConfig(databases={"alpha": location}))
[ref] = DatabaseScope.resolve(config, database_name="alpha").databases [ref] = DatabaseScope.resolve(config, database_name="alpha").databases
one, db_path = ref.connection(config) return SingleDatabaseSession(ref, config)
return SingleDatabaseSession(
db_path if db_path is not None else default_db_path(one),
one,
source="alpha",
)
def test_a_named_remote_database_reports_its_uri(self): def test_a_named_remote_database_reports_its_uri(self):
session = self._session("s3://bucket/alpha.lancedb") session = self._session("s3://bucket/alpha.lancedb")
assert isinstance(session.db_path, Path) assert session.db_path is None
assert session.location == "s3://bucket/alpha.lancedb" assert session.location == "s3://bucket/alpha.lancedb"
assert session.source == "alpha"
def test_a_named_local_database_reports_its_path(self): def test_a_named_local_database_reports_its_path(self):
session = self._session("/data/alpha.lancedb") session = self._session("/data/alpha.lancedb")

View file

@ -4,53 +4,44 @@ from unittest.mock import AsyncMock, patch
import pytest import pytest
from pydantic import ValidationError from pydantic import ValidationError
from haiku.rag.config import get_config
from haiku.rag.config.models import AppConfig, LanceDBConfig from haiku.rag.config.models import AppConfig, LanceDBConfig
from haiku.rag.store.engine import ConnectionMode, Store, connect_lancedb from haiku.rag.store.engine import ConnectionMode, Store, connect_lancedb
class TestConnectionMode: class TestConnectionMode:
def test_local_when_uri_empty(self): """A location is classified by itself: a path is local, `db://` is LanceDB
config = AppConfig(lancedb=LanceDBConfig(uri="")) Cloud, any other scheme is object storage."""
assert ConnectionMode.from_config(config) == ConnectionMode.LOCAL
def test_a_path_is_local(self, tmp_path):
assert ConnectionMode.of(tmp_path / "db.lancedb") == ConnectionMode.LOCAL
def test_a_schemeless_string_is_local(self):
assert ConnectionMode.of("/data/db.lancedb") == ConnectionMode.LOCAL
def test_cloud_when_db_uri(self): def test_cloud_when_db_uri(self):
config = AppConfig( assert ConnectionMode.of("db://my-database") == ConnectionMode.CLOUD
lancedb=LanceDBConfig(
uri="db://my-database", api_key="key", region="us-east-1"
)
)
assert ConnectionMode.from_config(config) == ConnectionMode.CLOUD
def test_object_storage_s3(self): @pytest.mark.parametrize(
config = AppConfig(lancedb=LanceDBConfig(uri="s3://bucket/path")) "uri",
assert ConnectionMode.from_config(config) == ConnectionMode.OBJECT_STORAGE [
"s3://bucket/path",
def test_object_storage_gs(self): "gs://bucket/path",
config = AppConfig(lancedb=LanceDBConfig(uri="gs://bucket/path")) "az://container/path",
assert ConnectionMode.from_config(config) == ConnectionMode.OBJECT_STORAGE "hdfs://namenode/path",
"custom://something",
def test_object_storage_az(self): ],
config = AppConfig(lancedb=LanceDBConfig(uri="az://container/path")) )
assert ConnectionMode.from_config(config) == ConnectionMode.OBJECT_STORAGE def test_any_other_scheme_is_object_storage(self, uri):
assert ConnectionMode.of(uri) == ConnectionMode.OBJECT_STORAGE
def test_object_storage_hdfs(self):
config = AppConfig(lancedb=LanceDBConfig(uri="hdfs://namenode/path"))
assert ConnectionMode.from_config(config) == ConnectionMode.OBJECT_STORAGE
def test_unknown_uri_treated_as_object_storage(self):
config = AppConfig(lancedb=LanceDBConfig(uri="custom://something"))
assert ConnectionMode.from_config(config) == ConnectionMode.OBJECT_STORAGE
class TestConnectLancedb: class TestConnectLancedb:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_local_passes_absolute_db_path(self, temp_db_path): async def test_local_passes_absolute_db_path(self, temp_db_path):
config = AppConfig(lancedb=LanceDBConfig(uri=""))
with patch( with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
) as mock_connect: ) as mock_connect:
await connect_lancedb(config, db_path=temp_db_path) await connect_lancedb(temp_db_path, AppConfig())
mock_connect.assert_awaited_once() mock_connect.assert_awaited_once()
assert mock_connect.call_args.args == (temp_db_path.absolute(),) assert mock_connect.call_args.args == (temp_db_path.absolute(),)
@ -60,25 +51,34 @@ class TestConnectLancedb:
monkeypatch.chdir(tmp_path) monkeypatch.chdir(tmp_path)
relative = Path("db/rag.lancedb") relative = Path("db/rag.lancedb")
config = AppConfig(lancedb=LanceDBConfig(uri=""))
with patch( with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
) as mock_connect: ) as mock_connect:
await connect_lancedb(config, db_path=relative) await connect_lancedb(relative, AppConfig())
mock_connect.assert_awaited_once() mock_connect.assert_awaited_once()
assert mock_connect.call_args.args == (relative.absolute(),) assert mock_connect.call_args.args == (relative.absolute(),)
@pytest.mark.asyncio
async def test_the_configured_uri_is_not_consulted(self, temp_db_path):
"""Storage connects to the location it is handed; placement is the
caller's, and the configuration's own `uri` never redirects it."""
config = AppConfig(lancedb=LanceDBConfig(uri="s3://elsewhere/db.lancedb"))
with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
) as mock_connect:
await connect_lancedb(temp_db_path, config)
assert mock_connect.call_args.args == (temp_db_path.absolute(),)
assert "uri" not in mock_connect.call_args.kwargs
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_cloud_passes_uri_api_key_region(self): async def test_cloud_passes_uri_api_key_region(self):
config = AppConfig( config = AppConfig(
lancedb=LanceDBConfig( lancedb=LanceDBConfig(api_key="test-key", region="us-west-2")
uri="db://my-database", api_key="test-key", region="us-west-2"
)
) )
with patch( with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
) as mock_connect: ) as mock_connect:
await connect_lancedb(config) await connect_lancedb("db://my-database", config)
mock_connect.assert_awaited_once() mock_connect.assert_awaited_once()
kwargs = mock_connect.call_args.kwargs kwargs = mock_connect.call_args.kwargs
assert kwargs["uri"] == "db://my-database" assert kwargs["uri"] == "db://my-database"
@ -89,7 +89,6 @@ class TestConnectLancedb:
async def test_object_storage_passes_uri_and_storage_options(self): async def test_object_storage_passes_uri_and_storage_options(self):
config = AppConfig( config = AppConfig(
lancedb=LanceDBConfig( lancedb=LanceDBConfig(
uri="s3://bucket/path",
storage_options={ storage_options={
"endpoint": "http://minio:9000", "endpoint": "http://minio:9000",
"region": "us-east-1", "region": "us-east-1",
@ -99,7 +98,7 @@ class TestConnectLancedb:
with patch( with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
) as mock_connect: ) as mock_connect:
await connect_lancedb(config) await connect_lancedb("s3://bucket/path", config)
mock_connect.assert_awaited_once() mock_connect.assert_awaited_once()
kwargs = mock_connect.call_args.kwargs kwargs = mock_connect.call_args.kwargs
assert kwargs["uri"] == "s3://bucket/path" assert kwargs["uri"] == "s3://bucket/path"
@ -110,23 +109,25 @@ class TestConnectLancedb:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_object_storage_without_storage_options(self): async def test_object_storage_without_storage_options(self):
config = AppConfig(lancedb=LanceDBConfig(uri="s3://bucket/path"))
with patch( with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
) as mock_connect: ) as mock_connect:
await connect_lancedb(config) await connect_lancedb("s3://bucket/path", AppConfig())
mock_connect.assert_awaited_once() mock_connect.assert_awaited_once()
kwargs = mock_connect.call_args.kwargs kwargs = mock_connect.call_args.kwargs
assert kwargs["uri"] == "s3://bucket/path" assert kwargs["uri"] == "s3://bucket/path"
assert "storage_options" not in kwargs assert "storage_options" not in kwargs
@pytest.mark.asyncio
async def test_local_without_db_path_raises(self): def _remote_store(location: str, config: AppConfig | None = None) -> Store:
config = AppConfig(lancedb=LanceDBConfig(uri="")) """A store over a remote location, opened against a mocked connection."""
with pytest.raises( return Store(
ValueError, match="No lancedb.uri configured and no db_path provided" location,
): config=config,
await connect_lancedb(config) create=True,
skip_validation=True,
skip_migration_check=True,
)
class TestStoreConnectionMode: class TestStoreConnectionMode:
@ -134,132 +135,127 @@ class TestStoreConnectionMode:
async def test_store_connection_mode_local(self, temp_db_path): async def test_store_connection_mode_local(self, temp_db_path):
async with Store(temp_db_path, create=True) as store: async with Store(temp_db_path, create=True) as store:
assert store._connection_mode == ConnectionMode.LOCAL assert store._connection_mode == ConnectionMode.LOCAL
assert store.location == temp_db_path
assert store.db_path == temp_db_path
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_store_connection_mode_cloud(self, temp_db_path): async def test_a_local_store_ignores_the_configured_uri(self, temp_db_path):
async with Store(temp_db_path, create=True) as store: config = AppConfig(lancedb=LanceDBConfig(uri="s3://elsewhere/db.lancedb"))
with ( async with Store(temp_db_path, config=config, create=True) as store:
patch.object(get_config().lancedb, "uri", "db://test-database"), assert store._connection_mode == ConnectionMode.LOCAL
patch.object(get_config().lancedb, "api_key", "test-api-key"), assert store.db_path == temp_db_path
patch.object(get_config().lancedb, "region", "us-east-1"),
): @pytest.mark.asyncio
async def test_store_connection_mode_cloud(self):
config = AppConfig(lancedb=LanceDBConfig(api_key="key", region="us-east-1"))
with (
patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
),
patch.object(Store, "_init_tables", new_callable=AsyncMock),
):
async with _remote_store("db://test-database", config) as store:
assert store._connection_mode == ConnectionMode.CLOUD assert store._connection_mode == ConnectionMode.CLOUD
assert store.location == "db://test-database"
assert store.db_path is None
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_store_connection_mode_object_storage(self, temp_db_path): async def test_store_connection_mode_object_storage(self):
async with Store(temp_db_path, create=True) as store: with (
with patch.object(get_config().lancedb, "uri", "s3://bucket/path"): patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
),
patch.object(Store, "_init_tables", new_callable=AsyncMock),
):
async with _remote_store("s3://bucket/path") as store:
assert store._connection_mode == ConnectionMode.OBJECT_STORAGE assert store._connection_mode == ConnectionMode.OBJECT_STORAGE
assert store.db_path is None
def _remote_store_with_mock_tables(location: str) -> Store:
"""A remote store whose tables are mocks: the mode decision is under test,
not the tables."""
store = _remote_store(location)
store.chunks_table = AsyncMock()
return store
class TestVacuumByConnectionMode: class TestVacuumByConnectionMode:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_cloud_skips_vacuum(self, temp_db_path): async def test_cloud_skips_vacuum(self):
async with Store(temp_db_path, create=True) as store: with (
with ( patch(
patch.object(get_config().lancedb, "uri", "db://test-database"), "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
patch.object(get_config().lancedb, "api_key", "test-api-key"), ),
patch.object(get_config().lancedb, "region", "us-east-1"), patch.object(Store, "_init_tables", new_callable=AsyncMock),
): ):
with patch.object( async with _remote_store_with_mock_tables("db://test-database") as store:
store.chunks_table, "optimize", new_callable=AsyncMock await store.vacuum()
) as mock_optimize: store.chunks_table.optimize.assert_not_awaited()
await store.vacuum()
mock_optimize.assert_not_called()
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_object_storage_runs_vacuum(self, temp_db_path): async def test_object_storage_runs_vacuum(self):
async with Store(temp_db_path, create=True) as store: with (
with patch.object(get_config().lancedb, "uri", "s3://bucket/path"): patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
),
patch.object(Store, "_init_tables", new_callable=AsyncMock),
):
async with _remote_store_with_mock_tables("s3://bucket/path") as store:
store.chunks_table.tags.list = AsyncMock(return_value={})
with patch.object( with patch.object(
store.chunks_table, "optimize", new_callable=AsyncMock store, "_tables", return_value={"chunks": store.chunks_table}
) as mock_optimize: ):
await store.vacuum() await store.vacuum()
mock_optimize.assert_called() store.chunks_table.optimize.assert_awaited_once()
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_local_runs_vacuum(self, temp_db_path): async def test_local_runs_vacuum(self, temp_db_path):
async with Store(temp_db_path, create=True) as store: async with Store(temp_db_path, create=True) as store:
with patch.object(get_config().lancedb, "uri", ""): with patch.object(
with patch.object( store.chunks_table, "optimize", new_callable=AsyncMock
store.chunks_table, "optimize", new_callable=AsyncMock ) as mock_optimize:
) as mock_optimize: await store.vacuum()
await store.vacuum() mock_optimize.assert_called()
mock_optimize.assert_called()
class TestVectorIndexByConnectionMode: class TestVectorIndexByConnectionMode:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_cloud_skips_index_creation(self, temp_db_path): async def test_cloud_skips_index_creation(self):
async with Store(temp_db_path, create=True) as store: with (
with ( patch(
patch.object(get_config().lancedb, "uri", "db://test-database"), "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
patch.object(get_config().lancedb, "api_key", "test-api-key"), ),
patch.object(get_config().lancedb, "region", "us-east-1"), patch.object(Store, "_init_tables", new_callable=AsyncMock),
):
with patch.object(
store.chunks_table, "count_rows", new_callable=AsyncMock
) as mock_count:
await store._ensure_vector_index()
mock_count.assert_not_called()
@pytest.mark.asyncio
async def test_object_storage_runs_index_creation(self, temp_db_path):
async with Store(temp_db_path, create=True) as store:
with patch.object(get_config().lancedb, "uri", "s3://bucket/path"):
with patch.object(
store.chunks_table,
"count_rows",
new_callable=AsyncMock,
return_value=0,
) as mock_count:
await store._ensure_vector_index()
mock_count.assert_called()
class TestStoreSkipsPathValidationForRemote:
@pytest.mark.asyncio
async def test_skips_path_check_for_cloud(self, tmp_path):
nonexistent = tmp_path / "does_not_exist" / "db.lancedb"
config = AppConfig(
lancedb=LanceDBConfig(
uri="db://test-database", api_key="key", region="us-east-1"
)
)
with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
): ):
with patch.object(Store, "_init_tables", new_callable=AsyncMock): async with _remote_store_with_mock_tables("db://test-database") as store:
async with Store( await store._ensure_vector_index()
nonexistent, store.chunks_table.count_rows.assert_not_awaited()
config=config,
create=True,
skip_validation=True,
skip_migration_check=True,
) as store:
assert store is not None
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_skips_path_check_for_object_storage(self, tmp_path): async def test_object_storage_runs_index_creation(self):
nonexistent = tmp_path / "does_not_exist" / "db.lancedb" with (
config = AppConfig( patch(
lancedb=LanceDBConfig( "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
uri="s3://bucket/path", ),
storage_options={"endpoint": "http://localhost:9000"}, patch.object(Store, "_init_tables", new_callable=AsyncMock),
)
)
with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
): ):
with patch.object(Store, "_init_tables", new_callable=AsyncMock): async with _remote_store_with_mock_tables("s3://bucket/path") as store:
async with Store( store.chunks_table.count_rows = AsyncMock(return_value=0)
nonexistent, await store._ensure_vector_index()
config=config, store.chunks_table.count_rows.assert_awaited_once()
create=True,
skip_validation=True,
skip_migration_check=True, class TestLocationIsFixed:
) as store: @pytest.mark.asyncio
assert store is not None async def test_a_store_keeps_the_location_it_opened(self, temp_db_path):
"""`db_path` and the connection mode derive from the location once; a
store cannot be pointed elsewhere after it is built."""
async with Store(temp_db_path, create=True) as store:
with pytest.raises(AttributeError):
store.location = "s3://bucket/path" # type: ignore[misc]
assert store.location == temp_db_path
assert store._connection_mode == ConnectionMode.LOCAL
class TestInitFailureCleanup: class TestInitFailureCleanup:
@ -412,33 +408,25 @@ class TestStoreMiscellany:
class TestSessionAndConsistency: class TestSessionAndConsistency:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_session_is_shared_across_connections(self): async def test_session_is_shared_across_connections(self):
config = AppConfig(lancedb=LanceDBConfig(uri="s3://bucket/path")) config = AppConfig()
with patch( with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
) as mock_connect: ) as mock_connect:
await connect_lancedb(config) await connect_lancedb("s3://bucket/path", config)
await connect_lancedb(config) await connect_lancedb("s3://bucket/path", config)
sessions = [c.kwargs["session"] for c in mock_connect.call_args_list] sessions = [c.kwargs["session"] for c in mock_connect.call_args_list]
assert sessions[0] is sessions[1] assert sessions[0] is sessions[1]
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_cache_sizes_select_distinct_sessions(self): async def test_cache_sizes_select_distinct_sessions(self):
small = AppConfig( small = AppConfig(lancedb=LanceDBConfig(index_cache_size_bytes=1 << 20))
lancedb=LanceDBConfig( large = AppConfig(lancedb=LanceDBConfig(index_cache_size_bytes=1 << 30))
uri="s3://bucket/path", index_cache_size_bytes=1 << 20
)
)
large = AppConfig(
lancedb=LanceDBConfig(
uri="s3://bucket/path", index_cache_size_bytes=1 << 30
)
)
with patch( with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
) as mock_connect: ) as mock_connect:
await connect_lancedb(small) await connect_lancedb("s3://bucket/path", small)
await connect_lancedb(large) await connect_lancedb("s3://bucket/path", large)
sessions = [c.kwargs["session"] for c in mock_connect.call_args_list] sessions = [c.kwargs["session"] for c in mock_connect.call_args_list]
assert sessions[0] is not sessions[1] assert sessions[0] is not sessions[1]
@ -447,7 +435,6 @@ class TestSessionAndConsistency:
async def test_both_cache_sizes_are_applied(self): async def test_both_cache_sizes_are_applied(self):
config = AppConfig( config = AppConfig(
lancedb=LanceDBConfig( lancedb=LanceDBConfig(
uri="s3://bucket/path",
index_cache_size_bytes=2 << 20, index_cache_size_bytes=2 << 20,
metadata_cache_size_bytes=4 << 20, metadata_cache_size_bytes=4 << 20,
) )
@ -458,7 +445,7 @@ class TestSessionAndConsistency:
), ),
patch("haiku.rag.store.engine.lancedb.Session") as mock_session, patch("haiku.rag.store.engine.lancedb.Session") as mock_session,
): ):
await connect_lancedb(config) await connect_lancedb("s3://bucket/path", config)
mock_session.assert_called_once_with( mock_session.assert_called_once_with(
index_cache_size_bytes=2 << 20, metadata_cache_size_bytes=4 << 20 index_cache_size_bytes=2 << 20, metadata_cache_size_bytes=4 << 20
@ -466,15 +453,11 @@ class TestSessionAndConsistency:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_read_consistency_interval_is_forwarded(self): async def test_read_consistency_interval_is_forwarded(self):
config = AppConfig( config = AppConfig(lancedb=LanceDBConfig(read_consistency_interval_seconds=5))
lancedb=LanceDBConfig(
uri="s3://bucket/path", read_consistency_interval_seconds=5
)
)
with patch( with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
) as mock_connect: ) as mock_connect:
await connect_lancedb(config) await connect_lancedb("s3://bucket/path", config)
assert mock_connect.call_args.kwargs["read_consistency_interval"] == timedelta( assert mock_connect.call_args.kwargs["read_consistency_interval"] == timedelta(
seconds=5 seconds=5
@ -483,14 +466,12 @@ class TestSessionAndConsistency:
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_read_consistency_interval_omitted_when_disabled(self): async def test_read_consistency_interval_omitted_when_disabled(self):
config = AppConfig( config = AppConfig(
lancedb=LanceDBConfig( lancedb=LanceDBConfig(read_consistency_interval_seconds=None)
uri="s3://bucket/path", read_consistency_interval_seconds=None
)
) )
with patch( with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
) as mock_connect: ) as mock_connect:
await connect_lancedb(config) await connect_lancedb("s3://bucket/path", config)
assert mock_connect.call_args.kwargs["read_consistency_interval"] is None assert mock_connect.call_args.kwargs["read_consistency_interval"] is None
@ -500,7 +481,7 @@ class TestSessionAndConsistency:
with patch( with patch(
"haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock "haiku.rag.store.engine.lancedb.connect_async", new_callable=AsyncMock
) as mock_connect: ) as mock_connect:
await connect_lancedb(config, tmp_path / "db.lancedb") await connect_lancedb(tmp_path / "db.lancedb", config)
assert mock_connect.call_args.kwargs["session"] is not None assert mock_connect.call_args.kwargs["session"] is not None
assert mock_connect.call_args.kwargs["read_consistency_interval"] == timedelta( assert mock_connect.call_args.kwargs["read_consistency_interval"] == timedelta(

View file

@ -89,7 +89,7 @@ async def _remote_client(config: AppConfig):
async def test_store_connect_and_create(tmp_path, config): async def test_store_connect_and_create(tmp_path, config):
from haiku.rag.store.info import get_database_stats from haiku.rag.store.info import get_database_stats
async with Store(tmp_path / "unused", config=config, create=True) as store: async with Store(config.lancedb.uri, config=config, create=True) as store:
stats = await get_database_stats(store.db) stats = await get_database_stats(store.db)
assert stats["documents"]["exists"] assert stats["documents"]["exists"]
assert stats["chunks"]["exists"] assert stats["chunks"]["exists"]
@ -97,7 +97,7 @@ async def test_store_connect_and_create(tmp_path, config):
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_store_vacuum(tmp_path, config): async def test_store_vacuum(tmp_path, config):
async with Store(tmp_path / "unused", config=config, create=True) as store: async with Store(config.lancedb.uri, config=config, create=True) as store:
await store.vacuum() await store.vacuum()
@ -106,7 +106,7 @@ async def test_store_add_document(tmp_path, config):
from haiku.rag.store.info import get_database_stats from haiku.rag.store.info import get_database_stats
from haiku.rag.store.schema import DocumentRecord from haiku.rag.store.schema import DocumentRecord
async with Store(tmp_path / "unused", config=config, create=True) as store: async with Store(config.lancedb.uri, config=config, create=True) as store:
doc = DocumentRecord(content="The quick brown fox jumps over the lazy dog.") doc = DocumentRecord(content="The quick brown fox jumps over the lazy dog.")
await store.documents_table.add([doc]) await store.documents_table.add([doc])