Take a schemeless lancedb.uri as a local path
Closes #582. `lancedb.databases` entries already classify a location by whether it carries a scheme; `lancedb.uri` was taken as a URI whatever it said, so a local path was opened as object storage and a mistyped one became a new empty database instead of failing. Both settings now place a database the same way, and `--db PATH` overrides either.
This commit is contained in:
parent
49580228c2
commit
5b420a9b23
6 changed files with 162 additions and 3 deletions
|
|
@ -16,6 +16,7 @@
|
|||
- `doctor`'s docling-serve probe sends `X-Api-Key`, so an instance requiring a key is reported reachable rather than unreachable.
|
||||
- The picture-description request to the public OpenAI endpoint sends `OPENAI_API_KEY`; it carried no authorization header.
|
||||
- `haiku-rag` prints the message and exits when the configured embedder does not match the database, instead of raising a traceback.
|
||||
- A `lancedb.uri` with no scheme is a local path, as it already is in `lancedb.databases`: `haiku-rag init` creates it and every command that opens an existing database requires it to exist, where a missing path was opened as object storage and became an empty database. `--db PATH` overrides `lancedb.uri`.
|
||||
|
||||
## [0.77.0] - 2026-08-21
|
||||
|
||||
|
|
|
|||
|
|
@ -66,6 +66,29 @@ If that number times six exceeds available RAM, use one of:
|
|||
|
||||
This is an upstream limitation rather than a `haiku.rag` setting. Compaction bounds itself by row count instead of bytes, and LanceDB's async API exposes no batch size or fragment target to override it. Tracked at [lancedb/lancedb#2325](https://github.com/lancedb/lancedb/issues/2325). The requirement above will drop once compaction batches by bytes.
|
||||
|
||||
### Changing the Default Database Path
|
||||
|
||||
`storage.data_dir` holds the default database, always called
|
||||
`haiku.rag.lancedb`. To put the database somewhere else for every command, give
|
||||
`lancedb.uri` a local path:
|
||||
|
||||
```yaml
|
||||
lancedb:
|
||||
uri: /data/notes.lancedb
|
||||
```
|
||||
|
||||
An explicit `--db PATH` overrides `lancedb.uri` for that invocation.
|
||||
|
||||
This places one database without naming it. Its `source` is `None` in search
|
||||
results, citations and documents, since only [`lancedb.databases`](#several-databases)
|
||||
assigns the names that carry provenance. A path here changes where the database
|
||||
lives, not what it is called.
|
||||
|
||||
A value with no scheme is a local path wherever it is configured, so
|
||||
`haiku-rag init` creates it and every command that opens an existing database
|
||||
requires it to exist. A mistyped path fails rather than becoming a new empty
|
||||
database.
|
||||
|
||||
## Database Creation
|
||||
|
||||
Databases must be explicitly created before use:
|
||||
|
|
@ -145,6 +168,7 @@ lancedb:
|
|||
- **LanceDB Cloud** (`db://`): Requires `api_key` and `region`. Table optimization and indexing are managed server-side.
|
||||
- **Object storage** (`s3://`, `gs://`, `az://`, `hdfs://`): Uses `storage_options` for credentials and endpoint configuration. Authentication can also be provided via environment variables (`AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, etc.) or cloud provider SDK defaults (AWS CLI, Azure CLI, gcloud).
|
||||
- **S3-compatible stores** (MinIO, Tigris, etc.): Set `endpoint` in `storage_options`. When using `http://` endpoints, also set `allow_http: "true"`.
|
||||
- **Local path** (no scheme): `uri` also takes a local path, which is how the default database is pointed elsewhere. See [Changing the Default Database Path](#changing-the-default-database-path).
|
||||
|
||||
The `storage_options` keys are case-insensitive and passed directly to the underlying object store library. Available keys depend on the backend. See the [LanceDB storage docs](https://lancedb.com/docs/storage/) for details.
|
||||
|
||||
|
|
|
|||
|
|
@ -42,8 +42,9 @@ class DatabaseRef:
|
|||
|
||||
@classmethod
|
||||
def configured(cls, name: str | None, location: str) -> "DatabaseRef":
|
||||
"""A database from ``lancedb.databases``, where the configured value is a
|
||||
URI or a local path depending on whether it carries a scheme."""
|
||||
"""A database the configuration placed, by ``lancedb.uri`` or by an entry
|
||||
in ``lancedb.databases``. A value carrying a scheme is a URI and anything
|
||||
else is a local path, so the two settings place a database alike."""
|
||||
uri, db_path = locate_database(location)
|
||||
return cls(name=name, uri=uri, db_path=db_path)
|
||||
|
||||
|
|
@ -123,7 +124,7 @@ class DatabaseScope:
|
|||
)
|
||||
|
||||
if config.lancedb.uri:
|
||||
return cls((DatabaseRef(name=None, uri=config.lancedb.uri, db_path=None),))
|
||||
return cls((DatabaseRef.configured(None, config.lancedb.uri),))
|
||||
|
||||
return cls((DatabaseRef.at(config.storage.data_dir / "haiku.rag.lancedb"),))
|
||||
|
||||
|
|
|
|||
|
|
@ -334,6 +334,74 @@ class TestResolvingTheDatabasePath:
|
|||
resolve_scope(Path("/data/other.lancedb"))
|
||||
|
||||
|
||||
class TestConfiguredLocalUri:
|
||||
"""`lancedb.uri` with a local path, the surface issue #582 reports."""
|
||||
|
||||
def _config_file(self, tmp_path, located: Path) -> Path:
|
||||
config_file = tmp_path / "haiku.rag.yaml"
|
||||
config_file.write_text(f"lancedb:\n uri: {located}\n")
|
||||
return config_file
|
||||
|
||||
def _fresh(self, monkeypatch) -> None:
|
||||
import haiku.rag.cli as cli_module
|
||||
import haiku.rag.config as config_module
|
||||
|
||||
monkeypatch.setattr(config_module, "_config", None)
|
||||
monkeypatch.setattr(cli_module, "_database", None)
|
||||
|
||||
def _reports(self, result, located: Path) -> bool:
|
||||
"""Rich wraps a long path to the terminal width, so compare without it."""
|
||||
return str(located) in "".join(result.output.split())
|
||||
|
||||
def test_init_creates_the_configured_local_database(self, tmp_path, monkeypatch):
|
||||
self._fresh(monkeypatch)
|
||||
located = tmp_path / "notes.lancedb"
|
||||
config_file = self._config_file(tmp_path, located)
|
||||
|
||||
result = runner.invoke(cli, ["--config", str(config_file), "init"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert located.exists()
|
||||
|
||||
self._fresh(monkeypatch)
|
||||
result = runner.invoke(cli, ["--config", str(config_file), "info"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert self._reports(result, located)
|
||||
|
||||
def test_a_missing_configured_path_is_refused(self, tmp_path, monkeypatch):
|
||||
"""A schemeless value is a local path, so a typo fails instead of
|
||||
becoming a new empty database."""
|
||||
self._fresh(monkeypatch)
|
||||
located = tmp_path / "typo.lancedb"
|
||||
config_file = self._config_file(tmp_path, located)
|
||||
|
||||
result = runner.invoke(cli, ["--config", str(config_file), "info"])
|
||||
|
||||
assert "does not exist" in result.output
|
||||
assert not located.exists()
|
||||
|
||||
def test_db_overrides_the_configured_uri(self, tmp_path, monkeypatch):
|
||||
self._fresh(monkeypatch)
|
||||
configured = tmp_path / "configured.lancedb"
|
||||
chosen = tmp_path / "chosen.lancedb"
|
||||
config_file = self._config_file(tmp_path, configured)
|
||||
|
||||
result = runner.invoke(
|
||||
cli, ["--config", str(config_file), "init", "--db", str(chosen)]
|
||||
)
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert chosen.exists()
|
||||
assert not configured.exists()
|
||||
|
||||
self._fresh(monkeypatch)
|
||||
result = runner.invoke(
|
||||
cli, ["--config", str(config_file), "info", "--db", str(chosen)]
|
||||
)
|
||||
assert result.exit_code == 0, result.output
|
||||
assert self._reports(result, chosen)
|
||||
|
||||
|
||||
class TestCliConfigMismatchError:
|
||||
def test_a_config_mismatch_exits_with_its_remedy(self):
|
||||
"""The message says which database and what to run, so it is worth more
|
||||
|
|
|
|||
|
|
@ -74,6 +74,17 @@ class TestResolution:
|
|||
|
||||
assert scope.databases == (DatabaseRef(None, "s3://bucket/one.lancedb", None),)
|
||||
|
||||
def test_a_bare_uri_without_a_scheme_is_a_local_path(self):
|
||||
"""`lancedb.uri` places one database the same way an entry in
|
||||
`lancedb.databases` does, so a schemeless value is a path and gets the
|
||||
existence check a local database gets."""
|
||||
scope = DatabaseScope.resolve(_config(uri="/data/notes.lancedb"))
|
||||
|
||||
[ref] = scope.databases
|
||||
assert ref.name is None
|
||||
assert ref.db_path == Path("/data/notes.lancedb")
|
||||
assert ref.uri == ""
|
||||
|
||||
def test_a_path_selects_the_database_over_a_configured_uri(self):
|
||||
"""`--db` exists to override what is configured, and the configuration
|
||||
derived from the ref is what makes the connection follow it."""
|
||||
|
|
|
|||
|
|
@ -145,6 +145,60 @@ class TestNamingADatabaseDirectly:
|
|||
assert [r.source for r in results] == ["alpha"]
|
||||
|
||||
|
||||
class TestOneConfiguredLocation:
|
||||
"""`lancedb.uri` places one unnamed database, at a URI or at a local path."""
|
||||
|
||||
def _config(self, location) -> AppConfig:
|
||||
return AppConfig(lancedb=LanceDBConfig(uri=str(location)))
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_local_uri_opens_the_configured_database(self, tmp_path):
|
||||
located = tmp_path / "notes.lancedb"
|
||||
config = self._config(located)
|
||||
|
||||
async with HaikuRAG(config=config, create=True) as rag:
|
||||
assert rag.store.db_path == located
|
||||
# It places a database without naming one: only `lancedb.databases`
|
||||
# assigns the name results and citations carry.
|
||||
assert rag.source is None
|
||||
assert located.exists()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_explicit_path_overrides_a_local_uri(self, tmp_path):
|
||||
"""`--db` overrides the configured location for one invocation."""
|
||||
config = self._config(tmp_path / "configured.lancedb")
|
||||
chosen = tmp_path / "chosen.lancedb"
|
||||
|
||||
async with HaikuRAG(chosen, config=config, create=True) as rag:
|
||||
assert rag.store.db_path == chosen
|
||||
assert chosen.exists()
|
||||
assert not (tmp_path / "configured.lancedb").exists()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_local_uri_that_does_not_exist_is_refused(self, tmp_path):
|
||||
"""A mistyped path fails instead of quietly becoming an empty database,
|
||||
which is what a value carrying a scheme would do."""
|
||||
config = self._config(tmp_path / "typo.lancedb")
|
||||
|
||||
with pytest.raises(FileNotFoundError):
|
||||
async with HaikuRAG(config=config):
|
||||
pass
|
||||
assert not (tmp_path / "typo.lancedb").exists()
|
||||
|
||||
def test_a_uri_with_a_scheme_stays_a_uri(self, tmp_path):
|
||||
"""Object storage has no local path to check, and a location that does
|
||||
not exist yet is normal there."""
|
||||
from haiku.rag.store.engine import ConnectionMode
|
||||
|
||||
config = self._config("s3://bucket/one.lancedb")
|
||||
|
||||
[ref] = DatabaseScope.resolve(config).databases
|
||||
one, db_path = ref.connection(config)
|
||||
|
||||
assert db_path is None
|
||||
assert ConnectionMode.from_config(one) == ConnectionMode.OBJECT_STORAGE
|
||||
|
||||
|
||||
class TestOpeningDatabases:
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_databases_open_together(self, tmp_path):
|
||||
|
|
|
|||
Loading…
Reference in a new issue