From db5f61d740f6f57e05f6d583238133813d3dd3e1 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 26 Aug 2026 15:09:28 +0300 Subject: [PATCH] Rename the database selector to --db-name `--database` and `--db` read as one word abbreviated but take different things, a configured name against a filesystem path, and they sit in different positions: `haiku-rag --db /path info` fails with `No such option`. The CLI already says "database" to the user everywhere it prints one, so the name it selects by should say the same. Unreleased, so no deprecation. Also: without a reranker, raise `search.limit` with the number of databases searched, since each contributes its best matches to a list truncated back to the limit. --- CHANGELOG.md | 2 +- docs/cli.md | 6 +-- docs/configuration/storage.md | 21 +++++--- haiku_rag_slim/haiku/rag/chat/__init__.py | 2 +- haiku_rag_slim/haiku/rag/cli.py | 22 ++++----- tests/chat/test_chat_app.py | 2 +- tests/multi_db/test_documents.py | 2 +- tests/multi_db/test_scope.py | 2 +- tests/test_cli.py | 58 +++++++++++------------ 9 files changed, 61 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e527e66..e0fb215b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- `lancedb.databases`: a name-to-location mapping for searching several databases at once, mutually exclusive with `lancedb.uri`. `client.search(..., sources=[...])` selects which to search, `sources=None` searches all of them, and `SearchResult.source` carries the configured name a result came from. `Document.source` names it on a document from a listing or a lookup, so a listing that spans databases says which one each came from. Candidates are fused by the configured reranker over the union, or by reciprocal rank fusion when none is configured. Databases searched together must have been written with the same embedder; two that disagree raise `ConfigMismatchError`. The query is embedded once for the whole selection. `SearchResult.format_for_agent` names the database, so the model can attribute evidence to one while it answers. `haiku-rag search`, `ask`, `analyze` and `chat` cover the configured set and label each result and citation with its database. `settings`, `init-config` and `download-models` open no database; every other command works on one, named with `--database NAME` or `--db PATH`, or resolved from a configured set of one. +- `lancedb.databases`: a name-to-location mapping for searching several databases at once, mutually exclusive with `lancedb.uri`. `client.search(..., sources=[...])` selects which to search, `sources=None` searches all of them, and `SearchResult.source` carries the configured name a result came from. `Document.source` names it on a document from a listing or a lookup, so a listing that spans databases says which one each came from. Candidates are fused by the configured reranker over the union, or by reciprocal rank fusion when none is configured. Databases searched together must have been written with the same embedder; two that disagree raise `ConfigMismatchError`. The query is embedded once for the whole selection. `SearchResult.format_for_agent` names the database, so the model can attribute evidence to one while it answers. `haiku-rag search`, `ask`, `analyze` and `chat` cover the configured set and label each result and citation with its database. `settings`, `init-config` and `download-models` open no database; every other command works on one, named with `--db-name NAME` or `--db PATH`, or resolved from a configured set of one. - `client.ask(..., sources=[...])` asks across the selected databases, and `Citation.source` names the one a cited chunk came from. The cite fallback for an id absent from the run's results looks only in the selected databases, so a question scoped to some cannot cite another. A chunk id held by two of the databases searched raises `AmbiguousCitationError`, which reaches the model as a retry; the fallback refuses it too, rather than answering from the first database that holds it. - `client.analyze(..., sources=[...])` analyzes across the selected databases: the sandbox mounts their documents under one flat `/documents/{id}/` namespace, resolving each id to the database holding it, and in-code `search()` covers the same selection. diff --git a/docs/cli.md b/docs/cli.md index dfea37ec..b1fc857b 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -7,7 +7,7 @@ The `haiku-rag` CLI provides complete document management functionality. - `--config` - Specify custom configuration file - `--read-only` - Open database in read-only mode (blocks writes, skips upgrades) - - `--database` - Name of a database from `lancedb.databases` to work on + - `--db-name` - Name of a database from `lancedb.databases` to work on - `--version` / `-v` - Show version and exit Per-command options: @@ -20,14 +20,14 @@ The `haiku-rag` CLI provides complete document management functionality. haiku-rag --config /path/to/config.yaml list haiku-rag --config /path/to/config.yaml list --db /path/to/custom.db haiku-rag --read-only search "query" - haiku-rag --database medic list + haiku-rag --db-name medic list haiku-rag add -h ``` With `lancedb.databases` configured, `search`, `ask`, `analyze` and `chat` cover every database in it. `settings`, `init-config` and `download-models` open no database at all. Every other command works on one, named with - `--database` or `--db`. See + `--db-name` or `--db`. See [Several Databases](configuration/storage.md#several-databases). ## Document Management diff --git a/docs/configuration/storage.md b/docs/configuration/storage.md index 1994dbee..0e35eab4 100644 --- a/docs/configuration/storage.md +++ b/docs/configuration/storage.md @@ -273,6 +273,13 @@ the same queries: retrieval MAP 0.9914 with a reranker against 0.9918 for the same corpus in a single database, and 0.6044 without one against 0.9798. The cost is that a reranker scores candidates in proportion to the number of databases. +Without one, consider raising `search.limit` with the number of databases +searched. Each contributes its own best matches to a list that is then truncated +at the limit, so with three full rankings and a limit of 5 any one database may +contribute only one or two results. Raising the limit raises what the caller and +the model receive, since without a reranker nothing is over-fetched to absorb +it. + Creating names a database: `create=True` on a client covering the set raises `AmbiguousDatabaseError`, and `HaikuRAG(config=config, create=True, sources=["name"])` creates that one. @@ -292,21 +299,21 @@ from a complete one. Commands fall into three groups: - **Set-capable**: `search`, `ask`, `analyze` and `chat` cover the whole - configured set, or the subset named by `--database`. + configured set, or the subset named by `--db-name`. - **Config-only**: `settings`, `init-config` and `download-models` open no database, so the set is irrelevant to them. - **Single-database**: everything else — document writes, `rebuild`, `vacuum`, `migrate`, `init`, `info`, `history`, `tag`, `doctor`, `list`, `inspect`, `visualize` and `mcp` — works on one database, named with the global - `--database` option. + `--db-name` option. ```bash haiku-rag search "query" # every configured database -haiku-rag --database medic list # one of them -haiku-rag --database medic migrate +haiku-rag --db-name medic list # one of them +haiku-rag --db-name medic migrate ``` -`--database` takes a name from `lancedb.databases`, which is how a database +`--db-name` takes a name from `lancedb.databases`, which is how a database behind a URI is reached. `--db` takes a path, and overrides the configured location with that one database. A single-database command given neither fails rather than choosing for you, unless `lancedb.databases` names exactly one: a @@ -315,8 +322,8 @@ set of one is unambiguous and is used, keeping its configured name. Each database is created, migrated and vacuumed on its own: ```bash -haiku-rag --database medic init -haiku-rag --database st init +haiku-rag --db-name medic init +haiku-rag --db-name st init ``` ## Vector Indexing diff --git a/haiku_rag_slim/haiku/rag/chat/__init__.py b/haiku_rag_slim/haiku/rag/chat/__init__.py index 9dd2c2b4..51817129 100644 --- a/haiku_rag_slim/haiku/rag/chat/__init__.py +++ b/haiku_rag_slim/haiku/rag/chat/__init__.py @@ -44,7 +44,7 @@ def run_chat( # The capabilities read the databases the scope covers, not whatever the # configuration happens to name: `--db PATH` would otherwise leave them on - # the default database, and `--database NAME` on the whole set. + # the default database, and `--db-name NAME` on the whole set. if scope.covers_multiple: capability_config, capability_db_path = config, None else: diff --git a/haiku_rag_slim/haiku/rag/cli.py b/haiku_rag_slim/haiku/rag/cli.py index db1d580c..ca876c73 100644 --- a/haiku_rag_slim/haiku/rag/cli.py +++ b/haiku_rag_slim/haiku/rag/cli.py @@ -59,7 +59,7 @@ def cli(): # Module-level flags set by callback _read_only: bool = False -_database: str | None = None +_db_name: str | None = None def create_app(db: Path | None = None, *, covers_set: bool = False) -> "HaikuRAGApp": @@ -70,7 +70,7 @@ def create_app(db: Path | None = None, *, covers_set: bool = False) -> "HaikuRAG Raises: AmbiguousDatabaseError: several databases are configured and this - command works on one, without `--db` or `--database` naming which. + command works on one, without `--db` or `--db-name` naming which. """ from haiku.rag.app import HaikuRAGApp @@ -86,24 +86,24 @@ def resolve_scope( ) -> "DatabaseScope": """The databases a command works on, resolved once. - The CLI decides only what it alone knows: that `--db` and `--database` are + The CLI decides only what it alone knows: that `--db` and `--db-name` are the same thing said twice, and whether this command can read more than one. Everything else — an unknown name, a legacy `uri`, the default location — is `DatabaseScope.resolve`'s to answer, so there is one table and not two. """ from haiku.rag.client.scope import DatabaseScope - if db is not None and _database is not None: + if db is not None and _db_name is not None: raise AmbiguousDatabaseError( - "pass --db or --database, not both: they name the same thing" + "pass --db or --db-name, not both: they name the same thing" ) scope = DatabaseScope.resolve( - get_config(), database_name=_database, database_path=db + get_config(), database_name=_db_name, database_path=db ) if scope.covers_multiple and not covers_set: raise AmbiguousDatabaseError( f"lancedb.databases names {', '.join(sorted(scope.names))}; this " - "command works on a single database: pass --database NAME, or " + "command works on a single database: pass --db-name NAME, or " "--db PATH." ) return scope @@ -145,16 +145,16 @@ def main( "--read-only", help="Open database in read-only mode", ), - database: str | None = typer.Option( + db_name: str | None = typer.Option( None, - "--database", + "--db-name", help="Name of a database from lancedb.databases to work on", ), ): """haiku.rag CLI - Vector database RAG system""" - global _read_only, _database + global _read_only, _db_name _read_only = read_only - _database = database + _db_name = db_name # Load config from --config, local folder, or default directory config_path = find_config_file(cli_path=config) if config_path: diff --git a/tests/chat/test_chat_app.py b/tests/chat/test_chat_app.py index cf6b730f..2b7f574a 100644 --- a/tests/chat/test_chat_app.py +++ b/tests/chat/test_chat_app.py @@ -63,7 +63,7 @@ def test_run_chat_covers_a_configured_set(tmp_path, monkeypatch): def test_chat_capabilities_read_the_named_database(tmp_path, monkeypatch): - """`--db PATH` and `--database NAME` have to reach the capabilities too, or + """`--db PATH` and `--db-name NAME` have to reach the capabilities too, or they answer from the default database or the whole set.""" import haiku.rag.config as config_module from haiku.rag.client.scope import DatabaseScope diff --git a/tests/multi_db/test_documents.py b/tests/multi_db/test_documents.py index 9a7fe21d..a7333d9f 100644 --- a/tests/multi_db/test_documents.py +++ b/tests/multi_db/test_documents.py @@ -197,7 +197,7 @@ class TestDocumentsNameTheirDatabase: @pytest.mark.asyncio async def test_one_named_database_still_names_itself(self, tmp_path): - """`haiku-rag --database alpha list` opens one database, and its name is + """`haiku-rag --db-name alpha list` opens one database, and its name is the whole reason the option exists.""" config = _config(tmp_path, ["alpha", "beta"]) await _seed(config, "alpha", ["alpha one"]) diff --git a/tests/multi_db/test_scope.py b/tests/multi_db/test_scope.py index a081b18f..82966e55 100644 --- a/tests/multi_db/test_scope.py +++ b/tests/multi_db/test_scope.py @@ -249,7 +249,7 @@ class TestPlacingADatabase: class TestNamingOneOfTheSetOnTheCommandLine: - """`--database NAME` reaches the application layer as a name, and every + """`--db-name NAME` reaches the application layer as a name, and every client it opens has to honour it — one that ignores it covers the set and quietly answers from the wrong database.""" diff --git a/tests/test_cli.py b/tests/test_cli.py index 9a3be545..d6e9f7cc 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -109,7 +109,7 @@ class TestOneDatabaseCommands: import haiku.rag.config as config_module monkeypatch.setattr(config_module, "_config", None) - monkeypatch.setattr("haiku.rag.cli._database", None) + monkeypatch.setattr("haiku.rag.cli._db_name", None) set_config(AppConfig(lancedb=LanceDBConfig(databases=databases))) def test_a_configured_set_refuses_a_one_database_command(self, monkeypatch): @@ -156,7 +156,7 @@ class TestOneDatabaseCommands: import haiku.rag.config as config_module monkeypatch.setattr(config_module, "_config", None) - monkeypatch.setattr("haiku.rag.cli._database", None) + monkeypatch.setattr("haiku.rag.cli._db_name", None) set_config(AppConfig(storage=StorageConfig(data_dir=tmp_path))) [ref] = resolve_scope(None).databases @@ -172,7 +172,7 @@ class TestOneDatabaseCommands: class TestSelectingADatabaseByName: - """`--database NAME` is the only way to reach a configured database whose + """`--db-name NAME` is the only way to reach a configured database whose location is a URI, since `--db` takes a path.""" @staticmethod @@ -186,13 +186,13 @@ class TestSelectingADatabaseByName: """Not resolved to a path: the name is what results and citations carry, and rewriting the configuration is what used to lose it.""" self._install(monkeypatch, medic="s3://bucket/prefix/medic.lancedb") - monkeypatch.setattr("haiku.rag.cli._database", "medic") + monkeypatch.setattr("haiku.rag.cli._db_name", "medic") assert resolve_scope(None).names == ("medic",) def test_naming_a_database_leaves_the_configuration_alone(self, monkeypatch): self._install(monkeypatch, st="/data/st.lancedb", other="/data/o.lancedb") - monkeypatch.setattr("haiku.rag.cli._database", "st") + monkeypatch.setattr("haiku.rag.cli._db_name", "st") resolve_scope(None) @@ -205,14 +205,14 @@ class TestSelectingADatabaseByName: def test_an_unknown_name_names_the_configured_ones(self, monkeypatch): self._install(monkeypatch, alpha="/data/a.lancedb", beta="/data/b.lancedb") - monkeypatch.setattr("haiku.rag.cli._database", "gamma") + monkeypatch.setattr("haiku.rag.cli._db_name", "gamma") with pytest.raises(AmbiguousDatabaseError, match="alpha, beta"): resolve_scope(None) def test_an_unknown_name_does_not_leak_locations(self, monkeypatch): self._install(monkeypatch, medic="s3://bucket/prefix/medic.lancedb") - monkeypatch.setattr("haiku.rag.cli._database", "gamma") + monkeypatch.setattr("haiku.rag.cli._db_name", "gamma") with pytest.raises(AmbiguousDatabaseError) as raised: resolve_scope(None) @@ -221,13 +221,13 @@ class TestSelectingADatabaseByName: def test_selecting_nothing_reports_an_empty_mapping(self, monkeypatch): self._install(monkeypatch) - monkeypatch.setattr("haiku.rag.cli._database", "medic") + monkeypatch.setattr("haiku.rag.cli._db_name", "medic") with pytest.raises(AmbiguousDatabaseError, match="nothing"): resolve_scope(None) def test_the_callback_selects_before_a_command_runs(self, tmp_path, monkeypatch): - """`--database` is resolved once the config is loaded, so every command + """`--db-name` is resolved once the config is loaded, so every command and both TUIs see the selected database.""" import haiku.rag.config as config_module @@ -236,7 +236,7 @@ class TestSelectingADatabaseByName: config_file.write_text("lancedb:\n databases:\n alpha: /data/a.lancedb\n") result = runner.invoke( - cli, ["--config", str(config_file), "--database", "nope", "list"] + cli, ["--config", str(config_file), "--db-name", "nope", "list"] ) assert result.exit_code != 0 @@ -245,24 +245,22 @@ class TestSelectingADatabaseByName: def test_a_selection_does_not_outlive_its_invocation(self, tmp_path, monkeypatch): """The selector is module state, so a second invocation without - `--database` must not inherit the first one's database.""" + `--db-name` must not inherit the first one's database.""" import haiku.rag.cli as cli_module import haiku.rag.config as config_module monkeypatch.setattr(config_module, "_config", None) - monkeypatch.setattr(cli_module, "_database", None) + monkeypatch.setattr(cli_module, "_db_name", None) config_file = tmp_path / "haiku.rag.yaml" selected = tmp_path / "alpha.lancedb" config_file.write_text(f"lancedb:\n databases:\n alpha: {selected}\n") - runner.invoke( - cli, ["--config", str(config_file), "--database", "alpha", "info"] - ) - assert cli_module._database == "alpha" + runner.invoke(cli, ["--config", str(config_file), "--db-name", "alpha", "info"]) + assert cli_module._db_name == "alpha" runner.invoke(cli, ["--config", str(config_file), "settings"]) - assert cli_module._database is None + assert cli_module._db_name is None def test_a_selection_does_not_outlive_its_invocation_in_process( self, tmp_path, monkeypatch @@ -273,7 +271,7 @@ class TestSelectingADatabaseByName: import haiku.rag.config as config_module monkeypatch.setattr(config_module, "_config", None) - monkeypatch.setattr(cli_module, "_database", None) + monkeypatch.setattr(cli_module, "_db_name", None) config_file = tmp_path / "haiku.rag.yaml" config_file.write_text( f"lancedb:\n databases:\n alpha: {tmp_path / 'alpha.lancedb'}\n" @@ -281,7 +279,7 @@ class TestSelectingADatabaseByName: ) runner.invoke( - cli, ["--config", str(config_file), "--database", "alpha", "settings"] + cli, ["--config", str(config_file), "--db-name", "alpha", "settings"] ) # Naming one leaves the configuration naming both. assert get_config().lancedb.uri == "" @@ -290,7 +288,7 @@ class TestSelectingADatabaseByName: runner.invoke(cli, ["--config", str(config_file), "settings"]) assert set(get_config().lancedb.databases) == {"alpha", "beta"} - assert cli_module._database is None + assert cli_module._db_name is None def test_a_selection_does_not_outlive_an_invocation_without_a_config_file( self, tmp_path, monkeypatch @@ -301,7 +299,7 @@ class TestSelectingADatabaseByName: import haiku.rag.config as config_module monkeypatch.setattr(config_module, "_config", None) - monkeypatch.setattr(cli_module, "_database", None) + monkeypatch.setattr(cli_module, "_db_name", None) monkeypatch.delenv("HAIKU_RAG_CONFIG_PATH", raising=False) monkeypatch.chdir(tmp_path) config_file = tmp_path / "selected.yaml" @@ -310,25 +308,25 @@ class TestSelectingADatabaseByName: ) runner.invoke( - cli, ["--config", str(config_file), "--database", "medic", "settings"] + cli, ["--config", str(config_file), "--db-name", "medic", "settings"] ) assert get_config().lancedb.databases == {"medic": "s3://bucket/medic.lancedb"} runner.invoke(cli, ["settings"]) assert get_config().lancedb.databases == {} - assert cli_module._database is None + assert cli_module._db_name is None class TestResolvingTheDatabasePath: def test_a_path_wins_when_nothing_is_selected(self, monkeypatch): - monkeypatch.setattr("haiku.rag.cli._database", None) + monkeypatch.setattr("haiku.rag.cli._db_name", None) [ref] = resolve_scope(Path("/data/one.lancedb")).databases assert ref.db_path == Path("/data/one.lancedb") def test_naming_a_database_twice_is_refused(self, monkeypatch): - monkeypatch.setattr("haiku.rag.cli._database", "st") + monkeypatch.setattr("haiku.rag.cli._db_name", "st") with pytest.raises(AmbiguousDatabaseError, match="not both"): resolve_scope(Path("/data/other.lancedb")) @@ -347,7 +345,7 @@ class TestConfiguredLocalUri: import haiku.rag.config as config_module monkeypatch.setattr(config_module, "_config", None) - monkeypatch.setattr(cli_module, "_database", None) + monkeypatch.setattr(cli_module, "_db_name", None) def _reports(self, result, located: Path) -> bool: """Rich wraps a long path to the terminal width, so compare without it.""" @@ -695,7 +693,7 @@ class TestChatCoversTheSet: import haiku.rag.config as config_module monkeypatch.setattr(config_module, "_config", None) - monkeypatch.setattr(cli_module, "_database", None) + monkeypatch.setattr(cli_module, "_db_name", None) with patch("haiku.rag.chat.run_chat") as run_chat: result = runner.invoke( @@ -711,7 +709,7 @@ class TestChatCoversTheSet: import haiku.rag.config as config_module monkeypatch.setattr(config_module, "_config", None) - monkeypatch.setattr(cli_module, "_database", None) + monkeypatch.setattr(cli_module, "_db_name", None) with patch("haiku.rag.chat.run_chat") as run_chat: result = runner.invoke( @@ -719,7 +717,7 @@ class TestChatCoversTheSet: [ "--config", str(self._config_file(tmp_path)), - "--database", + "--db-name", "wiki", "chat", ], @@ -735,7 +733,7 @@ class TestChatCoversTheSet: import haiku.rag.config as config_module monkeypatch.setattr(config_module, "_config", None) - monkeypatch.setattr(cli_module, "_database", None) + monkeypatch.setattr(cli_module, "_db_name", None) with patch("haiku.rag.chat.run_chat") as run_chat: result = runner.invoke(cli, ["chat", "--db", str(tmp_path / "one.lancedb")])