diff --git a/CHANGELOG.md b/CHANGELOG.md index b10e8b5d..e27bcd2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,12 +22,11 @@ ### Fixed -- `haiku-rag settings` prints YAML instead of Python dict reprs. -- The chat document filter pages results, searches the database for the rest, and - lists the selected separately, instead of mounting a checkbox per document. - Selection is by document ID. +- `haiku-rag settings` prints YAML. +- The chat document filter pages results and lists the selected separately. + Selection is by document ID, and a typed search applies on enter. - `haiku-rag list` prints only the fields a document has. -- `haiku-rag` and `haiku-ingester` print the message and exit when the configured embedder does not match the database, instead of raising a traceback. +- `haiku-rag` and `haiku-ingester` exit with a message on an embedder mismatch. - Capabilities created without a client honor `lancedb.uri`. - A `lancedb.uri` without a scheme is treated as a local path. `--db PATH` overrides it. diff --git a/evaluations/evaluations/config.py b/evaluations/evaluations/config.py index 96c56232..8a75c413 100644 --- a/evaluations/evaluations/config.py +++ b/evaluations/evaluations/config.py @@ -7,6 +7,7 @@ from datasets import Dataset from pydantic import BaseModel, model_validator from pydantic_evals import Case from pydantic_evals.evaluators import Evaluator +from haiku.rag.config.models import AppConfig class Turn(BaseModel): @@ -84,7 +85,7 @@ class DatasetSpec: experiment_metadata: dict[str, Any] | None = None def uses_configured_databases( - self, config, override_path: Path | None = None + self, config: AppConfig, override_path: Path | None = None ) -> bool: """Whether `lancedb.databases` places the databases to evaluate over. diff --git a/evaluations/tests/test_config.py b/evaluations/tests/test_config.py index a3123e89..81ceb222 100644 --- a/evaluations/tests/test_config.py +++ b/evaluations/tests/test_config.py @@ -179,7 +179,18 @@ class TestCoversASet: assert spec.uses_configured_databases(config, _Path("/chosen.lancedb")) is False - def test_one_database_is_not_a_set(self): + def test_a_configured_set_of_one_is_still_configured(self): + """A mapping of one is a named database like any other.""" + from haiku.rag.config.models import AppConfig, LanceDBConfig + + from evaluations.datasets import DATASETS + + spec = next(iter(DATASETS.values())) + config = AppConfig(lancedb=LanceDBConfig(databases={"a": "/a.lancedb"})) + + assert spec.uses_configured_databases(config) is True + + def test_naming_no_database_is_not_a_set(self): from haiku.rag.config.models import AppConfig from evaluations.datasets import DATASETS diff --git a/tests/multi_db/test_capabilities.py b/tests/multi_db/test_capabilities.py index a3a50422..661a2195 100644 --- a/tests/multi_db/test_capabilities.py +++ b/tests/multi_db/test_capabilities.py @@ -81,7 +81,6 @@ class TestStandaloneCapabilities: assert "beta document" in formatted @pytest.mark.asyncio - @pytest.mark.vcr() async def test_an_analysis_capability_mounts_the_configured_set(self, tmp_path): from haiku.rag.capabilities.analysis import ( create_capability as create_analysis, diff --git a/tests/multi_db/test_search.py b/tests/multi_db/test_search.py index 637d2b51..33048087 100644 --- a/tests/multi_db/test_search.py +++ b/tests/multi_db/test_search.py @@ -268,6 +268,84 @@ class TestRerankerFusion: assert stub.attached == {"alpha": b"bytes-alpha", "beta": b"bytes-beta"} +class TestOverFetchingForAReranker: + """A reranker needs more candidates than it returns. Ranking without one does + not, and an image query keeps its vector ranking either way.""" + + @staticmethod + def _limits_asked(monkeypatch) -> list[int]: + from haiku.rag.store.repositories.chunk import ChunkRepository + + asked: list[int] = [] + search = ChunkRepository.search + + async def spy(self, *args, **kwargs): + asked.append(kwargs["limit"]) + return await search(self, *args, **kwargs) + + monkeypatch.setattr(ChunkRepository, "search", spy) + return asked + + @pytest.mark.asyncio + async def test_a_text_query_over_fetches_for_a_reranker( + self, tmp_path, monkeypatch + ): + config = _config(tmp_path, ["alpha", "beta"]) + await _seed(config, "alpha", ["alpha document about cats"]) + await _seed(config, "beta", ["beta document about cats"]) + monkeypatch.setattr(HaikuRAG, "reranker", property(lambda self: StubReranker())) + asked = self._limits_asked(monkeypatch) + + async with HaikuRAG(config=config) as rag: + await rag.search("cats", limit=3, search_type="fts") + per_database = list(asked) + asked.clear() + await rag.search("cats", limit=3, search_type="fts", sources=["alpha"]) + + assert per_database == [30, 30] + assert asked == [30] + + @pytest.mark.asyncio + async def test_a_text_query_without_a_reranker_fetches_what_it_returns( + self, tmp_path, monkeypatch + ): + config = _config(tmp_path, ["alpha", "beta"]) + await _seed(config, "alpha", ["alpha document about cats"]) + await _seed(config, "beta", ["beta document about cats"]) + monkeypatch.setattr(HaikuRAG, "reranker", property(lambda self: None)) + asked = self._limits_asked(monkeypatch) + + async with HaikuRAG(config=config) as rag: + await rag.search("cats", limit=3, search_type="fts") + per_database = list(asked) + asked.clear() + await rag.search("cats", limit=3, search_type="fts", sources=["alpha"]) + + assert per_database == [3, 3] + assert asked == [3] + + @pytest.mark.asyncio + async def test_an_image_query_fetches_what_it_returns(self, tmp_path, monkeypatch): + config = _config(tmp_path, ["alpha", "beta"]) + await _seed(config, "alpha", ["alpha document about cats"]) + await _seed(config, "beta", ["beta document about cats"]) + monkeypatch.setattr(HaikuRAG, "reranker", property(lambda self: StubReranker())) + + dim = get_config().embeddings.model.vector_dim + + async def embed_image(self, image): # noqa: ARG001 + return [0.1] * dim + + monkeypatch.setattr(EmbedderWrapper, "supports_images", True) + monkeypatch.setattr(EmbedderWrapper, "embed_image", embed_image) + asked = self._limits_asked(monkeypatch) + + async with HaikuRAG(config=config) as rag: + await rag.search(b"\x89PNG\r\n\x1a\n", limit=3) + + assert asked == [3, 3] + + class TestOneReranker: @pytest.mark.asyncio async def test_the_set_builds_one_reranker_for_a_text_query( diff --git a/tests/sandbox/test_sandbox_multi_db.py b/tests/sandbox/test_sandbox_multi_db.py index 86655ca7..598b3786 100644 --- a/tests/sandbox/test_sandbox_multi_db.py +++ b/tests/sandbox/test_sandbox_multi_db.py @@ -5,6 +5,7 @@ import pytest from haiku.rag.client import HaikuRAG from haiku.rag.client.scope import DatabaseRef, DatabaseScope from haiku.rag.sandbox import AnalysisContext, Sandbox +from haiku.rag.store.exceptions import UnknownDatabaseError from tests.multi_db.helpers import _config, _seed @@ -300,7 +301,7 @@ class TestSelectionOnOneDatabase: await _seed(config, "alpha", ["alpha document about cats"]) async with HaikuRAG(config=config) as rag: - with pytest.raises(KeyError, match="beta"): + with pytest.raises(UnknownDatabaseError, match="beta"): await _mounted(rag, sources=["beta"]) @pytest.mark.asyncio