diff --git a/CHANGELOG.md b/CHANGELOG.md index 99b8987e..a428fb4e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,8 +31,9 @@ vacuum` also repairs it. - Migration to 0.38.0 no longer fails with `UnicodeDecodeError` on a `docling_document` blob written as zstd. -- Cross-database fusion without a reranker breaks rank ties by retrieval - score instead of database declaration order. +- Cross-database fusion without a reranker orders the union by retrieval + score, with within-database rank breaking ties, instead of round-robin by + database declaration order. Fused results carry the retrieval score. ## [0.79.0] - 2026-08-28 diff --git a/docs/configuration/storage.md b/docs/configuration/storage.md index a94b2b3c..10400dc8 100644 --- a/docs/configuration/storage.md +++ b/docs/configuration/storage.md @@ -227,7 +227,7 @@ results = await client.search("query") # every database results = await client.search("query", sources=["papers"]) # one of them ``` -Candidates are combined into one ranked list with the configured reranker, or with reciprocal rank fusion when reranking is disabled. `SearchResult.source`, `Citation.source`, and `Document.source` contain the configured database name. The name is retained when a client covers only one named database. Databases configured through `lancedb.uri` are unnamed, so their `source` is `None`. +Candidates are combined into one ranked list with the configured reranker, or by retrieval score when reranking is disabled, with within-database rank breaking score ties. `SearchResult.source`, `Citation.source`, and `Document.source` contain the configured database name. The name is retained when a client covers only one named database. Databases configured through `lancedb.uri` are unnamed, so their `source` is `None`. The CLI labels results and citations only when the operation spans multiple databases. A command already narrowed with `--db-name` does not repeat the name on every result. diff --git a/haiku_rag_slim/haiku/rag/client/search.py b/haiku_rag_slim/haiku/rag/client/search.py index b241fda8..e1f8adcf 100644 --- a/haiku_rag_slim/haiku/rag/client/search.py +++ b/haiku_rag_slim/haiku/rag/client/search.py @@ -154,16 +154,16 @@ async def _fuse( A configured reranker scores the union directly, which is what makes ranking across databases tractable: it compares query against document and does not - care where a candidate came from. Without one, reciprocal rank fusion over the - per-database rankings, since scores from separate indexes are not comparable. - The corpora are disjoint, so every database's rank-r candidate carries the - same RRF score; those ties break on the raw retrieval score, which is not - calibrated across indexes but only ever orders candidates within one rank - and never lifts a candidate above another rank. A hybrid score is itself - rank-derived (each database fuses its own vector and FTS rankings with - lancedb's RRFReranker), so there the tiebreak compares rank agreement, not - relevance magnitude, and databases agreeing exactly still tie, resolving - deterministically to configured order. + care where a candidate came from. Without one, the union is ordered by the + raw retrieval score. Scores from separate indexes are not calibrated, but a + hybrid score is each database's own rank agreement (lancedb fuses vector and + FTS with RRF inside the database), which carries across databases; rank + interleaving instead guarantees every database slots regardless of content, + which on domain-split collections allocates no better than chance. Equal + scores resolve by within-database rank — the candidate nothing in its own + database beat wins — and only a tie on both falls to configured order. The + returned score is the candidate's own, so downstream re-sorts (context + expansion) preserve this order. """ owned = [ (client, chunk, score) @@ -203,8 +203,8 @@ async def _fuse( for client, candidates in zip(clients, per_source, strict=True): for rank, (chunk, score) in enumerate(candidates): scored.append((1.0 / (_RRF_K + rank + 1), score, client, chunk)) - scored.sort(key=lambda item: (item[0], item[1]), reverse=True) - return [(client, chunk, fused) for fused, _, client, chunk in scored[:limit]] + scored.sort(key=lambda item: (item[1], item[0]), reverse=True) + return [(client, chunk, score) for _, score, client, chunk in scored[:limit]] # Reciprocal rank fusion's smoothing constant, the value the literature uses. diff --git a/tests/multi_db/test_search.py b/tests/multi_db/test_search.py index de588c7e..b55329b6 100644 --- a/tests/multi_db/test_search.py +++ b/tests/multi_db/test_search.py @@ -473,9 +473,10 @@ class TestNarrowingToOneDatabase: assert results == [] -class TestReciprocalRankFusion: - """Without a reranker, scores from separate indexes are not comparable, so - fusion ranks by position. These pin what that produces.""" +class TestFusionWithoutAReranker: + """Without a reranker, the union is ordered by retrieval score; score ties + resolve by within-database rank, and only a tie on both falls to configured + order. These pin what that produces.""" @staticmethod def _ranked(source: str, count: int, top: float) -> list[tuple[Chunk, float]]: @@ -502,38 +503,46 @@ class TestReciprocalRankFusion: return [(owner.source, chunk.id, score) for owner, chunk, score in fused] @pytest.mark.asyncio - async def test_databases_interleave_by_rank(self, tmp_path): - """Each contributes its rank-1 before either contributes its rank-2.""" + async def test_the_score_orders_the_union(self, tmp_path): + """A stronger database takes consecutive slots; breadth is not + guaranteed.""" fused = await self._fuse_over(tmp_path, self._lopsided(3), 10) assert [(source, cid) for source, cid, _ in fused] == [ ("alpha", "a0"), - ("beta", "b0"), ("alpha", "a1"), - ("beta", "b1"), ("alpha", "a2"), + ("beta", "b0"), + ("beta", "b1"), ("beta", "b2"), ] @pytest.mark.asyncio - async def test_the_score_is_the_reciprocal_of_the_rank(self, tmp_path): + async def test_the_score_is_the_retrieval_score(self, tmp_path): + """The fused score is the candidate's own, so re-sorting downstream + (context expansion) preserves the fused order.""" fused = await self._fuse_over(tmp_path, self._lopsided(2), 10) - assert [score for _, _, score in fused] == [ - 1 / 61, - 1 / 61, - 1 / 62, - 1 / 62, - ] + assert [score for _, _, score in fused] == [0.9, 0.89, 0.2, 0.19] @pytest.mark.asyncio - async def test_ranks_tie_and_the_retrieval_score_breaks_them(self, tmp_path): - """Every rank ties across databases, so the tiebreak decides all of it, - and it must be relevance, not the order databases were configured in.""" - per_source = [self._ranked("a", 2, 0.2), self._ranked("b", 2, 0.9)] + async def test_score_ties_break_by_rank_within_the_database(self, tmp_path): + """Equal scores can sit at different ranks: rank depends on what the + rest of a database scored. The candidate nothing in its own database + beat wins the tie.""" + per_source = [ + [ + (Chunk(id="a0", content="a 0"), 0.9), + (Chunk(id="a1", content="a 1"), 0.5), + ], + [ + (Chunk(id="b0", content="b 0"), 0.5), + (Chunk(id="b1", content="b 1"), 0.3), + ], + ] fused = await self._fuse_over(tmp_path, per_source, 10) - assert [source for source, _, _ in fused] == ["beta", "alpha", "beta", "alpha"] + assert [cid for _, cid, _ in fused] == ["a0", "b0", "a1", "b1"] @pytest.mark.asyncio async def test_the_configured_order_does_not_matter(self, tmp_path): @@ -561,12 +570,12 @@ class TestReciprocalRankFusion: assert [source for source, _, _ in fused] == ["alpha", "beta", "alpha", "beta"] @pytest.mark.asyncio - async def test_the_retrieval_score_never_overrides_the_rank(self, tmp_path): - """Raw scores are not calibrated across indexes, so a database with - inflated scores still contributes one candidate per rank.""" + async def test_rank_never_overrides_the_score(self, tmp_path): + """A database's rank-2 with a higher score precedes another's rank-0: + allocation is content-driven, not round-robin.""" fused = await self._fuse_over(tmp_path, self._lopsided(2), 10) - assert [cid for _, cid, _ in fused] == ["a0", "b0", "a1", "b1"] + assert [cid for _, cid, _ in fused] == ["a0", "a1", "b0", "b1"] @pytest.mark.asyncio async def test_the_limit_cuts_the_fused_list(self, tmp_path): @@ -575,8 +584,8 @@ class TestReciprocalRankFusion: assert [(source, cid) for source, cid, _ in fused] == [ ("alpha", "a0"), - ("beta", "b0"), ("alpha", "a1"), + ("alpha", "a2"), ]