From d06277408b7be5ffea47bb79fc866a7fb875f1da Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Mon, 31 Aug 2026 13:15:05 +0300 Subject: [PATCH] Break cross-database RRF rank ties by retrieval score Disjoint corpora give every database's rank-r candidate the same RRF score, and the stable sort resolved those ties to lancedb.databases declaration order, discarding the retrieval scores entirely. Ties now break on the raw retrieval score, which is uncalibrated across indexes but only ever orders candidates within one rank tier: the databases in a fusion share an embedder and ran the same search type, and it can never lift a candidate above another rank. Hybrid per-database scores are themselves rank-derived, so exact agreement still ties and keeps configured order, deterministically. The n > limit depth quota is unchanged, pending the retrieval eval. --- CHANGELOG.md | 5 +++ haiku_rag_slim/haiku/rag/client/search.py | 18 +++++++--- tests/multi_db/test_search.py | 41 +++++++++++++++++++++-- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1309ffc7..0233636a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ - `docs/configuration/storage.md` "Vector Indexing" carries the measured with/without IVF_PQ retrieval comparison; `docs/benchmarks.md` states the published numbers are measured without a vector index. - Re-indexing note corrected: `optimize()` adds new chunks to an existing vector index, a rebuild retrains centroids. +### Fixed + +- Cross-database fusion without a reranker breaks rank ties by retrieval + score instead of database declaration order. + ## [0.80.0] - 2026-08-31 ### Changed diff --git a/haiku_rag_slim/haiku/rag/client/search.py b/haiku_rag_slim/haiku/rag/client/search.py index 9c3441a6..b241fda8 100644 --- a/haiku_rag_slim/haiku/rag/client/search.py +++ b/haiku_rag_slim/haiku/rag/client/search.py @@ -156,6 +156,14 @@ async def _fuse( across databases tractable: it compares query against document and does not care where a candidate came from. Without one, reciprocal rank fusion over the per-database rankings, since scores from separate indexes are not comparable. + The corpora are disjoint, so every database's rank-r candidate carries the + same RRF score; those ties break on the raw retrieval score, which is not + calibrated across indexes but only ever orders candidates within one rank + and never lifts a candidate above another rank. A hybrid score is itself + rank-derived (each database fuses its own vector and FTS rankings with + lancedb's RRFReranker), so there the tiebreak compares rank agreement, not + relevance magnitude, and databases agreeing exactly still tie, resolving + deterministically to configured order. """ owned = [ (client, chunk, score) @@ -191,12 +199,12 @@ async def _fuse( ) return [(owner_of[id(chunk)], chunk, score) for chunk, score in reranked] - scored: list[tuple[float, HaikuRAG, Chunk]] = [] + scored: list[tuple[float, float, HaikuRAG, Chunk]] = [] for client, candidates in zip(clients, per_source, strict=True): - for rank, (chunk, _) in enumerate(candidates): - scored.append((1.0 / (_RRF_K + rank + 1), client, chunk)) - scored.sort(key=lambda item: item[0], reverse=True) - return [(client, chunk, score) for score, client, chunk in scored[:limit]] + for rank, (chunk, score) in enumerate(candidates): + scored.append((1.0 / (_RRF_K + rank + 1), score, client, chunk)) + scored.sort(key=lambda item: (item[0], item[1]), reverse=True) + return [(client, chunk, fused) for fused, _, client, chunk in scored[:limit]] # Reciprocal rank fusion's smoothing constant, the value the literature uses. diff --git a/tests/multi_db/test_search.py b/tests/multi_db/test_search.py index 3332d9b3..de588c7e 100644 --- a/tests/multi_db/test_search.py +++ b/tests/multi_db/test_search.py @@ -527,12 +527,47 @@ class TestReciprocalRankFusion: ] @pytest.mark.asyncio - async def test_equal_scores_keep_the_configured_order(self, tmp_path): - """Every rank ties across databases, so the tiebreak decides all of it.""" - fused = await self._fuse_over(tmp_path, self._lopsided(2), 10) + async def test_ranks_tie_and_the_retrieval_score_breaks_them(self, tmp_path): + """Every rank ties across databases, so the tiebreak decides all of it, + and it must be relevance, not the order databases were configured in.""" + per_source = [self._ranked("a", 2, 0.2), self._ranked("b", 2, 0.9)] + fused = await self._fuse_over(tmp_path, per_source, 10) + + assert [source for source, _, _ in fused] == ["beta", "alpha", "beta", "alpha"] + + @pytest.mark.asyncio + async def test_the_configured_order_does_not_matter(self, tmp_path): + """The same candidates fuse to the same list whichever database is + declared first.""" + forward = await self._fuse_over(tmp_path, self._lopsided(3), 10) + (tmp_path / "swapped").mkdir() + backward = await self._fuse_over( + tmp_path / "swapped", + [self._ranked("b", 3, 0.2), self._ranked("a", 3, 0.9)], + 10, + ) + + assert [(cid, score) for _, cid, score in forward] == [ + (cid, score) for _, cid, score in backward + ] + + @pytest.mark.asyncio + async def test_exact_ties_keep_the_configured_order(self, tmp_path): + """Hybrid scores are rank-derived and tie exactly when databases agree, + so a genuine tie must still resolve deterministically.""" + per_source = [self._ranked("a", 2, 0.9), self._ranked("b", 2, 0.9)] + fused = await self._fuse_over(tmp_path, per_source, 10) assert [source for source, _, _ in fused] == ["alpha", "beta", "alpha", "beta"] + @pytest.mark.asyncio + async def test_the_retrieval_score_never_overrides_the_rank(self, tmp_path): + """Raw scores are not calibrated across indexes, so a database with + inflated scores still contributes one candidate per rank.""" + fused = await self._fuse_over(tmp_path, self._lopsided(2), 10) + + assert [cid for _, cid, _ in fused] == ["a0", "b0", "a1", "b1"] + @pytest.mark.asyncio async def test_the_limit_cuts_the_fused_list(self, tmp_path): """Each database was asked for enough to fill the window on its own."""