diff --git a/CHANGELOG.md b/CHANGELOG.md index 1309ffc7..0233636a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ - `docs/configuration/storage.md` "Vector Indexing" carries the measured with/without IVF_PQ retrieval comparison; `docs/benchmarks.md` states the published numbers are measured without a vector index. - Re-indexing note corrected: `optimize()` adds new chunks to an existing vector index, a rebuild retrains centroids. +### Fixed + +- Cross-database fusion without a reranker breaks rank ties by retrieval + score instead of database declaration order. + ## [0.80.0] - 2026-08-31 ### Changed diff --git a/haiku_rag_slim/haiku/rag/client/search.py b/haiku_rag_slim/haiku/rag/client/search.py index 9c3441a6..b241fda8 100644 --- a/haiku_rag_slim/haiku/rag/client/search.py +++ b/haiku_rag_slim/haiku/rag/client/search.py @@ -156,6 +156,14 @@ async def _fuse( across databases tractable: it compares query against document and does not care where a candidate came from. Without one, reciprocal rank fusion over the per-database rankings, since scores from separate indexes are not comparable. + The corpora are disjoint, so every database's rank-r candidate carries the + same RRF score; those ties break on the raw retrieval score, which is not + calibrated across indexes but only ever orders candidates within one rank + and never lifts a candidate above another rank. A hybrid score is itself + rank-derived (each database fuses its own vector and FTS rankings with + lancedb's RRFReranker), so there the tiebreak compares rank agreement, not + relevance magnitude, and databases agreeing exactly still tie, resolving + deterministically to configured order. """ owned = [ (client, chunk, score) @@ -191,12 +199,12 @@ async def _fuse( ) return [(owner_of[id(chunk)], chunk, score) for chunk, score in reranked] - scored: list[tuple[float, HaikuRAG, Chunk]] = [] + scored: list[tuple[float, float, HaikuRAG, Chunk]] = [] for client, candidates in zip(clients, per_source, strict=True): - for rank, (chunk, _) in enumerate(candidates): - scored.append((1.0 / (_RRF_K + rank + 1), client, chunk)) - scored.sort(key=lambda item: item[0], reverse=True) - return [(client, chunk, score) for score, client, chunk in scored[:limit]] + for rank, (chunk, score) in enumerate(candidates): + scored.append((1.0 / (_RRF_K + rank + 1), score, client, chunk)) + scored.sort(key=lambda item: (item[0], item[1]), reverse=True) + return [(client, chunk, fused) for fused, _, client, chunk in scored[:limit]] # Reciprocal rank fusion's smoothing constant, the value the literature uses. diff --git a/tests/multi_db/test_search.py b/tests/multi_db/test_search.py index 3332d9b3..de588c7e 100644 --- a/tests/multi_db/test_search.py +++ b/tests/multi_db/test_search.py @@ -527,12 +527,47 @@ class TestReciprocalRankFusion: ] @pytest.mark.asyncio - async def test_equal_scores_keep_the_configured_order(self, tmp_path): - """Every rank ties across databases, so the tiebreak decides all of it.""" - fused = await self._fuse_over(tmp_path, self._lopsided(2), 10) + async def test_ranks_tie_and_the_retrieval_score_breaks_them(self, tmp_path): + """Every rank ties across databases, so the tiebreak decides all of it, + and it must be relevance, not the order databases were configured in.""" + per_source = [self._ranked("a", 2, 0.2), self._ranked("b", 2, 0.9)] + fused = await self._fuse_over(tmp_path, per_source, 10) + + assert [source for source, _, _ in fused] == ["beta", "alpha", "beta", "alpha"] + + @pytest.mark.asyncio + async def test_the_configured_order_does_not_matter(self, tmp_path): + """The same candidates fuse to the same list whichever database is + declared first.""" + forward = await self._fuse_over(tmp_path, self._lopsided(3), 10) + (tmp_path / "swapped").mkdir() + backward = await self._fuse_over( + tmp_path / "swapped", + [self._ranked("b", 3, 0.2), self._ranked("a", 3, 0.9)], + 10, + ) + + assert [(cid, score) for _, cid, score in forward] == [ + (cid, score) for _, cid, score in backward + ] + + @pytest.mark.asyncio + async def test_exact_ties_keep_the_configured_order(self, tmp_path): + """Hybrid scores are rank-derived and tie exactly when databases agree, + so a genuine tie must still resolve deterministically.""" + per_source = [self._ranked("a", 2, 0.9), self._ranked("b", 2, 0.9)] + fused = await self._fuse_over(tmp_path, per_source, 10) assert [source for source, _, _ in fused] == ["alpha", "beta", "alpha", "beta"] + @pytest.mark.asyncio + async def test_the_retrieval_score_never_overrides_the_rank(self, tmp_path): + """Raw scores are not calibrated across indexes, so a database with + inflated scores still contributes one candidate per rank.""" + fused = await self._fuse_over(tmp_path, self._lopsided(2), 10) + + assert [cid for _, cid, _ in fused] == ["a0", "b0", "a1", "b1"] + @pytest.mark.asyncio async def test_the_limit_cuts_the_fused_list(self, tmp_path): """Each database was asked for enough to fill the window on its own."""