Break cross-database RRF rank ties by retrieval score
Disjoint corpora give every database's rank-r candidate the same RRF score, and the stable sort resolved those ties to lancedb.databases declaration order, discarding the retrieval scores entirely. Ties now break on the raw retrieval score, which is uncalibrated across indexes but only ever orders candidates within one rank tier: the databases in a fusion share an embedder and ran the same search type, and it can never lift a candidate above another rank. Hybrid per-database scores are themselves rank-derived, so exact agreement still ties and keeps configured order, deterministically. The n > limit depth quota is unchanged, pending the retrieval eval.
This commit is contained in:
parent
33bd0be702
commit
16a1d9598b
3 changed files with 53 additions and 9 deletions
|
|
@ -31,7 +31,8 @@
|
|||
vacuum` also repairs it.
|
||||
- Migration to 0.38.0 no longer fails with `UnicodeDecodeError` on a
|
||||
`docling_document` blob written as zstd.
|
||||
- Vector search sets `search.vector_index_metric` on every query.
|
||||
- Cross-database fusion without a reranker breaks rank ties by retrieval
|
||||
score instead of database declaration order.
|
||||
|
||||
## [0.79.0] - 2026-08-28
|
||||
|
||||
|
|
|
|||
|
|
@ -156,6 +156,14 @@ async def _fuse(
|
|||
across databases tractable: it compares query against document and does not
|
||||
care where a candidate came from. Without one, reciprocal rank fusion over the
|
||||
per-database rankings, since scores from separate indexes are not comparable.
|
||||
The corpora are disjoint, so every database's rank-r candidate carries the
|
||||
same RRF score; those ties break on the raw retrieval score, which is not
|
||||
calibrated across indexes but only ever orders candidates within one rank
|
||||
and never lifts a candidate above another rank. A hybrid score is itself
|
||||
rank-derived (each database fuses its own vector and FTS rankings with
|
||||
lancedb's RRFReranker), so there the tiebreak compares rank agreement, not
|
||||
relevance magnitude, and databases agreeing exactly still tie, resolving
|
||||
deterministically to configured order.
|
||||
"""
|
||||
owned = [
|
||||
(client, chunk, score)
|
||||
|
|
@ -191,12 +199,12 @@ async def _fuse(
|
|||
)
|
||||
return [(owner_of[id(chunk)], chunk, score) for chunk, score in reranked]
|
||||
|
||||
scored: list[tuple[float, HaikuRAG, Chunk]] = []
|
||||
scored: list[tuple[float, float, HaikuRAG, Chunk]] = []
|
||||
for client, candidates in zip(clients, per_source, strict=True):
|
||||
for rank, (chunk, _) in enumerate(candidates):
|
||||
scored.append((1.0 / (_RRF_K + rank + 1), client, chunk))
|
||||
scored.sort(key=lambda item: item[0], reverse=True)
|
||||
return [(client, chunk, score) for score, client, chunk in scored[:limit]]
|
||||
for rank, (chunk, score) in enumerate(candidates):
|
||||
scored.append((1.0 / (_RRF_K + rank + 1), score, client, chunk))
|
||||
scored.sort(key=lambda item: (item[0], item[1]), reverse=True)
|
||||
return [(client, chunk, fused) for fused, _, client, chunk in scored[:limit]]
|
||||
|
||||
|
||||
# Reciprocal rank fusion's smoothing constant, the value the literature uses.
|
||||
|
|
|
|||
|
|
@ -527,12 +527,47 @@ class TestReciprocalRankFusion:
|
|||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_equal_scores_keep_the_configured_order(self, tmp_path):
|
||||
"""Every rank ties across databases, so the tiebreak decides all of it."""
|
||||
fused = await self._fuse_over(tmp_path, self._lopsided(2), 10)
|
||||
async def test_ranks_tie_and_the_retrieval_score_breaks_them(self, tmp_path):
|
||||
"""Every rank ties across databases, so the tiebreak decides all of it,
|
||||
and it must be relevance, not the order databases were configured in."""
|
||||
per_source = [self._ranked("a", 2, 0.2), self._ranked("b", 2, 0.9)]
|
||||
fused = await self._fuse_over(tmp_path, per_source, 10)
|
||||
|
||||
assert [source for source, _, _ in fused] == ["beta", "alpha", "beta", "alpha"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_configured_order_does_not_matter(self, tmp_path):
|
||||
"""The same candidates fuse to the same list whichever database is
|
||||
declared first."""
|
||||
forward = await self._fuse_over(tmp_path, self._lopsided(3), 10)
|
||||
(tmp_path / "swapped").mkdir()
|
||||
backward = await self._fuse_over(
|
||||
tmp_path / "swapped",
|
||||
[self._ranked("b", 3, 0.2), self._ranked("a", 3, 0.9)],
|
||||
10,
|
||||
)
|
||||
|
||||
assert [(cid, score) for _, cid, score in forward] == [
|
||||
(cid, score) for _, cid, score in backward
|
||||
]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exact_ties_keep_the_configured_order(self, tmp_path):
|
||||
"""Hybrid scores are rank-derived and tie exactly when databases agree,
|
||||
so a genuine tie must still resolve deterministically."""
|
||||
per_source = [self._ranked("a", 2, 0.9), self._ranked("b", 2, 0.9)]
|
||||
fused = await self._fuse_over(tmp_path, per_source, 10)
|
||||
|
||||
assert [source for source, _, _ in fused] == ["alpha", "beta", "alpha", "beta"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_retrieval_score_never_overrides_the_rank(self, tmp_path):
|
||||
"""Raw scores are not calibrated across indexes, so a database with
|
||||
inflated scores still contributes one candidate per rank."""
|
||||
fused = await self._fuse_over(tmp_path, self._lopsided(2), 10)
|
||||
|
||||
assert [cid for _, cid, _ in fused] == ["a0", "b0", "a1", "b1"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_limit_cuts_the_fused_list(self, tmp_path):
|
||||
"""Each database was asked for enough to fill the window on its own."""
|
||||
|
|
|
|||
Loading…
Reference in a new issue