diff --git a/CHANGELOG.md b/CHANGELOG.md index 19aa9590..ac6aa5d8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ ## [Unreleased] +### Fixed + +- Cross-database fusion without a reranker breaks rank ties by retrieval + score instead of database declaration order. + +## [0.80.0] - 2026-08-31 + ### Changed - lancedb 0.37.1. @@ -13,9 +20,6 @@ text untruncated. `format_citations_rich` takes a `full` argument. - `doctor` fails when the chunks FTS index covers no rows. - FTS and hybrid searches log a warning when the FTS index covers no rows. -- `evaluations run --retrieval-limit N`: candidates each database fetches during the retrieval benchmark, overriding the dataset's `retrieval_limit`. -- `mtrag_pooled` evaluation dataset and its reference config `evaluations/configs/mtrag_pooled.yaml`: all four MTRAG domains pooled and partitioned across `n` collections, `--alpha` interpolating between one domain per collection and a uniform shard. -- `mtrag_federated` evaluation dataset and its reference config `evaluations/configs/mtrag_federated.yaml`: MTRAG ClapNQ partitioned by article title into `n` collections, scored on retrieval only with Recall@5/@10, nDCG@5 and MAP. `python -m evaluations.datasets.mtrag_federated --config REF --n N --out PATH` builds the partition and emits the config that searches it. ### Removed @@ -24,8 +28,6 @@ ### Fixed -- Batched evaluation ingest converts inline content as text instead of letting `HaikuRAG.convert` disambiguate it, so a passage beginning with a URL is stored rather than fetched over HTTP. 187 MTRAG cloud and fiqa passages start with one; no clapnq passage does, so no existing dataset's numbers change. -- `mtrag_federated` builds vacuum each collection after ingest and assert the chunks FTS index covers every row. Without the vacuum the index stays at zero rows, and full-text search returns near-arbitrary rows while still returning results. - FTS and hybrid search on a database whose FTS index covers no rows. Chunk writes now build the index and rebuild it if it covers none; `haiku-rag vacuum` also repairs it. @@ -2296,7 +2298,8 @@ Existing documents without DoclingDocument data will work but won't have provena - Initial version tracking -[Unreleased]: https://github.com/ggozad/haiku.rag/compare/0.79.0...HEAD +[Unreleased]: https://github.com/ggozad/haiku.rag/compare/0.80.0...HEAD +[0.80.0]: https://github.com/ggozad/haiku.rag/compare/0.79.0...0.80.0 [0.79.0]: https://github.com/ggozad/haiku.rag/compare/0.78.0...0.79.0 [0.78.0]: https://github.com/ggozad/haiku.rag/compare/0.77.0...0.78.0 [0.77.0]: https://github.com/ggozad/haiku.rag/compare/0.77.0...0.77.0 diff --git a/haiku_rag_slim/haiku/rag/client/search.py b/haiku_rag_slim/haiku/rag/client/search.py index 9c3441a6..b241fda8 100644 --- a/haiku_rag_slim/haiku/rag/client/search.py +++ b/haiku_rag_slim/haiku/rag/client/search.py @@ -156,6 +156,14 @@ async def _fuse( across databases tractable: it compares query against document and does not care where a candidate came from. Without one, reciprocal rank fusion over the per-database rankings, since scores from separate indexes are not comparable. + The corpora are disjoint, so every database's rank-r candidate carries the + same RRF score; those ties break on the raw retrieval score, which is not + calibrated across indexes but only ever orders candidates within one rank + and never lifts a candidate above another rank. A hybrid score is itself + rank-derived (each database fuses its own vector and FTS rankings with + lancedb's RRFReranker), so there the tiebreak compares rank agreement, not + relevance magnitude, and databases agreeing exactly still tie, resolving + deterministically to configured order. """ owned = [ (client, chunk, score) @@ -191,12 +199,12 @@ async def _fuse( ) return [(owner_of[id(chunk)], chunk, score) for chunk, score in reranked] - scored: list[tuple[float, HaikuRAG, Chunk]] = [] + scored: list[tuple[float, float, HaikuRAG, Chunk]] = [] for client, candidates in zip(clients, per_source, strict=True): - for rank, (chunk, _) in enumerate(candidates): - scored.append((1.0 / (_RRF_K + rank + 1), client, chunk)) - scored.sort(key=lambda item: item[0], reverse=True) - return [(client, chunk, score) for score, client, chunk in scored[:limit]] + for rank, (chunk, score) in enumerate(candidates): + scored.append((1.0 / (_RRF_K + rank + 1), score, client, chunk)) + scored.sort(key=lambda item: (item[0], item[1]), reverse=True) + return [(client, chunk, fused) for fused, _, client, chunk in scored[:limit]] # Reciprocal rank fusion's smoothing constant, the value the literature uses. diff --git a/tests/multi_db/test_search.py b/tests/multi_db/test_search.py index 3332d9b3..de588c7e 100644 --- a/tests/multi_db/test_search.py +++ b/tests/multi_db/test_search.py @@ -527,12 +527,47 @@ class TestReciprocalRankFusion: ] @pytest.mark.asyncio - async def test_equal_scores_keep_the_configured_order(self, tmp_path): - """Every rank ties across databases, so the tiebreak decides all of it.""" - fused = await self._fuse_over(tmp_path, self._lopsided(2), 10) + async def test_ranks_tie_and_the_retrieval_score_breaks_them(self, tmp_path): + """Every rank ties across databases, so the tiebreak decides all of it, + and it must be relevance, not the order databases were configured in.""" + per_source = [self._ranked("a", 2, 0.2), self._ranked("b", 2, 0.9)] + fused = await self._fuse_over(tmp_path, per_source, 10) + + assert [source for source, _, _ in fused] == ["beta", "alpha", "beta", "alpha"] + + @pytest.mark.asyncio + async def test_the_configured_order_does_not_matter(self, tmp_path): + """The same candidates fuse to the same list whichever database is + declared first.""" + forward = await self._fuse_over(tmp_path, self._lopsided(3), 10) + (tmp_path / "swapped").mkdir() + backward = await self._fuse_over( + tmp_path / "swapped", + [self._ranked("b", 3, 0.2), self._ranked("a", 3, 0.9)], + 10, + ) + + assert [(cid, score) for _, cid, score in forward] == [ + (cid, score) for _, cid, score in backward + ] + + @pytest.mark.asyncio + async def test_exact_ties_keep_the_configured_order(self, tmp_path): + """Hybrid scores are rank-derived and tie exactly when databases agree, + so a genuine tie must still resolve deterministically.""" + per_source = [self._ranked("a", 2, 0.9), self._ranked("b", 2, 0.9)] + fused = await self._fuse_over(tmp_path, per_source, 10) assert [source for source, _, _ in fused] == ["alpha", "beta", "alpha", "beta"] + @pytest.mark.asyncio + async def test_the_retrieval_score_never_overrides_the_rank(self, tmp_path): + """Raw scores are not calibrated across indexes, so a database with + inflated scores still contributes one candidate per rank.""" + fused = await self._fuse_over(tmp_path, self._lopsided(2), 10) + + assert [cid for _, cid, _ in fused] == ["a0", "b0", "a1", "b1"] + @pytest.mark.asyncio async def test_the_limit_cuts_the_fused_list(self, tmp_path): """Each database was asked for enough to fill the window on its own."""