diff --git a/backend/app/services/search_service.py b/backend/app/services/search_service.py index 8783570..035b481 100644 --- a/backend/app/services/search_service.py +++ b/backend/app/services/search_service.py @@ -225,12 +225,29 @@ def article_ids_with_sections(db: Session, query_text: str, ranked, _ = hybrid_ids(db, query_text, "article", limit=limit) section_ranked, _ = hybrid_ids(db, query_text, "article_section", limit=limit) + + rows = {row.id: row for row in db.query(ArticleSectionIndex).filter( + ArticleSectionIndex.id.in_(section_ranked)).all()} if section_ranked else {} by_article: dict[int, list] = {} - if section_ranked: - position = {row_id: index for index, row_id in enumerate(section_ranked)} - rows = db.query(ArticleSectionIndex).filter( - ArticleSectionIndex.id.in_(section_ranked)).all() - for row in sorted(rows, key=lambda r: position.get(r.id, len(position))): - by_article.setdefault(row.article_id, []).append(row) - ordered = list(dict.fromkeys([*ranked, *by_article.keys()]))[:limit] + best: dict[int, int] = {} + for index, row_id in enumerate(section_ranked): + row = rows.get(row_id) + if row is None: + continue + by_article.setdefault(row.article_id, []).append(row) + # Only an article's best-placed section scores. Counting them all would + # rank a fourteen-section article above a better two-section one on + # length alone. + best.setdefault(row.article_id, index) + + # Fused, not concatenated. Appending the section hits behind the article + # hits put the strongest evidence there is — a section that matched at rank + # one — behind every weak whole-article match, so it never reached the page. + scores: dict[int, float] = {} + for index, article_id in enumerate(ranked): + scores[article_id] = scores.get(article_id, 0.0) + 1.0 / (RRF_K + index + 1) + for article_id, index in best.items(): + scores[article_id] = scores.get(article_id, 0.0) + 1.0 / (RRF_K + index + 1) + + ordered = sorted(scores, key=lambda article_id: (-scores[article_id], article_id))[:limit] return ordered, by_article diff --git a/backend/tests/test_article_search_index.py b/backend/tests/test_article_search_index.py index 8c5b8e8..d17767e 100644 --- a/backend/tests/test_article_search_index.py +++ b/backend/tests/test_article_search_index.py @@ -109,6 +109,57 @@ class BatchEmbeddingTests(unittest.TestCase): self.assertEqual([out[0][0], out[3][0]], [0.0, 1.0]) +class RollupFusionTests(unittest.TestCase): + """A section that matched at rank one is the strongest evidence there is.""" + + def setUp(self): + self.bank = fixtures.BuilderTests() + self.bank.setUp() + self.db = self.bank.db + self.db.add_all([ + Article(id=91, slug='a', title='Weak whole-article match', sections=[], status='published'), + Article(id=92, slug='b', title='Another weak one', sections=[], status='published'), + Article(id=93, slug='c', title='Body match only', sections=[], status='published'), + ]) + self.db.add(ArticleSectionIndex(id=1, article_id=93, section_id='c' * 32, + title='Treatment', content='Griseofulvin')) + self.db.flush() + + def tearDown(self): + self.bank.tearDown() + + def test_a_section_only_hit_is_not_parked_behind_every_article_hit(self): + from app.services import search_service + + def fake(db, query, kind='question', limit=200): + return ([91, 92], set()) if kind == 'article' else ([1], set()) + + with patch.object(search_service, 'hybrid_ids', side_effect=fake): + ranked, by_article = search_service.article_ids_with_sections(self.db, 'griseofulvin') + + # Concatenation put 93 last however well its section scored. + self.assertLess(ranked.index(93), ranked.index(92)) + self.assertEqual([row.title for row in by_article[93]], ['Treatment']) + + def test_one_article_cannot_win_on_number_of_matching_sections(self): + from app.services import search_service + + self.db.add_all([ + ArticleSectionIndex(id=2, article_id=91, section_id='d' * 32, title='B', content='x'), + ArticleSectionIndex(id=3, article_id=91, section_id='e' * 32, title='C', content='x'), + ArticleSectionIndex(id=4, article_id=91, section_id='f' * 32, title='D', content='x'), + ]) + self.db.flush() + + def fake(db, query, kind='question', limit=200): + # 93's single section leads; 91 has three, all behind it. + return ([], set()) if kind == 'article' else ([1, 2, 3, 4], set()) + + with patch.object(search_service, 'hybrid_ids', side_effect=fake): + ranked, _ = search_service.article_ids_with_sections(self.db, 'griseofulvin') + self.assertEqual(ranked[0], 93, "a longer article outranked a better one on volume") + + class SectionIndexInStepTests(unittest.TestCase): def setUp(self): self.bank = fixtures.BuilderTests()