031de53034
Retrieval fused a bi-encoder and BM25 by reciprocal rank. A bi-encoder embeds a document long before the question exists, so the two never meet: it is good at "same topic" and mediocre at "answers this". A cross-encoder reads the pair. The proxy already serves three — `cohere-rerank-v4.0-pro` is the default and measurably better than the fast variant. Query text goes exactly where the embeddings already go, and nothing new was signed up for. It found a defect nobody was looking for. In AI Mode each finder scored `1/(1+rank)` *within its own corpus*, so the best article, section, question and card all scored 1.0 and the shortlist was a meaningless round-robin. A cross-encoder is the first thing in this system that can compare a question with a section. Candidates per kind widened so it can select rather than merely reorder. Measured against labels neither ranker produced. Questions, 60 disease tags: precision@3 0.394 → 0.483. Sections, 60 article titles: 0.772 → 0.833. "Management of bronchiolitis" led with influenza transmission and a pregnancy question; "when do you image a first febrile seizure" returned the definition rather than the sentence saying imaging is unnecessary. And the honest negative, in docs/reranking.md: board vignettes are written *not* to name their diagnosis, so on "what causes croup" it prefers a question that says the word in passing over the barking-cough vignette that never says it. Some of the bi-encoder's strength is traded away. Not on the typeahead. A page of results is a choice being made and worth a third of a second; a typeahead is a word being finished, runs on every keystroke, and has nothing to judge yet. The three-state thresholds stay on cosine, argued at the constant: a reranker only ever sees a shortlist and structurally cannot answer the corpus-wide question those numbers ask, and whether an answer claims to come from the library is a promise that must not depend on a network hop. Every failure returns None and leaves the order alone — unconfigured, no proxy, connect error, bare 502, timeout, non-JSON, a duplicate or out-of-range index, a non-numeric score, a list the wrong length. Verified against the running site with a bogus model name: same results, fused order, no error to the reader. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN
226 lines
10 KiB
Python
226 lines
10 KiB
Python
"""What the cross-encoder is allowed to change, and what happens when it cannot.
|
|
|
|
No network: the point of these is not whether a reranker ranks well — that is
|
|
measured against the real corpus, in docs/reranking.md — but that every way it
|
|
can fail leaves the same rows on the page, in some order, with no error.
|
|
|
|
Run: DATABASE_URL=sqlite:// PYTHONPATH=backend python -m unittest discover -s backend/tests
|
|
"""
|
|
import os
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
os.environ.setdefault("DATABASE_URL", "sqlite://")
|
|
|
|
import httpx
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import Session
|
|
from sqlalchemy.pool import StaticPool
|
|
|
|
from app.database import Base
|
|
from app.models.article import Article, ArticleSectionIndex # noqa — mapper registration.
|
|
from app.models.course import Course # noqa — Quiz.course_id FK needs the table in metadata.
|
|
from app.models.media import MediaAsset # noqa — media is an embeddable corpus.
|
|
from app.models.question import Question
|
|
from app.models.user import User
|
|
from app.services import rerank_service, search_service
|
|
|
|
|
|
def response(payload, status=200):
|
|
"""An httpx reply in the shape the proxy sends one."""
|
|
return httpx.Response(status, json=payload, request=httpx.Request("POST", "http://proxy/v1/rerank"))
|
|
|
|
|
|
def scored(*pairs):
|
|
"""A rerank body: (index, score) pairs, deliberately not in index order."""
|
|
return {"id": "test", "results": [{"index": i, "relevance_score": s} for i, s in pairs]}
|
|
|
|
|
|
class ConfigurationTests(unittest.TestCase):
|
|
def test_no_model_means_no_call_and_no_opinion(self):
|
|
with patch.object(rerank_service, "rerank_model", return_value=""), \
|
|
patch("httpx.post", side_effect=AssertionError("must not reach the proxy")):
|
|
self.assertIsNone(rerank_service.rerank("croup", ["a", "b"]))
|
|
|
|
def test_a_model_without_a_proxy_is_not_configured(self):
|
|
with patch.object(rerank_service, "rerank_model", return_value="a-reranker"), \
|
|
patch.object(rerank_service.settings, "LITELLM_API_BASE", ""):
|
|
self.assertFalse(rerank_service.is_configured())
|
|
|
|
|
|
class DegradationTests(unittest.TestCase):
|
|
"""Every one of these must return None, which every caller reads as "keep it"."""
|
|
|
|
def setUp(self):
|
|
self.configured = patch.multiple(
|
|
rerank_service.settings, LITELLM_API_BASE="http://proxy",
|
|
LITELLM_API_KEY="k", LITELLM_RERANK_MODEL="a-reranker")
|
|
self.configured.start()
|
|
self.addCleanup(self.configured.stop)
|
|
# Redis is absent in tests; the cache must be optional, not required.
|
|
self.no_cache = patch.object(rerank_service, "_redis", return_value=None)
|
|
self.no_cache.start()
|
|
self.addCleanup(self.no_cache.stop)
|
|
|
|
def none_for(self, **post):
|
|
with patch("httpx.post", **post):
|
|
return rerank_service.rerank("croup", ["a", "b", "c"])
|
|
|
|
def test_unreachable_proxy(self):
|
|
self.assertIsNone(self.none_for(side_effect=httpx.ConnectError("refused")))
|
|
|
|
def test_a_bare_502_while_the_proxy_restarts(self):
|
|
self.assertIsNone(self.none_for(return_value=response({"error": "bad gateway"}, status=502)))
|
|
|
|
def test_a_timeout(self):
|
|
self.assertIsNone(self.none_for(side_effect=httpx.ReadTimeout("slow")))
|
|
|
|
def test_a_body_that_is_not_json(self):
|
|
self.assertIsNone(self.none_for(return_value=httpx.Response(
|
|
200, text="<html>gateway</html>",
|
|
request=httpx.Request("POST", "http://proxy/v1/rerank"))))
|
|
|
|
def test_a_reply_missing_a_document(self):
|
|
# Two scores for three documents: the third would have to be guessed,
|
|
# and a guess here is a reordering nobody asked for.
|
|
self.assertIsNone(self.none_for(return_value=response(scored((0, 0.1), (1, 0.9)))))
|
|
|
|
def test_a_reply_with_an_index_out_of_range(self):
|
|
self.assertIsNone(self.none_for(return_value=response(
|
|
scored((0, 0.1), (1, 0.9), (7, 0.5)))))
|
|
|
|
def test_a_reply_with_a_duplicated_index(self):
|
|
self.assertIsNone(self.none_for(return_value=response(
|
|
scored((0, 0.1), (0, 0.9), (2, 0.5)))))
|
|
|
|
def test_a_score_that_is_not_a_number(self):
|
|
self.assertIsNone(self.none_for(return_value=response(
|
|
{"results": [{"index": 0, "relevance_score": "high"},
|
|
{"index": 1, "relevance_score": 0.2},
|
|
{"index": 2, "relevance_score": 0.3}]})))
|
|
|
|
def test_scores_are_placed_by_the_index_the_server_echoed(self):
|
|
# The proxy answers best-first, so reading positionally would give the
|
|
# first document the best document's score.
|
|
with patch("httpx.post", return_value=response(scored((2, 0.9), (0, 0.1), (1, 0.4)))):
|
|
self.assertEqual(rerank_service.rerank("croup", ["a", "b", "c"]), [0.1, 0.4, 0.9])
|
|
|
|
|
|
class ReorderTests(unittest.TestCase):
|
|
def test_the_result_is_always_a_permutation(self):
|
|
items = list(range(8))
|
|
with patch.object(rerank_service, "rerank", return_value=[0.5] * 8):
|
|
self.assertCountEqual(rerank_service.reorder("q", items, str), items)
|
|
|
|
def test_no_opinion_leaves_the_order_alone(self):
|
|
items = [3, 1, 2]
|
|
with patch.object(rerank_service, "rerank", return_value=None):
|
|
self.assertEqual(rerank_service.reorder("q", items, str), items)
|
|
|
|
def test_ties_keep_the_order_fusion_gave_them(self):
|
|
with patch.object(rerank_service, "rerank", return_value=[0.4, 0.4, 0.9]):
|
|
self.assertEqual(rerank_service.reorder("q", [10, 20, 30], str), [30, 10, 20])
|
|
|
|
def test_only_the_head_is_scored_and_the_tail_is_kept(self):
|
|
items = list(range(rerank_service.MAX_CANDIDATES + 5))
|
|
seen = {}
|
|
|
|
def fake(query, documents):
|
|
seen["count"] = len(documents)
|
|
return list(range(len(documents))) # exactly reverses the head
|
|
|
|
with patch.object(rerank_service, "rerank", side_effect=fake):
|
|
out = rerank_service.reorder("q", items, str)
|
|
self.assertEqual(seen["count"], rerank_service.MAX_CANDIDATES)
|
|
self.assertEqual(out[0], rerank_service.MAX_CANDIDATES - 1)
|
|
self.assertEqual(out[-5:], items[-5:])
|
|
self.assertCountEqual(out, items)
|
|
|
|
|
|
class CorpusReorderTests(unittest.TestCase):
|
|
"""`rerank_ids` against a real (SQLite) corpus, where the text comes from."""
|
|
|
|
def setUp(self):
|
|
self.engine = create_engine("sqlite://", connect_args={"check_same_thread": False},
|
|
poolclass=StaticPool)
|
|
Base.metadata.create_all(self.engine)
|
|
self.db = Session(self.engine)
|
|
self.db.add(User(id=1, name="Mod", email="mod@example.test",
|
|
hashed_password="unused", role="moderator"))
|
|
for question_id, text in [
|
|
(1, "A child with fever and a rash"),
|
|
(2, "A toddler with a barking cough and stridor"),
|
|
(3, "An infant with jaundice on day three"),
|
|
]:
|
|
self.db.add(Question(id=question_id, user_id=1, question_text=text,
|
|
question_type="mcq", options=["yes", "no"], correct_answer="yes"))
|
|
self.db.commit()
|
|
self.addCleanup(self.engine.dispose)
|
|
self.addCleanup(self.db.close)
|
|
|
|
def test_the_reranker_sees_the_row_text_and_reorders_by_it(self):
|
|
seen = {}
|
|
|
|
def fake(query, documents):
|
|
seen["documents"] = documents
|
|
return [0.1, 0.9, 0.2]
|
|
|
|
with patch.object(rerank_service, "is_configured", return_value=True), \
|
|
patch.object(rerank_service, "rerank", side_effect=fake):
|
|
out = search_service.rerank_ids(self.db, "croup", "question", [1, 2, 3])
|
|
self.assertEqual(out, [2, 3, 1])
|
|
self.assertIn("barking cough", " ".join(seen["documents"]))
|
|
|
|
def test_an_unconfigured_reranker_costs_neither_a_query_nor_the_order(self):
|
|
with patch.object(rerank_service, "is_configured", return_value=False), \
|
|
patch.object(rerank_service, "rerank", side_effect=AssertionError("no call")):
|
|
self.assertEqual(search_service.rerank_ids(self.db, "croup", "question", [3, 1, 2]),
|
|
[3, 1, 2])
|
|
|
|
def test_ids_that_are_no_longer_in_the_table_are_still_returned(self):
|
|
# Retrieval decided these exist; a row deleted between the two queries
|
|
# is scored on empty text, and is still on the page afterwards.
|
|
with patch.object(rerank_service, "is_configured", return_value=True), \
|
|
patch.object(rerank_service, "rerank", return_value=[0.2, 0.8, 0.5]):
|
|
out = search_service.rerank_ids(self.db, "croup", "question", [1, 2, 999])
|
|
self.assertEqual(out, [2, 999, 1])
|
|
|
|
def test_a_score_list_that_does_not_match_the_shortlist_is_refused(self):
|
|
with patch.object(rerank_service, "is_configured", return_value=True), \
|
|
patch.object(rerank_service, "rerank", return_value=[0.2, 0.8]):
|
|
self.assertEqual(search_service.rerank_ids(self.db, "croup", "question", [1, 2, 3]),
|
|
[1, 2, 3])
|
|
|
|
|
|
class AiModeShortlistTests(unittest.TestCase):
|
|
"""The shortlist is the answer's evidence, so membership matters most."""
|
|
|
|
def sources(self):
|
|
return [
|
|
{"kind": "section", "id": 1, "title": "Croup", "text": "barking cough", "score": 1.0},
|
|
{"kind": "section", "id": 2, "title": "Asthma", "text": "wheeze", "score": 0.5},
|
|
{"kind": "question", "id": 9, "title": "A toddler", "text": "stridor", "score": 1.0},
|
|
]
|
|
|
|
def test_one_scale_replaces_four_incomparable_ones(self):
|
|
from app.services import ai_mode_service
|
|
|
|
sources = self.sources()
|
|
with patch.object(ai_mode_service.rerank_service, "rerank",
|
|
return_value=[0.2, 0.1, 0.9]):
|
|
ai_mode_service._cross_encode("croup", sources)
|
|
# The best section and the best question both scored 1.0 before; now
|
|
# every source sits on one ranking, so the sort can mean something.
|
|
self.assertEqual([s["score"] for s in sources], [0.5, 1 / 3, 1.0])
|
|
|
|
def test_silence_leaves_every_score_exactly_as_it_was(self):
|
|
from app.services import ai_mode_service
|
|
|
|
sources = self.sources()
|
|
with patch.object(ai_mode_service.rerank_service, "rerank", return_value=None):
|
|
ai_mode_service._cross_encode("croup", sources)
|
|
self.assertEqual([s["score"] for s in sources], [1.0, 0.5, 1.0])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|