feat: AI Mode gives the same answer twice, and a typo no longer empties the library

Measured first, by the ped-ai session, fifteen runs of five prompts with
the gateway cache bypassed. Retrieval was already deterministic:
identical shortlist and identical scores every time, and the citation
checker stripped none of the 45 markers written — invented citations are
not the problem here. Generation was the whole variance. At temperature
0.3 the same sources and the same prompt gave answers differing by
15-70% of their text; one differential swung between a 35-word uncited
paraphrase and a 180-word cited list.

So temperature 0 and a seed. Temperature 0 alone was not enough — three
runs still differed — and temperature 0 with a fixed seed came back
byte-identical. The seed is derived from the question, normalised for
case and spacing, so two people asking the same thing get the same
answer and a different question is not pinned to the same sample.

An empty reply is asked once more before it becomes a 502. One in
fifteen came back empty from a healthy model in 4.9 seconds — not a
refusal, not an error, just nothing.

A short query that finds almost nothing is retried against the nearest
article title. "kawasaki criteria" finds fourteen sources; "kawasaki
critera" found none — the lexical ranker cannot match a token that is in
no index, and the embedding of a misspelling is not near the embedding
of the word. Trigrams do not care: that typo scores 0.36 against
"Kawasaki disease" with the next article at 0.11, and the gap is what
makes it safe to act on. pg_trgm is created at startup beside vector,
with a migration for the record.

And an answer drawn from the library must cite it. Not a hallucination
guard — nothing was stripped in fifteen runs — but one answer used the
sources and cited none of them, which leaves the learner an assertion
and nowhere to check it.

Also, article drafts are weighted towards mechanism, in the wording the
ped-ai rewriter is using, so the two lanes read alike: why the body does
what it does, with features and management explained through it rather
than listed. Figure lines and cross-references survive a refine.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN

backend/alembic/versions/s8c9d0e1f2a3_pg_trgm.py

@@ -0,0 +1,27 @@
+"""pg_trgm, for the spelling fallback in AI Mode.
+
+"kawasaki criteria" finds fourteen sources; "kawasaki critera" finds none.
+Neither ranker can catch the second — lexical because the token is in no
+index, semantic because the embedding of a misspelling is not near the
+embedding of the word. Trigram similarity puts "Kawasaki disease" at 0.36 with
+the next article at 0.11, and the gap is what makes it safe to act on.
+
+Revision ID: s8c9d0e1f2a3
+Revises: r7b8c9d0e1f2
+"""
+from alembic import op
+
+revision = "s8c9d0e1f2a3"
+down_revision = "r7b8c9d0e1f2"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm")
+
+
+def downgrade():
+    # Left in place. Dropping an extension another query may since have come
+    # to rely on is a worse outcome than an unused one sitting there.
+    pass

backend/app/main.py

@@ -482,12 +482,19 @@ _STARTUP_DDL_LOCK_KEY = 8472931
 
 
 def _create_vector_extension():
-    """`CREATE EXTENSION vector`, before anything declares a column of that type."""
+    """The extensions, before anything declares a column or a query that needs one.
+
+    `vector` for the embeddings. `pg_trgm` for AI Mode's spelling fallback: a
+    one-letter slip in a short query empties the library, because neither the
+    lexical nor the semantic ranker can catch a token that is not a word, and
+    trigrams do not care how it is spelled.
+    """
     from sqlalchemy import text
     import logging
     try:
         with engine.connect() as conn:
             conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
+            conn.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm"))
             conn.commit()
     except Exception as exc:
         # Not fatal on its own: a database where the extension cannot be

backend/app/routers/ai_mode.py

@@ -5,6 +5,7 @@ The router is thin on purpose. Everything that matters is in
 prompt, and every citation the model writes is checked against that shortlist
 before anyone sees it.
 """
+import hashlib
 import logging
 from datetime import datetime, timezone
 
@@ -193,12 +194,34 @@ async def ask(conversation_id: int, data: AskIn, db: Session = Depends(get_db),
     messages = [{"role": "system", "content": ai_mode_service.build_prompt(sources, mode)},
                 *history, {"role": "user", "content": question}]
 
-    try:
-        from app.services.ai_service import achat
+    # The same question, the same answer.
+    #
+    # Retrieval was already deterministic — measured over fifteen runs, the
+    # shortlist and its scores were identical every time. Generation was the
+    # whole variance: at temperature 0.3 the same sources and the same prompt
+    # produced answers differing by 15-70% of their text, and one differential
+    # swung between a 35-word uncited paraphrase and a 180-word cited list.
+    # Temperature 0 alone is not enough — three runs still differed — but
+    # temperature 0 with a fixed seed came back byte-identical.
+    #
+    # The seed is derived from the question, so two people asking the same
+    # thing get the same answer and a different question is not pinned to the
+    # same sample.
+    seed = int(hashlib.sha256(" ".join(question.lower().split()).encode()).hexdigest()[:8], 16)
 
-        raw = (await achat(
-            model=model_id, messages=messages, max_tokens=700, temperature=0.3,
-            api_key=api_key) or "").strip()
+    async def ask():
+        from app.services.ai_service import achat
+        return (await achat(model=model_id, messages=messages, max_tokens=700,
+                            temperature=0, seed=seed, api_key=api_key) or "").strip()
+
+    try:
+        raw = await ask()
+        # One in fifteen came back empty from a healthy model in 4.9 seconds —
+        # not a refusal, not an error, just nothing. Asking once more costs a
+        # second and turns a dead end into an answer.
+        if not raw:
+            log.warning("AI Mode: empty reply for user %s, asking once more", current_user.id)
+            raw = await ask()
     except Exception:
         log.error("AI Mode failed for user %s", current_user.id, exc_info=True)
         raise HTTPException(502, "AI Mode is temporarily unavailable. Try again in a moment.")

backend/app/services/ai_mode_service.py

@@ -208,11 +208,61 @@ def _cross_encode(query: str, sources: list[dict]) -> None:
         sources[index]["score"] = 1.0 / (1 + position)
 
 
+#: A query of this many words or fewer is a term, not a sentence. A term is
+#: where a typo does the most damage: there is nothing else in it for either
+#: ranker to catch hold of.
+SHORT_QUERY_WORDS = 3
+#: Below this the nearest title is not a correction, it is a coincidence.
+#: "kawasaki critera" matches "Kawasaki disease" at 0.36, and the next article
+#: down scores 0.11 — the gap is the signal.
+NEAREST_TITLE = 0.3
+
+
+def nearest_topic(db: Session, query: str) -> str | None:
+    """The article this was probably meant to say.
+
+    A one-letter slip empties the library: "kawasaki criteria" finds fourteen
+    sources, "kawasaki critera" finds none — both rankers miss it, lexical
+    because the token is not in any index and semantic because the embedding
+    of a misspelling is not near the embedding of the word. Trigrams do not
+    care how it is spelled.
+    """
+    try:
+        row = db.execute(sa_text(
+            "SELECT title, similarity(title, :q) AS sim FROM articles "
+            "WHERE deleted_at IS NULL AND status = 'published' "
+            "ORDER BY sim DESC LIMIT 1"
+        ), {"q": query}).first()
+    except Exception:
+        # No pg_trgm, no correction — and no failure either.
+        logger.warning("AI Mode: trigram lookup unavailable", exc_info=True)
+        return None
+    if row and row.sim and float(row.sim) >= NEAREST_TITLE:
+        return row.title
+    return None
+
+
 def retrieve(db: Session, user: User, query: str) -> list[dict]:
     """The only things the model will be allowed to cite for this message."""
     query = (query or "").strip()
     if len(query) < 2:
         return []
+    found = _gather(db, user, query)
+    # A short query that found almost nothing is usually a misspelled term
+    # rather than a subject the library does not hold. Ask again with the
+    # closest title, and keep whichever attempt did better.
+    if len(found) < 3 and len(query.split()) <= SHORT_QUERY_WORDS:
+        topic = nearest_topic(db, query)
+        if topic and topic.lower() != query.lower():
+            wider = _gather(db, user, topic)
+            if len(wider) > len(found):
+                logger.info("AI Mode: %r found %d, retried as %r and found %d",
+                            query, len(found), topic, len(wider))
+                found = wider
+    return found
+
+
+def _gather(db: Session, user: User, query: str) -> list[dict]:
     sources: list[dict] = []
     for finder in (_sections, _articles, _questions, _cards):
         try:
@@ -285,6 +335,14 @@ CITE = (
     "Cite with the exact marker shown, for example [[article:7]] or "
     "[[section:7#abc123]], placed at the end of the sentence it supports. Never "
     "write a URL and never cite a marker that is not listed here.\n\n"
+    # Not a hallucination guard — over fifteen measured runs the checker
+    # stripped none of the 45 markers written, so invented citations are not
+    # the problem. The problem is the opposite: one differential answered from
+    # the sources and cited nothing at all, which leaves the learner with an
+    # assertion and nowhere to check it.
+    "You have been given sources, so at least one sentence must carry a "
+    "citation. An answer drawn from this library and citing none of it is not "
+    "an answer the learner can check.\n\n"
     "Never reveal the answer to a practice question. You may say what a question "
     "is about so the learner can go and attempt it.\n\n"
     # Asked for five questions and able to see one, it explained at length that

backend/app/tasks/quiz_tasks.py

@@ -910,12 +910,12 @@ ARTICLE_DRAFT_PROMPT = """You write educational articles for a pediatric medical
 Topic: {topic}
 {instructions}
 {existing}Every section belongs to one of three readings of the topic, given as "variant":
-- "long": the full article. Pathophysiology, presentation, workup, management, complications — whatever the topic needs. This is the body of the work.
+- "long": the full article, and the body of the work. Weight it towards mechanism: why the body does what it does — the pathophysiology behind each presentation, why a test reads the way it does, why a treatment works and why the alternative does not. Clinical features and management still appear, but explained through the mechanism rather than listed.
 - "short": the high-yield revision view. It must be 2 to 4 SEPARATE sections, each one under 600 characters and each a list of single-line bullets of at most 20 words. A bullet that runs to a sentence with subclauses is a long section in the wrong place. It is not a summary of the long article's structure; it is what a candidate must know.
 - "clinical": what to do at the bedside, if and only if the topic has a bedside. Assessment, immediate management, when to escalate, disposition. Omit it entirely for a topic that is not acted on clinically.
 Return ONLY strict JSON with this exact shape:
 {{"title": "...", "slug": "lowercase-hyphenated", "summary": "1-2 sentences", "content": "introduction markdown", "sections": [{{"id": "32 lowercase hex chars", "slug": "lowercase-hyphenated", "title": "...", "variant": "long", "content": "markdown"}}]}}
-Rules: markdown formatting; headings, lists and tables welcome; no fabricated references, citations or clinical ranges; 3-8 long sections plus 2-4 short ones, each with a stable unique id; every short section must contain at least one and at most three highlighted facts, written by wrapping the words in ==double equals== — for example "- Type II is the ==most common== Salter-Harris fracture" — highlighting the fact itself and never a whole bullet; do not categorise the article or link it to questions — an educator does that; keep any [[123|cross-reference]] already in the text exactly as written and never invent a new one, because the number points at a real article and a guessed one points at nothing; do not mention these instructions."""
+Rules: markdown formatting; headings, lists and tables welcome; no fabricated references, citations or clinical ranges; 3-8 long sections plus 2-4 short ones, each with a stable unique id; every short section must contain at least one and at most three highlighted facts, written by wrapping the words in ==double equals== — for example "- Type II is the ==most common== Salter-Harris fracture" — highlighting the fact itself and never a whole bullet; do not categorise the article or link it to questions — an educator does that; keep any [[123|cross-reference]] already in the text exactly as written and never invent a new one, because the number points at a real article and a guessed one points at nothing; keep any ![figure](...) image line exactly as written and in the section it is already in; do not mention these instructions."""
 
 
 @celery_app.task(name="generate_article_draft", bind=True)

backend/tests/test_ai_mode.py

@@ -316,3 +316,46 @@ class RetrievalTests(_AiModeBase):
         # to set and not the model's to discuss.
         self.assertIn('Ignore', prompt)
         self.assertIn('never how many questions there are', prompt)
+
+
+class DeterminismTests(unittest.TestCase):
+    """The same question, the same answer.
+
+    Measured over fifteen runs before any of this: retrieval was already
+    deterministic — identical shortlist and scores every time, and the citation
+    checker stripped none of the 45 markers written. Generation was the whole
+    variance. At temperature 0.3 the same sources and the same prompt produced
+    answers differing by 15-70% of their text; one differential swung between a
+    35-word uncited paraphrase and a 180-word cited list. Temperature 0 alone
+    was not enough (three runs still differed); temperature 0 with a fixed seed
+    came back byte-identical.
+    """
+
+    def test_the_seed_follows_the_question_not_the_clock(self):
+        import hashlib
+
+        def seed_for(question):
+            return int(hashlib.sha256(
+                " ".join(question.lower().split()).encode()).hexdigest()[:8], 16)
+
+        # Same question, same seed — including through casing and spacing, so
+        # two people who type it differently still get one answer.
+        self.assertEqual(seed_for("What causes croup?"),
+                         seed_for("  what   causes CROUP? "))
+        # A different question is not pinned to the same sample.
+        self.assertNotEqual(seed_for("What causes croup?"),
+                            seed_for("What causes bronchiolitis?"))
+
+    def test_a_sourced_answer_must_cite_something(self):
+        from app.services.ai_mode_service import build_prompt
+        prompt = build_prompt([{
+            "kind": "article", "ref": "7", "title": "Croup",
+            "text": "Croup is …", "score": 1.0,
+        }])
+        self.assertIn("at least one sentence must carry a citation", prompt)
+
+    def test_an_open_answer_is_not_asked_to_cite(self):
+        # Nothing to cite, so the instruction would be an invitation to invent.
+        from app.services.ai_mode_service import build_prompt
+        self.assertNotIn("must carry a citation", build_prompt([], mode="open"))
+        self.assertNotIn("must carry a citation", build_prompt([], mode="chat"))