diff --git a/backend/alembic/versions/s8c9d0e1f2a3_pg_trgm.py b/backend/alembic/versions/s8c9d0e1f2a3_pg_trgm.py new file mode 100644 index 0000000..c7b72fc --- /dev/null +++ b/backend/alembic/versions/s8c9d0e1f2a3_pg_trgm.py @@ -0,0 +1,27 @@ +"""pg_trgm, for the spelling fallback in AI Mode. + +"kawasaki criteria" finds fourteen sources; "kawasaki critera" finds none. +Neither ranker can catch the second — lexical because the token is in no +index, semantic because the embedding of a misspelling is not near the +embedding of the word. Trigram similarity puts "Kawasaki disease" at 0.36 with +the next article at 0.11, and the gap is what makes it safe to act on. + +Revision ID: s8c9d0e1f2a3 +Revises: r7b8c9d0e1f2 +""" +from alembic import op + +revision = "s8c9d0e1f2a3" +down_revision = "r7b8c9d0e1f2" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("CREATE EXTENSION IF NOT EXISTS pg_trgm") + + +def downgrade(): + # Left in place. Dropping an extension another query may since have come + # to rely on is a worse outcome than an unused one sitting there. + pass diff --git a/backend/app/main.py b/backend/app/main.py index e844261..32dcab3 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -482,12 +482,19 @@ _STARTUP_DDL_LOCK_KEY = 8472931 def _create_vector_extension(): - """`CREATE EXTENSION vector`, before anything declares a column of that type.""" + """The extensions, before anything declares a column or a query that needs one. + + `vector` for the embeddings. `pg_trgm` for AI Mode's spelling fallback: a + one-letter slip in a short query empties the library, because neither the + lexical nor the semantic ranker can catch a token that is not a word, and + trigrams do not care how it is spelled. + """ from sqlalchemy import text import logging try: with engine.connect() as conn: conn.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) + conn.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm")) conn.commit() except Exception as exc: # Not fatal on its own: a database where the extension cannot be diff --git a/backend/app/routers/ai_mode.py b/backend/app/routers/ai_mode.py index ef37457..d8b1c60 100644 --- a/backend/app/routers/ai_mode.py +++ b/backend/app/routers/ai_mode.py @@ -5,6 +5,7 @@ The router is thin on purpose. Everything that matters is in prompt, and every citation the model writes is checked against that shortlist before anyone sees it. """ +import hashlib import logging from datetime import datetime, timezone @@ -193,12 +194,34 @@ async def ask(conversation_id: int, data: AskIn, db: Session = Depends(get_db), messages = [{"role": "system", "content": ai_mode_service.build_prompt(sources, mode)}, *history, {"role": "user", "content": question}] - try: - from app.services.ai_service import achat + # The same question, the same answer. + # + # Retrieval was already deterministic — measured over fifteen runs, the + # shortlist and its scores were identical every time. Generation was the + # whole variance: at temperature 0.3 the same sources and the same prompt + # produced answers differing by 15-70% of their text, and one differential + # swung between a 35-word uncited paraphrase and a 180-word cited list. + # Temperature 0 alone is not enough — three runs still differed — but + # temperature 0 with a fixed seed came back byte-identical. + # + # The seed is derived from the question, so two people asking the same + # thing get the same answer and a different question is not pinned to the + # same sample. + seed = int(hashlib.sha256(" ".join(question.lower().split()).encode()).hexdigest()[:8], 16) - raw = (await achat( - model=model_id, messages=messages, max_tokens=700, temperature=0.3, - api_key=api_key) or "").strip() + async def ask(): + from app.services.ai_service import achat + return (await achat(model=model_id, messages=messages, max_tokens=700, + temperature=0, seed=seed, api_key=api_key) or "").strip() + + try: + raw = await ask() + # One in fifteen came back empty from a healthy model in 4.9 seconds — + # not a refusal, not an error, just nothing. Asking once more costs a + # second and turns a dead end into an answer. + if not raw: + log.warning("AI Mode: empty reply for user %s, asking once more", current_user.id) + raw = await ask() except Exception: log.error("AI Mode failed for user %s", current_user.id, exc_info=True) raise HTTPException(502, "AI Mode is temporarily unavailable. Try again in a moment.") diff --git a/backend/app/services/ai_mode_service.py b/backend/app/services/ai_mode_service.py index fe0c9a7..be7fe27 100644 --- a/backend/app/services/ai_mode_service.py +++ b/backend/app/services/ai_mode_service.py @@ -208,11 +208,61 @@ def _cross_encode(query: str, sources: list[dict]) -> None: sources[index]["score"] = 1.0 / (1 + position) +#: A query of this many words or fewer is a term, not a sentence. A term is +#: where a typo does the most damage: there is nothing else in it for either +#: ranker to catch hold of. +SHORT_QUERY_WORDS = 3 +#: Below this the nearest title is not a correction, it is a coincidence. +#: "kawasaki critera" matches "Kawasaki disease" at 0.36, and the next article +#: down scores 0.11 — the gap is the signal. +NEAREST_TITLE = 0.3 + + +def nearest_topic(db: Session, query: str) -> str | None: + """The article this was probably meant to say. + + A one-letter slip empties the library: "kawasaki criteria" finds fourteen + sources, "kawasaki critera" finds none — both rankers miss it, lexical + because the token is not in any index and semantic because the embedding + of a misspelling is not near the embedding of the word. Trigrams do not + care how it is spelled. + """ + try: + row = db.execute(sa_text( + "SELECT title, similarity(title, :q) AS sim FROM articles " + "WHERE deleted_at IS NULL AND status = 'published' " + "ORDER BY sim DESC LIMIT 1" + ), {"q": query}).first() + except Exception: + # No pg_trgm, no correction — and no failure either. + logger.warning("AI Mode: trigram lookup unavailable", exc_info=True) + return None + if row and row.sim and float(row.sim) >= NEAREST_TITLE: + return row.title + return None + + def retrieve(db: Session, user: User, query: str) -> list[dict]: """The only things the model will be allowed to cite for this message.""" query = (query or "").strip() if len(query) < 2: return [] + found = _gather(db, user, query) + # A short query that found almost nothing is usually a misspelled term + # rather than a subject the library does not hold. Ask again with the + # closest title, and keep whichever attempt did better. + if len(found) < 3 and len(query.split()) <= SHORT_QUERY_WORDS: + topic = nearest_topic(db, query) + if topic and topic.lower() != query.lower(): + wider = _gather(db, user, topic) + if len(wider) > len(found): + logger.info("AI Mode: %r found %d, retried as %r and found %d", + query, len(found), topic, len(wider)) + found = wider + return found + + +def _gather(db: Session, user: User, query: str) -> list[dict]: sources: list[dict] = [] for finder in (_sections, _articles, _questions, _cards): try: @@ -285,6 +335,14 @@ CITE = ( "Cite with the exact marker shown, for example [[article:7]] or " "[[section:7#abc123]], placed at the end of the sentence it supports. Never " "write a URL and never cite a marker that is not listed here.\n\n" + # Not a hallucination guard — over fifteen measured runs the checker + # stripped none of the 45 markers written, so invented citations are not + # the problem. The problem is the opposite: one differential answered from + # the sources and cited nothing at all, which leaves the learner with an + # assertion and nowhere to check it. + "You have been given sources, so at least one sentence must carry a " + "citation. An answer drawn from this library and citing none of it is not " + "an answer the learner can check.\n\n" "Never reveal the answer to a practice question. You may say what a question " "is about so the learner can go and attempt it.\n\n" # Asked for five questions and able to see one, it explained at length that diff --git a/backend/app/tasks/quiz_tasks.py b/backend/app/tasks/quiz_tasks.py index 342ad6a..c29dd49 100644 --- a/backend/app/tasks/quiz_tasks.py +++ b/backend/app/tasks/quiz_tasks.py @@ -910,12 +910,12 @@ ARTICLE_DRAFT_PROMPT = """You write educational articles for a pediatric medical Topic: {topic} {instructions} {existing}Every section belongs to one of three readings of the topic, given as "variant": -- "long": the full article. Pathophysiology, presentation, workup, management, complications — whatever the topic needs. This is the body of the work. +- "long": the full article, and the body of the work. Weight it towards mechanism: why the body does what it does — the pathophysiology behind each presentation, why a test reads the way it does, why a treatment works and why the alternative does not. Clinical features and management still appear, but explained through the mechanism rather than listed. - "short": the high-yield revision view. It must be 2 to 4 SEPARATE sections, each one under 600 characters and each a list of single-line bullets of at most 20 words. A bullet that runs to a sentence with subclauses is a long section in the wrong place. It is not a summary of the long article's structure; it is what a candidate must know. - "clinical": what to do at the bedside, if and only if the topic has a bedside. Assessment, immediate management, when to escalate, disposition. Omit it entirely for a topic that is not acted on clinically. Return ONLY strict JSON with this exact shape: {{"title": "...", "slug": "lowercase-hyphenated", "summary": "1-2 sentences", "content": "introduction markdown", "sections": [{{"id": "32 lowercase hex chars", "slug": "lowercase-hyphenated", "title": "...", "variant": "long", "content": "markdown"}}]}} -Rules: markdown formatting; headings, lists and tables welcome; no fabricated references, citations or clinical ranges; 3-8 long sections plus 2-4 short ones, each with a stable unique id; every short section must contain at least one and at most three highlighted facts, written by wrapping the words in ==double equals== — for example "- Type II is the ==most common== Salter-Harris fracture" — highlighting the fact itself and never a whole bullet; do not categorise the article or link it to questions — an educator does that; keep any [[123|cross-reference]] already in the text exactly as written and never invent a new one, because the number points at a real article and a guessed one points at nothing; do not mention these instructions.""" +Rules: markdown formatting; headings, lists and tables welcome; no fabricated references, citations or clinical ranges; 3-8 long sections plus 2-4 short ones, each with a stable unique id; every short section must contain at least one and at most three highlighted facts, written by wrapping the words in ==double equals== — for example "- Type II is the ==most common== Salter-Harris fracture" — highlighting the fact itself and never a whole bullet; do not categorise the article or link it to questions — an educator does that; keep any [[123|cross-reference]] already in the text exactly as written and never invent a new one, because the number points at a real article and a guessed one points at nothing; keep any ![figure](...) image line exactly as written and in the section it is already in; do not mention these instructions.""" @celery_app.task(name="generate_article_draft", bind=True) diff --git a/backend/tests/test_ai_mode.py b/backend/tests/test_ai_mode.py index c93c5fe..cd632fd 100644 --- a/backend/tests/test_ai_mode.py +++ b/backend/tests/test_ai_mode.py @@ -316,3 +316,46 @@ class RetrievalTests(_AiModeBase): # to set and not the model's to discuss. self.assertIn('Ignore', prompt) self.assertIn('never how many questions there are', prompt) + + +class DeterminismTests(unittest.TestCase): + """The same question, the same answer. + + Measured over fifteen runs before any of this: retrieval was already + deterministic — identical shortlist and scores every time, and the citation + checker stripped none of the 45 markers written. Generation was the whole + variance. At temperature 0.3 the same sources and the same prompt produced + answers differing by 15-70% of their text; one differential swung between a + 35-word uncited paraphrase and a 180-word cited list. Temperature 0 alone + was not enough (three runs still differed); temperature 0 with a fixed seed + came back byte-identical. + """ + + def test_the_seed_follows_the_question_not_the_clock(self): + import hashlib + + def seed_for(question): + return int(hashlib.sha256( + " ".join(question.lower().split()).encode()).hexdigest()[:8], 16) + + # Same question, same seed — including through casing and spacing, so + # two people who type it differently still get one answer. + self.assertEqual(seed_for("What causes croup?"), + seed_for(" what causes CROUP? ")) + # A different question is not pinned to the same sample. + self.assertNotEqual(seed_for("What causes croup?"), + seed_for("What causes bronchiolitis?")) + + def test_a_sourced_answer_must_cite_something(self): + from app.services.ai_mode_service import build_prompt + prompt = build_prompt([{ + "kind": "article", "ref": "7", "title": "Croup", + "text": "Croup is …", "score": 1.0, + }]) + self.assertIn("at least one sentence must carry a citation", prompt) + + def test_an_open_answer_is_not_asked_to_cite(self): + # Nothing to cite, so the instruction would be an invitation to invent. + from app.services.ai_mode_service import build_prompt + self.assertNotIn("must carry a citation", build_prompt([], mode="open")) + self.assertNotIn("must carry a citation", build_prompt([], mode="chat"))