diff --git a/backend/app/tasks/quiz_tasks.py b/backend/app/tasks/quiz_tasks.py index f5631b3..7b057f1 100644 --- a/backend/app/tasks/quiz_tasks.py +++ b/backend/app/tasks/quiz_tasks.py @@ -1028,11 +1028,11 @@ Topic: {topic} {instructions} {existing}Every section belongs to one of three readings of the topic, given as "variant": - "long": the full article. Pathophysiology, presentation, workup, management, complications — whatever the topic needs. This is the body of the work. -- "short": the high-yield revision view. 2-4 short sections of the facts worth carrying into an exam, written as tight lists, not prose. It is not a summary of the long article's structure; it is what a candidate must know. +- "short": the high-yield revision view. It must be 2 to 4 SEPARATE sections, each one under 600 characters and each a list of single-line bullets of at most 20 words. A bullet that runs to a sentence with subclauses is a long section in the wrong place. It is not a summary of the long article's structure; it is what a candidate must know. - "clinical": what to do at the bedside, if and only if the topic has a bedside. Assessment, immediate management, when to escalate, disposition. Omit it entirely for a topic that is not acted on clinically. Return ONLY strict JSON with this exact shape: {{"title": "...", "slug": "lowercase-hyphenated", "summary": "1-2 sentences", "content": "introduction markdown", "sections": [{{"id": "32 lowercase hex chars", "slug": "lowercase-hyphenated", "title": "...", "variant": "long", "content": "markdown"}}]}} -Rules: markdown formatting; headings, lists and tables welcome; no fabricated references, citations or clinical ranges; 3-8 long sections plus 2-4 short ones, each with a stable unique id; mark a key fact in the short sections by wrapping it in ==double equals== so it is highlighted for the reader, sparingly and never a whole paragraph; do not categorise the article or link it to questions — an educator does that; do not mention these instructions.""" +Rules: markdown formatting; headings, lists and tables welcome; no fabricated references, citations or clinical ranges; 3-8 long sections plus 2-4 short ones, each with a stable unique id; every short section must contain at least one and at most three highlighted facts, written by wrapping the words in ==double equals== — for example "- Type II is the ==most common== Salter-Harris fracture" — highlighting the fact itself and never a whole bullet; do not categorise the article or link it to questions — an educator does that; do not mention these instructions.""" @celery_app.task(name="generate_article_draft", bind=True) @@ -1063,9 +1063,17 @@ def generate_article_draft(self, job_id: str, user_id: int, topic: str, _push_step(r, job_id, "ai", "Drafting article…") existing_block = "" if existing: + # The variant travels with the section. Without it the model was + # shown a flat list of headings and had to guess which readings + # they belonged to all over again — so refining an article could + # quietly turn its bedside section into part of the long read, and + # the high-yield view into prose. sections_text = "\n\n".join( - f"## {s.get('title', 'Section')}\n{s.get('content', '')}" for s in (existing.sections or [])) - existing_block = (f"Existing draft to refine (preserve and improve its content):\n" + f"## [{s.get('variant', 'long')}] {s.get('title', 'Section')}\n{s.get('content', '')}" + for s in (existing.sections or [])) + existing_block = (f"Existing draft to refine (preserve and improve its content; the " + f"bracketed word before each heading is that section's variant and " + f"must be kept):\n" f"Summary: {existing.summary or ''}\nIntro: {existing.content or ''}\n" f"{sections_text}\n\n") prompt = ARTICLE_DRAFT_PROMPT.format( diff --git a/backend/tests/test_article_ai.py b/backend/tests/test_article_ai.py index c55de78..f6a8f0b 100644 --- a/backend/tests/test_article_ai.py +++ b/backend/tests/test_article_ai.py @@ -172,3 +172,75 @@ class ArticleAiTests(unittest.TestCase): if __name__ == '__main__': unittest.main() + + +class DraftPromptTests(unittest.TestCase): + """What the prompt asks for, measured against what 323 drafts produced. + + The three readings of a topic — the full article, the high-yield revision + view, the bedside — were described and then largely ignored for the second + of them: across every AI draft in the bank, the "short" view came out as + one section of 1,808 characters (against 721 for a long one), and 4 of 326 + short sections carried a highlighted fact. The revision view was the + longest prose in the article and none of it was marked. + + So the instruction is a measurement rather than an adjective. + """ + + def test_the_short_view_is_asked_for_in_numbers(self): + from app.tasks.quiz_tasks import ARTICLE_DRAFT_PROMPT + self.assertIn("2 to 4 SEPARATE sections", ARTICLE_DRAFT_PROMPT) + self.assertIn("under 600 characters", ARTICLE_DRAFT_PROMPT) + self.assertIn("at most 20 words", ARTICLE_DRAFT_PROMPT) + + def test_highlights_are_required_and_shown(self): + from app.tasks.quiz_tasks import ARTICLE_DRAFT_PROMPT + self.assertIn("at least one and at most three highlighted facts", + ARTICLE_DRAFT_PROMPT) + # An example, because "wrap it in double equals" produced 1.2% uptake. + self.assertIn("==most common==", ARTICLE_DRAFT_PROMPT) + + def test_refining_is_told_which_reading_each_section_is(self): + """Otherwise a refine re-guesses every variant. + + The existing draft was handed over as a flat list of `## Heading`, so + the model could not know which sections were the bedside and which were + the revision view — and a refine could quietly move one into the other. + """ + self.bank = fixtures.BuilderTests() + self.bank.setUp() + try: + from unittest.mock import patch + from sqlalchemy.orm import sessionmaker + from app.models.article import Article + + self.bank.db.add(Article( + slug='refine-me', title='Refine me', content='Intro', user_id=None, + status='draft', sections=[ + {'id': 'a' * 32, 'slug': 'bedside', 'title': 'At the bedside', + 'variant': 'clinical', 'content': 'Do this'}, + {'id': 'b' * 32, 'slug': 'high-yield', 'title': 'High yield', + 'variant': 'short', 'content': '- A fact'}, + ])) + self.bank.db.commit() + article = self.bank.db.query(Article).filter_by(slug='refine-me').one() + + redis = Mock() + redis.from_url.return_value = redis + redis.get.return_value = None + redis.lrange.return_value = [] + with patch.dict(sys.modules, {'redis': redis}), \ + patch('app.tasks.quiz_tasks.SessionLocal', + sessionmaker(bind=self.bank.engine)), \ + patch('app.services.ai_service.get_model_for_task', + return_value=('synthetic', None)), \ + patch('app.services.ai_service.chat') as ai: + ai.return_value = json.dumps(DRAFT_RESPONSE) + generate_article_draft('job-refine', 3, 'Refine me', '', article.id) + prompt = ai.call_args.kwargs['messages'][0]['content'] + self.assertIn('## [clinical] At the bedside', prompt) + self.assertIn('## [short] High yield', prompt) + self.assertIn("must be kept", prompt) + finally: + patch.stopall() + self.bank.tearDown()