8836993ca1
"Prefer clinical application questions over pure recall when the content allows" produced sets of vignettes — recognition without understanding. Both generators now ask for an even split: about half clinical, a child in front of you and what to do next, and about half mechanism, why the body behaves as it does. Pathophysiology is what makes the clinical half stick. The question generator is also told outright that the patient is a child and that the ages, doses and norms are the paediatric ones. Its opening line said "pediatric medical education expert" and left the rest implied. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN
271 lines
14 KiB
Python
271 lines
14 KiB
Python
"""AI authoring endpoints on disposable SQLite; no network, no model calls."""
|
|
import json
|
|
import re
|
|
import sys
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
|
|
import test_quiz_builder as fixtures
|
|
from app.models.article import Article
|
|
from app.models.flashcard import Flashcard, FlashcardDeck, FlashcardArticleLink
|
|
from app.models.question import Question
|
|
from app.routers import articles
|
|
from app.tasks.quiz_tasks import generate_article_draft, generate_article_cards
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
DRAFT_RESPONSE = {
|
|
"title": "AI draft topic", "slug": "ai-draft-topic", "summary": "Draft summary",
|
|
"content": "Draft intro", "sections": [
|
|
{"id": "a" * 32, "slug": "first", "title": "First", "content": "Body"},
|
|
{"id": "b" * 32, "slug": "second", "title": "Second", "content": "More"},
|
|
],
|
|
}
|
|
|
|
|
|
class ArticleAiTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.bank = fixtures.BuilderTests()
|
|
self.bank.setUp()
|
|
self.client = self.bank.client
|
|
self.client.app.include_router(articles.router, prefix='/articles')
|
|
self.redis = Mock()
|
|
self.redis.from_url.return_value = self.redis
|
|
self.redis.get.return_value = None
|
|
self.redis.lrange.return_value = []
|
|
self.redis.lpush.return_value = 1
|
|
self.redis.expire.return_value = True
|
|
self.redis.incr.side_effect = [1, 21]
|
|
self.delay = patch.dict(sys.modules, {'redis': self.redis})
|
|
self.delay.start()
|
|
self.queue = patch('app.tasks.quiz_tasks.generate_article_draft.delay').start()
|
|
self.card_queue = patch('app.tasks.quiz_tasks.generate_article_cards.delay').start()
|
|
|
|
def tearDown(self):
|
|
patch.stopall()
|
|
self.bank.tearDown()
|
|
|
|
def publish(self, article_id):
|
|
return self.client.post(f'/articles/{article_id}/publish', json={'published': True})
|
|
|
|
def test_ai_endpoints_queue_and_poll(self):
|
|
self.bank.user = self.bank.mod
|
|
response = self.client.post('/articles/ai-draft', json={'topic': 'Neonatal jaundice', 'instructions': 'Two sections'})
|
|
self.assertEqual(response.status_code, 200, response.text)
|
|
job_id = response.json()['job_id']
|
|
self.queue.assert_called_once()
|
|
self.redis.get.return_value = 'completed'
|
|
poll = self.client.get(f'/articles/job/{job_id}').json()
|
|
self.assertEqual(poll['status'], 'completed')
|
|
self.bank.user = self.bank.owner
|
|
self.assertEqual(self.client.get(f'/articles/job/{job_id}').status_code, 404)
|
|
self.bank.user = self.bank.mod
|
|
article = self.client.post('/articles/', json={
|
|
"title": "Refine me", "slug": "refine-me", "content": "Intro", "sections": []}).json()
|
|
self.assertEqual(self.client.post(f"/articles/{article['id']}/ai-refine", json={'instructions': 'Shorten'}).status_code, 200)
|
|
self.assertEqual(self.client.post(f"/articles/{article['id']}/ai-cards").status_code, 200)
|
|
self.card_queue.assert_called_once()
|
|
self.bank.user = self.bank.owner
|
|
self.assertEqual(self.client.post('/articles/ai-draft', json={'topic': 'Learner'}).status_code, 403)
|
|
self.assertEqual(self.client.post(f"/articles/{article['id']}/ai-refine", json={'instructions': 'x'}).status_code, 403)
|
|
|
|
def test_draft_task_creates_and_refines_without_publishing(self):
|
|
self.redis.set.return_value = True
|
|
ai = patch('app.services.ai_service.chat').start()
|
|
ai.return_value = json.dumps(DRAFT_RESPONSE)
|
|
with patch('app.tasks.quiz_tasks.SessionLocal', sessionmaker(bind=self.bank.engine)), \
|
|
patch('app.services.ai_service.get_model_for_task', return_value=('synthetic', None)):
|
|
generate_article_draft('job-1', 3, 'Neonatal jaundice', '')
|
|
article = self.bank.db.query(Article).filter_by(slug='ai-draft-topic').one()
|
|
self.assertEqual(article.status, 'draft')
|
|
self.assertEqual([s['id'] for s in article.sections], ['a' * 32, 'b' * 32])
|
|
self.redis.set.assert_any_call('extraction:status:job-1', 'completed', ex=3600)
|
|
# Refine must send the existing body to the model, not just the title.
|
|
article.content = 'Unique draft body to preserve'
|
|
article.sections = [{'id': 'c' * 32, 'slug': 'kept', 'title': 'Kept', 'content': 'Kept body'}]
|
|
self.bank.db.commit()
|
|
generate_article_draft('job-2', 3, 'Neonatal jaundice', 'Shorten', article.id)
|
|
prompt = ai.call_args.kwargs['messages'][0]['content']
|
|
self.assertIn('Unique draft body to preserve', prompt)
|
|
self.assertIn('Kept body', prompt)
|
|
refreshed = self.bank.db.query(Article).filter_by(id=article.id).one()
|
|
self.assertEqual(refreshed.status, 'draft')
|
|
self.assertEqual(refreshed.slug, 'ai-draft-topic')
|
|
# Invalid model section ids are replaced with valid hex ids.
|
|
bad = dict(DRAFT_RESPONSE, sections=[{'id': 'bad-id', 'slug': 'bad', 'title': 'Bad', 'content': ''}])
|
|
ai.return_value = json.dumps(bad)
|
|
generate_article_draft('job-3', 3, 'Neonatal jaundice', 'Again', article.id)
|
|
fixed = self.bank.db.query(Article).filter_by(id=article.id).one()
|
|
self.assertTrue(all(re.fullmatch(r'[0-9a-f]{32}', s['id']) for s in fixed.sections), fixed.sections)
|
|
patch.stopall()
|
|
self.redis.reset_mock()
|
|
|
|
def test_cards_task_builds_private_linked_deck(self):
|
|
self.redis.set.return_value = True
|
|
self.bank.db.add(Article(slug='cards-source', title='Cards source', content='Body',
|
|
sections=[{'id': 'c' * 32, 'slug': 's', 'title': 'S', 'content': 'More'}],
|
|
user_id=3, status='published'))
|
|
self.bank.db.commit()
|
|
article = self.bank.db.query(Article).filter_by(slug='cards-source').one()
|
|
with patch('app.tasks.quiz_tasks.SessionLocal', sessionmaker(bind=self.bank.engine)), \
|
|
patch('app.services.extraction_modes.generate_flashcards',
|
|
return_value=[{'front': 'F1', 'back': 'B1'}, {'front': 'F2', 'back': 'B2', 'page_reference': 3}]), \
|
|
patch('app.services.ai_service.get_model_for_task', return_value=('synthetic', None)):
|
|
generate_article_cards('job-3', 3, article.id)
|
|
deck = self.bank.db.query(FlashcardDeck).filter_by(title='Cards: Cards source').one()
|
|
self.assertEqual(deck.is_shared, 0)
|
|
cards = self.bank.db.query(Flashcard).filter_by(deck_id=deck.id).all()
|
|
self.assertEqual(len(cards), 2)
|
|
self.assertEqual(deck.card_count, 2)
|
|
linked = self.bank.db.query(FlashcardArticleLink).filter_by(article_id=article.id).count()
|
|
self.assertEqual(linked, 2)
|
|
self.redis.set.assert_any_call('extraction:status:job-3', 'completed', ex=3600)
|
|
patch.stopall()
|
|
self.redis.reset_mock()
|
|
|
|
def test_pubmed_references_are_fields_the_reader_draws(self):
|
|
"""A paper is stored like a book: fields, not one flat sentence.
|
|
|
|
Everything else on an article is {title, author, pages}, and the reader
|
|
reads those keys — so the line PubMed used to write appeared as a blank
|
|
row under a References heading. A draft with six real papers looked
|
|
like a draft with none.
|
|
"""
|
|
from app.services import pubmed
|
|
records = [{
|
|
'pmid': '37419787', 'title': 'Management of diabetic ketoacidosis.',
|
|
'journal': 'Eur J Intern Med', 'year': '2023',
|
|
'authors': 'Barski L, Golbets E, et al',
|
|
'url': 'https://pubmed.ncbi.nlm.nih.gov/37419787/', 'abstract': '',
|
|
}]
|
|
reference, = pubmed.as_references(records)
|
|
self.assertEqual(reference['title'], 'Management of diabetic ketoacidosis')
|
|
self.assertEqual(reference['pmid'], '37419787')
|
|
self.assertEqual(reference['journal'], 'Eur J Intern Med')
|
|
self.assertEqual(reference['pages'], [])
|
|
|
|
# And a row written before this pulls itself apart on the way out,
|
|
# rather than being rewritten in the database.
|
|
healed = articles._reference_json(
|
|
'Barski L, et al. Management of diabetic ketoacidosis.. '
|
|
'Eur J Intern Med. 2023. PMID: 37419787.')
|
|
self.assertEqual(healed['pmid'], '37419787')
|
|
self.assertNotIn('PMID', healed['title'])
|
|
self.assertEqual(articles._reference_json({'title': 'A book'}), {'title': 'A book'})
|
|
|
|
def test_reading_shows_nobody_drafts(self):
|
|
"""Reading is the published library, Editorial is the unfinished work.
|
|
|
|
An educator saw their own drafts filed among the published articles
|
|
with a tag on them, which made Reading and Editorial two views of one
|
|
list. Not even an admin sees a draft here.
|
|
"""
|
|
self.bank.db.add(Article(slug='half-written', title='Half written',
|
|
content='Body', sections=[], user_id=3, status='draft'))
|
|
self.bank.db.add(Article(slug='finished', title='Finished',
|
|
content='Body', sections=[], user_id=3, status='published'))
|
|
self.bank.db.commit()
|
|
listed = self.client.get('/articles/').json()
|
|
slugs = [a['slug'] for a in listed]
|
|
self.assertIn('finished', slugs)
|
|
self.assertNotIn('half-written', slugs)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|
|
|
|
|
|
class DraftPromptTests(unittest.TestCase):
|
|
"""What the prompt asks for, measured against what 323 drafts produced.
|
|
|
|
The three readings of a topic — the full article, the high-yield revision
|
|
view, the bedside — were described and then largely ignored for the second
|
|
of them: across every AI draft in the bank, the "short" view came out as
|
|
one section of 1,808 characters (against 721 for a long one), and 4 of 326
|
|
short sections carried a highlighted fact. The revision view was the
|
|
longest prose in the article and none of it was marked.
|
|
|
|
So the instruction is a measurement rather than an adjective.
|
|
"""
|
|
|
|
def test_the_short_view_is_asked_for_in_numbers(self):
|
|
from app.tasks.quiz_tasks import ARTICLE_DRAFT_PROMPT
|
|
self.assertIn("2 to 4 SEPARATE sections", ARTICLE_DRAFT_PROMPT)
|
|
self.assertIn("under 600 characters", ARTICLE_DRAFT_PROMPT)
|
|
self.assertIn("at most 20 words", ARTICLE_DRAFT_PROMPT)
|
|
|
|
def test_highlights_are_required_and_shown(self):
|
|
from app.tasks.quiz_tasks import ARTICLE_DRAFT_PROMPT
|
|
self.assertIn("at least one and at most three highlighted facts",
|
|
ARTICLE_DRAFT_PROMPT)
|
|
# An example, because "wrap it in double equals" produced 1.2% uptake.
|
|
self.assertIn("==most common==", ARTICLE_DRAFT_PROMPT)
|
|
|
|
def test_refining_is_told_which_reading_each_section_is(self):
|
|
"""Otherwise a refine re-guesses every variant.
|
|
|
|
The existing draft was handed over as a flat list of `## Heading`, so
|
|
the model could not know which sections were the bedside and which were
|
|
the revision view — and a refine could quietly move one into the other.
|
|
"""
|
|
self.bank = fixtures.BuilderTests()
|
|
self.bank.setUp()
|
|
try:
|
|
from unittest.mock import patch
|
|
from sqlalchemy.orm import sessionmaker
|
|
from app.models.article import Article
|
|
|
|
self.bank.db.add(Article(
|
|
slug='refine-me', title='Refine me', content='Intro', user_id=None,
|
|
status='draft', sections=[
|
|
{'id': 'a' * 32, 'slug': 'bedside', 'title': 'At the bedside',
|
|
'variant': 'clinical', 'content': 'Do this'},
|
|
{'id': 'b' * 32, 'slug': 'high-yield', 'title': 'High yield',
|
|
'variant': 'short', 'content': '- A fact'},
|
|
]))
|
|
self.bank.db.commit()
|
|
article = self.bank.db.query(Article).filter_by(slug='refine-me').one()
|
|
|
|
redis = Mock()
|
|
redis.from_url.return_value = redis
|
|
redis.get.return_value = None
|
|
redis.lrange.return_value = []
|
|
with patch.dict(sys.modules, {'redis': redis}), \
|
|
patch('app.tasks.quiz_tasks.SessionLocal',
|
|
sessionmaker(bind=self.bank.engine)), \
|
|
patch('app.services.ai_service.get_model_for_task',
|
|
return_value=('synthetic', None)), \
|
|
patch('app.services.ai_service.chat') as ai:
|
|
ai.return_value = json.dumps(DRAFT_RESPONSE)
|
|
generate_article_draft('job-refine', 3, 'Refine me', '', article.id)
|
|
prompt = ai.call_args.kwargs['messages'][0]['content']
|
|
# Variant and id both travel: the variant so a bedside section is
|
|
# not rewritten as part of the long read, the id so the links
|
|
# pointing at it survive the refine.
|
|
self.assertIn(f"## [clinical] [id:{'a' * 32}] At the bedside", prompt)
|
|
self.assertIn(f"## [short] [id:{'b' * 32}] High yield", prompt)
|
|
self.assertIn("must be kept", prompt)
|
|
self.assertIn("returned unchanged", prompt)
|
|
finally:
|
|
patch.stopall()
|
|
self.bank.tearDown()
|
|
|
|
|
|
class GenerationBalanceTests(unittest.TestCase):
|
|
"""Half the bedside, half the mechanism, and always a child.
|
|
|
|
"Prefer clinical application questions over pure recall" produced sets of
|
|
vignettes: recognition without understanding. The user reads for
|
|
pathophysiology — how things work — so both generators are asked for an
|
|
even split rather than a preference, and the question generator is told
|
|
the patient is a child, which the prompt only implied in its opening line.
|
|
"""
|
|
|
|
def test_questions_ask_for_both_halves(self):
|
|
from app.services.extraction_modes import GENERATE_PROMPT
|
|
self.assertIn("about half clinical", GENERATE_PROMPT)
|
|
self.assertIn("half mechanism", GENERATE_PROMPT)
|
|
self.assertIn("paediatric", GENERATE_PROMPT)
|
|
|
|
def test_cards_ask_for_both_halves(self):
|
|
from app.services.extraction_modes import FLASHCARD_PROMPT
|
|
self.assertIn("Split them evenly", FLASHCARD_PROMPT)
|