deb8ee7830
Measured across the 323 AI drafts in the bank before touching anything. The prompt describes three readings of a topic and the second of them was not happening: short 326 sections 1 per article 1,808 chars 4 highlighted long 2,112 sections 6.5 per article 721 chars clinical 1,060 sections 531 chars So the revision view — the thing meant to be tight lists of what a candidate must know — came out as a single section two and a half times longer than a full one, written as bullets that were paragraphs, and 4 of 326 carried the ==highlight== the reader has a renderer for. "Tight lists, not prose" and "sparingly" are adjectives, and the model read them as suggestions. They are numbers now: 2 to 4 separate sections, each under 600 characters, bullets of at most 20 words, and one to three highlighted facts per section with an example of what that looks like. Two drafts written against the live model afterwards: Kawasaki disease 3 short sections, 292 chars avg, 3/3 highlighted Neonatal hypoglycaemia 3 short sections, 227 chars avg, 3/3 highlighted and the reader draws nine key-point marks on the first of them. Also, refining no longer loses a section's variant. The existing draft went to the model as a flat list of `## Heading`, so it had to guess all over again which sections were the bedside and which were the revision view — a refine could quietly move one into the other. The variant travels in the heading now and the prompt says to keep it. Left alone deliberately: the 16,000-token ceiling (no truncation failure in the logs to justify moving it) and temperature 0.4. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN
246 lines
13 KiB
Python
246 lines
13 KiB
Python
"""AI authoring endpoints on disposable SQLite; no network, no model calls."""
|
|
import json
|
|
import re
|
|
import sys
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
|
|
import test_quiz_builder as fixtures
|
|
from app.models.article import Article
|
|
from app.models.flashcard import Flashcard, FlashcardDeck, FlashcardArticleLink
|
|
from app.models.question import Question
|
|
from app.routers import articles
|
|
from app.tasks.quiz_tasks import generate_article_draft, generate_article_cards
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
DRAFT_RESPONSE = {
|
|
"title": "AI draft topic", "slug": "ai-draft-topic", "summary": "Draft summary",
|
|
"content": "Draft intro", "sections": [
|
|
{"id": "a" * 32, "slug": "first", "title": "First", "content": "Body"},
|
|
{"id": "b" * 32, "slug": "second", "title": "Second", "content": "More"},
|
|
],
|
|
}
|
|
|
|
|
|
class ArticleAiTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.bank = fixtures.BuilderTests()
|
|
self.bank.setUp()
|
|
self.client = self.bank.client
|
|
self.client.app.include_router(articles.router, prefix='/articles')
|
|
self.redis = Mock()
|
|
self.redis.from_url.return_value = self.redis
|
|
self.redis.get.return_value = None
|
|
self.redis.lrange.return_value = []
|
|
self.redis.lpush.return_value = 1
|
|
self.redis.expire.return_value = True
|
|
self.redis.incr.side_effect = [1, 21]
|
|
self.delay = patch.dict(sys.modules, {'redis': self.redis})
|
|
self.delay.start()
|
|
self.queue = patch('app.tasks.quiz_tasks.generate_article_draft.delay').start()
|
|
self.card_queue = patch('app.tasks.quiz_tasks.generate_article_cards.delay').start()
|
|
|
|
def tearDown(self):
|
|
patch.stopall()
|
|
self.bank.tearDown()
|
|
|
|
def publish(self, article_id):
|
|
return self.client.post(f'/articles/{article_id}/publish', json={'published': True})
|
|
|
|
def test_ai_endpoints_queue_and_poll(self):
|
|
self.bank.user = self.bank.mod
|
|
response = self.client.post('/articles/ai-draft', json={'topic': 'Neonatal jaundice', 'instructions': 'Two sections'})
|
|
self.assertEqual(response.status_code, 200, response.text)
|
|
job_id = response.json()['job_id']
|
|
self.queue.assert_called_once()
|
|
self.redis.get.return_value = 'completed'
|
|
poll = self.client.get(f'/articles/job/{job_id}').json()
|
|
self.assertEqual(poll['status'], 'completed')
|
|
self.bank.user = self.bank.owner
|
|
self.assertEqual(self.client.get(f'/articles/job/{job_id}').status_code, 404)
|
|
self.bank.user = self.bank.mod
|
|
article = self.client.post('/articles/', json={
|
|
"title": "Refine me", "slug": "refine-me", "content": "Intro", "sections": []}).json()
|
|
self.assertEqual(self.client.post(f"/articles/{article['id']}/ai-refine", json={'instructions': 'Shorten'}).status_code, 200)
|
|
self.assertEqual(self.client.post(f"/articles/{article['id']}/ai-cards").status_code, 200)
|
|
self.card_queue.assert_called_once()
|
|
self.bank.user = self.bank.owner
|
|
self.assertEqual(self.client.post('/articles/ai-draft', json={'topic': 'Learner'}).status_code, 403)
|
|
self.assertEqual(self.client.post(f"/articles/{article['id']}/ai-refine", json={'instructions': 'x'}).status_code, 403)
|
|
|
|
def test_draft_task_creates_and_refines_without_publishing(self):
|
|
self.redis.set.return_value = True
|
|
ai = patch('app.services.ai_service.chat').start()
|
|
ai.return_value = json.dumps(DRAFT_RESPONSE)
|
|
with patch('app.tasks.quiz_tasks.SessionLocal', sessionmaker(bind=self.bank.engine)), \
|
|
patch('app.services.ai_service.get_model_for_task', return_value=('synthetic', None)):
|
|
generate_article_draft('job-1', 3, 'Neonatal jaundice', '')
|
|
article = self.bank.db.query(Article).filter_by(slug='ai-draft-topic').one()
|
|
self.assertEqual(article.status, 'draft')
|
|
self.assertEqual([s['id'] for s in article.sections], ['a' * 32, 'b' * 32])
|
|
self.redis.set.assert_any_call('extraction:status:job-1', 'completed', ex=3600)
|
|
# Refine must send the existing body to the model, not just the title.
|
|
article.content = 'Unique draft body to preserve'
|
|
article.sections = [{'id': 'c' * 32, 'slug': 'kept', 'title': 'Kept', 'content': 'Kept body'}]
|
|
self.bank.db.commit()
|
|
generate_article_draft('job-2', 3, 'Neonatal jaundice', 'Shorten', article.id)
|
|
prompt = ai.call_args.kwargs['messages'][0]['content']
|
|
self.assertIn('Unique draft body to preserve', prompt)
|
|
self.assertIn('Kept body', prompt)
|
|
refreshed = self.bank.db.query(Article).filter_by(id=article.id).one()
|
|
self.assertEqual(refreshed.status, 'draft')
|
|
self.assertEqual(refreshed.slug, 'ai-draft-topic')
|
|
# Invalid model section ids are replaced with valid hex ids.
|
|
bad = dict(DRAFT_RESPONSE, sections=[{'id': 'bad-id', 'slug': 'bad', 'title': 'Bad', 'content': ''}])
|
|
ai.return_value = json.dumps(bad)
|
|
generate_article_draft('job-3', 3, 'Neonatal jaundice', 'Again', article.id)
|
|
fixed = self.bank.db.query(Article).filter_by(id=article.id).one()
|
|
self.assertTrue(all(re.fullmatch(r'[0-9a-f]{32}', s['id']) for s in fixed.sections), fixed.sections)
|
|
patch.stopall()
|
|
self.redis.reset_mock()
|
|
|
|
def test_cards_task_builds_private_linked_deck(self):
|
|
self.redis.set.return_value = True
|
|
self.bank.db.add(Article(slug='cards-source', title='Cards source', content='Body',
|
|
sections=[{'id': 'c' * 32, 'slug': 's', 'title': 'S', 'content': 'More'}],
|
|
user_id=3, status='published'))
|
|
self.bank.db.commit()
|
|
article = self.bank.db.query(Article).filter_by(slug='cards-source').one()
|
|
with patch('app.tasks.quiz_tasks.SessionLocal', sessionmaker(bind=self.bank.engine)), \
|
|
patch('app.services.extraction_modes.generate_flashcards',
|
|
return_value=[{'front': 'F1', 'back': 'B1'}, {'front': 'F2', 'back': 'B2', 'page_reference': 3}]), \
|
|
patch('app.services.ai_service.get_model_for_task', return_value=('synthetic', None)):
|
|
generate_article_cards('job-3', 3, article.id)
|
|
deck = self.bank.db.query(FlashcardDeck).filter_by(title='Cards: Cards source').one()
|
|
self.assertEqual(deck.is_shared, 0)
|
|
cards = self.bank.db.query(Flashcard).filter_by(deck_id=deck.id).all()
|
|
self.assertEqual(len(cards), 2)
|
|
self.assertEqual(deck.card_count, 2)
|
|
linked = self.bank.db.query(FlashcardArticleLink).filter_by(article_id=article.id).count()
|
|
self.assertEqual(linked, 2)
|
|
self.redis.set.assert_any_call('extraction:status:job-3', 'completed', ex=3600)
|
|
patch.stopall()
|
|
self.redis.reset_mock()
|
|
|
|
def test_pubmed_references_are_fields_the_reader_draws(self):
|
|
"""A paper is stored like a book: fields, not one flat sentence.
|
|
|
|
Everything else on an article is {title, author, pages}, and the reader
|
|
reads those keys — so the line PubMed used to write appeared as a blank
|
|
row under a References heading. A draft with six real papers looked
|
|
like a draft with none.
|
|
"""
|
|
from app.services import pubmed
|
|
records = [{
|
|
'pmid': '37419787', 'title': 'Management of diabetic ketoacidosis.',
|
|
'journal': 'Eur J Intern Med', 'year': '2023',
|
|
'authors': 'Barski L, Golbets E, et al',
|
|
'url': 'https://pubmed.ncbi.nlm.nih.gov/37419787/', 'abstract': '',
|
|
}]
|
|
reference, = pubmed.as_references(records)
|
|
self.assertEqual(reference['title'], 'Management of diabetic ketoacidosis')
|
|
self.assertEqual(reference['pmid'], '37419787')
|
|
self.assertEqual(reference['journal'], 'Eur J Intern Med')
|
|
self.assertEqual(reference['pages'], [])
|
|
|
|
# And a row written before this pulls itself apart on the way out,
|
|
# rather than being rewritten in the database.
|
|
healed = articles._reference_json(
|
|
'Barski L, et al. Management of diabetic ketoacidosis.. '
|
|
'Eur J Intern Med. 2023. PMID: 37419787.')
|
|
self.assertEqual(healed['pmid'], '37419787')
|
|
self.assertNotIn('PMID', healed['title'])
|
|
self.assertEqual(articles._reference_json({'title': 'A book'}), {'title': 'A book'})
|
|
|
|
def test_reading_shows_nobody_drafts(self):
|
|
"""Reading is the published library, Editorial is the unfinished work.
|
|
|
|
An educator saw their own drafts filed among the published articles
|
|
with a tag on them, which made Reading and Editorial two views of one
|
|
list. Not even an admin sees a draft here.
|
|
"""
|
|
self.bank.db.add(Article(slug='half-written', title='Half written',
|
|
content='Body', sections=[], user_id=3, status='draft'))
|
|
self.bank.db.add(Article(slug='finished', title='Finished',
|
|
content='Body', sections=[], user_id=3, status='published'))
|
|
self.bank.db.commit()
|
|
listed = self.client.get('/articles/').json()
|
|
slugs = [a['slug'] for a in listed]
|
|
self.assertIn('finished', slugs)
|
|
self.assertNotIn('half-written', slugs)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|
|
|
|
|
|
class DraftPromptTests(unittest.TestCase):
|
|
"""What the prompt asks for, measured against what 323 drafts produced.
|
|
|
|
The three readings of a topic — the full article, the high-yield revision
|
|
view, the bedside — were described and then largely ignored for the second
|
|
of them: across every AI draft in the bank, the "short" view came out as
|
|
one section of 1,808 characters (against 721 for a long one), and 4 of 326
|
|
short sections carried a highlighted fact. The revision view was the
|
|
longest prose in the article and none of it was marked.
|
|
|
|
So the instruction is a measurement rather than an adjective.
|
|
"""
|
|
|
|
def test_the_short_view_is_asked_for_in_numbers(self):
|
|
from app.tasks.quiz_tasks import ARTICLE_DRAFT_PROMPT
|
|
self.assertIn("2 to 4 SEPARATE sections", ARTICLE_DRAFT_PROMPT)
|
|
self.assertIn("under 600 characters", ARTICLE_DRAFT_PROMPT)
|
|
self.assertIn("at most 20 words", ARTICLE_DRAFT_PROMPT)
|
|
|
|
def test_highlights_are_required_and_shown(self):
|
|
from app.tasks.quiz_tasks import ARTICLE_DRAFT_PROMPT
|
|
self.assertIn("at least one and at most three highlighted facts",
|
|
ARTICLE_DRAFT_PROMPT)
|
|
# An example, because "wrap it in double equals" produced 1.2% uptake.
|
|
self.assertIn("==most common==", ARTICLE_DRAFT_PROMPT)
|
|
|
|
def test_refining_is_told_which_reading_each_section_is(self):
|
|
"""Otherwise a refine re-guesses every variant.
|
|
|
|
The existing draft was handed over as a flat list of `## Heading`, so
|
|
the model could not know which sections were the bedside and which were
|
|
the revision view — and a refine could quietly move one into the other.
|
|
"""
|
|
self.bank = fixtures.BuilderTests()
|
|
self.bank.setUp()
|
|
try:
|
|
from unittest.mock import patch
|
|
from sqlalchemy.orm import sessionmaker
|
|
from app.models.article import Article
|
|
|
|
self.bank.db.add(Article(
|
|
slug='refine-me', title='Refine me', content='Intro', user_id=None,
|
|
status='draft', sections=[
|
|
{'id': 'a' * 32, 'slug': 'bedside', 'title': 'At the bedside',
|
|
'variant': 'clinical', 'content': 'Do this'},
|
|
{'id': 'b' * 32, 'slug': 'high-yield', 'title': 'High yield',
|
|
'variant': 'short', 'content': '- A fact'},
|
|
]))
|
|
self.bank.db.commit()
|
|
article = self.bank.db.query(Article).filter_by(slug='refine-me').one()
|
|
|
|
redis = Mock()
|
|
redis.from_url.return_value = redis
|
|
redis.get.return_value = None
|
|
redis.lrange.return_value = []
|
|
with patch.dict(sys.modules, {'redis': redis}), \
|
|
patch('app.tasks.quiz_tasks.SessionLocal',
|
|
sessionmaker(bind=self.bank.engine)), \
|
|
patch('app.services.ai_service.get_model_for_task',
|
|
return_value=('synthetic', None)), \
|
|
patch('app.services.ai_service.chat') as ai:
|
|
ai.return_value = json.dumps(DRAFT_RESPONSE)
|
|
generate_article_draft('job-refine', 3, 'Refine me', '', article.id)
|
|
prompt = ai.call_args.kwargs['messages'][0]['content']
|
|
self.assertIn('## [clinical] At the bedside', prompt)
|
|
self.assertIn('## [short] High yield', prompt)
|
|
self.assertIn("must be kept", prompt)
|
|
finally:
|
|
patch.stopall()
|
|
self.bank.tearDown()
|