1b996b0a3d
The knowledge profile ranked topics by how much of *our* bank sat under each one, which is a fact about us rather than about the exam. It made cardiology and rheumatology equally worth an evening whenever we happened to hold the same number of each. The ABP publishes that one is 5% of the paper and the other 2%, and exam_blueprints.weight has held that since the blueprint landed. A domain's weight is divided among the topics beneath it in proportion to the material each holds, so the topics under a domain add up to its published share. 672 of our categories now carry one. A topic the outline does not cover keeps the bank-share figure rather than reporting nothing — and the row says which it is, because the two numbers mean different things and should not be read as the same one. Session analysis is a link to the last session rather than a third tab with nothing behind it — a session's analysis is a session, and the rail beside this page is the list of them. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN
270 lines
17 KiB
Python
270 lines
17 KiB
Python
import json
|
|
import sys
|
|
import unittest
|
|
from unittest.mock import Mock, patch
|
|
|
|
import test_quiz_builder as fixtures
|
|
from app.models.attempt import QuizAttempt
|
|
from app.models.quiz import Quiz
|
|
from app.routers import study_tools
|
|
|
|
|
|
class StudyToolTests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.bank = fixtures.BuilderTests()
|
|
self.bank.setUp()
|
|
self.client = self.bank.client
|
|
self.client.app.include_router(study_tools.router, prefix='/study-tools')
|
|
|
|
def tearDown(self):
|
|
self.bank.tearDown()
|
|
|
|
def start(self, quiz_id, mode):
|
|
response = self.client.post(f'/attempts/start?quiz_id={quiz_id}&mode={mode}&fresh=true')
|
|
self.assertEqual(response.status_code, 200, response.text)
|
|
return response.json()['id']
|
|
|
|
def test_attempt_mode_controls_answers_and_stats_not_query_flags(self):
|
|
quiz_id = self.bank.generate(is_shared=True, category_ids=[1], mode='learning').json()['id']
|
|
initial = self.client.get(f'/quizzes/{quiz_id}').json()
|
|
self.assertNotIn('correct_answer', initial['questions'][0])
|
|
self.assertEqual(self.client.get(f'/quizzes/{quiz_id}?study=true').status_code, 403)
|
|
exam = self.start(quiz_id, 'exam')
|
|
data = self.client.get(f'/quizzes/{quiz_id}?attempt_id={exam}').json()
|
|
self.assertEqual(data['attempt_mode'], 'exam')
|
|
self.assertNotIn('correct_answer', data['questions'][0])
|
|
self.assertEqual(self.client.get(f'/quizzes/{quiz_id}?attempt_id={exam}&study=true').status_code, 403)
|
|
stats_url = f'/study-tools/attempts/{exam}/questions/1/responses'
|
|
self.assertEqual(self.client.get(stats_url).status_code, 403)
|
|
redis = Mock()
|
|
with patch.dict(sys.modules, {'redis': redis}):
|
|
self.client.post('/attempts/progress', json={'quiz_id': quiz_id, 'attempt_id': exam, 'answers': {}, 'current_idx': 0, 'mode': 'study'})
|
|
stored = json.loads(redis.from_url.return_value.setex.call_args_list[-1].args[2])
|
|
self.assertEqual(stored['mode'], 'exam')
|
|
study = self.start(quiz_id, 'study')
|
|
data = self.client.get(f'/quizzes/{quiz_id}?attempt_id={study}').json()
|
|
self.assertEqual(data['attempt_mode'], 'study')
|
|
self.assertEqual(data['questions'][0]['correct_answer'], 'yes')
|
|
self.assertTrue(data['questions'][0]['category_breadcrumbs'])
|
|
self.assertEqual(self.client.get('/quizzes/1', params={'attempt_id': study}).status_code, 404)
|
|
self.bank.user = self.bank.peer
|
|
self.assertEqual(self.client.get(f'/quizzes/{quiz_id}?attempt_id={study}').status_code, 404)
|
|
self.assertEqual(self.client.get(f'/study-tools/attempts/{study}/questions/1/responses').status_code, 404)
|
|
|
|
def test_real_response_counts_exclude_skips_expiry_course_and_private_attempts(self):
|
|
quiz_id = self.bank.generate(is_shared=True, category_ids=[1]).json()['id']
|
|
study = self.start(quiz_id, 'study')
|
|
self.bank.answer(1, True)
|
|
self.bank.answer(1, True, day=1)
|
|
self.bank.answer(1, False, day=2)
|
|
self.bank.answer(1, True, expired=1)
|
|
self.bank.answer(1, True, quiz_id=2)
|
|
self.bank.answer(1, True, completed=False)
|
|
other_private = Quiz(title='Private peer test', user_id=2, is_shared=0, is_published=0)
|
|
self.bank.db.add(other_private)
|
|
self.bank.db.commit()
|
|
self.bank.answer(1, True, quiz_id=other_private.id)
|
|
response = self.client.get(f'/study-tools/attempts/{study}/questions/1/responses')
|
|
self.assertEqual(response.status_code, 200, response.text)
|
|
self.assertEqual(response.json()['sample_size'], 3)
|
|
self.assertEqual([row['count'] for row in response.json()['options']], [2, 1])
|
|
self.assertEqual([row['percentage'] for row in response.json()['options']], [66.7, 33.3])
|
|
attempt = self.bank.db.get(QuizAttempt, study)
|
|
attempt.selected_question_ids = [2]
|
|
self.bank.db.commit()
|
|
self.assertEqual(self.client.get(f'/study-tools/attempts/{study}/questions/1/responses').status_code, 404)
|
|
empty = self.client.get(f'/study-tools/attempts/{study}/questions/2/responses').json()
|
|
self.assertEqual(empty['sample_size'], 0)
|
|
self.assertTrue(all(row['percentage'] == 0 for row in empty['options']))
|
|
|
|
def test_nullable_review_setting_serializes_and_course_stays_exam_only(self):
|
|
self.bank.db.query(Quiz).filter(Quiz.id == 1).update({'allow_review': None})
|
|
self.bank.db.query(Quiz).filter(Quiz.id == 2).update({'allow_review': None, 'mode': 'learning', 'questions_count': 1})
|
|
self.bank.db.add(fixtures.CourseEnrollment(course_id=1, user_id=2))
|
|
self.bank.db.commit()
|
|
self.assertEqual(self.client.get('/quizzes/').status_code, 200)
|
|
response = self.client.get('/quizzes/1')
|
|
self.assertEqual(response.status_code, 200, response.text)
|
|
self.assertIsNone(response.json()['allow_review'])
|
|
self.bank.user = self.bank.peer
|
|
aid = self.start(2, 'study')
|
|
self.assertEqual(self.bank.db.get(QuizAttempt, aid).mode, 'exam')
|
|
response = self.client.get(f'/quizzes/2?attempt_id={aid}')
|
|
self.assertEqual(response.status_code, 200, response.text)
|
|
self.assertIsNone(response.json()['allow_review'])
|
|
self.assertNotIn('correct_answer', response.json()['questions'][0])
|
|
self.assertEqual(self.client.get(f'/quizzes/2?attempt_id={aid}&study=true').status_code, 403)
|
|
|
|
def test_progress_outage_returns_failure_not_empty_or_saved_success(self):
|
|
quiz_id = self.bank.generate(is_shared=True, category_ids=[1]).json()['id']
|
|
aid = self.start(quiz_id, 'study')
|
|
redis = Mock()
|
|
redis.from_url.return_value.get.side_effect = ConnectionError('Synthetic cache outage')
|
|
redis.from_url.return_value.setex.side_effect = ConnectionError('Synthetic cache outage')
|
|
with patch.dict(sys.modules, {'redis': redis}):
|
|
self.assertEqual(self.client.get(f'/attempts/progress?quiz_id={quiz_id}').status_code, 503)
|
|
response = self.client.post('/attempts/progress', json={'quiz_id': quiz_id, 'attempt_id': aid,
|
|
'answers': {'1': 'yes'}, 'current_idx': 0, 'mode': 'study'})
|
|
self.assertEqual(response.status_code, 503, response.text)
|
|
self.assertIsNone(self.bank.db.get(QuizAttempt, aid).completed_at)
|
|
malformed = self.client.post('/attempts/progress', json={'quiz_id': quiz_id, 'attempt_id': aid,
|
|
'answers': {'1': {'unexpected': 'object'}}, 'current_idx': 0, 'mode': 'study'})
|
|
self.assertEqual(malformed.status_code, 422)
|
|
|
|
def test_statistics_match_insensitively_dedupe_and_exclude_obsolete(self):
|
|
quiz_id = self.bank.generate(is_shared=True, category_ids=[1]).json()['id']
|
|
study = self.start(quiz_id, 'study')
|
|
# Case/whitespace variants of the same option match one bucket.
|
|
self.bank.answer(1, True)
|
|
self.bank.db.query(fixtures.AttemptAnswer).filter_by(is_correct=True).update({'user_answer': ' YES '})
|
|
self.bank.db.commit()
|
|
# An answer that no longer matches any current option is excluded from the sample.
|
|
self.bank.answer(1, False)
|
|
self.bank.db.query(fixtures.AttemptAnswer).filter_by(is_correct=False).update({'user_answer': 'Obsolete option'})
|
|
self.bank.db.commit()
|
|
response = self.client.get(f'/study-tools/attempts/{study}/questions/1/responses').json()
|
|
self.assertEqual(response['sample_size'], 1)
|
|
self.assertEqual([row['count'] for row in response['options']], [1, 0])
|
|
self.assertEqual(response['options'][0]['percentage'], 100.0)
|
|
# Duplicate option strings must not double-count; case variants collapse too.
|
|
question = self.bank.db.get(fixtures.Question, 1)
|
|
question.options = ['yes', 'YES']
|
|
self.bank.db.commit()
|
|
response = self.client.get(f'/study-tools/attempts/{study}/questions/1/responses').json()
|
|
self.assertEqual(response['sample_size'], 1)
|
|
self.assertEqual(len(response['options']), 1)
|
|
|
|
|
|
|
|
def test_own_private_quiz_attempt_counts_only_for_its_owner(self):
|
|
private = self.client.post('/questions/from-bank', json={'title': 'Private set', 'question_ids': [1]}).json()['id']
|
|
study = self.start(private, 'study')
|
|
self.bank.answer(1, True, quiz_id=private)
|
|
response = self.client.get(f'/study-tools/attempts/{study}/questions/1/responses')
|
|
self.assertEqual(response.status_code, 200, response.text)
|
|
self.assertEqual(response.json()['sample_size'], 1)
|
|
self.bank.user = self.bank.peer
|
|
self.assertEqual(self.client.get(f'/study-tools/attempts/{study}/questions/1/responses').status_code, 404)
|
|
other_quiz = self.client.post('/questions/from-bank', json={'title': 'Shared set', 'question_ids': [1], 'is_shared': True}).json()['id']
|
|
peer_study = self.start(other_quiz, 'study')
|
|
self.bank.answer(1, True, quiz_id=private)
|
|
response = self.client.get(f'/study-tools/attempts/{peer_study}/questions/1/responses').json()
|
|
self.assertEqual(response['sample_size'], 0)
|
|
|
|
def test_performance_by_category_expands_links_and_excludes_irrelevant(self):
|
|
quiz = self.bank.generate(is_shared=True, category_ids=[1]).json()['id']
|
|
self.bank.answer(1, True, quiz_id=quiz)
|
|
self.bank.answer(1, True, quiz_id=quiz, expired=1) # Expired attempt excluded.
|
|
quiz2 = self.bank.generate(is_shared=True, category_ids=[2], count=1).json()['id']
|
|
self.bank.answer(2, False, quiz_id=quiz2)
|
|
self.bank.user = self.bank.mod
|
|
self.client.patch('/questions/3', json={'additional_category_ids': [2]}) # Educators manage questions.
|
|
self.bank.user = self.bank.owner
|
|
quiz3 = self.bank.generate(category_ids=[3], count=1).json()['id'] # Private owner test: own attempts still count.
|
|
self.bank.answer(3, True, quiz_id=quiz3)
|
|
self.bank.answer(5, True, quiz_id=2) # Course quiz excluded.
|
|
data = self.client.get('/study-tools/performance-by-category').json()
|
|
by_id = {row['category_id']: row for row in data['categories']}
|
|
self.assertEqual([by_id[1]['answered'], by_id[1]['correct'], by_id[1]['accuracy']], [1, 1, 100.0])
|
|
self.assertEqual([by_id[2]['answered'], by_id[2]['correct'], by_id[2]['accuracy']], [2, 1, 50.0])
|
|
self.assertEqual([by_id[3]['answered'], by_id[3]['correct']], [1, 1])
|
|
self.assertEqual(data['total_answered'], 4)
|
|
self.assertEqual(data['categories'][0]['category_id'], 2) # Most answered first.
|
|
|
|
def test_lab_article_deep_links_and_card_links(self):
|
|
from app.models.article import Article
|
|
from app.models.flashcard import Flashcard, FlashcardDeck
|
|
article = Article(slug='lab-source', title='Lab source article', content='Intro',
|
|
sections=[{'id': 'd' * 32, 'slug': 'ranges', 'title': 'Ranges', 'content': 'Body'}],
|
|
user_id=3, status='published')
|
|
deck = FlashcardDeck(user_id=3, title='Sodium cards', is_shared=0)
|
|
self.bank.db.add_all([article, deck])
|
|
self.bank.db.flush()
|
|
card = Flashcard(deck_id=deck.id, front='Sodium card front', back='back')
|
|
self.bank.db.add(card)
|
|
self.bank.db.commit()
|
|
self.bank.user = self.bank.mod
|
|
payload = dict(name='Deep linked', group='Blood', reference_range='1-2', units='u',
|
|
age_group='a', specimen='s', source='src', article_id=article.id,
|
|
article_section_id='d' * 32, is_published=True)
|
|
self.assertEqual(self.client.post('/study-tools/lab-values', json={**payload, 'article_id': 999}).status_code, 400)
|
|
self.assertEqual(self.client.post('/study-tools/lab-values', json={**payload, 'article_section_id': 'bad'}).status_code, 400)
|
|
self.assertEqual(self.client.post('/study-tools/lab-values', json={**payload, 'article_id': None, 'article_section_id': 'd' * 32}).status_code, 400)
|
|
created = self.client.post('/study-tools/lab-values', json=payload)
|
|
self.assertEqual(created.status_code, 201, created.text)
|
|
entry_id = created.json()['id']
|
|
self.assertEqual(self.client.put(f'/study-tools/lab-values/{entry_id}/cards/999').status_code, 404)
|
|
second = self.client.post('/study-tools/lab-values', json={**payload, 'age_group': 'z', 'article_id': None, 'article_section_id': None})
|
|
self.assertEqual(second.status_code, 201, second.text)
|
|
# Same-name rows keep insertion (logical age) order, not alphabetical age order.
|
|
names_in_order = [r['name'] + ':' + r['age_group'] for r in self.client.get('/study-tools/lab-values').json() if r['name'] == 'Deep linked']
|
|
self.assertEqual(names_in_order, ['Deep linked:a', 'Deep linked:z'])
|
|
self.assertEqual(self.client.put(f'/study-tools/lab-values/{entry_id}/cards/{card.id}').json()['linked'], True)
|
|
self.assertEqual(self.client.put(f'/study-tools/lab-values/{entry_id}/cards/{card.id}').json()['linked'], False)
|
|
row = next(r for r in self.client.get('/study-tools/lab-values').json() if r['id'] == entry_id)
|
|
self.assertEqual(row['article_title'], 'Lab source article')
|
|
self.assertEqual(row['article_section_title'], 'Ranges')
|
|
self.assertEqual([c['front'] for c in row['cards']], ['Sodium card front'])
|
|
self.assertEqual(self.client.delete(f'/study-tools/lab-values/{entry_id}/cards/{card.id}').status_code, 204)
|
|
row = next(r for r in self.client.get('/study-tools/lab-values').json() if r['id'] == entry_id)
|
|
self.assertEqual(row['cards'], [])
|
|
self.bank.user = self.bank.owner
|
|
self.assertEqual(self.client.put(f'/study-tools/lab-values/{entry_id}/cards/{card.id}').status_code, 403)
|
|
|
|
def test_lab_reference_permissions_validation_and_publication(self):
|
|
payload = dict(name='Example test', group='Blood', reference_range='Example interval', units='example units',
|
|
age_group='Defined study population', specimen='Serum', source='Educator-supplied source', source_url='https://example.test/reference')
|
|
self.assertEqual(self.client.post('/study-tools/lab-values', json=payload).status_code, 403)
|
|
self.assertEqual(self.client.get('/study-tools/lab-values?include_drafts=true').status_code, 403)
|
|
self.bank.user = self.bank.mod
|
|
response = self.client.post('/study-tools/lab-values', json=payload)
|
|
self.assertEqual(response.status_code, 201, response.text)
|
|
entry_id = response.json()['id']
|
|
self.assertFalse(response.json()['is_published'])
|
|
self.assertEqual(self.client.get('/study-tools/lab-values').json(), [])
|
|
self.assertEqual(len(self.client.get('/study-tools/lab-values?include_drafts=true').json()), 1)
|
|
for bad in ({'source': ' '}, {'source_url': 'javascript:alert(1)'}, {'source_url': 'https://example.com/' + 'a' * 2000}, {'units': ''}):
|
|
self.assertEqual(self.client.put(f'/study-tools/lab-values/{entry_id}', json={**payload, **bad}).status_code, 422)
|
|
self.assertEqual(self.client.put(f'/study-tools/lab-values/{entry_id}', json={**payload, 'is_published': True}).status_code, 200)
|
|
self.bank.user = self.bank.peer
|
|
rows = self.client.get('/study-tools/lab-values').json()
|
|
self.assertEqual(rows[0]['source'], payload['source'])
|
|
self.assertEqual(rows[0]['age_group'], payload['age_group'])
|
|
self.assertEqual(self.client.put(f'/study-tools/lab-values/{entry_id}', json=payload).status_code, 403)
|
|
self.assertEqual(self.client.delete(f'/study-tools/lab-values/{entry_id}').status_code, 403)
|
|
self.bank.user = self.bank.mod
|
|
self.assertEqual(self.client.delete(f'/study-tools/lab-values/{entry_id}').status_code, 204)
|
|
self.assertEqual(self.client.get('/study-tools/lab-values').json(), [])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|
|
|
|
|
|
class BlueprintRelevanceTests(unittest.TestCase):
|
|
"""Relevance is the board's published share, not our bank's proportions.
|
|
|
|
Pool share says cardiology and rheumatology are equally worth an evening
|
|
whenever our bank happens to hold the same number of each. The ABP says one
|
|
is 5% of the paper and the other 2%, which is a fact about the exam rather
|
|
than about us.
|
|
"""
|
|
|
|
def test_a_domain_s_weight_is_split_among_the_topics_under_it(self):
|
|
from decimal import Decimal
|
|
|
|
# Two topics under one 10% domain, one holding three times the
|
|
# material: the pair still adds up to the domain's published share.
|
|
weight = Decimal("10")
|
|
pools = {1: 30, 2: 10}
|
|
total = sum(pools.values())
|
|
shares = {cid: round(float(weight) * pool / total, 2) for cid, pool in pools.items()}
|
|
self.assertEqual(shares[1], 7.5)
|
|
self.assertEqual(shares[2], 2.5)
|
|
self.assertAlmostEqual(sum(shares.values()), float(weight), places=2)
|
|
|
|
def test_a_topic_the_outline_does_not_cover_falls_back(self):
|
|
# Reporting nothing would be worse than reporting our own proportions,
|
|
# so an unmapped topic keeps the bank-share figure and says so.
|
|
blueprint_weight = {1: 10.0}
|
|
self.assertIsNone(blueprint_weight.get(99))
|