dd71bed184
analysis is a real tab Adaptive selection knew what you were weak at and nothing about what the exam is made of, so being weak at something worth 5% of the paper ranked the same as being weak at something worth 1%. Every score is now multiplied by the weight the board publishes for that topic's domain — the same `exam_blueprints.weight` behind the Relevance column. A topic the blueprint does not cover takes the median published weight. A zero would make unmapped material unreachable and the highest would make it the priority; neither is a claim the blueprint supports. With no study objective the multiplier is absent and selection is about weakness alone, exactly as before. Weight scales weakness, it does not replace it: a topic you are certain of does not surface because it is worth 5% of the paper, because (1 − accuracy) is near zero and no multiplier rescues that. docs/adaptive- sessions.md says all of this, including what is still open. Session analysis is the third tab rather than a link out of the page — two of the three used to change what you were reading and the third took you somewhere else. The tab bar is one component both routes wear, AnalysisSessionPage's body is a component the tab renders in place, and the tab lives in the address so a link opens where it says. Two things that were wrong turned up in that work: a session nobody had sat showed 0% in the figures and "0% correct" in the donut — two separate statements of a score on a session that had none — and the old third tab disappeared entirely for anyone with no attempts, so the strip silently changed shape. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN
193 lines
9.2 KiB
Python
193 lines
9.2 KiB
Python
"""An exam's published shape, and drawing a paper that matches it.
|
||
|
||
The arithmetic is the part worth pinning. Twenty-four percentages rounded
|
||
independently do not add up to forty questions, and a domain with a thin corner
|
||
of the bank behind it must give its shortfall back rather than shortening the
|
||
paper.
|
||
"""
|
||
import random
|
||
import unittest
|
||
from unittest.mock import patch
|
||
from decimal import Decimal
|
||
|
||
from sqlalchemy import create_engine
|
||
from sqlalchemy.orm import Session
|
||
from sqlalchemy.pool import StaticPool
|
||
|
||
import test_quiz_builder as fixtures # noqa: F401 — imports every model, so the metadata resolves
|
||
from app.database import Base
|
||
from app.models.exam import BlueprintCategoryLink, Exam, ExamBlueprint, QuestionExamLink
|
||
from app.models.question import Question
|
||
from app.models.question_category import QuestionCategory
|
||
from app.services import quiz_builder
|
||
from app.models.user import User
|
||
from app.services import exam_blueprint
|
||
|
||
|
||
class AllocationTests(unittest.TestCase):
|
||
"""No database: this is the arithmetic on its own."""
|
||
|
||
def test_shares_add_up_to_the_whole_paper(self):
|
||
# Twenty-four domains of the real outline, rounded independently, do
|
||
# not come to forty. Largest remainder is why the block is never short.
|
||
weights = {i: Decimal(w) for i, w in enumerate(
|
||
[12, 4, 5, 7, 6, 3, 4, 3, 4, 6, 5, 5, 5, 4, 4, 3, 3, 3, 3, 3, 2, 2, 2, 2])}
|
||
plan = exam_blueprint.allocate(weights, 40, {i: 999 for i in weights})
|
||
self.assertEqual(sum(plan.values()), 40)
|
||
self.assertEqual(plan[0], 5) # 12% of 40 is 4.8, and the remainder carries it
|
||
self.assertEqual(plan[23], 1) # 2% of 40 is 0.8 — still worth a question
|
||
|
||
def test_a_thin_domain_gives_its_shortfall_back(self):
|
||
weights = {1: Decimal(50), 2: Decimal(50)}
|
||
plan = exam_blueprint.allocate(weights, 20, {1: 3, 2: 100})
|
||
self.assertEqual(plan[1], 3)
|
||
# The paper stays twenty questions long; it is its accuracy that suffers.
|
||
self.assertEqual(sum(plan.values()), 20)
|
||
|
||
def test_overflow_lands_where_the_exam_is_heaviest(self):
|
||
weights = {1: Decimal(10), 2: Decimal(60), 3: Decimal(30)}
|
||
plan = exam_blueprint.allocate(weights, 10, {1: 0, 2: 100, 3: 100})
|
||
self.assertNotIn(1, plan)
|
||
self.assertEqual(sum(plan.values()), 10)
|
||
self.assertGreater(plan[2], plan[3])
|
||
|
||
def test_nothing_to_draw_from_returns_nothing_rather_than_looping(self):
|
||
weights = {1: Decimal(100)}
|
||
self.assertEqual(exam_blueprint.allocate(weights, 10, {1: 0}), {})
|
||
self.assertEqual(exam_blueprint.allocate({}, 10, {}), {})
|
||
self.assertEqual(exam_blueprint.allocate(weights, 0, {1: 5}), {})
|
||
|
||
|
||
class BlueprintTests(unittest.TestCase):
|
||
def setUp(self):
|
||
self.engine = create_engine("sqlite://", connect_args={"check_same_thread": False},
|
||
poolclass=StaticPool)
|
||
Base.metadata.create_all(self.engine)
|
||
self.db = Session(self.engine)
|
||
self.db.add(User(id=1, name="Mod", email="m@example.test",
|
||
hashed_password="unused", role="moderator"))
|
||
self.exam = Exam(id=1, slug="boards", name="Boards")
|
||
self.db.add(self.exam)
|
||
# Cardiology holds its questions two levels down, which is the case the
|
||
# naive count got wrong: filed on the root, nothing would be found.
|
||
self.db.add_all([
|
||
QuestionCategory(id=1, name="Cardiology", user_id=1),
|
||
QuestionCategory(id=2, name="Congenital", parent_id=1, user_id=1),
|
||
QuestionCategory(id=3, name="Tetralogy", parent_id=2, user_id=1),
|
||
QuestionCategory(id=4, name="Dermatology", user_id=1),
|
||
])
|
||
self.db.flush()
|
||
for qid in range(1, 21):
|
||
category = 3 if qid <= 15 else 4
|
||
self.db.add(Question(id=qid, question_category_id=category, user_id=1,
|
||
question_text=f"Q{qid}", question_type="mcq",
|
||
options=["a", "b"], correct_answer="a"))
|
||
self.db.add(QuestionExamLink(question_id=qid, exam_id=1))
|
||
self.cardio = ExamBlueprint(id=1, exam_id=1, code="11", title="Cardiology",
|
||
weight=Decimal("75"), sort_order=0)
|
||
self.derm = ExamBlueprint(id=2, exam_id=1, code="15", title="Skin",
|
||
weight=Decimal("25"), sort_order=1)
|
||
self.db.add_all([self.cardio, self.derm])
|
||
self.db.flush()
|
||
self.db.add_all([BlueprintCategoryLink(blueprint_id=1, category_id=1),
|
||
BlueprintCategoryLink(blueprint_id=2, category_id=4)])
|
||
self.db.commit()
|
||
|
||
def tearDown(self):
|
||
self.db.close()
|
||
self.engine.dispose()
|
||
|
||
def test_a_domain_counts_everything_beneath_its_categories(self):
|
||
rows = {row["code"]: row for row in exam_blueprint.coverage(self.db, 1)}
|
||
# Mapped to the Cardiology root; the questions are on a grandchild.
|
||
self.assertEqual(rows["11"]["questions"], 15)
|
||
self.assertEqual(rows["15"]["questions"], 5)
|
||
|
||
def test_subdomains_carry_no_weight_of_their_own(self):
|
||
self.db.add(ExamBlueprint(id=3, exam_id=1, parent_id=1, code="11.A",
|
||
title="Congenital", weight=None))
|
||
self.db.commit()
|
||
self.assertEqual([line.code for line in exam_blueprint.domains(self.db, 1)], ["11", "15"])
|
||
self.assertEqual(exam_blueprint.total_weight(self.db, 1), 100.0)
|
||
|
||
def test_a_paper_comes_out_shaped_like_the_blueprint(self):
|
||
ids, report = exam_blueprint.sample(self.db, 1, 8, rng=random.Random(1))
|
||
self.assertEqual(len(ids), 8)
|
||
self.assertEqual(len(set(ids)), 8, "no question twice")
|
||
given = {row["code"]: row["given"] for row in report}
|
||
self.assertEqual(given["11"], 6) # 75% of 8
|
||
self.assertEqual(given["15"], 2) # 25% of 8
|
||
|
||
def test_an_unmapped_domain_is_named_rather_than_silently_ignored(self):
|
||
self.db.add(ExamBlueprint(id=4, exam_id=1, code="6", title="Psychosocial",
|
||
weight=Decimal("10"), sort_order=2))
|
||
self.db.commit()
|
||
rows = {row["code"]: row for row in exam_blueprint.coverage(self.db, 1)}
|
||
self.assertEqual(rows["6"]["questions"], 0)
|
||
self.assertEqual(rows["6"]["category_ids"], [])
|
||
|
||
def test_an_exam_with_no_blueprint_draws_nothing_rather_than_guessing(self):
|
||
other = Exam(id=2, slug="step", name="Step")
|
||
self.db.add(other)
|
||
self.db.commit()
|
||
self.assertEqual(exam_blueprint.sample(self.db, 2, 40), ([], []))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|
||
|
||
|
||
class AdaptiveWeightedByTheBlueprintTests(unittest.TestCase):
|
||
"""Weakness is scaled by how much of the real paper a topic accounts for.
|
||
|
||
Being weak at something worth 5% of the exam is worth more study than being
|
||
equally weak at something worth 1%. Until this was added the two ranked the
|
||
same, because selection knew about the learner and nothing about the paper.
|
||
"""
|
||
|
||
def setUp(self):
|
||
self.bank = fixtures.BuilderTests()
|
||
self.bank.setUp()
|
||
self.db = self.bank.db
|
||
|
||
def tearDown(self):
|
||
self.bank.tearDown()
|
||
|
||
def weigh(self, mapping):
|
||
"""Patch the blueprint so the test states the weights it is about."""
|
||
return patch.object(quiz_builder, "blueprint_weights", lambda db, user: mapping)
|
||
|
||
def test_the_heavier_topic_is_drawn_first_when_both_are_untouched(self):
|
||
# Categories 1 and 2 both unanswered, so weakness is equal at 0.5.
|
||
# Question 1 sits under category 1, question 2 under category 2.
|
||
with self.weigh({1: 10.0, 2: 1.0}):
|
||
ids = quiz_builder.adaptive_select(self.db, self.bank.owner, 1, [], "all", None)
|
||
self.assertEqual(ids, [1])
|
||
|
||
with self.weigh({1: 1.0, 2: 10.0}):
|
||
ids = quiz_builder.adaptive_select(self.db, self.bank.owner, 1, [], "all", None)
|
||
self.assertEqual(ids, [2])
|
||
|
||
def test_weakness_still_beats_weight_when_it_is_lopsided(self):
|
||
# Right in the heavy topic, wrong in the light one: the light topic is
|
||
# where the learner actually is weak, and 10× the weight does not undo
|
||
# being certain about the other.
|
||
self.bank.answer(1, True, day=0)
|
||
self.bank.answer(2, False, day=1)
|
||
with self.weigh({1: 10.0, 2: 1.0}):
|
||
ids = quiz_builder.adaptive_select(self.db, self.bank.owner, 1, [], "all", None)
|
||
# 3 and 6 are unanswered, so they come before anything recycled.
|
||
self.assertTrue(set(ids) <= {3, 6}, ids)
|
||
|
||
def test_a_topic_the_blueprint_does_not_cover_is_neither_promoted_nor_lost(self):
|
||
# Only category 1 is mapped. Category 2 takes the median, so a question
|
||
# under it is still reachable — a zero would make it invisible.
|
||
with self.weigh({1: 4.0}):
|
||
ids = quiz_builder.adaptive_select(self.db, self.bank.owner, 4, [], "all", None)
|
||
self.assertIn(2, ids)
|
||
|
||
def test_with_no_study_objective_it_is_about_weakness_alone(self):
|
||
# No exam, no weights: selection is what it always was.
|
||
self.assertEqual(quiz_builder.blueprint_weights(self.db, self.bank.owner), {})
|
||
ids = quiz_builder.adaptive_select(self.db, self.bank.owner, 2, [], "all", None)
|
||
self.assertEqual(len(ids), 2)
|