diff --git a/backend/app/services/quiz_builder.py b/backend/app/services/quiz_builder.py index 4bfb8df..9d761c5 100644 --- a/backend/app/services/quiz_builder.py +++ b/backend/app/services/quiz_builder.py @@ -354,6 +354,49 @@ PRIOR_ANSWERS = 2.0 #: How fast a topic's priority falls as the session keeps drawing from it. CATEGORY_DAMPING = 0.5 +#: The three labels, in order, and the readiness at which each becomes the +#: right level to be working at. +#: +#: A learner getting a third of a topic right is not helped by its hardest +#: questions — they will miss those too and learn nothing from the miss — and +#: one getting nine in ten right is not helped by its easiest. The bands are +#: readiness, which is shrunk accuracy, so a topic answered twice sits near +#: 0.5 and gets medium: the middle is also the honest default when almost +#: nothing is known about somebody. +DIFFICULTY_ORDER = ("easy", "medium", "hard") +#: 0.45 and not 0.55 for the first edge: a topic with no evidence sits at +#: exactly NEUTRAL_RECALL, and a boundary above that would hand every untouched +#: topic its easiest questions — which is a poor way to find out what somebody +#: knows and a slow way to start. +DIFFICULTY_BANDS = ((0.45, "easy"), (0.78, "medium"), (1.01, "hard")) + +#: What a question at the wrong level is still worth. One step away is most of +#: the value — a medium question is a perfectly good thing to answer while you +#: are on easy ones — and two steps is half. +#: +#: A multiplier and not a filter, deliberately, for the same reason the +#: reranker is a permutation: a bank thinned to one difficulty is a bank three +#: times smaller, and on a narrow topic that means the same eight questions +#: every time. Nothing is ever excluded for being the wrong level; it is +#: preferred against. +DIFFICULTY_FIT = (1.0, 0.72, 0.5) + +#: And what an unlabelled question is worth. Just under a perfect fit, so a +#: labelled question at the right level wins a tie, and comfortably above a +#: two-step miss — an unknown difficulty is unknown, not wrong. Every question +#: in the bank carries a label today; this is for what is written tomorrow. +UNRATED_FIT = 0.92 + + +def target_difficulty(readiness: float | None) -> str: + """The level a learner working at this readiness should be asked at.""" + if readiness is None: + return "medium" + for ceiling, level in DIFFICULTY_BANDS: + if readiness < ceiling: + return level + return "hard" + def recency_weight(age_days: float) -> float: """How much evidence that old still counts for.""" @@ -416,6 +459,11 @@ class CandidateRanking: self.rows = sorted(((row[0], row[1]) for row in query.with_entities( Question.id, Question.question_category_id).all()), key=lambda row: row[0]) self.category_of = dict(self.rows) + #: question id → "easy" | "medium" | "hard", where one is recorded. A + #: third integer column per question, fetched with the other two. + self.difficulty_of = { + row[0]: row[1] for row in query.with_entities( + Question.id, Question.difficulty).all() if row[1]} answered = db.query(AttemptAnswer.question_id, AttemptAnswer.is_correct, QuizAttempt.completed_at).join( QuizAttempt, QuizAttempt.id == AttemptAnswer.attempt_id).join(Quiz, Quiz.id == QuizAttempt.quiz_id).filter( @@ -494,6 +542,28 @@ class CandidateRanking: """How much a question here could move the score.""" return (1 - self.accuracy(category)) * self.weight(category) + def difficulty_fit(self, row) -> float: + """How well this question's level suits where the learner is on its topic. + + This is the half of "adaptive" that was missing: until now difficulty + was a filter a learner could set and nothing the session did on its + own, so somebody at 30% on a topic and somebody at 90% were asked the + same questions in the same order. + + Measured per topic rather than overall, because a learner is not one + level: strong on growth and weak on arrhythmias is the normal case, and + an average across the two describes nobody. + """ + level = self.difficulty_of.get(row[0]) + if level is None: + return UNRATED_FIT + wanted = target_difficulty(self.accuracy(row[1])) + try: + distance = abs(DIFFICULTY_ORDER.index(level) - DIFFICULTY_ORDER.index(wanted)) + except ValueError: + return UNRATED_FIT + return DIFFICULTY_FIT[min(distance, len(DIFFICULTY_FIT) - 1)] + def _unseen_value(self, row) -> float: return self.impact(row[1]) @@ -501,8 +571,14 @@ class CandidateRanking: return (1 - self.recall[row[0]]) * self.weight(row[1]) def value(self, row) -> float: - """What one candidate is worth, whichever pool it came from.""" - return self._review_value(row) if row[0] in self.recall else self._unseen_value(row) + """What one candidate is worth, whichever pool it came from. + + Topic first, level second. Which topic to spend a question on is the + decision that moves a score; how hard that question should be is how + the session meets the learner where they are inside it. + """ + base = self._review_value(row) if row[0] in self.recall else self._unseen_value(row) + return base * self.difficulty_fit(row) def review_budget(self, count: int) -> int: """Slots this session gives to questions already seen.""" diff --git a/backend/tests/test_quiz_builder.py b/backend/tests/test_quiz_builder.py index cf407e8..52ca5b5 100644 --- a/backend/tests/test_quiz_builder.py +++ b/backend/tests/test_quiz_builder.py @@ -460,6 +460,47 @@ class AdaptiveSelectionTests(unittest.TestCase): # the queue the way a raw 0% would. self.assertLess(ranking().impact(1), ranking().weight(1)) + def test_the_level_follows_the_learner_topic_by_topic(self): + """The half of "adaptive" that was missing until the bank had labels. + + Difficulty was a filter a learner could set and nothing the session did + on its own, so somebody at 30% on a topic and somebody at 90% were + asked the same questions in the same order. + """ + from app.services.quiz_builder import ( + CandidateRanking, DIFFICULTY_FIT, UNRATED_FIT, target_difficulty) + + # Knowing nothing about somebody is the middle, not the bottom: the + # easiest questions are a poor way to find out what they know. + self.assertEqual(target_difficulty(None), 'medium') + self.assertEqual(target_difficulty(0.30), 'easy') + self.assertEqual(target_difficulty(0.65), 'medium') + self.assertEqual(target_difficulty(0.95), 'hard') + + for qid, level in [(1, 'easy'), (2, 'medium'), (4, 'hard')]: + self.db.get(Question, qid).difficulty = level + self.db.commit() + + as_of = datetime(2026, 1, 1) + ranking = CandidateRanking(self.db, self.user, now=as_of) + # Nothing answered anywhere, so every topic is at neutral readiness and + # every topic wants medium. + self.assertEqual(ranking.difficulty_fit((2, 2)), DIFFICULTY_FIT[0]) + self.assertEqual(ranking.difficulty_fit((1, 1)), DIFFICULTY_FIT[1]) + self.assertEqual(ranking.difficulty_fit((4, 2)), DIFFICULTY_FIT[1]) + # An unlabelled question is unknown, not wrong. + self.assertEqual(ranking.difficulty_fit((3, 3)), UNRATED_FIT) + + # It is a preference, never a filter: a bank thinned to one level is a + # bank three times smaller, and on a narrow topic that is the same + # eight questions every time. + self.assertGreater(min(DIFFICULTY_FIT), 0) + picked = self.bank.client.post('/questions/builder', json={ + 'title': 'Adaptive', 'count': 5, 'state': 'all', 'mode': 'learning', + 'algorithm': 'adaptive', 'category_ids': [], 'expected_count': 5, + }) + self.assertEqual(picked.status_code, 200, picked.text) + def test_it_can_see_the_whole_bank_not_the_first_page_of_it(self): from app.services.quiz_builder import adaptive_select, bank_query # The bank this learner may actually see — not every row in the table. diff --git a/docs/TODO.md b/docs/TODO.md index 1e6e5f9..99ca98a 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -88,14 +88,16 @@ Captured so nothing is lost while the article writing runs. that system exists to hold them, which is a classification gap rather than a filing error. `Radial Head Subluxation` merged into `Nursemaid's Elbow (Radial Head Subluxation)` — one idea under two names. -- [ ] **Adaptive session: the difficulty ladder** — reopened 2026-09-12, hours +- [x] **Adaptive session: the difficulty ladder** — done 2026-09-12, hours after being closed as unbuildable, because the reason it was unbuildable - is gone: every one of the 2,924 questions now carries a difficulty (622 + went away: every one of the 2,924 questions now carries a difficulty (622 easy, 1,634 medium, 668 hard, no failures), labelled in batches of 25 against a written rubric by `classify_question_difficulty`. The ordering - can now move a session along the range as a learner does well or badly, - which is the thing the design has always been missing. The rest of the - item, below, stays closed. + now moves a session along the range as a learner does well or badly: + `difficulty_fit()` scores a question by how far its level is from the one + the learner's readiness on that topic calls for, per topic rather than + overall, as a multiplier rather than a filter. Written up in + `docs/adaptive-sessions.md`. - [x] **Adaptive session (the other halves)** — closed 2026-09-12. *Shrunk readiness*: already done and now pinned by a test — `CandidateRanking.accuracy()` pulls a topic towards `NEUTRAL_RECALL` by diff --git a/docs/adaptive-sessions.md b/docs/adaptive-sessions.md index 2e809e5..36b390c 100644 --- a/docs/adaptive-sessions.md +++ b/docs/adaptive-sessions.md @@ -18,7 +18,7 @@ show its working could not do that. ## What decides which questions -Four rules, then one multiplier. +Five rules, then one multiplier. **1. Unseen material is most of the session.** A question never met teaches more than one already answered, so it takes every slot review is not holding. @@ -41,6 +41,35 @@ session of twenty becomes twenty questions from the single worst subject — and a learner with no history at all, whose topics are all equally unknown, gets handed the heaviest domain entire rather than a spread. +**5. And how hard the question is follows how the learner is doing on that +topic.** `difficulty_fit()`. Readiness under 0.45 wants **easy**, up to 0.78 +wants **medium**, above that **hard**; a question at the target level scores in +full, one step away keeps 0.72 of its value, two steps 0.5. An unlabelled +question keeps 0.92 — unknown is not wrong. + +Three things about this are deliberate. + +*Per topic, not per learner.* Somebody strong on growth and weak on +arrhythmias is the normal case, and one number across the two describes +nobody. The target is computed from the same shrunk accuracy that ranks the +topic. + +*A multiplier, not a filter.* Thinning the bank to one level makes it three +times smaller, and on a narrow topic that is the same eight questions every +time. Nothing is ever excluded for being the wrong level; it is preferred +against — the same reasoning as the reranker being a permutation. + +*The middle is the default.* A topic with no evidence sits at exactly +`NEUTRAL_RECALL`, and the first band's edge is 0.45 rather than 0.55 precisely +so that untouched topics get medium questions. Handing somebody the easiest +questions in a subject nobody has measured them on is a poor way to find out +what they know and a slow way to start. + +This could not exist until 2026-09-12, when `classify_question_difficulty` +labelled the bank: the column had been NULL on all 2,924 rows since it was +added, so difficulty was a filter a learner could set and nothing the session +ever did on its own. + **And all of it is scaled by the topic's share of the real paper.** Weakness alone said that being weak at something worth 5% of the exam and something worth 1% were the same problem. They are not. `exam_blueprints.weight` holds