diff --git a/backend/app/services/quiz_builder.py b/backend/app/services/quiz_builder.py index ff1a625..48d3e5b 100644 --- a/backend/app/services/quiz_builder.py +++ b/backend/app/services/quiz_builder.py @@ -215,9 +215,14 @@ def adaptive_select(db, user, count, category_ids, state, difficulty): query = filtered_bank_query(db, user, category_ids, state) if difficulty: query = query.filter(Question.difficulty == difficulty) - rows = query.with_entities(Question.id, Question.question_category_id).limit(2000).all() + # No cap. This used to take the first 2,000 rows, which on a 2,948-question + # bank meant adaptive selection could not see about a third of it, and which + # third depended on database order. Two integer columns per question is not + # a size worth protecting against. + rows = query.with_entities(Question.id, Question.question_category_id).all() if not rows: return [] + category_of = dict(rows) answered = db.query(AttemptAnswer.question_id, AttemptAnswer.is_correct, QuizAttempt.completed_at).join( QuizAttempt, QuizAttempt.id == AttemptAnswer.attempt_id).join(Quiz, Quiz.id == QuizAttempt.quiz_id).filter( QuizAttempt.user_id == user.id, QuizAttempt.completed_at.isnot(None), @@ -229,7 +234,9 @@ def adaptive_select(db, user, count, category_ids, state, difficulty): latest.setdefault(qid, (ok, when)) cat_correct: dict = defaultdict(lambda: [0, 0]) for qid, ok, _ in answered: - category = next((r[1] for r in rows if r[0] == qid), None) + # Looked up, not scanned. This was a linear search through every + # candidate for every answer — the slowest part of building a session. + category = category_of.get(qid) if category is None: continue cat_correct[category][0] += 1 diff --git a/backend/tests/test_quiz_builder.py b/backend/tests/test_quiz_builder.py index 68d7136..790fe7d 100644 --- a/backend/tests/test_quiz_builder.py +++ b/backend/tests/test_quiz_builder.py @@ -376,3 +376,43 @@ class BuilderTests(unittest.TestCase): if __name__ == "__main__": unittest.main() + + +class AdaptiveSelectionTests(unittest.TestCase): + """What the adaptive session actually picks, and in what order.""" + + def setUp(self): + self.bank = BuilderTests() + self.bank.setUp() + self.db = self.bank.db + self.user = self.bank.owner + + def tearDown(self): + self.bank.tearDown() + + def test_unanswered_comes_first_and_the_weakest_category_leads(self): + from app.services.quiz_builder import adaptive_select + # Answer one question in category 2 wrongly, one in category 3 rightly, + # so category 2 is the weaker of the two. + self.bank.answer(1, correct=False) + self.bank.answer(5, correct=True) + picked = adaptive_select(self.db, self.user, 2, [], "all", None) + self.assertEqual(len(picked), 2) + # Neither already-answered question is recycled while unseen ones remain. + self.assertNotIn(1, picked) + self.assertNotIn(5, picked) + + def test_it_can_see_the_whole_bank_not_the_first_page_of_it(self): + from app.services.quiz_builder import adaptive_select, bank_query + # The bank this learner may actually see — not every row in the table. + every = {row.id for row in bank_query(self.db, self.user).all()} + picked = set(adaptive_select(self.db, self.user, len(every), [], "all", None)) + self.assertEqual(picked, every) + + def test_a_question_answered_wrongly_returns_before_one_answered_rightly(self): + from app.services.quiz_builder import adaptive_select + for qid in [row.id for row in self.db.query(Question).all()]: + self.bank.answer(qid, correct=(qid != 1)) + # Everything has been seen, so the whole selection is recycled. + picked = adaptive_select(self.db, self.user, 1, [], "all", None) + self.assertEqual(picked, [1]) diff --git a/docs/study-recommendations.md b/docs/study-recommendations.md new file mode 100644 index 0000000..fc9befd --- /dev/null +++ b/docs/study-recommendations.md @@ -0,0 +1,134 @@ +# How the study recommendations and the adaptive session work + +Two questions were asked and owed an answer: *how do the study recommendations +work?* and *how would an adaptive session work?* Both are built; this is what +the code actually does, including the parts that are weaker than they look. + +--- + +## 1. Study recommendations + +`GET /study-tools/recommendations` — `backend/app/routers/study_tools.py`. + +### What goes in + +Only the learner's **completed, non-expired, non-course** answers: + +```python +QuizAttempt.user_id == user.id, +QuizAttempt.completed_at.isnot(None), +or_(QuizAttempt.expired == 0, QuizAttempt.expired.is_(None)), +Quiz.course_id.is_(None), +*([exam_filter] if exam_filter is not None else []), +``` + +`exam_filter` is the learner's chosen study objective. Answers to questions +linked to a *different* exam are excluded, and so is that material from the +denominator — someone revising for a paediatrics board who happens to sit a +step-exam plan does not have it steer their recommendations. Questions linked +to no exam stay in: unlinked means unclassified, not excluded. + +### Membership: the same answers, three ways + +Every answer is attributed to a set of **group keys**, and the grouping decides +what a key is: + +| Tab | Key | How a question reaches it | +|---|---|---| +| Articles | article id | the question's categories → the published article filed under one | +| Disciplines | top-level category id | the question's categories, rolled up to the top of the tree | +| Systems | organ-system tag id | the question's symptom keyword → the system that keyword is filed under | + +A question counts towards **every** ancestor it has, not just its primary +category, and `question_category_links` adds cross-filed categories on top. So +one answer on *Kawasaki disease* raises the count for Kawasaki, for Cardiology, +and for the Kawasaki article. + +Systems is the odd one: no question is tagged with an organ system directly. +It carries a symptom keyword (`question_tags.type = 'keyword'`) and 726 of +those keywords have a system as `parent_id`. That path reaches **1,502 of +2,948 questions**, which is why that tab states its own coverage instead of +implying it can see the whole bank. + +### The four numbers per row + +```python +accuracy = 100 * correct / answered # what you actually scored +coverage = 100 * seen / available # how much of it you have met +relevance = 100 * available / denominator # how much of the bank it is +readiness = 100 * (c + 8 * overall) / (n + 8) # accuracy, shrunk +``` + +**Readiness** is the only one that is not arithmetic on raw counts. It is a +plain empirical-Bayes shrinkage: a category's accuracy is pulled toward the +learner's own overall accuracy in proportion to how few answers it has. With +`READINESS_PRIOR_ANSWERS = 8`, one lucky question in a category does not read +as mastery and one unlucky one does not read as a gap — the category has to +earn its distance from your average. It stays locked until +`READINESS_UNLOCK_ANSWERS = 40` answers exist at all, because before that the +learner's own average is not a stable thing to shrink toward. + +It is **not** a psychometric score and not a prediction of any exam. The +`basis` string returned with every response says so, in those words, because a +number on a dashboard acquires authority it has not earned unless something +tells the reader what it is. + +**Relevance** divides by `bank_total` normally, but by `grouped_total` for +Systems — dividing by the whole bank when the grouping can only see half of it +would make every system look half as relevant as it is. + +### Ranking + +```python +gap = max(0, (baseline - score) / 100) # how far below your own average +unseen = 1 - coverage / 100 +priority = gap * relevance + 0.25 * unseen * relevance +``` + +`baseline` is the learner's overall accuracy (70% before there is one). So a +topic rises when it is **weak** *and* **a big part of the bank**. Material never +touched has no gap to measure, so it enters at `gap = 0.5` and is carried by +the `unseen` term at quarter weight — enough to surface, not enough to crowd +out a topic you are demonstrably failing. + +The top three rows with any answers are flagged `is_focus_area`. + +--- + +## 2. The adaptive session + +`adaptive_select()` — `backend/app/services/quiz_builder.py`. + +Four steps: + +1. **Candidates.** The filtered bank for the chosen categories/state/difficulty. +2. **Unanswered first**, sorted by the accuracy of their category — so a + question you have never seen, in the area you are worst at, comes first. A + category you have never answered in scores `0.5`, which puts it mid-pack. +3. **Then recycle.** If there are not enough unanswered, previously answered + questions are pulled back in, weighted `1 - 0.85` if you got it right and + `1 - 0.25` if you got it wrong — so a question you failed is roughly five + times likelier to return than one you passed. Ties break toward the oldest + incorrect answer. +4. **Damping.** Each time a category is picked, its weight is halved + (`damping[category] *= 0.5`). Without this a single weak category would fill + the whole session; with it, the session walks across your weak areas. + +### Where it is weaker than it looks + +- **`.limit(2000)` on the candidate query.** The bank is 2,948 questions, so + adaptive selection cannot currently see about a third of it, and which third + depends on database order. This is a real cap, not a tuning choice. +- **`accuracy()` is computed per category from a linear scan** — + `next((r[1] for r in rows if r[0] == qid), None)` inside a loop over every + answer. That is O(answers × candidates); at 2,000 candidates and a few + thousand answers it is the slowest part of building a session. +- **Difficulty is not used in the ordering**, only as a filter. A genuine + adaptive scheme would move up and down the difficulty range as you do well + or badly; this one does not. +- **Step 2 sorts by raw category accuracy, not readiness.** The shrinkage that + the recommendations page applies precisely so small samples do not mislead is + not applied here, so a category with one wrong answer can sort first. + +The first two are defects and are worth fixing. The last two are honest +limitations of the current design and would be the substance of a better one.