diff --git a/backend/scripts/promote_primary_categories.py b/backend/scripts/promote_primary_categories.py new file mode 100644 index 0000000..f2fe28e --- /dev/null +++ b/backend/scripts/promote_primary_categories.py @@ -0,0 +1,145 @@ +"""Push each question's primary category down from its system to its condition. + +93% of questions have a top-level system as their primary category, so the +condition tree is reached almost entirely through cross-links. Filtering still +works, but the counts on every deep node are link counts, and a question about +Tetralogy of Fallot reads as "Cardiology" rather than as what it is about. + +Where a question already links to a condition *inside* its own system, that link +is the specific classification and can simply become the primary; the system it +came from is kept as a cross-link so nothing narrows. + +The top level follows the standard Nelson organisation — body systems plus +neonatology, genetics/metabolism, adolescent medicine, immunology/allergy, +rheumatology, infectious disease, emergency medicine and prevention — which the +27 existing roots already match, so this moves questions within that structure +rather than reshaping it. + +Three outcomes, and only the first is acted on automatically: + * exactly one candidate -> promoted + * several candidates -> the deepest wins; a tie at the same depth goes + to the one holding more questions, and a tie + there too is left alone and reported + * no candidate in its system -> nothing to promote; reported as genuinely + unclassified beyond its system + + docker compose exec backend python -m scripts.promote_primary_categories + docker compose exec backend python -m scripts.promote_primary_categories --apply +""" +import sys +from collections import defaultdict + +from sqlalchemy import text as sa_text + +from app.database import SessionLocal + + +def build_tree(db): + rows = db.execute(sa_text("SELECT id, name, parent_id FROM question_categories")).fetchall() + parent = {cid: pid for cid, _n, pid in rows} + name = {cid: n for cid, n, _p in rows} + + def depth(cid): + d, cursor, guard = 0, cid, 0 + while parent.get(cursor) is not None and guard < 8: + cursor, d, guard = parent[cursor], d + 1, guard + 1 + return d + + def root(cid): + cursor, guard = cid, 0 + while parent.get(cursor) is not None and guard < 8: + cursor, guard = parent[cursor], guard + 1 + return cursor + + return parent, name, {cid: depth(cid) for cid in parent}, {cid: root(cid) for cid in parent} + + +def main(): + apply_changes = "--apply" in sys.argv + db = SessionLocal() + try: + parent, name, depth, root = build_tree(db) + systems = {cid for cid, pid in parent.items() if pid is None} + + # Questions whose primary category is the system itself. + rows = db.execute(sa_text(""" + SELECT q.id, q.question_category_id FROM questions q + WHERE q.question_category_id IS NOT NULL + """)).fetchall() + at_system = {qid: cid for qid, cid in rows if cid in systems} + + links = defaultdict(set) + for qid, cid in db.execute(sa_text( + "SELECT question_id, category_id FROM question_category_links")).fetchall(): + links[qid].add(cid) + + counts = dict(db.execute(sa_text(""" + SELECT category_id, COUNT(*) FROM question_category_links GROUP BY category_id + """)).fetchall()) + + promoted, ambiguous, orphaned = [], [], [] + for qid, system_id in at_system.items(): + # Candidates are links that live inside this question's own system. + candidates = [cid for cid in links.get(qid, ()) + if cid != system_id and root.get(cid) == system_id] + if not candidates: + orphaned.append((qid, system_id)) + continue + if len(candidates) == 1: + promoted.append((qid, candidates[0], system_id)) + continue + # Most specific first, then the better-populated of equals. + ranked = sorted(candidates, key=lambda cid: (-depth[cid], -counts.get(cid, 0), cid)) + best, second = ranked[0], ranked[1] + if (depth[best], counts.get(best, 0)) == (depth[second], counts.get(second, 0)): + ambiguous.append((qid, system_id, ranked)) + else: + promoted.append((qid, best, system_id)) + + if apply_changes: + for qid, new_primary, old_system in promoted: + db.execute(sa_text( + "UPDATE questions SET question_category_id = :c WHERE id = :q"), + {"c": new_primary, "q": qid}) + # The system stays reachable, so no filter narrows. + db.execute(sa_text(""" + INSERT INTO question_category_links (question_id, category_id) + SELECT :q, :sys WHERE NOT EXISTS ( + SELECT 1 FROM question_category_links + WHERE question_id = :q AND category_id = :sys) + """), {"q": qid, "sys": old_system}) + db.commit() + + print("APPLIED" if apply_changes else "DRY RUN") + print(f" filed at system level : {len(at_system)}") + print(f" promoted to condition : {len(promoted)}") + print(f" ambiguous, left alone : {len(ambiguous)}") + print(f" nothing to promote : {len(orphaned)}\n") + + by_target = defaultdict(int) + for _qid, target, _sys in promoted: + by_target[target] += 1 + print(" Biggest destinations:") + for target, n in sorted(by_target.items(), key=lambda kv: -kv[1])[:15]: + trail = [] + cursor, guard = target, 0 + while cursor is not None and guard < 5: + trail.append(name.get(cursor, "?")) + cursor, guard = parent.get(cursor), guard + 1 + print(f" {' < '.join(trail)} — {n}") + + unclassified = defaultdict(int) + for _qid, system_id in orphaned: + unclassified[name.get(system_id, "?")] += 1 + print("\n No condition inside their own system (need classifying, not moving):") + for system, n in sorted(unclassified.items(), key=lambda kv: -kv[1])[:12]: + print(f" {system} — {n}") + + if not apply_changes: + print("\n Re-run with --apply to promote.") + finally: + db.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/TODO.md b/docs/TODO.md index bf122ae..5c1e2d5 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -93,10 +93,15 @@ Updated 2026-09-10. `scripts/merge_duplicate_conditions.py` to list them; the largest are Pneumonia, Behavioral Disorder, Intestinal Obstruction, Acute Rheumatic Fever, Hypoxic-Ischemic Encephalopathy. -- [ ] **Most questions are filed at system level** — 2,730 of 2,948 point their - primary category at a top-level system rather than a condition, so the - condition tree is reached almost entirely through cross-links. Worth - deciding whether primary categories should be pushed down the tree. +- [x] **Primary categories pushed down** — done 2026-09-10. 1,244 questions + promoted from system to condition; 2,730 → 1,486 filed at system level. + The system stays as a cross-link so no filter narrows. +- [ ] **1,455 questions have no condition inside their own system** — nothing to + promote, they are genuinely unclassified below the system. Worst: + General Pediatrics 396 (which has no subcategories at all), Emergency + Medicine 158, Adolescent Medicine 137. Needs classification work, not a + data move. +- [ ] **31 questions tie between equally specific conditions** — left alone. - [x] **Grant dimensions** — done 2026-09-10. A grant names any combination of exam, discipline and category, and covers what matches all of them. - [x] **Exam-scoped disciplines** — done 2026-09-10. A tag may be shared