diff --git a/backend/scripts/merge_duplicate_conditions.py b/backend/scripts/merge_duplicate_conditions.py new file mode 100644 index 0000000..0f2befc --- /dev/null +++ b/backend/scripts/merge_duplicate_conditions.py @@ -0,0 +1,147 @@ +"""Give each condition one home, and keep its other systems as cross-links. + +The tag→category conversion made every question's subject tag a parent, so a +question tagged both Pulmonology and Emergency Medicine produced an "Asthma" row +under each. "Asthma" is therefore not one topic with 45 questions but several +fragments no filter ever adds up — 330 of 618 condition names live in more than +one system, a few in eight. + +Merging keeps the row with the most questions and folds the rest into it. The +association with the other systems is not thrown away: each moved question gains +a `question_category_links` row pointing at the system it came from, which is the +mechanism built for "this matters to nephrology *and* emergency medicine". +Duplicate rows are the wrong way to say that; a link is the right one. + +Only names appearing in several systems are touched, and only when one row is a +clear plurality — an even split is reported for a human to decide rather than +broken by an arbitrary rule. + + docker compose exec backend python -m scripts.merge_duplicate_conditions + docker compose exec backend python -m scripts.merge_duplicate_conditions --apply + docker compose exec backend python -m scripts.merge_duplicate_conditions --min-questions 5 +""" +import sys +from collections import defaultdict + +from sqlalchemy import text as sa_text + +from app.database import SessionLocal + + +def direct_count(db, category_id: int) -> int: + return db.execute(sa_text(""" + SELECT (SELECT COUNT(*) FROM questions WHERE question_category_id = :c) + + (SELECT COUNT(*) FROM question_category_links WHERE category_id = :c) + """), {"c": category_id}).scalar() or 0 + + +def root_of(db, category_id: int) -> int: + """The system a row sits under, walking up at most a few levels.""" + cursor, guard = category_id, 0 + while guard < 8: + parent = db.execute(sa_text( + "SELECT parent_id FROM question_categories WHERE id = :c"), {"c": cursor}).scalar() + if parent is None: + return cursor + cursor, guard = parent, guard + 1 + return cursor + + +def main(): + apply_changes = "--apply" in sys.argv + threshold = 0 + if "--min-questions" in sys.argv: + threshold = int(sys.argv[sys.argv.index("--min-questions") + 1]) + + db = SessionLocal() + try: + rows = db.execute(sa_text(""" + SELECT c.id, c.name, c.parent_id FROM question_categories c + WHERE c.parent_id IS NOT NULL + """)).fetchall() + names = dict(db.execute(sa_text("SELECT id, name FROM question_categories")).fetchall()) + + by_name: dict[str, list] = defaultdict(list) + for cid, name, _parent in rows: + by_name[name.strip().lower()].append(cid) + + plans, ambiguous = [], [] + for key, ids in by_name.items(): + if len(ids) < 2: + continue + counts = {cid: direct_count(db, cid) for cid in ids} + total = sum(counts.values()) + if total < threshold: + continue + ranked = sorted(ids, key=lambda cid: (-counts[cid], cid)) + best, runner_up = counts[ranked[0]], counts[ranked[1]] + entry = (names[ranked[0]], ranked[0], ranked[1:], counts, total) + # An even split has no evidence-based winner, so leave it to a human. + if best == runner_up: + ambiguous.append(entry) + else: + plans.append(entry) + + plans.sort(key=lambda e: -e[4]) + ambiguous.sort(key=lambda e: -e[4]) + + moved_questions = 0 + if apply_changes: + for _name, keeper, losers, _counts, _total in plans: + for loser in losers: + system = root_of(db, loser) + # Preserve the association with the system being folded away. + db.execute(sa_text(""" + INSERT INTO question_category_links (question_id, category_id) + SELECT q.id, :sys FROM questions q + WHERE q.question_category_id = :loser + AND NOT EXISTS (SELECT 1 FROM question_category_links l + WHERE l.question_id = q.id AND l.category_id = :sys) + """), {"sys": system, "loser": loser}) + db.execute(sa_text( + "UPDATE questions SET question_category_id = :k WHERE question_category_id = :l"), + {"k": keeper, "l": loser}) + db.execute(sa_text(""" + UPDATE question_category_links SET category_id = :k + WHERE category_id = :l AND question_id NOT IN ( + SELECT question_id FROM question_category_links WHERE category_id = :k) + """), {"k": keeper, "l": loser}) + db.execute(sa_text("DELETE FROM question_category_links WHERE category_id = :l"), + {"l": loser}) + db.execute(sa_text("UPDATE articles SET category_id = :k WHERE category_id = :l"), + {"k": keeper, "l": loser}) + db.execute(sa_text("UPDATE flashcard_decks SET category_id = :k WHERE category_id = :l"), + {"k": keeper, "l": loser}) + db.execute(sa_text("UPDATE question_categories SET parent_id = :k WHERE parent_id = :l"), + {"k": keeper, "l": loser}) + db.execute(sa_text("DELETE FROM category_grants WHERE category_id = :l"), {"l": loser}) + db.execute(sa_text("DELETE FROM question_categories WHERE id = :l"), {"l": loser}) + moved_questions += 1 + db.commit() + + print("APPLIED" if apply_changes else "DRY RUN") + print(f" duplicated names : {len(plans) + len(ambiguous)}") + print(f" merges planned : {len(plans)} (rows removed: {sum(len(p[2]) for p in plans)})") + print(f" need a human call : {len(ambiguous)}\n") + + print(" Largest merges (keeper shown first):") + for name, keeper, losers, counts, total in plans[:25]: + homes = " + ".join( + f"{names.get(root_of(db, cid), '?')}:{counts[cid]}" for cid in [keeper, *losers]) + print(f" {name} — {total} questions across {len(losers) + 1} systems [{homes}]") + + if ambiguous: + print("\n Even splits — pick a home yourself, nothing was merged:") + for name, keeper, losers, counts, total in ambiguous[:15]: + homes = ", ".join( + f"{names.get(root_of(db, cid), '?')}" for cid in [keeper, *losers]) + print(f" {name} — {total} questions, tied across {homes}") + + if not apply_changes: + print("\n Re-run with --apply to merge, or --min-questions N to limit scope.") + finally: + db.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/scripts/nest_systems.py b/backend/scripts/nest_systems.py new file mode 100644 index 0000000..52da2a1 --- /dev/null +++ b/backend/scripts/nest_systems.py @@ -0,0 +1,286 @@ +"""Give the system tree a middle level: System → Subsystem → Condition. + +The tree came from the old subject tags, so every condition sits directly under +its system — 27 roots and about 1,050 leaves, one level deep. That makes the +Systems picker a flat alphabetical wall. + +The subsystem names here are standard textbook divisions, not invented ones, and +each is matched by an explicit pattern so a dry run shows exactly what will move. + +Deliberately conservative in three ways: + * a condition that matches no pattern stays directly under its system, rather + than being forced into an approximate bucket; + * a subsystem that would hold fewer than MIN_CHILDREN is not created, so the + tree does not sprout one-item branches; + * nothing is reclassified across systems, and nothing is merged. Conditions + that look misfiled or synonymous are *reported* for a human to judge, because + deciding that "Trisomy 21" and "Down Syndrome" are one thing, or that Marfan + belongs under Genetics, is a medical call and not a string comparison. + + docker compose exec backend python -m scripts.nest_systems + docker compose exec backend python -m scripts.nest_systems --apply +""" +import re +import sys +from collections import defaultdict + +from sqlalchemy import text as sa_text + +from app.database import SessionLocal + +MIN_CHILDREN = 2 + +# system -> [(subsystem, pattern matched against the condition name)] +SUBSYSTEMS: dict[str, list[tuple[str, str]]] = { + "Cardiology": [ + ("Congenital Heart Disease", r"septal defect|tetralogy|transposition|ductus arteriosus|" + r"hypoplastic left|bicuspid|coarctation|congenital heart|" + r"cyanotic heart|truncus|ebstein|pulmonary atresia"), + ("Arrhythmias", r"arrhythmi|long qt|wolff|tachycardi|bradycardi|heart block|sudden cardiac"), + ("Cardiomyopathy", r"cardiomyopath"), + ("Heart Failure", r"heart failure"), + ("Inflammatory Heart Disease", r"myocarditis|pericarditis|endocarditis|rheumatic"), + ("Vascular", r"hypertension|hyperlipid|hypercholesterol|vasculitis"), + ], + "Neurology": [ + ("Seizures & Epilepsy", r"seizure|epilep|status epilepticus|febrile convulsion|infantile spasm"), + ("Headache", r"headache|migraine"), + ("Neuromuscular", r"muscular dystroph|myopath|neuropath|myasthen|guillain|spinal muscular|hypotoni"), + ("Neurocutaneous", r"neurofibromatos|tuberous sclerosis|sturge|von hippel"), + ("CNS Infection", r"meningitis|encephalitis|brain abscess"), + ("Movement Disorders", r"ataxia|chorea|dystonia|tic disorder|tourette"), + ("Developmental Brain Disorders", r"cerebral palsy|hydrocephal|spina bifida|neural tube|" + r"microcephal|macrocephal|craniosynostos"), + ], + "Infectious Disease": [ + ("Bacterial Infections", r"bacter|strep|staph|pertussis|tuberculosis|lyme|salmonell|" + r"shigell|cellulitis|impetigo|osteomyelitis|scarlet"), + ("Viral Infections", r"viral|virus|measles|mumps|rubella|varicella|influenza|herpes|" + r"hepatitis|hiv|mononucleosis|roseola|hand foot|croup|rsv"), + ("Parasitic & Fungal", r"parasit|malaria|giardia|pinworm|helminth|fungal|candid|tinea|scabies|lice"), + ("Immunisation & Prophylaxis", r"vaccin|immuniz|immunis|prophylax"), + ("Congenital Infections", r"congenital infection|toxoplasm|cytomegalovirus|congenital syphilis"), + ], + "Neonatology": [ + ("Neonatal Jaundice", r"jaundice|bilirubin|kernicterus"), + ("Neonatal Respiratory", r"respiratory distress|surfactant|meconium aspiration|apnea|bronchopulmonary|" + r"transient tachypnea|pneumothorax"), + ("Prematurity", r"prematur|preterm|low birth weight|intraventricular h|retinopathy of prematurity|" + r"necrotizing enterocolitis"), + ("Neonatal Infection", r"neonatal sepsis|early onset sepsis|late onset sepsis|group b strep"), + ("Birth Injury & Asphyxia", r"birth injur|asphyxia|hypoxic ischemic|brachial plexus|cephalohematoma|" + r"caput"), + ("Newborn Screening", r"newborn screen|metabolic screen"), + ], + "Pulmonology": [ + ("Asthma & Wheezing", r"asthma|wheez|reactive airway"), + ("Lower Airway Infection", r"bronchiolitis|pneumonia|bronchitis"), + ("Upper Airway", r"croup|epiglottitis|tracheitis|stridor|laryngomalacia|obstructive sleep"), + ("Cystic Fibrosis", r"cystic fibrosis"), + ("Foreign Body & Aspiration", r"foreign body|aspiration pneumon"), + ], + "Gastroenterology & Nutrition": [ + ("Diarrhoea & Gastroenteritis", r"diarrhea|diarrhoea|gastroenteritis|dehydration"), + ("Constipation & Motility", r"constipation|hirschsprung|encopresis|motility"), + ("Liver Disease", r"hepat|liver|biliary|cirrhosis"), + ("Malabsorption", r"celiac|malabsorption|lactose|short bowel"), + ("Inflammatory Bowel Disease", r"crohn|ulcerative colitis|inflammatory bowel"), + ("Surgical Abdomen", r"appendicitis|intussusception|volvulus|pyloric stenosis|obstruction|" + r"malrotation|acute abdomen"), + ("Nutrition & Growth", r"nutrition|failure to thrive|obesity|vitamin|malnutrition|rickets|" + r"breastfeed|deficiency"), + ("Reflux & Peptic Disease", r"reflux|gerd|peptic|gastritis|ulcer"), + ], + "Endocrinology": [ + ("Diabetes", r"diabet|ketoacidosis|hypoglycem|insulin"), + ("Thyroid", r"thyroid|hypothyroid|hyperthyroid|goiter"), + ("Growth Disorders", r"short stature|growth hormone|tall stature|growth failure"), + ("Puberty", r"puberty|precocious|delayed puberty|gynecomastia"), + ("Adrenal", r"adrenal|cushing|addison|congenital adrenal"), + ("Calcium & Bone", r"calcium|rickets|parathyroid|vitamin d"), + ], + "Hematology-Oncology": [ + ("Anaemia", r"anemia|anaemia|thalassem|sickle|spherocytos|g6pd|iron deficiency"), + ("Bleeding & Clotting", r"hemophil|thrombocytopen|coagulop|von willebrand|purpura|thrombo"), + ("Leukaemia & Lymphoma", r"leukemi|leukaemi|lymphoma|hodgkin"), + ("Solid Tumours", r"neuroblastoma|wilms|sarcoma|retinoblastoma|brain tumor|tumour|osteosarcoma"), + ], + "Nephrology & Urology": [ + ("Urinary Tract Infection", r"urinary tract infection|pyelonephritis|cystitis|uti"), + ("Glomerular Disease", r"glomerulonephritis|nephrotic|nephritic|proteinuria|hematuria|iga"), + ("Acute & Chronic Kidney Disease", r"kidney injury|renal failure|kidney disease|renal insufficiency"), + ("Congenital Urinary Tract", r"vesicoureteral|hydronephrosis|posterior urethral|reflux nephropathy"), + ("Electrolytes & Acid-Base", r"hyponatrem|hypernatrem|hypokalem|hyperkalem|acidosis|alkalosis|electrolyte"), + ("Genital & Scrotal", r"testicular torsion|hydrocele|cryptorchid|phimosis|hypospadias|scrotal"), + ], + "Dermatology": [ + ("Eczema & Dermatitis", r"eczema|dermatitis|atopic"), + ("Skin Infections", r"impetigo|cellulitis|tinea|scabies|warts|molluscum|abscess"), + ("Rashes & Exanthems", r"exanthem|\brash\b|urticaria|\berythema\b"), + ("Birthmarks & Vascular Lesions", r"hemangioma|nevus|port wine|birthmark"), + ("Acne", r"acne"), + ], + "Allergy/Immunology": [ + ("Food Allergy", r"food allerg|anaphylax|peanut|milk protein"), + ("Allergic Rhinitis & Atopy", r"allergic rhinitis|atopy|hay fever"), + ("Immunodeficiency", r"immunodefic|scid|agammaglobulin|chronic granulomatous|complement deficiency"), + ("Drug Allergy", r"drug allerg|drug reaction|penicillin allerg"), + ], + "Orthopedics & Sports": [ + ("Fractures & Trauma", r"fracture|dislocation|sprain|trauma|injury"), + ("Hip Disorders", r"hip dysplasia|slipped capital|legg|perthes|developmental dysplasia"), + ("Spine", r"scoliosis|kyphosis|spondylolisthesis|back pain"), + ("Bone & Joint Infection", r"osteomyelitis|septic arthritis"), + ("Limb & Gait", r"limp|gait|bowleg|genu|clubfoot|torsion"), + ], + "Psychiatry & Psychology": [ + ("Mood Disorders", r"depress|bipolar|mood"), + ("Anxiety Disorders", r"anxiety|panic|phobia|obsessive"), + ("Neurodevelopmental", r"adhd|attention deficit|autism|learning disab|intellectual disab"), + ("Behaviour Disorders", r"conduct disorder|oppositional|impulse|aggression"), + ("Eating Disorders", r"anorexia|bulimia|eating disorder"), + ("Trauma & Stress", r"ptsd|trauma|stress disorder|adjustment disorder"), + ], + "Emergency Medicine": [ + ("Shock & Resuscitation", r"shock|resuscitat|cardiac arrest|cpr"), + ("Trauma", r"trauma|fracture|burn|head injury|drowning"), + ("Poisoning", r"poison|overdose|ingestion|toxic"), + ("Respiratory Emergencies", r"respiratory failure|respiratory distress|airway|anaphylax"), + ("Altered Consciousness & Seizure", r"seizure|status epilepticus|altered mental|coma|syncope"), + ], + "Genetics & Metabolism": [ + ("Chromosomal Disorders", r"trisomy|down syndrome|turner|klinefelter|deletion syndrome|" + r"chromosom|22q11"), + ("Inborn Errors of Metabolism", r"phenylketonuria|galactosem|urea cycle|mitochondrial|" + r"glycogen storage|lysosomal|metabolic disorder|maple syrup"), + ("Connective Tissue Disorders", r"marfan|ehlers|osteogenesis imperfecta|connective tissue"), + ("Skeletal Dysplasias", r"achondroplasia|dysplasia|dwarfism"), + ], + "Adolescent Medicine": [ + ("Sexual & Reproductive Health", r"sexually transmitted|contracept|pregnan|menstrua|" + r"dysmenorrh|amenorrh|uterine bleeding"), + ("Substance Use", r"substance|alcohol|smoking|vaping|drug use|marijuana|opioid"), + ("Mental Health", r"depress|suicid|self harm|anxiety|eating disorder"), + ("Growth & Puberty", r"puberty|acne|gynecomastia|scoliosis"), + ], +} + + + +def report_misfiled(db) -> None: + """List conditions that match another system's patterns better than their own. + + Reported, never moved: deciding that Down Syndrome belongs under Genetics + rather than Cardiology is a medical call, and a pattern match is only a hint. + """ + roots = db.execute(sa_text( + "SELECT id, name FROM question_categories WHERE parent_id IS NULL")).fetchall() + root_name = {row[0]: row[1] for row in roots} + compiled = {system: [(sub, re.compile(pattern, re.I)) for sub, pattern in rules] + for system, rules in SUBSYSTEMS.items()} + + rows = db.execute(sa_text(""" + SELECT c.id, c.name, r.id + FROM question_categories c + JOIN question_categories r ON r.id = c.parent_id + WHERE r.parent_id IS NULL + ORDER BY c.name + """)).fetchall() + + suspects = [] + for _cid, name, parent_id in rows: + home = root_name.get(parent_id) + if home in compiled and any(p.search(name) for _s, p in compiled[home]): + continue # it fits where it already is + elsewhere = [system for system, rules in compiled.items() + if system != home and any(p.search(name) for _s, p in rules)] + if elsewhere: + suspects.append((name, home, elsewhere)) + + print(f"\n Possibly misfiled ({len(suspects)}) — review, nothing was moved:") + for name, home, elsewhere in suspects[:40]: + print(f" {name!r} is under {home} but matches {', '.join(elsewhere)}") + if len(suspects) > 40: + print(f" … and {len(suspects) - 40} more") + + +def main(): + apply_changes = "--apply" in sys.argv + db = SessionLocal() + try: + roots = db.execute(sa_text( + "SELECT id, name FROM question_categories WHERE parent_id IS NULL ORDER BY name")).fetchall() + by_name = {row[1]: row[0] for row in roots} + + planned: dict[str, list[tuple[str, int, str]]] = defaultdict(list) + untouched: dict[str, int] = {} + + for system, rules in SUBSYSTEMS.items(): + root_id = by_name.get(system) + if root_id is None: + print(f" system {system!r} not found; skipping") + continue + children = db.execute(sa_text(""" + SELECT id, name FROM question_categories + WHERE parent_id = :p ORDER BY name + """), {"p": root_id}).fetchall() + existing_subs = {name for _cid, name in children if name in {s for s, _ in rules}} + + claimed = set() + for subsystem, pattern in rules: + matcher = re.compile(pattern, re.I) + hits = [(cid, name) for cid, name in children + if cid not in claimed and name not in existing_subs and matcher.search(name)] + if len(hits) < MIN_CHILDREN: + continue # not worth a branch of its own + for cid, name in hits: + claimed.add(cid) + planned[system].append((subsystem, cid, name)) + untouched[system] = len([c for c, _ in children if c not in claimed]) + + if apply_changes: + for system, moves in planned.items(): + root_id = by_name[system] + sub_ids: dict[str, int] = {} + for subsystem, cid, _name in moves: + if subsystem not in sub_ids: + found = db.execute(sa_text(""" + SELECT id FROM question_categories + WHERE parent_id = :p AND lower(name) = lower(:n) + """), {"p": root_id, "n": subsystem}).first() + if found: + sub_ids[subsystem] = found[0] + else: + sub_ids[subsystem] = db.execute(sa_text(""" + INSERT INTO question_categories (name, parent_id, user_id, created_at) + VALUES (:n, :p, (SELECT id FROM users ORDER BY id LIMIT 1), NOW()) + RETURNING id + """), {"n": subsystem, "p": root_id}).scalar() + db.execute(sa_text("UPDATE question_categories SET parent_id = :s WHERE id = :c"), + {"s": sub_ids[subsystem], "c": cid}) + db.commit() + + print("APPLIED" if apply_changes else "DRY RUN") + total_moved = sum(len(v) for v in planned.values()) + total_subs = sum(len({s for s, _, _ in v}) for v in planned.values()) + print(f" subsystems created : {total_subs}") + print(f" conditions nested : {total_moved}") + print(f" left at top level : {sum(untouched.values())}\n") + for system in sorted(planned): + groups: dict[str, list[str]] = defaultdict(list) + for subsystem, _cid, name in planned[system]: + groups[subsystem].append(name) + print(f" {system} (+{untouched.get(system, 0)} stay directly under it)") + for subsystem, names in sorted(groups.items()): + shown = ", ".join(names[:4]) + more = f" … +{len(names) - 4}" if len(names) > 4 else "" + print(f" {subsystem} ({len(names)}): {shown}{more}") + report_misfiled(db) + if not apply_changes: + print("\n Re-run with --apply to write this structure.") + finally: + db.close() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/TODO.md b/docs/TODO.md index e603d06..bf122ae 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -83,8 +83,20 @@ Updated 2026-09-10. ## Taxonomy -- [ ] **Systems need subsystems** — the current tree came from the old subject - tags and is flat where it should nest. Disciplines are fine. +- [x] **Systems subsystems** — done 2026-09-10. 69 subsystems created, 305 + conditions nested; the tree is now System → Subsystem → Condition. +- [x] **Cross-system duplicate conditions merged** — done 2026-09-10. 216 names + folded to one home each (347 rows removed, 1075 → 786), keeping the other + systems as `question_category_links` rather than duplicate rows. +- [ ] **114 tied conditions need a human call** — an even split across systems + has no evidence-based winner, so nothing was merged. Run + `scripts/merge_duplicate_conditions.py` to list them; the largest are + Pneumonia, Behavioral Disorder, Intestinal Obstruction, Acute Rheumatic + Fever, Hypoxic-Ischemic Encephalopathy. +- [ ] **Most questions are filed at system level** — 2,730 of 2,948 point their + primary category at a top-level system rather than a condition, so the + condition tree is reached almost entirely through cross-links. Worth + deciding whether primary categories should be pushed down the tree. - [x] **Grant dimensions** — done 2026-09-10. A grant names any combination of exam, discipline and category, and covers what matches all of them. - [x] **Exam-scoped disciplines** — done 2026-09-10. A tag may be shared