"""Give the system tree a middle level: System → Subsystem → Condition. The tree came from the old subject tags, so every condition sits directly under its system — 27 roots and about 1,050 leaves, one level deep. That makes the Systems picker a flat alphabetical wall. The subsystem names here are standard textbook divisions, not invented ones, and each is matched by an explicit pattern so a dry run shows exactly what will move. Deliberately conservative in three ways: * a condition that matches no pattern stays directly under its system, rather than being forced into an approximate bucket; * a subsystem that would hold fewer than MIN_CHILDREN is not created, so the tree does not sprout one-item branches; * nothing is reclassified across systems, and nothing is merged. Conditions that look misfiled or synonymous are *reported* for a human to judge, because deciding that "Trisomy 21" and "Down Syndrome" are one thing, or that Marfan belongs under Genetics, is a medical call and not a string comparison. docker compose exec backend python -m scripts.nest_systems docker compose exec backend python -m scripts.nest_systems --apply """ import re import sys from collections import defaultdict from sqlalchemy import text as sa_text from app.database import SessionLocal MIN_CHILDREN = 2 # system -> [(subsystem, pattern matched against the condition name)] SUBSYSTEMS: dict[str, list[tuple[str, str]]] = { "Cardiology": [ ("Congenital Heart Disease", r"septal defect|tetralogy|transposition|ductus arteriosus|" r"hypoplastic left|bicuspid|coarctation|congenital heart|" r"cyanotic heart|truncus|ebstein|pulmonary atresia"), ("Arrhythmias", r"arrhythmi|long qt|wolff|tachycardi|bradycardi|heart block|sudden cardiac"), ("Cardiomyopathy", r"cardiomyopath"), ("Heart Failure", r"heart failure"), ("Inflammatory Heart Disease", r"myocarditis|pericarditis|endocarditis|rheumatic"), ("Vascular", r"hypertension|hyperlipid|hypercholesterol|vasculitis"), ], "Neurology": [ ("Seizures & Epilepsy", r"seizure|epilep|status epilepticus|febrile convulsion|infantile spasm"), ("Headache", r"headache|migraine"), ("Neuromuscular", r"muscular dystroph|myopath|neuropath|myasthen|guillain|spinal muscular|hypotoni"), ("Neurocutaneous", r"neurofibromatos|tuberous sclerosis|sturge|von hippel"), ("CNS Infection", r"meningitis|encephalitis|brain abscess"), ("Movement Disorders", r"ataxia|chorea|dystonia|tic disorder|tourette"), ("Developmental Brain Disorders", r"cerebral palsy|hydrocephal|spina bifida|neural tube|" r"microcephal|macrocephal|craniosynostos"), ], "Infectious Disease": [ ("Bacterial Infections", r"bacter|strep|staph|pertussis|tuberculosis|lyme|salmonell|" r"shigell|cellulitis|impetigo|osteomyelitis|scarlet"), ("Viral Infections", r"viral|virus|measles|mumps|rubella|varicella|influenza|herpes|" r"hepatitis|hiv|mononucleosis|roseola|hand foot|croup|rsv"), ("Parasitic & Fungal", r"parasit|malaria|giardia|pinworm|helminth|fungal|candid|tinea|scabies|lice"), ("Immunisation & Prophylaxis", r"vaccin|immuniz|immunis|prophylax"), ("Congenital Infections", r"congenital infection|toxoplasm|cytomegalovirus|congenital syphilis"), ], "Neonatology": [ ("Neonatal Jaundice", r"jaundice|bilirubin|kernicterus"), ("Neonatal Respiratory", r"respiratory distress|surfactant|meconium aspiration|apnea|bronchopulmonary|" r"transient tachypnea|pneumothorax"), ("Prematurity", r"prematur|preterm|low birth weight|intraventricular h|retinopathy of prematurity|" r"necrotizing enterocolitis"), ("Neonatal Infection", r"neonatal sepsis|early onset sepsis|late onset sepsis|group b strep"), ("Birth Injury & Asphyxia", r"birth injur|asphyxia|hypoxic ischemic|brachial plexus|cephalohematoma|" r"caput"), ("Newborn Screening", r"newborn screen|metabolic screen"), ], "Pulmonology": [ ("Asthma & Wheezing", r"asthma|wheez|reactive airway"), ("Lower Airway Infection", r"bronchiolitis|pneumonia|bronchitis"), ("Upper Airway", r"croup|epiglottitis|tracheitis|stridor|laryngomalacia|obstructive sleep"), ("Cystic Fibrosis", r"cystic fibrosis"), ("Foreign Body & Aspiration", r"foreign body|aspiration pneumon"), ], "Gastroenterology & Nutrition": [ ("Diarrhoea & Gastroenteritis", r"diarrhea|diarrhoea|gastroenteritis|dehydration"), ("Constipation & Motility", r"constipation|hirschsprung|encopresis|motility"), ("Liver Disease", r"hepat|liver|biliary|cirrhosis"), ("Malabsorption", r"celiac|malabsorption|lactose|short bowel"), ("Inflammatory Bowel Disease", r"crohn|ulcerative colitis|inflammatory bowel"), ("Surgical Abdomen", r"appendicitis|intussusception|volvulus|pyloric stenosis|obstruction|" r"malrotation|acute abdomen"), ("Nutrition & Growth", r"nutrition|failure to thrive|obesity|vitamin|malnutrition|rickets|" r"breastfeed|deficiency"), ("Reflux & Peptic Disease", r"reflux|gerd|peptic|gastritis|ulcer"), ], "Endocrinology": [ ("Diabetes", r"diabet|ketoacidosis|hypoglycem|insulin"), ("Thyroid", r"thyroid|hypothyroid|hyperthyroid|goiter"), ("Growth Disorders", r"short stature|growth hormone|tall stature|growth failure"), ("Puberty", r"puberty|precocious|delayed puberty|gynecomastia"), ("Adrenal", r"adrenal|cushing|addison|congenital adrenal"), ("Calcium & Bone", r"calcium|rickets|parathyroid|vitamin d"), ], "Hematology-Oncology": [ ("Anaemia", r"anemia|anaemia|thalassem|sickle|spherocytos|g6pd|iron deficiency"), ("Bleeding & Clotting", r"hemophil|thrombocytopen|coagulop|von willebrand|purpura|thrombo"), ("Leukaemia & Lymphoma", r"leukemi|leukaemi|lymphoma|hodgkin"), ("Solid Tumours", r"neuroblastoma|wilms|sarcoma|retinoblastoma|brain tumor|tumour|osteosarcoma"), ], "Nephrology & Urology": [ ("Urinary Tract Infection", r"urinary tract infection|pyelonephritis|cystitis|uti"), ("Glomerular Disease", r"glomerulonephritis|nephrotic|nephritic|proteinuria|hematuria|iga"), ("Acute & Chronic Kidney Disease", r"kidney injury|renal failure|kidney disease|renal insufficiency"), ("Congenital Urinary Tract", r"vesicoureteral|hydronephrosis|posterior urethral|reflux nephropathy"), ("Electrolytes & Acid-Base", r"hyponatrem|hypernatrem|hypokalem|hyperkalem|acidosis|alkalosis|electrolyte"), ("Genital & Scrotal", r"testicular torsion|hydrocele|cryptorchid|phimosis|hypospadias|scrotal"), ], "Dermatology": [ ("Eczema & Dermatitis", r"eczema|dermatitis|atopic"), ("Skin Infections", r"impetigo|cellulitis|tinea|scabies|warts|molluscum|abscess"), ("Rashes & Exanthems", r"exanthem|\brash\b|urticaria|\berythema\b"), ("Birthmarks & Vascular Lesions", r"hemangioma|nevus|port wine|birthmark"), ("Acne", r"acne"), ], "Allergy/Immunology": [ ("Food Allergy", r"food allerg|anaphylax|peanut|milk protein"), ("Allergic Rhinitis & Atopy", r"allergic rhinitis|atopy|hay fever"), ("Immunodeficiency", r"immunodefic|scid|agammaglobulin|chronic granulomatous|complement deficiency"), ("Drug Allergy", r"drug allerg|drug reaction|penicillin allerg"), ], "Orthopedics & Sports": [ ("Fractures & Trauma", r"fracture|dislocation|sprain|trauma|injury"), ("Hip Disorders", r"hip dysplasia|slipped capital|legg|perthes|developmental dysplasia"), ("Spine", r"scoliosis|kyphosis|spondylolisthesis|back pain"), ("Bone & Joint Infection", r"osteomyelitis|septic arthritis"), ("Limb & Gait", r"limp|gait|bowleg|genu|clubfoot|torsion"), ], "Psychiatry & Psychology": [ ("Mood Disorders", r"depress|bipolar|mood"), ("Anxiety Disorders", r"anxiety|panic|phobia|obsessive"), ("Neurodevelopmental", r"adhd|attention deficit|autism|learning disab|intellectual disab"), ("Behaviour Disorders", r"conduct disorder|oppositional|impulse|aggression"), ("Eating Disorders", r"anorexia|bulimia|eating disorder"), ("Trauma & Stress", r"ptsd|trauma|stress disorder|adjustment disorder"), ], "Emergency Medicine": [ ("Shock & Resuscitation", r"shock|resuscitat|cardiac arrest|cpr"), ("Trauma", r"trauma|fracture|burn|head injury|drowning"), ("Poisoning", r"poison|overdose|ingestion|toxic"), ("Respiratory Emergencies", r"respiratory failure|respiratory distress|airway|anaphylax"), ("Altered Consciousness & Seizure", r"seizure|status epilepticus|altered mental|coma|syncope"), ], "Genetics & Metabolism": [ ("Chromosomal Disorders", r"trisomy|down syndrome|turner|klinefelter|deletion syndrome|" r"chromosom|22q11"), ("Inborn Errors of Metabolism", r"phenylketonuria|galactosem|urea cycle|mitochondrial|" r"glycogen storage|lysosomal|metabolic disorder|maple syrup"), ("Connective Tissue Disorders", r"marfan|ehlers|osteogenesis imperfecta|connective tissue"), ("Skeletal Dysplasias", r"achondroplasia|dysplasia|dwarfism"), ], "Adolescent Medicine": [ ("Sexual & Reproductive Health", r"sexually transmitted|contracept|pregnan|menstrua|" r"dysmenorrh|amenorrh|uterine bleeding"), ("Substance Use", r"substance|alcohol|smoking|vaping|drug use|marijuana|opioid"), ("Mental Health", r"depress|suicid|self harm|anxiety|eating disorder"), ("Growth & Puberty", r"puberty|acne|gynecomastia|scoliosis"), ], } def report_misfiled(db) -> None: """List conditions that match another system's patterns better than their own. Reported, never moved: deciding that Down Syndrome belongs under Genetics rather than Cardiology is a medical call, and a pattern match is only a hint. """ roots = db.execute(sa_text( "SELECT id, name FROM question_categories WHERE parent_id IS NULL")).fetchall() root_name = {row[0]: row[1] for row in roots} compiled = {system: [(sub, re.compile(pattern, re.I)) for sub, pattern in rules] for system, rules in SUBSYSTEMS.items()} rows = db.execute(sa_text(""" SELECT c.id, c.name, r.id FROM question_categories c JOIN question_categories r ON r.id = c.parent_id WHERE r.parent_id IS NULL ORDER BY c.name """)).fetchall() suspects = [] for _cid, name, parent_id in rows: home = root_name.get(parent_id) if home in compiled and any(p.search(name) for _s, p in compiled[home]): continue # it fits where it already is elsewhere = [system for system, rules in compiled.items() if system != home and any(p.search(name) for _s, p in rules)] if elsewhere: suspects.append((name, home, elsewhere)) print(f"\n Possibly misfiled ({len(suspects)}) — review, nothing was moved:") for name, home, elsewhere in suspects[:40]: print(f" {name!r} is under {home} but matches {', '.join(elsewhere)}") if len(suspects) > 40: print(f" … and {len(suspects) - 40} more") def main(): apply_changes = "--apply" in sys.argv db = SessionLocal() try: roots = db.execute(sa_text( "SELECT id, name FROM question_categories WHERE parent_id IS NULL ORDER BY name")).fetchall() by_name = {row[1]: row[0] for row in roots} planned: dict[str, list[tuple[str, int, str]]] = defaultdict(list) untouched: dict[str, int] = {} for system, rules in SUBSYSTEMS.items(): root_id = by_name.get(system) if root_id is None: print(f" system {system!r} not found; skipping") continue children = db.execute(sa_text(""" SELECT id, name FROM question_categories WHERE parent_id = :p ORDER BY name """), {"p": root_id}).fetchall() existing_subs = {name for _cid, name in children if name in {s for s, _ in rules}} claimed = set() for subsystem, pattern in rules: matcher = re.compile(pattern, re.I) hits = [(cid, name) for cid, name in children if cid not in claimed and name not in existing_subs and matcher.search(name)] if len(hits) < MIN_CHILDREN: continue # not worth a branch of its own for cid, name in hits: claimed.add(cid) planned[system].append((subsystem, cid, name)) untouched[system] = len([c for c, _ in children if c not in claimed]) if apply_changes: for system, moves in planned.items(): root_id = by_name[system] sub_ids: dict[str, int] = {} for subsystem, cid, _name in moves: if subsystem not in sub_ids: found = db.execute(sa_text(""" SELECT id FROM question_categories WHERE parent_id = :p AND lower(name) = lower(:n) """), {"p": root_id, "n": subsystem}).first() if found: sub_ids[subsystem] = found[0] else: sub_ids[subsystem] = db.execute(sa_text(""" INSERT INTO question_categories (name, parent_id, user_id, created_at) VALUES (:n, :p, (SELECT id FROM users ORDER BY id LIMIT 1), NOW()) RETURNING id """), {"n": subsystem, "p": root_id}).scalar() db.execute(sa_text("UPDATE question_categories SET parent_id = :s WHERE id = :c"), {"s": sub_ids[subsystem], "c": cid}) db.commit() print("APPLIED" if apply_changes else "DRY RUN") total_moved = sum(len(v) for v in planned.values()) total_subs = sum(len({s for s, _, _ in v}) for v in planned.values()) print(f" subsystems created : {total_subs}") print(f" conditions nested : {total_moved}") print(f" left at top level : {sum(untouched.values())}\n") for system in sorted(planned): groups: dict[str, list[str]] = defaultdict(list) for subsystem, _cid, name in planned[system]: groups[subsystem].append(name) print(f" {system} (+{untouched.get(system, 0)} stay directly under it)") for subsystem, names in sorted(groups.items()): shown = ", ".join(names[:4]) more = f" … +{len(names) - 4}" if len(names) > 4 else "" print(f" {subsystem} ({len(names)}): {shown}{more}") report_misfiled(db) if not apply_changes: print("\n Re-run with --apply to write this structure.") finally: db.close() if __name__ == "__main__": sys.exit(main())