"""File orphan symptom keywords under the organ system they actually describe. A question reaches an organ system through the keyword it carries: keyword → parent system. Only 726 of our keywords have such a parent, so the Systems grouping saw about half the bank while Disciplines saw all of it. The obvious fix — a hand-written table of keyword synonyms — would be three thousand clinical judgements made by whoever wrote the table. This asks the data instead: every question already sits under a discipline, disciplines map onto systems, and a keyword's system is the one its own questions agree on. A keyword used almost entirely on cardiology questions belongs to the cardiovascular system, and nobody had to assert that. A keyword whose questions do *not* agree is left alone. "Family History" and "Differential Diagnosis" are spread across every discipline, and that is the evidence that they are not organ-system keywords at all. Provenance keywords are skipped: "Board Review 2019" says where a question came from, not what it is about. Idempotent, and a dry run by default: docker compose exec backend python -m scripts.parent_keywords_to_systems docker compose exec backend python -m scripts.parent_keywords_to_systems --apply """ import re import sys from collections import Counter, defaultdict from sqlalchemy import text as sa_text from app.database import SessionLocal #: The discipline a question is filed under, and the organ system that implies. #: Left out on purpose: General Pediatrics, Primary Care & Prevention and #: Surgery span every system, so a keyword under them is evidence of nothing. #: Otolaryngology goes to Special Senses because that is what hearing is. ROOT_TO_SYSTEM = { "Adolescent Medicine": "Human Development", "Allergy/Immunology": "Immune System", "Cardiology": "Cardiovascular System", "Child Abuse and Neglect": "Social Sciences & Ethics", "Critical Care": "Multisystem Processes & Disorders", "Dermatology": "Skin & Subcutaneous Tissue", "Developmental & Behavioral": "Human Development", "Emergency Medicine": "Multisystem Processes & Disorders", "Endocrinology": "Endocrine System", "Gastroenterology & Nutrition": "Gastrointestinal System", "Genetics & Metabolism": "Multisystem Processes & Disorders", "Hematology-Oncology": "Blood & Lymphoreticular System", "Infectious Disease": "Multisystem Processes & Disorders", "Neonatology": "Human Development", "Nephrology & Urology": "Renal & Urinary System", "Neurology": "Nervous System & Special Senses", "Ophthalmology": "Nervous System & Special Senses", "Orthopedics & Sports": "Musculoskeletal System", "Otolaryngology": "Nervous System & Special Senses", "Professional Topics": "Social Sciences & Ethics", "Psychiatry & Psychology": "Behavioral Health", "Psychosocial Issues": "Behavioral Health", "Pulmonology": "Respiratory System", "Rheumatology": "Musculoskeletal System", "Toxicology": "Multisystem Processes & Disorders", } #: Where a question came from, not what it is about. PROVENANCE = re.compile(r"^board review\b", re.I) #: How much of a keyword's own evidence must point one way. Below this the #: keyword is used across systems, which is a finding, not a gap. AGREEMENT = 0.6 #: One question is an anecdote. MIN_QUESTIONS = 2 def load_roots(db) -> dict[int, str]: """Every category mapped to the name of the root above it.""" rows = db.execute(sa_text("SELECT id, name, parent_id FROM question_categories")).fetchall() parents = {r.id: r.parent_id for r in rows} names = {r.id: r.name for r in rows} roots: dict[int, str] = {} for cid in parents: walk, guard = cid, 0 while parents.get(walk) is not None and guard < 20: walk = parents[walk] guard += 1 roots[cid] = names.get(walk, "") return roots def main(apply: bool) -> int: db = SessionLocal() try: systems = {r.name: r.id for r in db.execute(sa_text( "SELECT id, name FROM question_tags WHERE type = 'system' AND parent_id IS NULL")).fetchall()} missing = sorted(set(ROOT_TO_SYSTEM.values()) - set(systems)) if missing: print(f"! these systems do not exist as tags: {', '.join(missing)}") return 1 roots = load_roots(db) # Which discipline each question sits under. The primary category # first; an extra link only where there is no primary. question_root: dict[int, str] = {} for qid, cid in db.execute(sa_text( "SELECT id, question_category_id FROM questions " "WHERE deleted_at IS NULL AND question_category_id IS NOT NULL")).fetchall(): question_root[qid] = roots.get(cid, "") for qid, cid in db.execute(sa_text( "SELECT question_id, category_id FROM question_category_links")).fetchall(): question_root.setdefault(qid, roots.get(cid, "")) orphans = db.execute(sa_text( "SELECT id, name FROM question_tags WHERE type = 'keyword' AND parent_id IS NULL")).fetchall() links = defaultdict(list) for tag_id, qid in db.execute(sa_text( "SELECT tag_id, question_id FROM question_tag_links")).fetchall(): links[tag_id].append(qid) decided: list[tuple[int, str, str, int, int]] = [] spread, thin, provenance = [], [], 0 for tag in orphans: if PROVENANCE.match(tag.name): provenance += 1 continue votes = Counter() for qid in links.get(tag.id, ()): system = ROOT_TO_SYSTEM.get(question_root.get(qid, "")) if system: votes[system] += 1 total = sum(votes.values()) if total < MIN_QUESTIONS: thin.append(tag.name) continue system, count = votes.most_common(1)[0] if count / total < AGREEMENT: spread.append(tag.name) continue decided.append((tag.id, tag.name, system, count, total)) print(f"orphan keywords {len(orphans)}") print(f" provenance, skipped {provenance}") print(f" too few questions {len(thin)}") print(f" used across systems {len(spread)}") print(f" filed under a system {len(decided)}") by_system = Counter(system for _, _, system, _, _ in decided) for system, n in by_system.most_common(): print(f" {system:<38} {n}") print("\n a sample of what moves:") for _, name, system, count, total in sorted( decided, key=lambda row: -row[4])[:15]: print(f" {name:<34} → {system:<34} ({count}/{total})") if spread[:8]: print("\n left alone, used across systems: " + ", ".join(sorted(spread)[:8])) if not apply: print("\ndry run. Pass --apply to write.") return 0 for tag_id, _, system, _, _ in decided: db.execute(sa_text("UPDATE question_tags SET parent_id = :p WHERE id = :id"), {"p": systems[system], "id": tag_id}) db.commit() print(f"\nfiled {len(decided)} keywords.") return 0 finally: db.close() if __name__ == "__main__": sys.exit(main("--apply" in sys.argv))