diff --git a/backend/alembic/versions/a3b4c5d6e7f8_category_system.py b/backend/alembic/versions/a3b4c5d6e7f8_category_system.py new file mode 100644 index 0000000..9d1047f --- /dev/null +++ b/backend/alembic/versions/a3b4c5d6e7f8_category_system.py @@ -0,0 +1,32 @@ +"""Give each topic an organ system of its own. + +A question reached a system through a keyword it happened to carry, and only +about half of them carried one that had been filed. The system is a fact about +the topic, not about the wording of one question, so it belongs on the +category: every question has a category, so every question reaches a system. + +Nullable, and null means nobody has said yet — not "no system". + +Revision ID: a3b4c5d6e7f8 +Revises: f2a3b4c5d6e7 +""" +import sqlalchemy as sa +from alembic import op + +revision = "a3b4c5d6e7f8" +down_revision = "f2a3b4c5d6e7" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("question_categories", sa.Column("system_id", sa.Integer(), nullable=True)) + op.create_foreign_key("fk_question_categories_system", "question_categories", + "question_tags", ["system_id"], ["id"], ondelete="SET NULL") + op.create_index("ix_question_categories_system", "question_categories", ["system_id"]) + + +def downgrade() -> None: + op.drop_index("ix_question_categories_system", table_name="question_categories") + op.drop_constraint("fk_question_categories_system", "question_categories", type_="foreignkey") + op.drop_column("question_categories", "system_id") diff --git a/backend/app/models/question_category.py b/backend/app/models/question_category.py index bea318c..b0957bf 100644 --- a/backend/app/models/question_category.py +++ b/backend/app/models/question_category.py @@ -12,6 +12,15 @@ class QuestionCategory(Base): name = Column(String, nullable=False) description = Column(Text, nullable=True) user_id = Column(Integer, ForeignKey("users.id"), nullable=False) + # The organ system this topic belongs to, said once here rather than + # inferred from a keyword on each question. A category's discipline is + # where it sits in the tree; its system is a separate fact about it — + # conjunctivitis is filed under Infectious Disease and is an eye. + # + # No ForeignKey in the model: `question_tags` is a raw-SQL table with no + # ORM class, so declaring one here leaves every metadata build unable to + # resolve it. The constraint is real in Postgres; the migration adds it. + system_id = Column(Integer, nullable=True) created_at = Column(DateTime, default=datetime.utcnow) questions = relationship("Question", back_populates="question_category", diff --git a/backend/app/services/knowledge_groups.py b/backend/app/services/knowledge_groups.py index 54d6b4c..b9a4e4a 100644 --- a/backend/app/services/knowledge_groups.py +++ b/backend/app/services/knowledge_groups.py @@ -68,28 +68,30 @@ class Grouping: self.article_by_id = {a.id: a for a in self.articles.values()} self.category_of_article = {a.id: cid for cid, a in self.articles.items()} - # A question reaches an organ system through the symptom keyword it - # carries, which is filed under one. No question is tagged with a - # system directly. Raw SQL because question_tags has no ORM model; the + # A question reaches an organ system through the topic it is filed + # under, which carries one. It used to go through a symptom keyword the + # question happened to mention, and only about half of them mentioned + # one that had been filed — so the Systems tab saw half the bank while + # Disciplines saw all of it. + # + # A category's system is a separate fact from where it sits in the + # tree, which is what keeps this a third way of asking rather than the + # discipline tree relabelled: conjunctivitis is filed under Infectious + # Disease and is an eye. + # + # Raw SQL for the names because question_tags has no ORM model; the # tags router is the same, and one spelling of this table is better - # than two. Only read for the systems grouping, and only when the table - # is there — it is not part of the ORM metadata a fresh database is - # built from. + # than two. self.system_names: dict[int, str] = {} - self.systems_of_question: dict[int, set[int]] = defaultdict(set) + self.system_of_category: dict[int, int] = {} if group == "systems" and inspect(db.get_bind()).has_table("question_tags"): for tag_id, name in db.execute(sa_text( "SELECT id, name FROM question_tags WHERE type = 'system' AND parent_id IS NULL")).all(): self.system_names[tag_id] = name - if self.system_names: - for question_id, parent in db.execute(sa_text(''' - SELECT l.question_id, t.parent_id - FROM question_tag_links l - JOIN question_tags t ON t.id = l.tag_id - WHERE t.parent_id IS NOT NULL - ''')).all(): - if parent in self.system_names: - self.systems_of_question[question_id].add(parent) + for cat in self.categories: + system_id = getattr(cat, "system_id", None) + if system_id in self.system_names: + self.system_of_category[cat.id] = system_id def categories_for(self, question_id: int, primary: int | None) -> set[int]: """Every category a question counts towards, ancestors included.""" @@ -101,9 +103,13 @@ class Grouping: def keys_for(self, question_id: int, primary: int | None) -> set[int]: """Which rows of this grouping the question belongs to.""" - if self.group == "systems": - return self.systems_of_question.get(question_id, set()) rolled = self.categories_for(question_id, primary) + if self.group == "systems": + # Every category the question counts towards, including the ones it + # inherits up the tree — a topic with no system of its own still + # reaches one through the topic above it. + return {self.system_of_category[cid] for cid in rolled + if cid in self.system_of_category} if self.group == "articles": return {self.articles[cid].id for cid in rolled if cid in self.articles} return rolled & self.top_level diff --git a/backend/scripts/assign_category_systems.py b/backend/scripts/assign_category_systems.py new file mode 100644 index 0000000..9e8366e --- /dev/null +++ b/backend/scripts/assign_category_systems.py @@ -0,0 +1,185 @@ +"""Say which organ system each topic belongs to. + +Two passes, and the order matters. + +First the discipline it sits under: a topic beneath Cardiology is +cardiovascular unless something says otherwise. That is right for most of the +tree and wrong for exactly the cases that make an organ-system axis worth +having — conjunctivitis is filed under Infectious Disease and is an eye, +osteomyelitis is filed under Infectious Disease and is a bone. + +So the second pass reads the topic's own name. A name that names an organ wins +over the shelf it was put on, which is the whole reason this axis is not just +the discipline tree relabelled. + +Disciplines that span every system — General Pediatrics, Primary Care, Surgery +— assign nothing from the first pass; only a name that says an organ does. + +Idempotent, and a dry run by default: + + docker compose exec backend python -m scripts.assign_category_systems + docker compose exec backend python -m scripts.assign_category_systems --apply + docker compose exec backend python -m scripts.assign_category_systems --apply --force +""" +import re +import sys +from collections import Counter + +from sqlalchemy import text as sa_text + +from app.database import SessionLocal + +ROOT_TO_SYSTEM = { + "Adolescent Medicine": "Human Development", + "Allergy/Immunology": "Immune System", + "Cardiology": "Cardiovascular System", + "Child Abuse and Neglect": "Social Sciences & Ethics", + "Critical Care": "Multisystem Processes & Disorders", + "Dermatology": "Skin & Subcutaneous Tissue", + "Developmental & Behavioral": "Human Development", + "Emergency Medicine": "Multisystem Processes & Disorders", + "Endocrinology": "Endocrine System", + "Gastroenterology & Nutrition": "Gastrointestinal System", + "Genetics & Metabolism": "Multisystem Processes & Disorders", + "Hematology-Oncology": "Blood & Lymphoreticular System", + "Infectious Disease": "Multisystem Processes & Disorders", + "Neonatology": "Human Development", + "Nephrology & Urology": "Renal & Urinary System", + "Neurology": "Nervous System & Special Senses", + "Ophthalmology": "Nervous System & Special Senses", + "Orthopedics & Sports": "Musculoskeletal System", + "Otolaryngology": "Nervous System & Special Senses", + "Professional Topics": "Social Sciences & Ethics", + "Psychiatry & Psychology": "Behavioral Health", + "Psychosocial Issues": "Behavioral Health", + "Pulmonology": "Respiratory System", + "Rheumatology": "Musculoskeletal System", + "Toxicology": "Multisystem Processes & Disorders", +} + +#: A name that names an organ beats the shelf the topic was put on. Ordered: +#: the first match wins, so the specific sits above the general. +BY_NAME: list[tuple[str, str]] = [ + (r"conjunctiv|uveitis|retin|cornea|glaucoma|strabism|amblyop|orbital|ocular|" + r"\beye|vision|optic|periorbital|chalazi|blephar|nystagmus", "Nervous System & Special Senses"), + (r"otitis|hearing|deaf|tympan|cochlea|mastoid|\bear\b|vestibul|labyrinth", + "Nervous System & Special Senses"), + (r"meningitis|encephal|seizure|epilep|migraine|headache|neuropath|myelin|" + r"cerebr|hydrocephal|neural tube|spina bifida|ataxia|palsy|concussion|coma", + "Nervous System & Special Senses"), + (r"osteomyel|arthritis|fracture|scoliosis|myosit|muscular dystroph|rickets|" + r"limp|\bbone|joint|septic arthritis|slipped capital|legg-calv", "Musculoskeletal System"), + (r"cellulit|impetigo|eczema|dermatitis|urticaria|psoria|acne|scabies|tinea|" + r"exanthem|rash|abscess|burn|molluscum|wart|alopecia|nevus|hemangioma", + "Skin & Subcutaneous Tissue"), + (r"pneumon|bronchiol|asthma|croup|epiglott|pertussis|cystic fibrosis|apnea|" + r"stridor|wheez|tuberculosis|pleural|respiratory|airway|sinusitis|pharyngitis|" + r"tonsill|rhinitis|\blung", "Respiratory System"), + (r"gastroenter|diarrhea|constipat|hepatitis|jaundice|intussuscept|pylor|" + r"appendicit|celiac|reflux|colitis|\bliver|biliary|pancreat|volvulus|" + r"hirschsprung|encopresis|\bstool", "Gastrointestinal System"), + (r"nephr|urinary|pyelonephritis|glomerul|proteinuria|hematuria|enuresis|" + r"\bkidney|bladder|vesicoureter|hydronephro|\brenal\b", "Renal & Urinary System"), + (r"endocardit|myocardit|kawasaki|congenital heart|arrhythm|hypertension|" + r"murmur|cardiomyopath|rheumatic fever|\bheart", "Cardiovascular System"), + (r"anemia|sickle|thalass|leukemi|lymphom|neutropen|thrombocytopen|hemophil|" + r"coagul|purpura|splenomegal|transfusion|bone marrow|epistaxis", + "Blood & Lymphoreticular System"), + (r"diabet|thyroid|adrenal|growth hormone|puberty|precocious|short stature|" + r"obesity|hypoglycem|congenital adrenal|calcium|rickets", "Endocrine System"), + (r"immunodefic|hiv\b|allerg|anaphyla|urticaria|vaccine|immuniz|scid|" + r"complement deficiency", "Immune System"), + (r"testic|scrotal|ovarian|menstrual|amenorrh|vaginal|pregnan|contracept|" + r"sexually transmitted|gonorrh|chlamydia|breast", "Reproductive System"), + (r"depress|anxiety|adhd|autism|suicide|eating disorder|substance|behaviou?r|" + r"psychos|bullying|school refusal", "Behavioral Health"), + (r"development|milestone|growth chart|newborn|premature|breastfeed|" + r"adolescen|puberty", "Human Development"), + (r"ethic|consent|confidential|disparit|advocacy|statistic|epidemiolog|" + r"screening test|sensitivity|specificity", "Social Sciences & Ethics"), +] + +#: Read before BY_NAME. These are the names that a general rule below would +#: otherwise catch by accident — "adrenal" is not a kidney and "abnormal +#: uterine bleeding" is not a clotting disorder. +FIRST: list[tuple[str, str]] = [ + (r"uterine|menorrh|menstrual|dysmenorrh", "Reproductive System"), + (r"adrenal|pituitar|parathyroid", "Endocrine System"), +] + +COMPILED = [(re.compile(pattern, re.I), system) for pattern, system in FIRST + BY_NAME] + + +def by_name(name: str) -> str | None: + for pattern, system in COMPILED: + if pattern.search(name): + return system + return None + + +def main(apply: bool, force: bool) -> int: + db = SessionLocal() + try: + systems = {r.name: r.id for r in db.execute(sa_text( + "SELECT id, name FROM question_tags WHERE type = 'system' AND parent_id IS NULL")).fetchall()} + unknown = sorted({s for s in ROOT_TO_SYSTEM.values()} | {s for _, s in BY_NAME}) + missing = [s for s in unknown if s not in systems] + if missing: + print(f"! these systems do not exist as tags: {', '.join(missing)}") + return 1 + + rows = db.execute(sa_text( + "SELECT id, name, parent_id, system_id FROM question_categories")).fetchall() + parents = {r.id: r.parent_id for r in rows} + names = {r.id: r.name for r in rows} + + def root_of(cid): + walk, guard = cid, 0 + while parents.get(walk) is not None and guard < 20: + walk = parents[walk] + guard += 1 + return names.get(walk, "") + + planned: dict[int, tuple[str, str]] = {} # id -> (system, why) + for row in rows: + if row.system_id and not force: + continue + named = by_name(row.name) + if named: + planned[row.id] = (named, "name") + continue + inherited = ROOT_TO_SYSTEM.get(root_of(row.id)) + if inherited: + planned[row.id] = (inherited, "discipline") + + already = sum(1 for r in rows if r.system_id) + print(f"categories {len(rows)}") + print(f" already assigned {already}") + print(f" from the topic's name {sum(1 for s, why in planned.values() if why == 'name')}") + print(f" from its discipline {sum(1 for s, why in planned.values() if why == 'discipline')}") + print(f" left unassigned {len(rows) - already - len(planned) if not force else len(rows) - len(planned)}") + for system, n in Counter(s for s, _ in planned.values()).most_common(): + print(f" {system:<38} {n}") + + overrides = [(names[cid], s) for cid, (s, why) in planned.items() + if why == "name" and ROOT_TO_SYSTEM.get(root_of(cid)) not in (None, s)] + print(f"\n {len(overrides)} topics where the name beat the shelf. A sample:") + for name, system in sorted(overrides)[:14]: + print(f" {name:<40} → {system}") + + if not apply: + print("\ndry run. Pass --apply to write.") + return 0 + + for cid, (system, _) in planned.items(): + db.execute(sa_text("UPDATE question_categories SET system_id = :s WHERE id = :id"), + {"s": systems[system], "id": cid}) + db.commit() + print(f"\nassigned {len(planned)} topics.") + return 0 + finally: + db.close() + + +if __name__ == "__main__": + sys.exit(main("--apply" in sys.argv, "--force" in sys.argv)) diff --git a/backend/scripts/parent_keywords_to_systems.py b/backend/scripts/parent_keywords_to_systems.py new file mode 100644 index 0000000..2c63424 --- /dev/null +++ b/backend/scripts/parent_keywords_to_systems.py @@ -0,0 +1,172 @@ +"""File orphan symptom keywords under the organ system they actually describe. + +A question reaches an organ system through the keyword it carries: keyword → +parent system. Only 726 of our keywords have such a parent, so the Systems +grouping saw about half the bank while Disciplines saw all of it. + +The obvious fix — a hand-written table of keyword synonyms — would be three +thousand clinical judgements made by whoever wrote the table. This asks the +data instead: every question already sits under a discipline, disciplines map +onto systems, and a keyword's system is the one its own questions agree on. A +keyword used almost entirely on cardiology questions belongs to the +cardiovascular system, and nobody had to assert that. + +A keyword whose questions do *not* agree is left alone. "Family History" and +"Differential Diagnosis" are spread across every discipline, and that is the +evidence that they are not organ-system keywords at all. + +Provenance keywords are skipped: "Board Review 2019" says where a question came +from, not what it is about. + +Idempotent, and a dry run by default: + + docker compose exec backend python -m scripts.parent_keywords_to_systems + docker compose exec backend python -m scripts.parent_keywords_to_systems --apply +""" +import re +import sys +from collections import Counter, defaultdict + +from sqlalchemy import text as sa_text + +from app.database import SessionLocal + +#: The discipline a question is filed under, and the organ system that implies. +#: Left out on purpose: General Pediatrics, Primary Care & Prevention and +#: Surgery span every system, so a keyword under them is evidence of nothing. +#: Otolaryngology goes to Special Senses because that is what hearing is. +ROOT_TO_SYSTEM = { + "Adolescent Medicine": "Human Development", + "Allergy/Immunology": "Immune System", + "Cardiology": "Cardiovascular System", + "Child Abuse and Neglect": "Social Sciences & Ethics", + "Critical Care": "Multisystem Processes & Disorders", + "Dermatology": "Skin & Subcutaneous Tissue", + "Developmental & Behavioral": "Human Development", + "Emergency Medicine": "Multisystem Processes & Disorders", + "Endocrinology": "Endocrine System", + "Gastroenterology & Nutrition": "Gastrointestinal System", + "Genetics & Metabolism": "Multisystem Processes & Disorders", + "Hematology-Oncology": "Blood & Lymphoreticular System", + "Infectious Disease": "Multisystem Processes & Disorders", + "Neonatology": "Human Development", + "Nephrology & Urology": "Renal & Urinary System", + "Neurology": "Nervous System & Special Senses", + "Ophthalmology": "Nervous System & Special Senses", + "Orthopedics & Sports": "Musculoskeletal System", + "Otolaryngology": "Nervous System & Special Senses", + "Professional Topics": "Social Sciences & Ethics", + "Psychiatry & Psychology": "Behavioral Health", + "Psychosocial Issues": "Behavioral Health", + "Pulmonology": "Respiratory System", + "Rheumatology": "Musculoskeletal System", + "Toxicology": "Multisystem Processes & Disorders", +} + +#: Where a question came from, not what it is about. +PROVENANCE = re.compile(r"^board review\b", re.I) + +#: How much of a keyword's own evidence must point one way. Below this the +#: keyword is used across systems, which is a finding, not a gap. +AGREEMENT = 0.6 +#: One question is an anecdote. +MIN_QUESTIONS = 2 + + +def load_roots(db) -> dict[int, str]: + """Every category mapped to the name of the root above it.""" + rows = db.execute(sa_text("SELECT id, name, parent_id FROM question_categories")).fetchall() + parents = {r.id: r.parent_id for r in rows} + names = {r.id: r.name for r in rows} + roots: dict[int, str] = {} + for cid in parents: + walk, guard = cid, 0 + while parents.get(walk) is not None and guard < 20: + walk = parents[walk] + guard += 1 + roots[cid] = names.get(walk, "") + return roots + + +def main(apply: bool) -> int: + db = SessionLocal() + try: + systems = {r.name: r.id for r in db.execute(sa_text( + "SELECT id, name FROM question_tags WHERE type = 'system' AND parent_id IS NULL")).fetchall()} + missing = sorted(set(ROOT_TO_SYSTEM.values()) - set(systems)) + if missing: + print(f"! these systems do not exist as tags: {', '.join(missing)}") + return 1 + + roots = load_roots(db) + + # Which discipline each question sits under. The primary category + # first; an extra link only where there is no primary. + question_root: dict[int, str] = {} + for qid, cid in db.execute(sa_text( + "SELECT id, question_category_id FROM questions " + "WHERE deleted_at IS NULL AND question_category_id IS NOT NULL")).fetchall(): + question_root[qid] = roots.get(cid, "") + for qid, cid in db.execute(sa_text( + "SELECT question_id, category_id FROM question_category_links")).fetchall(): + question_root.setdefault(qid, roots.get(cid, "")) + + orphans = db.execute(sa_text( + "SELECT id, name FROM question_tags WHERE type = 'keyword' AND parent_id IS NULL")).fetchall() + links = defaultdict(list) + for tag_id, qid in db.execute(sa_text( + "SELECT tag_id, question_id FROM question_tag_links")).fetchall(): + links[tag_id].append(qid) + + decided: list[tuple[int, str, str, int, int]] = [] + spread, thin, provenance = [], [], 0 + for tag in orphans: + if PROVENANCE.match(tag.name): + provenance += 1 + continue + votes = Counter() + for qid in links.get(tag.id, ()): + system = ROOT_TO_SYSTEM.get(question_root.get(qid, "")) + if system: + votes[system] += 1 + total = sum(votes.values()) + if total < MIN_QUESTIONS: + thin.append(tag.name) + continue + system, count = votes.most_common(1)[0] + if count / total < AGREEMENT: + spread.append(tag.name) + continue + decided.append((tag.id, tag.name, system, count, total)) + + print(f"orphan keywords {len(orphans)}") + print(f" provenance, skipped {provenance}") + print(f" too few questions {len(thin)}") + print(f" used across systems {len(spread)}") + print(f" filed under a system {len(decided)}") + by_system = Counter(system for _, _, system, _, _ in decided) + for system, n in by_system.most_common(): + print(f" {system:<38} {n}") + print("\n a sample of what moves:") + for _, name, system, count, total in sorted( + decided, key=lambda row: -row[4])[:15]: + print(f" {name:<34} → {system:<34} ({count}/{total})") + if spread[:8]: + print("\n left alone, used across systems: " + ", ".join(sorted(spread)[:8])) + + if not apply: + print("\ndry run. Pass --apply to write.") + return 0 + + for tag_id, _, system, _, _ in decided: + db.execute(sa_text("UPDATE question_tags SET parent_id = :p WHERE id = :id"), + {"p": systems[system], "id": tag_id}) + db.commit() + print(f"\nfiled {len(decided)} keywords.") + return 0 + finally: + db.close() + + +if __name__ == "__main__": + sys.exit(main("--apply" in sys.argv)) diff --git a/backend/tests/test_study_tools.py b/backend/tests/test_study_tools.py index 6ac8eea..c1cfbed 100644 --- a/backend/tests/test_study_tools.py +++ b/backend/tests/test_study_tools.py @@ -615,3 +615,68 @@ class SessionRecommendationTests(CompletionTests): self.bank.user = self.bank.peer self.assertEqual( self.client.get(f'/attempts/{aid}/recommendations').status_code, 404) + + +class SystemAxisTests(CompletionTests): + """A question reaches an organ system through the topic it is filed under. + + It used to go through a symptom keyword the question happened to mention, + and only about half of them mentioned one that had been filed — so the + Systems tab saw half the bank while Disciplines saw all of it. + + What makes this a third way of asking rather than the discipline tree + relabelled: a topic's system is a separate fact from where it sits in the + tree. Conjunctivitis is filed under Infectious Disease and is an eye. + """ + + def systems(self): + from app.services.knowledge_groups import Grouping + return Grouping(self.bank.db, "systems") + + def make_system(self, name): + from sqlalchemy import text as sa_text + self.bank.db.execute(sa_text( + "CREATE TABLE IF NOT EXISTS question_tags " + "(id INTEGER PRIMARY KEY, name TEXT, type TEXT, parent_id INTEGER, sort_order INTEGER)")) + self.bank.db.execute(sa_text( + "INSERT INTO question_tags (name, type, parent_id) VALUES (:n, 'system', NULL)"), + {"n": name}) + self.bank.db.commit() + return self.bank.db.execute(sa_text( + "SELECT id FROM question_tags WHERE name = :n"), {"n": name}).scalar() + + def file_under(self, category_id, system_id): + from app.models.question_category import QuestionCategory + self.bank.db.get(QuestionCategory, category_id).system_id = system_id + self.bank.db.commit() + + def test_a_question_reaches_the_system_of_its_topic(self): + eyes = self.make_system("Nervous System & Special Senses") + # Question 2 is filed under category 2 in the fixture tree. + self.file_under(2, eyes) + grouping = self.systems() + self.assertEqual(grouping.keys_for(2, 2), {eyes}) + self.assertEqual(grouping.describe(eyes)["name"], "Nervous System & Special Senses") + self.assertEqual(grouping.describe(eyes)["system_id"], eyes) + + def test_a_topic_inherits_the_system_of_the_topic_above_it(self): + chest = self.make_system("Respiratory System") + # Category 1 is the root; question 3 sits two levels below it. + self.file_under(1, chest) + # Reached through the tree, not through anything on the question. + self.assertEqual(self.systems().keys_for(3, 3), {chest}) + + def test_a_topic_says_its_own_system_over_the_one_above_it(self): + chest = self.make_system("Respiratory System") + eyes = self.make_system("Nervous System & Special Senses") + self.file_under(1, chest) + self.file_under(2, eyes) + # Both, because the question counts towards both topics — which is the + # honest answer for a question filed under a subtopic of another. + self.assertEqual(self.systems().keys_for(2, 2), {chest, eyes}) + + def test_a_topic_nobody_has_filed_reaches_nothing(self): + self.make_system("Respiratory System") + # Null means nobody has said yet, and the grouping says nothing rather + # than guessing a system from the shelf. + self.assertEqual(self.systems().keys_for(2, 2), set())