feat: an organ system is a fact about a topic, not about a keyword

A question reached a system through a symptom keyword it happened to
mention — question → keyword → parent system — and only 726 of 4,281
keywords had ever been given a parent. The Systems tab saw 1,492 of 2,924
questions while Disciplines saw all of them.

The system now sits on the category: question_categories.system_id. Every
question has a category, so every question reaches a system. 2,919 of
2,924, and all sixteen buckets have real content.

It stays a third way of asking rather than the discipline tree relabelled
because a topic's system is assigned separately from where it sits in the
tree. scripts/assign_category_systems takes the discipline as a default
and lets the topic's own name overrule it, which is exactly the case that
makes the axis worth having: conjunctivitis is filed under Infectious
Disease and is an eye, osteomyelitis is filed there and is a bone. 110 of
660 topics were decided that way.

Two regex traps caught in the dry run and fixed before applying:
"adRENAL" matched the kidney rule, and "Abnormal Uterine Bleeding" matched
the bleeding rule. Both now have a specific rule above the general one.

I first tried to fix this by parenting the orphan keywords to systems,
deriving each keyword's system from the questions carrying it. The dry run
showed why that was the wrong shape: it reached only 534 of 3,555 orphans,
and inherited every coarse edge of the discipline map — conjunctivitis came
out as Multisystem because conjunctivitis questions are filed under
Infectious Disease. That script is left in place, unapplied, as the record
of a measurement worth keeping.

No ForeignKey on system_id in the model: question_tags is a raw-SQL table
with no ORM class, and declaring one leaves every metadata build unable to
resolve it. The constraint is real in Postgres.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN
This commit is contained in:
Daniel 2026-09-12 04:32:25 +02:00
parent 56ecbb0889
commit 13d5f50c58
6 changed files with 487 additions and 18 deletions

View file

@ -0,0 +1,32 @@
"""Give each topic an organ system of its own.
A question reached a system through a keyword it happened to carry, and only
about half of them carried one that had been filed. The system is a fact about
the topic, not about the wording of one question, so it belongs on the
category: every question has a category, so every question reaches a system.
Nullable, and null means nobody has said yet not "no system".
Revision ID: a3b4c5d6e7f8
Revises: f2a3b4c5d6e7
"""
import sqlalchemy as sa
from alembic import op
revision = "a3b4c5d6e7f8"
down_revision = "f2a3b4c5d6e7"
branch_labels = None
depends_on = None
def upgrade() -> None:
op.add_column("question_categories", sa.Column("system_id", sa.Integer(), nullable=True))
op.create_foreign_key("fk_question_categories_system", "question_categories",
"question_tags", ["system_id"], ["id"], ondelete="SET NULL")
op.create_index("ix_question_categories_system", "question_categories", ["system_id"])
def downgrade() -> None:
op.drop_index("ix_question_categories_system", table_name="question_categories")
op.drop_constraint("fk_question_categories_system", "question_categories", type_="foreignkey")
op.drop_column("question_categories", "system_id")

View file

@ -12,6 +12,15 @@ class QuestionCategory(Base):
name = Column(String, nullable=False)
description = Column(Text, nullable=True)
user_id = Column(Integer, ForeignKey("users.id"), nullable=False)
# The organ system this topic belongs to, said once here rather than
# inferred from a keyword on each question. A category's discipline is
# where it sits in the tree; its system is a separate fact about it —
# conjunctivitis is filed under Infectious Disease and is an eye.
#
# No ForeignKey in the model: `question_tags` is a raw-SQL table with no
# ORM class, so declaring one here leaves every metadata build unable to
# resolve it. The constraint is real in Postgres; the migration adds it.
system_id = Column(Integer, nullable=True)
created_at = Column(DateTime, default=datetime.utcnow)
questions = relationship("Question", back_populates="question_category",

View file

@ -68,28 +68,30 @@ class Grouping:
self.article_by_id = {a.id: a for a in self.articles.values()}
self.category_of_article = {a.id: cid for cid, a in self.articles.items()}
# A question reaches an organ system through the symptom keyword it
# carries, which is filed under one. No question is tagged with a
# system directly. Raw SQL because question_tags has no ORM model; the
# A question reaches an organ system through the topic it is filed
# under, which carries one. It used to go through a symptom keyword the
# question happened to mention, and only about half of them mentioned
# one that had been filed — so the Systems tab saw half the bank while
# Disciplines saw all of it.
#
# A category's system is a separate fact from where it sits in the
# tree, which is what keeps this a third way of asking rather than the
# discipline tree relabelled: conjunctivitis is filed under Infectious
# Disease and is an eye.
#
# Raw SQL for the names because question_tags has no ORM model; the
# tags router is the same, and one spelling of this table is better
# than two. Only read for the systems grouping, and only when the table
# is there — it is not part of the ORM metadata a fresh database is
# built from.
# than two.
self.system_names: dict[int, str] = {}
self.systems_of_question: dict[int, set[int]] = defaultdict(set)
self.system_of_category: dict[int, int] = {}
if group == "systems" and inspect(db.get_bind()).has_table("question_tags"):
for tag_id, name in db.execute(sa_text(
"SELECT id, name FROM question_tags WHERE type = 'system' AND parent_id IS NULL")).all():
self.system_names[tag_id] = name
if self.system_names:
for question_id, parent in db.execute(sa_text('''
SELECT l.question_id, t.parent_id
FROM question_tag_links l
JOIN question_tags t ON t.id = l.tag_id
WHERE t.parent_id IS NOT NULL
''')).all():
if parent in self.system_names:
self.systems_of_question[question_id].add(parent)
for cat in self.categories:
system_id = getattr(cat, "system_id", None)
if system_id in self.system_names:
self.system_of_category[cat.id] = system_id
def categories_for(self, question_id: int, primary: int | None) -> set[int]:
"""Every category a question counts towards, ancestors included."""
@ -101,9 +103,13 @@ class Grouping:
def keys_for(self, question_id: int, primary: int | None) -> set[int]:
"""Which rows of this grouping the question belongs to."""
if self.group == "systems":
return self.systems_of_question.get(question_id, set())
rolled = self.categories_for(question_id, primary)
if self.group == "systems":
# Every category the question counts towards, including the ones it
# inherits up the tree — a topic with no system of its own still
# reaches one through the topic above it.
return {self.system_of_category[cid] for cid in rolled
if cid in self.system_of_category}
if self.group == "articles":
return {self.articles[cid].id for cid in rolled if cid in self.articles}
return rolled & self.top_level

View file

@ -0,0 +1,185 @@
"""Say which organ system each topic belongs to.
Two passes, and the order matters.
First the discipline it sits under: a topic beneath Cardiology is
cardiovascular unless something says otherwise. That is right for most of the
tree and wrong for exactly the cases that make an organ-system axis worth
having conjunctivitis is filed under Infectious Disease and is an eye,
osteomyelitis is filed under Infectious Disease and is a bone.
So the second pass reads the topic's own name. A name that names an organ wins
over the shelf it was put on, which is the whole reason this axis is not just
the discipline tree relabelled.
Disciplines that span every system General Pediatrics, Primary Care, Surgery
assign nothing from the first pass; only a name that says an organ does.
Idempotent, and a dry run by default:
docker compose exec backend python -m scripts.assign_category_systems
docker compose exec backend python -m scripts.assign_category_systems --apply
docker compose exec backend python -m scripts.assign_category_systems --apply --force
"""
import re
import sys
from collections import Counter
from sqlalchemy import text as sa_text
from app.database import SessionLocal
ROOT_TO_SYSTEM = {
"Adolescent Medicine": "Human Development",
"Allergy/Immunology": "Immune System",
"Cardiology": "Cardiovascular System",
"Child Abuse and Neglect": "Social Sciences & Ethics",
"Critical Care": "Multisystem Processes & Disorders",
"Dermatology": "Skin & Subcutaneous Tissue",
"Developmental & Behavioral": "Human Development",
"Emergency Medicine": "Multisystem Processes & Disorders",
"Endocrinology": "Endocrine System",
"Gastroenterology & Nutrition": "Gastrointestinal System",
"Genetics & Metabolism": "Multisystem Processes & Disorders",
"Hematology-Oncology": "Blood & Lymphoreticular System",
"Infectious Disease": "Multisystem Processes & Disorders",
"Neonatology": "Human Development",
"Nephrology & Urology": "Renal & Urinary System",
"Neurology": "Nervous System & Special Senses",
"Ophthalmology": "Nervous System & Special Senses",
"Orthopedics & Sports": "Musculoskeletal System",
"Otolaryngology": "Nervous System & Special Senses",
"Professional Topics": "Social Sciences & Ethics",
"Psychiatry & Psychology": "Behavioral Health",
"Psychosocial Issues": "Behavioral Health",
"Pulmonology": "Respiratory System",
"Rheumatology": "Musculoskeletal System",
"Toxicology": "Multisystem Processes & Disorders",
}
#: A name that names an organ beats the shelf the topic was put on. Ordered:
#: the first match wins, so the specific sits above the general.
BY_NAME: list[tuple[str, str]] = [
(r"conjunctiv|uveitis|retin|cornea|glaucoma|strabism|amblyop|orbital|ocular|"
r"\beye|vision|optic|periorbital|chalazi|blephar|nystagmus", "Nervous System & Special Senses"),
(r"otitis|hearing|deaf|tympan|cochlea|mastoid|\bear\b|vestibul|labyrinth",
"Nervous System & Special Senses"),
(r"meningitis|encephal|seizure|epilep|migraine|headache|neuropath|myelin|"
r"cerebr|hydrocephal|neural tube|spina bifida|ataxia|palsy|concussion|coma",
"Nervous System & Special Senses"),
(r"osteomyel|arthritis|fracture|scoliosis|myosit|muscular dystroph|rickets|"
r"limp|\bbone|joint|septic arthritis|slipped capital|legg-calv", "Musculoskeletal System"),
(r"cellulit|impetigo|eczema|dermatitis|urticaria|psoria|acne|scabies|tinea|"
r"exanthem|rash|abscess|burn|molluscum|wart|alopecia|nevus|hemangioma",
"Skin & Subcutaneous Tissue"),
(r"pneumon|bronchiol|asthma|croup|epiglott|pertussis|cystic fibrosis|apnea|"
r"stridor|wheez|tuberculosis|pleural|respiratory|airway|sinusitis|pharyngitis|"
r"tonsill|rhinitis|\blung", "Respiratory System"),
(r"gastroenter|diarrhea|constipat|hepatitis|jaundice|intussuscept|pylor|"
r"appendicit|celiac|reflux|colitis|\bliver|biliary|pancreat|volvulus|"
r"hirschsprung|encopresis|\bstool", "Gastrointestinal System"),
(r"nephr|urinary|pyelonephritis|glomerul|proteinuria|hematuria|enuresis|"
r"\bkidney|bladder|vesicoureter|hydronephro|\brenal\b", "Renal & Urinary System"),
(r"endocardit|myocardit|kawasaki|congenital heart|arrhythm|hypertension|"
r"murmur|cardiomyopath|rheumatic fever|\bheart", "Cardiovascular System"),
(r"anemia|sickle|thalass|leukemi|lymphom|neutropen|thrombocytopen|hemophil|"
r"coagul|purpura|splenomegal|transfusion|bone marrow|epistaxis",
"Blood & Lymphoreticular System"),
(r"diabet|thyroid|adrenal|growth hormone|puberty|precocious|short stature|"
r"obesity|hypoglycem|congenital adrenal|calcium|rickets", "Endocrine System"),
(r"immunodefic|hiv\b|allerg|anaphyla|urticaria|vaccine|immuniz|scid|"
r"complement deficiency", "Immune System"),
(r"testic|scrotal|ovarian|menstrual|amenorrh|vaginal|pregnan|contracept|"
r"sexually transmitted|gonorrh|chlamydia|breast", "Reproductive System"),
(r"depress|anxiety|adhd|autism|suicide|eating disorder|substance|behaviou?r|"
r"psychos|bullying|school refusal", "Behavioral Health"),
(r"development|milestone|growth chart|newborn|premature|breastfeed|"
r"adolescen|puberty", "Human Development"),
(r"ethic|consent|confidential|disparit|advocacy|statistic|epidemiolog|"
r"screening test|sensitivity|specificity", "Social Sciences & Ethics"),
]
#: Read before BY_NAME. These are the names that a general rule below would
#: otherwise catch by accident — "adrenal" is not a kidney and "abnormal
#: uterine bleeding" is not a clotting disorder.
FIRST: list[tuple[str, str]] = [
(r"uterine|menorrh|menstrual|dysmenorrh", "Reproductive System"),
(r"adrenal|pituitar|parathyroid", "Endocrine System"),
]
COMPILED = [(re.compile(pattern, re.I), system) for pattern, system in FIRST + BY_NAME]
def by_name(name: str) -> str | None:
for pattern, system in COMPILED:
if pattern.search(name):
return system
return None
def main(apply: bool, force: bool) -> int:
db = SessionLocal()
try:
systems = {r.name: r.id for r in db.execute(sa_text(
"SELECT id, name FROM question_tags WHERE type = 'system' AND parent_id IS NULL")).fetchall()}
unknown = sorted({s for s in ROOT_TO_SYSTEM.values()} | {s for _, s in BY_NAME})
missing = [s for s in unknown if s not in systems]
if missing:
print(f"! these systems do not exist as tags: {', '.join(missing)}")
return 1
rows = db.execute(sa_text(
"SELECT id, name, parent_id, system_id FROM question_categories")).fetchall()
parents = {r.id: r.parent_id for r in rows}
names = {r.id: r.name for r in rows}
def root_of(cid):
walk, guard = cid, 0
while parents.get(walk) is not None and guard < 20:
walk = parents[walk]
guard += 1
return names.get(walk, "")
planned: dict[int, tuple[str, str]] = {} # id -> (system, why)
for row in rows:
if row.system_id and not force:
continue
named = by_name(row.name)
if named:
planned[row.id] = (named, "name")
continue
inherited = ROOT_TO_SYSTEM.get(root_of(row.id))
if inherited:
planned[row.id] = (inherited, "discipline")
already = sum(1 for r in rows if r.system_id)
print(f"categories {len(rows)}")
print(f" already assigned {already}")
print(f" from the topic's name {sum(1 for s, why in planned.values() if why == 'name')}")
print(f" from its discipline {sum(1 for s, why in planned.values() if why == 'discipline')}")
print(f" left unassigned {len(rows) - already - len(planned) if not force else len(rows) - len(planned)}")
for system, n in Counter(s for s, _ in planned.values()).most_common():
print(f" {system:<38} {n}")
overrides = [(names[cid], s) for cid, (s, why) in planned.items()
if why == "name" and ROOT_TO_SYSTEM.get(root_of(cid)) not in (None, s)]
print(f"\n {len(overrides)} topics where the name beat the shelf. A sample:")
for name, system in sorted(overrides)[:14]:
print(f" {name:<40}{system}")
if not apply:
print("\ndry run. Pass --apply to write.")
return 0
for cid, (system, _) in planned.items():
db.execute(sa_text("UPDATE question_categories SET system_id = :s WHERE id = :id"),
{"s": systems[system], "id": cid})
db.commit()
print(f"\nassigned {len(planned)} topics.")
return 0
finally:
db.close()
if __name__ == "__main__":
sys.exit(main("--apply" in sys.argv, "--force" in sys.argv))

View file

@ -0,0 +1,172 @@
"""File orphan symptom keywords under the organ system they actually describe.
A question reaches an organ system through the keyword it carries: keyword
parent system. Only 726 of our keywords have such a parent, so the Systems
grouping saw about half the bank while Disciplines saw all of it.
The obvious fix a hand-written table of keyword synonyms would be three
thousand clinical judgements made by whoever wrote the table. This asks the
data instead: every question already sits under a discipline, disciplines map
onto systems, and a keyword's system is the one its own questions agree on. A
keyword used almost entirely on cardiology questions belongs to the
cardiovascular system, and nobody had to assert that.
A keyword whose questions do *not* agree is left alone. "Family History" and
"Differential Diagnosis" are spread across every discipline, and that is the
evidence that they are not organ-system keywords at all.
Provenance keywords are skipped: "Board Review 2019" says where a question came
from, not what it is about.
Idempotent, and a dry run by default:
docker compose exec backend python -m scripts.parent_keywords_to_systems
docker compose exec backend python -m scripts.parent_keywords_to_systems --apply
"""
import re
import sys
from collections import Counter, defaultdict
from sqlalchemy import text as sa_text
from app.database import SessionLocal
#: The discipline a question is filed under, and the organ system that implies.
#: Left out on purpose: General Pediatrics, Primary Care & Prevention and
#: Surgery span every system, so a keyword under them is evidence of nothing.
#: Otolaryngology goes to Special Senses because that is what hearing is.
ROOT_TO_SYSTEM = {
"Adolescent Medicine": "Human Development",
"Allergy/Immunology": "Immune System",
"Cardiology": "Cardiovascular System",
"Child Abuse and Neglect": "Social Sciences & Ethics",
"Critical Care": "Multisystem Processes & Disorders",
"Dermatology": "Skin & Subcutaneous Tissue",
"Developmental & Behavioral": "Human Development",
"Emergency Medicine": "Multisystem Processes & Disorders",
"Endocrinology": "Endocrine System",
"Gastroenterology & Nutrition": "Gastrointestinal System",
"Genetics & Metabolism": "Multisystem Processes & Disorders",
"Hematology-Oncology": "Blood & Lymphoreticular System",
"Infectious Disease": "Multisystem Processes & Disorders",
"Neonatology": "Human Development",
"Nephrology & Urology": "Renal & Urinary System",
"Neurology": "Nervous System & Special Senses",
"Ophthalmology": "Nervous System & Special Senses",
"Orthopedics & Sports": "Musculoskeletal System",
"Otolaryngology": "Nervous System & Special Senses",
"Professional Topics": "Social Sciences & Ethics",
"Psychiatry & Psychology": "Behavioral Health",
"Psychosocial Issues": "Behavioral Health",
"Pulmonology": "Respiratory System",
"Rheumatology": "Musculoskeletal System",
"Toxicology": "Multisystem Processes & Disorders",
}
#: Where a question came from, not what it is about.
PROVENANCE = re.compile(r"^board review\b", re.I)
#: How much of a keyword's own evidence must point one way. Below this the
#: keyword is used across systems, which is a finding, not a gap.
AGREEMENT = 0.6
#: One question is an anecdote.
MIN_QUESTIONS = 2
def load_roots(db) -> dict[int, str]:
"""Every category mapped to the name of the root above it."""
rows = db.execute(sa_text("SELECT id, name, parent_id FROM question_categories")).fetchall()
parents = {r.id: r.parent_id for r in rows}
names = {r.id: r.name for r in rows}
roots: dict[int, str] = {}
for cid in parents:
walk, guard = cid, 0
while parents.get(walk) is not None and guard < 20:
walk = parents[walk]
guard += 1
roots[cid] = names.get(walk, "")
return roots
def main(apply: bool) -> int:
db = SessionLocal()
try:
systems = {r.name: r.id for r in db.execute(sa_text(
"SELECT id, name FROM question_tags WHERE type = 'system' AND parent_id IS NULL")).fetchall()}
missing = sorted(set(ROOT_TO_SYSTEM.values()) - set(systems))
if missing:
print(f"! these systems do not exist as tags: {', '.join(missing)}")
return 1
roots = load_roots(db)
# Which discipline each question sits under. The primary category
# first; an extra link only where there is no primary.
question_root: dict[int, str] = {}
for qid, cid in db.execute(sa_text(
"SELECT id, question_category_id FROM questions "
"WHERE deleted_at IS NULL AND question_category_id IS NOT NULL")).fetchall():
question_root[qid] = roots.get(cid, "")
for qid, cid in db.execute(sa_text(
"SELECT question_id, category_id FROM question_category_links")).fetchall():
question_root.setdefault(qid, roots.get(cid, ""))
orphans = db.execute(sa_text(
"SELECT id, name FROM question_tags WHERE type = 'keyword' AND parent_id IS NULL")).fetchall()
links = defaultdict(list)
for tag_id, qid in db.execute(sa_text(
"SELECT tag_id, question_id FROM question_tag_links")).fetchall():
links[tag_id].append(qid)
decided: list[tuple[int, str, str, int, int]] = []
spread, thin, provenance = [], [], 0
for tag in orphans:
if PROVENANCE.match(tag.name):
provenance += 1
continue
votes = Counter()
for qid in links.get(tag.id, ()):
system = ROOT_TO_SYSTEM.get(question_root.get(qid, ""))
if system:
votes[system] += 1
total = sum(votes.values())
if total < MIN_QUESTIONS:
thin.append(tag.name)
continue
system, count = votes.most_common(1)[0]
if count / total < AGREEMENT:
spread.append(tag.name)
continue
decided.append((tag.id, tag.name, system, count, total))
print(f"orphan keywords {len(orphans)}")
print(f" provenance, skipped {provenance}")
print(f" too few questions {len(thin)}")
print(f" used across systems {len(spread)}")
print(f" filed under a system {len(decided)}")
by_system = Counter(system for _, _, system, _, _ in decided)
for system, n in by_system.most_common():
print(f" {system:<38} {n}")
print("\n a sample of what moves:")
for _, name, system, count, total in sorted(
decided, key=lambda row: -row[4])[:15]:
print(f" {name:<34}{system:<34} ({count}/{total})")
if spread[:8]:
print("\n left alone, used across systems: " + ", ".join(sorted(spread)[:8]))
if not apply:
print("\ndry run. Pass --apply to write.")
return 0
for tag_id, _, system, _, _ in decided:
db.execute(sa_text("UPDATE question_tags SET parent_id = :p WHERE id = :id"),
{"p": systems[system], "id": tag_id})
db.commit()
print(f"\nfiled {len(decided)} keywords.")
return 0
finally:
db.close()
if __name__ == "__main__":
sys.exit(main("--apply" in sys.argv))

View file

@ -615,3 +615,68 @@ class SessionRecommendationTests(CompletionTests):
self.bank.user = self.bank.peer
self.assertEqual(
self.client.get(f'/attempts/{aid}/recommendations').status_code, 404)
class SystemAxisTests(CompletionTests):
"""A question reaches an organ system through the topic it is filed under.
It used to go through a symptom keyword the question happened to mention,
and only about half of them mentioned one that had been filed so the
Systems tab saw half the bank while Disciplines saw all of it.
What makes this a third way of asking rather than the discipline tree
relabelled: a topic's system is a separate fact from where it sits in the
tree. Conjunctivitis is filed under Infectious Disease and is an eye.
"""
def systems(self):
from app.services.knowledge_groups import Grouping
return Grouping(self.bank.db, "systems")
def make_system(self, name):
from sqlalchemy import text as sa_text
self.bank.db.execute(sa_text(
"CREATE TABLE IF NOT EXISTS question_tags "
"(id INTEGER PRIMARY KEY, name TEXT, type TEXT, parent_id INTEGER, sort_order INTEGER)"))
self.bank.db.execute(sa_text(
"INSERT INTO question_tags (name, type, parent_id) VALUES (:n, 'system', NULL)"),
{"n": name})
self.bank.db.commit()
return self.bank.db.execute(sa_text(
"SELECT id FROM question_tags WHERE name = :n"), {"n": name}).scalar()
def file_under(self, category_id, system_id):
from app.models.question_category import QuestionCategory
self.bank.db.get(QuestionCategory, category_id).system_id = system_id
self.bank.db.commit()
def test_a_question_reaches_the_system_of_its_topic(self):
eyes = self.make_system("Nervous System & Special Senses")
# Question 2 is filed under category 2 in the fixture tree.
self.file_under(2, eyes)
grouping = self.systems()
self.assertEqual(grouping.keys_for(2, 2), {eyes})
self.assertEqual(grouping.describe(eyes)["name"], "Nervous System & Special Senses")
self.assertEqual(grouping.describe(eyes)["system_id"], eyes)
def test_a_topic_inherits_the_system_of_the_topic_above_it(self):
chest = self.make_system("Respiratory System")
# Category 1 is the root; question 3 sits two levels below it.
self.file_under(1, chest)
# Reached through the tree, not through anything on the question.
self.assertEqual(self.systems().keys_for(3, 3), {chest})
def test_a_topic_says_its_own_system_over_the_one_above_it(self):
chest = self.make_system("Respiratory System")
eyes = self.make_system("Nervous System & Special Senses")
self.file_under(1, chest)
self.file_under(2, eyes)
# Both, because the question counts towards both topics — which is the
# honest answer for a question filed under a subtopic of another.
self.assertEqual(self.systems().keys_for(2, 2), {chest, eyes})
def test_a_topic_nobody_has_filed_reaches_nothing(self):
self.make_system("Respiratory System")
# Null means nobody has said yet, and the grouping says nothing rather
# than guessing a system from the shelf.
self.assertEqual(self.systems().keys_for(2, 2), set())