"""Cross-reference the articles to each other, by id. The marker system exists — `[[7|Febrile seizures]]` resolves by id, survives a rename and shows a preview on hover. This reads what is written and applies it. The first version linked the first mention in every *section*, which produced 3,560 links dominated by a handful of hub terms: Seizures 155 times, Sepsis 104, Meningitis 82. A reader does not need "seizures may occur" to be a link in every article that mentions seizures; that is noise, and noise trains people to stop clicking. The rule now: 1. **First mention per view, not per section.** Short, Long and Clinical are read separately so each earns its own first link, but a term is linked once within a view rather than once per heading. 2. **Lists are jump lists; prose is not.** In a differential, causes or complications list every distinct condition keeps its link — that is the one place a reader wants ten links in a row. In running prose, only the first mention. 3. **Hub terms link from lists only.** A title mentioned across more than HUB_SHARE of all articles is too general to be worth a jump from prose. It still links as a list item, where it is something you might pick. Measure this on clean prose: counting mentions in text that is already linked hides most of them behind markers the word pattern will not match, which made the corpus look four times less repetitive than it is. 4. **Specific beats general.** Longest title first, and a marker once made is protected, so "Otitis media with effusion" cannot be re-cut into "Otitis media". 5. **Never** inside a heading, table, code span, fenced block, existing link or marker; never an article to itself; never a title under MIN_TITLE characters, which collide. 6. **The educator wins.** A link whose label differs from the target's title was written by hand and is left alone — stripping and re-linking only ever touches links this script could have made. NEVER_LINK holds terms that should not auto-link at all. Re-runnable: --apply strips the links it owns and reapplies the rule, so changing the rule or the prose does not leave the old pass behind. docker compose exec backend python -m scripts.link_articles docker compose exec backend python -m scripts.link_articles --apply docker compose exec backend python -m scripts.link_articles --strip --apply """ import re import sys from collections import Counter, defaultdict from sqlalchemy.orm.attributes import flag_modified from app.database import SessionLocal from app.models.article import Article MIN_TITLE = 4 #: A title mentioned in more than this share of articles links from lists only. #: At 5% of 333 articles that is 23 terms — Seizures (mentioned in 50), Sepsis #: (46), Pneumonia (34), Respiratory Distress (34) and the like: the vocabulary #: of paediatrics rather than a topic anyone would break off reading to visit. HUB_SHARE = 0.05 #: Terms that are never worth a jump, however specific the match looks. NEVER_LINK = {"history", "examination", "management", "treatment", "prognosis"} MARKER = re.compile(r"\[\[(\d+)\|([^\]]+)\]\]") #: Spans within a line that must not be linked into. PROTECTED = re.compile(r"(\[\[[^\]]*\]\]|\[[^\]]*\]\([^)]*\)|`[^`]*`)") HEADING = re.compile(r"^\s{0,3}#{1,6}\s") TABLE = re.compile(r"^\s*\|") FENCE = re.compile(r"^\s*(```|~~~)") LIST_ITEM = re.compile(r"^\s*([-*+]|\d+[.)])\s") def word_pattern(title: str) -> re.Pattern: """Whole-word, case-insensitive, and never biting into an existing marker.""" return re.compile(rf"(? tuple[str, int]: """Remove the links this script owns, leaving the words behind. A link is ours when its label is the target's title. Anything else — a link an educator wrote as `[[7|this condition]]` — is left exactly as it is. """ if not text: return text, 0 removed = 0 def drop(match: re.Match) -> str: nonlocal removed title = titles_by_id.get(int(match.group(1))) label = match.group(2) if title and label.strip().lower() == title.strip().lower(): removed += 1 return label return match.group(0) return MARKER.sub(drop, text), removed def link_text(text, targets, self_id, seen, hubs): """Link `text` under the rule. `seen` is shared across one view and mutated. `targets` is (lowered title, id, is_hub), longest first. """ if not text: return text, 0 out, linked, in_fence = [], 0, False list_scope: set[int] | None = None for line in text.split("\n"): if FENCE.match(line): in_fence = not in_fence out.append(line) continue if in_fence or HEADING.match(line) or TABLE.match(line) or not line.strip(): # A blank line ends a list block, so the next list starts fresh. if not line.strip(): list_scope = None out.append(line) continue is_item = bool(LIST_ITEM.match(line)) if is_item: if list_scope is None: list_scope = set() else: list_scope = None # Carve out spans that must not be linked into, link the rest, restore. holes: list[str] = [] def stash(match: re.Match) -> str: holes.append(match.group(0)) return f"\x00{len(holes) - 1}\x00" working = PROTECTED.sub(stash, line) for lowered, article_id, is_hub in targets: if article_id == self_id or lowered in NEVER_LINK: continue if is_item: # Rule 2: a jump list links each distinct target once per list. if article_id in list_scope: continue else: # Rule 1 and 3: prose links a target once per view, and never # links a hub term at all. if is_hub or article_id in seen: continue match = word_pattern(lowered).search(working) if not match: continue # Keep the author's casing; only the target is decided here. The # finished marker is stashed so a shorter title cannot re-cut it. holes.append(f"[[{article_id}|{match.group(0)}]]") working = f"{working[:match.start()]}\x00{len(holes) - 1}\x00{working[match.end():]}" linked += 1 seen.add(article_id) if is_item: list_scope.add(article_id) out.append(re.sub(r"\x00(\d+)\x00", lambda m: holes[int(m.group(1))], working)) return "\n".join(out), linked def scopes(article) -> dict[str, list[dict]]: """Sections grouped by the view they belong to; each view links independently.""" grouped: dict[str, list[dict]] = defaultdict(list) for section in article.sections or []: grouped[section.get("variant") or "long"].append(section) return grouped def main() -> int: apply_changes = "--apply" in sys.argv strip_only = "--strip" in sys.argv db = SessionLocal() try: articles = db.query(Article).all() titles_by_id = {a.id: a.title for a in articles} # Strip first, always: the rule is applied to clean prose so a re-run # cannot layer a new pass on top of an old one. stripped = 0 for article in articles: for section in article.sections or []: section["content"], n = strip_owned(section.get("content"), titles_by_id) stripped += n article.summary, n = strip_owned(article.summary, titles_by_id) stripped += n article.content, n = strip_owned(article.content, titles_by_id) stripped += n print(f" existing auto-links stripped: {stripped}") if strip_only: if apply_changes: for article in articles: flag_modified(article, "sections") db.commit() print(" stripped and committed; prose is clean.") else: print("\n Re-run with --apply.") return 0 candidates = sorted( ((a.title.lower(), a.id) for a in articles if len(a.title or "") >= MIN_TITLE), key=lambda row: -len(row[0]), ) # How widely each title is mentioned decides whether it is a hub. Counted # over the corpus as written, not over the links the last run happened # to make, so the threshold does not drift with its own output. bodies = {} for article in articles: parts = [article.summary or "", article.content or ""] parts += [s.get("content") or "" for s in (article.sections or [])] bodies[article.id] = "\n".join(parts).lower() mentions: Counter[int] = Counter() for lowered, article_id in candidates: pattern = word_pattern(lowered) for other_id, body in bodies.items(): if other_id != article_id and lowered in body and pattern.search(body): mentions[article_id] += 1 cutoff = max(2, int(HUB_SHARE * len(articles))) hubs = {aid for aid, n in mentions.items() if n > cutoff} targets = [(lowered, aid, aid in hubs) for lowered, aid in candidates] print(f" articles: {len(articles)} linkable titles: {len(candidates)}") print(f" hub cutoff: mentioned in more than {cutoff} articles -> {len(hubs)} hubs, list-only") for aid in sorted(hubs, key=lambda a: -mentions[a])[:12]: print(f" {mentions[aid]:4d} {titles_by_id[aid]}") touched = links = 0 for article in articles: changed = False # The lead is its own scope: it is shown above every view. seen: set[int] = set() article.summary, n = link_text(article.summary, targets, article.id, seen, hubs) links += n changed |= bool(n) article.content, n = link_text(article.content, targets, article.id, seen, hubs) links += n changed |= bool(n) for _variant, sections in scopes(article).items(): seen = set() for section in sections: section["content"], n = link_text( section.get("content"), targets, article.id, seen, hubs) links += n changed |= bool(n) if changed: touched += 1 if apply_changes: flag_modified(article, "sections") print(f" articles gaining links: {touched}") print(f" links added : {links}") if not apply_changes: print("\n Re-run with --apply.") return 0 db.commit() print("\n Linked by id, so renaming an article cannot break them.") finally: db.close() return 0 if __name__ == "__main__": sys.exit(main())