pdf-quiz-generator/backend/scripts/link_articles.py

"""Cross-reference the articles to each other, by id.

The marker system exists — `[[7|Febrile seizures]]` resolves by id, survives a
rename and shows a preview on hover. This reads what is written and applies it.

The first version linked the first mention in every *section*, which produced
3,560 links dominated by a handful of hub terms: Seizures 155 times, Sepsis
104, Meningitis 82. A reader does not need "seizures may occur" to be a link in
every article that mentions seizures; that is noise, and noise trains people to
stop clicking. The rule now:

  1. **First mention per view, not per section.** Short, Long and Clinical are
     read separately so each earns its own first link, but a term is linked
     once within a view rather than once per heading.
  2. **Lists are jump lists; prose is not.** In a differential, causes or
     complications list every distinct condition keeps its link — that is the
     one place a reader wants ten links in a row. In running prose, only the
     first mention.
  3. **Hub terms link from lists only.** A title mentioned across more than
     HUB_SHARE of all articles is too general to be worth a jump from prose.
     It still links as a list item, where it is something you might pick.
     Measure this on clean prose: counting mentions in text that is already
     linked hides most of them behind markers the word pattern will not match,
     which made the corpus look four times less repetitive than it is.
  4. **Specific beats general.** Longest title first, and a marker once made is
     protected, so "Otitis media with effusion" cannot be re-cut into
     "Otitis media".
  5. **Never** inside a heading, table, code span, fenced block, existing link
     or marker; never an article to itself; never a title under MIN_TITLE
     characters, which collide.
  6. **The educator wins.** A link whose label differs from the target's title
     was written by hand and is left alone — stripping and re-linking only ever
     touches links this script could have made. NEVER_LINK holds terms that
     should not auto-link at all.

Re-runnable: --apply strips the links it owns and reapplies the rule, so
changing the rule or the prose does not leave the old pass behind.

    docker compose exec backend python -m scripts.link_articles
    docker compose exec backend python -m scripts.link_articles --apply
    docker compose exec backend python -m scripts.link_articles --strip --apply
"""
import re
import sys
from collections import Counter, defaultdict

from sqlalchemy.orm.attributes import flag_modified

from app.database import SessionLocal
from app.models.article import Article

MIN_TITLE = 4
#: A title mentioned in more than this share of articles links from lists only.
#: At 5% of 333 articles that is 23 terms — Seizures (mentioned in 50), Sepsis
#: (46), Pneumonia (34), Respiratory Distress (34) and the like: the vocabulary
#: of paediatrics rather than a topic anyone would break off reading to visit.
HUB_SHARE = 0.05
#: Terms that are never worth a jump, however specific the match looks.
NEVER_LINK = {"history", "examination", "management", "treatment", "prognosis"}

MARKER = re.compile(r"\[\[(\d+)\|([^\]]+)\]\]")
#: Spans within a line that must not be linked into.
PROTECTED = re.compile(r"(\[\[[^\]]*\]\]|\[[^\]]*\]\([^)]*\)|`[^`]*`)")
HEADING = re.compile(r"^\s{0,3}#{1,6}\s")
TABLE = re.compile(r"^\s*\|")
FENCE = re.compile(r"^\s*(```|~~~)")
LIST_ITEM = re.compile(r"^\s*([-*+]|\d+[.)])\s")


def word_pattern(title: str) -> re.Pattern:
    """Whole-word, case-insensitive, and never biting into an existing marker."""
    return re.compile(rf"(?<![\w\[|]){re.escape(title)}(?![\w\]])", re.I)


#: A marker whose label swallowed another marker. The first version of this
#: script linked the longest title first and then let a shorter one cut into
#: the result, producing `[[363|[[245|gastroesophageal reflux]] disease]]` —
#: which renders as literal brackets and resolves to nothing.
NESTED = re.compile(r"\[\[(\d+)\|([^\[\]]*)\[\[\d+\|([^\[\]]*)\]\]([^\[\]]*)\]\]")


def unnest(text: str) -> tuple[str, int]:
    """Unwrap the inner marker, keeping the outer.

    The outer is the longer, more specific title — "Gastroesophageal reflux
    disease" over "Gastroesophageal reflux" — so it is the link worth keeping,
    and the inner one's label becomes plain words again.
    """
    if not text:
        return text, 0
    fixed = 0
    while True:
        text, count = NESTED.subn(lambda m: f"[[{m.group(1)}|{m.group(2)}{m.group(3)}{m.group(4)}]]", text)
        fixed += count
        if not count:            # nesting can be more than one deep
            return text, fixed


def strip_owned(text: str, titles_by_id: dict[int, str]) -> tuple[str, int]:
    """Remove the links this script owns, leaving the words behind.

    A link is ours when its label is the target's title. Anything else — a link
    an educator wrote as `[[7|this condition]]` — is left exactly as it is.
    """
    if not text:
        return text, 0
    removed = 0

    def drop(match: re.Match) -> str:
        nonlocal removed
        title = titles_by_id.get(int(match.group(1)))
        label = match.group(2)
        if title and label.strip().lower() == title.strip().lower():
            removed += 1
            return label
        return match.group(0)

    return MARKER.sub(drop, text), removed


def link_text(text, targets, self_id, seen, hubs):
    """Link `text` under the rule. `seen` is shared across one view and mutated.

    `targets` is (lowered title, id, is_hub), longest first.
    """
    if not text:
        return text, 0

    out, linked, in_fence = [], 0, False
    list_scope: set[int] | None = None

    for line in text.split("\n"):
        if FENCE.match(line):
            in_fence = not in_fence
            out.append(line)
            continue
        if in_fence or HEADING.match(line) or TABLE.match(line) or not line.strip():
            # A blank line ends a list block, so the next list starts fresh.
            if not line.strip():
                list_scope = None
            out.append(line)
            continue

        is_item = bool(LIST_ITEM.match(line))
        if is_item:
            if list_scope is None:
                list_scope = set()
        else:
            list_scope = None

        # Carve out spans that must not be linked into, link the rest, restore.
        holes: list[str] = []

        def stash(match: re.Match) -> str:
            holes.append(match.group(0))
            return f"\x00{len(holes) - 1}\x00"

        working = PROTECTED.sub(stash, line)

        for lowered, article_id, is_hub in targets:
            if article_id == self_id or lowered in NEVER_LINK:
                continue
            if is_item:
                # Rule 2: a jump list links each distinct target once per list.
                if article_id in list_scope:
                    continue
            else:
                # Rule 1 and 3: prose links a target once per view, and never
                # links a hub term at all.
                if is_hub or article_id in seen:
                    continue

            match = word_pattern(lowered).search(working)
            if not match:
                continue
            # Keep the author's casing; only the target is decided here. The
            # finished marker is stashed so a shorter title cannot re-cut it.
            holes.append(f"[[{article_id}|{match.group(0)}]]")
            working = f"{working[:match.start()]}\x00{len(holes) - 1}\x00{working[match.end():]}"
            linked += 1
            seen.add(article_id)
            if is_item:
                list_scope.add(article_id)

        out.append(re.sub(r"\x00(\d+)\x00", lambda m: holes[int(m.group(1))], working))

    return "\n".join(out), linked


def scopes(article) -> dict[str, list[dict]]:
    """Sections grouped by the view they belong to; each view links independently."""
    grouped: dict[str, list[dict]] = defaultdict(list)
    for section in article.sections or []:
        grouped[section.get("variant") or "long"].append(section)
    return grouped


def main() -> int:
    apply_changes = "--apply" in sys.argv
    strip_only = "--strip" in sys.argv
    db = SessionLocal()
    try:
        articles = db.query(Article).all()
        titles_by_id = {a.id: a.title for a in articles}

        # Strip first, always: the rule is applied to clean prose so a re-run
        # cannot layer a new pass on top of an old one.
        stripped = repaired = 0

        def clean(value):
            nonlocal stripped, repaired
            value, fixed = unnest(value)
            repaired += fixed
            value, removed = strip_owned(value, titles_by_id)
            stripped += removed
            return value

        for article in articles:
            for section in article.sections or []:
                section["content"] = clean(section.get("content"))
            article.summary = clean(article.summary)
            article.content = clean(article.content)
        print(f"  nested markers repaired      : {repaired}")
        print(f"  existing auto-links stripped: {stripped}")

        if strip_only:
            if apply_changes:
                for article in articles:
                    flag_modified(article, "sections")
                db.commit()
                print("  stripped and committed; prose is clean.")
            else:
                print("\n  Re-run with --apply.")
            return 0

        candidates = sorted(
            ((a.title.lower(), a.id) for a in articles if len(a.title or "") >= MIN_TITLE),
            key=lambda row: -len(row[0]),
        )

        # How widely each title is mentioned decides whether it is a hub. Counted
        # over the corpus as written, not over the links the last run happened
        # to make, so the threshold does not drift with its own output.
        bodies = {}
        for article in articles:
            parts = [article.summary or "", article.content or ""]
            parts += [s.get("content") or "" for s in (article.sections or [])]
            bodies[article.id] = "\n".join(parts).lower()

        mentions: Counter[int] = Counter()
        for lowered, article_id in candidates:
            pattern = word_pattern(lowered)
            for other_id, body in bodies.items():
                if other_id != article_id and lowered in body and pattern.search(body):
                    mentions[article_id] += 1

        cutoff = max(2, int(HUB_SHARE * len(articles)))
        hubs = {aid for aid, n in mentions.items() if n > cutoff}
        targets = [(lowered, aid, aid in hubs) for lowered, aid in candidates]
        print(f"  articles: {len(articles)}   linkable titles: {len(candidates)}")
        print(f"  hub cutoff: mentioned in more than {cutoff} articles -> {len(hubs)} hubs, list-only")
        for aid in sorted(hubs, key=lambda a: -mentions[a])[:12]:
            print(f"      {mentions[aid]:4d}  {titles_by_id[aid]}")

        touched = links = 0
        for article in articles:
            changed = False
            # The lead is its own scope: it is shown above every view.
            seen: set[int] = set()
            article.summary, n = link_text(article.summary, targets, article.id, seen, hubs)
            links += n
            changed |= bool(n)
            article.content, n = link_text(article.content, targets, article.id, seen, hubs)
            links += n
            changed |= bool(n)

            for _variant, sections in scopes(article).items():
                seen = set()
                for section in sections:
                    section["content"], n = link_text(
                        section.get("content"), targets, article.id, seen, hubs)
                    links += n
                    changed |= bool(n)
            if changed:
                touched += 1
            if apply_changes:
                flag_modified(article, "sections")

        print(f"  articles gaining links: {touched}")
        print(f"  links added           : {links}")
        if not apply_changes:
            print("\n  Re-run with --apply.")
            return 0
        db.commit()
        print("\n  Linked by id, so renaming an article cannot break them.")
    finally:
        db.close()
    return 0


if __name__ == "__main__":
    sys.exit(main())