diff --git a/backend/scripts/link_articles.py b/backend/scripts/link_articles.py index 07bf8b3..352a3c6 100644 --- a/backend/scripts/link_articles.py +++ b/backend/scripts/link_articles.py @@ -19,6 +19,9 @@ stop clicking. The rule now: 3. **Hub terms link from lists only.** A title mentioned across more than HUB_SHARE of all articles is too general to be worth a jump from prose. It still links as a list item, where it is something you might pick. + Measure this on clean prose: counting mentions in text that is already + linked hides most of them behind markers the word pattern will not match, + which made the corpus look four times less repetitive than it is. 4. **Specific beats general.** Longest title first, and a marker once made is protected, so "Otitis media with effusion" cannot be re-cut into "Otitis media". @@ -48,7 +51,10 @@ from app.models.article import Article MIN_TITLE = 4 #: A title mentioned in more than this share of articles links from lists only. -HUB_SHARE = 0.25 +#: At 5% of 333 articles that is 23 terms — Seizures (mentioned in 50), Sepsis +#: (46), Pneumonia (34), Respiratory Distress (34) and the like: the vocabulary +#: of paediatrics rather than a topic anyone would break off reading to visit. +HUB_SHARE = 0.05 #: Terms that are never worth a jump, however specific the match looks. NEVER_LINK = {"history", "examination", "management", "treatment", "prognosis"} @@ -60,8 +66,6 @@ TABLE = re.compile(r"^\s*\|") FENCE = re.compile(r"^\s*(```|~~~)") LIST_ITEM = re.compile(r"^\s*([-*+]|\d+[.)])\s") -LEAD = "__lead__" # summary and the whole-article introduction - def word_pattern(title: str) -> re.Pattern: """Whole-word, case-insensitive, and never biting into an existing marker."""