From efa0af224482e42270d3b3aaa9d155f0ba9335de Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 11 Sep 2026 13:16:05 +0200 Subject: [PATCH] fix: the imperative detector only ever saw sentence starts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'When delay is suspected, obtain a detailed history' is as much an instruction to the reader as 'Obtain a detailed history', and the first pass could not see it — so the 819 I reported cleared was the count of one kind. 71 more were buried mid-sentence, after a comma or a conjunction. The detector now finds both, and export can be limited to particular variants so a pass does not collide with one already running over another view. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN --- backend/scripts/prose_pass.py | 36 +++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/backend/scripts/prose_pass.py b/backend/scripts/prose_pass.py index 08cc6ef..5809879 100644 --- a/backend/scripts/prose_pass.py +++ b/backend/scripts/prose_pass.py @@ -38,24 +38,38 @@ from app.models.article import Article MARKER = re.compile(r"\[\[(\d+)\|([^\]]+)\]\]") NUMBER = re.compile(r"\d+(?:\.\d+)?") -#: Verbs that open a sentence or a bullet as an instruction to the reader. -IMPERATIVE = re.compile( - r"(?:(?<=^)|(?<=[.!?] )|(?<=^- )|(?<=^\* )|(?<=\n- )|(?<=\n\* ))" - r"(Obtain|Order|Check|Give|Start|Administer|Consider|Assess|Evaluate|Perform|" - r"Measure|Repeat|Refer|Avoid|Ensure|Monitor|Treat|Begin|Initiate|Stop|Use|Send|" - r"Screen|Look|Ask|Confirm|Rule out|Do not|Don't|Reserve|Counsel|Admit|Discharge|" - r"Suspect|Exclude|Document|Reassess|Titrate|Correct|Repair|Remove|Apply|Prescribe)\b") +_VERBS = (r"Obtain|Order|Check|Give|Start|Administer|Consider|Assess|Evaluate|Perform|" + r"Measure|Repeat|Refer|Avoid|Ensure|Monitor|Treat|Begin|Initiate|Stop|Use|Send|" + r"Screen|Look|Ask|Confirm|Rule out|Do not|Don't|Reserve|Counsel|Admit|Discharge|" + r"Suspect|Exclude|Document|Reassess|Titrate|Correct|Repair|Remove|Apply|Prescribe") + +#: An instruction to the reader opening a sentence or a bullet. +IMPERATIVE_START = re.compile( + r"(?:(?<=^)|(?<=[.!?] )|(?<=^- )|(?<=^\* )|(?<=\n- )|(?<=\n\* ))(" + _VERBS + r")\b") + +#: And one buried mid-sentence, after a clause break — "…is suspected, obtain a +#: detailed history". These are the ones the first pass missed entirely, +#: because it only ever looked at what a sentence started with. +IMPERATIVE_MID = re.compile( + r"(?:,|;|\band\b|\bthen\b|\bor\b)\s+(" + _VERBS + r")\b\s+" + r"(?:a|an|the|for|all|both|age|serum|blood|urine|IV|oral)\b", re.I) + + +def imperatives(text: str) -> list[str]: + return IMPERATIVE_START.findall(text) + IMPERATIVE_MID.findall(text) MIN_RATIO, MAX_RATIO = 0.7, 1.4 -def affected(db): +def affected(db, variants=None): """Every section carrying the instructional voice, with its article.""" out = [] for article in db.query(Article).order_by(Article.id).all(): for section in article.sections or []: + if variants and (section.get("variant") or "long") not in variants: + continue content = section.get("content") or "" - hits = IMPERATIVE.findall(content) + hits = imperatives(content) if hits: out.append({ "article_id": article.id, @@ -100,7 +114,7 @@ def cmd_export(args) -> int: out.mkdir(parents=True, exist_ok=True) db = SessionLocal() try: - rows = affected(db) + rows = affected(db, args.variants.split(",") if args.variants else None) finally: db.close() if not rows: @@ -191,6 +205,8 @@ def main() -> int: export = sub.add_parser("export") export.add_argument("--out", required=True) export.add_argument("--batches", type=int, default=8) + export.add_argument("--variants", default=None, + help="Comma-separated variants to export, e.g. short,long") export.set_defaults(func=cmd_export) imp = sub.add_parser("import")