From 9bc202d967cb7e496cf1b95fcb1b0cada8a44025 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 11 Sep 2026 05:14:56 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20prose=5Fpass=20=E2=80=94=20move=20artic?= =?UTF-8?q?le=20voice=20to=20third=20person,=20with=20a=20gate=20that=20re?= =?UTF-8?q?fuses=20fact=20drift?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 348 sections carry 819 imperatives ('Obtain a CBC', 'Counsel parents'). Reference prose states what is done. This script exports those sections, takes rewrites back, and refuses any that lost a cross-reference, changed a number, or landed outside 0.7-1.4x length — so a voice pass cannot quietly become a content pass. Measured while building it: first person is zero in the corpus. An earlier count of 136 was matching 'US' the country as the pronoun 'us'. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01TqXevQJhxFrM7jJg82cgZN --- backend/scripts/prose_pass.py | 209 ++++++++++++++++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 backend/scripts/prose_pass.py diff --git a/backend/scripts/prose_pass.py b/backend/scripts/prose_pass.py new file mode 100644 index 0000000..08cc6ef --- /dev/null +++ b/backend/scripts/prose_pass.py @@ -0,0 +1,209 @@ +"""Move article prose into the third person, impersonal, and check the result. + +The articles were written as instructions to a clinician — "Obtain a CBC", +"Do not order a scrotal ultrasound", "Counsel parents". Reference prose states +what is done rather than telling the reader to do it: *diagnosis is made by*, +*response to treatment is assessed by*. 819 imperatives across 348 sections +carry that voice. + +This does the moving of text, not the writing. `export` writes the affected +sections to batch files; something that can write English rewrites the +`content` of each and saves alongside; `import` checks and applies. + +What `import` refuses, because a rewrite must not quietly change the article: + + * a lost or invented cross-reference — the set of `[[id|…]]` targets must match + * a changed number — every numeric token must survive, so doses, ages, + thresholds and percentages cannot drift + * a length outside 0.7-1.4x, which is how a summary or a padding pass looks + * an empty result + +It reports, but allows, imperatives that survive: some are genuinely the right +form inside a quoted protocol. + + python -m scripts.prose_pass export --out /tmp/prose --batches 8 + python -m scripts.prose_pass import --in /tmp/prose + python -m scripts.prose_pass check +""" +import argparse +import json +import re +import sys +from pathlib import Path + +from sqlalchemy.orm.attributes import flag_modified + +from app.database import SessionLocal +from app.models.article import Article + +MARKER = re.compile(r"\[\[(\d+)\|([^\]]+)\]\]") +NUMBER = re.compile(r"\d+(?:\.\d+)?") +#: Verbs that open a sentence or a bullet as an instruction to the reader. +IMPERATIVE = re.compile( + r"(?:(?<=^)|(?<=[.!?] )|(?<=^- )|(?<=^\* )|(?<=\n- )|(?<=\n\* ))" + r"(Obtain|Order|Check|Give|Start|Administer|Consider|Assess|Evaluate|Perform|" + r"Measure|Repeat|Refer|Avoid|Ensure|Monitor|Treat|Begin|Initiate|Stop|Use|Send|" + r"Screen|Look|Ask|Confirm|Rule out|Do not|Don't|Reserve|Counsel|Admit|Discharge|" + r"Suspect|Exclude|Document|Reassess|Titrate|Correct|Repair|Remove|Apply|Prescribe)\b") + +MIN_RATIO, MAX_RATIO = 0.7, 1.4 + + +def affected(db): + """Every section carrying the instructional voice, with its article.""" + out = [] + for article in db.query(Article).order_by(Article.id).all(): + for section in article.sections or []: + content = section.get("content") or "" + hits = IMPERATIVE.findall(content) + if hits: + out.append({ + "article_id": article.id, + "article_title": article.title, + "section_id": section.get("id"), + "section_title": section.get("title"), + "variant": section.get("variant") or "long", + "imperatives": len(hits), + "content": content, + }) + return out + + +def validate(before: str, after: str) -> list[str]: + """Reasons this rewrite must not be applied. Empty means it is safe.""" + problems = [] + if not after or not after.strip(): + return ["empty"] + + want = sorted(m.group(1) for m in MARKER.finditer(before)) + got = sorted(m.group(1) for m in MARKER.finditer(after)) + if want != got: + lost = sorted(set(want) - set(got)) + added = sorted(set(got) - set(want)) + problems.append(f"cross-references changed (lost {lost or '-'}, added {added or '-'})") + + want_n = sorted(NUMBER.findall(before)) + got_n = sorted(NUMBER.findall(after)) + if want_n != got_n: + lost = sorted(set(want_n) - set(got_n)) + added = sorted(set(got_n) - set(want_n)) + problems.append(f"numbers changed (lost {lost or '-'}, added {added or '-'})") + + ratio = len(after) / max(len(before), 1) + if not MIN_RATIO <= ratio <= MAX_RATIO: + problems.append(f"length {ratio:.2f}x of the original") + return problems + + +def cmd_export(args) -> int: + out = Path(args.out) + out.mkdir(parents=True, exist_ok=True) + db = SessionLocal() + try: + rows = affected(db) + finally: + db.close() + if not rows: + print("Nothing to rewrite.") + return 0 + + # Whole articles stay together where they can: a rewriter seeing every + # section of one topic keeps its terminology consistent. + rows.sort(key=lambda r: (r["article_id"], r["section_title"] or "")) + per = max(1, -(-len(rows) // args.batches)) + written = 0 + for index in range(args.batches): + chunk = rows[index * per:(index + 1) * per] + if not chunk: + break + path = out / f"batch-{index + 1:02d}.json" + path.write_text(json.dumps(chunk, indent=2, ensure_ascii=False)) + written += 1 + print(f" {path} {len(chunk):3d} sections, {sum(c['imperatives'] for c in chunk):4d} imperatives") + print(f"\n{len(rows)} sections in {written} batches.") + print("Rewrite each entry's `content` and save as batch-NN.done.json alongside.") + return 0 + + +def cmd_import(args) -> int: + folder = Path(args.folder) + done = sorted(folder.glob("*.done.json")) + if not done: + print(f"No *.done.json in {folder}") + return 1 + + db = SessionLocal() + try: + articles = {a.id: a for a in db.query(Article).all()} + applied = skipped = 0 + for path in done: + entries = json.loads(path.read_text()) + for entry in entries: + article = articles.get(entry["article_id"]) + if not article: + continue + section = next((s for s in (article.sections or []) + if s.get("id") == entry["section_id"]), None) + if not section: + print(f" ! {entry['article_title']} / {entry['section_title']}: section is gone") + skipped += 1 + continue + before, after = section.get("content") or "", entry["content"] + if before == after: + continue + problems = validate(before, after) + if problems: + print(f" ! {entry['article_title']} / {entry['section_title']}: {'; '.join(problems)}") + skipped += 1 + continue + section["content"] = after + flag_modified(article, "sections") + applied += 1 + print(f"\n applied: {applied} refused: {skipped}") + if args.apply: + db.commit() + print(" committed.") + else: + print("\n Re-run with --apply to write.") + finally: + db.close() + return 0 + + +def cmd_check(args) -> int: + db = SessionLocal() + try: + rows = affected(db) + total = sum(r["imperatives"] for r in rows) + articles = len({r["article_id"] for r in rows}) + print(f" {total} imperatives in {len(rows)} sections across {articles} articles") + for row in sorted(rows, key=lambda r: -r["imperatives"])[:10]: + print(f" {row['imperatives']:3d} {row['article_title']} / {row['section_title']}") + finally: + db.close() + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="cmd", required=True) + + export = sub.add_parser("export") + export.add_argument("--out", required=True) + export.add_argument("--batches", type=int, default=8) + export.set_defaults(func=cmd_export) + + imp = sub.add_parser("import") + imp.add_argument("--in", dest="folder", required=True) + imp.add_argument("--apply", action="store_true") + imp.set_defaults(func=cmd_import) + + check = sub.add_parser("check") + check.set_defaults(func=cmd_check) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main())