"""Move article prose into the third person, impersonal, and check the result. The articles were written as instructions to a clinician — "Obtain a CBC", "Do not order a scrotal ultrasound", "Counsel parents". Reference prose states what is done rather than telling the reader to do it: *diagnosis is made by*, *response to treatment is assessed by*. 819 imperatives across 348 sections carry that voice. This does the moving of text, not the writing. `export` writes the affected sections to batch files; something that can write English rewrites the `content` of each and saves alongside; `import` checks and applies. What `import` refuses, because a rewrite must not quietly change the article: * a lost or invented cross-reference — the set of `[[id|…]]` targets must match * a changed number — every numeric token must survive, so doses, ages, thresholds and percentages cannot drift * a length outside 0.7-1.4x, which is how a summary or a padding pass looks * an empty result It reports, but allows, imperatives that survive: some are genuinely the right form inside a quoted protocol. python -m scripts.prose_pass export --out /tmp/prose --batches 8 python -m scripts.prose_pass import --in /tmp/prose python -m scripts.prose_pass check """ import argparse import json import re import sys from pathlib import Path from sqlalchemy.orm.attributes import flag_modified from app.database import SessionLocal from app.models.article import Article MARKER = re.compile(r"\[\[(\d+)\|([^\]]+)\]\]") NUMBER = re.compile(r"\d+(?:\.\d+)?") #: Verbs that open a sentence or a bullet as an instruction to the reader. IMPERATIVE = re.compile( r"(?:(?<=^)|(?<=[.!?] )|(?<=^- )|(?<=^\* )|(?<=\n- )|(?<=\n\* ))" r"(Obtain|Order|Check|Give|Start|Administer|Consider|Assess|Evaluate|Perform|" r"Measure|Repeat|Refer|Avoid|Ensure|Monitor|Treat|Begin|Initiate|Stop|Use|Send|" r"Screen|Look|Ask|Confirm|Rule out|Do not|Don't|Reserve|Counsel|Admit|Discharge|" r"Suspect|Exclude|Document|Reassess|Titrate|Correct|Repair|Remove|Apply|Prescribe)\b") MIN_RATIO, MAX_RATIO = 0.7, 1.4 def affected(db): """Every section carrying the instructional voice, with its article.""" out = [] for article in db.query(Article).order_by(Article.id).all(): for section in article.sections or []: content = section.get("content") or "" hits = IMPERATIVE.findall(content) if hits: out.append({ "article_id": article.id, "article_title": article.title, "section_id": section.get("id"), "section_title": section.get("title"), "variant": section.get("variant") or "long", "imperatives": len(hits), "content": content, }) return out def validate(before: str, after: str) -> list[str]: """Reasons this rewrite must not be applied. Empty means it is safe.""" problems = [] if not after or not after.strip(): return ["empty"] want = sorted(m.group(1) for m in MARKER.finditer(before)) got = sorted(m.group(1) for m in MARKER.finditer(after)) if want != got: lost = sorted(set(want) - set(got)) added = sorted(set(got) - set(want)) problems.append(f"cross-references changed (lost {lost or '-'}, added {added or '-'})") want_n = sorted(NUMBER.findall(before)) got_n = sorted(NUMBER.findall(after)) if want_n != got_n: lost = sorted(set(want_n) - set(got_n)) added = sorted(set(got_n) - set(want_n)) problems.append(f"numbers changed (lost {lost or '-'}, added {added or '-'})") ratio = len(after) / max(len(before), 1) if not MIN_RATIO <= ratio <= MAX_RATIO: problems.append(f"length {ratio:.2f}x of the original") return problems def cmd_export(args) -> int: out = Path(args.out) out.mkdir(parents=True, exist_ok=True) db = SessionLocal() try: rows = affected(db) finally: db.close() if not rows: print("Nothing to rewrite.") return 0 # Whole articles stay together where they can: a rewriter seeing every # section of one topic keeps its terminology consistent. rows.sort(key=lambda r: (r["article_id"], r["section_title"] or "")) per = max(1, -(-len(rows) // args.batches)) written = 0 for index in range(args.batches): chunk = rows[index * per:(index + 1) * per] if not chunk: break path = out / f"batch-{index + 1:02d}.json" path.write_text(json.dumps(chunk, indent=2, ensure_ascii=False)) written += 1 print(f" {path} {len(chunk):3d} sections, {sum(c['imperatives'] for c in chunk):4d} imperatives") print(f"\n{len(rows)} sections in {written} batches.") print("Rewrite each entry's `content` and save as batch-NN.done.json alongside.") return 0 def cmd_import(args) -> int: folder = Path(args.folder) done = sorted(folder.glob("*.done.json")) if not done: print(f"No *.done.json in {folder}") return 1 db = SessionLocal() try: articles = {a.id: a for a in db.query(Article).all()} applied = skipped = 0 for path in done: entries = json.loads(path.read_text()) for entry in entries: article = articles.get(entry["article_id"]) if not article: continue section = next((s for s in (article.sections or []) if s.get("id") == entry["section_id"]), None) if not section: print(f" ! {entry['article_title']} / {entry['section_title']}: section is gone") skipped += 1 continue before, after = section.get("content") or "", entry["content"] if before == after: continue problems = validate(before, after) if problems: print(f" ! {entry['article_title']} / {entry['section_title']}: {'; '.join(problems)}") skipped += 1 continue section["content"] = after flag_modified(article, "sections") applied += 1 print(f"\n applied: {applied} refused: {skipped}") if args.apply: db.commit() print(" committed.") else: print("\n Re-run with --apply to write.") finally: db.close() return 0 def cmd_check(args) -> int: db = SessionLocal() try: rows = affected(db) total = sum(r["imperatives"] for r in rows) articles = len({r["article_id"] for r in rows}) print(f" {total} imperatives in {len(rows)} sections across {articles} articles") for row in sorted(rows, key=lambda r: -r["imperatives"])[:10]: print(f" {row['imperatives']:3d} {row['article_title']} / {row['section_title']}") finally: db.close() return 0 def main() -> int: parser = argparse.ArgumentParser(description=__doc__) sub = parser.add_subparsers(dest="cmd", required=True) export = sub.add_parser("export") export.add_argument("--out", required=True) export.add_argument("--batches", type=int, default=8) export.set_defaults(func=cmd_export) imp = sub.add_parser("import") imp.add_argument("--in", dest="folder", required=True) imp.add_argument("--apply", action="store_true") imp.set_defaults(func=cmd_import) check = sub.add_parser("check") check.set_defaults(func=cmd_check) args = parser.parse_args() return args.func(args) if __name__ == "__main__": sys.exit(main())