diff --git a/backend/scripts/mdm_pass.py b/backend/scripts/mdm_pass.py index 6a8abfb..046604b 100644 --- a/backend/scripts/mdm_pass.py +++ b/backend/scripts/mdm_pass.py @@ -29,6 +29,7 @@ import json import re import sys import uuid +from collections import Counter from pathlib import Path from sqlalchemy.orm.attributes import flag_modified @@ -105,6 +106,26 @@ def conforms(article) -> bool: return all(required in titles for required in REQUIRED) +def _multiset_problem(what: str, before: list[str], after: list[str]) -> list[str]: + """Describe how two multisets differ, counting repeats. + + Comparing sets alone reported "lost -, added -" when a token simply appeared + a different number of times — which reads as nothing having changed, on a + line explaining a refusal. + """ + want, got = Counter(before), Counter(after) + if want == got: + return [] + lost = {k: want[k] - got.get(k, 0) for k in want if want[k] > got.get(k, 0)} + gained = {k: got[k] - want.get(k, 0) for k in got if got[k] > want.get(k, 0)} + parts = [] + if lost: + parts.append("lost " + ", ".join(f"{k}x{n}" for k, n in sorted(lost.items())[:6])) + if gained: + parts.append("duplicated " + ", ".join(f"{k}x{n}" for k, n in sorted(gained.items())[:6])) + return [f"{what} changed ({'; '.join(parts)})"] + + def validate(before: list[dict], after: list[dict]) -> list[str]: """Reasons this restructure must not be applied. Empty means it is safe.""" problems = [] @@ -128,15 +149,10 @@ def validate(before: list[dict], after: list[dict]) -> list[str]: if not new_text.strip(): return ["empty"] - want, got = sorted(MARKER.findall(old_text)), sorted(MARKER.findall(new_text)) - if want != got: - problems.append(f"cross-references changed (lost {sorted(set(want) - set(got)) or '-'}, " - f"added {sorted(set(got) - set(want)) or '-'})") - want_n = sorted(NUMBER.findall(facts_only(old_text))) - got_n = sorted(NUMBER.findall(facts_only(new_text))) - if want_n != got_n: - problems.append(f"numbers changed (lost {sorted(set(want_n) - set(got_n)) or '-'}, " - f"added {sorted(set(got_n) - set(want_n)) or '-'})") + problems += _multiset_problem("cross-references", MARKER.findall(old_text), + MARKER.findall(new_text)) + problems += _multiset_problem("numbers", NUMBER.findall(facts_only(old_text)), + NUMBER.findall(facts_only(new_text))) ratio = len(new_text) / max(len(old_text), 1) if not MIN_RATIO <= ratio <= MAX_RATIO: problems.append(f"length {ratio:.2f}x of the original")