From 044ac62e49f3d66cc832997f71c1fddcc57cb209 Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 26 Jun 2026 16:04:51 +0300 Subject: [PATCH] Drop boilerplate handling; make duplicate report readable --- CHANGELOG.md | 2 +- docs/cli.md | 2 +- docs/configuration/index.md | 11 ++- haiku_rag_slim/haiku/rag/config/models.py | 5 +- haiku_rag_slim/haiku/rag/doctor.py | 86 ++++++++++++----------- tests/test_doctor.py | 47 +++++++------ 6 files changed, 79 insertions(+), 74 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4adbaa23..0da1cddb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### Added -- `doctor` reports groups of near-duplicate documents (revisions sharing most of their chunks), flagging the largest member as the likely one to keep. Corpus-wide boilerplate chunks are excluded so shared templates do not create false groups. Tuned via the `doctor.duplicates` config block (`containment_threshold`, `candidate_threshold`, `twin_similarity`, `min_chunks`, `boilerplate_doc_fraction`). +- `doctor` reports groups of near-duplicate documents (revisions sharing most of their chunks), flagging the largest member as the likely one to keep. Tuned via the `doctor.duplicates` config block (`containment_threshold`, `candidate_threshold`, `twin_similarity`, `min_chunks`). ### Security diff --git a/docs/cli.md b/docs/cli.md index a580b474..589f519e 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -307,7 +307,7 @@ Checks include: - the configured embedding identity matches the stored settings - no database migrations are pending - the vector index covers all chunks -- near-duplicate documents (revisions sharing most of their chunks) are grouped and reported, with the largest member flagged as the likely one to keep (advisory only, never deleted; corpus-wide boilerplate chunks are ignored so shared templates do not create false groups; tuned via `doctor.duplicates` in config) +- near-duplicate documents (revisions sharing most of their chunks) are grouped and reported, with the largest member flagged as the likely one to keep (advisory only, never deleted; tuned via `doctor.duplicates` in config) - API keys are set for configured providers It also probes the external endpoints the config uses and reports them under a Providers section: diff --git a/docs/configuration/index.md b/docs/configuration/index.md index ef90e1b4..4c3c558d 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -115,12 +115,11 @@ search: vector_refine_factor: 30 doctor: - duplicates: # Near-duplicate document detection (doctor command) - containment_threshold: 0.75 # flag a group when one doc shares >= this fraction of the smaller's chunks - candidate_threshold: 0.85 # centroid similarity gate for proposing candidate pairs (recall) - twin_similarity: 0.95 # cosine at which two chunks count as the same chunk - min_chunks: 3 # documents with fewer chunks are excluded - boilerplate_doc_fraction: 0.01 # chunks appearing in more than this fraction of docs are ignored as boilerplate + duplicates: # Near-duplicate document detection (doctor command) + containment_threshold: 0.75 # flag a group when one doc shares >= this fraction of the smaller's chunks + candidate_threshold: 0.85 # centroid similarity gate for proposing candidate pairs (recall) + twin_similarity: 0.95 # cosine at which two chunks count as the same chunk + min_chunks: 3 # documents with fewer chunks are excluded prompts: domain_preamble: "" # Prepended to skill instructions diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index c3aaca17..f3a1a564 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -108,15 +108,14 @@ class DuplicateDetectionConfig(BaseModel): Detection clusters documents that share most of their chunks (revisions of one another). ``containment_threshold`` is the decision knob; the others - tune the cheap centroid pre-filter, what counts as a shared chunk, which - tiny documents to skip, and which ubiquitous (boilerplate) chunks to ignore. + tune the cheap centroid pre-filter, what counts as a shared chunk, and which + tiny documents to skip. """ containment_threshold: float = 0.75 candidate_threshold: float = 0.85 twin_similarity: float = 0.95 min_chunks: int = 3 - boilerplate_doc_fraction: float = 0.01 class DoctorConfig(BaseModel): diff --git a/haiku_rag_slim/haiku/rag/doctor.py b/haiku_rag_slim/haiku/rag/doctor.py index 35ff1e7a..23eec97b 100644 --- a/haiku_rag_slim/haiku/rag/doctor.py +++ b/haiku_rag_slim/haiku/rag/doctor.py @@ -1,5 +1,6 @@ import asyncio import json +from collections.abc import Mapping from enum import StrEnum from pathlib import Path @@ -237,32 +238,6 @@ async def _column_values(table, column: str) -> list: # runtime on a pathologically self-similar corpus. MAX_CANDIDATE_PAIRS = 200_000 -# A chunk must appear in more than this many documents before its document -# frequency is even considered for the boilerplate cutoff, so small duplicate -# families on small corpora are never mistaken for boilerplate. -_MIN_BOILERPLATE_DOCS = 10 - - -def _vector_key(row: np.ndarray) -> int: - return hash(row.round(4).tobytes()) - - -def _boilerplate_keys(doc_vectors: dict[str, np.ndarray], fraction: float) -> set[int]: - """Vector keys of chunks that recur across more documents than the cutoff. - - Such chunks (navigation, FAQ blocks, license headers) carry no - document-identity signal and would inflate both centroid and containment. - """ - cutoff = max(_MIN_BOILERPLATE_DOCS, fraction * len(doc_vectors)) - cluster_docs: dict[int, set[str]] = {} - for doc_id, matrix in doc_vectors.items(): - m = np.asarray(matrix, dtype=float) - if m.ndim != 2: - continue - for row in m: - cluster_docs.setdefault(_vector_key(row), set()).add(doc_id) - return {key for key, docs in cluster_docs.items() if len(docs) > cutoff} - class _DuplicateFamily(BaseModel): members: list[str] @@ -289,18 +264,14 @@ def _duplicate_families( pairs, then directed chunk-overlap containment confirms them. Returns one entry per connected component of confirmed pairs. """ - # Drop boilerplate (corpus-wide ubiquitous chunks), unembedded (zero) - # vectors, and documents left below the small-document floor; normalize. - boilerplate = _boilerplate_keys(doc_vectors, cfg.boilerplate_doc_fraction) + # Drop unembedded (zero) vectors and documents below the small-document + # floor; normalize the rest to unit length. normalized: dict[str, np.ndarray] = {} for doc_id, matrix in doc_vectors.items(): m = np.asarray(matrix, dtype=float) if m.ndim != 2 or m.shape[0] == 0: continue - keep = np.linalg.norm(m, axis=1) > 0 - if boilerplate: - keep &= np.array([_vector_key(row) not in boilerplate for row in m]) - m = m[keep] + m = m[np.linalg.norm(m, axis=1) > 0] if m.shape[0] < cfg.min_chunks: continue normalized[doc_id] = m / np.linalg.norm(m, axis=1)[:, None] @@ -369,10 +340,26 @@ def _duplicate_families( return sorted(families, key=lambda f: f.members) +def _common_path_prefix(labels: list[str]) -> str: + """Longest shared prefix across labels, trimmed to a path boundary. + + Returns "" unless the shared prefix is long enough to be worth factoring out + of every line (deep URI trees are otherwise unreadable). + """ + if len(labels) < 2: + return "" + lo, hi = min(labels), max(labels) + end = 0 + while end < len(lo) and lo[end] == hi[end]: + end += 1 + cut = lo.rfind("/", 0, end) + return lo[: cut + 1] if cut > 16 else "" + + def _check_duplicate_documents( doc_vectors: dict[str, np.ndarray], - uri_by_doc: dict[str, str | None], - title_by_doc: dict[str, str | None], + uri_by_doc: Mapping[str, str | None], + title_by_doc: Mapping[str, str | None], cfg: DuplicateDetectionConfig, ) -> CheckResult: families = _duplicate_families(doc_vectors, cfg) @@ -386,14 +373,29 @@ def _check_duplicate_documents( def label(doc_id: str) -> str: return uri_by_doc.get(doc_id) or title_by_doc.get(doc_id) or doc_id - lines: list[str] = [] - for family in families: - members = ", ".join(label(m) for m in family.members) - overlaps = "; ".join( - f"{label(a)}→{label(b)}: {ab:.2f}, {label(b)}→{label(a)}: {ba:.2f}" + prefix = _common_path_prefix([label(m) for f in families for m in f.members]) + + def short(doc_id: str) -> str: + text = label(doc_id) + return text[len(prefix) :] if prefix and text.startswith(prefix) else text + + # One block per group: a header, each member on its own numbered line, then a + # compact overlap summary referencing those numbers. Not truncated. + details: list[str] = [] + if prefix: + details.append(f"common path: {prefix}") + for n, family in enumerate(families, start=1): + number = {member: i for i, member in enumerate(family.members, start=1)} + details.append( + f"group {n} — {len(family.members)} docs, keep #{number[family.superset]}:" + ) + for member in family.members: + details.append(f" #{number[member]} {short(member)}") + overlaps = ", ".join( + f"#{number[a]}→#{number[b]} {ab:.0%}, #{number[b]}→#{number[a]} {ba:.0%}" for a, b, ab, ba in family.pairs ) - lines.append(f"[{members}] keep≈{label(family.superset)} ({overlaps})") + details.append(f" overlap: {overlaps}") total_docs = sum(len(f.members) for f in families) return CheckResult( @@ -406,7 +408,7 @@ def _check_duplicate_documents( remediation=( "Review each group and remove redundant revisions; overlap may be intentional." ), - details=_sample(lines), + details=details, ) diff --git a/tests/test_doctor.py b/tests/test_doctor.py index a20740ec..84b45412 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -27,6 +27,7 @@ from haiku.rag.doctor import ( Severity, _active_models, _check_api_keys, + _check_duplicate_documents, _check_embedding_drift, _check_vector_index, _duplicate_families, @@ -1021,29 +1022,33 @@ def test_duplicate_families_threshold_is_configurable(): assert set(flagged[0].members) == {"a", "b"} -def test_duplicate_families_strips_boilerplate(): - # 12 documents, each = 3 shared boilerplate chunks (0,1,2) + 1 unique chunk. - # Sharing 3 of 4 chunks would pair every document with every other (0.75) - # if boilerplate counted. - spec = {f"d{i}": [0, 1, 2, 3 + i] for i in range(12)} - docs = _docs(spec, dim=20) - # Stripping disabled: boilerplate inflates them into one big false family. - inflated = _duplicate_families(docs, _stage2_cfg(boilerplate_doc_fraction=1.0)) - assert len(inflated) == 1 and len(inflated[0].members) == 12 - # Default stripping removes the ubiquitous chunks; each doc drops below - # min_chunks and nothing is flagged. - assert _duplicate_families(docs, _stage2_cfg()) == [] +def test_duplicate_documents_report_lists_all_groups_untruncated(): + pairs = 7 # more than the old detail cap of 5 + spec: dict[str, list[int]] = {} + for k in range(pairs): + idx = [3 * k, 3 * k + 1, 3 * k + 2] + spec[f"a{k}"] = idx + spec[f"b{k}"] = list(idx) + docs = _docs(spec, dim=3 * pairs) + uris = {d: f"file:///srv/shared/library/docs/{d}.pdf" for d in spec} + result = _check_duplicate_documents(docs, uris, {}, _stage2_cfg()) + assert result.severity is Severity.WARN + assert f"{pairs} group(s)" in result.message + assert sum(1 for d in result.details if d.startswith("group ")) == pairs + assert not any("more)" in d for d in result.details) + assert any("keep #" in d for d in result.details) + assert any("100%" in d for d in result.details) -def test_duplicate_families_boilerplate_strip_keeps_real_pair(): - # A real duplicate pair (a,b) shares 4 content chunks present in only those - # two documents; 12 filler docs carry the corpus boilerplate (0,1,2). - spec = {f"f{i}": [0, 1, 2, 7 + i] for i in range(12)} - spec["a"] = [3, 4, 5, 6] - spec["b"] = [3, 4, 5, 6] - families = _duplicate_families(_docs(spec, dim=24), _stage2_cfg()) - assert len(families) == 1 - assert set(families[0].members) == {"a", "b"} +def test_duplicate_documents_report_factors_common_path(): + docs = _docs({"a": [0, 1, 2], "b": [0, 1, 2]}, dim=3) + base = "file:///srv/shared/library/docs/" + uris = {"a": base + "alpha.pdf", "b": base + "beta.pdf"} + result = _check_duplicate_documents(docs, uris, {}, _stage2_cfg()) + assert f"common path: {base}" in result.details + member_lines = [d for d in result.details if d.lstrip().startswith("#")] + assert {d.strip() for d in member_lines} == {"#1 alpha.pdf", "#2 beta.pdf"} + assert not any(base in d for d in member_lines) async def _build_dup_db(path, docs: dict[str, list[int]], *, vector_dim: int = 8):