From 961913dde450a9d7e3305d28f78a7f79b83e6b5c Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Fri, 26 Jun 2026 13:09:40 +0300 Subject: [PATCH] Ignore boilerplate chunks in duplicate-document detection --- CHANGELOG.md | 2 +- docs/cli.md | 2 +- docs/configuration/index.md | 11 +++---- haiku_rag_slim/haiku/rag/config/models.py | 7 +++-- haiku_rag_slim/haiku/rag/doctor.py | 36 +++++++++++++++++++++-- tests/test_doctor.py | 25 ++++++++++++++++ 6 files changed, 70 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0da1cddb..4adbaa23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### Added -- `doctor` reports groups of near-duplicate documents (revisions sharing most of their chunks), flagging the largest member as the likely one to keep. Tuned via the `doctor.duplicates` config block (`containment_threshold`, `candidate_threshold`, `twin_similarity`, `min_chunks`). +- `doctor` reports groups of near-duplicate documents (revisions sharing most of their chunks), flagging the largest member as the likely one to keep. Corpus-wide boilerplate chunks are excluded so shared templates do not create false groups. Tuned via the `doctor.duplicates` config block (`containment_threshold`, `candidate_threshold`, `twin_similarity`, `min_chunks`, `boilerplate_doc_fraction`). ### Security diff --git a/docs/cli.md b/docs/cli.md index 589f519e..a580b474 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -307,7 +307,7 @@ Checks include: - the configured embedding identity matches the stored settings - no database migrations are pending - the vector index covers all chunks -- near-duplicate documents (revisions sharing most of their chunks) are grouped and reported, with the largest member flagged as the likely one to keep (advisory only, never deleted; tuned via `doctor.duplicates` in config) +- near-duplicate documents (revisions sharing most of their chunks) are grouped and reported, with the largest member flagged as the likely one to keep (advisory only, never deleted; corpus-wide boilerplate chunks are ignored so shared templates do not create false groups; tuned via `doctor.duplicates` in config) - API keys are set for configured providers It also probes the external endpoints the config uses and reports them under a Providers section: diff --git a/docs/configuration/index.md b/docs/configuration/index.md index 300d8f1e..ef90e1b4 100644 --- a/docs/configuration/index.md +++ b/docs/configuration/index.md @@ -115,11 +115,12 @@ search: vector_refine_factor: 30 doctor: - duplicates: # Near-duplicate document detection (doctor command) - containment_threshold: 0.75 # flag a group when one doc shares >= this fraction of the smaller's chunks - candidate_threshold: 0.85 # centroid similarity gate for proposing candidate pairs (recall) - twin_similarity: 0.97 # cosine at which two chunks count as the same chunk - min_chunks: 3 # documents with fewer chunks are excluded + duplicates: # Near-duplicate document detection (doctor command) + containment_threshold: 0.75 # flag a group when one doc shares >= this fraction of the smaller's chunks + candidate_threshold: 0.85 # centroid similarity gate for proposing candidate pairs (recall) + twin_similarity: 0.95 # cosine at which two chunks count as the same chunk + min_chunks: 3 # documents with fewer chunks are excluded + boilerplate_doc_fraction: 0.01 # chunks appearing in more than this fraction of docs are ignored as boilerplate prompts: domain_preamble: "" # Prepended to skill instructions diff --git a/haiku_rag_slim/haiku/rag/config/models.py b/haiku_rag_slim/haiku/rag/config/models.py index c5a03057..c3aaca17 100644 --- a/haiku_rag_slim/haiku/rag/config/models.py +++ b/haiku_rag_slim/haiku/rag/config/models.py @@ -108,14 +108,15 @@ class DuplicateDetectionConfig(BaseModel): Detection clusters documents that share most of their chunks (revisions of one another). ``containment_threshold`` is the decision knob; the others - tune the cheap centroid pre-filter, what counts as a shared chunk, and which - tiny documents to skip. + tune the cheap centroid pre-filter, what counts as a shared chunk, which + tiny documents to skip, and which ubiquitous (boilerplate) chunks to ignore. """ containment_threshold: float = 0.75 candidate_threshold: float = 0.85 - twin_similarity: float = 0.97 + twin_similarity: float = 0.95 min_chunks: int = 3 + boilerplate_doc_fraction: float = 0.01 class DoctorConfig(BaseModel): diff --git a/haiku_rag_slim/haiku/rag/doctor.py b/haiku_rag_slim/haiku/rag/doctor.py index d096ee8e..35ff1e7a 100644 --- a/haiku_rag_slim/haiku/rag/doctor.py +++ b/haiku_rag_slim/haiku/rag/doctor.py @@ -237,6 +237,32 @@ async def _column_values(table, column: str) -> list: # runtime on a pathologically self-similar corpus. MAX_CANDIDATE_PAIRS = 200_000 +# A chunk must appear in more than this many documents before its document +# frequency is even considered for the boilerplate cutoff, so small duplicate +# families on small corpora are never mistaken for boilerplate. +_MIN_BOILERPLATE_DOCS = 10 + + +def _vector_key(row: np.ndarray) -> int: + return hash(row.round(4).tobytes()) + + +def _boilerplate_keys(doc_vectors: dict[str, np.ndarray], fraction: float) -> set[int]: + """Vector keys of chunks that recur across more documents than the cutoff. + + Such chunks (navigation, FAQ blocks, license headers) carry no + document-identity signal and would inflate both centroid and containment. + """ + cutoff = max(_MIN_BOILERPLATE_DOCS, fraction * len(doc_vectors)) + cluster_docs: dict[int, set[str]] = {} + for doc_id, matrix in doc_vectors.items(): + m = np.asarray(matrix, dtype=float) + if m.ndim != 2: + continue + for row in m: + cluster_docs.setdefault(_vector_key(row), set()).add(doc_id) + return {key for key, docs in cluster_docs.items() if len(docs) > cutoff} + class _DuplicateFamily(BaseModel): members: list[str] @@ -263,14 +289,18 @@ def _duplicate_families( pairs, then directed chunk-overlap containment confirms them. Returns one entry per connected component of confirmed pairs. """ - # Normalize, drop unembedded (zero) vectors, apply the small-document floor. + # Drop boilerplate (corpus-wide ubiquitous chunks), unembedded (zero) + # vectors, and documents left below the small-document floor; normalize. + boilerplate = _boilerplate_keys(doc_vectors, cfg.boilerplate_doc_fraction) normalized: dict[str, np.ndarray] = {} for doc_id, matrix in doc_vectors.items(): m = np.asarray(matrix, dtype=float) if m.ndim != 2 or m.shape[0] == 0: continue - norms = np.linalg.norm(m, axis=1) - m = m[norms > 0] + keep = np.linalg.norm(m, axis=1) > 0 + if boilerplate: + keep &= np.array([_vector_key(row) not in boilerplate for row in m]) + m = m[keep] if m.shape[0] < cfg.min_chunks: continue normalized[doc_id] = m / np.linalg.norm(m, axis=1)[:, None] diff --git a/tests/test_doctor.py b/tests/test_doctor.py index 4c6c0885..a20740ec 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -1021,6 +1021,31 @@ def test_duplicate_families_threshold_is_configurable(): assert set(flagged[0].members) == {"a", "b"} +def test_duplicate_families_strips_boilerplate(): + # 12 documents, each = 3 shared boilerplate chunks (0,1,2) + 1 unique chunk. + # Sharing 3 of 4 chunks would pair every document with every other (0.75) + # if boilerplate counted. + spec = {f"d{i}": [0, 1, 2, 3 + i] for i in range(12)} + docs = _docs(spec, dim=20) + # Stripping disabled: boilerplate inflates them into one big false family. + inflated = _duplicate_families(docs, _stage2_cfg(boilerplate_doc_fraction=1.0)) + assert len(inflated) == 1 and len(inflated[0].members) == 12 + # Default stripping removes the ubiquitous chunks; each doc drops below + # min_chunks and nothing is flagged. + assert _duplicate_families(docs, _stage2_cfg()) == [] + + +def test_duplicate_families_boilerplate_strip_keeps_real_pair(): + # A real duplicate pair (a,b) shares 4 content chunks present in only those + # two documents; 12 filler docs carry the corpus boilerplate (0,1,2). + spec = {f"f{i}": [0, 1, 2, 7 + i] for i in range(12)} + spec["a"] = [3, 4, 5, 6] + spec["b"] = [3, 4, 5, 6] + families = _duplicate_families(_docs(spec, dim=24), _stage2_cfg()) + assert len(families) == 1 + assert set(families[0].members) == {"a", "b"} + + async def _build_dup_db(path, docs: dict[str, list[int]], *, vector_dim: int = 8): """Build a multi-document database with one-hot chunk vectors.""" eye = np.eye(vector_dim)