Ignore boilerplate chunks in duplicate-document detection

CHANGELOG.md

@@ -3,7 +3,7 @@
 
 ### Added
 
-- `doctor` reports groups of near-duplicate documents (revisions sharing most of their chunks), flagging the largest member as the likely one to keep. Tuned via the `doctor.duplicates` config block (`containment_threshold`, `candidate_threshold`, `twin_similarity`, `min_chunks`).
+- `doctor` reports groups of near-duplicate documents (revisions sharing most of their chunks), flagging the largest member as the likely one to keep. Corpus-wide boilerplate chunks are excluded so shared templates do not create false groups. Tuned via the `doctor.duplicates` config block (`containment_threshold`, `candidate_threshold`, `twin_similarity`, `min_chunks`, `boilerplate_doc_fraction`).
 
 ### Security
 

docs/cli.md

@@ -307,7 +307,7 @@ Checks include:
 - the configured embedding identity matches the stored settings
 - no database migrations are pending
 - the vector index covers all chunks
-- near-duplicate documents (revisions sharing most of their chunks) are grouped and reported, with the largest member flagged as the likely one to keep (advisory only, never deleted; tuned via `doctor.duplicates` in config)
+- near-duplicate documents (revisions sharing most of their chunks) are grouped and reported, with the largest member flagged as the likely one to keep (advisory only, never deleted; corpus-wide boilerplate chunks are ignored so shared templates do not create false groups; tuned via `doctor.duplicates` in config)
 - API keys are set for configured providers
 
 It also probes the external endpoints the config uses and reports them under a Providers section:

docs/configuration/index.md

@@ -115,11 +115,12 @@ search:
   vector_refine_factor: 30
 
 doctor:
-  duplicates:                    # Near-duplicate document detection (doctor command)
-    containment_threshold: 0.75  # flag a group when one doc shares >= this fraction of the smaller's chunks
-    candidate_threshold: 0.85    # centroid similarity gate for proposing candidate pairs (recall)
-    twin_similarity: 0.97        # cosine at which two chunks count as the same chunk
-    min_chunks: 3                # documents with fewer chunks are excluded
+  duplicates:                     # Near-duplicate document detection (doctor command)
+    containment_threshold: 0.75   # flag a group when one doc shares >= this fraction of the smaller's chunks
+    candidate_threshold: 0.85     # centroid similarity gate for proposing candidate pairs (recall)
+    twin_similarity: 0.95         # cosine at which two chunks count as the same chunk
+    min_chunks: 3                 # documents with fewer chunks are excluded
+    boilerplate_doc_fraction: 0.01  # chunks appearing in more than this fraction of docs are ignored as boilerplate
 
 prompts:
   domain_preamble: ""  # Prepended to skill instructions

haiku_rag_slim/haiku/rag/config/models.py

@@ -108,14 +108,15 @@ class DuplicateDetectionConfig(BaseModel):
 
     Detection clusters documents that share most of their chunks (revisions of
     one another). ``containment_threshold`` is the decision knob; the others
-    tune the cheap centroid pre-filter, what counts as a shared chunk, and which
-    tiny documents to skip.
+    tune the cheap centroid pre-filter, what counts as a shared chunk, which
+    tiny documents to skip, and which ubiquitous (boilerplate) chunks to ignore.
     """
 
     containment_threshold: float = 0.75
     candidate_threshold: float = 0.85
-    twin_similarity: float = 0.97
+    twin_similarity: float = 0.95
     min_chunks: int = 3
+    boilerplate_doc_fraction: float = 0.01
 
 
 class DoctorConfig(BaseModel):

haiku_rag_slim/haiku/rag/doctor.py

@@ -237,6 +237,32 @@ async def _column_values(table, column: str) -> list:
 # runtime on a pathologically self-similar corpus.
 MAX_CANDIDATE_PAIRS = 200_000
 
+# A chunk must appear in more than this many documents before its document
+# frequency is even considered for the boilerplate cutoff, so small duplicate
+# families on small corpora are never mistaken for boilerplate.
+_MIN_BOILERPLATE_DOCS = 10
+
+
+def _vector_key(row: np.ndarray) -> int:
+    return hash(row.round(4).tobytes())
+
+
+def _boilerplate_keys(doc_vectors: dict[str, np.ndarray], fraction: float) -> set[int]:
+    """Vector keys of chunks that recur across more documents than the cutoff.
+
+    Such chunks (navigation, FAQ blocks, license headers) carry no
+    document-identity signal and would inflate both centroid and containment.
+    """
+    cutoff = max(_MIN_BOILERPLATE_DOCS, fraction * len(doc_vectors))
+    cluster_docs: dict[int, set[str]] = {}
+    for doc_id, matrix in doc_vectors.items():
+        m = np.asarray(matrix, dtype=float)
+        if m.ndim != 2:
+            continue
+        for row in m:
+            cluster_docs.setdefault(_vector_key(row), set()).add(doc_id)
+    return {key for key, docs in cluster_docs.items() if len(docs) > cutoff}
+
 
 class _DuplicateFamily(BaseModel):
     members: list[str]
@@ -263,14 +289,18 @@ def _duplicate_families(
     pairs, then directed chunk-overlap containment confirms them. Returns one
     entry per connected component of confirmed pairs.
     """
-    # Normalize, drop unembedded (zero) vectors, apply the small-document floor.
+    # Drop boilerplate (corpus-wide ubiquitous chunks), unembedded (zero)
+    # vectors, and documents left below the small-document floor; normalize.
+    boilerplate = _boilerplate_keys(doc_vectors, cfg.boilerplate_doc_fraction)
     normalized: dict[str, np.ndarray] = {}
     for doc_id, matrix in doc_vectors.items():
         m = np.asarray(matrix, dtype=float)
         if m.ndim != 2 or m.shape[0] == 0:
             continue
-        norms = np.linalg.norm(m, axis=1)
-        m = m[norms > 0]
+        keep = np.linalg.norm(m, axis=1) > 0
+        if boilerplate:
+            keep &= np.array([_vector_key(row) not in boilerplate for row in m])
+        m = m[keep]
         if m.shape[0] < cfg.min_chunks:
             continue
         normalized[doc_id] = m / np.linalg.norm(m, axis=1)[:, None]

tests/test_doctor.py

@@ -1021,6 +1021,31 @@ def test_duplicate_families_threshold_is_configurable():
     assert set(flagged[0].members) == {"a", "b"}
 
 
+def test_duplicate_families_strips_boilerplate():
+    # 12 documents, each = 3 shared boilerplate chunks (0,1,2) + 1 unique chunk.
+    # Sharing 3 of 4 chunks would pair every document with every other (0.75)
+    # if boilerplate counted.
+    spec = {f"d{i}": [0, 1, 2, 3 + i] for i in range(12)}
+    docs = _docs(spec, dim=20)
+    # Stripping disabled: boilerplate inflates them into one big false family.
+    inflated = _duplicate_families(docs, _stage2_cfg(boilerplate_doc_fraction=1.0))
+    assert len(inflated) == 1 and len(inflated[0].members) == 12
+    # Default stripping removes the ubiquitous chunks; each doc drops below
+    # min_chunks and nothing is flagged.
+    assert _duplicate_families(docs, _stage2_cfg()) == []
+
+
+def test_duplicate_families_boilerplate_strip_keeps_real_pair():
+    # A real duplicate pair (a,b) shares 4 content chunks present in only those
+    # two documents; 12 filler docs carry the corpus boilerplate (0,1,2).
+    spec = {f"f{i}": [0, 1, 2, 7 + i] for i in range(12)}
+    spec["a"] = [3, 4, 5, 6]
+    spec["b"] = [3, 4, 5, 6]
+    families = _duplicate_families(_docs(spec, dim=24), _stage2_cfg())
+    assert len(families) == 1
+    assert set(families[0].members) == {"a", "b"}
+
+
 async def _build_dup_db(path, docs: dict[str, list[int]], *, vector_dim: int = 8):
     """Build a multi-document database with one-hot chunk vectors."""
     eye = np.eye(vector_dim)