Drop boilerplate handling; make duplicate report readable

CHANGELOG.md

@@ -3,7 +3,7 @@
 
 ### Added
 
-- `doctor` reports groups of near-duplicate documents (revisions sharing most of their chunks), flagging the largest member as the likely one to keep. Corpus-wide boilerplate chunks are excluded so shared templates do not create false groups. Tuned via the `doctor.duplicates` config block (`containment_threshold`, `candidate_threshold`, `twin_similarity`, `min_chunks`, `boilerplate_doc_fraction`).
+- `doctor` reports groups of near-duplicate documents (revisions sharing most of their chunks), flagging the largest member as the likely one to keep. Tuned via the `doctor.duplicates` config block (`containment_threshold`, `candidate_threshold`, `twin_similarity`, `min_chunks`).
 
 ### Security
 

docs/cli.md

@@ -307,7 +307,7 @@ Checks include:
 - the configured embedding identity matches the stored settings
 - no database migrations are pending
 - the vector index covers all chunks
-- near-duplicate documents (revisions sharing most of their chunks) are grouped and reported, with the largest member flagged as the likely one to keep (advisory only, never deleted; corpus-wide boilerplate chunks are ignored so shared templates do not create false groups; tuned via `doctor.duplicates` in config)
+- near-duplicate documents (revisions sharing most of their chunks) are grouped and reported, with the largest member flagged as the likely one to keep (advisory only, never deleted; tuned via `doctor.duplicates` in config)
 - API keys are set for configured providers
 
 It also probes the external endpoints the config uses and reports them under a Providers section:

docs/configuration/index.md

@@ -115,12 +115,11 @@ search:
   vector_refine_factor: 30
 
 doctor:
-  duplicates:                     # Near-duplicate document detection (doctor command)
+  duplicates:                    # Near-duplicate document detection (doctor command)
-    containment_threshold: 0.75   # flag a group when one doc shares >= this fraction of the smaller's chunks
+    containment_threshold: 0.75  # flag a group when one doc shares >= this fraction of the smaller's chunks
-    candidate_threshold: 0.85     # centroid similarity gate for proposing candidate pairs (recall)
+    candidate_threshold: 0.85    # centroid similarity gate for proposing candidate pairs (recall)
-    twin_similarity: 0.95         # cosine at which two chunks count as the same chunk
+    twin_similarity: 0.95        # cosine at which two chunks count as the same chunk
-    min_chunks: 3                 # documents with fewer chunks are excluded
+    min_chunks: 3                # documents with fewer chunks are excluded
-    boilerplate_doc_fraction: 0.01  # chunks appearing in more than this fraction of docs are ignored as boilerplate
 
 prompts:
   domain_preamble: ""  # Prepended to skill instructions

haiku_rag_slim/haiku/rag/config/models.py

@@ -108,15 +108,14 @@ class DuplicateDetectionConfig(BaseModel):
 
     Detection clusters documents that share most of their chunks (revisions of
     one another). ``containment_threshold`` is the decision knob; the others
-    tune the cheap centroid pre-filter, what counts as a shared chunk, which
+    tune the cheap centroid pre-filter, what counts as a shared chunk, and which
-    tiny documents to skip, and which ubiquitous (boilerplate) chunks to ignore.
+    tiny documents to skip.
     """
 
     containment_threshold: float = 0.75
     candidate_threshold: float = 0.85
     twin_similarity: float = 0.95
     min_chunks: int = 3
-    boilerplate_doc_fraction: float = 0.01
 
 
 class DoctorConfig(BaseModel):

haiku_rag_slim/haiku/rag/doctor.py

@@ -1,5 +1,6 @@
 import asyncio
 import json
+from collections.abc import Mapping
 from enum import StrEnum
 from pathlib import Path
 
@@ -237,32 +238,6 @@ async def _column_values(table, column: str) -> list:
 # runtime on a pathologically self-similar corpus.
 MAX_CANDIDATE_PAIRS = 200_000
 
-# A chunk must appear in more than this many documents before its document
-# frequency is even considered for the boilerplate cutoff, so small duplicate
-# families on small corpora are never mistaken for boilerplate.
-_MIN_BOILERPLATE_DOCS = 10
-
-
-def _vector_key(row: np.ndarray) -> int:
-    return hash(row.round(4).tobytes())
-
-
-def _boilerplate_keys(doc_vectors: dict[str, np.ndarray], fraction: float) -> set[int]:
-    """Vector keys of chunks that recur across more documents than the cutoff.
-
-    Such chunks (navigation, FAQ blocks, license headers) carry no
-    document-identity signal and would inflate both centroid and containment.
-    """
-    cutoff = max(_MIN_BOILERPLATE_DOCS, fraction * len(doc_vectors))
-    cluster_docs: dict[int, set[str]] = {}
-    for doc_id, matrix in doc_vectors.items():
-        m = np.asarray(matrix, dtype=float)
-        if m.ndim != 2:
-            continue
-        for row in m:
-            cluster_docs.setdefault(_vector_key(row), set()).add(doc_id)
-    return {key for key, docs in cluster_docs.items() if len(docs) > cutoff}
-
 
 class _DuplicateFamily(BaseModel):
     members: list[str]
@@ -289,18 +264,14 @@ def _duplicate_families(
     pairs, then directed chunk-overlap containment confirms them. Returns one
     entry per connected component of confirmed pairs.
     """
-    # Drop boilerplate (corpus-wide ubiquitous chunks), unembedded (zero)
+    # Drop unembedded (zero) vectors and documents below the small-document
-    # vectors, and documents left below the small-document floor; normalize.
+    # floor; normalize the rest to unit length.
-    boilerplate = _boilerplate_keys(doc_vectors, cfg.boilerplate_doc_fraction)
     normalized: dict[str, np.ndarray] = {}
     for doc_id, matrix in doc_vectors.items():
         m = np.asarray(matrix, dtype=float)
         if m.ndim != 2 or m.shape[0] == 0:
             continue
-        keep = np.linalg.norm(m, axis=1) > 0
+        m = m[np.linalg.norm(m, axis=1) > 0]
-        if boilerplate:
-            keep &= np.array([_vector_key(row) not in boilerplate for row in m])
-        m = m[keep]
         if m.shape[0] < cfg.min_chunks:
             continue
         normalized[doc_id] = m / np.linalg.norm(m, axis=1)[:, None]
@@ -369,10 +340,26 @@ def _duplicate_families(
     return sorted(families, key=lambda f: f.members)
 
 
+def _common_path_prefix(labels: list[str]) -> str:
+    """Longest shared prefix across labels, trimmed to a path boundary.
+
+    Returns "" unless the shared prefix is long enough to be worth factoring out
+    of every line (deep URI trees are otherwise unreadable).
+    """
+    if len(labels) < 2:
+        return ""
+    lo, hi = min(labels), max(labels)
+    end = 0
+    while end < len(lo) and lo[end] == hi[end]:
+        end += 1
+    cut = lo.rfind("/", 0, end)
+    return lo[: cut + 1] if cut > 16 else ""
+
+
 def _check_duplicate_documents(
     doc_vectors: dict[str, np.ndarray],
-    uri_by_doc: dict[str, str | None],
+    uri_by_doc: Mapping[str, str | None],
-    title_by_doc: dict[str, str | None],
+    title_by_doc: Mapping[str, str | None],
     cfg: DuplicateDetectionConfig,
 ) -> CheckResult:
     families = _duplicate_families(doc_vectors, cfg)
@@ -386,14 +373,29 @@ def _check_duplicate_documents(
     def label(doc_id: str) -> str:
         return uri_by_doc.get(doc_id) or title_by_doc.get(doc_id) or doc_id
 
-    lines: list[str] = []
+    prefix = _common_path_prefix([label(m) for f in families for m in f.members])
-    for family in families:
+
-        members = ", ".join(label(m) for m in family.members)
+    def short(doc_id: str) -> str:
-        overlaps = "; ".join(
+        text = label(doc_id)
-            f"{label(a)}→{label(b)}: {ab:.2f}, {label(b)}→{label(a)}: {ba:.2f}"
+        return text[len(prefix) :] if prefix and text.startswith(prefix) else text
+
+    # One block per group: a header, each member on its own numbered line, then a
+    # compact overlap summary referencing those numbers. Not truncated.
+    details: list[str] = []
+    if prefix:
+        details.append(f"common path: {prefix}")
+    for n, family in enumerate(families, start=1):
+        number = {member: i for i, member in enumerate(family.members, start=1)}
+        details.append(
+            f"group {n} — {len(family.members)} docs, keep #{number[family.superset]}:"
+        )
+        for member in family.members:
+            details.append(f"  #{number[member]} {short(member)}")
+        overlaps = ", ".join(
+            f"#{number[a]}→#{number[b]} {ab:.0%}, #{number[b]}→#{number[a]} {ba:.0%}"
             for a, b, ab, ba in family.pairs
         )
-        lines.append(f"[{members}] keep≈{label(family.superset)} ({overlaps})")
+        details.append(f"  overlap: {overlaps}")
 
     total_docs = sum(len(f.members) for f in families)
     return CheckResult(
@@ -406,7 +408,7 @@ def _check_duplicate_documents(
         remediation=(
             "Review each group and remove redundant revisions; overlap may be intentional."
         ),
-        details=_sample(lines),
+        details=details,
     )
 
 

tests/test_doctor.py

@@ -27,6 +27,7 @@ from haiku.rag.doctor import (
     Severity,
     _active_models,
     _check_api_keys,
+    _check_duplicate_documents,
     _check_embedding_drift,
     _check_vector_index,
     _duplicate_families,
@@ -1021,29 +1022,33 @@ def test_duplicate_families_threshold_is_configurable():
     assert set(flagged[0].members) == {"a", "b"}
 
 
-def test_duplicate_families_strips_boilerplate():
+def test_duplicate_documents_report_lists_all_groups_untruncated():
-    # 12 documents, each = 3 shared boilerplate chunks (0,1,2) + 1 unique chunk.
+    pairs = 7  # more than the old detail cap of 5
-    # Sharing 3 of 4 chunks would pair every document with every other (0.75)
+    spec: dict[str, list[int]] = {}
-    # if boilerplate counted.
+    for k in range(pairs):
-    spec = {f"d{i}": [0, 1, 2, 3 + i] for i in range(12)}
+        idx = [3 * k, 3 * k + 1, 3 * k + 2]
-    docs = _docs(spec, dim=20)
+        spec[f"a{k}"] = idx
-    # Stripping disabled: boilerplate inflates them into one big false family.
+        spec[f"b{k}"] = list(idx)
-    inflated = _duplicate_families(docs, _stage2_cfg(boilerplate_doc_fraction=1.0))
+    docs = _docs(spec, dim=3 * pairs)
-    assert len(inflated) == 1 and len(inflated[0].members) == 12
+    uris = {d: f"file:///srv/shared/library/docs/{d}.pdf" for d in spec}
-    # Default stripping removes the ubiquitous chunks; each doc drops below
+    result = _check_duplicate_documents(docs, uris, {}, _stage2_cfg())
-    # min_chunks and nothing is flagged.
+    assert result.severity is Severity.WARN
-    assert _duplicate_families(docs, _stage2_cfg()) == []
+    assert f"{pairs} group(s)" in result.message
+    assert sum(1 for d in result.details if d.startswith("group ")) == pairs
+    assert not any("more)" in d for d in result.details)
+    assert any("keep #" in d for d in result.details)
+    assert any("100%" in d for d in result.details)
 
 
-def test_duplicate_families_boilerplate_strip_keeps_real_pair():
+def test_duplicate_documents_report_factors_common_path():
-    # A real duplicate pair (a,b) shares 4 content chunks present in only those
+    docs = _docs({"a": [0, 1, 2], "b": [0, 1, 2]}, dim=3)
-    # two documents; 12 filler docs carry the corpus boilerplate (0,1,2).
+    base = "file:///srv/shared/library/docs/"
-    spec = {f"f{i}": [0, 1, 2, 7 + i] for i in range(12)}
+    uris = {"a": base + "alpha.pdf", "b": base + "beta.pdf"}
-    spec["a"] = [3, 4, 5, 6]
+    result = _check_duplicate_documents(docs, uris, {}, _stage2_cfg())
-    spec["b"] = [3, 4, 5, 6]
+    assert f"common path: {base}" in result.details
-    families = _duplicate_families(_docs(spec, dim=24), _stage2_cfg())
+    member_lines = [d for d in result.details if d.lstrip().startswith("#")]
-    assert len(families) == 1
+    assert {d.strip() for d in member_lines} == {"#1 alpha.pdf", "#2 beta.pdf"}
-    assert set(families[0].members) == {"a", "b"}
+    assert not any(base in d for d in member_lines)
 
 
 async def _build_dup_db(path, docs: dict[str, list[int]], *, vector_dim: int = 8):