4b59ef2330
Adds the pure-domain hash function that summarises a chunkset for stale-detection purposes. Recorded on each DocumentStoreLink at push time (#203 ships the column slot); compared against the recomputed current hash to flip a link to Stale when the source has drifted. domain/hashing.py - chunkset_hash(chunks: Iterable[ChunkResult]) -> str - SHA-256 over (text, source_page, headings) per chunk - Excludes bboxes / doc_items / token_count by design - 0x1F separator between chunks defends against the join-attack (split A+B vs concat AB) Tests - 9 tests: determinism, sensitivity per included field, exclusion of rendering-only fields, join-attack resistance, order sensitivity, empty-input invariant - Locked fixture: a hand-built 3-chunk input has a fixed expected hash; CI fails loud if anyone changes the canonical inputs without updating the fixture deliberately (and the release notes) Service integration (recompute on chunk write, set on push) lands with #205 once chunks are first-class — direct integration on the legacy chunks_json path is deliberately deferred to keep #204 focused. Refs #204
102 lines
3.9 KiB
Python
102 lines
3.9 KiB
Python
"""Tests for the chunkset hash function (#204)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from domain.hashing import chunkset_hash
|
|
from domain.value_objects import ChunkBbox, ChunkDocItem, ChunkResult
|
|
|
|
|
|
def _chunk(text: str, *, page: int | None = 1, headings=()) -> ChunkResult:
|
|
return ChunkResult(text=text, headings=list(headings), source_page=page)
|
|
|
|
|
|
def test_empty_chunkset_returns_empty_sha256() -> None:
|
|
"""Empty input has a stable, well-known hash."""
|
|
h = chunkset_hash([])
|
|
# SHA-256 of nothing is the well-known constant.
|
|
assert h == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
|
|
|
|
|
|
def test_determinism_across_invocations() -> None:
|
|
chunks = [_chunk("a"), _chunk("b"), _chunk("c")]
|
|
assert chunkset_hash(chunks) == chunkset_hash(chunks)
|
|
|
|
|
|
def test_text_change_changes_hash() -> None:
|
|
base = [_chunk("alpha")]
|
|
edited = [_chunk("alpha!")]
|
|
assert chunkset_hash(base) != chunkset_hash(edited)
|
|
|
|
|
|
def test_page_change_changes_hash() -> None:
|
|
a = [_chunk("alpha", page=1)]
|
|
b = [_chunk("alpha", page=2)]
|
|
assert chunkset_hash(a) != chunkset_hash(b)
|
|
|
|
|
|
def test_headings_change_changes_hash() -> None:
|
|
a = [_chunk("alpha", headings=("Section A",))]
|
|
b = [_chunk("alpha", headings=("Section B",))]
|
|
assert chunkset_hash(a) != chunkset_hash(b)
|
|
|
|
|
|
def test_excluded_fields_do_not_change_hash() -> None:
|
|
"""token_count, bboxes, doc_items are deliberately excluded."""
|
|
a = ChunkResult(text="alpha", headings=[], source_page=1, token_count=100, bboxes=[])
|
|
b = ChunkResult(
|
|
text="alpha",
|
|
headings=[],
|
|
source_page=1,
|
|
token_count=999, # different
|
|
bboxes=[ChunkBbox(page=1, bbox=[0, 0, 10, 10])], # different
|
|
doc_items=[ChunkDocItem(self_ref="#/x", label="text")], # different
|
|
)
|
|
assert chunkset_hash([a]) == chunkset_hash([b])
|
|
|
|
|
|
def test_join_attack_resistance() -> None:
|
|
"""Splitting one chunk into two with the same combined content must
|
|
produce a different hash from the original single-chunk version."""
|
|
one = [_chunk("HelloWorld")]
|
|
two = [_chunk("Hello"), _chunk("World")]
|
|
assert chunkset_hash(one) != chunkset_hash(two)
|
|
|
|
|
|
def test_order_matters() -> None:
|
|
a = [_chunk("alpha"), _chunk("bravo")]
|
|
b = [_chunk("bravo"), _chunk("alpha")]
|
|
assert chunkset_hash(a) != chunkset_hash(b)
|
|
|
|
|
|
def test_locked_fixture_three_chunks() -> None:
|
|
"""Locked fixture: a hand-built 3-chunk input has a fixed expected
|
|
hash. CI fails loud if anyone changes the canonical input list
|
|
without updating the fixture deliberately.
|
|
|
|
The expected hash below was computed once with the function as
|
|
pinned in this commit. To regenerate after a deliberate canonical
|
|
change, run:
|
|
|
|
python -c 'from domain.hashing import chunkset_hash; \\
|
|
from domain.value_objects import ChunkResult; \\
|
|
print(chunkset_hash([
|
|
ChunkResult(text="Intro paragraph.", source_page=1,
|
|
headings=["Intro"]),
|
|
ChunkResult(text="Body of section A.", source_page=2,
|
|
headings=["A"]),
|
|
ChunkResult(text="Conclusion.", source_page=3,
|
|
headings=["Conclusion"]),
|
|
]))'
|
|
"""
|
|
chunks = [
|
|
ChunkResult(text="Intro paragraph.", source_page=1, headings=["Intro"]),
|
|
ChunkResult(text="Body of section A.", source_page=2, headings=["A"]),
|
|
ChunkResult(text="Conclusion.", source_page=3, headings=["Conclusion"]),
|
|
]
|
|
expected = "6ac365ae403e53675e57884b69b0629684f2209c39730093231caa11a40e5225"
|
|
actual = chunkset_hash(chunks)
|
|
assert actual == expected, (
|
|
f"Hash drift detected. Expected {expected}, got {actual}. "
|
|
"If you intentionally changed the canonical inputs, update this "
|
|
"fixture and document the breaking change in the release notes."
|
|
)
|