Cite a repeated chunk from its last occurrence
Collision detection replaced the lookup's dict comprehension with `setdefault`, which also flipped a chunk found by several searches from its last occurrence to its first. The copies differ in everything the expansion window decides, figures included, so that silently changed what a citation renders. The rules are separate now: a repeated (source, chunk_id) takes the later result, a chunk_id under two sources is still refused.
This commit is contained in:
parent
8a7cfb949f
commit
b8bf846bb7
3 changed files with 53 additions and 8 deletions
|
|
@ -74,19 +74,23 @@ def resolve_citations(
|
|||
) -> list[Citation]:
|
||||
"""Resolve chunk IDs to full Citation objects with metadata.
|
||||
|
||||
Raises ``AmbiguousCitationError`` when a cited id names a chunk in more than
|
||||
one of the databases searched, as after copying a database. A citation
|
||||
records the id alone, so resolving one would attribute the answer to a
|
||||
database it may not have come from.
|
||||
A chunk returned by more than one search resolves to its last occurrence.
|
||||
Raises ``AmbiguousCitationError`` instead when a cited id names a chunk in
|
||||
more than one of the databases searched, as after copying a database: a
|
||||
citation records the id alone, so resolving one would attribute the answer
|
||||
to a database it may not have come from.
|
||||
"""
|
||||
by_id: dict[str, SearchResult] = {}
|
||||
ambiguous: dict[str, set[str | None]] = {}
|
||||
for r in search_results:
|
||||
# A result built by hand carries no id and nothing can cite it.
|
||||
if cid := r.chunk_id:
|
||||
held = by_id.setdefault(cid, r)
|
||||
if held.source != r.source:
|
||||
if (held := by_id.get(cid)) is not None and held.source != r.source:
|
||||
ambiguous.setdefault(cid, {held.source}).add(r.source)
|
||||
# A chunk found by several searches is expanded once per search, so
|
||||
# the copies differ in content, window and figures. The later entry
|
||||
# wins.
|
||||
by_id[cid] = r
|
||||
|
||||
citations = []
|
||||
for raw_id in cited_chunk_ids:
|
||||
|
|
|
|||
|
|
@ -66,7 +66,9 @@ class TestSharedChunkIds:
|
|||
|
||||
def test_a_repeated_id_from_one_database_still_collapses(self):
|
||||
"""One database cannot hold two chunks under one id, so seeing it twice
|
||||
is the same chunk seen twice."""
|
||||
is the same chunk seen twice, and it resolves rather than raising. Which
|
||||
copy supplies the content is `resolve_citations`' own rule, pinned in
|
||||
`tests/store/test_citation.py`."""
|
||||
results = [
|
||||
SearchResult(
|
||||
content="first",
|
||||
|
|
@ -88,7 +90,8 @@ class TestSharedChunkIds:
|
|||
|
||||
[citation] = resolve_citations(["c1"], results)
|
||||
|
||||
assert citation.content == "first"
|
||||
assert citation.chunk_id == "c1"
|
||||
assert citation.source == "alpha"
|
||||
|
||||
def test_only_a_cited_id_has_to_be_unambiguous(self):
|
||||
"""An id the answer never cites attributes nothing."""
|
||||
|
|
|
|||
|
|
@ -62,3 +62,41 @@ def test_resolve_citations_copies_chunk_meta():
|
|||
result = _result("c1", chunk_meta={"para_no": "12", "speaker": "MR SMITH"})
|
||||
citations = resolve_citations(["c1"], [result])
|
||||
assert citations[0].chunk_meta == {"para_no": "12", "speaker": "MR SMITH"}
|
||||
|
||||
|
||||
def test_a_repeated_chunk_is_cited_from_its_last_occurrence():
|
||||
"""One chunk is returned by several searches, each expanded against what
|
||||
that search found in the same document, so the copies differ in everything
|
||||
the window decides. The later entry supplies them."""
|
||||
earlier = SearchResult(
|
||||
content="narrow window",
|
||||
score=0.9,
|
||||
chunk_id="c1",
|
||||
chunk_ids=["c1"],
|
||||
document_id="doc-1",
|
||||
document_uri="test://doc",
|
||||
doc_item_refs=["#/texts/4"],
|
||||
page_numbers=[2],
|
||||
headings=["Maintenance"],
|
||||
)
|
||||
later = SearchResult(
|
||||
content="wider window",
|
||||
score=0.4,
|
||||
chunk_id="c1",
|
||||
chunk_ids=["c1", "c2"],
|
||||
document_id="doc-1",
|
||||
document_uri="test://doc",
|
||||
doc_item_refs=["#/texts/4", "#/texts/5", "#/pictures/0"],
|
||||
page_numbers=[2, 3],
|
||||
headings=["Maintenance", "Calibration"],
|
||||
)
|
||||
|
||||
[citation] = resolve_citations(["c1"], [earlier, later])
|
||||
|
||||
assert citation.content == "wider window"
|
||||
assert citation.chunk_ids == ["c1", "c2"]
|
||||
assert citation.doc_item_refs == ["#/texts/4", "#/texts/5", "#/pictures/0"]
|
||||
assert citation.page_numbers == [2, 3]
|
||||
assert citation.headings == ["Maintenance", "Calibration"]
|
||||
# The window decides which figures travel with the citation.
|
||||
assert citation.picture_refs == ["#/pictures/0"]
|
||||
|
|
|
|||
Loading…
Reference in a new issue