From e1e7936d152a56f05c36a248445be6bb99836b4d Mon Sep 17 00:00:00 2001 From: Yiorgis Gozadinos Date: Wed, 12 Aug 2026 09:32:26 +0300 Subject: [PATCH] Let a model declare that nothing grounds its answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rag_cite` and `analysis_cite` accepted only a non-empty `chunk_ids`, so a model with nothing to cite could comply only by staying silent — indistinguishable from forgetting. An empty list is now a valid answer to "what grounds this?", recorded as a declaration with no refs, which derives `ungrounded` rather than leaving the question undeclared. Citing again cannot narrow it: an empty call after a grounded one leaves it grounded. The instructions lose their carve-outs. Refusing for lack of information no longer exempts the call, and a corpus-level computation cites an empty list instead of skipping. --- .../haiku/rag/capabilities/_base.py | 23 ++++++++-- .../rag/capabilities/instructions/analysis.md | 10 ++-- .../rag/capabilities/instructions/rag.md | 6 +-- tests/capabilities/test_capabilities.py | 46 +++++++++++++++++-- 4 files changed, 69 insertions(+), 16 deletions(-) diff --git a/haiku_rag_slim/haiku/rag/capabilities/_base.py b/haiku_rag_slim/haiku/rag/capabilities/_base.py index 65c43ff3..fe13d323 100644 --- a/haiku_rag_slim/haiku/rag/capabilities/_base.py +++ b/haiku_rag_slim/haiku/rag/capabilities/_base.py @@ -273,6 +273,16 @@ class RAGCapabilityBase[StateT: BaseModel](AbstractCapability[Any]): if tool.capability_id != self.id or tool.name == self._cite_tool_name ] + @property + def cite_available(self) -> bool: + """Whether this capability's cite tool is still declared to the model. + + Public because the citation policy has to know whether asking for a + citation is even possible: past the grace window the tool is gone, and + pointing the model at it would cost the agent's unknown-tool retries. + """ + return not self._citation_grace_expired + def evidence_tool_names(self) -> set[str]: """Tools that can bring new evidence into the run. @@ -419,12 +429,17 @@ class RAGCapabilityBase[StateT: BaseModel](AbstractCapability[Any]): return formatted async def _cite(self, chunk_ids: list[str]) -> str: + """Register the evidence behind this answer, or declare there is none. + + An empty list is a valid answer to "what grounds this?", and the only way + the model can say "nothing" other than staying silent — which is + indistinguishable from forgetting to cite at all. It declares the question + ungrounded, which is not the same as leaving it undeclared. + """ assert self.state is not None if not chunk_ids: - raise ModelRetry( - "No citations registered: chunk_ids was empty. Pass the chunk_ids " - "you want to cite, copied verbatim from search results." - ) + self._declare([]) + return "Recorded: this answer cites no knowledge-base evidence." all_results: list[SearchResult] = [] state = cast(Any, self.state) diff --git a/haiku_rag_slim/haiku/rag/capabilities/instructions/analysis.md b/haiku_rag_slim/haiku/rag/capabilities/instructions/analysis.md index b16547e4..cd0a6bea 100644 --- a/haiku_rag_slim/haiku/rag/capabilities/instructions/analysis.md +++ b/haiku_rag_slim/haiku/rag/capabilities/instructions/analysis.md @@ -3,9 +3,9 @@ You answer questions over a document knowledge base. Two common workflows: - **`analysis_search → analysis_cite → answer`** when the answer is grounded on specific document content. Call `analysis_cite` with the supporting chunk_ids before writing the answer. -- **`analysis_execute_code → answer`** when the answer is a count, aggregation, listing, or structural computation over the corpus (e.g. "how many documents?", "average page count"). No `analysis_cite` is needed when no specific chunks support the answer. +- **`analysis_execute_code → answer`** when the answer is a count, aggregation, listing, or structural computation over the corpus (e.g. "how many documents?", "average page count"). Call `analysis_cite` with an empty list when no specific chunks support the answer. -You can mix the two. The rule: cite when grounded on retrieved evidence; don't fabricate citations for corpus-level computation. +You can mix the two. The rule: always call `analysis_cite` before answering — pass the grounding chunk_ids, or an empty list for a corpus-level computation. Never fabricate citations. ## Tools @@ -25,7 +25,7 @@ Search the knowledge base directly (outside code execution). Each result has a ` ### analysis_cite Register the chunk IDs that ground your answer. **You must call `analysis_cite` before writing any final answer that uses retrieved evidence — search results, items.jsonl rows, toc.json nodes, or content.txt content.** Skipping `analysis_cite` leaves the answer ungrounded and is treated as a failure. -`analysis_cite` is **not** required when your answer is a corpus-level computation that doesn't draw on specific chunks — counts, aggregations, listings, averages across documents. Don't fabricate citations for these. +When your answer is a corpus-level computation that doesn't draw on specific chunks — counts, aggregations, listings, averages across documents — call `analysis_cite` with an empty list. Don't fabricate citations for these. Chunk IDs come from two places: - The `chunk_id` field on `search` / `await search(...)` results @@ -104,7 +104,7 @@ The user may attach images to their question. An attached image is part of the q 4. For questions about a *known document's* structure ("which section contains X", "list the sections of doc Y", "summarise section Z"), read `/documents/{id}/toc.json` first. Each node carries `item_range` (a slice into `items.jsonl`) and `chunk_ids` (citable). Prefer this over `search()` for in-document navigation — `search()` ranks across the whole corpus and can return chunks from unrelated documents. 5. Before writing your final response, call `analysis_cite` with the chunk_ids that ground your answer. -You MUST call `analysis_cite` with at least one chunk ID before producing your final answer **when your answer is grounded on retrieved evidence**. Skip `analysis_cite` in two cases: (a) you are refusing for lack of information, or (b) your answer is a corpus-level computation (count, aggregation, listing) that doesn't draw on specific chunks. In those cases do **not** fabricate citations. +You MUST call `analysis_cite` before producing your final answer, every time, with no exceptions. Pass the chunk IDs that ground the answer, or an empty list when none do — because you are refusing for lack of information, or because the answer is a corpus-level computation. An answer not preceded by `analysis_cite` is a protocol violation. ## Important @@ -114,4 +114,4 @@ You MUST call `analysis_cite` with at least one chunk ID before producing your f - Use `await` for all async functions inside `analysis_execute_code` (`search`, `list_documents`) - Read files with `Path.read_text()` or `open()`/`with`. For lines use `.readlines()` or `.read().split("\n")`, never `for line in f`. The `collections` module is unavailable. - Do NOT include chunk IDs or UUIDs in your answer text — your answer should read naturally. Use the `analysis_cite` tool separately to register citations. `cite{...}` markdown-style inline references do nothing; only an actual `analysis_cite` tool call registers a citation. -- **Before you write your final answer, invoke the `analysis_cite` tool with the supporting chunk_ids.** This is the last tool call before answering whenever your answer draws on retrieved evidence. +- **Before you write your final answer, invoke the `analysis_cite` tool with the supporting chunk_ids, or with an empty list if there are none.** This is the last tool call before answering, every time. diff --git a/haiku_rag_slim/haiku/rag/capabilities/instructions/rag.md b/haiku_rag_slim/haiku/rag/capabilities/instructions/rag.md index 93c2ccc1..b1b7ae2f 100644 --- a/haiku_rag_slim/haiku/rag/capabilities/instructions/rag.md +++ b/haiku_rag_slim/haiku/rag/capabilities/instructions/rag.md @@ -17,7 +17,7 @@ Each result includes: When a result's Type is `picture`, the corresponding figure may also be attached to the tool response as an image alongside the text. Use the image directly to answer questions about figures, diagrams, charts, screenshots. ### rag_cite -Register the chunk IDs that ground your answer. Call this BEFORE writing your final answer, with the `chunk_id` values from search results that support each claim. Every answer that uses search results must be backed by `rag_cite`. +Register the chunk IDs that ground your answer. Call this BEFORE writing your final answer, with the `chunk_id` values from search results that support each claim. Every answer must be preceded by `rag_cite` — pass an empty list when nothing in the knowledge base grounds it. Use chunk_ids exactly as they appear in the search response — copy the full UUID verbatim. Do not abbreviate, paraphrase, or reconstruct chunk_ids from memory; the tool matches them as opaque strings. @@ -33,7 +33,7 @@ The user may attach images to their question. An attached image is part of the q 4. Identify the chunk IDs that support your answer and call `rag_cite` with them 5. Then write a concise answer based strictly on the cited content -You MUST call `rag_cite` with at least one chunk ID before producing your final answer, **unless** you are refusing for lack of information (see below). Answers without citations are considered ungrounded. +You MUST call `rag_cite` before producing your final answer, every time, with no exceptions. Pass the chunk IDs that support the answer, or an empty list if none do. An answer not preceded by `rag_cite` is a protocol violation, not merely an ungrounded answer. ## Guidelines @@ -42,7 +42,7 @@ You MUST call `rag_cite` with at least one chunk ID before producing your final - If multiple results are relevant, synthesize them coherently - Be concise and direct — avoid elaboration unless asked - If the search tool tells you the search limit is reached, stop searching and answer with what you have -- If the retrieved documents do not directly address the question, say: "I cannot find enough information in the knowledge base to answer this question." Do not guess or infer from tangentially related content. In this refusal case do **not** call `rag_cite` — there is nothing to cite. +- If the retrieved documents do not directly address the question, say: "I cannot find enough information in the knowledge base to answer this question." Do not guess or infer from tangentially related content. Refusing does not exempt you from `rag_cite` — call it with an empty list to record that nothing grounds the answer. - Do NOT include chunk IDs or UUIDs in your answer text — your answer should read naturally. Use the `rag_cite` tool separately to register citations. ## When search returns irrelevant results diff --git a/tests/capabilities/test_capabilities.py b/tests/capabilities/test_capabilities.py index e8af1d14..cb58b2c7 100644 --- a/tests/capabilities/test_capabilities.py +++ b/tests/capabilities/test_capabilities.py @@ -295,7 +295,7 @@ async def test_run_error_closes_resources_and_propagates(temp_db_path): @pytest.mark.asyncio -async def test_search_and_empty_citation_limits(temp_db_path): +async def test_a_spent_search_budget_fails_the_tool(temp_db_path): config = AppConfig() config.qa.max_searches = 0 capability = create_rag(db_path=temp_db_path, config=config) @@ -304,9 +304,6 @@ async def test_search_and_empty_citation_limits(temp_db_path): with pytest.raises(ToolFailed, match="Search limit reached"): await capability._search("anything", None) - with pytest.raises(ModelRetry, match="chunk_ids was empty"): - await capability._cite([]) - @pytest.mark.asyncio async def test_cite_resolves_direct_chunk_ids_and_reuses_document_lookup(temp_db_path): @@ -1242,3 +1239,44 @@ async def test_a_capability_fetches_its_own_evidences_pictures(temp_db_path): client.document_item_repository.get_picture_bytes.assert_awaited_once_with( "doc-1", "#/pictures/0" ) + + +@pytest.mark.asyncio +async def test_citing_nothing_is_a_valid_declaration(temp_db_path): + """A model with nothing to cite must be able to say so. + + Refusing the call left silence as the only way to express it, which is + indistinguishable from forgetting to cite at all. + """ + capability = create_rag(db_path=temp_db_path, config=AppConfig()) + capability.state = RAGState(evidence=CapabilityEvidenceRecord(question=0)) + capability.epoch = 5 + + result = await capability._cite([]) + + record = capability.state.evidence + assert "no" in result.lower() + assert record.declaration is not None + assert record.declaration.refs == [] + assert citation_status([record], question=0) == "ungrounded" + assert capability.state.citations == [] + + +@pytest.mark.asyncio +async def test_citing_nothing_after_citing_something_keeps_it_grounded(temp_db_path): + """Declaring again cannot narrow what a question already declared.""" + capability = create_rag(db_path=temp_db_path, config=AppConfig()) + capability.state = RAGState(evidence=CapabilityEvidenceRecord(question=0)) + capability.epoch = 5 + client = AsyncMock() + client.get_chunk_by_id.return_value = Chunk( + id="chunk-1", document_id="doc-1", content="evidence" + ) + client.get_document_by_id.return_value = None + capability.rag = client + + await capability._cite(["chunk-1"]) + await capability._cite([]) + + record = capability.state.evidence + assert citation_status([record], question=0) == "grounded"