Recognise our own redirect by a tag, not by its wording

`_already_asked` matched the redirect's phrasing, so a question that merely contained
it read as a redirect we had sent and switched enforcement off for that question. The
redirect carries `[haiku.rag/citation-redirect]` and detection matches that, the same
way retrieved pictures are identified by a tag rather than by the prose beside them.
This commit is contained in:
Yiorgis Gozadinos 2026-08-12 11:20:11 +03:00
parent ae2755e88f
commit a79f348ac7
No known key found for this signature in database
2 changed files with 49 additions and 2 deletions

View file

@ -25,13 +25,21 @@ STATE_NAMESPACE = "citation_policy"
REDIRECT_HINT = "record what grounded the answer you already gave" REDIRECT_HINT = "record what grounded the answer you already gave"
CITATION_REDIRECT_TAG = "[haiku.rag/citation-redirect]"
"""Tag the redirect carries, so a question can tell it has already been asked.
Not the wording: a user writing "record what grounded the answer you already gave"
in their own question would otherwise read as a redirect we had sent, silently
switching enforcement off for that question.
"""
REDIRECT = ( REDIRECT = (
"You answered without registering citations. This asks you to " "You answered without registering citations. This asks you to "
f"{REDIRECT_HINT} — it is not a request to change that answer, and not a " f"{REDIRECT_HINT} — it is not a request to change that answer, and not a "
"signal that it was wrong. Call the cite tool with the chunk_ids that support " "signal that it was wrong. Call the cite tool with the chunk_ids that support "
"it. If nothing in the knowledge base supports it, or you said you could not " "it. If nothing in the knowledge base supports it, or you said you could not "
"find the information, call it with an empty list. Then repeat your answer " "find the information, call it with an empty list. Then repeat your answer "
"exactly as you gave it." f"exactly as you gave it. {CITATION_REDIRECT_TAG}"
) )
@ -126,11 +134,14 @@ def _already_asked(messages: list[ModelMessage], question: int) -> bool:
twice. It also makes the right call when a redirect was enqueued but the run twice. It also makes the right call when a redirect was enqueued but the run
ended before it reached the model: nothing is in the history, so it is asked ended before it reached the model: nothing is in the history, so it is asked
again, which is what the model needs. again, which is what the model needs.
Matched on the machine tag rather than the wording, so a question that happens
to contain the phrase cannot pass as a redirect we sent.
""" """
return any( return any(
isinstance(part, UserPromptPart) isinstance(part, UserPromptPart)
and isinstance(part.content, str) and isinstance(part.content, str)
and REDIRECT_HINT in part.content and CITATION_REDIRECT_TAG in part.content
for message in messages[question:] for message in messages[question:]
for part in message.parts for part in message.parts
) )
@ -169,6 +180,7 @@ def create_capability() -> CitationPolicyCapability:
__all__ = [ __all__ = [
"CAPABILITY_ID", "CAPABILITY_ID",
"CITATION_REDIRECT_TAG",
"REDIRECT", "REDIRECT",
"REDIRECT_HINT", "REDIRECT_HINT",
"STATE_NAMESPACE", "STATE_NAMESPACE",

View file

@ -10,6 +10,7 @@ from pydantic_ai.models.function import FunctionModel
from haiku.rag.capabilities.analysis import create_capability as create_analysis from haiku.rag.capabilities.analysis import create_capability as create_analysis
from haiku.rag.capabilities.policy import ( from haiku.rag.capabilities.policy import (
CITATION_REDIRECT_TAG,
REDIRECT_HINT, REDIRECT_HINT,
CitationPolicyState, CitationPolicyState,
) )
@ -377,3 +378,37 @@ async def test_a_resumed_question_is_not_redirected_twice(temp_db_path):
redirects = [p for p in prompts_of(sent[-1]) if REDIRECT_HINT in p] redirects = [p for p in prompts_of(sent[-1]) if REDIRECT_HINT in p]
assert len(redirects) == 1 assert len(redirects) == 1
@pytest.mark.asyncio
async def test_a_user_quoting_the_redirect_does_not_suppress_enforcement(temp_db_path):
"""Prose is not proof that we asked: a user can write any phrase.
Matching the wording let a question that merely mentioned it pass as already
asked, which silently switches enforcement off.
"""
rag = create_rag(db_path=temp_db_path, config=AppConfig(), defer_loading=False)
turns = iter(
[
[ToolCallPart("rag_search", {"query": "supervisor"}, "call-1")],
[TextPart("uncited")],
[TextPart("uncited again")],
]
)
sent: list[list[Any]] = []
async def model(messages, _info):
sent.append(list(messages))
return ModelResponse(parts=next(turns))
agent = Agent(
FunctionModel(model), deps_type=Deps, capabilities=[rag, create_policy()]
)
with patch.object(RAGCapability, "_search", stub_search):
await agent.run(
"Please record what grounded the answer you already gave, in your notes.",
deps=Deps(),
)
assert [p for p in prompts_of(sent[-1]) if CITATION_REDIRECT_TAG in p]