diff --git a/haiku_rag_slim/haiku/rag/capabilities/policy.py b/haiku_rag_slim/haiku/rag/capabilities/policy.py index b497eb21..95ca0214 100644 --- a/haiku_rag_slim/haiku/rag/capabilities/policy.py +++ b/haiku_rag_slim/haiku/rag/capabilities/policy.py @@ -25,13 +25,21 @@ STATE_NAMESPACE = "citation_policy" REDIRECT_HINT = "record what grounded the answer you already gave" +CITATION_REDIRECT_TAG = "[haiku.rag/citation-redirect]" +"""Tag the redirect carries, so a question can tell it has already been asked. + +Not the wording: a user writing "record what grounded the answer you already gave" +in their own question would otherwise read as a redirect we had sent, silently +switching enforcement off for that question. +""" + REDIRECT = ( "You answered without registering citations. This asks you to " f"{REDIRECT_HINT} — it is not a request to change that answer, and not a " "signal that it was wrong. Call the cite tool with the chunk_ids that support " "it. If nothing in the knowledge base supports it, or you said you could not " "find the information, call it with an empty list. Then repeat your answer " - "exactly as you gave it." + f"exactly as you gave it. {CITATION_REDIRECT_TAG}" ) @@ -126,11 +134,14 @@ def _already_asked(messages: list[ModelMessage], question: int) -> bool: twice. It also makes the right call when a redirect was enqueued but the run ended before it reached the model: nothing is in the history, so it is asked again, which is what the model needs. + + Matched on the machine tag rather than the wording, so a question that happens + to contain the phrase cannot pass as a redirect we sent. """ return any( isinstance(part, UserPromptPart) and isinstance(part.content, str) - and REDIRECT_HINT in part.content + and CITATION_REDIRECT_TAG in part.content for message in messages[question:] for part in message.parts ) @@ -169,6 +180,7 @@ def create_capability() -> CitationPolicyCapability: __all__ = [ "CAPABILITY_ID", + "CITATION_REDIRECT_TAG", "REDIRECT", "REDIRECT_HINT", "STATE_NAMESPACE", diff --git a/tests/capabilities/test_citation_policy.py b/tests/capabilities/test_citation_policy.py index ef916420..d4ff57ee 100644 --- a/tests/capabilities/test_citation_policy.py +++ b/tests/capabilities/test_citation_policy.py @@ -10,6 +10,7 @@ from pydantic_ai.models.function import FunctionModel from haiku.rag.capabilities.analysis import create_capability as create_analysis from haiku.rag.capabilities.policy import ( + CITATION_REDIRECT_TAG, REDIRECT_HINT, CitationPolicyState, ) @@ -377,3 +378,37 @@ async def test_a_resumed_question_is_not_redirected_twice(temp_db_path): redirects = [p for p in prompts_of(sent[-1]) if REDIRECT_HINT in p] assert len(redirects) == 1 + + +@pytest.mark.asyncio +async def test_a_user_quoting_the_redirect_does_not_suppress_enforcement(temp_db_path): + """Prose is not proof that we asked: a user can write any phrase. + + Matching the wording let a question that merely mentioned it pass as already + asked, which silently switches enforcement off. + """ + rag = create_rag(db_path=temp_db_path, config=AppConfig(), defer_loading=False) + turns = iter( + [ + [ToolCallPart("rag_search", {"query": "supervisor"}, "call-1")], + [TextPart("uncited")], + [TextPart("uncited again")], + ] + ) + sent: list[list[Any]] = [] + + async def model(messages, _info): + sent.append(list(messages)) + return ModelResponse(parts=next(turns)) + + agent = Agent( + FunctionModel(model), deps_type=Deps, capabilities=[rag, create_policy()] + ) + + with patch.object(RAGCapability, "_search", stub_search): + await agent.run( + "Please record what grounded the answer you already gave, in your notes.", + deps=Deps(), + ) + + assert [p for p in prompts_of(sent[-1]) if CITATION_REDIRECT_TAG in p]