cite raises ModelRetry on unresolved chunk_ids
This commit is contained in:
parent
7569fdcea1
commit
57d077e2c3
4 changed files with 77 additions and 2 deletions
|
|
@ -2,7 +2,7 @@ from pathlib import Path
|
|||
from typing import Any
|
||||
|
||||
from pydantic import BaseModel
|
||||
from pydantic_ai import RunContext
|
||||
from pydantic_ai import ModelRetry, RunContext
|
||||
from pydantic_ai.messages import ToolReturn
|
||||
|
||||
from haiku.rag.agents.research.models import Citation
|
||||
|
|
@ -314,7 +314,24 @@ def create_skill_tools(
|
|||
citations = resolve_citations(chunk_ids, all_results)
|
||||
if citations:
|
||||
_register_citations(state, citations)
|
||||
return f"Registered {len(citations)} citation(s)."
|
||||
return f"Registered {len(citations)} citation(s)."
|
||||
|
||||
if not chunk_ids:
|
||||
return "Registered 0 citations (empty chunk_ids)."
|
||||
|
||||
if not any(r.chunk_id for r in all_results):
|
||||
raise ModelRetry(
|
||||
f"None of the supplied chunk_ids {list(chunk_ids)} can be "
|
||||
"resolved: no search results have been recorded in this "
|
||||
"session yet. Call `search` first, then cite chunk_ids "
|
||||
"from its response."
|
||||
)
|
||||
raise ModelRetry(
|
||||
f"None of the supplied chunk_ids {list(chunk_ids)} match a "
|
||||
"chunk_id from search results. Copy chunk_ids verbatim from "
|
||||
"the search response — never reconstruct, abbreviate, or "
|
||||
"paraphrase them."
|
||||
)
|
||||
|
||||
tools["cite"] = cite
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ Search the knowledge base directly (outside code execution). Each result has a `
|
|||
### cite
|
||||
Register the chunk IDs that ground your answer. Call this BEFORE writing your final answer, with the `chunk_id` values from search results (from either the `search` tool or `await search(...)` inside `execute_code`) that support each claim. Every answer that uses search results must be backed by `cite`.
|
||||
|
||||
Use chunk_ids exactly as they appear in the search response — copy the full UUID verbatim. Do not abbreviate, paraphrase, or reconstruct chunk_ids from memory; the tool matches them as opaque strings.
|
||||
|
||||
## Document Filesystem (inside execute_code)
|
||||
|
||||
All documents are mounted as a virtual filesystem at `/documents/`:
|
||||
|
|
|
|||
|
|
@ -30,6 +30,8 @@ Retrieve a document by ID, title, or URI. Partial matches work. Use when the use
|
|||
### cite
|
||||
Register the chunk IDs that ground your answer. Call this BEFORE writing your final answer, with the `chunk_id` values from search results that support each claim. Every answer that uses search results must be backed by `cite`.
|
||||
|
||||
Use chunk_ids exactly as they appear in the search response — copy the full UUID verbatim. Do not abbreviate, paraphrase, or reconstruct chunk_ids from memory; the tool matches them as opaque strings.
|
||||
|
||||
## How to answer questions
|
||||
|
||||
1. Call `search` with relevant keywords from the question
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import pytest
|
||||
|
||||
from haiku.rag.config.models import AppConfig
|
||||
from haiku.rag.skills.rag import (
|
||||
STATE_NAMESPACE,
|
||||
|
|
@ -319,6 +321,58 @@ class TestCiteTool:
|
|||
result = await cite(ctx, chunk_ids=["nonexistent"])
|
||||
assert "No state" in result
|
||||
|
||||
async def test_cite_raises_modelretry_when_chunk_ids_unresolved(
|
||||
self, rag_db, rag_client
|
||||
):
|
||||
"""When supplied chunk_ids don't match any search result, cite raises
|
||||
ModelRetry so pydantic-ai prompts the model to retry with valid ids
|
||||
instead of silently registering zero citations."""
|
||||
from pydantic_ai import ModelRetry
|
||||
|
||||
from haiku.rag.skills.rag import RAGState, create_skill
|
||||
|
||||
skill = create_skill(db_path=rag_db)
|
||||
search = _get_tool(skill, "search")
|
||||
cite = _get_tool(skill, "cite")
|
||||
state = RAGState()
|
||||
ctx = _make_ctx(state, rag=rag_client)
|
||||
|
||||
await search(ctx, query="artificial intelligence")
|
||||
assert state.searches, "fixture should have produced some search results"
|
||||
|
||||
with pytest.raises(ModelRetry) as exc_info:
|
||||
await cite(ctx, chunk_ids=["372c9ddf-not-a-real-id"])
|
||||
message = str(exc_info.value)
|
||||
assert "verbatim" in message
|
||||
assert "372c9ddf-not-a-real-id" in message
|
||||
|
||||
async def test_cite_raises_modelretry_when_no_searches_recorded(self, rag_db):
|
||||
"""If cite is called before any search has populated state.searches,
|
||||
the retry message tells the model to call search first."""
|
||||
from pydantic_ai import ModelRetry
|
||||
|
||||
from haiku.rag.skills.rag import RAGState, create_skill
|
||||
|
||||
skill = create_skill(db_path=rag_db)
|
||||
cite = _get_tool(skill, "cite")
|
||||
state = RAGState()
|
||||
ctx = _make_ctx(state)
|
||||
|
||||
with pytest.raises(ModelRetry) as exc_info:
|
||||
await cite(ctx, chunk_ids=["any-id"])
|
||||
assert "search" in str(exc_info.value).lower()
|
||||
|
||||
async def test_cite_returns_message_when_chunk_ids_empty(self, rag_db):
|
||||
"""An empty chunk_ids list is a no-op, not a retry trigger."""
|
||||
from haiku.rag.skills.rag import RAGState, create_skill
|
||||
|
||||
skill = create_skill(db_path=rag_db)
|
||||
cite = _get_tool(skill, "cite")
|
||||
state = RAGState()
|
||||
ctx = _make_ctx(state)
|
||||
result = await cite(ctx, chunk_ids=[])
|
||||
assert "0" in result
|
||||
|
||||
|
||||
class TestLifespan:
|
||||
async def test_opens_one_client_per_invocation(self, rag_db):
|
||||
|
|
|
|||
Loading…
Reference in a new issue