cite raises ModelRetry on unresolved chunk_ids
This commit is contained in:
parent
7569fdcea1
commit
57d077e2c3
4 changed files with 77 additions and 2 deletions
|
|
@ -2,7 +2,7 @@ from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from pydantic import BaseModel
|
from pydantic import BaseModel
|
||||||
from pydantic_ai import RunContext
|
from pydantic_ai import ModelRetry, RunContext
|
||||||
from pydantic_ai.messages import ToolReturn
|
from pydantic_ai.messages import ToolReturn
|
||||||
|
|
||||||
from haiku.rag.agents.research.models import Citation
|
from haiku.rag.agents.research.models import Citation
|
||||||
|
|
@ -314,7 +314,24 @@ def create_skill_tools(
|
||||||
citations = resolve_citations(chunk_ids, all_results)
|
citations = resolve_citations(chunk_ids, all_results)
|
||||||
if citations:
|
if citations:
|
||||||
_register_citations(state, citations)
|
_register_citations(state, citations)
|
||||||
return f"Registered {len(citations)} citation(s)."
|
return f"Registered {len(citations)} citation(s)."
|
||||||
|
|
||||||
|
if not chunk_ids:
|
||||||
|
return "Registered 0 citations (empty chunk_ids)."
|
||||||
|
|
||||||
|
if not any(r.chunk_id for r in all_results):
|
||||||
|
raise ModelRetry(
|
||||||
|
f"None of the supplied chunk_ids {list(chunk_ids)} can be "
|
||||||
|
"resolved: no search results have been recorded in this "
|
||||||
|
"session yet. Call `search` first, then cite chunk_ids "
|
||||||
|
"from its response."
|
||||||
|
)
|
||||||
|
raise ModelRetry(
|
||||||
|
f"None of the supplied chunk_ids {list(chunk_ids)} match a "
|
||||||
|
"chunk_id from search results. Copy chunk_ids verbatim from "
|
||||||
|
"the search response — never reconstruct, abbreviate, or "
|
||||||
|
"paraphrase them."
|
||||||
|
)
|
||||||
|
|
||||||
tools["cite"] = cite
|
tools["cite"] = cite
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,8 @@ Search the knowledge base directly (outside code execution). Each result has a `
|
||||||
### cite
|
### cite
|
||||||
Register the chunk IDs that ground your answer. Call this BEFORE writing your final answer, with the `chunk_id` values from search results (from either the `search` tool or `await search(...)` inside `execute_code`) that support each claim. Every answer that uses search results must be backed by `cite`.
|
Register the chunk IDs that ground your answer. Call this BEFORE writing your final answer, with the `chunk_id` values from search results (from either the `search` tool or `await search(...)` inside `execute_code`) that support each claim. Every answer that uses search results must be backed by `cite`.
|
||||||
|
|
||||||
|
Use chunk_ids exactly as they appear in the search response — copy the full UUID verbatim. Do not abbreviate, paraphrase, or reconstruct chunk_ids from memory; the tool matches them as opaque strings.
|
||||||
|
|
||||||
## Document Filesystem (inside execute_code)
|
## Document Filesystem (inside execute_code)
|
||||||
|
|
||||||
All documents are mounted as a virtual filesystem at `/documents/`:
|
All documents are mounted as a virtual filesystem at `/documents/`:
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,8 @@ Retrieve a document by ID, title, or URI. Partial matches work. Use when the use
|
||||||
### cite
|
### cite
|
||||||
Register the chunk IDs that ground your answer. Call this BEFORE writing your final answer, with the `chunk_id` values from search results that support each claim. Every answer that uses search results must be backed by `cite`.
|
Register the chunk IDs that ground your answer. Call this BEFORE writing your final answer, with the `chunk_id` values from search results that support each claim. Every answer that uses search results must be backed by `cite`.
|
||||||
|
|
||||||
|
Use chunk_ids exactly as they appear in the search response — copy the full UUID verbatim. Do not abbreviate, paraphrase, or reconstruct chunk_ids from memory; the tool matches them as opaque strings.
|
||||||
|
|
||||||
## How to answer questions
|
## How to answer questions
|
||||||
|
|
||||||
1. Call `search` with relevant keywords from the question
|
1. Call `search` with relevant keywords from the question
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,5 @@
|
||||||
|
import pytest
|
||||||
|
|
||||||
from haiku.rag.config.models import AppConfig
|
from haiku.rag.config.models import AppConfig
|
||||||
from haiku.rag.skills.rag import (
|
from haiku.rag.skills.rag import (
|
||||||
STATE_NAMESPACE,
|
STATE_NAMESPACE,
|
||||||
|
|
@ -319,6 +321,58 @@ class TestCiteTool:
|
||||||
result = await cite(ctx, chunk_ids=["nonexistent"])
|
result = await cite(ctx, chunk_ids=["nonexistent"])
|
||||||
assert "No state" in result
|
assert "No state" in result
|
||||||
|
|
||||||
|
async def test_cite_raises_modelretry_when_chunk_ids_unresolved(
|
||||||
|
self, rag_db, rag_client
|
||||||
|
):
|
||||||
|
"""When supplied chunk_ids don't match any search result, cite raises
|
||||||
|
ModelRetry so pydantic-ai prompts the model to retry with valid ids
|
||||||
|
instead of silently registering zero citations."""
|
||||||
|
from pydantic_ai import ModelRetry
|
||||||
|
|
||||||
|
from haiku.rag.skills.rag import RAGState, create_skill
|
||||||
|
|
||||||
|
skill = create_skill(db_path=rag_db)
|
||||||
|
search = _get_tool(skill, "search")
|
||||||
|
cite = _get_tool(skill, "cite")
|
||||||
|
state = RAGState()
|
||||||
|
ctx = _make_ctx(state, rag=rag_client)
|
||||||
|
|
||||||
|
await search(ctx, query="artificial intelligence")
|
||||||
|
assert state.searches, "fixture should have produced some search results"
|
||||||
|
|
||||||
|
with pytest.raises(ModelRetry) as exc_info:
|
||||||
|
await cite(ctx, chunk_ids=["372c9ddf-not-a-real-id"])
|
||||||
|
message = str(exc_info.value)
|
||||||
|
assert "verbatim" in message
|
||||||
|
assert "372c9ddf-not-a-real-id" in message
|
||||||
|
|
||||||
|
async def test_cite_raises_modelretry_when_no_searches_recorded(self, rag_db):
|
||||||
|
"""If cite is called before any search has populated state.searches,
|
||||||
|
the retry message tells the model to call search first."""
|
||||||
|
from pydantic_ai import ModelRetry
|
||||||
|
|
||||||
|
from haiku.rag.skills.rag import RAGState, create_skill
|
||||||
|
|
||||||
|
skill = create_skill(db_path=rag_db)
|
||||||
|
cite = _get_tool(skill, "cite")
|
||||||
|
state = RAGState()
|
||||||
|
ctx = _make_ctx(state)
|
||||||
|
|
||||||
|
with pytest.raises(ModelRetry) as exc_info:
|
||||||
|
await cite(ctx, chunk_ids=["any-id"])
|
||||||
|
assert "search" in str(exc_info.value).lower()
|
||||||
|
|
||||||
|
async def test_cite_returns_message_when_chunk_ids_empty(self, rag_db):
|
||||||
|
"""An empty chunk_ids list is a no-op, not a retry trigger."""
|
||||||
|
from haiku.rag.skills.rag import RAGState, create_skill
|
||||||
|
|
||||||
|
skill = create_skill(db_path=rag_db)
|
||||||
|
cite = _get_tool(skill, "cite")
|
||||||
|
state = RAGState()
|
||||||
|
ctx = _make_ctx(state)
|
||||||
|
result = await cite(ctx, chunk_ids=[])
|
||||||
|
assert "0" in result
|
||||||
|
|
||||||
|
|
||||||
class TestLifespan:
|
class TestLifespan:
|
||||||
async def test_opens_one_client_per_invocation(self, rag_db):
|
async def test_opens_one_client_per_invocation(self, rag_db):
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue