Remove analyze tool from chat agent, not yet ready for integration
This commit is contained in:
Yiorgis Gozadinos
parent
9a480cee90
commit
fb6bbca124
5 changed files with 1 additions and 1319 deletions
|
|
@ -62,14 +62,13 @@ Key features:
|
|||
|
||||
### Tools
|
||||
|
||||
The chat agent uses six tools:
|
||||
The chat agent uses five tools:
|
||||
|
||||
- `list_documents` — Browse available documents in the knowledge base
|
||||
- `summarize_document` — Generate a summary of a specific document
|
||||
- `get_document` — Retrieve a specific document by title or URI
|
||||
- `search` — Hybrid search with optional document filter
|
||||
- `ask` — Answer questions using the conversational research graph (automatically recalls prior answers)
|
||||
- `analyze` — Complex analytical questions via code execution (counting, aggregation, comparison)
|
||||
|
||||
The `ask` tool automatically checks conversation history before running research. It uses embedding similarity (0.7 cosine threshold) to find semantically matching prior answers, which are passed to the research planner as context. When prior answers are sufficient, the planner can skip searching entirely.
|
||||
|
||||
|
|
|
|||
|
|
@ -447,65 +447,4 @@ def create_chat_agent(config: AppConfig) -> Agent[ChatDeps, str]:
|
|||
|
||||
return f"**Summary of {doc.title or doc.uri}:**\n\n{result.output}"
|
||||
|
||||
@agent.tool
|
||||
async def analyze(
|
||||
ctx: RunContext[ChatDeps],
|
||||
task: str,
|
||||
document_name: str | None = None,
|
||||
) -> str:
|
||||
"""Execute a computational task via code execution.
|
||||
|
||||
IMPORTANT: Provide a clear, specific task instruction that describes
|
||||
exactly what to compute. Do NOT pass the user's question directly.
|
||||
|
||||
Examples of good task instructions:
|
||||
- "Count the total number of documents using list_documents()"
|
||||
- "Search for 'Python' and return the titles of all matching documents"
|
||||
- "Calculate the average word count across all documents"
|
||||
|
||||
Args:
|
||||
task: A specific, actionable instruction describing what to compute
|
||||
document_name: Optional document to focus on
|
||||
"""
|
||||
from haiku.rag.agents.rlm import RLMContext, RLMDeps, create_rlm_agent
|
||||
|
||||
client = ctx.deps.client
|
||||
session_state = ctx.deps.session_state
|
||||
|
||||
# Build session filter from document_filter
|
||||
session_filter = build_multi_document_filter(session_state.document_filter)
|
||||
|
||||
# Build tool filter from document_name parameter
|
||||
tool_filter = build_document_filter(document_name) if document_name else None
|
||||
|
||||
# Combine filters: session AND tool
|
||||
filter_clause = combine_filters(session_filter, tool_filter)
|
||||
|
||||
# Call RLM agent directly to access code executions
|
||||
rlm_context = RLMContext(filter=filter_clause)
|
||||
deps = RLMDeps(
|
||||
client=client,
|
||||
config=ctx.deps.config,
|
||||
context=rlm_context,
|
||||
)
|
||||
|
||||
rlm_agent = create_rlm_agent(ctx.deps.config)
|
||||
result = await rlm_agent.run(task, deps=deps)
|
||||
|
||||
# Format response with code executions
|
||||
answer = result.output.answer
|
||||
code_executions = rlm_context.code_executions
|
||||
|
||||
if code_executions:
|
||||
code_section = "\n\n---\n**Code executed:**\n"
|
||||
for i, execution in enumerate(code_executions, 1):
|
||||
code_section += f"\n```python\n# Execution {i}\n{execution.code}\n```\n"
|
||||
if execution.stdout.strip():
|
||||
code_section += f"Output:\n```\n{execution.stdout.strip()}\n```\n"
|
||||
if execution.stderr.strip():
|
||||
code_section += f"Errors:\n```\n{execution.stderr.strip()}\n```\n"
|
||||
return answer + code_section
|
||||
|
||||
return answer
|
||||
|
||||
return agent
|
||||
|
|
|
|||
|
|
@ -15,19 +15,6 @@ How to decide which tool to use:
|
|||
- "get_document" - Use when the user wants the FULL content of a specific document (e.g., "get the paper about Y", "fetch 2412.00566", "show me the full document").
|
||||
- "ask" - Use for CONTENT questions: "What does X say about Y?", "What are the main findings?", "Explain concept Z from the documents". This tool retrieves and synthesizes text from documents.
|
||||
- "search" - Use when the user explicitly asks to search/find/explore documents. Call it ONCE. After calling search, copy the ENTIRE tool response to your output INCLUDING the content snippets. Do NOT shorten, summarize, or omit any part of the results.
|
||||
- "analyze" - Use for COMPUTATIONAL tasks. IMPORTANT: Do NOT pass the user's question directly. Instead, write a specific task instruction describing what to compute.
|
||||
|
||||
IMPORTANT - Choosing between "ask" and "analyze":
|
||||
- "ask" answers WHAT questions about content (retrieval + synthesis)
|
||||
- "analyze" answers HOW MANY/HOW MUCH questions requiring computation
|
||||
|
||||
CRITICAL - When using "analyze", reformulate the user's question into a specific task:
|
||||
- User: "How many documents are there?" → task="Count the total number of documents using list_documents()"
|
||||
- User: "What is the total revenue across all reports?" → task="Search for revenue figures in all documents, extract the numeric values, and calculate the sum"
|
||||
- User: "How many documents discuss climate change?" → task="Search for 'climate change' and count the number of unique documents returned"
|
||||
- User: "List all the dates mentioned" → task="Search across documents, extract all date patterns, and return a deduplicated list"
|
||||
|
||||
When "analyze" returns results, include both the answer AND the "Code executed" section in your response to the user. This shows transparency about how the computation was performed.
|
||||
|
||||
IMPORTANT - When user mentions a document in search/ask:
|
||||
- If user says "search in <doc>", "find in <doc>", "answer from <doc>", or "<topic> in <doc>":
|
||||
|
|
|
|||
|
|
@ -1265,47 +1265,3 @@ async def test_summarization_task_cleanup_on_completion():
|
|||
|
||||
# Task should be cleaned up
|
||||
assert session_id not in _summarization_tasks
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# analyze Tool Tests
|
||||
# =============================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.vcr()
|
||||
async def test_analyze_tool(allow_model_requests, temp_db_path):
|
||||
"""Test the analyze tool for complex analytical questions."""
|
||||
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||
# Add test documents
|
||||
await client.create_document(
|
||||
content=DOCLAYNET_CLASS_LABELS,
|
||||
uri="doclaynet-labels",
|
||||
title="DocLayNet Class Labels",
|
||||
)
|
||||
await client.create_document(
|
||||
content=DOCLAYNET_ANNOTATION,
|
||||
uri="doclaynet-annotation",
|
||||
title="DocLayNet Annotation",
|
||||
)
|
||||
await client.create_document(
|
||||
content=DOCLAYNET_DATA_SOURCES,
|
||||
uri="doclaynet-sources",
|
||||
title="DocLayNet Sources",
|
||||
)
|
||||
|
||||
agent = create_chat_agent(Config)
|
||||
deps = ChatDeps(
|
||||
client=client,
|
||||
config=Config,
|
||||
)
|
||||
|
||||
# Ask an analytical question that requires computation
|
||||
result = await agent.run(
|
||||
"How many documents are in the database?",
|
||||
deps=deps,
|
||||
)
|
||||
|
||||
assert result.output is not None
|
||||
# The answer should mention 3 documents
|
||||
assert "3" in result.output or "three" in result.output.lower()
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
Loading…
Reference in a new issue