Search results now show rank position instead of raw scores. Consolidate cassettes

This commit is contained in:
Yiorgis Gozadinos 2026-01-21 14:53:27 +02:00
parent dff313fa74
commit 7f51c63ad7
No known key found for this signature in database
38 changed files with 10904 additions and 18445 deletions

View file

@ -5,6 +5,19 @@
- **Model Downloads**: `download-models` now pre-downloads HuggingFace models for `sentence-transformers`, `mxbai`, and `jina-local`
- **Reranker Factory**: Removed unreliable `id(config)`-based caching from `get_reranker()`; factory now always instantiates fresh
### Changed
- **Agent Search Result Display**: Search results now show rank position instead of raw scores
- `SearchResult.format_for_agent()` accepts optional `rank` and `total` parameters
- Output changes from `(score: 0.02)` to `[rank 1 of 5]` when rank is provided
- Prevents LLMs from misinterpreting low RRF hybrid search scores as "2% relevant"
- QA and Research agents updated to pass rank/total to formatted results
- Agent prompts updated to reference rank-based ordering instead of scores
### Fixed
- **Test Cassette Organization**: Consolidated all VCR cassettes to `tests/cassettes/`
## [0.26.7] - 2026-01-20
### Added

View file

@ -57,8 +57,12 @@ class QuestionAnswerAgent:
results = await ctx.deps.client.expand_context(results)
# Store results for citation resolution
ctx.deps.search_results = results
# Format with metadata for agent context
parts = [r.format_for_agent() for r in results]
# Format with rank instead of raw score to avoid confusing LLMs
total = len(results)
parts = [
r.format_for_agent(rank=i + 1, total=total)
for i, r in enumerate(results)
]
return "\n\n".join(parts) if parts else "No results found."
async def answer(

View file

@ -2,18 +2,18 @@ QA_SYSTEM_PROMPT = """You are a knowledgeable assistant that answers questions u
Process:
1. Call search_documents with relevant keywords from the question
2. Review the results and their relevance scores
2. Review the results ordered by relevance
3. If needed, perform follow-up searches with different keywords (max 3 total)
4. Provide a concise answer based strictly on the retrieved content
The search tool returns results like:
[chunk_abc123] (score: 0.85)
[chunk_abc123] [rank 1 of 5]
Source: "Document Title" > Section > Subsection
Type: paragraph
Content:
The actual text content here...
[chunk_def456] (score: 0.72)
[chunk_def456] [rank 2 of 5]
Source: "Another Document"
Type: table
Content:
@ -21,7 +21,7 @@ Content:
...
Each result includes:
- chunk_id in brackets and relevance score
- chunk_id in brackets and rank position (rank 1 = most relevant)
- Source: document title and section hierarchy (when available)
- Type: content type like paragraph, table, code, list_item (when available)
- Content: the actual text
@ -34,5 +34,5 @@ Guidelines:
- If multiple results are relevant, synthesize them coherently
- If information is insufficient, say: "I cannot find enough information in the knowledge base to answer this question."
- Be concise and direct - avoid elaboration unless asked
- Higher scores indicate more relevant results
- Results are ordered by relevance, with rank 1 being most relevant
"""

View file

@ -191,7 +191,12 @@ async def _search_one_step_logic(
)
results = await ctx2.deps.client.expand_context(results)
ctx2.deps.search_results = results
parts = [r.format_for_agent() for r in results]
# Format with rank instead of raw score to avoid confusing LLMs
total = len(results)
parts = [
r.format_for_agent(rank=i + 1, total=total)
for i, r in enumerate(results)
]
if not parts:
return f"No relevant information found for: {query}"
return "\n\n".join(parts)

View file

@ -49,18 +49,18 @@ SEARCH_PROMPT = """You are a search and question-answering specialist.
Process:
1. Call search_and_answer with relevant keywords from the question.
2. Review the results and their relevance scores.
2. Review the results ordered by relevance.
3. If needed, perform follow-up searches with different keywords (max 3 total).
4. Provide a concise answer based strictly on the retrieved content.
The search tool returns results like:
[9bde5847-44c9-400a-8997-0e6b65babf92] (score: 0.85)
[9bde5847-44c9-400a-8997-0e6b65babf92] [rank 1 of 5]
Source: "Document Title" > Section > Subsection
Type: paragraph
Content:
The actual text content here...
[d5a63c82-cb40-439f-9b2e-de7d177829b7] (score: 0.72)
[d5a63c82-cb40-439f-9b2e-de7d177829b7] [rank 2 of 5]
Source: "Another Document"
Type: table
Content:
@ -68,7 +68,7 @@ Content:
...
Each result includes:
- chunk_id in brackets and relevance score
- chunk_id in brackets and rank position (rank 1 = most relevant)
- Source: document title and section hierarchy (when available)
- Type: content type like paragraph, table, code, list_item (when available)
- Content: the actual text
@ -87,7 +87,7 @@ Guidelines:
- If multiple results are relevant, synthesize them coherently.
- If information is insufficient, say so clearly.
- Be concise and direct; avoid meta commentary about the process.
- Higher scores indicate more relevant results."""
- Results are ordered by relevance, with rank 1 being most relevant."""
DECISION_PROMPT = """You are the research evaluator responsible for assessing
whether gathered evidence sufficiently answers the research question.

View file

@ -143,13 +143,25 @@ class SearchResult(BaseModel):
labels=meta.labels,
)
def format_for_agent(self) -> str:
def format_for_agent(
self, rank: int | None = None, total: int | None = None
) -> str:
"""Format this search result for inclusion in agent context.
Args:
rank: 1-based position in results (1 = most relevant)
total: Total number of results returned
Produces a structured format with metadata that helps LLMs understand
the source and nature of the content.
the source and nature of the content. When rank is provided, shows
position instead of raw score to avoid confusing LLMs with low RRF scores.
"""
parts = [f"[{self.chunk_id}] (score: {self.score:.2f})"]
if rank is not None and total is not None:
parts = [f"[{self.chunk_id}] [rank {rank} of {total}]"]
elif rank is not None:
parts = [f"[{self.chunk_id}] [rank {rank}]"]
else:
parts = [f"[{self.chunk_id}] (score: {self.score:.2f})"]
# Document source info
source_parts = []

File diff suppressed because one or more lines are too long

View file

@ -1,3 +1,5 @@
from pathlib import Path
import pytest
from haiku.rag.agents.research.dependencies import ResearchContext
@ -6,6 +8,11 @@ from haiku.rag.agents.research.state import ResearchDeps, ResearchState
from haiku.rag.client import HaikuRAG
@pytest.fixture(scope="module")
def vcr_cassette_dir():
return str(Path(__file__).parent.parent.parent / "cassettes" / "test_search_filter")
@pytest.fixture
async def client_with_docs(temp_db_path):
"""Create a client with two distinct documents."""

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -126,7 +126,178 @@ interactions:
response:
headers:
content-length:
- '585'
- '559'
content-type:
- application/json
parsed_body:
choices:
- finish_reason: tool_calls
index: 0
message:
content: ''
reasoning: User wants nonexistent document. We can use get_document but it may not exist. We'll try get_document.
role: assistant
tool_calls:
- function:
arguments: '{"query":"nonexistent document"}'
name: get_document
id: call_31uy8050
index: 0
type: function
created: 1768998264
id: chatcmpl-114
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 47
prompt_tokens: 842
total_tokens: 889
status:
code: 200
message: OK
- request:
headers:
accept:
- application/json
accept-encoding:
- gzip, deflate, zstd
connection:
- keep-alive
content-length:
- '4470'
content-type:
- application/json
host:
- localhost:11434
method: POST
parsed_body:
messages:
- content: |-
You are a helpful research assistant powered by haiku.rag, a knowledge base system.
You have access to a knowledge base of documents. Use your tools to search and answer questions.
CRITICAL RULES:
1. For greetings or casual chat: respond directly WITHOUT using any tools
2. For questions: Use the "ask" tool EXACTLY ONCE - it handles query expansion internally
3. For searches: Use the "search" tool EXACTLY ONCE - it handles multi-query expansion internally
4. NEVER call the same tool multiple times for a single user message
5. NEVER make up information - always use tools to get facts from the knowledge base
How to decide which tool to use:
- "get_document" - Use when the user references a SPECIFIC document by name, title, or URI (e.g., "summarize document X", "get the paper about Y", "fetch 2412.00566"). Retrieves the full document content.
- "ask" - Use for general questions about topics in the knowledge base when no specific document is named. It searches across all documents and returns answers with citations.
- "search" - Use when the user explicitly asks to search/find/explore documents. Call it ONCE. After calling search, copy the ENTIRE tool response to your output INCLUDING the content snippets. Do NOT shorten, summarize, or omit any part of the results.
IMPORTANT - When user mentions a document in search/ask:
- If user says "search in <doc>", "find in <doc>", "answer from <doc>", or "<topic> in <doc>":
- Extract the TOPIC as `query`/`question`
- Extract the DOCUMENT NAME as `document_name`
- Examples for search:
- "search for embeddings in the ML paper" → query="embeddings", document_name="ML paper"
- "find transformer architecture in 2412.00566" → query="transformer architecture", document_name="2412.00566"
- Examples for ask:
- "what does the ML paper say about embeddings?" → question="what are the embedding methods?", document_name="ML paper"
- "answer from 2412.00566 about model training" → question="how is the model trained?", document_name="2412.00566"
Be friendly and conversational. When you use the "ask" tool, summarize the key findings for the user.
role: system
- content: Get me the nonexistent document
role: user
- content: |-
<think>
User wants nonexistent document. We can use get_document but it may not exist. We'll try get_document.
</think>
role: assistant
tool_calls:
- function:
arguments: '{"query":"nonexistent document"}'
name: get_document
id: call_31uy8050
type: function
- content: 'Document not found: nonexistent document'
role: tool
tool_call_id: call_31uy8050
model: gpt-oss
reasoning_effort: low
stream: false
tool_choice: auto
tools:
- function:
description: |-
Search the knowledge base for relevant documents.
Use this when you need to find documents or explore the knowledge base.
Results are displayed to the user - just list the titles found.
name: search
parameters:
additionalProperties: false
properties:
document_name:
anyOf:
- type: string
- type: 'null'
default: null
description: Optional document name/title to search within
limit:
anyOf:
- type: integer
- type: 'null'
default: null
description: 'Number of results to return (default: 5)'
query:
description: The search query (what to search for)
type: string
required:
- query
type: object
type: function
- function:
description: |-
Answer a specific question using the knowledge base.
Use this for direct questions that need a focused answer with citations.
Uses a research graph for planning, searching, and synthesis.
name: ask
parameters:
additionalProperties: false
properties:
document_name:
anyOf:
- type: string
- type: 'null'
default: null
description: Optional document name/title to search within (e.g., "tbmed593", "army manual")
question:
description: The question to answer
type: string
required:
- question
type: object
type: function
- function:
description: |-
Retrieve a specific document by title or URI.
Use this when the user wants to fetch/get/retrieve a specific document.
name: get_document
parameters:
additionalProperties: false
properties:
query:
description: The document title or URI to look up
type: string
required:
- query
type: object
strict: true
type: function
uri: http://localhost:11434/v1/chat/completions
response:
headers:
content-length:
- '459'
content-type:
- application/json
parsed_body:
@ -134,20 +305,18 @@ interactions:
- finish_reason: stop
index: 0
message:
content: Im sorry—I couldnt find a document matching that name in the knowledge base. If you have any other request
or need help locating a different resource, just let me know!
reasoning: User asks for nonexistent document. Use get_document? but tool should not fabricate. We can explain not
found.
content: Im sorry, but I couldnt find a document titled “nonexistent document.” If you have another title or some
details to share, let me know and Ill look it up for you!
role: assistant
created: 1768225927
id: chatcmpl-215
created: 1768998265
id: chatcmpl-968
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 68
prompt_tokens: 842
total_tokens: 910
completion_tokens: 44
prompt_tokens: 912
total_tokens: 956
status:
code: 200
message: OK

View file

@ -195,11 +195,11 @@ interactions:
- function:
arguments: '{"query":"DocLayNet Class Labels"}'
name: get_document
id: call_geowshoi
id: call_m5izt73j
index: 0
type: function
created: 1768225921
id: chatcmpl-162
created: 1768998258
id: chatcmpl-390
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
@ -268,14 +268,14 @@ interactions:
- function:
arguments: '{"query":"DocLayNet Class Labels"}'
name: get_document
id: call_geowshoi
id: call_m5izt73j
type: function
- content: |-
**DocLayNet Class Labels**
- ID: 50eeb7f5-49be-47d4-b1bd-19d221da9794
- ID: cd0fc5b8-ec16-4f1c-99fe-20979ecaab1c
- URI: doclaynet-labels
- Created: 2026-01-12 15:52
- Created: 2026-01-21 14:24
**Content:**
DocLayNet Dataset - Class Labels
@ -296,7 +296,7 @@ interactions:
The Text class has the highest count with 510,377 instances in the dataset.
role: tool
tool_call_id: call_geowshoi
tool_call_id: call_m5izt73j
model: gpt-oss
reasoning_effort: low
stream: false
@ -375,7 +375,7 @@ interactions:
response:
headers:
content-length:
- '1105'
- '1131'
content-type:
- application/json
parsed_body:
@ -383,23 +383,24 @@ interactions:
- finish_reason: stop
index: 0
message:
content: "Heres the document you requested:\n\n**DocLayNet Class Labels**\n\nDocLayNet defines 11 distinct class
labels for document layout analysis:\n\n1. **Caption** Text describing figures or tables \n2. **Footnote**
Notes at the bottom of pages \n3. **Formula** Mathematical expressions \n4. **Listitem** Items in bulleted
or numbered lists \n5. **Pagefooter** Footer content on pages \n6. **Pageheader** Header content on pages
\ \n7. **Picture** Images and diagrams \n8. **Sectionheader** Headings for document sections \n9. **Table**
Tabular data \n10. **Text** Regular paragraph text (highest count: 510,377 instances) \n11. **Title**
Document titles \n\nThats the full list of labels used in DocLayNet. Let me know if youd like any more details!"
content: "Heres the full **DocLayNet Class Labels** document:\n\n---\n\n## DocLayNet Dataset Class Labels\n\nDocLayNet
defines **11 distinct class labels** for document layout analysis:\n\n1. **Caption** Text describing figures
or tables \n2. **Footnote** Notes at the bottom of pages \n3. **Formula** Mathematical expressions \n4.
**Listitem** Items in bulleted or numbered lists \n5. **Pagefooter** Footer content on pages \n6. **Pageheader**
Header content on pages \n7. **Picture** Images and diagrams \n8. **Sectionheader** Headings for document
sections \n9. **Table** Tabular data \n10. **Text** Regular paragraph text (highest count: **510,377** instances)
\ \n11. **Title** Document titles \n\nThe **Text** class has the highest count with 510,377 instances in the
dataset."
role: assistant
created: 1768225925
id: chatcmpl-313
created: 1768998262
id: chatcmpl-746
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 196
completion_tokens: 202
prompt_tokens: 1110
total_tokens: 1306
total_tokens: 1312
status:
code: 200
message: OK

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -227,7 +227,7 @@ interactions:
response:
headers:
content-length:
- '493'
- '502'
content-type:
- application/json
parsed_body:
@ -236,17 +236,17 @@ interactions:
index: 0
message:
content: ''
reasoning: Need search.
reasoning: Need to use search once.
role: assistant
tool_calls:
- function:
arguments: '{"document_name":null,"limit":null,"query":"class labels"}'
arguments: '{"query":"class labels","document_name":null,"limit":5}'
name: search
id: call_bji3vk31
id: call_m70fxfgn
index: 0
type: function
created: 1768403358
id: chatcmpl-424
created: 1768998240
id: chatcmpl-236
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
@ -317,7 +317,7 @@ interactions:
response:
headers:
content-length:
- '565'
- '478'
content-type:
- application/json
parsed_body:
@ -326,24 +326,24 @@ interactions:
index: 0
message:
content: ''
reasoning: Need to search "class labels". Likely referring to machine learning classification labels. Provide search.
reasoning: Need to run search.
role: assistant
tool_calls:
- function:
arguments: '{"limit":5,"query":"class labels"}'
arguments: '{"query":"class labels","limit":5}'
name: run_search
id: call_gcwrxbl4
id: call_6dvnsbzh
index: 0
type: function
created: 1768403359
id: chatcmpl-9
created: 1768998241
id: chatcmpl-107
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 48
completion_tokens: 34
prompt_tokens: 263
total_tokens: 311
total_tokens: 297
status:
code: 200
message: OK
@ -396,7 +396,7 @@ interactions:
connection:
- keep-alive
content-length:
- '1457'
- '1368'
content-type:
- application/json
host:
@ -421,18 +421,18 @@ interactions:
role: user
- content: |-
<think>
Need to search "class labels". Likely referring to machine learning classification labels. Provide search.
Need to run search.
</think>
role: assistant
tool_calls:
- function:
arguments: '{"limit":5,"query":"class labels"}'
arguments: '{"query":"class labels","limit":5}'
name: run_search
id: call_gcwrxbl4
id: call_6dvnsbzh
type: function
- content: 'Found 2 results for: class labels'
role: tool
tool_call_id: call_gcwrxbl4
tool_call_id: call_6dvnsbzh
model: gpt-oss
reasoning_effort: low
stream: false
@ -461,7 +461,7 @@ interactions:
response:
headers:
content-length:
- '452'
- '524'
content-type:
- application/json
parsed_body:
@ -470,23 +470,24 @@ interactions:
index: 0
message:
content: ''
reasoning: 'Now an alternative query: maybe "label classification".'
role: assistant
tool_calls:
- function:
arguments: '{"limit":5,"query":"label classification"}'
name: run_search
id: call_di042ewg
id: call_eonlxxeh
index: 0
type: function
created: 1768403360
id: chatcmpl-890
created: 1768998242
id: chatcmpl-552
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 23
prompt_tokens: 336
total_tokens: 359
completion_tokens: 39
prompt_tokens: 322
total_tokens: 361
status:
code: 200
message: OK
@ -539,7 +540,7 @@ interactions:
connection:
- keep-alive
content-length:
- '1746'
- '1731'
content-type:
- application/json
host:
@ -564,29 +565,32 @@ interactions:
role: user
- content: |-
<think>
Need to search "class labels". Likely referring to machine learning classification labels. Provide search.
Need to run search.
</think>
role: assistant
tool_calls:
- function:
arguments: '{"limit":5,"query":"class labels"}'
arguments: '{"query":"class labels","limit":5}'
name: run_search
id: call_gcwrxbl4
id: call_6dvnsbzh
type: function
- content: 'Found 2 results for: class labels'
role: tool
tool_call_id: call_gcwrxbl4
- content: null
tool_call_id: call_6dvnsbzh
- content: |-
<think>
Now an alternative query: maybe "label classification".
</think>
role: assistant
tool_calls:
- function:
arguments: '{"limit":5,"query":"label classification"}'
name: run_search
id: call_di042ewg
id: call_eonlxxeh
type: function
- content: 'Found 2 results for: label classification'
role: tool
tool_call_id: call_di042ewg
tool_call_id: call_eonlxxeh
model: gpt-oss
reasoning_effort: low
stream: false
@ -615,7 +619,7 @@ interactions:
response:
headers:
content-length:
- '297'
- '298'
content-type:
- application/json
parsed_body:
@ -625,15 +629,15 @@ interactions:
message:
content: Search complete
role: assistant
created: 1768403361
id: chatcmpl-58
created: 1768998242
id: chatcmpl-801
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 6
prompt_tokens: 378
total_tokens: 384
prompt_tokens: 386
total_tokens: 392
status:
code: 200
message: OK
@ -646,7 +650,7 @@ interactions:
connection:
- keep-alive
content-length:
- '5071'
- '5080'
content-type:
- application/json
host:
@ -688,14 +692,14 @@ interactions:
role: user
- content: |-
<think>
Need search.
Need to use search once.
</think>
role: assistant
tool_calls:
- function:
arguments: '{"document_name":null,"limit":null,"query":"class labels"}'
arguments: '{"query":"class labels","document_name":null,"limit":5}'
name: search
id: call_bji3vk31
id: call_m70fxfgn
type: function
- content: |-
Found 2 results:
@ -706,7 +710,7 @@ interactions:
[2] **DocLayNet Annotation**
DocLayNet Dataset - Annotation Process The annotation process was organized into 4 phases: Phase 1: Data selection and preparation by a small team of experts Phase 2: Label selection and guideline definition Phase 3: Annotation by 40 dedicated annotators Phase 4: Quality control and continuous...
role: tool
tool_call_id: call_bji3vk31
tool_call_id: call_m70fxfgn
model: gpt-oss
reasoning_effort: low
stream: false
@ -785,7 +789,7 @@ interactions:
response:
headers:
content-length:
- '660'
- '700'
content-type:
- application/json
parsed_body:
@ -793,22 +797,20 @@ interactions:
- finish_reason: stop
index: 0
message:
content: |-
Here are the documents that mention **class labels**:
1. **DocLayNet Class Labels** Describes the 11 distinct class labels used for document layout analysis, such as Caption, Footnote, Formula, etc.
2. **DocLayNet Annotation** Discusses the annotation process for the DocLayNet dataset, including how the class labels were selected and applied during labeling.
content: "Here are two documents that discuss class labels:\n\n1. **DocLayNet Class Labels** \n - Describes the
11 distinct class labels used for document layout analysis (e.g., Caption, Footnote, Formula, Listitem, Pagefooter,
Pageheader, etc.).\n\n2. **DocLayNet Annotation** \n - Covers the annotation process, including how labels
were selected and guided during the construction of the DocLayNet dataset."
role: assistant
created: 1768403363
id: chatcmpl-294
created: 1768998245
id: chatcmpl-955
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 83
prompt_tokens: 1048
total_tokens: 1131
completion_tokens: 96
prompt_tokens: 1051
total_tokens: 1147
status:
code: 200
message: OK

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -39,63 +39,6 @@ interactions:
status:
code: 200
message: OK
- request:
body: !!binary |
H4sIAJMwUGkC/61VzY8URRTf5XOpAXa315UFJZZNTBbCNt0zs8wMCTGIFycxGkATA5tOTXf1TLHd
1W1VNcu4mYMHEjxw8eBFOZsQEv8AD/g3eCGakHg0Xr34ERNf9cdM77ArHJzLVL/v+r3fe4WeLqMn
+1EHLUkq7jCPWoxLRbg++IaJcIO2HUKdTrPZoU3bbnWcDm1f9HqNdiOwfa+BTPSqoiGNqBJDS/qb
Vkh4PyV9asyhQ8lQDWKO3kLGThtOImrMo2NxQvlYg95EyzvN7lAhWcx1JMdqdCwHTI6WhWYxFtF8
yjd5vMXdQo6WUC0RsUeltBK4w4GV777dD36vlEKRcsWiwv8IOuwVRZ5BJ6ZNKvkbltOwWugj9Nq0
kU+lJ1iitOFF1MwN8WpEGD+Pr9MEO21ct+vr53G9fmm9fqnRPItvXtUogdhyrCbeQJfRfHmtMuc5
tLrebjZ9u+Wt+806sQN/HbC32831estptr1O3Sa+0wpIz/jSQMtw66FPoCRvjTAjx8s2HuxDC7Nf
k8tP+I8Ljw9+8feFn68+M+Y6vyU/Pf5zuGXOPfgrceQ7vz89dwQ6xhWGS9mzneSPt8X91XsrV8pD
9wRCUezT0C1R6ydqLZayu4xQ5pkrDqOD2Vf3dbQI/y5hVvZt7dSeQrUw7gdMUCuSfaOGJtm7Z9Bi
qZMJ4a4aJtQ4jo4CU3zG+64Wdi+gk6VRKU+I0GVAxw20YE/9Ppu5NzszOzNj/Dq/Gx7vz37D9h36
+Ctz7nvj4eenFn94dO6oNyAKF7cESO4/PCIzSMpD9w20XFwRWCyI7n9+zUPogHbunkTHCgM5lIpG
mkZxGJKIdIGOhUrQT1MqlZWhWwX2NDquKUGFRXxfAOE0TGHskXAQS9U1UK1QJ7FQxv6VR590/6mh
lbxLRVQNClSkgFLGLzX0rLZtBin3dKmuiuNQmpfwzW1TVw0nU1IivIHrx14aAZbSPI/NSQT3tgQ3
6Q1oRMB624S6mA5Fwg+FhkAxqgMGJJRUe1aF2yYUJIbZSXc0S6cENM4cgW3IIqYypU8Dkob6zNMw
BBXhww+CvM7SkcHO6FOhPSdCbW6ONkY6nL4+cMPXbkXiDRCXpnHvNvVUlrgyvFpzPUMAqwHFeqmE
1O9T3COS4iAWWMByugMjhscIWbf4LX6NqlRwCWoJlUu8xdQAewPYS/i9dyUm3C89PYqlF4Odpb0C
CowFUenXG+rETOS+wGTMOPaYor6bSaSlGxKn0IxsKnzXZ55yN+lwgpbG1FOVLkjNBNgKJKwINxnX
2IzJoONCk4lPFJnE0tsNshWCkUawl7JQsQp5NvKKklS5mnk6qNaZE3EB9jhqId6FfgEDKrk5Gi+g
3p7cmurnDWhkNgnwDeDCRG8R3RG5RTU9KpyokDFX7xUu12IVZywpg+8Rqtq93QJqfsRB3nCJU0l9
HRi4FmXRi0ogEMSJ5F7TU4qIEGSY5415wHzNrqmhciz7edZfHVvn9MzIrgasRAqv2mvO2Ywk5C6L
0mgcKGK8+Laz78kwRj09oHsM4wTkXcZSK/9js+wytVfGLSE432E4y1OdRFGO2/8/RTmjX3qGgLtR
omlRTgKNkpAoWll5YRhvgfyuKmwqSXMdi+DBnFKOuptoqXwVKwNj3EDXtncBeWqG9no4dpCu3Jyj
0cu906d3PvSgrj6rL3zG235AWnbDaTWa9aDht7rm+M2FNZHEXNLnHs7yqf8XMrooUNoKAAA=
headers:
accept:
- '*/*'
accept-encoding:
- gzip, deflate, zstd
connection:
- keep-alive
content-encoding:
- gzip
content-length:
- '1307'
content-type:
- application/x-protobuf
method: POST
uri: https://logfire-eu.pydantic.dev/v1/traces
response:
body:
string: ''
headers:
connection:
- keep-alive
content-length:
- '0'
nel:
- '{"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}'
report-to:
- '{"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=mCozVqidqvTBt1g2DayKq8ombI9t7flItid4fd%2FjD%2FuxzCWRX%2FJ%2B8A2RaiZms8Btosj5mYmlsr4jYgoEnIPDu%2B5TkzrpLfdunaX%2B0BFbPLlp2gqt"}]}'
vary:
- origin, access-control-request-method, access-control-request-headers
status:
code: 200
message: OK
- request:
headers:
accept:
@ -105,7 +48,7 @@ interactions:
connection:
- keep-alive
content-length:
- '2630'
- '2668'
content-type:
- application/json
host:
@ -118,18 +61,18 @@ interactions:
Process:
1. Call search_documents with relevant keywords from the question
2. Review the results and their relevance scores
2. Review the results ordered by relevance
3. If needed, perform follow-up searches with different keywords (max 3 total)
4. Provide a concise answer based strictly on the retrieved content
The search tool returns results like:
[chunk_abc123] (score: 0.85)
[chunk_abc123] [rank 1 of 5]
Source: "Document Title" > Section > Subsection
Type: paragraph
Content:
The actual text content here...
[chunk_def456] (score: 0.72)
[chunk_def456] [rank 2 of 5]
Source: "Another Document"
Type: table
Content:
@ -137,7 +80,7 @@ interactions:
...
Each result includes:
- chunk_id in brackets and relevance score
- chunk_id in brackets and rank position (rank 1 = most relevant)
- Source: document title and section hierarchy (when available)
- Type: content type like paragraph, table, code, list_item (when available)
- Content: the actual text
@ -150,7 +93,7 @@ interactions:
- If multiple results are relevant, synthesize them coherently
- If information is insufficient, say: "I cannot find enough information in the knowledge base to answer this question."
- Be concise and direct - avoid elaboration unless asked
- Higher scores indicate more relevant results
- Results are ordered by relevance, with rank 1 being most relevant
role: system
- content: What is Python?
role: user
@ -212,7 +155,7 @@ interactions:
response:
headers:
content-length:
- '482'
- '484'
content-type:
- application/json
parsed_body:
@ -221,101 +164,24 @@ interactions:
index: 0
message:
content: ''
reasoning: Need search.
reasoning: Need to search.
role: assistant
tool_calls:
- function:
arguments: '{"limit":5,"query":"Python definition"}'
arguments: '{"query":"Python definition","limit":5}'
name: search_documents
id: call_snyo88ta
id: call_zzot5v63
index: 0
type: function
created: 1766862998
id: chatcmpl-511
created: 1768998999
id: chatcmpl-36
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 32
prompt_tokens: 532
total_tokens: 564
status:
code: 200
message: OK
- request:
body: !!binary |
H4sIAJYwUGkC/61YS4wcRxn2xE5Y13q963Zs1raSFB0wu8tuZ3oeu7OLArLXPDwSxIodQeRZjWq6
q2cq21096ere9WS9Bw5GyQFOSBwiSxEgIRIeJySkSIQDVw7kQkBEwJELQkII8ZL4/+rumZ7xrMkh
l1F31V//46vvf/SQ+yvkneNkk5xVPNoTDreEVDGT+OAaJqFV3rAZtzdrtU1eK5c3Nu1N3lh3OtVG
1Su7TpWY5HzMfR7wOBpYyt21fCa7CetyY4Y81h/EvVCSTxBjXEaygBvzZC7sczncIR8j58bF9nik
RChRk21VNy0bRE7ljmodZ8h8IndluC/b2To5S2b7Uehwpaw+xHBi8SffPw7nHs8Xo0TGIsjOnyQf
cTInnyYfnRQp2K9adtXaIC+QS5NCLldOJPoxCq6TWipIlwIm5Cq9yfvUbtBKuVJfpZXKVr2yVa0t
09vbiBIsW7ZVozvkWTKfh5XbXCFL9Uat5pY3nLpbq7Cy59YB+3KjVq9s2LWGs1kpM9fe8FjHeO8y
OQdRD1wGLjlrTBgpXmXjR5QslF5nz74j31344aOv/fuZ322/b8y8bdz/2sUzv3jTnPnGv/q2uvq3
36yccnospt1+vBYqVS5tvnr/pHp16d7ila/+4K1rr8FD8ylyrstlmwkLbi1iGG+K4WPkBB5uXiBz
mYAaqJgHCFvo+yxgTYA/24r4ywlXsRWELvcR/sxk8wlyGiHgkcVcNwKAjVly0g8d5vdCFTcNMptt
98MoNo4vvvli87+zZFHraWda230WgUcxQGj8cZa8P3tgeol00NV2HIa+Mrfo7QMTvYYnU3EWOb22
GzpJADxU5io1RxraLyk4ppweDxhIH5jgl0BVzL8RIQSx4KjQY77ieLK4eGCCQ9FAP8WDfmoujoTs
mocg64tAxHrT5R5LfHyWie/DFpOD57zUz/yggBzp8ghPjhZR3DzcOUR1GL6IuIvHMsM7sJyLhp2X
uBNrwwWy4s5NjQCNe5xiEvnc7XLaYYpTL4xoBMm4B5SiQ4SslmzJ53mcRFLBtgLPFd0XcY86PchD
ev2aoky6+UmHU+WEIGfhKY9HHJfyc50BGhZRerYtXCokdUTM3bZeURZeSJjAZbQxFLftCidu7/LB
CC3E1IkLt6CQCZAFzC8s7gqJ2AzJgHrhkpnLYjbShdkM1rKFQ0Swkwg/FgXy7KQe9ZO4jcxDpbhn
jpYzsIdas+Up9PMEUKmdovF/qHcktybu8xZcpM4EeAdwIaP3Gd6I2udIjwInCmRMt49Sl+7SONQs
yZUfoap4e9MUIj9CL71wRRPFXVQMXAu09swTUAR6AnVU9uRLLIrYILUbSk+4yK6JpLKt8oOs3x5K
p/TUZI97IkeKLpXX7GVNEnZHBEkwVBQImb2X9fsoGYMOJugRyTgCeUpa4uZDKsuUrL0yvBJG0xpG
tZ1iJkZ5un34WZQy+gPnEHA36CMt8kzgQd9nMS+UPN8P92H9TpzJFIymeyKAgWJi87B5iZzxw64H
YFuqz6QOD7sRvkA/mc03A9U1TpOxDtfsDbuZkKDUCqDjgA1lPEe+BDkahb6+I6BolOVmrMarMvpr
ptyLAStc+gpaABrd0BPFZ7E87zS//Qg5nzdOHcDI1n9K5J+lgjWmlMDpKz7KZE/IXcyCcbNf5pBH
KRGs8SaBVacNLVQXKKFvD9/aSg7CRgMuDpYf1g1Z1M1eQOCglfatlrlVX22l1IbnlpmGSyHrID2Q
pC3z0NR3jwuqBzWOqZS6I4cAmV/NkotjE0Mbh88o0TVaGT+eJW/NPhzwF8OEMkhgNmpfrOPDew5k
WgTT5FPD6oWlB2CEY3m0E+1P97kb6aC31ZK2RbfBZzoJUZpwwzYJObUfRq6iHlB+rFy2ZMWiz/M9
wff1et4CsVemPXCyY7Zk1aLXPSrhbrm7SqEq6DLphZgQa0k/c4ZnTrjC0wlf8AJG0Du0CmUiZv5y
S9YsChHtQeGDuAFDR6hhdceYgUK6DPgDqpsHegkLfA92MsgRFewJWdHBy0ShsWnAF7scILudNnXW
cexKdYcu6aiwbDbq4MvNMIkceG2Z1/ILuCVin7dM+hmYmTUB8CnpqPQF7AILtij2yG7E+r2W3E59
2kpdYk6cMJ8iR3JvaQ8AsSx9l5k7QNFafb3ozkZlzJ0rMoTII5q71TJzyzESq2j1Lt0O/SSQ1KbD
xwq925KZyc8xp5ehAoON4ydQxeHY2ti404mYs8vjqVMTyuaeDXkaI0xaOkOG9gRM43AdA7q03+OS
sj0mfHR2GRWkzueIYCrpGxoBuZpGtgoyLvz6kDdt7L5TteXRp916BDnGe13SAXiLIfchxUBXFrUW
Hs2HIJR2/slpD5V8IQF++kJmUF1lQ46qMXo+QE26BhhRuD3UTcElHkE7HaU1antBpb6koGoQER6a
tzHsqIl0wVaMe1ozxraGiRjAPYq+X8jdiA9Tf5WqgQTVSryiTQRwuKfT0R9k54XE/NWfTdgioNIl
niccwfVpNkD2XacOkxgD1E2Xchkm3d74QTltVsdBIBsJcIjJi46F5AUMeSHbXSgUERAH4GJ7IXCQ
w+WG6dccxO5DvYPiuctdPPlF0cVcSAsS2IbBAfo2DcJC5DkcWM//USJn855bmGCNP5TI70sHU+ae
ibF2akMeGwJHE9/0jvpw4SlN5ogDR31Vjonnn1WHh01KLmQ2EvQjDQFm/l0Ofez44rdOgMTFMYnh
Z4EWeWSRNs3hRJJn0IMfyR8nxqSMcPPZxoG5aq1u202bPDkpNdaIlTG/Mkfg83rYjV85dq90rHTs
mPHn0rT/C17/y69vXV164pfmzG/vfmr93vfmPr8yFyVSYg/VHzb4hwH/qf6f4Er+0HyKPKo3jfMr
jxODLEx2TxAYG9EWyOlcqa3VNp+eNuBBtH0OXJTdth70niEXcqF8HS4MDECJNcBsw/XYRrlqb1Rr
Fa/qbgxj/c6JabFemnl38+Xlv//JnHnv53/9+rmd/hsrp4qxQqj0mz9LQ80fmpfJQoY4iqR/i0yL
+DI5W5RD6PEC58nc2FjW/DQ5re9mOH8Zy+STB/nfBvXV/AvwwdHLPARAxmCl5Mmi/1sPTDHNt4/I
2++WyBsfJG/HXZ2aI6vZ6JcT8iihSRRRbnI9Q01vQe59KCRxNzecSqe63qizWtmtreck+R93K9BS
IBUAAA==
headers:
accept:
- '*/*'
accept-encoding:
- gzip, deflate, zstd
connection:
- keep-alive
content-encoding:
- gzip
content-length:
- '2398'
content-type:
- application/x-protobuf
method: POST
uri: https://logfire-eu.pydantic.dev/v1/traces
response:
body:
string: ''
headers:
connection:
- keep-alive
content-length:
- '0'
nel:
- '{"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}'
report-to:
- '{"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=FU9znOmvL1vinIEsd7b2RUqQ01rUYjwmY39v06KsjlLZsINKwrK%2FSMQQGGJ5a0kdVF9sSY0cTX5cb3B9X7UHiqXfqhvzZ7xuELCcPThBVvNPLEm%2F"}]}'
vary:
- origin, access-control-request-method, access-control-request-headers
completion_tokens: 33
prompt_tokens: 545
total_tokens: 578
status:
code: 200
message: OK
@ -359,68 +225,6 @@ interactions:
status:
code: 200
message: OK
- request:
body: !!binary |
H4sIAJcwUGkC/61WS2wbRRiO1Qfp5NlNU1JVpcNWQBLi7dpexw/UQwlCwheqtBWgNFqNd8f2Nvti
Z5zUBB+L2kO59sClEqAiVAkuCCQkegSO9MBLqoS4wA0QnHiJf/bhrB2n6oHb7v+af/7/+/5/0G/H
0N19qIJmGA02LYMqlss4ccWHKckIF2g5R2iuomkVqqlqqZKr0PKyUS+UCw3VNApIRkc5talDedBR
mLmh2MRttkmTSqPooN/hLc9FTyCp38YlDpWm0ITnU7enQY+j2X6zTRowy3NFpJxSqCg5MBlPEg1j
HEZTbXfD9bZcPZajGTTmB55BGVN8uMP+uQ9u7wO/I4kwaLvccmL/Q+gRI07yFHp00CR1fkHJFZQS
uoiODxqZlBmB5XNhuIy0yBDPO8Ryl/B56uNcGefVfHEJ5/PVYr5a0Bbw2oqoEoiVnKLhdXQGTSXX
Ss5cRPPFsqaZaskomlqeqA2zCLVXy1oxX8ppZaOSV4mZKzVIXbpzFM3CrTsmgZSMLLGkqF6q9Md+
NJ15i5y5696bvnPg+l+nv1+5L41++9mvb8yu+7fk0e9ef3r56jsTzy+Ow3VcC3LinmermQp+85Pn
rs9fnTvbevfeS+Kj9iSablJXJ5YiTKLySWiaURIYLd30jLYDzWRgN5O2M4htCyxBt8WnztyOVy5z
UnsGTQoDnQTNyFNaQE9ty7blWFyuFpfkV9s06MhV+VzYH2zShuVaosxyt3Yajdles2EFVHFYU8Lo
sXT+Vbwrq08zaCbxuMw8gIvRog6R3s6gW5ltmXd8Klex7NUvU4PLS1iGNvs04BZlIAeDvlQjUb9P
F5xCq4Ay33MZ3ctosIrCblAeVy1Udbu14+hwkjzziauLqNJBtF/81K5l0ETfwdIWaq9plTJdpqaW
JcZyKauVivlsuVKhWa1IKqW6WSAqKa/jeWZ4Aa1iVVELC+gChK1iTq9wtOIBMV1eRXH1LYYJblnN
Vtamm9TGUJ5mQBxHlDxhvPLayNXMSGZkRPowMwx2Cdjk0Rt/+jn27O/fLE6k28YAd9foRxHu3jv9
dYS7k+hAqJSOLh5BwwB3sh8L02gyCZoLwz6wer2Uf54alvLtly/++/kvX96UR7/4Sfn74xurXy2O
Gy3CcdPnWY+JjP2bP4aJnk0+IKHZuJcCQkRANqILnCmca8fQRGzAOoxTRwwYz7aJQ2owqGJVQIEA
jCuOZ1JbDKr4yNoJNCmGBQ0UYprQcyaNoUO2B5BpeYzXJDQWq30v4NK+ufdfqf0zhubCOHocVfcJ
dI9yGDbSD2Po/ti23Gi7hkhVD6sNyFvblmN4yoM1DwnSi6CnCBWCHvIKiUrsc2kWNYjN6G5qxTxP
kYXxANoXkiUeB6AE/pO2Lb7dtm2DiridFxtRnomjBaBt0kB47giFudxd74pw4vqAA0GstfjgdcHb
3SxNjXWhOR9WAPMWxWLd2NRsUlwnjOKGF+AA1tYmDF/cq5Byyb3krlLeDlwGagaZM7xl8RY2WrCx
8AvPAaFcM/E0KA6ZKPxWaYMGVIgSv3pHHGwFka9umdhysWFxauqhhCmiIV4bmhGC29RNy+D6Bu3s
VEvU1OCpLjCBBNgXxE4JNyxX1KYHBhEXmkxMwslOLLH34LRY0BUVrLctm1sp8KxHGfltrgvkiaBC
J++I42L3osbiIfCDwU/C8Sb6/2Do7YmtgX5egEaGTIB/KC4weouIjrAtKuCRwkQKjJF6r3CRFkZO
iJIk+B6h0t0bFlDgw2tEDWe4zagpAgPWnDB6nAkEgjgO24s9iYgEAelE53puwzIFugZIlVPU3ahf
6VlH8AzBzltWUik8r2ZzCyFIyBXLaTu9QLAY4n81/N8ho1MXBN2DjDtFHkJLoXzAZBnC2rO9lpD4
SYDDc9JMDBK6/f8sihD90BwC7Dq+gEXCBOr4NuE0NfJs29vSxYKObVKHRjrLgUU8oOzWNoY/fi6g
1Yd4+uy1OIa+b+CxcmrYup1E4/DWNwGYevhoOdG/tEGdXqvwvjuWqBM3OBgKDcNPvDrLZoOU1EKu
VNDyjYJZqsm9nZu8gnYtzmTV/wfevCNH9AwAAA==
headers:
accept:
- '*/*'
accept-encoding:
- gzip, deflate, zstd
connection:
- keep-alive
content-encoding:
- gzip
content-length:
- '1567'
content-type:
- application/x-protobuf
method: POST
uri: https://logfire-eu.pydantic.dev/v1/traces
response:
body:
string: ''
headers:
connection:
- keep-alive
content-length:
- '0'
nel:
- '{"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}'
report-to:
- '{"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=trOojjczzFXOqFvhWkkr%2BrMvxMbEH3hz%2FkGJP3CvoEJw8auFEMv%2Fmzm8oMmxMcpedH4EdA%2BTkzVPk5ECNP%2ByhN6FjH2W2XXLHAn%2Bwh9lVJH7H8DG"}]}'
vary:
- origin, access-control-request-method, access-control-request-headers
status:
code: 200
message: OK
- request:
headers:
accept:
@ -430,7 +234,7 @@ interactions:
connection:
- keep-alive
content-length:
- '3030'
- '3071'
content-type:
- application/json
host:
@ -443,18 +247,18 @@ interactions:
Process:
1. Call search_documents with relevant keywords from the question
2. Review the results and their relevance scores
2. Review the results ordered by relevance
3. If needed, perform follow-up searches with different keywords (max 3 total)
4. Provide a concise answer based strictly on the retrieved content
The search tool returns results like:
[chunk_abc123] (score: 0.85)
[chunk_abc123] [rank 1 of 5]
Source: "Document Title" > Section > Subsection
Type: paragraph
Content:
The actual text content here...
[chunk_def456] (score: 0.72)
[chunk_def456] [rank 2 of 5]
Source: "Another Document"
Type: table
Content:
@ -462,7 +266,7 @@ interactions:
...
Each result includes:
- chunk_id in brackets and relevance score
- chunk_id in brackets and rank position (rank 1 = most relevant)
- Source: document title and section hierarchy (when available)
- Type: content type like paragraph, table, code, list_item (when available)
- Content: the actual text
@ -475,28 +279,28 @@ interactions:
- If multiple results are relevant, synthesize them coherently
- If information is insufficient, say: "I cannot find enough information in the knowledge base to answer this question."
- Be concise and direct - avoid elaboration unless asked
- Higher scores indicate more relevant results
- Results are ordered by relevance, with rank 1 being most relevant
role: system
- content: What is Python?
role: user
- content: |-
<think>
Need search.
Need to search.
</think>
role: assistant
tool_calls:
- function:
arguments: '{"limit":5,"query":"Python definition"}'
arguments: '{"query":"Python definition","limit":5}'
name: search_documents
id: call_snyo88ta
id: call_zzot5v63
type: function
- content: |-
[498e6ed4-ac67-4752-899e-45a97bd3a0a8] (score: 0.03)
[8c03f6b5-b9e5-47fa-b785-83f659ff9198] [rank 1 of 1]
Type: text
Content:
Python is a high-level programming language.
role: tool
tool_call_id: call_snyo88ta
tool_call_id: call_zzot5v63
model: gpt-oss
reasoning_effort: low
stream: false
@ -555,7 +359,7 @@ interactions:
response:
headers:
content-length:
- '422'
- '457'
content-type:
- application/json
parsed_body:
@ -563,94 +367,18 @@ interactions:
- finish_reason: stop
index: 0
message:
content: I cannot find enough information in the knowledge base to answer this question.
reasoning: Only result very low. Probably insufficient.
content: '{"answer":"Python is a highlevel programming language.","cited_chunks":["8c03f6b5-b9e5-47fa-b785-83f659ff9198"],"confidence":0.93,"query":"What
is Python?"}'
role: assistant
created: 1766863000
id: chatcmpl-320
created: 1768999000
id: chatcmpl-740
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 32
prompt_tokens: 628
total_tokens: 660
status:
code: 200
message: OK
- request:
body: !!binary |
H4sIAJgwUGkC/+1YTYwcRxX2xkvi1Cax08ZhnQgo2kFaWzud+fXMLATkbBB4DsSyHazIsxrVdFfP
VLa7uumq3vVks8cccsiFCxIoEuICCogcuSD5ygEOcEEgIlAuQeIIEgiC4L3q7pme8awxkAOHvYx6
ql69evXe9733qsgHbXL3JOmSs4one8LljpBKM4kfnmUT2uCdGuO1brPZ5c1qtd2tdXnnsjtsdBp+
1XMbxCZPaR7wkOtk4ihv1wmYHKVsxK1T5OF4oseRJJ8l1ryMZCG3TpPHo5jL6Qz5DDk3L7bHEyUi
iZpqTqPr1EDkscJQo+NJcjqVuzLal4N8nJwla3ESuVwpJ4YzrK6/+72TsO7jxWCSSi3CfP2j5BE3
N/IC+cSiSGn/hlNrOG3yMnlmUcjjyk1ErFHwMmlmgnQjZEJu0hs8prUOrVfrrU1ar2+16luN5kV6
exu9BMNOzWnSHfI8OV0cq9jzEtlodZpNr9p2W16zzqq+1wLfVzvNVr1da3bcbr3KvFrbZ0Pru01y
Dk498RiY5FaYsDJ/Va23nyVnVr7Nnr8rf3nmhx978+/P/Wb7PevUTz9wPvzxW9d/YZ96629xTb3w
p19deswdM01Hsa5ESlVXuvE337/15sYb61fuvP/57+BH79Pk3IjLARMORC1heN7Mhw+TVVzcO08e
zwXURGkeotuiIGAh64H786mEfz3lSjth5PEA3Z9v2fskeQJdwBOHeV4CDrbWyKNB5LJgHCnds8ha
Ph1HibZOrr/zSu8fa2Td6BnkWgcxS8AiDS60fr9G3ls7sP1UumjqQEdRoOwtevvARqvhy1acJe54
4EVuGgIOlb1J7ZmGwasKlil3zEMG0gc22CVQFQuuJegCLTgq9FmgOK4sDx7YYFAyMV96Emfb6UTI
kX0IsoEIhTaTHvdZGuC3TIMAppicvORndhYLBXBkxBNcORtEcftw5xDV4fFFwj1clm+8A8OFaDR8
lbvabFwCK87cMB6geswpkijg3ojTIVOc+lFCEyDjHkCKTj3k9GVfXuc6TaSCaQWWK7ov9Ji6Y+Ah
vfqiokx6xUqXU+VGIOfgKp8nHIeKdcMJbiySbO1AeFRI6grNvYEZUQ4GJEohGAM8ijfwhKsHu3wy
8xb61NWlKChEArCABaXBXSHRN1MwoF4IMvOYZjNdyGbYLR84RA8OUxFoUQLPTmZRnOoBIg+V4pw9
G86dPdWaDy+Bny8ASoPMG/8GekdiayGeNyGQhgnwH5wLjN5nGBG1zxEeJUyUwJhNH6Uum6U6Migp
lB+hqhy9ZQoRH5GfBVzRVHEPFQPWQqM9twQUgZ5QHcWeYoglCZtk+0bSFx6ia4FUNad6L+q3p9IZ
PA3Y9VgUnqIb1UrtogEJuyPCNJwqCoXM/1fN/xkZwyES9Agyzpy8hJY4eZ/MsoS1V6YhYTTLYdTs
U2ZiUtDto2dRhugH5hBgN4wRFgUTeBgHTPNSyguCaB/G7+hcprRpNidCaCgWJg97z5Ang2jkg7Md
FTNpjofVCP9APVkrJkM1sp4gcxWu9+vVaTkTErQ6IZQc2ERZd1fJT1aBpkkUmDABSpOcnlrNJ2Y0
2c7gp8FdOHQLNwEkXTNNxRcxQ2PWLpQxpQT2V/oojWMhdxHn81q/yoEpWaid+TKAeWUARdKkIGHi
g/8GSk6iTgdCA8P3q3csGeV/QOCgn1Wmvr3V2uxn4IXvvp2dhgKvgAAIw759aJvo4oAaQxZjKgPn
zKC5g9/Pi8UKzIVxJBX/L8+Sp1KYvd3sdvhl7jUrzL3crjTbrXql0+3ySrPFuu2h12BV1tmhG4b+
hsyNi315EwzaohjUvtzOnL/Vl/nZMTvQsRiNK1DaeEAB1iPI15ARRrRoeR0M907vWw+Rp4pWyUB2
Bq5/rpAPV27/z3h4SQaTvI7SPWQ+kMShkDyGbAgz0MOnvi9cAeKLeLkXslepy6SMNIVYepTLKB2N
QQVmZdPlYVle0iFg+skTEabOojI4R+BC6SiGqd7P18jTc23iAG8cSWoKs7J+tEZ+sHZ/ir0SpZRB
1mYzi+DU8L/wZVb5MuNmhmG9wWCxaTuzcCLT3FzLunuIe82h24A+uoi0LMtOeyNIpPtR4inqQ56b
q5F9WXfodb4n+L4ZL/oebJCyxmexTerLhkOv+lQC3bm3SaEUmNroR5gFK2mcG8NzIzzhmyxfsgLu
HXdoA4KjWQCQbhpU7EG1g3ODD12hpiUdzwxZxeR+AE0kcythABDu0dzl6BVsBPJKg2xFobkWMBC7
HFx2O+vk2NCt1RtlenVaYMuNKE1c+Nu3XywCcFPogPdt+gW4KBkA4Fc6VNmfgpHYGAHX4nGZlqY3
cXXKAsPYwlo6Boc4jollbg5krWbrctmcdn3OnCsAflhGC7P69jQXILDKu75Ot6MgDSWt0elnnb7e
l/mWX2LuuCCmkG6QQumGZZW5HneYMHeX66WtMsoWlk1xqtFNRjr3DCQiuIJBOCZ0Y3/MJWV7TARo
7EVUkBlfeASpZCI0c+RmdrJNkPHgNwDeDLDlWqqtOH3Wos1cjue9KukErKVF5t4sTm2EZ5cCEMra
vcUWH5V8OQV8BkLmrnqBTTGq5uB5DzRpBXxEMXWBbgom8QR6qBmtUdvLKrMlc6pxIrqHFr0L5rFU
erCXxjmjGc9WQSKGEEcRByXuJnxK/U2qJhJUK/Ga2SKExWNDx2CSr5/LomouL8NqNkH0fcTpF8EL
PuQltnuQKBIADriL7UWAQQ7BjbIrPJw9gHwHyXOXe7jyK1DiQGmWkGBv6BahWaNhVDp54Q7M539Z
IWeLRqt0bbF+t0J+u3KwpNlduMssbcLmOv9Zm7+8qN5feEmROWLBUU8Jc+LFXfrwsEfJ+XyPFO3I
jgAXvV0Odezk+p9XQeLpOYnpXdCIPLROe/a0Cy0YdO/LyLPEWpQRXtHQutBMVxr1am+TfGpRaq4G
K4tcOkWwO4ZC/NqJN1ZOrJw4Yf3h9LL3ofo73uce+v4fiX3qG+/+7O3xLX7hnvehvz7yNfMsdKX4
OH4fOn4fOn4fOn4fOn4fOn4f+j95H9pd3pncJNcfoC/5T7uBC8seo6BIxxxaKDkaPMij1HPkfDFd
LIONwdGQ/CyLnOl4PmtXG7V2o1n3G177QdqHotT/C1v+K5XqGwAA
headers:
accept:
- '*/*'
accept-encoding:
- gzip, deflate, zstd
connection:
- keep-alive
content-encoding:
- gzip
content-length:
- '2376'
content-type:
- application/x-protobuf
method: POST
uri: https://logfire-eu.pydantic.dev/v1/traces
response:
body:
string: ''
headers:
connection:
- keep-alive
content-length:
- '0'
nel:
- '{"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}'
report-to:
- '{"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=u%2BdL5dvWixKrvXzzRkOqXalpRKpO2vLrtO1mg0NftN4VQ%2FhXvZ7%2B7v3RsAUrwkhx4rTftKfylbnqzzqNAsifN%2BkXJmwDtblzab4q25CJdU1VVTtU"}]}'
vary:
- origin, access-control-request-method, access-control-request-headers
completion_tokens: 62
prompt_tokens: 642
total_tokens: 704
status:
code: 200
message: OK
@ -663,7 +391,7 @@ interactions:
connection:
- keep-alive
content-length:
- '3338'
- '3406'
content-type:
- application/json
host:
@ -676,18 +404,18 @@ interactions:
Process:
1. Call search_documents with relevant keywords from the question
2. Review the results and their relevance scores
2. Review the results ordered by relevance
3. If needed, perform follow-up searches with different keywords (max 3 total)
4. Provide a concise answer based strictly on the retrieved content
The search tool returns results like:
[chunk_abc123] (score: 0.85)
[chunk_abc123] [rank 1 of 5]
Source: "Document Title" > Section > Subsection
Type: paragraph
Content:
The actual text content here...
[chunk_def456] (score: 0.72)
[chunk_def456] [rank 2 of 5]
Source: "Another Document"
Type: table
Content:
@ -695,7 +423,7 @@ interactions:
...
Each result includes:
- chunk_id in brackets and relevance score
- chunk_id in brackets and rank position (rank 1 = most relevant)
- Source: document title and section hierarchy (when available)
- Type: content type like paragraph, table, code, list_item (when available)
- Content: the actual text
@ -708,34 +436,30 @@ interactions:
- If multiple results are relevant, synthesize them coherently
- If information is insufficient, say: "I cannot find enough information in the knowledge base to answer this question."
- Be concise and direct - avoid elaboration unless asked
- Higher scores indicate more relevant results
- Results are ordered by relevance, with rank 1 being most relevant
role: system
- content: What is Python?
role: user
- content: |-
<think>
Need search.
Need to search.
</think>
role: assistant
tool_calls:
- function:
arguments: '{"limit":5,"query":"Python definition"}'
arguments: '{"query":"Python definition","limit":5}'
name: search_documents
id: call_snyo88ta
id: call_zzot5v63
type: function
- content: |-
[498e6ed4-ac67-4752-899e-45a97bd3a0a8] (score: 0.03)
[8c03f6b5-b9e5-47fa-b785-83f659ff9198] [rank 1 of 1]
Type: text
Content:
Python is a high-level programming language.
role: tool
tool_call_id: call_snyo88ta
- content: |-
<think>
Only result very low. Probably insufficient.
</think>
I cannot find enough information in the knowledge base to answer this question.
tool_call_id: call_zzot5v63
- content: '{"answer":"Python is a highlevel programming language.","cited_chunks":["8c03f6b5-b9e5-47fa-b785-83f659ff9198"],"confidence":0.93,"query":"What
is Python?"}'
role: assistant
- content: |-
Validation feedback:
@ -801,7 +525,7 @@ interactions:
response:
headers:
content-length:
- '609'
- '628'
content-type:
- application/json
parsed_body:
@ -810,25 +534,25 @@ interactions:
index: 0
message:
content: ''
reasoning: Need to use final_result.
reasoning: We need final result via function.
role: assistant
tool_calls:
- function:
arguments: '{"answer":"I cannot find enough information in the knowledge base to answer this question.","cited_chunks":[],"confidence":0,"query":"What
arguments: '{"answer":"Python is a highlevel programming language.","cited_chunks":["8c03f6b5-b9e5-47fa-b785-83f659ff9198"],"confidence":0.93,"query":"What
is Python?"}'
name: final_result
id: call_l8rd7d7f
id: call_sn6do4tl
index: 0
type: function
created: 1766863002
id: chatcmpl-578
created: 1768999002
id: chatcmpl-226
model: gpt-oss
object: chat.completion
system_fingerprint: fp_ollama
usage:
completion_tokens: 60
prompt_tokens: 685
total_tokens: 745
completion_tokens: 80
prompt_tokens: 725
total_tokens: 805
status:
code: 200
message: OK

View file

@ -39,53 +39,6 @@ interactions:
status:
code: 200
message: OK
- request:
body: !!binary |
H4sIAKIwUGkC/82SS0/UUBTHOwwwwxFkpjAKPmLBR4BA6WuYGSLJBFiZ6EJjYjQGL+2dmYbpbe1t
MRNjwgYTfHwCWbDThTziF8BPoPGRaExcuXLlwgVEE2Pb6SAP48Sw0LO4Ob33d+495/8vrLTCehRy
0EGxPaurmNcJdRDxE43tBU7GWRFhMacoOawIQiYn5nB2RJ2Ws3JB0FQZeuGQg8vYwI5d4ak2w5cR
KbqoiNk4NFsVp2QSOA3sToYgA7Pt0GZamGydQA+kdmKz2Ka6SfybRF7O8aKHtNYaDe5IQrtLZoh5
i0yF+9ABByzbVDGlvOXN0Ni1+jjq1XXWNm2XOLoR1rdATA2bPAmHdyPb3pd5UeYzcBmO7oY0TFVb
txwfHAGlCnJ9BtLJIHcJW5yY5SRBSg9ykjSalkZlpZ+7NuGr5G3zIq9w12EM2mtj1d4cgL50VlE0
IaOmNUVCQkFLe9oLWSUtZUQlq+YkAWlipoCm2c9xSHlTVzTktaQOIZ2t6iWwL+PQXcRkCum8WtY9
sXnHnMGEd6nv0PB5jKhrY8oR15jGNmcWOJ1YrsMhonGm6/hpUEA5l2LtSOx28HXn4kYMNqLQDW3h
5bRCHWz4QpnlMjIQnIBUeOR5bCNfnarizdColpDjOxICNr7pYurwhqnhsu9I0XKGTEq9Xyu1hVDL
JBTvZY5BMmSqgzkVC7MxaArGSD5/szq50Dff1b05+eLTfS/piTLV6A/Wt4t5ITF+NxJfPMXORSLM
PiKyz3MvzkXZCHOVT974shx03Rt/+M0S6fjXdwOJyCM0tk5eJ542LXwf/jDxUexsqBZ5PIPyS3V5
5xffRZfP3KvHL4c8CtalubweJKX5PGz+F84f/53zfhvBb5t88mrtj9YzU3nhoG/9jxbP+gbmH0Rk
r/WlZ2v1reeYLSvfi+NLf8NfebBS3/qzzHbrmXxoPXMhn4j8BD4ukZMxBgAA
headers:
accept:
- '*/*'
accept-encoding:
- gzip, deflate, zstd
connection:
- keep-alive
content-encoding:
- gzip
content-length:
- '729'
content-type:
- application/x-protobuf
method: POST
uri: https://logfire-eu.pydantic.dev/v1/metrics
response:
body:
string: "\n\0"
headers:
connection:
- keep-alive
content-length:
- '2'
nel:
- '{"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}'
report-to:
- '{"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=JzTAsIp8t3A57BJQl4mlG%2FfLA7%2BWUFJ9IsKcAJIIEeU%2BJ%2F%2BW4kihd7xaQIjse3baoJ79MeWUTKIFcB1xbCLrof%2B5yXY1xtklBwF4XA%2BVTc1MFqvW"}]}'
vary:
- origin, access-control-request-method, access-control-request-headers
status:
code: 200
message: OK
- request:
headers:
accept:

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -1,3 +1,5 @@
from pathlib import Path
import pytest
from haiku.rag.client import HaikuRAG
@ -8,6 +10,11 @@ from haiku.rag.store.repositories.document import DocumentRepository
from haiku.rag.store.repositories.settings import SettingsRepository
@pytest.fixture(scope="module")
def vcr_cassette_dir():
return str(Path(__file__).parent.parent / "cassettes" / "test_read_only")
class TestReadOnlyError:
def test_read_only_error_is_exception(self):
"""ReadOnlyError should be a subclass of Exception."""

View file

@ -254,8 +254,50 @@ def test_chunk_metadata_resolve_empty_refs():
assert doc_items == []
def test_search_result_format_for_agent_full():
"""Test format_for_agent with all metadata present."""
def test_search_result_format_for_agent_with_rank():
"""Test format_for_agent with rank and total parameters."""
result = SearchResult(
content="This is the chunk content about elections.",
score=0.02, # Low RRF score that would confuse agents
chunk_id="chunk-123",
document_id="doc-456",
document_uri="file:///docs/report.pdf",
document_title="Annual Report 2024",
headings=["Chapter 1", "Section 1.1", "Elections"],
labels=["paragraph", "table"],
page_numbers=[1, 2],
)
formatted = result.format_for_agent(rank=1, total=5)
assert "[chunk-123]" in formatted
assert "[rank 1 of 5]" in formatted
assert "score:" not in formatted # Score should NOT appear when rank is provided
assert (
'Source: "Annual Report 2024" > Chapter 1 > Section 1.1 > Elections'
in formatted
)
assert "Type: table" in formatted
assert "Content:\nThis is the chunk content about elections." in formatted
def test_search_result_format_for_agent_rank_only():
"""Test format_for_agent with rank but no total."""
result = SearchResult(
content="Some content.",
score=0.03,
chunk_id="chunk-abc",
)
formatted = result.format_for_agent(rank=2)
assert "[chunk-abc]" in formatted
assert "[rank 2]" in formatted
assert "score:" not in formatted
def test_search_result_format_for_agent_fallback():
"""Test format_for_agent falls back to score when no rank provided."""
result = SearchResult(
content="This is the chunk content about elections.",
score=0.85,

View file

@ -298,10 +298,11 @@ async def test_format_for_agent_output(temp_db_path, small_chunk_config):
assert len(table_results) > 0
expanded = await client.expand_context(table_results[:1])
formatted = expanded[0].format_for_agent()
# Format with rank (the way agents use it)
formatted = expanded[0].format_for_agent(rank=1, total=1)
# Check format structure
assert "score:" in formatted
assert "[rank 1 of 1]" in formatted
assert 'Source: "Format Test"' in formatted
assert "Type: table" in formatted
assert "Content:" in formatted

View file

@ -258,11 +258,12 @@ async def test_search_result_format_includes_metadata(temp_db_path):
results = await client.search("machine learning", limit=1)
assert len(results) > 0
formatted = results[0].format_for_agent()
# Format with rank (the way agents use it)
formatted = results[0].format_for_agent(rank=1, total=1)
# Should include chunk ID and score
# Should include chunk ID and rank
assert "[" in formatted and "]" in formatted
assert "score:" in formatted
assert "[rank 1 of 1]" in formatted
# Should include document title in Source
assert "ML Guide" in formatted