Merge pull request #252 from ggozad/fix/score-use

Search results now show rank position instead of raw scores.
2026-01-21 15:28:43 +02:00 · 2026-01-21 15:28:43 +02:00 · 4bdfb180d5
commit 4bdfb180d5
parent dff313fa74 b96e5a3be1
40 changed files with 11266 additions and 16426 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -5,6 +5,19 @@
 - **Model Downloads**: `download-models` now pre-downloads HuggingFace models for `sentence-transformers`, `mxbai`, and `jina-local`
 - **Reranker Factory**: Removed unreliable `id(config)`-based caching from `get_reranker()`; factory now always instantiates fresh

+### Changed
+
+- **Agent Search Result Display**: Search results now show rank position instead of raw scores
+  - `SearchResult.format_for_agent()` accepts optional `rank` and `total` parameters
+  - Output changes from `(score: 0.02)` to `[rank 1 of 5]` when rank is provided
+  - Prevents LLMs from misinterpreting low RRF hybrid search scores as "2% relevant"
+  - QA and Research agents updated to pass rank/total to formatted results
+  - Agent prompts updated to reference rank-based ordering instead of scores
+
+### Fixed
+
+- **Test Cassette Organization**: Consolidated all VCR cassettes to `tests/cassettes/`
+
 ## [0.26.7] - 2026-01-20

 ### Added
--- a/docs/configuration/prompts.md
+++ b/docs/configuration/prompts.md
@ -57,7 +57,7 @@ prompts:

    Process:
    1. Search for relevant documents using the search_documents tool
-    2. Review results and their relevance scores
+    2. Review results ordered by relevance (rank 1 = most relevant)
    3. Provide a brief, direct answer based on retrieved content

    Guidelines:
--- a/evaluations/evaluations/prompts.py
+++ b/evaluations/evaluations/prompts.py
@ -4,7 +4,7 @@ You are a WIX technical support expert helping users with questions about the WI
 Your process:
 1. When a user asks a question, use the search_documents tool to find relevant information
 2. Search with specific keywords and phrases from the user's question
-3. Review the search results and their relevance scores
+3. Review the search results ordered by relevance (rank 1 = most relevant)
 4. If you need additional context, perform follow-up searches with different keywords
 5. Provide a short and to the point comprehensive answer based only on the retrieved documents

--- a/haiku_rag_slim/haiku/rag/agents/qa/agent.py
+++ b/haiku_rag_slim/haiku/rag/agents/qa/agent.py
@ -48,7 +48,7 @@ class QuestionAnswerAgent:
        ) -> str:
            """Search the knowledge base for relevant documents.

-            Returns results with chunk IDs and relevance scores.
+            Returns results with chunk IDs and rank positions.
            Reference results by their chunk_id in cited_chunks.
            """
            results = await ctx.deps.client.search(
@ -57,8 +57,12 @@ class QuestionAnswerAgent:
            results = await ctx.deps.client.expand_context(results)
            # Store results for citation resolution
            ctx.deps.search_results = results
-            # Format with metadata for agent context
-            parts = [r.format_for_agent() for r in results]
+            # Format with rank instead of raw score to avoid confusing LLMs
+            total = len(results)
+            parts = [
+                r.format_for_agent(rank=i + 1, total=total)
+                for i, r in enumerate(results)
+            ]
            return "\n\n".join(parts) if parts else "No results found."

    async def answer(
--- a/haiku_rag_slim/haiku/rag/agents/qa/prompts.py
+++ b/haiku_rag_slim/haiku/rag/agents/qa/prompts.py
@ -2,18 +2,18 @@ QA_SYSTEM_PROMPT = """You are a knowledgeable assistant that answers questions u

 Process:
 1. Call search_documents with relevant keywords from the question
-2. Review the results and their relevance scores
+2. Review the results ordered by relevance
 3. If needed, perform follow-up searches with different keywords (max 3 total)
 4. Provide a concise answer based strictly on the retrieved content

 The search tool returns results like:
-[chunk_abc123] (score: 0.85)
+[chunk_abc123] [rank 1 of 5]
 Source: "Document Title" > Section > Subsection
 Type: paragraph
 Content:
 The actual text content here...

-[chunk_def456] (score: 0.72)
+[chunk_def456] [rank 2 of 5]
 Source: "Another Document"
 Type: table
 Content:
@ -21,7 +21,7 @@ Content:
 ...

 Each result includes:
- chunk_id in brackets and relevance score
+- chunk_id in brackets and rank position (rank 1 = most relevant)
 - Source: document title and section hierarchy (when available)
 - Type: content type like paragraph, table, code, list_item (when available)
 - Content: the actual text
@ -34,5 +34,5 @@ Guidelines:
 - If multiple results are relevant, synthesize them coherently
 - If information is insufficient, say: "I cannot find enough information in the knowledge base to answer this question."
 - Be concise and direct - avoid elaboration unless asked
- Higher scores indicate more relevant results
+- Results are ordered by relevance, with rank 1 being most relevant
 """
--- a/haiku_rag_slim/haiku/rag/agents/research/graph.py
+++ b/haiku_rag_slim/haiku/rag/agents/research/graph.py
@ -191,7 +191,12 @@ async def _search_one_step_logic(
            )
            results = await ctx2.deps.client.expand_context(results)
            ctx2.deps.search_results = results
-            parts = [r.format_for_agent() for r in results]
+            # Format with rank instead of raw score to avoid confusing LLMs
+            total = len(results)
+            parts = [
+                r.format_for_agent(rank=i + 1, total=total)
+                for i, r in enumerate(results)
+            ]
            if not parts:
                return f"No relevant information found for: {query}"
            return "\n\n".join(parts)
--- a/haiku_rag_slim/haiku/rag/agents/research/prompts.py
+++ b/haiku_rag_slim/haiku/rag/agents/research/prompts.py
@ -49,18 +49,18 @@ SEARCH_PROMPT = """You are a search and question-answering specialist.

 Process:
 1. Call search_and_answer with relevant keywords from the question.
-2. Review the results and their relevance scores.
+2. Review the results ordered by relevance.
 3. If needed, perform follow-up searches with different keywords (max 3 total).
 4. Provide a concise answer based strictly on the retrieved content.

 The search tool returns results like:
-[9bde5847-44c9-400a-8997-0e6b65babf92] (score: 0.85)
+[9bde5847-44c9-400a-8997-0e6b65babf92] [rank 1 of 5]
 Source: "Document Title" > Section > Subsection
 Type: paragraph
 Content:
 The actual text content here...

-[d5a63c82-cb40-439f-9b2e-de7d177829b7] (score: 0.72)
+[d5a63c82-cb40-439f-9b2e-de7d177829b7] [rank 2 of 5]
 Source: "Another Document"
 Type: table
 Content:
@ -68,7 +68,7 @@ Content:
 ...

 Each result includes:
- chunk_id in brackets and relevance score
+- chunk_id in brackets and rank position (rank 1 = most relevant)
 - Source: document title and section hierarchy (when available)
 - Type: content type like paragraph, table, code, list_item (when available)
 - Content: the actual text
@ -87,7 +87,7 @@ Guidelines:
 - If multiple results are relevant, synthesize them coherently.
 - If information is insufficient, say so clearly.
 - Be concise and direct; avoid meta commentary about the process.
- Higher scores indicate more relevant results."""
+- Results are ordered by relevance, with rank 1 being most relevant."""

 DECISION_PROMPT = """You are the research evaluator responsible for assessing
 whether gathered evidence sufficiently answers the research question.
--- a/haiku_rag_slim/haiku/rag/store/models/chunk.py
+++ b/haiku_rag_slim/haiku/rag/store/models/chunk.py
@ -143,13 +143,25 @@ class SearchResult(BaseModel):
            labels=meta.labels,
        )

-    def format_for_agent(self) -> str:
+    def format_for_agent(
+        self, rank: int | None = None, total: int | None = None
+    ) -> str:
        """Format this search result for inclusion in agent context.

+        Args:
+            rank: 1-based position in results (1 = most relevant)
+            total: Total number of results returned
+
        Produces a structured format with metadata that helps LLMs understand
-        the source and nature of the content.
+        the source and nature of the content. When rank is provided, shows
+        position instead of raw score to avoid confusing LLMs with low RRF scores.
        """
-        parts = [f"[{self.chunk_id}] (score: {self.score:.2f})"]
+        if rank is not None and total is not None:
+            parts = [f"[{self.chunk_id}] [rank {rank} of {total}]"]
+        elif rank is not None:
+            parts = [f"[{self.chunk_id}] [rank {rank}]"]
+        else:
+            parts = [f"[{self.chunk_id}] (score: {self.score:.2f})"]

        # Document source info
        source_parts = []
--- a/tests/agents/research/cassettes/test_research_graph/test_graph_end_to_end.yaml
+++ b/tests/agents/research/cassettes/test_research_graph/test_graph_end_to_end.yaml
--- a/tests/agents/research/cassettes/test_search_filter/test_research_graph_uses_search_filter.yaml
+++ b/tests/agents/research/cassettes/test_search_filter/test_research_graph_uses_search_filter.yaml
--- a/tests/agents/research/test_search_filter.py
+++ b/tests/agents/research/test_search_filter.py
@ -1,3 +1,5 @@
+from pathlib import Path
+
 import pytest

 from haiku.rag.agents.research.dependencies import ResearchContext
@ -6,6 +8,11 @@ from haiku.rag.agents.research.state import ResearchDeps, ResearchState
 from haiku.rag.client import HaikuRAG


+@pytest.fixture(scope="module")
+def vcr_cassette_dir():
+    return str(Path(__file__).parent.parent.parent / "cassettes" / "test_search_filter")
+
+
@pytest.fixture
 async def client_with_docs(temp_db_path):
    """Create a client with two distinct documents."""
--- a/tests/cassettes/test_chat_agent/test_chat_agent_ask_adds_citations.yaml
+++ b/tests/cassettes/test_chat_agent/test_chat_agent_ask_adds_citations.yaml
--- a/tests/cassettes/test_chat_agent/test_chat_agent_ask_with_state_key.yaml
+++ b/tests/cassettes/test_chat_agent/test_chat_agent_ask_with_state_key.yaml
--- a/tests/cassettes/test_chat_agent/test_chat_agent_get_document_not_found.yaml
+++ b/tests/cassettes/test_chat_agent/test_chat_agent_get_document_not_found.yaml
@ -126,7 +126,178 @@ interactions:
  response:
    headers:
      content-length:
-      - '585'
+      - '559'
+      content-type:
+      - application/json
+    parsed_body:
+      choices:
+      - finish_reason: tool_calls
+        index: 0
+        message:
+          content: ''
+          reasoning: User wants nonexistent document. We can use get_document but it may not exist. We'll try get_document.
+          role: assistant
+          tool_calls:
+          - function:
+              arguments: '{"query":"nonexistent document"}'
+              name: get_document
+            id: call_31uy8050
+            index: 0
+            type: function
+      created: 1768998264
+      id: chatcmpl-114
+      model: gpt-oss
+      object: chat.completion
+      system_fingerprint: fp_ollama
+      usage:
+        completion_tokens: 47
+        prompt_tokens: 842
+        total_tokens: 889
+    status:
+      code: 200
+      message: OK
+- request:
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate, zstd
+      connection:
+      - keep-alive
+      content-length:
+      - '4470'
+      content-type:
+      - application/json
+      host:
+      - localhost:11434
+    method: POST
+    parsed_body:
+      messages:
+      - content: |-
+          You are a helpful research assistant powered by haiku.rag, a knowledge base system.
+
+          You have access to a knowledge base of documents. Use your tools to search and answer questions.
+
+          CRITICAL RULES:
+          1. For greetings or casual chat: respond directly WITHOUT using any tools
+          2. For questions: Use the "ask" tool EXACTLY ONCE - it handles query expansion internally
+          3. For searches: Use the "search" tool EXACTLY ONCE - it handles multi-query expansion internally
+          4. NEVER call the same tool multiple times for a single user message
+          5. NEVER make up information - always use tools to get facts from the knowledge base
+
+          How to decide which tool to use:
+          - "get_document" - Use when the user references a SPECIFIC document by name, title, or URI (e.g., "summarize document X", "get the paper about Y", "fetch 2412.00566"). Retrieves the full document content.
+          - "ask" - Use for general questions about topics in the knowledge base when no specific document is named. It searches across all documents and returns answers with citations.
+          - "search" - Use when the user explicitly asks to search/find/explore documents. Call it ONCE. After calling search, copy the ENTIRE tool response to your output INCLUDING the content snippets. Do NOT shorten, summarize, or omit any part of the results.
+
+          IMPORTANT - When user mentions a document in search/ask:
+          - If user says "search in <doc>", "find in <doc>", "answer from <doc>", or "<topic> in <doc>":
+            - Extract the TOPIC as `query`/`question`
+            - Extract the DOCUMENT NAME as `document_name`
+          - Examples for search:
+            - "search for embeddings in the ML paper" → query="embeddings", document_name="ML paper"
+            - "find transformer architecture in 2412.00566" → query="transformer architecture", document_name="2412.00566"
+          - Examples for ask:
+            - "what does the ML paper say about embeddings?" → question="what are the embedding methods?", document_name="ML paper"
+            - "answer from 2412.00566 about model training" → question="how is the model trained?", document_name="2412.00566"
+
+          Be friendly and conversational. When you use the "ask" tool, summarize the key findings for the user.
+        role: system
+      - content: Get me the nonexistent document
+        role: user
+      - content: |-
+          <think>
+          User wants nonexistent document. We can use get_document but it may not exist. We'll try get_document.
+          </think>
+        role: assistant
+        tool_calls:
+        - function:
+            arguments: '{"query":"nonexistent document"}'
+            name: get_document
+          id: call_31uy8050
+          type: function
+      - content: 'Document not found: nonexistent document'
+        role: tool
+        tool_call_id: call_31uy8050
+      model: gpt-oss
+      reasoning_effort: low
+      stream: false
+      tool_choice: auto
+      tools:
+      - function:
+          description: |-
+            Search the knowledge base for relevant documents.
+
+            Use this when you need to find documents or explore the knowledge base.
+            Results are displayed to the user - just list the titles found.
+          name: search
+          parameters:
+            additionalProperties: false
+            properties:
+              document_name:
+                anyOf:
+                - type: string
+                - type: 'null'
+                default: null
+                description: Optional document name/title to search within
+              limit:
+                anyOf:
+                - type: integer
+                - type: 'null'
+                default: null
+                description: 'Number of results to return (default: 5)'
+              query:
+                description: The search query (what to search for)
+                type: string
+            required:
+            - query
+            type: object
+        type: function
+      - function:
+          description: |-
+            Answer a specific question using the knowledge base.
+
+            Use this for direct questions that need a focused answer with citations.
+            Uses a research graph for planning, searching, and synthesis.
+          name: ask
+          parameters:
+            additionalProperties: false
+            properties:
+              document_name:
+                anyOf:
+                - type: string
+                - type: 'null'
+                default: null
+                description: Optional document name/title to search within (e.g., "tbmed593", "army manual")
+              question:
+                description: The question to answer
+                type: string
+            required:
+            - question
+            type: object
+        type: function
+      - function:
+          description: |-
+            Retrieve a specific document by title or URI.
+
+            Use this when the user wants to fetch/get/retrieve a specific document.
+          name: get_document
+          parameters:
+            additionalProperties: false
+            properties:
+              query:
+                description: The document title or URI to look up
+                type: string
+            required:
+            - query
+            type: object
+          strict: true
+        type: function
+    uri: http://localhost:11434/v1/chat/completions
+  response:
+    headers:
+      content-length:
+      - '459'
      content-type:
      - application/json
    parsed_body:
@ -134,20 +305,18 @@ interactions:
      - finish_reason: stop
        index: 0
        message:
-          content: I’m sorry—I couldn’t find a document matching that name in the knowledge base. If you have any other request
-            or need help locating a different resource, just let me know!
-          reasoning: User asks for nonexistent document. Use get_document? but tool should not fabricate. We can explain not
-            found.
+          content: I’m sorry, but I couldn’t find a document titled “nonexistent document.” If you have another title or some
+            details to share, let me know and I’ll look it up for you!
          role: assistant
-      created: 1768225927
-      id: chatcmpl-215
+      created: 1768998265
+      id: chatcmpl-968
      model: gpt-oss
      object: chat.completion
      system_fingerprint: fp_ollama
      usage:
-        completion_tokens: 68
-        prompt_tokens: 842
-        total_tokens: 910
+        completion_tokens: 44
+        prompt_tokens: 912
+        total_tokens: 956
    status:
      code: 200
      message: OK
--- a/tests/cassettes/test_chat_agent/test_chat_agent_get_document_tool.yaml
+++ b/tests/cassettes/test_chat_agent/test_chat_agent_get_document_tool.yaml
@ -195,11 +195,11 @@ interactions:
          - function:
              arguments: '{"query":"DocLayNet Class Labels"}'
              name: get_document
-            id: call_geowshoi
+            id: call_m5izt73j
            index: 0
            type: function
-      created: 1768225921
-      id: chatcmpl-162
+      created: 1768998258
+      id: chatcmpl-390
      model: gpt-oss
      object: chat.completion
      system_fingerprint: fp_ollama
@ -268,14 +268,14 @@ interactions:
        - function:
            arguments: '{"query":"DocLayNet Class Labels"}'
            name: get_document
-          id: call_geowshoi
+          id: call_m5izt73j
          type: function
      - content: |-
          **DocLayNet Class Labels**

-          - ID: 50eeb7f5-49be-47d4-b1bd-19d221da9794
+          - ID: cd0fc5b8-ec16-4f1c-99fe-20979ecaab1c
          - URI: doclaynet-labels
-          - Created: 2026-01-12 15:52
+          - Created: 2026-01-21 14:24

          **Content:**
          DocLayNet Dataset - Class Labels
@ -296,7 +296,7 @@ interactions:

          The Text class has the highest count with 510,377 instances in the dataset.
        role: tool
-        tool_call_id: call_geowshoi
+        tool_call_id: call_m5izt73j
      model: gpt-oss
      reasoning_effort: low
      stream: false
@ -375,7 +375,7 @@ interactions:
  response:
    headers:
      content-length:
-      - '1105'
+      - '1131'
      content-type:
      - application/json
    parsed_body:
@ -383,23 +383,24 @@ interactions:
      - finish_reason: stop
        index: 0
        message:
-          content: "Here’s the document you requested:\n\n**DocLayNet Class Labels**\n\nDocLayNet defines 11 distinct class
-            labels for document layout analysis:\n\n1. **Caption** – Text describing figures or tables  \n2. **Footnote**
-            – Notes at the bottom of pages  \n3. **Formula** – Mathematical expressions  \n4. **List‑item** – Items in bulleted
-            or numbered lists  \n5. **Page‑footer** – Footer content on pages  \n6. **Page‑header** – Header content on pages
-            \ \n7. **Picture** – Images and diagrams  \n8. **Section‑header** – Headings for document sections  \n9. **Table**
-            – Tabular data  \n10. **Text** – Regular paragraph text (highest count: 510,377 instances)  \n11. **Title** –
-            Document titles  \n\nThat’s the full list of labels used in DocLayNet. Let me know if you’d like any more details!"
+          content: "Here’s the full **DocLayNet Class Labels** document:\n\n---\n\n## DocLayNet Dataset – Class Labels\n\nDocLayNet
+            defines **11 distinct class labels** for document layout analysis:\n\n1. **Caption** – Text describing figures
+            or tables  \n2. **Footnote** – Notes at the bottom of pages  \n3. **Formula** – Mathematical expressions  \n4.
+            **List‑item** – Items in bulleted or numbered lists  \n5. **Page‑footer** – Footer content on pages  \n6. **Page‑header**
+            – Header content on pages  \n7. **Picture** – Images and diagrams  \n8. **Section‑header** – Headings for document
+            sections  \n9. **Table** – Tabular data  \n10. **Text** – Regular paragraph text (highest count: **510,377** instances)
+            \ \n11. **Title** – Document titles  \n\nThe **Text** class has the highest count with 510,377 instances in the
+            dataset."
          role: assistant
-      created: 1768225925
-      id: chatcmpl-313
+      created: 1768998262
+      id: chatcmpl-746
      model: gpt-oss
      object: chat.completion
      system_fingerprint: fp_ollama
      usage:
-        completion_tokens: 196
+        completion_tokens: 202
        prompt_tokens: 1110
-        total_tokens: 1306
+        total_tokens: 1312
    status:
      code: 200
      message: OK
--- a/tests/cassettes/test_chat_agent/test_chat_agent_search_tool.yaml
+++ b/tests/cassettes/test_chat_agent/test_chat_agent_search_tool.yaml
--- a/tests/cassettes/test_chat_agent/test_chat_agent_search_tool_with_filter.yaml
+++ b/tests/cassettes/test_chat_agent/test_chat_agent_search_tool_with_filter.yaml
--- a/tests/cassettes/test_chat_agent/test_chat_agent_search_with_state_key.yaml
+++ b/tests/cassettes/test_chat_agent/test_chat_agent_search_with_state_key.yaml
@ -227,7 +227,7 @@ interactions:
  response:
    headers:
      content-length:
-      - '493'
+      - '502'
      content-type:
      - application/json
    parsed_body:
@ -236,17 +236,17 @@ interactions:
        index: 0
        message:
          content: ''
-          reasoning: Need search.
+          reasoning: Need to use search once.
          role: assistant
          tool_calls:
          - function:
-              arguments: '{"document_name":null,"limit":null,"query":"class labels"}'
+              arguments: '{"query":"class labels","document_name":null,"limit":5}'
              name: search
-            id: call_bji3vk31
+            id: call_m70fxfgn
            index: 0
            type: function
-      created: 1768403358
-      id: chatcmpl-424
+      created: 1768998240
+      id: chatcmpl-236
      model: gpt-oss
      object: chat.completion
      system_fingerprint: fp_ollama
@ -317,7 +317,7 @@ interactions:
  response:
    headers:
      content-length:
-      - '565'
+      - '478'
      content-type:
      - application/json
    parsed_body:
@ -326,24 +326,24 @@ interactions:
        index: 0
        message:
          content: ''
-          reasoning: Need to search "class labels". Likely referring to machine learning classification labels. Provide search.
+          reasoning: Need to run search.
          role: assistant
          tool_calls:
          - function:
-              arguments: '{"limit":5,"query":"class labels"}'
+              arguments: '{"query":"class labels","limit":5}'
              name: run_search
-            id: call_gcwrxbl4
+            id: call_6dvnsbzh
            index: 0
            type: function
-      created: 1768403359
-      id: chatcmpl-9
+      created: 1768998241
+      id: chatcmpl-107
      model: gpt-oss
      object: chat.completion
      system_fingerprint: fp_ollama
      usage:
-        completion_tokens: 48
+        completion_tokens: 34
        prompt_tokens: 263
-        total_tokens: 311
+        total_tokens: 297
    status:
      code: 200
      message: OK
@ -396,7 +396,7 @@ interactions:
      connection:
      - keep-alive
      content-length:
-      - '1457'
+      - '1368'
      content-type:
      - application/json
      host:
@ -421,18 +421,18 @@ interactions:
        role: user
      - content: |-
          <think>
-          Need to search "class labels". Likely referring to machine learning classification labels. Provide search.
+          Need to run search.
          </think>
        role: assistant
        tool_calls:
        - function:
-            arguments: '{"limit":5,"query":"class labels"}'
+            arguments: '{"query":"class labels","limit":5}'
            name: run_search
-          id: call_gcwrxbl4
+          id: call_6dvnsbzh
          type: function
      - content: 'Found 2 results for: class labels'
        role: tool
-        tool_call_id: call_gcwrxbl4
+        tool_call_id: call_6dvnsbzh
      model: gpt-oss
      reasoning_effort: low
      stream: false
@ -461,7 +461,7 @@ interactions:
  response:
    headers:
      content-length:
-      - '452'
+      - '524'
      content-type:
      - application/json
    parsed_body:
@ -470,23 +470,24 @@ interactions:
        index: 0
        message:
          content: ''
+          reasoning: 'Now an alternative query: maybe "label classification".'
          role: assistant
          tool_calls:
          - function:
              arguments: '{"limit":5,"query":"label classification"}'
              name: run_search
-            id: call_di042ewg
+            id: call_eonlxxeh
            index: 0
            type: function
-      created: 1768403360
-      id: chatcmpl-890
+      created: 1768998242
+      id: chatcmpl-552
      model: gpt-oss
      object: chat.completion
      system_fingerprint: fp_ollama
      usage:
-        completion_tokens: 23
-        prompt_tokens: 336
-        total_tokens: 359
+        completion_tokens: 39
+        prompt_tokens: 322
+        total_tokens: 361
    status:
      code: 200
      message: OK
@ -539,7 +540,7 @@ interactions:
      connection:
      - keep-alive
      content-length:
-      - '1746'
+      - '1731'
      content-type:
      - application/json
      host:
@ -564,29 +565,32 @@ interactions:
        role: user
      - content: |-
          <think>
-          Need to search "class labels". Likely referring to machine learning classification labels. Provide search.
+          Need to run search.
          </think>
        role: assistant
        tool_calls:
        - function:
-            arguments: '{"limit":5,"query":"class labels"}'
+            arguments: '{"query":"class labels","limit":5}'
            name: run_search
-          id: call_gcwrxbl4
+          id: call_6dvnsbzh
          type: function
      - content: 'Found 2 results for: class labels'
        role: tool
-        tool_call_id: call_gcwrxbl4
-      - content: null
+        tool_call_id: call_6dvnsbzh
+      - content: |-
+          <think>
+          Now an alternative query: maybe "label classification".
+          </think>
        role: assistant
        tool_calls:
        - function:
            arguments: '{"limit":5,"query":"label classification"}'
            name: run_search
-          id: call_di042ewg
+          id: call_eonlxxeh
          type: function
      - content: 'Found 2 results for: label classification'
        role: tool
-        tool_call_id: call_di042ewg
+        tool_call_id: call_eonlxxeh
      model: gpt-oss
      reasoning_effort: low
      stream: false
@ -615,7 +619,7 @@ interactions:
  response:
    headers:
      content-length:
-      - '297'
+      - '298'
      content-type:
      - application/json
    parsed_body:
@ -625,15 +629,15 @@ interactions:
        message:
          content: Search complete
          role: assistant
-      created: 1768403361
-      id: chatcmpl-58
+      created: 1768998242
+      id: chatcmpl-801
      model: gpt-oss
      object: chat.completion
      system_fingerprint: fp_ollama
      usage:
        completion_tokens: 6
-        prompt_tokens: 378
-        total_tokens: 384
+        prompt_tokens: 386
+        total_tokens: 392
    status:
      code: 200
      message: OK
@ -646,7 +650,7 @@ interactions:
      connection:
      - keep-alive
      content-length:
-      - '5071'
+      - '5080'
      content-type:
      - application/json
      host:
@ -688,14 +692,14 @@ interactions:
        role: user
      - content: |-
          <think>
-          Need search.
+          Need to use search once.
          </think>
        role: assistant
        tool_calls:
        - function:
-            arguments: '{"document_name":null,"limit":null,"query":"class labels"}'
+            arguments: '{"query":"class labels","document_name":null,"limit":5}'
            name: search
-          id: call_bji3vk31
+          id: call_m70fxfgn
          type: function
      - content: |-
          Found 2 results:
@ -706,7 +710,7 @@ interactions:
          [2] **DocLayNet Annotation**
              DocLayNet Dataset - Annotation Process  The annotation process was organized into 4 phases:  Phase 1: Data selection and preparation by a small team of experts  Phase 2: Label selection and guideline definition  Phase 3: Annotation by 40 dedicated annotators  Phase 4: Quality control and continuous...
        role: tool
-        tool_call_id: call_bji3vk31
+        tool_call_id: call_m70fxfgn
      model: gpt-oss
      reasoning_effort: low
      stream: false
@ -785,7 +789,7 @@ interactions:
  response:
    headers:
      content-length:
-      - '660'
+      - '700'
      content-type:
      - application/json
    parsed_body:
@ -793,22 +797,20 @@ interactions:
      - finish_reason: stop
        index: 0
        message:
-          content: |-
-            Here are the documents that mention **class labels**:
-
-            1. **DocLayNet Class Labels** – Describes the 11 distinct class labels used for document layout analysis, such as Caption, Footnote, Formula, etc.
-
-            2. **DocLayNet Annotation** – Discusses the annotation process for the DocLayNet dataset, including how the class labels were selected and applied during labeling.
+          content: "Here are two documents that discuss class labels:\n\n1. **DocLayNet Class Labels**  \n   - Describes the
+            11 distinct class labels used for document layout analysis (e.g., Caption, Footnote, Formula, List‑item, Page‑footer,
+            Page‑header, etc.).\n\n2. **DocLayNet Annotation**  \n   - Covers the annotation process, including how labels
+            were selected and guided during the construction of the DocLayNet dataset."
          role: assistant
-      created: 1768403363
-      id: chatcmpl-294
+      created: 1768998245
+      id: chatcmpl-955
      model: gpt-oss
      object: chat.completion
      system_fingerprint: fp_ollama
      usage:
-        completion_tokens: 83
-        prompt_tokens: 1048
-        total_tokens: 1131
+        completion_tokens: 96
+        prompt_tokens: 1051
+        total_tokens: 1147
    status:
      code: 200
      message: OK
--- a/tests/cassettes/test_chat_agent/test_chat_agent_with_qa_history_ranking.yaml
+++ b/tests/cassettes/test_chat_agent/test_chat_agent_with_qa_history_ranking.yaml
--- a/tests/cassettes/test_chat_agent/test_search_agent_deduplication.yaml
+++ b/tests/cassettes/test_chat_agent/test_search_agent_deduplication.yaml
--- a/tests/cassettes/test_chat_agent/test_search_agent_no_results.yaml
+++ b/tests/cassettes/test_chat_agent/test_search_agent_no_results.yaml
--- a/tests/cassettes/test_chat_agent/test_search_agent_with_context.yaml
+++ b/tests/cassettes/test_chat_agent/test_search_agent_with_context.yaml
--- a/tests/cassettes/test_chat_agent/test_search_agent_with_filter.yaml
+++ b/tests/cassettes/test_chat_agent/test_search_agent_with_filter.yaml
--- a/tests/cassettes/test_client/test_client_ask.yaml
+++ b/tests/cassettes/test_client/test_client_ask.yaml
--- a/tests/cassettes/test_client/test_client_update_document_stores_docling_json.yaml
+++ b/tests/cassettes/test_client/test_client_update_document_stores_docling_json.yaml
@ -39,53 +39,6 @@ interactions:
    status:
      code: 200
      message: OK
- request:
-    body: !!binary |
-      H4sIAKIwUGkC/82SS0/UUBTHOwwwwxFkpjAKPmLBR4BA6WuYGSLJBFiZ6EJjYjQGL+2dmYbpbe1t
-      MRNjwgYTfHwCWbDThTziF8BPoPGRaExcuXLlwgVEE2Pb6SAP48Sw0LO4Ob33d+495/8vrLTCehRy
-      0EGxPaurmNcJdRDxE43tBU7GWRFhMacoOawIQiYn5nB2RJ2Ws3JB0FQZeuGQg8vYwI5d4ak2w5cR
-      KbqoiNk4NFsVp2QSOA3sToYgA7Pt0GZamGydQA+kdmKz2Ka6SfybRF7O8aKHtNYaDe5IQrtLZoh5
-      i0yF+9ABByzbVDGlvOXN0Ni1+jjq1XXWNm2XOLoR1rdATA2bPAmHdyPb3pd5UeYzcBmO7oY0TFVb
-      txwfHAGlCnJ9BtLJIHcJW5yY5SRBSg9ykjSalkZlpZ+7NuGr5G3zIq9w12EM2mtj1d4cgL50VlE0
-      IaOmNUVCQkFLe9oLWSUtZUQlq+YkAWlipoCm2c9xSHlTVzTktaQOIZ2t6iWwL+PQXcRkCum8WtY9
-      sXnHnMGEd6nv0PB5jKhrY8oR15jGNmcWOJ1YrsMhonGm6/hpUEA5l2LtSOx28HXn4kYMNqLQDW3h
-      5bRCHWz4QpnlMjIQnIBUeOR5bCNfnarizdColpDjOxICNr7pYurwhqnhsu9I0XKGTEq9Xyu1hVDL
-      JBTvZY5BMmSqgzkVC7MxaArGSD5/szq50Dff1b05+eLTfS/piTLV6A/Wt4t5ITF+NxJfPMXORSLM
-      PiKyz3MvzkXZCHOVT974shx03Rt/+M0S6fjXdwOJyCM0tk5eJ542LXwf/jDxUexsqBZ5PIPyS3V5
-      5xffRZfP3KvHL4c8CtalubweJKX5PGz+F84f/53zfhvBb5t88mrtj9YzU3nhoG/9jxbP+gbmH0Rk
-      r/WlZ2v1reeYLSvfi+NLf8NfebBS3/qzzHbrmXxoPXMhn4j8BD4ukZMxBgAA
-    headers:
-      accept:
-      - '*/*'
-      accept-encoding:
-      - gzip, deflate, zstd
-      connection:
-      - keep-alive
-      content-encoding:
-      - gzip
-      content-length:
-      - '729'
-      content-type:
-      - application/x-protobuf
-    method: POST
-    uri: https://logfire-eu.pydantic.dev/v1/metrics
-  response:
-    body:
-      string: "\n\0"
-    headers:
-      connection:
-      - keep-alive
-      content-length:
-      - '2'
-      nel:
-      - '{"report_to":"cf-nel","success_fraction":0.0,"max_age":604800}'
-      report-to:
-      - '{"group":"cf-nel","max_age":604800,"endpoints":[{"url":"https://a.nel.cloudflare.com/report/v4?s=JzTAsIp8t3A57BJQl4mlG%2FfLA7%2BWUFJ9IsKcAJIIEeU%2BJ%2F%2BW4kihd7xaQIjse3baoJ79MeWUTKIFcB1xbCLrof%2B5yXY1xtklBwF4XA%2BVTc1MFqvW"}]}'
-      vary:
-      - origin, access-control-request-method, access-control-request-headers
-    status:
-      code: 200
-      message: OK
 - request:
    headers:
      accept:
--- a/tests/cassettes/test_qa/test_qa_anthropic.yaml
+++ b/tests/cassettes/test_qa/test_qa_anthropic.yaml
--- a/tests/cassettes/test_qa/test_qa_ollama.yaml
+++ b/tests/cassettes/test_qa/test_qa_ollama.yaml
--- a/tests/cassettes/test_qa/test_qa_openai.yaml
+++ b/tests/cassettes/test_qa/test_qa_openai.yaml
--- a/tests/store/cassettes/test_read_only/TestClientReadOnly.test_client_create_document_raises_when_read_only.yaml
+++ b/tests/store/cassettes/test_read_only/TestClientReadOnly.test_client_create_document_raises_when_read_only.yaml
--- a/tests/store/cassettes/test_read_only/TestClientReadOnly.test_client_delete_document_raises_when_read_only.yaml
+++ b/tests/store/cassettes/test_read_only/TestClientReadOnly.test_client_delete_document_raises_when_read_only.yaml
--- a/tests/store/cassettes/test_read_only/TestClientReadOnly.test_client_list_documents_works_when_read_only.yaml
+++ b/tests/store/cassettes/test_read_only/TestClientReadOnly.test_client_list_documents_works_when_read_only.yaml
--- a/tests/store/cassettes/test_read_only/TestClientReadOnly.test_client_search_works_when_read_only.yaml
+++ b/tests/store/cassettes/test_read_only/TestClientReadOnly.test_client_search_works_when_read_only.yaml
--- a/tests/cassettes/test_research_graph/test_graph_end_to_end.yaml
+++ b/tests/cassettes/test_research_graph/test_graph_end_to_end.yaml
--- a/tests/cassettes/test_search_filter/test_research_graph_uses_search_filter.yaml
+++ b/tests/cassettes/test_search_filter/test_research_graph_uses_search_filter.yaml
--- a/tests/agents/research/cassettes/test_search_filter/test_search_filter_none_searches_all.yaml
+++ b/tests/agents/research/cassettes/test_search_filter/test_search_filter_none_searches_all.yaml
--- a/tests/agents/research/cassettes/test_search_filter/test_search_filter_restricts_results.yaml
+++ b/tests/agents/research/cassettes/test_search_filter/test_search_filter_restricts_results.yaml
--- a/tests/store/test_read_only.py
+++ b/tests/store/test_read_only.py
@ -1,3 +1,5 @@
+from pathlib import Path
+
 import pytest

 from haiku.rag.client import HaikuRAG
@ -8,6 +10,11 @@ from haiku.rag.store.repositories.document import DocumentRepository
 from haiku.rag.store.repositories.settings import SettingsRepository


+@pytest.fixture(scope="module")
+def vcr_cassette_dir():
+    return str(Path(__file__).parent.parent / "cassettes" / "test_read_only")
+
+
 class TestReadOnlyError:
    def test_read_only_error_is_exception(self):
        """ReadOnlyError should be a subclass of Exception."""
--- a/tests/test_chunk.py
+++ b/tests/test_chunk.py
@ -254,8 +254,50 @@ def test_chunk_metadata_resolve_empty_refs():
    assert doc_items == []


-def test_search_result_format_for_agent_full():
-    """Test format_for_agent with all metadata present."""
+def test_search_result_format_for_agent_with_rank():
+    """Test format_for_agent with rank and total parameters."""
+    result = SearchResult(
+        content="This is the chunk content about elections.",
+        score=0.02,  # Low RRF score that would confuse agents
+        chunk_id="chunk-123",
+        document_id="doc-456",
+        document_uri="file:///docs/report.pdf",
+        document_title="Annual Report 2024",
+        headings=["Chapter 1", "Section 1.1", "Elections"],
+        labels=["paragraph", "table"],
+        page_numbers=[1, 2],
+    )
+
+    formatted = result.format_for_agent(rank=1, total=5)
+
+    assert "[chunk-123]" in formatted
+    assert "[rank 1 of 5]" in formatted
+    assert "score:" not in formatted  # Score should NOT appear when rank is provided
+    assert (
+        'Source: "Annual Report 2024" > Chapter 1 > Section 1.1 > Elections'
+        in formatted
+    )
+    assert "Type: table" in formatted
+    assert "Content:\nThis is the chunk content about elections." in formatted
+
+
+def test_search_result_format_for_agent_rank_only():
+    """Test format_for_agent with rank but no total."""
+    result = SearchResult(
+        content="Some content.",
+        score=0.03,
+        chunk_id="chunk-abc",
+    )
+
+    formatted = result.format_for_agent(rank=2)
+
+    assert "[chunk-abc]" in formatted
+    assert "[rank 2]" in formatted
+    assert "score:" not in formatted
+
+
+def test_search_result_format_for_agent_fallback():
+    """Test format_for_agent falls back to score when no rank provided."""
    result = SearchResult(
        content="This is the chunk content about elections.",
        score=0.85,
--- a/tests/test_context_enhancement.py
+++ b/tests/test_context_enhancement.py
@ -298,10 +298,11 @@ async def test_format_for_agent_output(temp_db_path, small_chunk_config):
        assert len(table_results) > 0

        expanded = await client.expand_context(table_results[:1])
-        formatted = expanded[0].format_for_agent()
+        # Format with rank (the way agents use it)
+        formatted = expanded[0].format_for_agent(rank=1, total=1)

        # Check format structure
-        assert "score:" in formatted
+        assert "[rank 1 of 1]" in formatted
        assert 'Source: "Format Test"' in formatted
        assert "Type: table" in formatted
        assert "Content:" in formatted
--- a/tests/test_search.py
+++ b/tests/test_search.py
@ -258,11 +258,12 @@ async def test_search_result_format_includes_metadata(temp_db_path):
        results = await client.search("machine learning", limit=1)
        assert len(results) > 0

-        formatted = results[0].format_for_agent()
+        # Format with rank (the way agents use it)
+        formatted = results[0].format_for_agent(rank=1, total=1)

-        # Should include chunk ID and score
+        # Should include chunk ID and rank
        assert "[" in formatted and "]" in formatted
-        assert "score:" in formatted
+        assert "[rank 1 of 1]" in formatted

        # Should include document title in Source
        assert "ML Guide" in formatted