Merge pull request #489 from ggozad/fix/excessive-citations
Scope citations to the matched content and make visual grounding faithful
This commit is contained in:
commit
c8497d7cd1
22 changed files with 1068 additions and 131 deletions
10
CHANGELOG.md
10
CHANGELOG.md
|
|
@ -1,6 +1,13 @@
|
||||||
# Changelog
|
# Changelog
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Added
|
||||||
|
|
||||||
|
- `SearchResult.chunk_ids` and `Citation.chunk_ids` carry the chunk ids merged into an expanded result.
|
||||||
|
- `Citation.doc_item_refs` carries the items the model saw; `visualize_chunk` accepts a `refs` argument and resolves bounding boxes from them, so visualizations match the cited content instead of re-expanding.
|
||||||
|
- Chunk visualizations draw matched content in a stronger highlight than surrounding context.
|
||||||
|
- `haiku-rag visualize --no-expand` highlights only the chunk itself, without its expanded context (`visualize_chunk` gains an `expand` argument).
|
||||||
|
|
||||||
### Changed
|
### Changed
|
||||||
|
|
||||||
- Duplicate images within a document produce a single picture chunk.
|
- Duplicate images within a document produce a single picture chunk.
|
||||||
|
|
@ -9,6 +16,9 @@
|
||||||
### Fixed
|
### Fixed
|
||||||
|
|
||||||
- vLLM embedding and vLLM/Jina reranking reuse one HTTP client across requests instead of opening one per request.
|
- vLLM embedding and vLLM/Jina reranking reuse one HTTP client across requests instead of opening one per request.
|
||||||
|
- Picture and table search hits expand within their section, never across section boundaries.
|
||||||
|
- A merged search result anchors its `chunk_id` on the highest-scoring constituent chunk instead of the one earliest in the document.
|
||||||
|
- A clipped citation's `page_numbers` and `doc_item_refs` reflect only the content that survived the budget, not the full expanded range.
|
||||||
|
|
||||||
## [0.64.0] - 2026-07-08
|
## [0.64.0] - 2026-07-08
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -203,22 +203,43 @@ async def db_info(_: Request) -> JSONResponse:
|
||||||
|
|
||||||
|
|
||||||
async def visualize_chunk(request: Request) -> JSONResponse:
|
async def visualize_chunk(request: Request) -> JSONResponse:
|
||||||
"""Return visual grounding images for a chunk as base64."""
|
"""Return visual grounding images for one or more chunks as base64.
|
||||||
|
|
||||||
|
The path param accepts comma-separated chunk ids (a merged citation's
|
||||||
|
constituent chunks). The optional ``refs`` query param is a JSON-encoded
|
||||||
|
list of the citation's ``doc_item_refs`` — the exact items the model saw —
|
||||||
|
so the highlight matches the cited content instead of re-expanding.
|
||||||
|
"""
|
||||||
import base64
|
import base64
|
||||||
|
import json
|
||||||
from io import BytesIO
|
from io import BytesIO
|
||||||
|
|
||||||
chunk_id = request.path_params["chunk_id"]
|
chunk_id = request.path_params["chunk_id"]
|
||||||
|
|
||||||
|
refs: list[str] | None = None
|
||||||
|
refs_param = request.query_params.get("refs")
|
||||||
|
if refs_param:
|
||||||
|
try:
|
||||||
|
parsed = json.loads(refs_param)
|
||||||
|
except ValueError:
|
||||||
|
parsed = None
|
||||||
|
if isinstance(parsed, list):
|
||||||
|
refs = [str(x) for x in parsed]
|
||||||
|
|
||||||
if not db_path.exists():
|
if not db_path.exists():
|
||||||
return JSONResponse({"error": "Database not found"}, status_code=404)
|
return JSONResponse({"error": "Database not found"}, status_code=404)
|
||||||
|
|
||||||
client = await get_client()
|
client = await get_client()
|
||||||
|
|
||||||
chunk = await client.chunk_repository.get_by_id(chunk_id)
|
chunks = []
|
||||||
if not chunk:
|
for cid in chunk_id.split(","):
|
||||||
|
chunk = await client.chunk_repository.get_by_id(cid)
|
||||||
|
if chunk:
|
||||||
|
chunks.append(chunk)
|
||||||
|
if not chunks:
|
||||||
return JSONResponse({"error": "Chunk not found"}, status_code=404)
|
return JSONResponse({"error": "Chunk not found"}, status_code=404)
|
||||||
|
|
||||||
images = await client.visualize_chunk(chunk)
|
images = await client.visualize_chunk(chunks, refs)
|
||||||
if not images:
|
if not images:
|
||||||
return JSONResponse({"images": [], "message": "No visual grounding available"})
|
return JSONResponse({"images": [], "message": "No visual grounding available"})
|
||||||
|
|
||||||
|
|
@ -233,7 +254,7 @@ async def visualize_chunk(request: Request) -> JSONResponse:
|
||||||
{
|
{
|
||||||
"images": base64_images,
|
"images": base64_images,
|
||||||
"chunk_id": chunk_id,
|
"chunk_id": chunk_id,
|
||||||
"document_uri": chunk.document_uri,
|
"document_uri": chunks[0].document_uri,
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,18 @@
|
||||||
import { NextResponse } from "next/server";
|
import { NextResponse } from "next/server";
|
||||||
|
|
||||||
export async function GET(
|
export async function GET(
|
||||||
_request: Request,
|
request: Request,
|
||||||
{ params }: { params: Promise<{ chunk_id: string }> },
|
{ params }: { params: Promise<{ chunk_id: string }> },
|
||||||
) {
|
) {
|
||||||
const { chunk_id } = await params;
|
const { chunk_id } = await params;
|
||||||
const backendUrl = process.env.BACKEND_URL || "http://backend:8000";
|
const backendUrl = process.env.BACKEND_URL || "http://backend:8000";
|
||||||
|
const refs = new URL(request.url).searchParams.get("refs");
|
||||||
|
const query = refs ? `?refs=${encodeURIComponent(refs)}` : "";
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetch(`${backendUrl}/api/visualize/${chunk_id}`);
|
const response = await fetch(
|
||||||
|
`${backendUrl}/api/visualize/${chunk_id}${query}`,
|
||||||
|
);
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,7 @@ function CitationItem({
|
||||||
onViewInDocument,
|
onViewInDocument,
|
||||||
}: {
|
}: {
|
||||||
citation: Citation;
|
citation: Citation;
|
||||||
onViewInDocument: (chunkId: string) => void;
|
onViewInDocument: (chunkId: string, refs?: string[]) => void;
|
||||||
}) {
|
}) {
|
||||||
const [expanded, setExpanded] = useState(false);
|
const [expanded, setExpanded] = useState(false);
|
||||||
|
|
||||||
|
|
@ -55,7 +55,14 @@ function CitationItem({
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
className="citation-view-btn"
|
className="citation-view-btn"
|
||||||
onClick={() => onViewInDocument(citation.chunk_id)}
|
onClick={() =>
|
||||||
|
onViewInDocument(
|
||||||
|
citation.chunk_ids?.length
|
||||||
|
? citation.chunk_ids.join(",")
|
||||||
|
: citation.chunk_id,
|
||||||
|
citation.doc_item_refs,
|
||||||
|
)
|
||||||
|
}
|
||||||
>
|
>
|
||||||
View in Document
|
View in Document
|
||||||
</button>
|
</button>
|
||||||
|
|
@ -82,45 +89,54 @@ export default function CitationBlock({ citations }: CitationBlockProps) {
|
||||||
return () => abortRef.current?.abort();
|
return () => abortRef.current?.abort();
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const fetchVisualGrounding = useCallback(async (chunkId: string) => {
|
const fetchVisualGrounding = useCallback(
|
||||||
abortRef.current?.abort();
|
async (chunkId: string, refs?: string[]) => {
|
||||||
const controller = new AbortController();
|
abortRef.current?.abort();
|
||||||
abortRef.current = controller;
|
const controller = new AbortController();
|
||||||
|
abortRef.current = controller;
|
||||||
|
|
||||||
setVisualGrounding({
|
setVisualGrounding({
|
||||||
isOpen: true,
|
isOpen: true,
|
||||||
chunkId,
|
chunkId,
|
||||||
images: [],
|
images: [],
|
||||||
loading: true,
|
loading: true,
|
||||||
error: null,
|
error: null,
|
||||||
});
|
|
||||||
|
|
||||||
try {
|
|
||||||
const response = await fetch(`/api/visualize/${chunkId}`, {
|
|
||||||
signal: controller.signal,
|
|
||||||
});
|
});
|
||||||
const data = await response.json();
|
|
||||||
|
|
||||||
if (controller.signal.aborted) return;
|
const query = refs?.length
|
||||||
if (!response.ok) {
|
? `?refs=${encodeURIComponent(JSON.stringify(refs))}`
|
||||||
throw new Error(data.error || "Failed to fetch visual grounding");
|
: "";
|
||||||
|
try {
|
||||||
|
const response = await fetch(
|
||||||
|
`/api/visualize/${encodeURIComponent(chunkId)}${query}`,
|
||||||
|
{
|
||||||
|
signal: controller.signal,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
const data = await response.json();
|
||||||
|
|
||||||
|
if (controller.signal.aborted) return;
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error(data.error || "Failed to fetch visual grounding");
|
||||||
|
}
|
||||||
|
|
||||||
|
setVisualGrounding((prev) => ({
|
||||||
|
...prev,
|
||||||
|
images: data.images || [],
|
||||||
|
loading: false,
|
||||||
|
error: data.images?.length === 0 ? data.message : null,
|
||||||
|
}));
|
||||||
|
} catch (err) {
|
||||||
|
if (controller.signal.aborted) return;
|
||||||
|
setVisualGrounding((prev) => ({
|
||||||
|
...prev,
|
||||||
|
loading: false,
|
||||||
|
error: err instanceof Error ? err.message : "Unknown error",
|
||||||
|
}));
|
||||||
}
|
}
|
||||||
|
},
|
||||||
setVisualGrounding((prev) => ({
|
[],
|
||||||
...prev,
|
);
|
||||||
images: data.images || [],
|
|
||||||
loading: false,
|
|
||||||
error: data.images?.length === 0 ? data.message : null,
|
|
||||||
}));
|
|
||||||
} catch (err) {
|
|
||||||
if (controller.signal.aborted) return;
|
|
||||||
setVisualGrounding((prev) => ({
|
|
||||||
...prev,
|
|
||||||
loading: false,
|
|
||||||
error: err instanceof Error ? err.message : "Unknown error",
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
const closeVisualGrounding = useCallback(() => {
|
const closeVisualGrounding = useCallback(() => {
|
||||||
abortRef.current?.abort();
|
abortRef.current?.abort();
|
||||||
|
|
|
||||||
|
|
@ -2,11 +2,13 @@ export interface Citation {
|
||||||
index: number;
|
index: number;
|
||||||
document_id: string;
|
document_id: string;
|
||||||
chunk_id: string;
|
chunk_id: string;
|
||||||
|
chunk_ids?: string[];
|
||||||
document_uri: string;
|
document_uri: string;
|
||||||
document_title: string | null;
|
document_title: string | null;
|
||||||
page_numbers: number[];
|
page_numbers: number[];
|
||||||
headings: string[] | null;
|
headings: string[] | null;
|
||||||
content: string;
|
content: string;
|
||||||
|
doc_item_refs?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
// Matches RAGState from the backend skill
|
// Matches RAGState from the backend skill
|
||||||
|
|
|
||||||
|
|
@ -246,7 +246,9 @@ Display visual grounding for a chunk - shows page images with highlighted boundi
|
||||||
haiku-rag visualize <chunk_id>
|
haiku-rag visualize <chunk_id>
|
||||||
```
|
```
|
||||||
|
|
||||||
This renders the source document pages with the chunk's location highlighted. Useful for verifying chunk boundaries and understanding document structure.
|
This renders the source document pages with the chunk's location highlighted. The chunk itself draws in a strong highlight, while surrounding context swept in by expansion draws fainter. Useful for verifying chunk boundaries and understanding document structure.
|
||||||
|
|
||||||
|
Pass `--no-expand` to highlight only the chunk itself, without its expanded context.
|
||||||
|
|
||||||
!!! note
|
!!! note
|
||||||
Requires a terminal with image support (iTerm2, Kitty, WezTerm, etc.) and documents processed with docling that have page images stored.
|
Requires a terminal with image support (iTerm2, Kitty, WezTerm, etc.) and documents processed with docling that have page images stored.
|
||||||
|
|
|
||||||
|
|
@ -13,7 +13,7 @@ search:
|
||||||
- **limit**: Default number of search results to return when no limit is specified. Used by CLI, MCP server, and QA. Default: 10
|
- **limit**: Default number of search results to return when no limit is specified. Used by CLI, MCP server, and QA. Default: 10
|
||||||
- **max_context_chars**: Hard limit on total characters in expanded content. Default: 10000.
|
- **max_context_chars**: Hard limit on total characters in expanded content. Default: 10000.
|
||||||
|
|
||||||
Context expansion is automatic and section-aware. For structured documents (with section headers), expansion includes the entire section containing the match. For sections that exceed the budget or are too small (e.g., a title+authors area), expansion grows outward item-by-item from the match center, skipping noise labels (footnotes, page headers). This naturally crosses into adjacent sections until the budget is filled. For unstructured documents, expansion grows outward item-by-item. Results without `doc_item_refs` (e.g., custom chunks passed to `import_document`) pass through unexpanded.
|
Context expansion is automatic and section-aware. For structured documents (with section headers), expansion includes the entire section containing the match. For sections that exceed the budget or are too small (e.g., a title+authors area), expansion grows outward item-by-item from the match center, skipping noise labels (footnotes, page headers). This naturally crosses into adjacent sections until the budget is filled. Picture and table matches are exempt: they return their enclosing section as-is and never cross section boundaries. For unstructured documents, expansion grows outward item-by-item. Results without `doc_item_refs` (e.g., custom chunks passed to `import_document`) pass through unexpanded.
|
||||||
|
|
||||||
!!! note "Reranking behavior"
|
!!! note "Reranking behavior"
|
||||||
When a reranker is configured, search automatically retrieves 10x the requested limit, then reranks to return the final count. This improves result quality without requiring you to adjust `limit`.
|
When a reranker is configured, search automatically retrieves 10x the requested limit, then reranks to return the final count. This improves result quality without requiring you to adjust `limit`.
|
||||||
|
|
|
||||||
|
|
@ -298,7 +298,7 @@ for result in expanded_results:
|
||||||
print(f"Expanded content: {result.content}")
|
print(f"Expanded content: {result.content}")
|
||||||
```
|
```
|
||||||
|
|
||||||
Context expansion is automatic and section-aware. For structured documents (with section headers), expansion includes the entire section containing the match. For sections that exceed the budget or are too small (e.g., a title+authors area), expansion grows outward item-by-item from the match center, skipping noise labels (footnotes, page headers). This naturally crosses into adjacent sections until the budget is filled. For unstructured documents, expansion grows outward item-by-item. Results without `doc_item_refs` (e.g., custom chunks passed to `import_document`) pass through unexpanded.
|
Context expansion is automatic and section-aware. For structured documents (with section headers), expansion includes the entire section containing the match. For sections that exceed the budget or are too small (e.g., a title+authors area), expansion grows outward item-by-item from the match center, skipping noise labels (footnotes, page headers). This naturally crosses into adjacent sections until the budget is filled. Picture and table matches are exempt: they return their enclosing section as-is and never cross section boundaries. For unstructured documents, expansion grows outward item-by-item. Results without `doc_item_refs` (e.g., custom chunks passed to `import_document`) pass through unexpanded.
|
||||||
|
|
||||||
Configuration:
|
Configuration:
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -457,7 +457,7 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
for result in results:
|
for result in results:
|
||||||
self._rich_print_search_result(result)
|
self._rich_print_search_result(result)
|
||||||
|
|
||||||
async def visualize_chunk(self, chunk_id: str):
|
async def visualize_chunk(self, chunk_id: str, expand: bool = True):
|
||||||
"""Display visual grounding images for a chunk."""
|
"""Display visual grounding images for a chunk."""
|
||||||
from textual_image.renderable import Image as RichImage
|
from textual_image.renderable import Image as RichImage
|
||||||
|
|
||||||
|
|
@ -472,7 +472,7 @@ class HaikuRAGApp: # pragma: no cover
|
||||||
self.console.print(f"[red]Chunk with id {chunk_id} not found.[/red]")
|
self.console.print(f"[red]Chunk with id {chunk_id} not found.[/red]")
|
||||||
return
|
return
|
||||||
|
|
||||||
images = await self.client.visualize_chunk(chunk)
|
images = await self.client.visualize_chunk(chunk, expand=expand)
|
||||||
if not images:
|
if not images:
|
||||||
self.console.print(
|
self.console.print(
|
||||||
"[yellow]No visual grounding available for this chunk.[/yellow]"
|
"[yellow]No visual grounding available for this chunk.[/yellow]"
|
||||||
|
|
|
||||||
|
|
@ -405,13 +405,24 @@ class ChatApp(App):
|
||||||
return
|
return
|
||||||
|
|
||||||
citation = selected_widgets[0].citation
|
citation = selected_widgets[0].citation
|
||||||
chunk = await self.client.get_chunk_by_id(citation.chunk_id)
|
chunk_ids = citation.chunk_ids or [citation.chunk_id]
|
||||||
if not chunk:
|
chunks = []
|
||||||
|
for cid in chunk_ids:
|
||||||
|
chunk = await self.client.get_chunk_by_id(cid)
|
||||||
|
if chunk:
|
||||||
|
chunks.append(chunk)
|
||||||
|
if not chunks:
|
||||||
return
|
return
|
||||||
|
|
||||||
from haiku.rag.inspector.widgets.visual_modal import VisualGroundingModal
|
from haiku.rag.inspector.widgets.visual_modal import VisualGroundingModal
|
||||||
|
|
||||||
await self.push_screen(VisualGroundingModal(chunk=chunk, client=self.client))
|
await self.push_screen(
|
||||||
|
VisualGroundingModal(
|
||||||
|
chunk=chunks,
|
||||||
|
client=self.client,
|
||||||
|
refs=citation.doc_item_refs or None,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
async def action_show_info(self) -> None:
|
async def action_show_info(self) -> None:
|
||||||
"""Show database info modal."""
|
"""Show database info modal."""
|
||||||
|
|
|
||||||
|
|
@ -346,9 +346,14 @@ def visualize( # pragma: no cover
|
||||||
"--db",
|
"--db",
|
||||||
help="Path to the LanceDB database file",
|
help="Path to the LanceDB database file",
|
||||||
),
|
),
|
||||||
|
no_expand: bool = typer.Option(
|
||||||
|
False,
|
||||||
|
"--no-expand",
|
||||||
|
help="Highlight only the chunk itself, without its expanded context",
|
||||||
|
),
|
||||||
):
|
):
|
||||||
app = create_app(db)
|
app = create_app(db)
|
||||||
asyncio.run(app.visualize_chunk(chunk_id=chunk_id))
|
asyncio.run(app.visualize_chunk(chunk_id=chunk_id, expand=not no_expand))
|
||||||
|
|
||||||
|
|
||||||
@_cli.command("ask", help="Ask a question using the QA agent")
|
@_cli.command("ask", help="Ask a question using the QA agent")
|
||||||
|
|
|
||||||
|
|
@ -4,7 +4,7 @@ import json
|
||||||
import logging
|
import logging
|
||||||
import mimetypes
|
import mimetypes
|
||||||
import tempfile
|
import tempfile
|
||||||
from collections.abc import AsyncGenerator
|
from collections.abc import AsyncGenerator, Sequence
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from functools import cached_property
|
from functools import cached_property
|
||||||
|
|
@ -504,10 +504,15 @@ class HaikuRAG:
|
||||||
|
|
||||||
return await analyze(self, question, filter)
|
return await analyze(self, question, filter)
|
||||||
|
|
||||||
async def visualize_chunk(self, chunk: Chunk) -> list:
|
async def visualize_chunk(
|
||||||
|
self,
|
||||||
|
chunk: Chunk | Sequence[Chunk],
|
||||||
|
refs: list[str] | None = None,
|
||||||
|
expand: bool = True,
|
||||||
|
) -> list:
|
||||||
from haiku.rag.client.search import visualize_chunk
|
from haiku.rag.client.search import visualize_chunk
|
||||||
|
|
||||||
return await visualize_chunk(self, chunk)
|
return await visualize_chunk(self, chunk, refs, expand)
|
||||||
|
|
||||||
async def rebuild_database(
|
async def rebuild_database(
|
||||||
self, mode: RebuildMode = RebuildMode.FULL
|
self, mode: RebuildMode = RebuildMode.FULL
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import base64
|
import base64
|
||||||
|
from collections.abc import Sequence
|
||||||
from typing import TYPE_CHECKING
|
from typing import TYPE_CHECKING
|
||||||
|
|
||||||
from haiku.rag.store.models.chunk import Chunk, SearchResult, SearchType
|
from haiku.rag.store.models.chunk import Chunk, SearchResult, SearchType
|
||||||
|
|
@ -223,12 +224,24 @@ async def expand_context(
|
||||||
return expanded_results
|
return expanded_results
|
||||||
|
|
||||||
|
|
||||||
async def visualize_chunk(client: "HaikuRAG", chunk: Chunk) -> list:
|
async def visualize_chunk(
|
||||||
"""Render page images with bounding box highlights for a chunk.
|
client: "HaikuRAG",
|
||||||
|
chunk: "Chunk | Sequence[Chunk]",
|
||||||
|
refs: list[str] | None = None,
|
||||||
|
expand: bool = True,
|
||||||
|
) -> list:
|
||||||
|
"""Render page images with bounding box highlights for one or more chunks.
|
||||||
|
|
||||||
Expands the chunk's context to find the full section, then resolves
|
When ``refs`` is given (the ``doc_item_refs`` of the citation, i.e. the
|
||||||
bounding boxes from all items in the expanded range. This ensures
|
exact items the model saw), bounding boxes are resolved from them directly
|
||||||
visualization covers all pages the expanded content spans.
|
so the visualization matches the cited context precisely. Otherwise, with
|
||||||
|
``expand=True`` (default) the chunks' context is re-expanded to recover the
|
||||||
|
surrounding section; with ``expand=False`` only the chunks' own items are
|
||||||
|
drawn, so the visualization shows just the retrieved chunk with no context.
|
||||||
|
|
||||||
|
The chunks' own items draw in a strong highlight; the remaining items draw
|
||||||
|
fainter, so the matched content stands out from its surrounding context.
|
||||||
|
Chunks from a different document than the first are ignored.
|
||||||
|
|
||||||
Returns a list of PIL Image objects, one per page with bounding boxes.
|
Returns a list of PIL Image objects, one per page with bounding boxes.
|
||||||
Empty list if no bounding boxes or page images available.
|
Empty list if no bounding boxes or page images available.
|
||||||
|
|
@ -239,10 +252,15 @@ async def visualize_chunk(client: "HaikuRAG", chunk: Chunk) -> list:
|
||||||
|
|
||||||
from haiku.rag.store.models.chunk import ChunkMetadata
|
from haiku.rag.store.models.chunk import ChunkMetadata
|
||||||
|
|
||||||
if not chunk.document_id:
|
chunks = [chunk] if isinstance(chunk, Chunk) else list(chunk)
|
||||||
|
if not chunks:
|
||||||
return []
|
return []
|
||||||
|
document_id = chunks[0].document_id
|
||||||
|
if not document_id:
|
||||||
|
return []
|
||||||
|
chunks = [c for c in chunks if c.document_id == document_id]
|
||||||
|
|
||||||
doc = await client.document_repository.get_docling_data(chunk.document_id)
|
doc = await client.document_repository.get_docling_data(document_id)
|
||||||
if not doc:
|
if not doc:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
@ -250,35 +268,60 @@ async def visualize_chunk(client: "HaikuRAG", chunk: Chunk) -> list:
|
||||||
if not docling_doc:
|
if not docling_doc:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# Expand context to get all doc_item_refs in the section
|
matched_refs = {r for c in chunks for r in c.get_chunk_metadata().doc_item_refs}
|
||||||
chunk_meta = chunk.get_chunk_metadata()
|
|
||||||
if chunk_meta.doc_item_refs:
|
if refs is not None:
|
||||||
search_result = SearchResult(
|
all_refs = list(refs)
|
||||||
content=chunk.content,
|
elif not expand:
|
||||||
score=1.0,
|
# Chunk-only: draw just the retrieved chunks' own items, no context.
|
||||||
chunk_id=chunk.id,
|
all_refs = list(matched_refs)
|
||||||
document_id=chunk.document_id,
|
|
||||||
doc_item_refs=chunk_meta.doc_item_refs,
|
|
||||||
page_numbers=chunk_meta.page_numbers,
|
|
||||||
)
|
|
||||||
expanded = await expand_context(client, [search_result])
|
|
||||||
refs = expanded[0].doc_item_refs if expanded else chunk_meta.doc_item_refs
|
|
||||||
meta = ChunkMetadata(doc_item_refs=refs)
|
|
||||||
else:
|
else:
|
||||||
meta = chunk_meta
|
# No stored context: re-expand the chunks to recover their section.
|
||||||
bounding_boxes = meta.resolve_bounding_boxes(docling_doc)
|
search_results = [
|
||||||
if not bounding_boxes:
|
SearchResult(
|
||||||
|
content=c.content,
|
||||||
|
score=1.0,
|
||||||
|
chunk_id=c.id,
|
||||||
|
document_id=c.document_id,
|
||||||
|
doc_item_refs=meta.doc_item_refs,
|
||||||
|
page_numbers=meta.page_numbers,
|
||||||
|
)
|
||||||
|
for c in chunks
|
||||||
|
if (meta := c.get_chunk_metadata()).doc_item_refs
|
||||||
|
]
|
||||||
|
if search_results:
|
||||||
|
expanded = await expand_context(client, search_results)
|
||||||
|
all_refs = []
|
||||||
|
for result in expanded:
|
||||||
|
all_refs.extend(r for r in result.doc_item_refs if r not in all_refs)
|
||||||
|
if not all_refs:
|
||||||
|
all_refs = [r for sr in search_results for r in sr.doc_item_refs]
|
||||||
|
else:
|
||||||
|
all_refs = list(chunks[0].get_chunk_metadata().doc_item_refs)
|
||||||
|
|
||||||
|
matched_draw = [r for r in all_refs if r in matched_refs]
|
||||||
|
swept_refs = [r for r in all_refs if r not in matched_refs]
|
||||||
|
|
||||||
|
matched_boxes = ChunkMetadata(doc_item_refs=matched_draw).resolve_bounding_boxes(
|
||||||
|
docling_doc
|
||||||
|
)
|
||||||
|
swept_boxes = ChunkMetadata(doc_item_refs=swept_refs).resolve_bounding_boxes(
|
||||||
|
docling_doc
|
||||||
|
)
|
||||||
|
if not matched_boxes and not swept_boxes:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# Group bounding boxes by page
|
# Group bounding boxes by page; swept boxes first so matched draw on top
|
||||||
boxes_by_page: dict[int, list] = {}
|
boxes_by_page: dict[int, list] = {}
|
||||||
for bbox in bounding_boxes:
|
for bbox, is_matched in [(b, False) for b in swept_boxes] + [
|
||||||
|
(b, True) for b in matched_boxes
|
||||||
|
]:
|
||||||
if bbox.page_no not in boxes_by_page:
|
if bbox.page_no not in boxes_by_page:
|
||||||
boxes_by_page[bbox.page_no] = []
|
boxes_by_page[bbox.page_no] = []
|
||||||
boxes_by_page[bbox.page_no].append(bbox)
|
boxes_by_page[bbox.page_no].append((bbox, is_matched))
|
||||||
|
|
||||||
# Load only the needed page images
|
# Load only the needed page images
|
||||||
pages_doc = await client.document_repository.get_pages_data(chunk.document_id)
|
pages_doc = await client.document_repository.get_pages_data(document_id)
|
||||||
if not pages_doc:
|
if not pages_doc:
|
||||||
return []
|
return []
|
||||||
page_images = pages_doc.get_page_images(list(boxes_by_page.keys()))
|
page_images = pages_doc.get_page_images(list(boxes_by_page.keys()))
|
||||||
|
|
@ -303,7 +346,7 @@ async def visualize_chunk(client: "HaikuRAG", chunk: Chunk) -> list:
|
||||||
image = deepcopy(pil_image)
|
image = deepcopy(pil_image)
|
||||||
draw = ImageDraw.Draw(image, "RGBA")
|
draw = ImageDraw.Draw(image, "RGBA")
|
||||||
|
|
||||||
for bbox in boxes_by_page[page_no]:
|
for bbox, is_matched in boxes_by_page[page_no]:
|
||||||
# Document coords are bottom-left origin; PIL uses top-left
|
# Document coords are bottom-left origin; PIL uses top-left
|
||||||
x0 = bbox.left * scale_x
|
x0 = bbox.left * scale_x
|
||||||
y0 = (page_height - bbox.top) * scale_y
|
y0 = (page_height - bbox.top) * scale_y
|
||||||
|
|
@ -313,8 +356,12 @@ async def visualize_chunk(client: "HaikuRAG", chunk: Chunk) -> list:
|
||||||
if y0 > y1:
|
if y0 > y1:
|
||||||
y0, y1 = y1, y0
|
y0, y1 = y1, y0
|
||||||
|
|
||||||
fill_color = (255, 255, 0, 40) # Yellow with transparency
|
if is_matched:
|
||||||
outline_color = (255, 165, 0, 100) # Orange outline
|
fill_color = (255, 150, 0, 55) # Orange, matched content
|
||||||
|
outline_color = (240, 130, 0, 150) # Orange outline
|
||||||
|
else:
|
||||||
|
fill_color = (255, 255, 0, 40) # Yellow, surrounding context
|
||||||
|
outline_color = (255, 165, 0, 100)
|
||||||
|
|
||||||
draw.rectangle([(x0, y0), (x1, y1)], fill=fill_color, outline=None)
|
draw.rectangle([(x0, y0), (x1, y1)], fill=fill_color, outline=None)
|
||||||
draw.rectangle([(x0, y0), (x1, y1)], outline=outline_color, width=1)
|
draw.rectangle([(x0, y0), (x1, y1)], outline=outline_color, width=1)
|
||||||
|
|
|
||||||
|
|
@ -12,7 +12,8 @@ For STRUCTURED documents (containing section_header or title labels):
|
||||||
5. If the section is too small (under 20% of max_context_chars), expand
|
5. If the section is too small (under 20% of max_context_chars), expand
|
||||||
item-by-item crossing into adjacent sections until the budget is filled.
|
item-by-item crossing into adjacent sections until the budget is filled.
|
||||||
This lets small sections (e.g., title+authors) grow into neighboring
|
This lets small sections (e.g., title+authors) grow into neighboring
|
||||||
content.
|
content. Picture and table matches are exempt: they return their
|
||||||
|
enclosing section as-is, never crossing section boundaries.
|
||||||
6. Merge overlapping ranges from multiple results in the same document.
|
6. Merge overlapping ranges from multiple results in the same document.
|
||||||
Adjacent but non-overlapping ranges stay separate to preserve section
|
Adjacent but non-overlapping ranges stay separate to preserve section
|
||||||
independence.
|
independence.
|
||||||
|
|
@ -36,6 +37,10 @@ from haiku.rag.store.repositories.document_item import DocumentItemRepository
|
||||||
_NOISE_LABELS = {"footnote", "page_header", "page_footer", "document_index"}
|
_NOISE_LABELS = {"footnote", "page_header", "page_footer", "document_index"}
|
||||||
_SECTION_BOUNDARY_LABELS = {"section_header", "title"}
|
_SECTION_BOUNDARY_LABELS = {"section_header", "title"}
|
||||||
|
|
||||||
|
# Labels whose pertinent unit is the item plus its own section: expansion
|
||||||
|
# never crosses section boundaries for these matches.
|
||||||
|
_TIGHT_LABELS = {"picture", "table"}
|
||||||
|
|
||||||
# Sections with fewer chars than this fraction of max_context_chars are
|
# Sections with fewer chars than this fraction of max_context_chars are
|
||||||
# considered too small — expansion falls through to item-by-item outward
|
# considered too small — expansion falls through to item-by-item outward
|
||||||
# growth, which naturally crosses into adjacent sections.
|
# growth, which naturally crosses into adjacent sections.
|
||||||
|
|
@ -99,16 +104,18 @@ def _evidence_anchors(content: str, max_chars: int) -> list[str]:
|
||||||
return anchors
|
return anchors
|
||||||
|
|
||||||
|
|
||||||
def _clip_to_budget(content: str, results: list[SearchResult], max_chars: int) -> str:
|
def _clip_window(
|
||||||
"""Clip expanded content to ``max_chars``, keeping the matched evidence.
|
content: str, results: list[SearchResult], max_chars: int
|
||||||
|
) -> tuple[int, int]:
|
||||||
|
"""Return the ``[start, end)`` char window to keep when clipping ``content``.
|
||||||
|
|
||||||
Anchors on the first locatable result in ``results`` order (the primary
|
Anchors on the first locatable result in ``results`` order (the primary
|
||||||
chunk that supplies the expanded result's identity) and returns a
|
chunk that supplies the expanded result's identity) and returns a
|
||||||
``max_chars``-wide window centered on it. Falls back to a prefix cut only
|
``max_chars``-wide window centered on it. Falls back to a prefix window only
|
||||||
when no anchor is locatable (heavy drift).
|
when no anchor is locatable (heavy drift).
|
||||||
"""
|
"""
|
||||||
if max_chars <= 0:
|
if max_chars <= 0:
|
||||||
return ""
|
return (0, 0)
|
||||||
evidence_start, evidence_len = -1, 0
|
evidence_start, evidence_len = -1, 0
|
||||||
for result in results:
|
for result in results:
|
||||||
for anchor in _evidence_anchors(result.content, max_chars):
|
for anchor in _evidence_anchors(result.content, max_chars):
|
||||||
|
|
@ -119,14 +126,45 @@ def _clip_to_budget(content: str, results: list[SearchResult], max_chars: int) -
|
||||||
if evidence_start != -1:
|
if evidence_start != -1:
|
||||||
break
|
break
|
||||||
if evidence_start == -1:
|
if evidence_start == -1:
|
||||||
return content[:max_chars]
|
return (0, min(len(content), max_chars))
|
||||||
center = evidence_start + evidence_len // 2
|
center = evidence_start + evidence_len // 2
|
||||||
start = max(0, center - max_chars // 2)
|
start = max(0, center - max_chars // 2)
|
||||||
end = min(len(content), start + max_chars)
|
end = min(len(content), start + max_chars)
|
||||||
start = max(0, end - max_chars)
|
start = max(0, end - max_chars)
|
||||||
|
return (start, end)
|
||||||
|
|
||||||
|
|
||||||
|
def _clip_to_budget(content: str, results: list[SearchResult], max_chars: int) -> str:
|
||||||
|
"""Clip expanded content to ``max_chars``, keeping the matched evidence."""
|
||||||
|
start, end = _clip_window(content, results, max_chars)
|
||||||
return content[start:end]
|
return content[start:end]
|
||||||
|
|
||||||
|
|
||||||
|
def _collect_meta(
|
||||||
|
spans: list[tuple[int, int, DocumentItem]],
|
||||||
|
) -> tuple[set[int], list[str], set[str]]:
|
||||||
|
"""Union the page numbers, refs, and labels of the given item spans."""
|
||||||
|
pages: set[int] = set()
|
||||||
|
refs: list[str] = []
|
||||||
|
labels: set[str] = set()
|
||||||
|
for _start, _end, item in spans:
|
||||||
|
refs.append(item.self_ref)
|
||||||
|
if item.label:
|
||||||
|
labels.add(item.label)
|
||||||
|
pages.update(item.page_numbers)
|
||||||
|
return pages, refs, labels
|
||||||
|
|
||||||
|
|
||||||
|
def _span_in_window(
|
||||||
|
span: tuple[int, int, DocumentItem], win_start: int, win_end: int
|
||||||
|
) -> bool:
|
||||||
|
"""Whether an item's char span overlaps the ``[win_start, win_end]`` clip window."""
|
||||||
|
start, end, _item = span
|
||||||
|
if start == end: # zero-width picture position
|
||||||
|
return win_start <= start <= win_end
|
||||||
|
return start < win_end and end > win_start
|
||||||
|
|
||||||
|
|
||||||
def _expand_outward(
|
def _expand_outward(
|
||||||
items: list[DocumentItem],
|
items: list[DocumentItem],
|
||||||
center_idx: int,
|
center_idx: int,
|
||||||
|
|
@ -222,6 +260,11 @@ def _find_expansion_range(
|
||||||
hi_bound=sec_end,
|
hi_bound=sec_end,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Picture/table hits stay section-bounded: their pertinent unit is the
|
||||||
|
# figure or table plus its section, never neighboring sections.
|
||||||
|
if any(items[i].label in _TIGHT_LABELS for i in matched_indices):
|
||||||
|
return (items[sec_start].position, items[sec_end].position)
|
||||||
|
|
||||||
# Section too small (e.g., title+authors) — expand across boundaries
|
# Section too small (e.g., title+authors) — expand across boundaries
|
||||||
return _expand_outward(items, center_idx, max_chars, skip_noise=True)
|
return _expand_outward(items, center_idx, max_chars, skip_noise=True)
|
||||||
|
|
||||||
|
|
@ -280,9 +323,11 @@ async def expand_with_items(
|
||||||
final_results: list[SearchResult] = []
|
final_results: list[SearchResult] = []
|
||||||
for range_start, range_end, original_results in merged:
|
for range_start, range_end, original_results in merged:
|
||||||
content_parts: list[str] = []
|
content_parts: list[str] = []
|
||||||
refs: list[str] = []
|
# Char span of each contributing item within the joined content, so
|
||||||
pages: set[int] = set()
|
# metadata can be narrowed to whatever survives a budget clip.
|
||||||
labels: set[str] = set()
|
item_spans: list[tuple[int, int, DocumentItem]] = []
|
||||||
|
cursor = 0
|
||||||
|
separator = "\n\n"
|
||||||
|
|
||||||
for pos in range(range_start, range_end + 1):
|
for pos in range(range_start, range_end + 1):
|
||||||
item = pos_to_item.get(pos)
|
item = pos_to_item.get(pos)
|
||||||
|
|
@ -291,56 +336,96 @@ async def expand_with_items(
|
||||||
if has_sections and item.label in _NOISE_LABELS:
|
if has_sections and item.label in _NOISE_LABELS:
|
||||||
continue
|
continue
|
||||||
if item.text:
|
if item.text:
|
||||||
|
if content_parts:
|
||||||
|
cursor += len(separator)
|
||||||
|
start = cursor
|
||||||
content_parts.append(item.text)
|
content_parts.append(item.text)
|
||||||
refs.append(item.self_ref)
|
cursor += len(item.text)
|
||||||
if item.label:
|
item_spans.append((start, cursor, item))
|
||||||
labels.add(item.label)
|
|
||||||
pages.update(item.page_numbers)
|
|
||||||
elif item.label == "picture":
|
elif item.label == "picture":
|
||||||
# Pictures may legitimately have empty text (no VLM
|
# Pictures may legitimately have empty text (no VLM
|
||||||
# description configured). Keep their self_ref so the
|
# description configured). Keep their self_ref so the
|
||||||
# downstream image_data lookup can still attach bytes.
|
# downstream image_data lookup can still attach bytes. They
|
||||||
refs.append(item.self_ref)
|
# occupy a zero-width position in reading order.
|
||||||
labels.add(item.label)
|
item_spans.append((cursor, cursor, item))
|
||||||
pages.update(item.page_numbers)
|
|
||||||
|
|
||||||
all_headings: list[str] = []
|
all_headings: list[str] = []
|
||||||
for r in original_results:
|
for r in original_results:
|
||||||
if r.headings:
|
if r.headings:
|
||||||
all_headings.extend(h for h in r.headings if h not in all_headings)
|
all_headings.extend(h for h in r.headings if h not in all_headings)
|
||||||
|
|
||||||
# Carry image_data and picture_captions through expansion so that
|
# Anchor identity (chunk_id, content/refs fallbacks) on the
|
||||||
# only pictures from the originally retrieved chunks get attached.
|
# best-scoring constituent — the chunk that earned the result its
|
||||||
# Pictures swept in by section expansion are referenced in `refs`
|
# rank — rather than whichever sits earliest in the document.
|
||||||
# for cross-referencing but their bytes are not re-fetched —
|
first = max(original_results, key=lambda r: r.score)
|
||||||
# otherwise a single search can balloon the response with adjacent
|
|
||||||
# figures the model did not actually retrieve.
|
chunk_ids: list[str] = []
|
||||||
|
for r in original_results:
|
||||||
|
if r.chunk_id and r.chunk_id not in chunk_ids:
|
||||||
|
chunk_ids.append(r.chunk_id)
|
||||||
|
|
||||||
|
joined = separator.join(content_parts)
|
||||||
|
|
||||||
|
# Expansion should never return less content than the original chunk.
|
||||||
|
# This can happen when item texts are fragmented (e.g., docling splits
|
||||||
|
# formatted HTML list items into many small text nodes); fall back to
|
||||||
|
# the chunk's own content, described by the chunk's own metadata.
|
||||||
|
if len(joined) < len(first.content):
|
||||||
|
base_content, base_spans = first.content, None
|
||||||
|
else:
|
||||||
|
base_content, base_spans = joined, item_spans
|
||||||
|
|
||||||
|
if len(base_content) > max_chars:
|
||||||
|
# Clip to the budget, anchored on the primary (highest-scoring)
|
||||||
|
# chunk, and narrow metadata to whatever survives the window.
|
||||||
|
win_start, win_end = _clip_window(
|
||||||
|
base_content, [first, *original_results], max_chars
|
||||||
|
)
|
||||||
|
expanded_content = base_content[win_start:win_end]
|
||||||
|
if base_spans is None:
|
||||||
|
pages, refs, labels = (
|
||||||
|
set(first.page_numbers),
|
||||||
|
list(first.doc_item_refs),
|
||||||
|
set(first.labels),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
pages, refs, labels = _collect_meta(
|
||||||
|
[s for s in base_spans if _span_in_window(s, win_start, win_end)]
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
expanded_content = base_content
|
||||||
|
if base_spans is None:
|
||||||
|
pages, refs, labels = (
|
||||||
|
set(first.page_numbers),
|
||||||
|
list(first.doc_item_refs),
|
||||||
|
set(first.labels),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
pages, refs, labels = _collect_meta(base_spans)
|
||||||
|
|
||||||
|
# Carry image_data and picture_captions from the originally retrieved
|
||||||
|
# chunks, but only for constituents whose refs survive the window — a
|
||||||
|
# chunk clipped out of the budget must not still ship its image to the
|
||||||
|
# model. Pictures swept in by section expansion are referenced in
|
||||||
|
# ``refs`` but their bytes are never re-fetched, so the multimodal
|
||||||
|
# payload stays bounded to what was actually retrieved and shown.
|
||||||
|
surviving_refs = set(refs)
|
||||||
merged_image_data: dict[str, str] = {}
|
merged_image_data: dict[str, str] = {}
|
||||||
merged_captions: dict[str, str] = {}
|
merged_captions: dict[str, str] = {}
|
||||||
for r in original_results:
|
for r in original_results:
|
||||||
|
if r.doc_item_refs and not surviving_refs.intersection(r.doc_item_refs):
|
||||||
|
continue
|
||||||
if r.image_data:
|
if r.image_data:
|
||||||
merged_image_data.update(r.image_data)
|
merged_image_data.update(r.image_data)
|
||||||
if r.picture_captions:
|
if r.picture_captions:
|
||||||
merged_captions.update(r.picture_captions)
|
merged_captions.update(r.picture_captions)
|
||||||
|
|
||||||
first = original_results[0]
|
|
||||||
|
|
||||||
# Expansion should never return less content than the original chunk.
|
|
||||||
# This can happen when item texts are fragmented (e.g., docling splits
|
|
||||||
# formatted HTML list items into many small text nodes).
|
|
||||||
expanded_content = "\n\n".join(content_parts)
|
|
||||||
if len(expanded_content) < len(first.content):
|
|
||||||
expanded_content = first.content
|
|
||||||
if len(expanded_content) > max_chars:
|
|
||||||
expanded_content = _clip_to_budget(
|
|
||||||
expanded_content, original_results, max_chars
|
|
||||||
)
|
|
||||||
|
|
||||||
final_results.append(
|
final_results.append(
|
||||||
SearchResult(
|
SearchResult(
|
||||||
content=expanded_content,
|
content=expanded_content,
|
||||||
score=max(r.score for r in original_results),
|
score=max(r.score for r in original_results),
|
||||||
chunk_id=first.chunk_id,
|
chunk_id=first.chunk_id,
|
||||||
|
chunk_ids=chunk_ids,
|
||||||
document_id=first.document_id,
|
document_id=first.document_id,
|
||||||
document_uri=first.document_uri,
|
document_uri=first.document_uri,
|
||||||
document_title=first.document_title,
|
document_title=first.document_title,
|
||||||
|
|
|
||||||
|
|
@ -57,14 +57,16 @@ class VisualGroundingModal(Screen):
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
chunk: "Chunk",
|
chunk: "Chunk | list[Chunk]",
|
||||||
client: "HaikuRAG",
|
client: "HaikuRAG",
|
||||||
document_uri: str | None = None,
|
document_uri: str | None = None,
|
||||||
|
refs: list[str] | None = None,
|
||||||
):
|
):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
self.chunk = chunk
|
self.chunks = chunk if isinstance(chunk, list) else [chunk]
|
||||||
self.client = client
|
self.client = client
|
||||||
self.document_uri = document_uri or chunk.document_uri
|
self.refs = refs
|
||||||
|
self.document_uri = document_uri or self.chunks[0].document_uri
|
||||||
self.images: list[PILImage] = []
|
self.images: list[PILImage] = []
|
||||||
self.current_page_idx = 0
|
self.current_page_idx = 0
|
||||||
self._image_widget: Widget = Static("Loading...", id="image-display")
|
self._image_widget: Widget = Static("Loading...", id="image-display")
|
||||||
|
|
@ -81,7 +83,7 @@ class VisualGroundingModal(Screen):
|
||||||
|
|
||||||
async def on_mount(self) -> None:
|
async def on_mount(self) -> None:
|
||||||
"""Load images and display the first page."""
|
"""Load images and display the first page."""
|
||||||
self.images = await self.client.visualize_chunk(self.chunk)
|
self.images = await self.client.visualize_chunk(self.chunks, self.refs)
|
||||||
await self._render_current_page()
|
await self._render_current_page()
|
||||||
|
|
||||||
async def _render_current_page(self) -> None:
|
async def _render_current_page(self) -> None:
|
||||||
|
|
|
||||||
|
|
@ -131,14 +131,19 @@ def create_skill_extras(
|
||||||
- 'visualize_chunk': returns visualizations for chunks in the database
|
- 'visualize_chunk': returns visualizations for chunks in the database
|
||||||
"""
|
"""
|
||||||
|
|
||||||
async def visualize_chunk(chunk_id: str) -> list:
|
async def visualize_chunk(chunk_id: str | list[str]) -> list:
|
||||||
from haiku.rag.client import HaikuRAG
|
from haiku.rag.client import HaikuRAG
|
||||||
|
|
||||||
|
chunk_ids = [chunk_id] if isinstance(chunk_id, str) else chunk_id
|
||||||
async with HaikuRAG(db_path, config=config, read_only=True) as rag:
|
async with HaikuRAG(db_path, config=config, read_only=True) as rag:
|
||||||
chunk = await rag.get_chunk_by_id(chunk_id)
|
chunks = []
|
||||||
if chunk is None:
|
for cid in chunk_ids:
|
||||||
|
chunk = await rag.get_chunk_by_id(cid)
|
||||||
|
if chunk is not None:
|
||||||
|
chunks.append(chunk)
|
||||||
|
if not chunks:
|
||||||
return []
|
return []
|
||||||
return await rag.visualize_chunk(chunk)
|
return await rag.visualize_chunk(chunks)
|
||||||
|
|
||||||
async def list_documents(
|
async def list_documents(
|
||||||
limit: int | None = None,
|
limit: int | None = None,
|
||||||
|
|
|
||||||
|
|
@ -124,11 +124,17 @@ class SearchResult(BaseModel):
|
||||||
when the caller asked to omit them via ``include_images=False`` on
|
when the caller asked to omit them via ``include_images=False`` on
|
||||||
``client.search``. Same shape is used everywhere — MCP, in-process search,
|
``client.search``. Same shape is used everywhere — MCP, in-process search,
|
||||||
agent toolsets — so non-vision callers see ``None`` and pay nothing.
|
agent toolsets — so non-vision callers see ``None`` and pay nothing.
|
||||||
|
|
||||||
|
``chunk_ids`` lists the ids of all chunks whose expansion ranges merged
|
||||||
|
into this result; empty means just ``chunk_id``. It lets citation
|
||||||
|
consumers (visual grounding) reproduce a merged expansion and is never
|
||||||
|
part of ``format_for_agent`` output.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
content: str
|
content: str
|
||||||
score: float
|
score: float
|
||||||
chunk_id: str | None = None
|
chunk_id: str | None = None
|
||||||
|
chunk_ids: list[str] = []
|
||||||
document_id: str | None = None
|
document_id: str | None = None
|
||||||
document_uri: str | None = None
|
document_uri: str | None = None
|
||||||
document_title: str | None = None
|
document_title: str | None = None
|
||||||
|
|
|
||||||
|
|
@ -18,16 +18,26 @@ class Citation(BaseModel):
|
||||||
cited chunk. Empty for text-only citations. UIs can fetch the picture
|
cited chunk. Empty for text-only citations. UIs can fetch the picture
|
||||||
bytes via ``DocumentItemRepository.get_picture_bytes(document_id, ref)``
|
bytes via ``DocumentItemRepository.get_picture_bytes(document_id, ref)``
|
||||||
and render them alongside the text content.
|
and render them alongside the text content.
|
||||||
|
|
||||||
|
``chunk_ids`` lists the ids of all chunks whose expansion ranges merged
|
||||||
|
into the cited result (always includes ``chunk_id``).
|
||||||
|
|
||||||
|
``doc_item_refs`` are the ``self_ref`` values of every item in the cited
|
||||||
|
content — the exact items the model saw. Visual grounding resolves bounding
|
||||||
|
boxes from them so the rendered pages match the citation precisely.
|
||||||
|
``picture_refs`` is the picture-labeled subset.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
index: int | None = None
|
index: int | None = None
|
||||||
document_id: str
|
document_id: str
|
||||||
chunk_id: str
|
chunk_id: str
|
||||||
|
chunk_ids: list[str] = Field(default_factory=list)
|
||||||
document_uri: str
|
document_uri: str
|
||||||
document_title: str | None = None
|
document_title: str | None = None
|
||||||
page_numbers: list[int] = Field(default_factory=list)
|
page_numbers: list[int] = Field(default_factory=list)
|
||||||
headings: list[str] | None = None
|
headings: list[str] | None = None
|
||||||
content: str
|
content: str
|
||||||
|
doc_item_refs: list[str] = Field(default_factory=list)
|
||||||
picture_refs: list[str] = Field(default_factory=list)
|
picture_refs: list[str] = Field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -51,11 +61,13 @@ def resolve_citations(
|
||||||
Citation(
|
Citation(
|
||||||
document_id=r.document_id or "",
|
document_id=r.document_id or "",
|
||||||
chunk_id=chunk_id,
|
chunk_id=chunk_id,
|
||||||
|
chunk_ids=r.chunk_ids or [chunk_id],
|
||||||
document_uri=r.document_uri or "",
|
document_uri=r.document_uri or "",
|
||||||
document_title=r.document_title,
|
document_title=r.document_title,
|
||||||
page_numbers=r.page_numbers,
|
page_numbers=r.page_numbers,
|
||||||
headings=r.headings,
|
headings=r.headings,
|
||||||
content=r.content,
|
content=r.content,
|
||||||
|
doc_item_refs=list(r.doc_item_refs),
|
||||||
picture_refs=picture_refs,
|
picture_refs=picture_refs,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
|
||||||
41
tests/store/test_citation.py
Normal file
41
tests/store/test_citation.py
Normal file
|
|
@ -0,0 +1,41 @@
|
||||||
|
from haiku.rag.store.models.chunk import SearchResult
|
||||||
|
from haiku.rag.store.models.citation import resolve_citations
|
||||||
|
|
||||||
|
|
||||||
|
def _result(chunk_id: str, chunk_ids: list[str] | None = None) -> SearchResult:
|
||||||
|
return SearchResult(
|
||||||
|
content="content",
|
||||||
|
score=0.9,
|
||||||
|
chunk_id=chunk_id,
|
||||||
|
chunk_ids=chunk_ids or [],
|
||||||
|
document_id="doc-1",
|
||||||
|
document_uri="test://doc",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_citations_copies_merged_chunk_ids():
|
||||||
|
result = _result("c1", chunk_ids=["c1", "c2"])
|
||||||
|
citations = resolve_citations(["c1"], [result])
|
||||||
|
assert len(citations) == 1
|
||||||
|
assert citations[0].chunk_id == "c1"
|
||||||
|
assert citations[0].chunk_ids == ["c1", "c2"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_citations_falls_back_to_chunk_id():
|
||||||
|
result = _result("c1")
|
||||||
|
citations = resolve_citations(["c1"], [result])
|
||||||
|
assert len(citations) == 1
|
||||||
|
assert citations[0].chunk_ids == ["c1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_citations_strips_brackets():
|
||||||
|
result = _result("c1")
|
||||||
|
citations = resolve_citations(["[c1]"], [result])
|
||||||
|
assert len(citations) == 1
|
||||||
|
assert citations[0].chunk_id == "c1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_citations_skips_unknown_ids():
|
||||||
|
result = _result("c1")
|
||||||
|
citations = resolve_citations(["c1", "missing"], [result])
|
||||||
|
assert len(citations) == 1
|
||||||
|
|
@ -1590,6 +1590,269 @@ async def test_client_visualize_chunk_multi_page(temp_db_path):
|
||||||
assert img.tobytes() != blank.tobytes()
|
assert img.tobytes() != blank.tobytes()
|
||||||
|
|
||||||
|
|
||||||
|
async def test_client_visualize_chunk_merged_chunks_union_pages(temp_db_path):
|
||||||
|
"""Visualizing all chunks of a merged result covers the union of their
|
||||||
|
expansions, which a single constituent chunk alone does not reach."""
|
||||||
|
from docling_core.types.doc.base import BoundingBox, Size
|
||||||
|
from docling_core.types.doc.document import (
|
||||||
|
DoclingDocument,
|
||||||
|
ImageRef,
|
||||||
|
ProvenanceItem,
|
||||||
|
)
|
||||||
|
from docling_core.types.doc.labels import DocItemLabel
|
||||||
|
from PIL import Image as PilImageModule
|
||||||
|
|
||||||
|
docling_doc = DoclingDocument(name="merged-viz-test")
|
||||||
|
page_size = Size(width=612.0, height=792.0)
|
||||||
|
for page_no in (1, 2):
|
||||||
|
docling_doc.add_page(
|
||||||
|
page_no=page_no,
|
||||||
|
size=page_size,
|
||||||
|
image=ImageRef.from_pil(
|
||||||
|
PilImageModule.new("RGB", (612, 792), color="white"), dpi=72
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Two sections, one per page, each large enough to be returned whole and
|
||||||
|
# under budget (so no cross-boundary expansion and no clip). A single
|
||||||
|
# chunk visualizes its own section's page; both chunks cover both pages.
|
||||||
|
layout = [
|
||||||
|
(DocItemLabel.SECTION_HEADER, "Section One", 1),
|
||||||
|
(DocItemLabel.PARAGRAPH, "Page one body. " + "x" * 3000, 1),
|
||||||
|
(DocItemLabel.SECTION_HEADER, "Section Two", 2),
|
||||||
|
(DocItemLabel.PARAGRAPH, "Page two body. " + "y" * 3000, 2),
|
||||||
|
]
|
||||||
|
for i, (label, text, page_no) in enumerate(layout):
|
||||||
|
docling_doc.add_text(
|
||||||
|
label=label,
|
||||||
|
text=text,
|
||||||
|
prov=ProvenanceItem(
|
||||||
|
page_no=page_no,
|
||||||
|
bbox=BoundingBox(l=50, t=700 - (i % 2) * 100, r=550, b=650),
|
||||||
|
charspan=(0, 20),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
chunks = [
|
||||||
|
Chunk(
|
||||||
|
content="Page one body. " + "x" * 3000,
|
||||||
|
metadata={
|
||||||
|
"doc_item_refs": ["#/texts/1"],
|
||||||
|
"page_numbers": [1],
|
||||||
|
"labels": ["paragraph"],
|
||||||
|
},
|
||||||
|
order=0,
|
||||||
|
embedding=[0.1] * 2560,
|
||||||
|
),
|
||||||
|
Chunk(
|
||||||
|
content="Page two body. " + "y" * 3000,
|
||||||
|
metadata={
|
||||||
|
"doc_item_refs": ["#/texts/3"],
|
||||||
|
"page_numbers": [2],
|
||||||
|
"labels": ["paragraph"],
|
||||||
|
},
|
||||||
|
order=1,
|
||||||
|
embedding=[0.1] * 2560,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||||
|
doc = await client.import_document(docling_doc, chunks, uri="test://merged")
|
||||||
|
|
||||||
|
stored_chunks = await client.chunk_repository.get_by_document_id(doc.id)
|
||||||
|
stored_chunks.sort(key=lambda c: c.order)
|
||||||
|
assert len(stored_chunks) == 2
|
||||||
|
c1, c2 = stored_chunks
|
||||||
|
|
||||||
|
solo_images = await client.visualize_chunk(c1)
|
||||||
|
assert len(solo_images) == 1
|
||||||
|
|
||||||
|
merged_images = await client.visualize_chunk([c1, c2])
|
||||||
|
assert len(merged_images) == 2
|
||||||
|
|
||||||
|
|
||||||
|
async def test_client_visualize_chunk_two_tone_highlights(temp_db_path):
|
||||||
|
"""Matched content draws stronger than context swept in by expansion."""
|
||||||
|
from docling_core.types.doc.base import BoundingBox, Size
|
||||||
|
from docling_core.types.doc.document import (
|
||||||
|
DoclingDocument,
|
||||||
|
ImageRef,
|
||||||
|
ProvenanceItem,
|
||||||
|
)
|
||||||
|
from docling_core.types.doc.labels import DocItemLabel
|
||||||
|
from PIL import Image as PilImageModule
|
||||||
|
|
||||||
|
docling_doc = DoclingDocument(name="two-tone-test")
|
||||||
|
page_size = Size(width=612.0, height=792.0)
|
||||||
|
docling_doc.add_page(
|
||||||
|
page_no=1,
|
||||||
|
size=page_size,
|
||||||
|
image=ImageRef.from_pil(
|
||||||
|
PilImageModule.new("RGB", (612, 792), color="white"), dpi=72
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
# Three small paragraphs; the chunk matches only the middle one, so
|
||||||
|
# expansion sweeps in its neighbors.
|
||||||
|
for i in range(3):
|
||||||
|
docling_doc.add_text(
|
||||||
|
label=DocItemLabel.PARAGRAPH,
|
||||||
|
text=f"Paragraph {i}.",
|
||||||
|
prov=ProvenanceItem(
|
||||||
|
page_no=1,
|
||||||
|
bbox=BoundingBox(l=50, t=700 - i * 100, r=550, b=650 - i * 100),
|
||||||
|
charspan=(0, 12),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
chunks = [
|
||||||
|
Chunk(
|
||||||
|
content="Paragraph 1.",
|
||||||
|
metadata={
|
||||||
|
"doc_item_refs": ["#/texts/1"],
|
||||||
|
"page_numbers": [1],
|
||||||
|
"labels": ["paragraph"],
|
||||||
|
},
|
||||||
|
order=0,
|
||||||
|
embedding=[0.1] * 2560,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||||
|
doc = await client.import_document(docling_doc, chunks, uri="test://two-tone")
|
||||||
|
stored_chunks = await client.chunk_repository.get_by_document_id(doc.id)
|
||||||
|
assert len(stored_chunks) == 1
|
||||||
|
|
||||||
|
images = await client.visualize_chunk(stored_chunks[0])
|
||||||
|
assert len(images) == 1
|
||||||
|
image = images[0]
|
||||||
|
|
||||||
|
# Page dpi 72 == document coords, bottom-left origin flipped to
|
||||||
|
# top-left: item i's box spans y = 92 + i * 100 .. 142 + i * 100.
|
||||||
|
matched = image.getpixel((300, 217)) # inside #/texts/1
|
||||||
|
swept = image.getpixel((300, 117)) # inside #/texts/0
|
||||||
|
background = image.getpixel((300, 30)) # outside all boxes
|
||||||
|
|
||||||
|
assert matched != background
|
||||||
|
assert swept != background
|
||||||
|
assert matched != swept
|
||||||
|
|
||||||
|
|
||||||
|
async def test_client_visualize_chunk_uses_given_refs(temp_db_path):
|
||||||
|
"""Explicit refs (the citation's doc_item_refs) restrict the visualization
|
||||||
|
to exactly those items, instead of re-expanding the chunk's context."""
|
||||||
|
from docling_core.types.doc.base import BoundingBox, Size
|
||||||
|
from docling_core.types.doc.document import (
|
||||||
|
DoclingDocument,
|
||||||
|
ImageRef,
|
||||||
|
ProvenanceItem,
|
||||||
|
)
|
||||||
|
from docling_core.types.doc.labels import DocItemLabel
|
||||||
|
from PIL import Image as PilImageModule
|
||||||
|
|
||||||
|
docling_doc = DoclingDocument(name="refs-test")
|
||||||
|
page_size = Size(width=612.0, height=792.0)
|
||||||
|
for page_no in (1, 2):
|
||||||
|
docling_doc.add_page(
|
||||||
|
page_no=page_no,
|
||||||
|
size=page_size,
|
||||||
|
image=ImageRef.from_pil(
|
||||||
|
PilImageModule.new("RGB", (612, 792), color="white"), dpi=72
|
||||||
|
),
|
||||||
|
)
|
||||||
|
docling_doc.add_text(
|
||||||
|
label=DocItemLabel.PARAGRAPH,
|
||||||
|
text="Content on page one.",
|
||||||
|
prov=ProvenanceItem(
|
||||||
|
page_no=1, bbox=BoundingBox(l=50, t=700, r=550, b=650), charspan=(0, 20)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
docling_doc.add_text(
|
||||||
|
label=DocItemLabel.PARAGRAPH,
|
||||||
|
text="Content on page two.",
|
||||||
|
prov=ProvenanceItem(
|
||||||
|
page_no=2, bbox=BoundingBox(l=50, t=700, r=550, b=650), charspan=(0, 20)
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
chunks = [
|
||||||
|
Chunk(
|
||||||
|
content="Content on page one.\nContent on page two.",
|
||||||
|
metadata={
|
||||||
|
"doc_item_refs": ["#/texts/0", "#/texts/1"],
|
||||||
|
"page_numbers": [1, 2],
|
||||||
|
"labels": ["paragraph", "paragraph"],
|
||||||
|
},
|
||||||
|
order=0,
|
||||||
|
embedding=[0.1] * 2560,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||||
|
doc = await client.import_document(docling_doc, chunks, uri="test://refs")
|
||||||
|
chunk = (await client.chunk_repository.get_by_document_id(doc.id))[0]
|
||||||
|
|
||||||
|
# No refs: re-expands the chunk's own refs → both pages.
|
||||||
|
assert len(await client.visualize_chunk(chunk)) == 2
|
||||||
|
# Given only the page-one ref → only page one is rendered.
|
||||||
|
assert len(await client.visualize_chunk(chunk, refs=["#/texts/0"])) == 1
|
||||||
|
|
||||||
|
|
||||||
|
async def test_client_visualize_chunk_no_expand_shows_only_chunk(temp_db_path):
|
||||||
|
"""expand=False draws only the chunk's own items, not the expanded section."""
|
||||||
|
from docling_core.types.doc.base import BoundingBox, Size
|
||||||
|
from docling_core.types.doc.document import (
|
||||||
|
DoclingDocument,
|
||||||
|
ImageRef,
|
||||||
|
ProvenanceItem,
|
||||||
|
)
|
||||||
|
from docling_core.types.doc.labels import DocItemLabel
|
||||||
|
from PIL import Image as PilImageModule
|
||||||
|
|
||||||
|
docling_doc = DoclingDocument(name="no-expand-test")
|
||||||
|
page_size = Size(width=612.0, height=792.0)
|
||||||
|
for page_no in (1, 2):
|
||||||
|
docling_doc.add_page(
|
||||||
|
page_no=page_no,
|
||||||
|
size=page_size,
|
||||||
|
image=ImageRef.from_pil(
|
||||||
|
PilImageModule.new("RGB", (612, 792), color="white"), dpi=72
|
||||||
|
),
|
||||||
|
)
|
||||||
|
for page_no in (1, 2):
|
||||||
|
docling_doc.add_text(
|
||||||
|
label=DocItemLabel.PARAGRAPH,
|
||||||
|
text=f"Short paragraph on page {page_no}.",
|
||||||
|
prov=ProvenanceItem(
|
||||||
|
page_no=page_no,
|
||||||
|
bbox=BoundingBox(l=50, t=700, r=550, b=650),
|
||||||
|
charspan=(0, 20),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
chunks = [
|
||||||
|
Chunk(
|
||||||
|
content="Short paragraph on page 1.",
|
||||||
|
metadata={
|
||||||
|
"doc_item_refs": ["#/texts/0"],
|
||||||
|
"page_numbers": [1],
|
||||||
|
"labels": ["paragraph"],
|
||||||
|
},
|
||||||
|
order=0,
|
||||||
|
embedding=[0.1] * 2560,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as client:
|
||||||
|
doc = await client.import_document(docling_doc, chunks, uri="test://no-expand")
|
||||||
|
chunk = (await client.chunk_repository.get_by_document_id(doc.id))[0]
|
||||||
|
|
||||||
|
# Default expands the chunk's context outward → reaches page two.
|
||||||
|
assert len(await client.visualize_chunk(chunk)) == 2
|
||||||
|
# expand=False draws only the chunk's own page-one item.
|
||||||
|
assert len(await client.visualize_chunk(chunk, expand=False)) == 1
|
||||||
|
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# convert() method tests
|
# convert() method tests
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|
|
||||||
|
|
@ -244,6 +244,46 @@ class TestFindExpansionRange:
|
||||||
# Should expand outward into the first section
|
# Should expand outward into the first section
|
||||||
assert hi >= 2
|
assert hi >= 2
|
||||||
|
|
||||||
|
def test_picture_in_small_section_stays_section_bounded(self):
|
||||||
|
items = [
|
||||||
|
_item(0, label="section_header", text="Chapter 1"),
|
||||||
|
_item(1, text="Chapter 1 prose. " * 100),
|
||||||
|
_item(2, label="section_header", text="Figure heading"),
|
||||||
|
_item(3, label="picture", text="Diagram description."),
|
||||||
|
_item(4, label="caption", text="Figure 2-3. Balance arm."),
|
||||||
|
_item(5, label="section_header", text="Chapter 3"),
|
||||||
|
_item(6, text="Chapter 3 prose. " * 100),
|
||||||
|
]
|
||||||
|
# Figure section (items 2-4) is far under 20% of 5000 chars.
|
||||||
|
lo, hi = _find_expansion_range(items, {3}, has_sections=True, max_chars=5000)
|
||||||
|
# Never crosses either header
|
||||||
|
assert (lo, hi) == (2, 4)
|
||||||
|
|
||||||
|
def test_table_in_small_section_stays_section_bounded(self):
|
||||||
|
items = [
|
||||||
|
_item(0, label="section_header", text="Chapter 1"),
|
||||||
|
_item(1, text="Chapter 1 prose. " * 100),
|
||||||
|
_item(2, label="section_header", text="Table heading"),
|
||||||
|
_item(3, label="table", text="Header | Value"),
|
||||||
|
_item(4, label="section_header", text="Chapter 3"),
|
||||||
|
_item(5, text="Chapter 3 prose. " * 100),
|
||||||
|
]
|
||||||
|
lo, hi = _find_expansion_range(items, {3}, has_sections=True, max_chars=5000)
|
||||||
|
assert (lo, hi) == (2, 3)
|
||||||
|
|
||||||
|
def test_text_in_small_section_still_expands_outward(self):
|
||||||
|
items = [
|
||||||
|
_item(0, label="section_header", text="Chapter 1"),
|
||||||
|
_item(1, text="Chapter 1 prose. " * 100),
|
||||||
|
_item(2, label="section_header", text="Short note"),
|
||||||
|
_item(3, text="A brief remark."),
|
||||||
|
_item(4, label="section_header", text="Chapter 3"),
|
||||||
|
_item(5, text="Chapter 3 prose. " * 100),
|
||||||
|
]
|
||||||
|
lo, hi = _find_expansion_range(items, {3}, has_sections=True, max_chars=5000)
|
||||||
|
# Text hits keep growing across section boundaries
|
||||||
|
assert lo < 2 or hi > 3
|
||||||
|
|
||||||
|
|
||||||
class TestEvidenceAnchors:
|
class TestEvidenceAnchors:
|
||||||
def test_empty_content(self):
|
def test_empty_content(self):
|
||||||
|
|
@ -369,6 +409,85 @@ class TestExpandWithItems:
|
||||||
# real content — so we get expanded content, not the fallback.
|
# real content — so we get expanded content, not the fallback.
|
||||||
assert len(expanded[0].content) > 0
|
assert len(expanded[0].content) > 0
|
||||||
|
|
||||||
|
async def test_picture_expansion_stays_within_section_pages(self, temp_db_path):
|
||||||
|
from haiku.rag.client import HaikuRAG
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as rag:
|
||||||
|
items = [
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=0,
|
||||||
|
self_ref="#/texts/0",
|
||||||
|
label="section_header",
|
||||||
|
text="Chapter 1",
|
||||||
|
page_numbers=[10],
|
||||||
|
),
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=1,
|
||||||
|
self_ref="#/texts/1",
|
||||||
|
label="text",
|
||||||
|
text="Chapter 1 prose. " * 200,
|
||||||
|
page_numbers=[10],
|
||||||
|
),
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=2,
|
||||||
|
self_ref="#/texts/2",
|
||||||
|
label="section_header",
|
||||||
|
text="Figure heading",
|
||||||
|
page_numbers=[13],
|
||||||
|
),
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=3,
|
||||||
|
self_ref="#/pictures/0",
|
||||||
|
label="picture",
|
||||||
|
text="Diagram description.",
|
||||||
|
page_numbers=[13],
|
||||||
|
),
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=4,
|
||||||
|
self_ref="#/texts/3",
|
||||||
|
label="caption",
|
||||||
|
text="Figure 2-3. Balance arm.",
|
||||||
|
page_numbers=[13],
|
||||||
|
),
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=5,
|
||||||
|
self_ref="#/texts/4",
|
||||||
|
label="section_header",
|
||||||
|
text="Chapter 3",
|
||||||
|
page_numbers=[15],
|
||||||
|
),
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=6,
|
||||||
|
self_ref="#/texts/5",
|
||||||
|
label="text",
|
||||||
|
text="Chapter 3 prose. " * 200,
|
||||||
|
page_numbers=[15],
|
||||||
|
),
|
||||||
|
]
|
||||||
|
await rag.document_item_repository.create_items("doc-1", items)
|
||||||
|
|
||||||
|
result = SearchResult(
|
||||||
|
content="Diagram description.",
|
||||||
|
score=0.9,
|
||||||
|
document_id="doc-1",
|
||||||
|
doc_item_refs=["#/pictures/0"],
|
||||||
|
page_numbers=[13],
|
||||||
|
)
|
||||||
|
expanded = await expand_with_items(
|
||||||
|
rag.document_item_repository, "doc-1", [result], 5000
|
||||||
|
)
|
||||||
|
assert len(expanded) == 1
|
||||||
|
assert expanded[0].page_numbers == [13]
|
||||||
|
assert "Chapter 1 prose" not in expanded[0].content
|
||||||
|
assert "Chapter 3 prose" not in expanded[0].content
|
||||||
|
|
||||||
async def test_fragmented_items_preserve_chunk(self, temp_db_path):
|
async def test_fragmented_items_preserve_chunk(self, temp_db_path):
|
||||||
"""When items are fragmented (e.g., list_item children), the original
|
"""When items are fragmented (e.g., list_item children), the original
|
||||||
chunk content is preserved if expansion produces less text."""
|
chunk content is preserved if expansion produces less text."""
|
||||||
|
|
@ -578,6 +697,187 @@ class TestExpandWithItems:
|
||||||
assert len(expanded[0].content) <= 5000
|
assert len(expanded[0].content) <= 5000
|
||||||
assert marker in expanded[0].content
|
assert marker in expanded[0].content
|
||||||
|
|
||||||
|
async def test_solo_result_carries_own_chunk_id(self, temp_db_path):
|
||||||
|
from haiku.rag.client import HaikuRAG
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as rag:
|
||||||
|
items = [
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=i,
|
||||||
|
self_ref=f"#/texts/{i}",
|
||||||
|
label="text",
|
||||||
|
text=f"Paragraph {i}. " * 10,
|
||||||
|
)
|
||||||
|
for i in range(5)
|
||||||
|
]
|
||||||
|
await rag.document_item_repository.create_items("doc-1", items)
|
||||||
|
|
||||||
|
result = SearchResult(
|
||||||
|
content="Paragraph 2.",
|
||||||
|
score=0.9,
|
||||||
|
chunk_id="c1",
|
||||||
|
document_id="doc-1",
|
||||||
|
doc_item_refs=["#/texts/2"],
|
||||||
|
)
|
||||||
|
expanded = await expand_with_items(
|
||||||
|
rag.document_item_repository, "doc-1", [result], 5000
|
||||||
|
)
|
||||||
|
assert len(expanded) == 1
|
||||||
|
assert expanded[0].chunk_ids == ["c1"]
|
||||||
|
|
||||||
|
async def test_merged_results_carry_all_chunk_ids(self, temp_db_path):
|
||||||
|
from haiku.rag.client import HaikuRAG
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as rag:
|
||||||
|
items = [
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=i,
|
||||||
|
self_ref=f"#/texts/{i}",
|
||||||
|
label="text",
|
||||||
|
text=f"Paragraph {i}. " * 10,
|
||||||
|
)
|
||||||
|
for i in range(5)
|
||||||
|
]
|
||||||
|
await rag.document_item_repository.create_items("doc-1", items)
|
||||||
|
|
||||||
|
r1 = SearchResult(
|
||||||
|
content="Paragraph 1.",
|
||||||
|
score=0.9,
|
||||||
|
chunk_id="c1",
|
||||||
|
document_id="doc-1",
|
||||||
|
doc_item_refs=["#/texts/1"],
|
||||||
|
)
|
||||||
|
r2 = SearchResult(
|
||||||
|
content="Paragraph 3.",
|
||||||
|
score=0.85,
|
||||||
|
chunk_id="c2",
|
||||||
|
document_id="doc-1",
|
||||||
|
doc_item_refs=["#/texts/3"],
|
||||||
|
)
|
||||||
|
expanded = await expand_with_items(
|
||||||
|
rag.document_item_repository, "doc-1", [r1, r2], 5000
|
||||||
|
)
|
||||||
|
# Ranges around positions 1 and 3 overlap → one merged result.
|
||||||
|
assert len(expanded) == 1
|
||||||
|
assert expanded[0].chunk_id == "c1"
|
||||||
|
assert expanded[0].chunk_ids == ["c1", "c2"]
|
||||||
|
# Sibling chunk ids are plumbing for visualization, never shown
|
||||||
|
# to the model.
|
||||||
|
assert "c2" not in expanded[0].format_for_agent()
|
||||||
|
|
||||||
|
async def test_merged_anchor_is_highest_scoring_constituent(self, temp_db_path):
|
||||||
|
"""A merged result's chunk_id anchors on the best-scoring constituent,
|
||||||
|
not whichever chunk sits earliest in the document."""
|
||||||
|
from haiku.rag.client import HaikuRAG
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as rag:
|
||||||
|
items = [
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=i,
|
||||||
|
self_ref=f"#/texts/{i}",
|
||||||
|
label="text",
|
||||||
|
text=f"Paragraph {i}. " * 10,
|
||||||
|
)
|
||||||
|
for i in range(5)
|
||||||
|
]
|
||||||
|
await rag.document_item_repository.create_items("doc-1", items)
|
||||||
|
|
||||||
|
# earlier in the document, lower score
|
||||||
|
r_early = SearchResult(
|
||||||
|
content="Paragraph 1.",
|
||||||
|
score=0.40,
|
||||||
|
chunk_id="c-early",
|
||||||
|
document_id="doc-1",
|
||||||
|
doc_item_refs=["#/texts/1"],
|
||||||
|
)
|
||||||
|
# later in the document, higher score — the real hit
|
||||||
|
r_best = SearchResult(
|
||||||
|
content="Paragraph 3.",
|
||||||
|
score=0.95,
|
||||||
|
chunk_id="c-best",
|
||||||
|
document_id="doc-1",
|
||||||
|
doc_item_refs=["#/texts/3"],
|
||||||
|
)
|
||||||
|
expanded = await expand_with_items(
|
||||||
|
rag.document_item_repository, "doc-1", [r_early, r_best], 5000
|
||||||
|
)
|
||||||
|
assert len(expanded) == 1
|
||||||
|
assert expanded[0].chunk_id == "c-best"
|
||||||
|
assert expanded[0].score == 0.95
|
||||||
|
# provenance still lists both
|
||||||
|
assert set(expanded[0].chunk_ids) == {"c-early", "c-best"}
|
||||||
|
|
||||||
|
async def test_clipped_merged_result_keeps_anchor_evidence_and_pages(
|
||||||
|
self, temp_db_path
|
||||||
|
):
|
||||||
|
"""When a merged result is clipped to budget, the surviving window is
|
||||||
|
centered on the anchor (highest-scoring) chunk, and page_numbers reflect
|
||||||
|
only the content that survived — not the full merged range."""
|
||||||
|
from haiku.rag.client import HaikuRAG
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as rag:
|
||||||
|
items = [
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=0,
|
||||||
|
self_ref="#/texts/0",
|
||||||
|
label="text",
|
||||||
|
text="LOWMARK " + "a" * 400,
|
||||||
|
page_numbers=[1],
|
||||||
|
),
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=1,
|
||||||
|
self_ref="#/texts/1",
|
||||||
|
label="text",
|
||||||
|
text="b" * 400,
|
||||||
|
page_numbers=[2],
|
||||||
|
),
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=2,
|
||||||
|
self_ref="#/texts/2",
|
||||||
|
label="text",
|
||||||
|
text="c" * 400 + " HIGHMARK",
|
||||||
|
page_numbers=[3],
|
||||||
|
),
|
||||||
|
]
|
||||||
|
await rag.document_item_repository.create_items("doc-1", items)
|
||||||
|
|
||||||
|
r_low = SearchResult(
|
||||||
|
content="LOWMARK " + "a" * 400,
|
||||||
|
score=0.4,
|
||||||
|
chunk_id="c-low",
|
||||||
|
document_id="doc-1",
|
||||||
|
doc_item_refs=["#/texts/0"],
|
||||||
|
page_numbers=[1],
|
||||||
|
)
|
||||||
|
r_high = SearchResult(
|
||||||
|
content="c" * 400 + " HIGHMARK",
|
||||||
|
score=0.9,
|
||||||
|
chunk_id="c-high",
|
||||||
|
document_id="doc-1",
|
||||||
|
doc_item_refs=["#/texts/2"],
|
||||||
|
page_numbers=[3],
|
||||||
|
)
|
||||||
|
expanded = await expand_with_items(
|
||||||
|
rag.document_item_repository, "doc-1", [r_low, r_high], 500
|
||||||
|
)
|
||||||
|
assert len(expanded) == 1
|
||||||
|
e = expanded[0]
|
||||||
|
# anchor is the high-scoring chunk, and its evidence survives clipping
|
||||||
|
assert e.chunk_id == "c-high"
|
||||||
|
assert "HIGHMARK" in e.content
|
||||||
|
assert "LOWMARK" not in e.content
|
||||||
|
# page_numbers reflect only the surviving window, not the full range
|
||||||
|
assert 3 in e.page_numbers
|
||||||
|
assert 1 not in e.page_numbers
|
||||||
|
# refs likewise exclude the clipped-out item
|
||||||
|
assert "#/texts/0" not in e.doc_item_refs
|
||||||
|
|
||||||
async def test_fuzzy_match_preserves_central_marker(self, temp_db_path):
|
async def test_fuzzy_match_preserves_central_marker(self, temp_db_path):
|
||||||
"""The chunk's text need not be verbatim in the joined item text: a clean
|
"""The chunk's text need not be verbatim in the joined item text: a clean
|
||||||
central marker is still located via the central-slice anchor."""
|
central marker is still located via the central-slice anchor."""
|
||||||
|
|
@ -715,3 +1015,61 @@ class TestExpandWithItemsPictureBytes:
|
||||||
"#/pictures/1": "A",
|
"#/pictures/1": "A",
|
||||||
"#/pictures/3": "B",
|
"#/pictures/3": "B",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async def test_clipped_out_picture_bytes_dropped(self, temp_db_path):
|
||||||
|
"""A lower-scoring picture chunk clipped out of the budget window no
|
||||||
|
longer contributes its image bytes — the model must not receive an
|
||||||
|
image the citation and visualization omit."""
|
||||||
|
from haiku.rag.client import HaikuRAG
|
||||||
|
|
||||||
|
async with HaikuRAG(temp_db_path, create=True) as rag:
|
||||||
|
items = [
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=0,
|
||||||
|
self_ref="#/pictures/0",
|
||||||
|
label="picture",
|
||||||
|
text="LOWPIC " + "a" * 400,
|
||||||
|
page_numbers=[1],
|
||||||
|
),
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=1,
|
||||||
|
self_ref="#/texts/1",
|
||||||
|
label="text",
|
||||||
|
text="b" * 400,
|
||||||
|
page_numbers=[2],
|
||||||
|
),
|
||||||
|
DocumentItem(
|
||||||
|
document_id="doc-1",
|
||||||
|
position=2,
|
||||||
|
self_ref="#/pictures/1",
|
||||||
|
label="picture",
|
||||||
|
text="c" * 400 + " HIGHPIC",
|
||||||
|
page_numbers=[3],
|
||||||
|
),
|
||||||
|
]
|
||||||
|
await rag.document_item_repository.create_items("doc-1", items)
|
||||||
|
|
||||||
|
r_low = SearchResult(
|
||||||
|
content="LOWPIC " + "a" * 400,
|
||||||
|
score=0.4,
|
||||||
|
chunk_id="c-low",
|
||||||
|
document_id="doc-1",
|
||||||
|
doc_item_refs=["#/pictures/0"],
|
||||||
|
image_data={"#/pictures/0": "LOWBYTES"},
|
||||||
|
)
|
||||||
|
r_high = SearchResult(
|
||||||
|
content="c" * 400 + " HIGHPIC",
|
||||||
|
score=0.9,
|
||||||
|
chunk_id="c-high",
|
||||||
|
document_id="doc-1",
|
||||||
|
doc_item_refs=["#/pictures/1"],
|
||||||
|
image_data={"#/pictures/1": "HIGHBYTES"},
|
||||||
|
)
|
||||||
|
expanded = await expand_with_items(
|
||||||
|
rag.document_item_repository, "doc-1", [r_low, r_high], 500
|
||||||
|
)
|
||||||
|
assert len(expanded) == 1
|
||||||
|
assert "#/pictures/0" not in expanded[0].doc_item_refs
|
||||||
|
assert expanded[0].image_data == {"#/pictures/1": "HIGHBYTES"}
|
||||||
|
|
|
||||||
42
tests/test_skill_extras.py
Normal file
42
tests/test_skill_extras.py
Normal file
|
|
@ -0,0 +1,42 @@
|
||||||
|
"""Tests for non-tool utilities from ``haiku.rag.skills._tools.create_skill_extras``."""
|
||||||
|
|
||||||
|
from docling_core.types.doc.document import DoclingDocument
|
||||||
|
|
||||||
|
from haiku.rag.client import HaikuRAG
|
||||||
|
from haiku.rag.config import AppConfig
|
||||||
|
from haiku.rag.skills._tools import create_skill_extras
|
||||||
|
from haiku.rag.store.models.chunk import Chunk
|
||||||
|
|
||||||
|
|
||||||
|
async def _seed_chunk(db_path) -> str:
|
||||||
|
docling_doc = DoclingDocument(name="extras-test")
|
||||||
|
chunk = Chunk(
|
||||||
|
content="Some content.",
|
||||||
|
metadata={"doc_item_refs": ["#/texts/0"], "page_numbers": [1]},
|
||||||
|
order=0,
|
||||||
|
embedding=[0.1] * 2560,
|
||||||
|
)
|
||||||
|
async with HaikuRAG(db_path, create=True) as client:
|
||||||
|
doc = await client.import_document(docling_doc, [chunk], uri="test://extras")
|
||||||
|
stored = await client.chunk_repository.get_by_document_id(doc.id)
|
||||||
|
return stored[0].id
|
||||||
|
|
||||||
|
|
||||||
|
async def test_extras_visualize_chunk_accepts_str_and_list(temp_db_path):
|
||||||
|
chunk_id = await _seed_chunk(temp_db_path)
|
||||||
|
extras = create_skill_extras(temp_db_path, AppConfig())
|
||||||
|
visualize_chunk = extras["visualize_chunk"]
|
||||||
|
|
||||||
|
# A document imported without page images yields no visualizations, but the
|
||||||
|
# str and list inputs must both resolve the chunk and reach visualize_chunk.
|
||||||
|
assert await visualize_chunk(chunk_id) == []
|
||||||
|
assert await visualize_chunk([chunk_id]) == []
|
||||||
|
|
||||||
|
|
||||||
|
async def test_extras_visualize_chunk_unknown_id_returns_empty(temp_db_path):
|
||||||
|
await _seed_chunk(temp_db_path)
|
||||||
|
extras = create_skill_extras(temp_db_path, AppConfig())
|
||||||
|
visualize_chunk = extras["visualize_chunk"]
|
||||||
|
|
||||||
|
assert await visualize_chunk("does-not-exist") == []
|
||||||
|
assert await visualize_chunk(["does-not-exist"]) == []
|
||||||
Loading…
Reference in a new issue