diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b255e2c..88a9e373 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,45 @@ ## [0.20.0] - 2025-11-28 +### Added + +- **DoclingDocument Storage**: Full DoclingDocument JSON is now stored with each document, enabling rich context and visual grounding + - Documents store the complete DoclingDocument structure (JSON) and schema version + - Chunks store metadata with JSON pointer references (`doc_item_refs`), semantic labels, section headings, and page numbers + - New `ChunkMetadata` model for structured chunk provenance: `doc_item_refs`, `headings`, `labels`, `page_numbers` + - `Document.get_docling_document()` method to parse stored DoclingDocument + - `ChunkMetadata.resolve_doc_items()` to resolve JSON pointer refs to actual DocItem objects + - `ChunkMetadata.resolve_bounding_boxes()` for visual grounding with page coordinates + - LRU cache (100 documents) for parsed DoclingDocument objects to avoid repeated JSON parsing +- **Enhanced Search Results**: `search()` and `expand_context()` now return full provenance information + - `SearchResult` includes `page_numbers`, `headings`, `labels`, and `doc_item_refs` + - QA and research agents use provenance for better citations (page numbers, section headings) +- **Inspector Visual Grounding**: New visual grounding modal in the database inspector + - View page images with highlighted bounding boxes for chunks + - Keyboard navigation between pages (←/→ arrows) + - Access from both main detail view and search results + - Requires `textual-image` dependency + +### Changed + +- **BREAKING: Chunker Interface**: `DocumentChunker.chunk()` now returns `list[ChunkWithMetadata]` instead of `list[str]` + - `ChunkWithMetadata` combines chunk text with `ChunkMetadata` (refs, labels, headings, page_numbers) + - All chunker implementations updated: `DoclingLocalChunker`, `DoclingServeChunker` +- **Page Image Generation**: `generate_page_images=True` is now the default for local docling converter + - Required for visual grounding features + - docling-serve already generates page images by default +- **QA Prompts**: Updated to use page numbers and section headings in citations when available + +### Migration + +This release requires a database rebuild to populate the new DoclingDocument fields: + +```bash +haiku-rag rebuild +``` + +Existing documents without DoclingDocument data will work but won't have provenance information. The `rebuild` command re-processes all documents to populate the new fields. + ## [0.19.6] - 2025-12-03 ### Changed diff --git a/haiku_rag_slim/haiku/rag/inspector/app.py b/haiku_rag_slim/haiku/rag/inspector/app.py index 7b9939c1..e3101ed2 100644 --- a/haiku_rag_slim/haiku/rag/inspector/app.py +++ b/haiku_rag_slim/haiku/rag/inspector/app.py @@ -68,6 +68,7 @@ class InspectorApp(App): # type: ignore[misc] BINDINGS = [ Binding("q", "quit", "Quit", show=True), Binding("/", "search", "Search", show=True), + Binding("v", "show_visual", "Visual", show=True), ] def __init__(self, db_path: Path): @@ -93,9 +94,7 @@ class InspectorApp(App): # type: ignore[misc] doc_list = self.query_one(DocumentList) await doc_list.load_documents(self.client) - # Focus the document list view - if doc_list.list_view: - doc_list.list_view.focus() + doc_list.list_view.focus() async def on_unmount(self) -> None: """Clean up when unmounting.""" @@ -106,9 +105,8 @@ class InspectorApp(App): # type: ignore[misc] """Helper to select a chunk after refresh.""" for idx, c in enumerate(chunk_list.chunks): if c.id == chunk_id: - if chunk_list.list_view: - chunk_list.list_view.index = idx - chunk_list.list_view.focus() + chunk_list.list_view.index = idx + chunk_list.list_view.focus() break async def action_search(self) -> None: @@ -135,8 +133,7 @@ class InspectorApp(App): # type: ignore[misc] # Find and select the document for idx, d in enumerate(doc_list.documents): if d.id == chunk.document_id: - if doc_list.list_view: - doc_list.list_view.index = idx + doc_list.list_view.index = idx break # Load chunks for this document @@ -179,6 +176,43 @@ class InspectorApp(App): # type: ignore[misc] detail_view = self.query_one(DetailView) await detail_view.show_chunk(message.chunk) + async def action_show_visual(self) -> None: + """Show visual grounding for the currently selected chunk.""" + if not self.client: + return + + chunk_list = self.query_one(ChunkList) + idx = chunk_list.list_view.index + if idx is None or idx >= len(chunk_list.chunks): + return + + chunk = chunk_list.chunks[idx] + if not chunk.document_id: + return + + document = await self.client.get_document_by_id(chunk.document_id) + if not document: + return + + docling_doc = document.get_docling_document() + if not docling_doc: + self.notify("No DoclingDocument available", severity="warning") + return + + meta = chunk.get_chunk_metadata() + bounding_boxes = meta.resolve_bounding_boxes(docling_doc) + + from haiku.rag.inspector.widgets.visual_modal import VisualGroundingModal + + await self.push_screen( + VisualGroundingModal( + docling_document=docling_doc, + bounding_boxes=bounding_boxes, + page_numbers=meta.page_numbers, + document_uri=chunk.document_uri, + ) + ) + def run_inspector(db_path: Path | None = None) -> None: """Run the inspector TUI. diff --git a/haiku_rag_slim/haiku/rag/inspector/widgets/__init__.py b/haiku_rag_slim/haiku/rag/inspector/widgets/__init__.py index a1d02e12..6ec3de34 100644 --- a/haiku_rag_slim/haiku/rag/inspector/widgets/__init__.py +++ b/haiku_rag_slim/haiku/rag/inspector/widgets/__init__.py @@ -1,5 +1,6 @@ from haiku.rag.inspector.widgets.chunk_list import ChunkList from haiku.rag.inspector.widgets.detail_view import DetailView from haiku.rag.inspector.widgets.document_list import DocumentList +from haiku.rag.inspector.widgets.visual_modal import VisualGroundingModal -__all__ = ["ChunkList", "DetailView", "DocumentList"] +__all__ = ["ChunkList", "DetailView", "DocumentList", "VisualGroundingModal"] diff --git a/haiku_rag_slim/haiku/rag/inspector/widgets/chunk_list.py b/haiku_rag_slim/haiku/rag/inspector/widgets/chunk_list.py index 420f3c31..dec0ef12 100644 --- a/haiku_rag_slim/haiku/rag/inspector/widgets/chunk_list.py +++ b/haiku_rag_slim/haiku/rag/inspector/widgets/chunk_list.py @@ -23,36 +23,24 @@ class ChunkList(VerticalScroll): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) self.chunks: list[Chunk] = [] - self.list_view: ListView | None = None + self.list_view = ListView() def compose(self) -> ComposeResult: """Compose the chunk list.""" yield Static("[bold]Chunks[/bold]", classes="title") - self.list_view = ListView() yield self.list_view async def load_chunks_for_document( self, client: HaikuRAG, document_id: str ) -> None: - """Load chunks for a specific document. - - Args: - client: HaikuRAG client instance - document_id: ID of the document to load chunks for - """ - if self.list_view is None: - return - + """Load chunks for a specific document.""" self.chunks = await client.chunk_repository.get_by_document_id(document_id) - - # Clear existing items await self.list_view.clear() - - # Add chunk items for chunk in self.chunks: first_line = chunk.content.split("\n")[0] - item = ListItem(Static(f"[{chunk.order}] {first_line}")) - await self.list_view.append(item) + await self.list_view.append( + ListItem(Static(f"[{chunk.order}] {first_line}")) + ) @on(ListView.Highlighted) @on(ListView.Selected) @@ -60,8 +48,8 @@ class ChunkList(VerticalScroll): self, event: ListView.Highlighted | ListView.Selected ) -> None: """Handle chunk selection (arrow keys or Enter).""" - if event.list_view == self.list_view: - idx = event.list_view.index - if idx is not None and 0 <= idx < len(self.chunks): - chunk = self.chunks[idx] - self.post_message(self.ChunkSelected(chunk)) + if event.list_view != self.list_view: + return + idx = event.list_view.index + if idx is not None and 0 <= idx < len(self.chunks): + self.post_message(self.ChunkSelected(self.chunks[idx])) diff --git a/haiku_rag_slim/haiku/rag/inspector/widgets/detail_view.py b/haiku_rag_slim/haiku/rag/inspector/widgets/detail_view.py index c599075d..5b9c4e14 100644 --- a/haiku_rag_slim/haiku/rag/inspector/widgets/detail_view.py +++ b/haiku_rag_slim/haiku/rag/inspector/widgets/detail_view.py @@ -1,8 +1,19 @@ +from typing import Protocol + from textual.app import ComposeResult from textual.containers import VerticalScroll from textual.widgets import Markdown, Static -from haiku.rag.store.models import Chunk, Document +from haiku.rag.store.models import Chunk, Document, SearchResult + + +class ProvenanceData(Protocol): + """Protocol for objects that have provenance metadata.""" + + page_numbers: list[int] + headings: list[str] | None + labels: list[str] + doc_item_refs: list[str] class DetailView(VerticalScroll): @@ -12,81 +23,108 @@ class DetailView(VerticalScroll): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - self.title_widget: Static | None = None - self.content_widget: Markdown | None = None - - def compose(self) -> ComposeResult: - """Compose the detail view.""" self.title_widget = Static("[bold]Detail View[/bold]", classes="title") - yield self.title_widget self.content_widget = Markdown("") self.content_widget.can_focus = True + + def compose(self) -> ComposeResult: + yield self.title_widget yield self.content_widget + def _format_provenance(self, prov: ProvenanceData) -> list[str]: + """Format provenance metadata as markdown lines.""" + parts: list[str] = [] + if prov.page_numbers: + pages_str = ", ".join(str(p) for p in prov.page_numbers) + parts.append(f"**Page(s):** {pages_str}") + if prov.headings: + headings_str = " > ".join(prov.headings) + parts.append(f"**Section:** {headings_str}") + if prov.labels: + labels_str = ", ".join(prov.labels) + parts.append(f"**Labels:** {labels_str}") + if prov.doc_item_refs: + refs_str = ", ".join(prov.doc_item_refs[:5]) + if len(prov.doc_item_refs) > 5: + refs_str += f" ... (+{len(prov.doc_item_refs) - 5} more)" + parts.append(f"**DocItem Refs:** `{refs_str}`") + return parts + async def show_document(self, document: Document) -> None: - """Display document details. + """Display document details.""" + title = document.title or document.uri or "Untitled Document" + self.title_widget.update(f"[bold]Document: {title}[/bold]") - Args: - document: Document to display - """ - if self.title_widget and self.content_widget: - title = document.title or document.uri or "Untitled Document" - self.title_widget.update(f"[bold]Document: {title}[/bold]") + content_parts: list[str] = [] + if document.id: + content_parts.append(f"**ID:** `{document.id}`") + if document.uri: + content_parts.append(f"**URI:** `{document.uri}`") + if document.metadata: + metadata_str = "\n".join( + f" - {k}: {v}" for k, v in document.metadata.items() + ) + content_parts.append(f"**Metadata:**\n{metadata_str}") + if document.created_at: + content_parts.append(f"**Created:** {document.created_at}") + if document.updated_at: + content_parts.append(f"**Updated:** {document.updated_at}") - # Build markdown content - content_parts = [] + content_parts.append("\n---\n") + content_parts.append(document.content) - if document.id: - content_parts.append(f"**ID:** `{document.id}`") - if document.uri: - content_parts.append(f"**URI:** `{document.uri}`") - if document.metadata: - metadata_str = "\n".join( - f" - {k}: {v}" for k, v in document.metadata.items() - ) - content_parts.append(f"**Metadata:**\n{metadata_str}") - if document.created_at: - content_parts.append(f"**Created:** {document.created_at}") - if document.updated_at: - content_parts.append(f"**Updated:** {document.updated_at}") - - content_parts.append("\n---\n") - content_parts.append(document.content) - - await self.content_widget.update("\n\n".join(content_parts)) + await self.content_widget.update("\n\n".join(content_parts)) async def show_chunk(self, chunk: Chunk) -> None: - """Display chunk details. + """Display chunk details.""" + self.title_widget.update(f"[bold]Chunk {chunk.order}[/bold]") - Args: - chunk: Chunk to display - """ - if self.title_widget and self.content_widget: - self.title_widget.update(f"[bold]Chunk {chunk.order}[/bold]") + content_parts: list[str] = [] + if chunk.id: + content_parts.append(f"**ID:** `{chunk.id}`") + if chunk.document_id: + content_parts.append(f"**Document ID:** `{chunk.document_id}`") + if chunk.document_title: + content_parts.append(f"**Document Title:** {chunk.document_title}") + if chunk.document_uri: + content_parts.append(f"**Document URI:** `{chunk.document_uri}`") + content_parts.append(f"**Order:** {chunk.order}") - # Build markdown content - content_parts = [] + chunk_meta = chunk.get_chunk_metadata() + content_parts.extend(self._format_provenance(chunk_meta)) - if chunk.id: - content_parts.append(f"**ID:** `{chunk.id}`") - if chunk.document_id: - content_parts.append(f"**Document ID:** `{chunk.document_id}`") - if chunk.document_title: - content_parts.append(f"**Document Title:** {chunk.document_title}") - if chunk.document_uri: - content_parts.append(f"**Document URI:** `{chunk.document_uri}`") - content_parts.append(f"**Order:** {chunk.order}") - if chunk.metadata: - metadata_str = "\n".join( - f" - {k}: {v}" for k, v in chunk.metadata.items() - ) - content_parts.append(f"**Metadata:**\n{metadata_str}") - if chunk.embedding: - content_parts.append( - f"**Embedding:** {len(chunk.embedding)} dimensions" - ) + if chunk.embedding: + content_parts.append(f"**Embedding:** {len(chunk.embedding)} dimensions") - content_parts.append("\n---\n") - content_parts.append(chunk.content) + content_parts.append("\n---\n") + content_parts.append(chunk.content) - await self.content_widget.update("\n\n".join(content_parts)) + await self.content_widget.update("\n\n".join(content_parts)) + + async def show_search_result( + self, chunk: Chunk, search_result: SearchResult + ) -> None: + """Display chunk details with search result metadata.""" + self.title_widget.update(f"[bold]Chunk {chunk.order}[/bold]") + + content_parts: list[str] = [] + if chunk.id: + content_parts.append(f"**ID:** `{chunk.id}`") + if chunk.document_id: + content_parts.append(f"**Document ID:** `{chunk.document_id}`") + if search_result.document_title: + content_parts.append(f"**Document Title:** {search_result.document_title}") + if search_result.document_uri: + content_parts.append(f"**Document URI:** `{search_result.document_uri}`") + content_parts.append(f"**Order:** {chunk.order}") + content_parts.append(f"**Score:** {search_result.score:.4f}") + + content_parts.extend(self._format_provenance(search_result)) + + if chunk.embedding: + content_parts.append(f"**Embedding:** {len(chunk.embedding)} dimensions") + + content_parts.append("\n---\n") + content_parts.append(chunk.content) + + await self.content_widget.update("\n\n".join(content_parts)) diff --git a/haiku_rag_slim/haiku/rag/inspector/widgets/document_list.py b/haiku_rag_slim/haiku/rag/inspector/widgets/document_list.py index ec52d40d..e2f172b8 100644 --- a/haiku_rag_slim/haiku/rag/inspector/widgets/document_list.py +++ b/haiku_rag_slim/haiku/rag/inspector/widgets/document_list.py @@ -23,33 +23,20 @@ class DocumentList(VerticalScroll): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) self.documents: list[Document] = [] - self.list_view: ListView | None = None + self.list_view = ListView() def compose(self) -> ComposeResult: """Compose the document list.""" yield Static("[bold]Documents[/bold]", classes="title") - self.list_view = ListView() yield self.list_view async def load_documents(self, client: HaikuRAG) -> None: - """Load all documents from the database. - - Args: - client: HaikuRAG client instance - """ - if self.list_view is None: - return - + """Load all documents from the database.""" self.documents = await client.list_documents(limit=None) - - # Clear existing items await self.list_view.clear() - - # Add document items for doc in self.documents: title = doc.title or doc.uri or doc.id - item = ListItem(Static(f"{title}")) - await self.list_view.append(item) + await self.list_view.append(ListItem(Static(f"{title}"))) @on(ListView.Highlighted) @on(ListView.Selected) @@ -57,8 +44,8 @@ class DocumentList(VerticalScroll): self, event: ListView.Highlighted | ListView.Selected ) -> None: """Handle document selection (arrow keys or Enter).""" - if event.list_view == self.list_view: - idx = event.list_view.index - if idx is not None and 0 <= idx < len(self.documents): - document = self.documents[idx] - self.post_message(self.DocumentSelected(document)) + if event.list_view != self.list_view: + return + idx = event.list_view.index + if idx is not None and 0 <= idx < len(self.documents): + self.post_message(self.DocumentSelected(self.documents[idx])) diff --git a/haiku_rag_slim/haiku/rag/inspector/widgets/search_modal.py b/haiku_rag_slim/haiku/rag/inspector/widgets/search_modal.py index 7ef8e256..1cdbc8a8 100644 --- a/haiku_rag_slim/haiku/rag/inspector/widgets/search_modal.py +++ b/haiku_rag_slim/haiku/rag/inspector/widgets/search_modal.py @@ -8,7 +8,7 @@ from textual.widgets import Input, ListItem, ListView, Static from haiku.rag.client import HaikuRAG from haiku.rag.inspector.widgets.detail_view import DetailView -from haiku.rag.store.models import Chunk +from haiku.rag.store.models import Chunk, SearchResult class SearchModal(Screen): @@ -16,6 +16,7 @@ class SearchModal(Screen): BINDINGS = [ Binding("escape", "dismiss", "Close", show=True), + Binding("v", "show_visual", "Visual", show=True), ] CSS = """ @@ -65,6 +66,7 @@ class SearchModal(Screen): super().__init__() self.client = client self.chunks: list[Chunk] = [] + self.search_results: list[SearchResult] = [] def compose(self) -> ComposeResult: """Compose the search screen.""" @@ -99,29 +101,42 @@ class SearchModal(Screen): status_label.update("Searching...") try: - # Perform search - results = await self.client.chunk_repository.search( - query=query, limit=50, search_type="hybrid" - ) + # Perform search using client API + self.search_results = await self.client.search(query=query, limit=50) - self.chunks = [chunk for chunk, _score in results] + # Get chunks for the results + self.chunks = [] + for result in self.search_results: + if result.chunk_id: + chunk = await self.client.chunk_repository.get_by_id( + result.chunk_id + ) + if chunk: + self.chunks.append(chunk) # Clear and populate results await list_view.clear() - for chunk, score in results: - first_line = chunk.content.split("\n")[0] - score_str = f"{score:.2f}" if score else "N/A" - item = ListItem(Static(f"[{score_str}] {first_line}")) + for result in self.search_results: + first_line = result.content.split("\n")[0][:60] + score_str = f"{result.score:.2f}" + # Add page info if available + page_info = "" + if result.page_numbers: + pages = ", ".join(str(p) for p in result.page_numbers[:3]) + page_info = f" (p.{pages})" + item = ListItem(Static(f"[{score_str}]{page_info} {first_line}")) await list_view.append(item) # Update status status_label.update(f"Found {len(self.chunks)} results") # Select first result, show in detail view, and focus list - if self.chunks: + if self.chunks and self.search_results: list_view.index = 0 detail_view = self.query_one("#search-detail", DetailView) - await detail_view.show_chunk(self.chunks[0]) + await detail_view.show_search_result( + self.chunks[0], self.search_results[0] + ) list_view.focus() except Exception as e: status_label.update(f"Error: {str(e)}") @@ -129,23 +144,61 @@ class SearchModal(Screen): async def on_list_view_highlighted(self, event: ListView.Highlighted) -> None: """Handle chunk navigation (arrow keys).""" list_view = self.query_one("#search-results", ListView) - if event.list_view == list_view and event.item is not None: - idx = event.list_view.index - if idx is not None and 0 <= idx < len(self.chunks): - chunk = self.chunks[idx] - detail_view = self.query_one("#search-detail", DetailView) - await detail_view.show_chunk(chunk) + if event.list_view != list_view or event.item is None: + return + idx = event.list_view.index + detail_view = self.query_one("#search-detail", DetailView) + await detail_view.show_search_result(self.chunks[idx], self.search_results[idx]) # type: ignore[index] async def on_list_view_selected(self, event: ListView.Selected) -> None: """Handle chunk selection (Enter key).""" list_view = self.query_one("#search-results", ListView) - if event.list_view == list_view: - idx = event.list_view.index - if idx is not None and 0 <= idx < len(self.chunks): - chunk = self.chunks[idx] - self.post_message(self.ChunkSelected(chunk)) - self.app.pop_screen() + if event.list_view != list_view: + return + idx = event.list_view.index + self.post_message(self.ChunkSelected(self.chunks[idx])) # type: ignore[index] + self.app.pop_screen() async def action_dismiss(self, result=None) -> None: - """Close the search screen.""" self.app.pop_screen() + + async def action_show_visual(self) -> None: + """Show visual grounding for the current chunk.""" + list_view = self.query_one("#search-results", ListView) + idx = list_view.index + if idx is None or not self.search_results: + return + + search_result = self.search_results[idx] + if not search_result.document_id or not search_result.chunk_id: + return + + status_label = self.query_one("#status-label", Static) + + document = await self.client.get_document_by_id(search_result.document_id) + if not document: + status_label.update("[yellow]Document not found[/yellow]") + return + + docling_doc = document.get_docling_document() + if not docling_doc: + status_label.update( + "[yellow]No DoclingDocument available for visual[/yellow]" + ) + return + + chunk = await self.client.chunk_repository.get_by_id(search_result.chunk_id) + bounding_boxes = [] + if chunk: + meta = chunk.get_chunk_metadata() + bounding_boxes = meta.resolve_bounding_boxes(docling_doc) + + from haiku.rag.inspector.widgets.visual_modal import VisualGroundingModal + + modal = VisualGroundingModal( + docling_document=docling_doc, + bounding_boxes=bounding_boxes, + page_numbers=search_result.page_numbers, + document_uri=search_result.document_uri, + ) + await self.app.push_screen(modal) diff --git a/haiku_rag_slim/haiku/rag/inspector/widgets/visual_modal.py b/haiku_rag_slim/haiku/rag/inspector/widgets/visual_modal.py new file mode 100644 index 00000000..b3078d15 --- /dev/null +++ b/haiku_rag_slim/haiku/rag/inspector/widgets/visual_modal.py @@ -0,0 +1,187 @@ +from copy import deepcopy +from typing import TYPE_CHECKING + +from textual.app import ComposeResult +from textual.binding import Binding +from textual.containers import Horizontal, Vertical +from textual.screen import Screen +from textual.widget import Widget +from textual.widgets import Static +from textual_image.widget import Image as TextualImage + +from haiku.rag.store.models import BoundingBox + +if TYPE_CHECKING: + from docling_core.types.doc.document import DoclingDocument + from PIL.Image import Image as PILImage + + +class VisualGroundingModal(Screen): + """Modal screen for displaying visual grounding with bounding boxes.""" + + BINDINGS = [ + Binding("escape", "dismiss", "Close", show=True), + Binding("left", "prev_page", "Previous Page"), + Binding("right", "next_page", "Next Page"), + ] + + CSS = """ + VisualGroundingModal { + background: $surface; + layout: vertical; + } + + #visual-header { + dock: top; + height: auto; + padding: 1; + } + + #visual-content { + height: 1fr; + width: 100%; + align: center middle; + } + + #visual-content Image { + width: auto; + height: 100%; + } + + #page-nav { + dock: bottom; + height: auto; + padding: 1; + } + """ + + def __init__( + self, + docling_document: "DoclingDocument", + bounding_boxes: list[BoundingBox], + page_numbers: list[int], + document_uri: str | None = None, + ): + super().__init__() + self.docling_document = docling_document + self.document_uri = document_uri + self.bounding_boxes = bounding_boxes + self.page_numbers = sorted(set(page_numbers)) if page_numbers else [] + self.current_page_idx = 0 + self._image_widget: Widget = Static("Loading...", id="image-display") + self._page_info = Static("", id="page-info") + + def compose(self) -> ComposeResult: + uri_display = self.document_uri or self.docling_document.name or "Document" + with Vertical(id="visual-header"): + yield Static(f"[bold]Visual Grounding[/bold] - {uri_display}") + with Horizontal(id="visual-content"): + yield self._image_widget + with Horizontal(id="page-nav"): + yield self._page_info + + async def on_mount(self) -> None: + """Load and display the first page.""" + await self._render_current_page() + + async def _render_current_page(self) -> None: + """Render the current page with bounding boxes.""" + if not self.page_numbers: + if isinstance(self._image_widget, Static): + self._image_widget.update("[red]No page information available[/red]") + self._page_info.update("") + return + + current_page = self.page_numbers[self.current_page_idx] + self._page_info.update( + f"Page {current_page} " + f"({self.current_page_idx + 1}/{len(self.page_numbers)}) " + f"- Use ←/→ to navigate" + ) + + try: + image = self._render_page_with_boxes(current_page) + if image: + new_widget = TextualImage(image, id="rendered-image") + await self._image_widget.remove() + content = self.query_one("#visual-content", Horizontal) + await content.mount(new_widget) + self._image_widget = new_widget + elif isinstance(self._image_widget, Static): + self._image_widget.update( + "[yellow]No page image available[/yellow]\n" + "This document was converted without page images." + ) + except Exception as e: + if isinstance(self._image_widget, Static): + self._image_widget.update(f"[red]Error: {e}[/red]") + + def _render_page_with_boxes(self, page_no: int) -> "PILImage | None": + """Render a page from DoclingDocument with bounding boxes.""" + from PIL import ImageDraw + + # Get the page from DoclingDocument + if page_no not in self.docling_document.pages: + return None + + page = self.docling_document.pages[page_no] + if page.image is None: + return None + + pil_image = page.image.pil_image + if pil_image is None: + return None + + # Get page dimensions + page_height = page.size.height + + # Calculate scale factor (image pixels vs document coordinates) + scale_x = pil_image.width / page.size.width + scale_y = pil_image.height / page.size.height + + # Get bounding boxes for this page + page_boxes = [bb for bb in self.bounding_boxes if bb.page_no == page_no] + + if page_boxes: + # Draw bounding boxes + image = deepcopy(pil_image) + draw = ImageDraw.Draw(image, "RGBA") + + for bbox in page_boxes: + # Convert from document coordinates to image coordinates + # Document coords are typically bottom-left origin + # PIL uses top-left origin + x0 = bbox.left * scale_x + y0 = (page_height - bbox.top) * scale_y # Flip Y + x1 = bbox.right * scale_x + y1 = (page_height - bbox.bottom) * scale_y # Flip Y + + # Ensure proper ordering (y0 should be less than y1 for PIL) + if y0 > y1: + y0, y1 = y1, y0 + + # Draw filled rectangle with transparency + fill_color = (255, 255, 0, 80) # Yellow with transparency + outline_color = (255, 165, 0, 255) # Orange outline + + draw.rectangle([(x0, y0), (x1, y1)], fill=fill_color, outline=None) + draw.rectangle([(x0, y0), (x1, y1)], outline=outline_color, width=3) + + return image + + return pil_image + + async def action_dismiss(self, result=None) -> None: + self.app.pop_screen() + + async def action_prev_page(self) -> None: + """Navigate to the previous page.""" + if self.current_page_idx > 0: + self.current_page_idx -= 1 + await self._render_current_page() + + async def action_next_page(self) -> None: + """Navigate to the next page.""" + if self.current_page_idx < len(self.page_numbers) - 1: + self.current_page_idx += 1 + await self._render_current_page() diff --git a/haiku_rag_slim/pyproject.toml b/haiku_rag_slim/pyproject.toml index 6b304761..6c401a4b 100644 --- a/haiku_rag_slim/pyproject.toml +++ b/haiku_rag_slim/pyproject.toml @@ -45,7 +45,7 @@ mxbai = ["mxbai-rerank>=0.1.6"] cohere = ["cohere>=5.20.0"] zeroentropy = ["zeroentropy>=0.1.0a6"] # Inspector TUI -inspector = ["textual>=1.0.0"] +inspector = ["textual>=1.0.0", "textual-image>=0.8.1"] # Model providers (delegated to pydantic-ai-slim) anthropic = ["pydantic-ai-slim[anthropic]"] groq = ["pydantic-ai-slim[groq]"] diff --git a/uv.lock b/uv.lock index ae96a4b5..9ae5ccc0 100644 --- a/uv.lock +++ b/uv.lock @@ -1371,6 +1371,7 @@ groq = [ ] inspector = [ { name = "textual" }, + { name = "textual-image" }, ] mistral = [ { name = "pydantic-ai-slim", extra = ["mistral"] }, @@ -1410,6 +1411,7 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6.0.3" }, { name = "rich", specifier = ">=14.2.0" }, { name = "textual", marker = "extra == 'inspector'", specifier = ">=1.0.0" }, + { name = "textual-image", marker = "extra == 'inspector'", specifier = ">=0.8.1" }, { name = "typer", specifier = ">=0.19.2,<0.20.0" }, { name = "voyageai", marker = "extra == 'voyageai'", specifier = ">=0.3.5" }, { name = "watchfiles", specifier = ">=1.1.1" }, @@ -4671,6 +4673,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/53/b3/95ab646b0c908823d71e49ab8b5949ec9f33346cee3897d1af6be28a8d91/textual-6.6.0-py3-none-any.whl", hash = "sha256:5a9484bd15ee8a6fd8ac4ed4849fb25ee56bed2cecc7b8a83c4cd7d5f19515e5", size = 712606, upload-time = "2025-11-10T17:49:58.391Z" }, ] +[[package]] +name = "textual-image" +version = "0.8.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pillow" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f7/3e/807c5a449e9d99ba3b860acf5b83cf1da7ac46477bfe0e9e4d0149b8ed90/textual_image-0.8.4.tar.gz", hash = "sha256:d13f960da07659cfac9d9e417ca7057b3ac0c17a7827ae8e47c3b164d43776fc", size = 109056, upload-time = "2025-09-02T19:09:11.895Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5b/0e/2c3c2972ee810595089d85a51107f45e41c0642f645c24675951c69fd648/textual_image-0.8.4-py3-none-any.whl", hash = "sha256:0f0256993348f5af619c930a4839ea190525a22a56e8d69e1cf0f8e32d59fa3b", size = 109608, upload-time = "2025-09-02T19:09:10.707Z" }, +] + [[package]] name = "tifffile" version = "2025.10.16"