diff --git a/app/backend/main.py b/app/backend/main.py index 77cc1fd1..0074f8e6 100644 --- a/app/backend/main.py +++ b/app/backend/main.py @@ -258,12 +258,49 @@ async def db_info(_: Request) -> JSONResponse: ) +async def visualize_chunk(request: Request) -> JSONResponse: + """Return visual grounding images for a chunk as base64.""" + import base64 + from io import BytesIO + + chunk_id = request.path_params["chunk_id"] + + if not db_path.exists(): + return JSONResponse({"error": "Database not found"}, status_code=404) + + client = get_client(db_path) + + chunk = await client.chunk_repository.get_by_id(chunk_id) + if not chunk: + return JSONResponse({"error": "Chunk not found"}, status_code=404) + + images = await client.visualize_chunk(chunk) + if not images: + return JSONResponse({"images": [], "message": "No visual grounding available"}) + + base64_images = [] + for img in images: + buffer = BytesIO() + img.save(buffer, format="PNG") + buffer.seek(0) + base64_images.append(base64.b64encode(buffer.read()).decode("utf-8")) + + return JSONResponse( + { + "images": base64_images, + "chunk_id": chunk_id, + "document_uri": chunk.document_uri, + } + ) + + # Create Starlette app app = Starlette( routes=[ Route("/v1/chat/stream", stream_chat, methods=["POST"]), Route("/api/documents", list_documents, methods=["GET"]), Route("/api/info", db_info, methods=["GET"]), + Route("/api/visualize/{chunk_id}", visualize_chunk, methods=["GET"]), Route("/health", health_check, methods=["GET"]), ], middleware=[ diff --git a/app/frontend/app/api/visualize/[chunk_id]/route.ts b/app/frontend/app/api/visualize/[chunk_id]/route.ts new file mode 100644 index 00000000..81f297ee --- /dev/null +++ b/app/frontend/app/api/visualize/[chunk_id]/route.ts @@ -0,0 +1,22 @@ +import { NextResponse } from "next/server"; + +export async function GET( + _request: Request, + { params }: { params: Promise<{ chunk_id: string }> }, +) { + const { chunk_id } = await params; + const backendUrl = process.env.BACKEND_URL || "http://backend:8000"; + + try { + const response = await fetch(`${backendUrl}/api/visualize/${chunk_id}`); + const data = await response.json(); + + if (!response.ok) { + return NextResponse.json(data, { status: response.status }); + } + + return NextResponse.json(data); + } catch { + return NextResponse.json({ error: "Backend unavailable" }, { status: 503 }); + } +} diff --git a/app/frontend/app/globals.css b/app/frontend/app/globals.css index bdf9491e..4ce84c9a 100644 --- a/app/frontend/app/globals.css +++ b/app/frontend/app/globals.css @@ -69,7 +69,8 @@ a { padding: 0.15em 0.4em; border-radius: 4px; font-size: 0.9em; - font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; + font-family: + ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace; } .copilotKitAssistantMessage pre { diff --git a/app/frontend/components/CitationBlock.tsx b/app/frontend/components/CitationBlock.tsx index a24615a5..17d07ab1 100644 --- a/app/frontend/components/CitationBlock.tsx +++ b/app/frontend/components/CitationBlock.tsx @@ -1,6 +1,6 @@ "use client"; -import { useState } from "react"; +import { useCallback, useState } from "react"; interface Citation { index: number; @@ -17,7 +17,21 @@ interface CitationBlockProps { citations: Citation[]; } -function CitationItem({ citation }: { citation: Citation }) { +interface VisualGroundingState { + isOpen: boolean; + chunkId: string | null; + images: string[]; + loading: boolean; + error: string | null; +} + +function CitationItem({ + citation, + onViewInDocument, +}: { + citation: Citation; + onViewInDocument: (chunkId: string) => void; +}) { const [expanded, setExpanded] = useState(false); const title = citation.document_title || citation.document_uri || "Unknown"; @@ -48,6 +62,13 @@ function CitationItem({ citation }: { citation: Citation }) { )}
{citation.content}
+ )} @@ -55,6 +76,57 @@ function CitationItem({ citation }: { citation: Citation }) { } export default function CitationBlock({ citations }: CitationBlockProps) { + const [visualGrounding, setVisualGrounding] = useState({ + isOpen: false, + chunkId: null, + images: [], + loading: false, + error: null, + }); + + const fetchVisualGrounding = useCallback(async (chunkId: string) => { + setVisualGrounding({ + isOpen: true, + chunkId, + images: [], + loading: true, + error: null, + }); + + try { + const backendUrl = process.env.NEXT_PUBLIC_BACKEND_URL || ""; + const response = await fetch(`${backendUrl}/api/visualize/${chunkId}`); + const data = await response.json(); + + if (!response.ok) { + throw new Error(data.error || "Failed to fetch visual grounding"); + } + + setVisualGrounding((prev) => ({ + ...prev, + images: data.images || [], + loading: false, + error: data.images?.length === 0 ? data.message : null, + })); + } catch (err) { + setVisualGrounding((prev) => ({ + ...prev, + loading: false, + error: err instanceof Error ? err.message : "Unknown error", + })); + } + }, []); + + const closeVisualGrounding = useCallback(() => { + setVisualGrounding({ + isOpen: false, + chunkId: null, + images: [], + loading: false, + error: null, + }); + }, []); + if (!citations || citations.length === 0) { return null; } @@ -141,15 +213,157 @@ export default function CitationBlock({ citations }: CitationBlockProps) { max-height: 200px; overflow-y: auto; } + .citation-view-btn { + margin-top: 0.5rem; + padding: 0.25rem 0.5rem; + font-size: 0.75rem; + background: #3b82f6; + color: white; + border: none; + border-radius: 4px; + cursor: pointer; + transition: background 0.15s; + } + .citation-view-btn:hover { + background: #2563eb; + } + .visual-modal-overlay { + position: fixed; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(0, 0, 0, 0.75); + display: flex; + align-items: center; + justify-content: center; + z-index: 1000; + } + .visual-modal { + background: white; + border-radius: 8px; + padding: 1.5rem; + max-width: 90vw; + max-height: 90vh; + overflow: auto; + position: relative; + } + .visual-modal-close { + position: absolute; + top: 0.5rem; + right: 0.5rem; + background: #ef4444; + color: white; + border: none; + border-radius: 50%; + width: 2rem; + height: 2rem; + cursor: pointer; + font-size: 1rem; + display: flex; + align-items: center; + justify-content: center; + } + .visual-modal-close:hover { + background: #dc2626; + } + .visual-modal-title { + margin: 0 0 1rem 0; + font-size: 1.125rem; + color: #1e293b; + } + .visual-modal-loading { + padding: 2rem; + text-align: center; + color: #64748b; + } + .visual-modal-error { + padding: 1rem; + background: #fef2f2; + color: #dc2626; + border-radius: 4px; + } + .visual-modal-images { + display: flex; + flex-direction: column; + gap: 1rem; + } + .visual-modal-page-label { + font-size: 0.75rem; + color: #64748b; + margin-bottom: 0.5rem; + } + .visual-modal-image { + max-width: 100%; + border: 1px solid #e2e8f0; + border-radius: 4px; + } `}
Sources ({citations.length})
{citations.map((citation) => ( - + ))}
+ + {visualGrounding.isOpen && ( +
e.key === "Escape" && closeVisualGrounding()} + role="dialog" + aria-modal="true" + aria-label="Visual grounding" + > + {/* biome-ignore lint/a11y/noStaticElementInteractions: modal content wrapper */} +
e.stopPropagation()} + onKeyDown={(e) => e.stopPropagation()} + > + +

Visual Grounding

+ {visualGrounding.loading && ( +
Loading...
+ )} + {visualGrounding.error && ( +
{visualGrounding.error}
+ )} + {!visualGrounding.loading && + !visualGrounding.error && + visualGrounding.images.length > 0 && ( +
+ {visualGrounding.images.map((img, idx) => ( + // biome-ignore lint/suspicious/noArrayIndexKey: images have no stable id +
+
+ Page {idx + 1} of {visualGrounding.images.length} +
+ {/* biome-ignore lint/performance/noImgElement: base64 data URLs require img element */} + {`Page +
+ ))} +
+ )} +
+
+ )} ); }