diff --git a/app/backend/main.py b/app/backend/main.py
index 77cc1fd1..0074f8e6 100644
--- a/app/backend/main.py
+++ b/app/backend/main.py
@@ -258,12 +258,49 @@ async def db_info(_: Request) -> JSONResponse:
)
+async def visualize_chunk(request: Request) -> JSONResponse:
+ """Return visual grounding images for a chunk as base64."""
+ import base64
+ from io import BytesIO
+
+ chunk_id = request.path_params["chunk_id"]
+
+ if not db_path.exists():
+ return JSONResponse({"error": "Database not found"}, status_code=404)
+
+ client = get_client(db_path)
+
+ chunk = await client.chunk_repository.get_by_id(chunk_id)
+ if not chunk:
+ return JSONResponse({"error": "Chunk not found"}, status_code=404)
+
+ images = await client.visualize_chunk(chunk)
+ if not images:
+ return JSONResponse({"images": [], "message": "No visual grounding available"})
+
+ base64_images = []
+ for img in images:
+ buffer = BytesIO()
+ img.save(buffer, format="PNG")
+ buffer.seek(0)
+ base64_images.append(base64.b64encode(buffer.read()).decode("utf-8"))
+
+ return JSONResponse(
+ {
+ "images": base64_images,
+ "chunk_id": chunk_id,
+ "document_uri": chunk.document_uri,
+ }
+ )
+
+
# Create Starlette app
app = Starlette(
routes=[
Route("/v1/chat/stream", stream_chat, methods=["POST"]),
Route("/api/documents", list_documents, methods=["GET"]),
Route("/api/info", db_info, methods=["GET"]),
+ Route("/api/visualize/{chunk_id}", visualize_chunk, methods=["GET"]),
Route("/health", health_check, methods=["GET"]),
],
middleware=[
diff --git a/app/frontend/app/api/visualize/[chunk_id]/route.ts b/app/frontend/app/api/visualize/[chunk_id]/route.ts
new file mode 100644
index 00000000..81f297ee
--- /dev/null
+++ b/app/frontend/app/api/visualize/[chunk_id]/route.ts
@@ -0,0 +1,22 @@
+import { NextResponse } from "next/server";
+
+export async function GET(
+ _request: Request,
+ { params }: { params: Promise<{ chunk_id: string }> },
+) {
+ const { chunk_id } = await params;
+ const backendUrl = process.env.BACKEND_URL || "http://backend:8000";
+
+ try {
+ const response = await fetch(`${backendUrl}/api/visualize/${chunk_id}`);
+ const data = await response.json();
+
+ if (!response.ok) {
+ return NextResponse.json(data, { status: response.status });
+ }
+
+ return NextResponse.json(data);
+ } catch {
+ return NextResponse.json({ error: "Backend unavailable" }, { status: 503 });
+ }
+}
diff --git a/app/frontend/app/globals.css b/app/frontend/app/globals.css
index bdf9491e..4ce84c9a 100644
--- a/app/frontend/app/globals.css
+++ b/app/frontend/app/globals.css
@@ -69,7 +69,8 @@ a {
padding: 0.15em 0.4em;
border-radius: 4px;
font-size: 0.9em;
- font-family: ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace;
+ font-family:
+ ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace;
}
.copilotKitAssistantMessage pre {
diff --git a/app/frontend/components/CitationBlock.tsx b/app/frontend/components/CitationBlock.tsx
index a24615a5..17d07ab1 100644
--- a/app/frontend/components/CitationBlock.tsx
+++ b/app/frontend/components/CitationBlock.tsx
@@ -1,6 +1,6 @@
"use client";
-import { useState } from "react";
+import { useCallback, useState } from "react";
interface Citation {
index: number;
@@ -17,7 +17,21 @@ interface CitationBlockProps {
citations: Citation[];
}
-function CitationItem({ citation }: { citation: Citation }) {
+interface VisualGroundingState {
+ isOpen: boolean;
+ chunkId: string | null;
+ images: string[];
+ loading: boolean;
+ error: string | null;
+}
+
+function CitationItem({
+ citation,
+ onViewInDocument,
+}: {
+ citation: Citation;
+ onViewInDocument: (chunkId: string) => void;
+}) {
const [expanded, setExpanded] = useState(false);
const title = citation.document_title || citation.document_uri || "Unknown";
@@ -48,6 +62,13 @@ function CitationItem({ citation }: { citation: Citation }) {
)}
{citation.content}
+
)}
@@ -55,6 +76,57 @@ function CitationItem({ citation }: { citation: Citation }) {
}
export default function CitationBlock({ citations }: CitationBlockProps) {
+ const [visualGrounding, setVisualGrounding] = useState({
+ isOpen: false,
+ chunkId: null,
+ images: [],
+ loading: false,
+ error: null,
+ });
+
+ const fetchVisualGrounding = useCallback(async (chunkId: string) => {
+ setVisualGrounding({
+ isOpen: true,
+ chunkId,
+ images: [],
+ loading: true,
+ error: null,
+ });
+
+ try {
+ const backendUrl = process.env.NEXT_PUBLIC_BACKEND_URL || "";
+ const response = await fetch(`${backendUrl}/api/visualize/${chunkId}`);
+ const data = await response.json();
+
+ if (!response.ok) {
+ throw new Error(data.error || "Failed to fetch visual grounding");
+ }
+
+ setVisualGrounding((prev) => ({
+ ...prev,
+ images: data.images || [],
+ loading: false,
+ error: data.images?.length === 0 ? data.message : null,
+ }));
+ } catch (err) {
+ setVisualGrounding((prev) => ({
+ ...prev,
+ loading: false,
+ error: err instanceof Error ? err.message : "Unknown error",
+ }));
+ }
+ }, []);
+
+ const closeVisualGrounding = useCallback(() => {
+ setVisualGrounding({
+ isOpen: false,
+ chunkId: null,
+ images: [],
+ loading: false,
+ error: null,
+ });
+ }, []);
+
if (!citations || citations.length === 0) {
return null;
}
@@ -141,15 +213,157 @@ export default function CitationBlock({ citations }: CitationBlockProps) {
max-height: 200px;
overflow-y: auto;
}
+ .citation-view-btn {
+ margin-top: 0.5rem;
+ padding: 0.25rem 0.5rem;
+ font-size: 0.75rem;
+ background: #3b82f6;
+ color: white;
+ border: none;
+ border-radius: 4px;
+ cursor: pointer;
+ transition: background 0.15s;
+ }
+ .citation-view-btn:hover {
+ background: #2563eb;
+ }
+ .visual-modal-overlay {
+ position: fixed;
+ top: 0;
+ left: 0;
+ right: 0;
+ bottom: 0;
+ background: rgba(0, 0, 0, 0.75);
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ z-index: 1000;
+ }
+ .visual-modal {
+ background: white;
+ border-radius: 8px;
+ padding: 1.5rem;
+ max-width: 90vw;
+ max-height: 90vh;
+ overflow: auto;
+ position: relative;
+ }
+ .visual-modal-close {
+ position: absolute;
+ top: 0.5rem;
+ right: 0.5rem;
+ background: #ef4444;
+ color: white;
+ border: none;
+ border-radius: 50%;
+ width: 2rem;
+ height: 2rem;
+ cursor: pointer;
+ font-size: 1rem;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ }
+ .visual-modal-close:hover {
+ background: #dc2626;
+ }
+ .visual-modal-title {
+ margin: 0 0 1rem 0;
+ font-size: 1.125rem;
+ color: #1e293b;
+ }
+ .visual-modal-loading {
+ padding: 2rem;
+ text-align: center;
+ color: #64748b;
+ }
+ .visual-modal-error {
+ padding: 1rem;
+ background: #fef2f2;
+ color: #dc2626;
+ border-radius: 4px;
+ }
+ .visual-modal-images {
+ display: flex;
+ flex-direction: column;
+ gap: 1rem;
+ }
+ .visual-modal-page-label {
+ font-size: 0.75rem;
+ color: #64748b;
+ margin-bottom: 0.5rem;
+ }
+ .visual-modal-image {
+ max-width: 100%;
+ border: 1px solid #e2e8f0;
+ border-radius: 4px;
+ }
`}
Sources ({citations.length})
{citations.map((citation) => (
-
+
))}
+
+ {visualGrounding.isOpen && (
+ e.key === "Escape" && closeVisualGrounding()}
+ role="dialog"
+ aria-modal="true"
+ aria-label="Visual grounding"
+ >
+ {/* biome-ignore lint/a11y/noStaticElementInteractions: modal content wrapper */}
+
e.stopPropagation()}
+ onKeyDown={(e) => e.stopPropagation()}
+ >
+
+
Visual Grounding
+ {visualGrounding.loading && (
+
Loading...
+ )}
+ {visualGrounding.error && (
+
{visualGrounding.error}
+ )}
+ {!visualGrounding.loading &&
+ !visualGrounding.error &&
+ visualGrounding.images.length > 0 && (
+
+ {visualGrounding.images.map((img, idx) => (
+ // biome-ignore lint/suspicious/noArrayIndexKey: images have no stable id
+
+
+ Page {idx + 1} of {visualGrounding.images.length}
+
+ {/* biome-ignore lint/performance/noImgElement: base64 data URLs require img element */}
+

+
+ ))}
+
+ )}
+
+
+ )}
>
);
}