feat(reasoning): bidirectional PDF ↔ graph focus + DocumentView mode

Propagate Docling `self_ref` through PageElement so bboxes and graph nodes
share a stable identity. Add a Document/Graph mode switch to the reasoning
workspace; selecting a node highlights its bbox (numbered badge, focus ring,
optional dim of non-visited) and clicking a bbox re-centers the graph.
This commit is contained in:
Pier-Jean Malandrino 2026-04-23 22:07:58 +02:00
parent bef7ec4686
commit 9ec64961fc
9 changed files with 446 additions and 29 deletions

View file

@ -19,6 +19,11 @@ class PageElement:
bbox: list[float]
content: str
level: int = 0
# Docling `self_ref` ("#/texts/12", "#/tables/3", …). Empty for items
# that don't have one (rare — defensive default). Lets callers correlate
# a rendered bbox with the corresponding node in the graph without
# resorting to fuzzy bbox matching.
self_ref: str = ""
@dataclass(frozen=True)

View file

@ -194,7 +194,13 @@ def _process_content_item(
content = item.export_to_markdown()
pages[page_no].elements.append(
PageElement(type=element_type, bbox=bbox, content=content, level=level)
PageElement(
type=element_type,
bbox=bbox,
content=content,
level=level,
self_ref=getattr(item, "self_ref", "") or "",
)
)
except (AttributeError, KeyError, TypeError, ValueError):
logger.warning(

View file

@ -19,7 +19,7 @@
</span>
<span class="graph-legend">
<button
v-for="chip in LEGEND_CHIPS"
v-for="chip in visibleChips"
:key="chip.key"
type="button"
class="legend-chip"
@ -82,6 +82,13 @@ const props = withDefaults(
}>(),
{ fetcher: fetchDocumentGraph },
)
const emit = defineEmits<{
/** Emitted on node tap with the element's `self_ref` (null when the tap
* cleared the selection, or when the tapped node has no self_ref
* Document / Page / Chunk). Consumers can mirror the selection elsewhere
* (e.g. the ReasoningWorkspace syncs it to the PDF viewer). */
nodeFocus: [selfRef: string | null]
}>()
const { t } = useI18n()
const containerRef = ref<HTMLDivElement | null>(null)
@ -129,6 +136,16 @@ const selectedNodeContents = computed<GraphNode[]>(() => {
return childrenByParent.value.get(id) ?? []
})
// Only surface chips that actually have matching nodes in the current
// payload. Keeps the legend in sync with the source (e.g. Reasoning view
// never emits Chunk nodes, so the Chunk chip would dangle) without
// hardcoding per-view chip lists.
const visibleChips = computed(() => {
const nodes = payload.value?.nodes ?? []
if (nodes.length === 0) return []
return LEGEND_CHIPS.filter((chip) => nodes.some((n) => chip.match(n)))
})
// Hover tooltip: position (px within .graph-canvas) + text. Null hides it.
const tooltip = ref<{ x: number; y: number; text: string } | null>(null)
@ -383,12 +400,17 @@ async function renderGraph(): Promise<void> {
// Visual feedback clear previous selection class first.
cy.value?.nodes('.nd-selected').removeClass('nd-selected')
evt.target.addClass('nd-selected')
// Let the outer workspace mirror the selection (e.g. into the PDF view).
// Nodes without a `self_ref` (Document / Page / Chunk) emit `null` so
// the consumer can reset its focus.
emit('nodeFocus', raw.self_ref ?? null)
})
// Click on background close details panel.
// Click on background close details panel + clear cross-view focus.
cy.value.on('tap', (evt) => {
if (evt.target === cy.value) {
selectedNode.value = null
cy.value?.nodes('.nd-selected').removeClass('nd-selected')
emit('nodeFocus', null)
}
})
@ -448,6 +470,18 @@ function navigateToNode(target: GraphNode): void {
}
}
/**
* Mirror an external selection (e.g. user clicked a bbox in the PDF view)
* onto the graph: select the matching node, scroll it into view, update
* the details panel. No-op if the element isn't in the current graph
* (common for a PDF-only element that the reasoning graph didn't emit).
*/
function selectBySelfRef(selfRef: string): void {
const node = payload.value?.nodes.find((n) => n.self_ref === selfRef) ?? null
if (!node) return
navigateToNode(node)
}
function disposeGraph(): void {
if (cy.value) {
cy.value.destroy()
@ -501,7 +535,7 @@ watch(
// Let parent components observe the live Cytoscape instance (e.g. the
// reasoning-trace overlay reads it via `graphViewRef.value?.cy`).
defineExpose({ cy, load })
defineExpose({ cy, load, selectBySelfRef })
</script>
<style scoped>

View file

@ -40,8 +40,10 @@
<canvas
ref="canvasRef"
class="overlay-canvas"
:class="{ selectable }"
@mousemove="onMouseMove"
@mouseleave="hoveredElement = null"
@click="onCanvasClick"
/>
<!-- Tooltip -->
<div v-if="hoveredElement" class="tooltip" :style="tooltipStyle">
@ -76,8 +78,39 @@ const ELEMENT_COLORS: Record<string, string> = {
const props = defineProps({
pages: { type: Array as () => Page[], default: () => [] },
documentId: String,
/**
* Reasoning-trace integration hooks. Optional when unset, StructureViewer
* renders like before (Studio "Structure" tab). When set, enables overlays
* for the reasoning viewer without forking the component:
*
* - `visitedBySelfRef`: elements whose `self_ref` is in this map render in
* the reasoning accent color with a numbered badge (the visit order).
* - `focusedSelfRef`: when it changes, auto-scroll to the page of that
* element and pulse its bbox briefly.
* - `selectable`: when true, clicking a bbox emits `elementFocus` so a
* parent can sync the selection with the graph view.
*/
visitedBySelfRef: {
type: Object as () => Map<string, number> | null,
default: null,
},
focusedSelfRef: { type: String as () => string | null, default: null },
selectable: { type: Boolean, default: false },
/**
* When true AND `visitedBySelfRef` is set, non-visited elements are drawn
* with reduced alpha so the visited ones pop. Matches the reasoning
* panel's "Focus" toggle behavior on the graph.
*/
dimNonVisited: { type: Boolean, default: false },
})
const emit = defineEmits<{
/** Fired when the user clicks a bbox — only if `selectable` is true. */
elementFocus: [selfRef: string]
}>()
const REASONING_COLOR = '#EA580C'
const selectedPage = ref(1)
const hiddenTypes = reactive(new Set<string>())
const containerRef = ref<HTMLDivElement | null>(null)
@ -136,39 +169,94 @@ function drawOverlay() {
const scale = computeScale(img.clientWidth, img.clientHeight, page.width, page.height)
// Two-pass draw so reasoning overlays (highlight + pulse) sit on top of
// the base element strokes without being painted over by subsequent
// elements. First pass = base, second pass = accents.
for (const el of visibleElements.value) {
const rect = bboxToRect(el.bbox, scale)
const color = ELEMENT_COLORS[el.type] || ELEMENT_COLORS.text
const baseColor = ELEMENT_COLORS[el.type] || ELEMENT_COLORS.text
const isVisited =
props.visitedBySelfRef !== null && !!el.self_ref && props.visitedBySelfRef.has(el.self_ref)
ctx.strokeStyle = color
ctx.lineWidth = 2
ctx.strokeRect(rect.x, rect.y, rect.w, rect.h)
ctx.fillStyle = color + '20'
ctx.fillRect(rect.x, rect.y, rect.w, rect.h)
if (isVisited) {
// Reasoning-visited element reasoning accent color, bolder stroke,
// more saturated fill than the base element. The visit-order badge
// is drawn in the second pass below.
ctx.strokeStyle = REASONING_COLOR
ctx.lineWidth = 3
ctx.strokeRect(rect.x, rect.y, rect.w, rect.h)
ctx.fillStyle = REASONING_COLOR + '33'
ctx.fillRect(rect.x, rect.y, rect.w, rect.h)
} else {
// Dim non-visited when focus mode is on and a visited set is present,
// so visited bboxes pop. Otherwise keep the regular styling.
const dim = props.dimNonVisited && props.visitedBySelfRef !== null
ctx.strokeStyle = baseColor + (dim ? '22' : '')
ctx.lineWidth = dim ? 1 : 2
ctx.strokeRect(rect.x, rect.y, rect.w, rect.h)
ctx.fillStyle = baseColor + (dim ? '08' : '20')
ctx.fillRect(rect.x, rect.y, rect.w, rect.h)
}
}
// Second pass numbered badges on visited elements + focus pulse ring.
for (const el of visibleElements.value) {
const rect = bboxToRect(el.bbox, scale)
const order =
props.visitedBySelfRef !== null && el.self_ref
? props.visitedBySelfRef.get(el.self_ref)
: undefined
if (order !== undefined) {
drawVisitBadge(ctx, rect.x, rect.y, order)
}
if (props.focusedSelfRef && el.self_ref === props.focusedSelfRef) {
ctx.strokeStyle = REASONING_COLOR
ctx.lineWidth = 2
ctx.setLineDash([6, 4])
ctx.strokeRect(rect.x - 4, rect.y - 4, rect.w + 8, rect.h + 8)
ctx.setLineDash([])
}
}
}
function drawVisitBadge(ctx: CanvasRenderingContext2D, x: number, y: number, order: number): void {
const radius = 10
const cx = x
const cy = y
ctx.fillStyle = REASONING_COLOR
ctx.beginPath()
ctx.arc(cx, cy, radius, 0, Math.PI * 2)
ctx.fill()
ctx.fillStyle = '#ffffff'
ctx.font = 'bold 11px -apple-system, sans-serif'
ctx.textAlign = 'center'
ctx.textBaseline = 'middle'
ctx.fillText(String(order), cx, cy + 0.5)
}
function elementAt(e: MouseEvent): PageElement | null {
const canvas = canvasRef.value
const page = currentPageData.value
const img = imageRef.value
if (!canvas || !page || !img) return null
const canvasRect = canvas.getBoundingClientRect()
const mx = e.clientX - canvasRect.left
const my = e.clientY - canvasRect.top
const scale = computeScale(img.clientWidth, img.clientHeight, page.width, page.height)
for (const el of visibleElements.value) {
if (pointInRect(mx, my, bboxToRect(el.bbox, scale))) return el
}
return null
}
function onMouseMove(e: MouseEvent) {
const canvas = canvasRef.value
const page = currentPageData.value
const img = imageRef.value
if (!canvas || !page || !img) return
if (!canvas) return
const canvasRect = canvas.getBoundingClientRect()
const mx = e.clientX - canvasRect.left
const my = e.clientY - canvasRect.top
const scale = computeScale(img.clientWidth, img.clientHeight, page.width, page.height)
let found: PageElement | null = null
for (const el of visibleElements.value) {
if (pointInRect(mx, my, bboxToRect(el.bbox, scale))) {
found = el
break
}
}
const found = elementAt(e)
hoveredElement.value = found
if (found) {
tooltipStyle.value = {
@ -178,9 +266,47 @@ function onMouseMove(e: MouseEvent) {
}
}
function onCanvasClick(e: MouseEvent): void {
if (!props.selectable) return
const el = elementAt(e)
if (el?.self_ref) emit('elementFocus', el.self_ref)
}
watch([() => props.pages, selectedPage, hiddenTypes], () => {
nextTick(drawOverlay)
})
watch(
() => [props.visitedBySelfRef, props.dimNonVisited],
() => nextTick(drawOverlay),
)
// When the caller sets a focused self_ref (e.g. the user clicked a node in
// the graph), find which page that element lives on and jump to it. The
// overlay redraw will then show the dashed focus ring around its bbox.
watch(
() => props.focusedSelfRef,
(ref) => {
if (!ref) {
nextTick(drawOverlay)
return
}
for (const page of props.pages) {
if (page.elements.some((e) => e.self_ref === ref)) {
if (selectedPage.value !== page.page_number) {
selectedPage.value = page.page_number
// Let <img> reload before drawing drawOverlay runs on @load.
} else {
nextTick(drawOverlay)
}
return
}
}
// Ref not on any page (e.g. a #/body node) just redraw to clear the
// previous focus ring.
nextTick(drawOverlay)
},
)
</script>
<style scoped>
@ -282,6 +408,10 @@ watch([() => props.pages, selectedPage, hiddenTypes], () => {
pointer-events: auto;
}
.overlay-canvas.selectable {
cursor: pointer;
}
.tooltip {
position: absolute;
background: var(--bg-surface);

View file

@ -96,6 +96,11 @@ export const useReasoningStore = defineStore('reasoning', () => {
}
function setActiveIteration(n: number | null): void {
// Flip via null first so a click on the *same* iteration still fires
// downstream watchers (PDF scroll, graph pan). Otherwise Vue sees
// "no change" and the second click is a silent no-op — painful UX when
// only one iteration exists.
activeIteration.value = null
activeIteration.value = n
}

View file

@ -0,0 +1,95 @@
<template>
<div class="rdv-root" data-e2e="reasoning-document-view">
<div v-if="!pages || pages.length === 0" class="rdv-empty">
{{ t('reasoning.docNoContent') }}
</div>
<StructureViewer
v-else
:pages="pages"
:document-id="docId"
:visited-by-self-ref="visitedBySelfRef"
:focused-self-ref="focusedSelfRef"
:dim-non-visited="reasoningStore.focusMode"
selectable
class="rdv-viewer"
@element-focus="(ref) => emit('elementFocus', ref)"
/>
</div>
</template>
<script setup lang="ts">
/**
* PDF rendering of the document (per-page PNG via /api/documents/:id/preview),
* augmented with reasoning overlays:
* - elements visited by the RAG loop get a bold orange stroke + numbered
* badge showing the visit order
* - the current `focusedSelfRef` (usually driven by a graph-node click
* upstream) auto-jumps to the right page and pulses its bbox
* - clicking a bbox emits `elementFocus` so the graph can mirror the
* selection (bidirectional sync handled in ReasoningWorkspace).
*/
import { computed } from 'vue'
import StructureViewer from '../../analysis/ui/StructureViewer.vue'
import { useAnalysisStore } from '../../analysis/store'
import { useI18n } from '../../../shared/i18n'
import type { Page } from '../../../shared/types'
import { useReasoningStore } from '../store'
const props = defineProps<{
docId: string
focusedSelfRef: string | null
}>()
const emit = defineEmits<{ elementFocus: [selfRef: string] }>()
const analysisStore = useAnalysisStore()
const reasoningStore = useReasoningStore()
const { t } = useI18n()
const pages = computed<Page[]>(() => {
const hit = analysisStore.analyses.find(
(a) => a.documentId === props.docId && a.status === 'COMPLETED' && a.pagesJson,
)
if (!hit?.pagesJson) return []
try {
return JSON.parse(hit.pagesJson) as Page[]
} catch {
return []
}
})
const visitedBySelfRef = computed<Map<string, number>>(() => {
const out = new Map<string, number>()
for (const it of reasoningStore.iterations) {
if (!it.present || !it.sectionRef) continue
if (!out.has(it.sectionRef)) out.set(it.sectionRef, it.iteration)
}
return out
})
</script>
<style scoped>
.rdv-root {
flex: 1 1 auto;
min-width: 0;
min-height: 0;
overflow-y: auto;
background: var(--bg);
padding: 16px 20px;
}
.rdv-viewer {
max-width: 960px;
margin: 0 auto;
}
.rdv-empty {
display: flex;
align-items: center;
justify-content: center;
height: 100%;
color: var(--text-muted);
font-size: 13px;
font-style: italic;
}
</style>

View file

@ -7,6 +7,35 @@
<div class="rw-doc-title" :title="docFilename ?? docId">
{{ docFilename ?? docId }}
</div>
<!-- Main-pane toggle: graph vs docling markdown. Overlay panel on the
right stays mounted in both modes so the iteration list keeps
living context only the main pane swaps. -->
<div class="rw-mode-switch" role="tablist" :aria-label="t('reasoning.modeSwitchLabel')">
<button
type="button"
role="tab"
class="rw-mode-btn"
:class="{ active: mode === 'graph' }"
:aria-selected="mode === 'graph'"
data-e2e="reasoning-mode-graph"
@click="mode = 'graph'"
>
{{ t('reasoning.modeGraph') }}
</button>
<button
type="button"
role="tab"
class="rw-mode-btn"
:class="{ active: mode === 'document' }"
:aria-selected="mode === 'document'"
data-e2e="reasoning-mode-document"
@click="mode = 'document'"
>
{{ t('reasoning.modeDocument') }}
</button>
</div>
<button
class="rw-action-btn rw-action-ghost"
data-e2e="reasoning-workspace-import"
@ -24,7 +53,26 @@
</header>
<div class="rw-body">
<GraphView ref="graphViewRef" :doc-id="docId" :fetcher="fetchReasoningGraph" />
<!-- Keep GraphView mounted via v-show rather than v-if so the Cytoscape
instance + its layout state survive a toggle to document mode and
back rebuilding is expensive and would reset pan/zoom. -->
<GraphView
v-show="mode === 'graph'"
ref="graphViewRef"
:doc-id="docId"
:fetcher="fetchReasoningGraph"
@node-focus="onGraphNodeFocus"
/>
<!-- v-show (not v-if) so the StructureViewer's scroll-to-focused watch
sees transitions from null sectionRef that happen while we're in
graph mode. Otherwise the viewer mounts with an already-set prop
and the initial scroll never fires. -->
<DocumentView
v-show="mode === 'document'"
:doc-id="docId"
:focused-self-ref="focusedSelfRef"
@element-focus="onPdfElementFocus"
/>
<ReasoningPanel :cy="graphCy" />
</div>
@ -39,9 +87,12 @@ import GraphView from '../../analysis/ui/GraphView.vue'
import { useI18n } from '../../../shared/i18n'
import { fetchReasoningGraph } from '../api'
import { useReasoningStore } from '../store'
import DocumentView from './DocumentView.vue'
import ReasoningPanel from './ReasoningPanel.vue'
import RunReasoningDialog from './RunReasoningDialog.vue'
type WorkspaceMode = 'graph' | 'document'
const props = defineProps<{
docId: string
docFilename?: string | null
@ -55,11 +106,55 @@ const reasoningStore = useReasoningStore()
const graphViewRef = ref<InstanceType<typeof GraphView> | null>(null)
const graphCy = computed(() => graphViewRef.value?.cy ?? null)
const mode = ref<WorkspaceMode>('graph')
// Shared focused element (Docling self_ref like "#/texts/12") the one
// bridge between graph and PDF. Clicking a node in the graph sets this,
// clicking a bbox in the PDF sets this. When set, both views highlight
// the corresponding element. Persists across mode toggles so jumping from
// Graph Document preserves the currently-looked-at element.
const focusedSelfRef = ref<string | null>(null)
function onGraphNodeFocus(selfRef: string | null): void {
focusedSelfRef.value = selfRef
}
function onPdfElementFocus(selfRef: string): void {
focusedSelfRef.value = selfRef
// Mirror the selection on the graph side if the user switches back to
// graph mode, they'll see the same element selected + centered.
graphViewRef.value?.selectBySelfRef(selfRef)
}
// Click on an iteration card in the reasoning panel flows through
// `reasoningStore.setActiveIteration(n)`. That path already focuses the
// cytoscape node (in ReasoningPanel.onFocus); we mirror it into the PDF
// viewer by resolving the active iteration's section_ref and piping it
// into our shared focus. Done at the workspace level not inside the
// panel because the panel doesn't know it has a PDF sibling.
watch(
() => reasoningStore.activeIteration,
(n) => {
if (n === null) return
const hit = reasoningStore.iterations.find((i) => i.iteration === n)
if (!hit?.present || !hit.sectionRef) return
// Flip via null so StructureViewer's watch on `focusedSelfRef` re-fires
// even when clicking the same iteration twice (same sectionRef).
focusedSelfRef.value = null
focusedSelfRef.value = hit.sectionRef
},
)
// Reset the reasoning store when switching docs a trace imported for one
// document is meaningless on another.
// document is meaningless on another. The main-pane mode resets too so a
// new doc opens on the graph (consistent default).
watch(
() => props.docId,
() => reasoningStore.reset(),
() => {
reasoningStore.reset()
mode.value = 'graph'
focusedSelfRef.value = null
},
)
// Clean up so a later navigation back to the workspace starts fresh.
@ -138,6 +233,40 @@ onBeforeUnmount(() => reasoningStore.reset())
filter: none;
}
/* Segmented control for the main-pane mode (graph vs document). Sits
* between the doc title and the action buttons. */
.rw-mode-switch {
display: inline-flex;
border: 1px solid var(--border);
border-radius: var(--radius-sm);
overflow: hidden;
}
.rw-mode-btn {
background: transparent;
border: 0;
padding: 5px 12px;
font-size: 12px;
color: var(--text-secondary);
cursor: pointer;
transition: all var(--transition);
}
.rw-mode-btn + .rw-mode-btn {
border-left: 1px solid var(--border);
}
.rw-mode-btn:hover:not(.active) {
background: var(--border-light);
color: var(--text);
}
.rw-mode-btn.active {
background: var(--accent);
color: #fff;
font-weight: 500;
}
.rw-body {
flex: 1 1 auto;
min-height: 0;
@ -146,7 +275,8 @@ onBeforeUnmount(() => reasoningStore.reset())
overflow: hidden;
}
.rw-body > :deep(.graph-view) {
.rw-body > :deep(.graph-view),
.rw-body > :deep(.rdv-root) {
flex: 1 1 auto;
min-width: 0;
}

View file

@ -175,6 +175,10 @@ const messages: Messages = {
'Aucun des documents existants n\u2019a encore été analysé — lance-en un depuis Studio, ou dépose un nouveau PDF ci-dessus.',
'reasoning.pagesCount': '{n} pages',
'reasoning.changeDoc': 'Changer de document',
'reasoning.modeSwitchLabel': 'Mode d\u2019affichage',
'reasoning.modeGraph': 'Graphe',
'reasoning.modeDocument': 'Document',
'reasoning.docNoContent': 'Aucun contenu rendu disponible pour ce document.',
'reasoning.analyzing': 'Analyse du document...',
'reasoning.analyzingHint':
'Docling analyse le PDF avec la configuration par défaut. Cela peut prendre 1 à 3 minutes selon la taille.',
@ -430,6 +434,10 @@ const messages: Messages = {
'None of your existing documents have been analyzed yet — run one from Studio, or drop a new PDF above.',
'reasoning.pagesCount': '{n} pages',
'reasoning.changeDoc': 'Change document',
'reasoning.modeSwitchLabel': 'View mode',
'reasoning.modeGraph': 'Graph',
'reasoning.modeDocument': 'Document',
'reasoning.docNoContent': 'No rendered content available for this document.',
'reasoning.analyzing': 'Analyzing document...',
'reasoning.analyzingHint':
'Docling is parsing the PDF with default settings. May take 13 minutes depending on size.',

View file

@ -67,6 +67,10 @@ export interface PageElement {
bbox: [number, number, number, number]
content: string
level: number
/** Docling `self_ref` "#/texts/12", "#/tables/3", etc. Empty string for
* items that don't have one (rare). Lets callers correlate a bbox with
* the matching graph node without fuzzy bbox matching. */
self_ref?: string
}
// Backend serializes with snake_case (dataclasses.asdict)