Fix display error on visual mode

This commit is contained in:
pjmalandrino 2026-03-17 10:28:50 +01:00
parent 9c2e42ace8
commit f1aa39b75f
9 changed files with 411 additions and 66 deletions

View file

@ -61,6 +61,7 @@ public class AnalysisService {
if (pageCount instanceof Number n && n.intValue() > 0) { if (pageCount instanceof Number n && n.intValue() > 0) {
Document doc = job.getDocument(); Document doc = job.getDocument();
doc.setPageCount(n.intValue()); doc.setPageCount(n.intValue());
documentService.save(doc);
} }
job.markCompleted(markdown, html, pagesJson); job.markCompleted(markdown, html, pagesJson);

View file

@ -53,6 +53,10 @@ public class DocumentService {
} }
} }
public Document save(Document document) {
return repository.save(document);
}
public List<Document> findAll() { public List<Document> findAll() {
return repository.findAll(); return repository.findAll();
} }

View file

@ -1,4 +1,5 @@
import io import io
import logging
import os import os
import tempfile import tempfile
from pathlib import Path from pathlib import Path
@ -12,6 +13,8 @@ from PIL import Image
from bbox import to_topleft_list from bbox import to_topleft_list
logger = logging.getLogger(__name__)
app = FastAPI(title="Docling Studio - Document Parser") app = FastAPI(title="Docling Studio - Document Parser")
converter = DocumentConverter() converter = DocumentConverter()
@ -45,7 +48,11 @@ class ParseResponse(BaseModel):
# --- Helpers --- # --- Helpers ---
def extract_pages_detail(doc_result) -> list[PageDetail]: def extract_pages_detail(doc_result) -> list[PageDetail]:
"""Extract per-page element details with bounding boxes from Docling result.""" """Extract per-page element details with bounding boxes from Docling result.
Uses Docling's iterate_items() API (preferred) or falls back to document.texts.
Both provide a flat iteration over all content items, avoiding duplicates.
"""
pages: dict[int, PageDetail] = {} pages: dict[int, PageDetail] = {}
document = doc_result.document document = doc_result.document
@ -62,12 +69,11 @@ def extract_pages_detail(doc_result) -> list[PageDetail]:
elements=[] elements=[]
) )
# Iterate document items to extract elements with bounding boxes # Use iterate_items() (Docling v2 API) — avoids duplicates
if hasattr(document, 'body') and hasattr(document.body, 'children'): if hasattr(document, 'iterate_items'):
_extract_elements_recursive(document, document.body.children, pages) for item, _level in document.iterate_items():
_process_content_item(item, pages)
# Fallback: if we have content items elif hasattr(document, 'texts'):
if hasattr(document, 'texts'):
for text_item in document.texts: for text_item in document.texts:
_process_content_item(text_item, pages) _process_content_item(text_item, pages)
@ -76,56 +82,47 @@ def extract_pages_detail(doc_result) -> list[PageDetail]:
def _process_content_item(item, pages: dict[int, PageDetail]): def _process_content_item(item, pages: dict[int, PageDetail]):
"""Process a single content item and add it to the appropriate page.""" """Process a single content item and add it to the appropriate page.
Silently skips items that lack provenance or fail to process,
so one bad item doesn't break the whole extraction.
"""
if not hasattr(item, 'prov') or not item.prov: if not hasattr(item, 'prov') or not item.prov:
return return
for prov in item.prov: for prov in item.prov:
page_no = prov.page_no if hasattr(prov, 'page_no') else 1 try:
page_no = prov.page_no if hasattr(prov, 'page_no') else 1
if page_no not in pages: if page_no not in pages:
pages[page_no] = PageDetail( pages[page_no] = PageDetail(
page_number=page_no, width=612.0, height=792.0, elements=[] page_number=page_no, width=612.0, height=792.0, elements=[]
) )
page_height = pages[page_no].height page_height = pages[page_no].height
bbox = [0, 0, 0, 0] bbox = [0, 0, 0, 0]
if hasattr(prov, 'bbox') and prov.bbox: if hasattr(prov, 'bbox') and prov.bbox:
b = prov.bbox b = prov.bbox
if hasattr(b, 'l'): if hasattr(b, 'l'):
bbox = to_topleft_list(b, page_height) bbox = to_topleft_list(b, page_height)
elif isinstance(b, (list, tuple)) and len(b) >= 4: elif isinstance(b, (list, tuple)) and len(b) >= 4:
bbox = list(b[:4]) bbox = list(b[:4])
element_type = _get_element_type(item) element_type = _get_element_type(item)
content = "" content = ""
if hasattr(item, 'text'): if hasattr(item, 'text'):
content = item.text or "" content = item.text or ""
elif hasattr(item, 'export_to_markdown'):
content = item.export_to_markdown()
pages[page_no].elements.append(PageElement( pages[page_no].elements.append(PageElement(
type=element_type, type=element_type,
bbox=bbox, bbox=bbox,
content=content[:500] # Truncate long content content=content[:500]
)) ))
except Exception:
logger.warning("Skipping item %s: failed to process", type(item).__name__, exc_info=True)
def _extract_elements_recursive(document, children, pages: dict[int, PageDetail]):
"""Recursively extract elements from document tree."""
if not children:
return
for child_ref in children:
# Resolve reference if needed
item = child_ref
if hasattr(child_ref, '__ref__'):
item = document.resolve_ref(child_ref)
if item:
_process_content_item(item, pages)
if hasattr(item, 'children') and item.children:
_extract_elements_recursive(document, item.children, pages)
def _get_element_type(item) -> str: def _get_element_type(item) -> str:
"""Determine the element type from a Docling document item.""" """Determine the element type from a Docling document item."""
@ -201,6 +198,7 @@ async def parse(file: UploadFile):
) )
except Exception as e: except Exception as e:
logger.exception("Failed to parse document: %s", file.filename)
raise HTTPException(status_code=422, detail=f"Failed to parse document: {str(e)}") raise HTTPException(status_code=422, detail=f"Failed to parse document: {str(e)}")
finally: finally:
if tmp_path and os.path.exists(tmp_path): if tmp_path and os.path.exists(tmp_path):

View file

@ -3,15 +3,19 @@
* to canvas pixel coordinates. * to canvas pixel coordinates.
* *
* Docling bbox format (after backend normalization): [l, t, r, b] * Docling bbox format (after backend normalization): [l, t, r, b]
* All values are in page coordinate space (points). * All values are in page coordinate space (points, 1pt = 1/72 inch).
* Origin: top-left (y=0 at top of page). * Origin: top-left (y=0 at top of page).
*
* The preview image is a faithful rasterization of the PDF page.
* CSS `max-width: 100%; height: auto` preserves the aspect ratio,
* so sx and sy are always equal. We compute both for completeness.
*/ */
/** /**
* Compute scale factors from page coordinates to displayed image pixels. * Compute scale factors from page coordinates to displayed pixels.
* *
* @param {number} displayWidth - Rendered image width in CSS pixels * @param {number} displayWidth - img.clientWidth (CSS pixels)
* @param {number} displayHeight - Rendered image height in CSS pixels * @param {number} displayHeight - img.clientHeight (CSS pixels)
* @param {number} pageWidth - Page width in Docling points * @param {number} pageWidth - Page width in Docling points
* @param {number} pageHeight - Page height in Docling points * @param {number} pageHeight - Page height in Docling points
* @returns {{ sx: number, sy: number }} * @returns {{ sx: number, sy: number }}
@ -26,8 +30,8 @@ export function computeScale(displayWidth, displayHeight, pageWidth, pageHeight)
/** /**
* Convert a Docling bbox [l, t, r, b] to a canvas rect { x, y, w, h }. * Convert a Docling bbox [l, t, r, b] to a canvas rect { x, y, w, h }.
* *
* @param {number[]} bbox - [left, top, right, bottom] in page points * @param {number[]} bbox - [left, top, right, bottom] in page points
* @param {{ sx: number, sy: number }} scale - Scale factors from computeScale * @param {{ sx: number, sy: number }} scale
* @returns {{ x: number, y: number, w: number, h: number }} * @returns {{ x: number, y: number, w: number, h: number }}
*/ */
export function bboxToRect(bbox, scale) { export function bboxToRect(bbox, scale) {

View file

@ -13,6 +13,15 @@ describe('computeScale', () => {
expect(s.sx).toBeCloseTo(0.5) expect(s.sx).toBeCloseTo(0.5)
expect(s.sy).toBeCloseTo(0.5) expect(s.sy).toBeCloseTo(0.5)
}) })
it('sx equals sy when aspect ratio is preserved', () => {
// Image at 150 DPI displayed at 700px wide
const pageW = 612, pageH = 792
const displayW = 700
const displayH = displayW * pageH / pageW // preserves ratio
const s = computeScale(displayW, displayH, pageW, pageH)
expect(s.sx).toBeCloseTo(s.sy, 5)
})
}) })
describe('bboxToRect', () => { describe('bboxToRect', () => {
@ -36,6 +45,15 @@ describe('bboxToRect', () => {
expect(rect.w).toBeCloseTo(100) expect(rect.w).toBeCloseTo(100)
expect(rect.h).toBeCloseTo(100) expect(rect.h).toBeCloseTo(100)
}) })
it('end-to-end: full page bbox fills display', () => {
const scale = computeScale(700, 907.84, 612, 792)
const rect = bboxToRect([0, 0, 612, 792], scale)
expect(rect.x).toBeCloseTo(0)
expect(rect.y).toBeCloseTo(0)
expect(rect.w).toBeCloseTo(700)
expect(rect.h).toBeCloseTo(907.84, 0)
})
}) })
describe('pointInRect', () => { describe('pointInRect', () => {

View file

@ -3,4 +3,5 @@ export { default as AnalysisPanel } from './ui/AnalysisPanel.vue'
export { default as ResultTabs } from './ui/ResultTabs.vue' export { default as ResultTabs } from './ui/ResultTabs.vue'
export { default as MarkdownViewer } from './ui/MarkdownViewer.vue' export { default as MarkdownViewer } from './ui/MarkdownViewer.vue'
export { default as StructureViewer } from './ui/StructureViewer.vue' export { default as StructureViewer } from './ui/StructureViewer.vue'
export { default as BboxOverlay } from './ui/BboxOverlay.vue'
export { default as ImageGallery } from './ui/ImageGallery.vue' export { default as ImageGallery } from './ui/ImageGallery.vue'

View file

@ -0,0 +1,250 @@
<template>
<div class="bbox-overlay">
<!-- Legend bar -->
<div class="overlay-legend" v-if="pageData">
<button
v-for="[type, color] in Object.entries(ELEMENT_COLORS)"
:key="type"
class="legend-chip"
:class="{ dimmed: hiddenTypes.has(type) }"
@click="toggleType(type)"
>
<span class="legend-dot" :style="{ background: color }" />
<span>{{ type }}</span>
<span class="legend-count">{{ countElements(type) }}</span>
</button>
</div>
<!-- Canvas + Tooltip (positioned over image via parent) -->
<canvas
ref="canvasRef"
class="overlay-canvas"
@mousemove="onMouseMove"
@mouseleave="hoveredElement = null"
/>
<div
v-if="hoveredElement"
class="tooltip"
:style="tooltipStyle"
>
<span class="tooltip-type" :style="{ color: ELEMENT_COLORS[hoveredElement.type] || ELEMENT_COLORS.text }">
{{ hoveredElement.type }}
</span>
<span class="tooltip-content">{{ hoveredElement.content?.substring(0, 150) }}</span>
</div>
</div>
</template>
<script setup>
import { ref, computed, watch, nextTick, reactive, onMounted, onBeforeUnmount } from 'vue'
import { computeScale, bboxToRect, pointInRect } from '../bboxScaling.js'
const ELEMENT_COLORS = {
section_header: '#F97316',
text: '#3B82F6',
table: '#8B5CF6',
picture: '#22C55E',
list: '#06B6D4',
formula: '#EC4899',
caption: '#EAB308'
}
const props = defineProps({
/** The <img> element to overlay onto */
imageEl: { type: Object, default: null },
/** Page data { page_number, width, height, elements[] } for current page */
pageData: { type: Object, default: null }
})
const hiddenTypes = reactive(new Set())
const canvasRef = ref(null)
const hoveredElement = ref(null)
const tooltipStyle = ref({})
const visibleElements = computed(() => {
if (!props.pageData) return []
return props.pageData.elements.filter(e => !hiddenTypes.has(e.type))
})
function toggleType(type) {
if (hiddenTypes.has(type)) hiddenTypes.delete(type)
else hiddenTypes.add(type)
draw()
}
function countElements(type) {
if (!props.pageData) return 0
return props.pageData.elements.filter(e => e.type === type).length
}
function draw() {
const canvas = canvasRef.value
const img = props.imageEl
if (!canvas || !img) return
// Wait until image is loaded (clientWidth > 0)
if (!img.clientWidth || !img.clientHeight) return
canvas.width = img.clientWidth
canvas.height = img.clientHeight
const ctx = canvas.getContext('2d')
ctx.clearRect(0, 0, canvas.width, canvas.height)
if (!props.pageData) return
const scale = computeScale(img.clientWidth, img.clientHeight, props.pageData.width, props.pageData.height)
for (const el of visibleElements.value) {
const rect = bboxToRect(el.bbox, scale)
const color = ELEMENT_COLORS[el.type] || ELEMENT_COLORS.text
ctx.strokeStyle = color
ctx.lineWidth = 2
ctx.strokeRect(rect.x, rect.y, rect.w, rect.h)
ctx.fillStyle = color + '20'
ctx.fillRect(rect.x, rect.y, rect.w, rect.h)
}
}
function onMouseMove(e) {
const canvas = canvasRef.value
const img = props.imageEl
if (!canvas || !img || !props.pageData) return
const canvasRect = canvas.getBoundingClientRect()
const mx = e.clientX - canvasRect.left
const my = e.clientY - canvasRect.top
const scale = computeScale(img.clientWidth, img.clientHeight, props.pageData.width, props.pageData.height)
let found = null
for (const el of visibleElements.value) {
if (pointInRect(mx, my, bboxToRect(el.bbox, scale))) {
found = el
break
}
}
hoveredElement.value = found
if (found) {
tooltipStyle.value = {
left: `${Math.min(mx + 12, canvas.width - 250)}px`,
top: `${my + 12}px`
}
}
}
// Redraw on resize
let resizeObserver = null
onMounted(() => {
if (props.imageEl) {
resizeObserver = new ResizeObserver(() => draw())
resizeObserver.observe(props.imageEl)
}
})
onBeforeUnmount(() => {
resizeObserver?.disconnect()
})
// Redraw when data or image changes
watch([() => props.pageData, () => props.imageEl, hiddenTypes], () => {
nextTick(draw)
})
// Expose draw so parent can call it after image load
defineExpose({ draw })
</script>
<style scoped>
.bbox-overlay {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
pointer-events: none;
}
.overlay-legend {
position: absolute;
top: 8px;
left: 8px;
right: 8px;
display: flex;
flex-wrap: wrap;
gap: 4px;
z-index: 5;
pointer-events: auto;
}
.legend-chip {
display: flex;
align-items: center;
gap: 4px;
padding: 3px 8px;
background: var(--bg-surface);
border: 1px solid var(--border);
border-radius: 12px;
font-size: 10px;
color: var(--text-secondary);
cursor: pointer;
transition: all var(--transition);
backdrop-filter: blur(8px);
opacity: 0.9;
}
.legend-chip:hover { opacity: 1; background: var(--bg-hover); }
.legend-chip.dimmed { opacity: 0.35; }
.legend-dot {
width: 6px;
height: 6px;
border-radius: 50%;
flex-shrink: 0;
}
.legend-count {
color: var(--text-muted);
font-family: 'IBM Plex Mono', monospace;
}
.overlay-canvas {
position: absolute;
top: 0;
left: 0;
pointer-events: auto;
/* No CSS width/height dimensions set purely by canvas.width/height in JS
to avoid browser stretching the canvas content */
}
.tooltip {
position: absolute;
background: var(--bg-surface);
border: 1px solid var(--border);
border-radius: var(--radius-sm);
padding: 8px 12px;
max-width: 250px;
pointer-events: none;
z-index: 10;
box-shadow: 0 4px 12px rgba(0,0,0,0.3);
}
.tooltip-type {
display: block;
font-size: 11px;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.05em;
margin-bottom: 4px;
}
.tooltip-content {
display: block;
font-size: 12px;
color: var(--text-secondary);
line-height: 1.4;
word-break: break-word;
}
</style>

View file

@ -13,11 +13,6 @@
</div> </div>
<div class="tab-content"> <div class="tab-content">
<MarkdownViewer v-if="activeTab === 'text'" :content="store.currentAnalysis.contentMarkdown" /> <MarkdownViewer v-if="activeTab === 'text'" :content="store.currentAnalysis.contentMarkdown" />
<StructureViewer
v-else-if="activeTab === 'visual'"
:pages="store.currentPages"
:document-id="store.currentAnalysis.documentId"
/>
<div v-else-if="activeTab === 'markdown'" class="raw-markdown"> <div v-else-if="activeTab === 'markdown'" class="raw-markdown">
<pre class="raw-content">{{ store.currentAnalysis.contentMarkdown }}</pre> <pre class="raw-content">{{ store.currentAnalysis.contentMarkdown }}</pre>
</div> </div>
@ -44,7 +39,6 @@
import { ref } from 'vue' import { ref } from 'vue'
import { useAnalysisStore } from '../store.js' import { useAnalysisStore } from '../store.js'
import MarkdownViewer from './MarkdownViewer.vue' import MarkdownViewer from './MarkdownViewer.vue'
import StructureViewer from './StructureViewer.vue'
import ImageGallery from './ImageGallery.vue' import ImageGallery from './ImageGallery.vue'
const store = useAnalysisStore() const store = useAnalysisStore()
@ -52,7 +46,6 @@ const activeTab = ref('text')
const tabs = [ const tabs = [
{ id: 'text', label: 'Résultat du texte' }, { id: 'text', label: 'Résultat du texte' },
{ id: 'visual', label: 'Visuel' },
{ id: 'markdown', label: 'Markdown' }, { id: 'markdown', label: 'Markdown' },
{ id: 'images', label: 'Images' } { id: 'images', label: 'Images' }
] ]

View file

@ -100,14 +100,35 @@
</button> </button>
<span class="pdf-separator" /> <span class="pdf-separator" />
<span class="pdf-zoom">100%</span> <span class="pdf-zoom">100%</span>
<template v-if="hasAnalysisResults">
<span class="pdf-separator" />
<button
class="visual-toggle"
:class="{ active: visualMode }"
@click="visualMode = !visualMode"
>
<svg viewBox="0 0 20 20" fill="currentColor" class="btn-icon"><path d="M10 12a2 2 0 100-4 2 2 0 000 4z"/><path fill-rule="evenodd" d="M.458 10C1.732 5.943 5.522 3 10 3s8.268 2.943 9.542 7c-1.274 4.057-5.064 7-9.542 7S1.732 14.057.458 10zM14 10a4 4 0 11-8 0 4 4 0 018 0z" clip-rule="evenodd"/></svg>
Visuel
</button>
</template>
</div> </div>
<div class="pdf-image-area"> <div class="pdf-image-area">
<img <div class="pdf-image-wrapper">
v-if="previewUrl" <img
:src="previewUrl" v-if="previewUrl"
:alt="`Page ${currentPage}`" ref="pdfImageRef"
class="pdf-image" :src="previewUrl"
/> :alt="`Page ${currentPage}`"
class="pdf-image"
@load="onPdfImageLoad"
/>
<BboxOverlay
v-if="visualMode && hasAnalysisResults"
ref="bboxOverlayRef"
:image-el="pdfImageRef"
:page-data="currentPageData"
/>
</div>
</div> </div>
</div> </div>
@ -189,11 +210,12 @@
</template> </template>
<script setup> <script setup>
import { ref, computed, watch, onMounted, reactive } from 'vue' import { ref, computed, watch, nextTick, onMounted, reactive } from 'vue'
import { useDocumentStore } from '../features/document/store.js' import { useDocumentStore } from '../features/document/store.js'
import { useAnalysisStore } from '../features/analysis/store.js' import { useAnalysisStore } from '../features/analysis/store.js'
import { DocumentUpload, DocumentList } from '../features/document/index.js' import { DocumentUpload, DocumentList } from '../features/document/index.js'
import { ResultTabs } from '../features/analysis/index.js' import { ResultTabs } from '../features/analysis/index.js'
import BboxOverlay from '../features/analysis/ui/BboxOverlay.vue'
import { getPreviewUrl } from '../features/document/api.js' import { getPreviewUrl } from '../features/document/api.js'
const documentStore = useDocumentStore() const documentStore = useDocumentStore()
@ -203,6 +225,22 @@ const mode = ref('configurer')
const currentPage = ref(1) const currentPage = ref(1)
const pageRange = ref('') const pageRange = ref('')
const tableMode = ref('markdown') const tableMode = ref('markdown')
const visualMode = ref(false)
const pdfImageRef = ref(null)
const bboxOverlayRef = ref(null)
const hasAnalysisResults = computed(() => {
return analysisStore.currentAnalysis?.status === 'COMPLETED' && analysisStore.currentPages?.length > 0
})
const currentPageData = computed(() => {
if (!analysisStore.currentPages) return null
return analysisStore.currentPages.find(p => p.page_number === currentPage.value) || null
})
function onPdfImageLoad() {
nextTick(() => bboxOverlayRef.value?.draw())
}
const extractOptions = [ const extractOptions = [
{ id: 'images', label: 'Images', icon: 'image' }, { id: 'images', label: 'Images', icon: 'image' },
@ -605,6 +643,37 @@ onMounted(() => {
padding: 3px 10px; padding: 3px 10px;
} }
.visual-toggle {
display: flex;
align-items: center;
gap: 5px;
padding: 4px 10px;
font-size: 12px;
font-weight: 500;
color: var(--text-secondary);
background: var(--bg-elevated);
border: 1px solid var(--border);
border-radius: 4px;
cursor: pointer;
transition: all var(--transition);
}
.visual-toggle .btn-icon {
width: 14px;
height: 14px;
}
.visual-toggle:hover {
background: var(--bg-hover);
color: var(--text);
}
.visual-toggle.active {
background: var(--accent-muted);
border-color: var(--accent);
color: var(--accent);
}
.pdf-image-area { .pdf-image-area {
flex: 1; flex: 1;
overflow: auto; overflow: auto;
@ -614,9 +683,16 @@ onMounted(() => {
padding: 20px; padding: 20px;
} }
.pdf-image-wrapper {
position: relative;
display: inline-block;
max-width: 100%;
}
.pdf-image { .pdf-image {
max-width: 100%; max-width: 100%;
height: auto; height: auto;
display: block;
box-shadow: 0 2px 20px rgba(0,0,0,0.4); box-shadow: 0 2px 20px rgba(0,0,0,0.4);
border-radius: 2px; border-radius: 2px;
} }