danvics/docling-studio
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Fix display error on visual mode

Browse source
This commit is contained in:
pjmalandrino 2026-03-17 10:28:50 +01:00
parent 9c2e42ace8
commit f1aa39b75f
9 changed files with 411 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
backend/src/main/java/com/docling/studio/analysis/AnalysisService.java
View file

@ -61,6 +61,7 @@ public class AnalysisService {
if (pageCount instanceof Number n && n.intValue() > 0) {
Document doc = job.getDocument();
doc.setPageCount(n.intValue());
documentService.save(doc);
}
job.markCompleted(markdown, html, pagesJson);

4
backend/src/main/java/com/docling/studio/document/DocumentService.java
View file

@ -53,6 +53,10 @@ public class DocumentService {
}
}
public Document save(Document document) {
return repository.save(document);
}
public List<Document> findAll() {
return repository.findAll();
}

90
document-parser/main.py
View file

@ -1,4 +1,5 @@
import io
import logging
import os
import tempfile
from pathlib import Path
@ -12,6 +13,8 @@ from PIL import Image
from bbox import to_topleft_list
logger = logging.getLogger(__name__)
app = FastAPI(title="Docling Studio - Document Parser")
converter = DocumentConverter()
@ -45,7 +48,11 @@ class ParseResponse(BaseModel):
# --- Helpers ---
def extract_pages_detail(doc_result) -> list[PageDetail]:
"""Extract per-page element details with bounding boxes from Docling result."""
"""Extract per-page element details with bounding boxes from Docling result.
Uses Docling's iterate_items() API (preferred) or falls back to document.texts.
Both provide a flat iteration over all content items, avoiding duplicates.
"""
pages: dict[int, PageDetail] = {}
document = doc_result.document
@ -62,12 +69,11 @@ def extract_pages_detail(doc_result) -> list[PageDetail]:
elements=[]
)
# Iterate document items to extract elements with bounding boxes
if hasattr(document, 'body') and hasattr(document.body, 'children'):
_extract_elements_recursive(document, document.body.children, pages)
# Fallback: if we have content items
if hasattr(document, 'texts'):
# Use iterate_items() (Docling v2 API) — avoids duplicates
if hasattr(document, 'iterate_items'):
for item, _level in document.iterate_items():
_process_content_item(item, pages)
elif hasattr(document, 'texts'):
for text_item in document.texts:
_process_content_item(text_item, pages)
@ -76,56 +82,47 @@ def extract_pages_detail(doc_result) -> list[PageDetail]:
def _process_content_item(item, pages: dict[int, PageDetail]):
"""Process a single content item and add it to the appropriate page."""
"""Process a single content item and add it to the appropriate page.
Silently skips items that lack provenance or fail to process,
so one bad item doesn't break the whole extraction.
"""
if not hasattr(item, 'prov') or not item.prov:
return
for prov in item.prov:
page_no = prov.page_no if hasattr(prov, 'page_no') else 1
try:
page_no = prov.page_no if hasattr(prov, 'page_no') else 1
if page_no not in pages:
pages[page_no] = PageDetail(
page_number=page_no, width=612.0, height=792.0, elements=[]
)
if page_no not in pages:
pages[page_no] = PageDetail(
page_number=page_no, width=612.0, height=792.0, elements=[]
)
page_height = pages[page_no].height
page_height = pages[page_no].height
bbox = [0, 0, 0, 0]
if hasattr(prov, 'bbox') and prov.bbox:
b = prov.bbox
if hasattr(b, 'l'):
bbox = to_topleft_list(b, page_height)
elif isinstance(b, (list, tuple)) and len(b) >= 4:
bbox = list(b[:4])
bbox = [0, 0, 0, 0]
if hasattr(prov, 'bbox') and prov.bbox:
b = prov.bbox
if hasattr(b, 'l'):
bbox = to_topleft_list(b, page_height)
elif isinstance(b, (list, tuple)) and len(b) >= 4:
bbox = list(b[:4])
element_type = _get_element_type(item)
content = ""
if hasattr(item, 'text'):
content = item.text or ""
elif hasattr(item, 'export_to_markdown'):
content = item.export_to_markdown()
element_type = _get_element_type(item)
content = ""
if hasattr(item, 'text'):
content = item.text or ""
pages[page_no].elements.append(PageElement(
type=element_type,
bbox=bbox,
content=content[:500] # Truncate long content
))
pages[page_no].elements.append(PageElement(
type=element_type,
bbox=bbox,
content=content[:500]
))
except Exception:
logger.warning("Skipping item %s: failed to process", type(item).__name__, exc_info=True)
def _extract_elements_recursive(document, children, pages: dict[int, PageDetail]):
"""Recursively extract elements from document tree."""
if not children:
return
for child_ref in children:
# Resolve reference if needed
item = child_ref
if hasattr(child_ref, '__ref__'):
item = document.resolve_ref(child_ref)
if item:
_process_content_item(item, pages)
if hasattr(item, 'children') and item.children:
_extract_elements_recursive(document, item.children, pages)
def _get_element_type(item) -> str:
"""Determine the element type from a Docling document item."""
@ -201,6 +198,7 @@ async def parse(file: UploadFile):
)
except Exception as e:
logger.exception("Failed to parse document: %s", file.filename)
raise HTTPException(status_code=422, detail=f"Failed to parse document: {str(e)}")
finally:
if tmp_path and os.path.exists(tmp_path):

16
frontend/src/features/analysis/bboxScaling.js
View file

@ -3,15 +3,19 @@
* to canvas pixel coordinates.
*
* Docling bbox format (after backend normalization): [l, t, r, b]
* All values are in page coordinate space (points).
* All values are in page coordinate space (points, 1pt = 1/72 inch).
* Origin: top-left (y=0 at top of page).
*
* The preview image is a faithful rasterization of the PDF page.
* CSS `max-width: 100%; height: auto` preserves the aspect ratio,
* so sx and sy are always equal. We compute both for completeness.
*/
/**
* Compute scale factors from page coordinates to displayed image pixels.
* Compute scale factors from page coordinates to displayed pixels.
*
* @param {number} displayWidth - Rendered image width in CSS pixels
* @param {number} displayHeight - Rendered image height in CSS pixels
* @param {number} displayWidth - img.clientWidth (CSS pixels)
* @param {number} displayHeight - img.clientHeight (CSS pixels)
* @param {number} pageWidth - Page width in Docling points
* @param {number} pageHeight - Page height in Docling points
* @returns {{ sx: number, sy: number }}
@ -26,8 +30,8 @@ export function computeScale(displayWidth, displayHeight, pageWidth, pageHeight)
/**
* Convert a Docling bbox [l, t, r, b] to a canvas rect { x, y, w, h }.
*
* @param {number[]} bbox - [left, top, right, bottom] in page points
* @param {{ sx: number, sy: number }} scale - Scale factors from computeScale
* @param {number[]} bbox - [left, top, right, bottom] in page points
* @param {{ sx: number, sy: number }} scale
* @returns {{ x: number, y: number, w: number, h: number }}
*/
export function bboxToRect(bbox, scale) {

18
frontend/src/features/analysis/bboxScaling.test.js
View file

@ -13,6 +13,15 @@ describe('computeScale', () => {
expect(s.sx).toBeCloseTo(0.5)
expect(s.sy).toBeCloseTo(0.5)
})
it('sx equals sy when aspect ratio is preserved', () => {
// Image at 150 DPI displayed at 700px wide
const pageW = 612, pageH = 792
const displayW = 700
const displayH = displayW * pageH / pageW // preserves ratio
const s = computeScale(displayW, displayH, pageW, pageH)
expect(s.sx).toBeCloseTo(s.sy, 5)
})
})
describe('bboxToRect', () => {
@ -36,6 +45,15 @@ describe('bboxToRect', () => {
expect(rect.w).toBeCloseTo(100)
expect(rect.h).toBeCloseTo(100)
})
it('end-to-end: full page bbox fills display', () => {
const scale = computeScale(700, 907.84, 612, 792)
const rect = bboxToRect([0, 0, 612, 792], scale)
expect(rect.x).toBeCloseTo(0)
expect(rect.y).toBeCloseTo(0)
expect(rect.w).toBeCloseTo(700)
expect(rect.h).toBeCloseTo(907.84, 0)
})
})
describe('pointInRect', () => {

1
frontend/src/features/analysis/index.js
View file

@ -3,4 +3,5 @@ export { default as AnalysisPanel } from './ui/AnalysisPanel.vue'
export { default as ResultTabs } from './ui/ResultTabs.vue'
export { default as MarkdownViewer } from './ui/MarkdownViewer.vue'
export { default as StructureViewer } from './ui/StructureViewer.vue'
export { default as BboxOverlay } from './ui/BboxOverlay.vue'
export { default as ImageGallery } from './ui/ImageGallery.vue'

250
frontend/src/features/analysis/ui/BboxOverlay.vue Normal file
View file

@ -0,0 +1,250 @@
<template>
<div class="bbox-overlay">
<!-- Legend bar -->
<div class="overlay-legend" v-if="pageData">
<button
v-for="[type, color] in Object.entries(ELEMENT_COLORS)"
:key="type"
class="legend-chip"
:class="{ dimmed: hiddenTypes.has(type) }"
@click="toggleType(type)"
>
<span class="legend-dot" :style="{ background: color }" />
<span>{{ type }}</span>
<span class="legend-count">{{ countElements(type) }}</span>
</button>
</div>
<!-- Canvas + Tooltip (positioned over image via parent) -->
<canvas
ref="canvasRef"
class="overlay-canvas"
@mousemove="onMouseMove"
@mouseleave="hoveredElement = null"
/>
<div
v-if="hoveredElement"
class="tooltip"
:style="tooltipStyle"
>
<span class="tooltip-type" :style="{ color: ELEMENT_COLORS[hoveredElement.type] || ELEMENT_COLORS.text }">
{{ hoveredElement.type }}
</span>
<span class="tooltip-content">{{ hoveredElement.content?.substring(0, 150) }}</span>
</div>
</div>
</template>
<script setup>
import { ref, computed, watch, nextTick, reactive, onMounted, onBeforeUnmount } from 'vue'
import { computeScale, bboxToRect, pointInRect } from '../bboxScaling.js'
const ELEMENT_COLORS = {
section_header: '#F97316',
text: '#3B82F6',
table: '#8B5CF6',
picture: '#22C55E',
list: '#06B6D4',
formula: '#EC4899',
caption: '#EAB308'
}
const props = defineProps({
/** The <img> element to overlay onto */
imageEl: { type: Object, default: null },
/** Page data { page_number, width, height, elements[] } for current page */
pageData: { type: Object, default: null }
})
const hiddenTypes = reactive(new Set())
const canvasRef = ref(null)
const hoveredElement = ref(null)
const tooltipStyle = ref({})
const visibleElements = computed(() => {
if (!props.pageData) return []
return props.pageData.elements.filter(e => !hiddenTypes.has(e.type))
})
function toggleType(type) {
if (hiddenTypes.has(type)) hiddenTypes.delete(type)
else hiddenTypes.add(type)
draw()
}
function countElements(type) {
if (!props.pageData) return 0
return props.pageData.elements.filter(e => e.type === type).length
}
function draw() {
const canvas = canvasRef.value
const img = props.imageEl
if (!canvas || !img) return
// Wait until image is loaded (clientWidth > 0)
if (!img.clientWidth || !img.clientHeight) return
canvas.width = img.clientWidth
canvas.height = img.clientHeight
const ctx = canvas.getContext('2d')
ctx.clearRect(0, 0, canvas.width, canvas.height)
if (!props.pageData) return
const scale = computeScale(img.clientWidth, img.clientHeight, props.pageData.width, props.pageData.height)
for (const el of visibleElements.value) {
const rect = bboxToRect(el.bbox, scale)
const color = ELEMENT_COLORS[el.type] || ELEMENT_COLORS.text
ctx.strokeStyle = color
ctx.lineWidth = 2
ctx.strokeRect(rect.x, rect.y, rect.w, rect.h)
ctx.fillStyle = color + '20'
ctx.fillRect(rect.x, rect.y, rect.w, rect.h)
}
}
function onMouseMove(e) {
const canvas = canvasRef.value
const img = props.imageEl
if (!canvas || !img || !props.pageData) return
const canvasRect = canvas.getBoundingClientRect()
const mx = e.clientX - canvasRect.left
const my = e.clientY - canvasRect.top
const scale = computeScale(img.clientWidth, img.clientHeight, props.pageData.width, props.pageData.height)
let found = null
for (const el of visibleElements.value) {
if (pointInRect(mx, my, bboxToRect(el.bbox, scale))) {
found = el
break
}
}
hoveredElement.value = found
if (found) {
tooltipStyle.value = {
left: `${Math.min(mx + 12, canvas.width - 250)}px`,
top: `${my + 12}px`
}
}
}
// Redraw on resize
let resizeObserver = null
onMounted(() => {
if (props.imageEl) {
resizeObserver = new ResizeObserver(() => draw())
resizeObserver.observe(props.imageEl)
}
})
onBeforeUnmount(() => {
resizeObserver?.disconnect()
})
// Redraw when data or image changes
watch([() => props.pageData, () => props.imageEl, hiddenTypes], () => {
nextTick(draw)
})
// Expose draw so parent can call it after image load
defineExpose({ draw })
</script>
<style scoped>
.bbox-overlay {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
pointer-events: none;
}
.overlay-legend {
position: absolute;
top: 8px;
left: 8px;
right: 8px;
display: flex;
flex-wrap: wrap;
gap: 4px;
z-index: 5;
pointer-events: auto;
}
.legend-chip {
display: flex;
align-items: center;
gap: 4px;
padding: 3px 8px;
background: var(--bg-surface);
border: 1px solid var(--border);
border-radius: 12px;
font-size: 10px;
color: var(--text-secondary);
cursor: pointer;
transition: all var(--transition);
backdrop-filter: blur(8px);
opacity: 0.9;
}
.legend-chip:hover { opacity: 1; background: var(--bg-hover); }
.legend-chip.dimmed { opacity: 0.35; }
.legend-dot {
width: 6px;
height: 6px;
border-radius: 50%;
flex-shrink: 0;
}
.legend-count {
color: var(--text-muted);
font-family: 'IBM Plex Mono', monospace;
}
.overlay-canvas {
position: absolute;
top: 0;
left: 0;
pointer-events: auto;
/* No CSS width/height — dimensions set purely by canvas.width/height in JS
to avoid browser stretching the canvas content */
}
.tooltip {
position: absolute;
background: var(--bg-surface);
border: 1px solid var(--border);
border-radius: var(--radius-sm);
padding: 8px 12px;
max-width: 250px;
pointer-events: none;
z-index: 10;
box-shadow: 0 4px 12px rgba(0,0,0,0.3);
}
.tooltip-type {
display: block;
font-size: 11px;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.05em;
margin-bottom: 4px;
}
.tooltip-content {
display: block;
font-size: 12px;
color: var(--text-secondary);
line-height: 1.4;
word-break: break-word;
}
</style>

7
frontend/src/features/analysis/ui/ResultTabs.vue
View file

@ -13,11 +13,6 @@
</div>
<div class="tab-content">
<MarkdownViewer v-if="activeTab === 'text'" :content="store.currentAnalysis.contentMarkdown" />
<StructureViewer
v-else-if="activeTab === 'visual'"
:pages="store.currentPages"
:document-id="store.currentAnalysis.documentId"
/>
<div v-else-if="activeTab === 'markdown'" class="raw-markdown">
<pre class="raw-content">{{ store.currentAnalysis.contentMarkdown }}</pre>
</div>
@ -44,7 +39,6 @@
import { ref } from 'vue'
import { useAnalysisStore } from '../store.js'
import MarkdownViewer from './MarkdownViewer.vue'
import StructureViewer from './StructureViewer.vue'
import ImageGallery from './ImageGallery.vue'
const store = useAnalysisStore()
@ -52,7 +46,6 @@ const activeTab = ref('text')
const tabs = [
{ id: 'text', label: 'Résultat du texte' },
{ id: 'visual', label: 'Visuel' },
{ id: 'markdown', label: 'Markdown' },
{ id: 'images', label: 'Images' }
]

90
frontend/src/pages/StudioPage.vue
View file

@ -100,14 +100,35 @@
</button>
<span class="pdf-separator" />
<span class="pdf-zoom">100%</span>
<template v-if="hasAnalysisResults">
<span class="pdf-separator" />
<button
class="visual-toggle"
:class="{ active: visualMode }"
@click="visualMode = !visualMode"
>
<svg viewBox="0 0 20 20" fill="currentColor" class="btn-icon"><path d="M10 12a2 2 0 100-4 2 2 0 000 4z"/><path fill-rule="evenodd" d="M.458 10C1.732 5.943 5.522 3 10 3s8.268 2.943 9.542 7c-1.274 4.057-5.064 7-9.542 7S1.732 14.057.458 10zM14 10a4 4 0 11-8 0 4 4 0 018 0z" clip-rule="evenodd"/></svg>
Visuel
</button>
</template>
</div>
<div class="pdf-image-area">
<img
v-if="previewUrl"
:src="previewUrl"
:alt="`Page ${currentPage}`"
class="pdf-image"
/>
<div class="pdf-image-wrapper">
<img
v-if="previewUrl"
ref="pdfImageRef"
:src="previewUrl"
:alt="`Page ${currentPage}`"
class="pdf-image"
@load="onPdfImageLoad"
/>
<BboxOverlay
v-if="visualMode && hasAnalysisResults"
ref="bboxOverlayRef"
:image-el="pdfImageRef"
:page-data="currentPageData"
/>
</div>
</div>
</div>
@ -189,11 +210,12 @@
</template>
<script setup>
import { ref, computed, watch, onMounted, reactive } from 'vue'
import { ref, computed, watch, nextTick, onMounted, reactive } from 'vue'
import { useDocumentStore } from '../features/document/store.js'
import { useAnalysisStore } from '../features/analysis/store.js'
import { DocumentUpload, DocumentList } from '../features/document/index.js'
import { ResultTabs } from '../features/analysis/index.js'
import BboxOverlay from '../features/analysis/ui/BboxOverlay.vue'
import { getPreviewUrl } from '../features/document/api.js'
const documentStore = useDocumentStore()
@ -203,6 +225,22 @@ const mode = ref('configurer')
const currentPage = ref(1)
const pageRange = ref('')
const tableMode = ref('markdown')
const visualMode = ref(false)
const pdfImageRef = ref(null)
const bboxOverlayRef = ref(null)
const hasAnalysisResults = computed(() => {
return analysisStore.currentAnalysis?.status === 'COMPLETED' && analysisStore.currentPages?.length > 0
})
const currentPageData = computed(() => {
if (!analysisStore.currentPages) return null
return analysisStore.currentPages.find(p => p.page_number === currentPage.value) || null
})
function onPdfImageLoad() {
nextTick(() => bboxOverlayRef.value?.draw())
}
const extractOptions = [
{ id: 'images', label: 'Images', icon: 'image' },
@ -605,6 +643,37 @@ onMounted(() => {
padding: 3px 10px;
}
.visual-toggle {
display: flex;
align-items: center;
gap: 5px;
padding: 4px 10px;
font-size: 12px;
font-weight: 500;
color: var(--text-secondary);
background: var(--bg-elevated);
border: 1px solid var(--border);
border-radius: 4px;
cursor: pointer;
transition: all var(--transition);
}
.visual-toggle .btn-icon {
width: 14px;
height: 14px;
}
.visual-toggle:hover {
background: var(--bg-hover);
color: var(--text);
}
.visual-toggle.active {
background: var(--accent-muted);
border-color: var(--accent);
color: var(--accent);
}
.pdf-image-area {
flex: 1;
overflow: auto;
@ -614,9 +683,16 @@ onMounted(() => {
padding: 20px;
}
.pdf-image-wrapper {
position: relative;
display: inline-block;
max-width: 100%;
}
.pdf-image {
max-width: 100%;
height: auto;
display: block;
box-shadow: 0 2px 20px rgba(0,0,0,0.4);
border-radius: 2px;
}