Add chunk-to-bbox hover highlighting in Prepare mode
Extract bounding boxes from chunk doc_items provenance in the chunker, propagate through domain/service/API layers, and render highlighted bboxes on canvas when hovering a chunk card. Reset highlights on mode and page changes to prevent stale visual state.
This commit is contained in:
parent
ace37429bd
commit
4af6b5b231
9 changed files with 124 additions and 17 deletions
|
|
@ -7,7 +7,13 @@ from typing import Annotated
|
|||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||
|
||||
from api.schemas import AnalysisResponse, ChunkResponse, CreateAnalysisRequest, RechunkRequest
|
||||
from api.schemas import (
|
||||
AnalysisResponse,
|
||||
ChunkBboxResponse,
|
||||
ChunkResponse,
|
||||
CreateAnalysisRequest,
|
||||
RechunkRequest,
|
||||
)
|
||||
from services.analysis_service import AnalysisService
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -94,6 +100,7 @@ async def rechunk_analysis(job_id: str, body: RechunkRequest, service: ServiceDe
|
|||
headings=c.headings,
|
||||
source_page=c.source_page,
|
||||
token_count=c.token_count,
|
||||
bboxes=[ChunkBboxResponse(page=b.page, bbox=b.bbox) for b in c.bboxes],
|
||||
)
|
||||
for c in chunks
|
||||
]
|
||||
|
|
|
|||
|
|
@ -103,11 +103,17 @@ class ChunkingOptionsRequest(BaseModel):
|
|||
return v
|
||||
|
||||
|
||||
class ChunkBboxResponse(_CamelModel):
|
||||
page: int
|
||||
bbox: list[float]
|
||||
|
||||
|
||||
class ChunkResponse(_CamelModel):
|
||||
text: str
|
||||
headings: list[str] = []
|
||||
source_page: int | None = None
|
||||
token_count: int = 0
|
||||
bboxes: list[ChunkBboxResponse] = []
|
||||
|
||||
|
||||
class CreateAnalysisRequest(BaseModel):
|
||||
|
|
|
|||
|
|
@ -63,9 +63,16 @@ class ChunkingOptions:
|
|||
return self == ChunkingOptions()
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkBbox:
|
||||
page: int
|
||||
bbox: list[float] # [left, top, right, bottom] in TOPLEFT origin
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChunkResult:
|
||||
text: str
|
||||
headings: list[str] = field(default_factory=list)
|
||||
source_page: int | None = None
|
||||
token_count: int = 0
|
||||
bboxes: list[ChunkBbox] = field(default_factory=list)
|
||||
|
|
|
|||
|
|
@ -15,7 +15,8 @@ from docling_core.transforms.chunker import HierarchicalChunker
|
|||
from docling_core.transforms.chunker.hybrid_chunker import HybridChunker
|
||||
from docling_core.types.doc.document import DoclingDocument
|
||||
|
||||
from domain.value_objects import ChunkingOptions, ChunkResult
|
||||
from domain.bbox import EMPTY_BBOX, to_topleft_list
|
||||
from domain.value_objects import ChunkBbox, ChunkingOptions, ChunkResult
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -37,12 +38,22 @@ def _chunk_sync(document_json: str, options: ChunkingOptions) -> list[ChunkResul
|
|||
for chunk in chunker.chunk(doc):
|
||||
source_page = None
|
||||
token_count = 0
|
||||
bboxes: list[ChunkBbox] = []
|
||||
|
||||
if hasattr(chunk, "meta") and chunk.meta and chunk.meta.doc_items:
|
||||
for doc_item in chunk.meta.doc_items:
|
||||
if hasattr(doc_item, "prov") and doc_item.prov:
|
||||
source_page = doc_item.prov[0].page_no
|
||||
break
|
||||
if not hasattr(doc_item, "prov") or not doc_item.prov:
|
||||
continue
|
||||
for prov in doc_item.prov:
|
||||
page_no = prov.page_no
|
||||
if source_page is None:
|
||||
source_page = page_no
|
||||
if prov.bbox:
|
||||
page_obj = doc.pages.get(page_no)
|
||||
if page_obj:
|
||||
bbox = to_topleft_list(prov.bbox, page_obj.size.height)
|
||||
if bbox != EMPTY_BBOX:
|
||||
bboxes.append(ChunkBbox(page=page_no, bbox=bbox))
|
||||
|
||||
if hasattr(chunker, "tokenizer") and chunker.tokenizer:
|
||||
token_count = chunker.tokenizer.count_tokens(chunk.text)
|
||||
|
|
@ -55,6 +66,7 @@ def _chunk_sync(document_json: str, options: ChunkingOptions) -> list[ChunkResul
|
|||
headings=headings,
|
||||
source_page=source_page,
|
||||
token_count=token_count,
|
||||
bboxes=bboxes,
|
||||
)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ def _chunk_to_dict(c: ChunkResult) -> dict:
|
|||
"headings": c.headings,
|
||||
"sourcePage": c.source_page,
|
||||
"tokenCount": c.token_count,
|
||||
"bboxes": [{"page": b.page, "bbox": b.bbox} for b in c.bboxes],
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -38,7 +38,7 @@
|
|||
<script setup lang="ts">
|
||||
import { ref, computed, watch, nextTick, reactive, onMounted, onBeforeUnmount } from 'vue'
|
||||
import { computeScale, bboxToRect, pointInRect } from '../bboxScaling'
|
||||
import type { Page, PageElement } from '../../../shared/types'
|
||||
import type { Page, PageElement, ChunkBbox } from '../../../shared/types'
|
||||
|
||||
const ELEMENT_COLORS: Record<string, string> = {
|
||||
title: '#EF4444',
|
||||
|
|
@ -52,11 +52,15 @@ const ELEMENT_COLORS: Record<string, string> = {
|
|||
caption: '#EAB308'
|
||||
}
|
||||
|
||||
const props = defineProps<{
|
||||
imageEl: HTMLImageElement | null
|
||||
pageData: Page | null
|
||||
highlightedIndex: number
|
||||
}>()
|
||||
const props = withDefaults(
|
||||
defineProps<{
|
||||
imageEl: HTMLImageElement | null
|
||||
pageData: Page | null
|
||||
highlightedIndex: number
|
||||
highlightedBboxes: ChunkBbox[]
|
||||
}>(),
|
||||
{ highlightedBboxes: () => [] },
|
||||
)
|
||||
|
||||
const emit = defineEmits<{
|
||||
'highlight-element': [index: number]
|
||||
|
|
@ -114,6 +118,8 @@ function draw(): void {
|
|||
|
||||
const scale = computeScale(img.clientWidth, img.clientHeight, props.pageData.width, props.pageData.height)
|
||||
|
||||
const hasChunkHighlight = props.highlightedBboxes.length > 0
|
||||
|
||||
for (const el of visibleElements.value) {
|
||||
const rect = bboxToRect(el.bbox, scale)
|
||||
const color = ELEMENT_COLORS[el.type] || ELEMENT_COLORS.text
|
||||
|
|
@ -121,13 +127,29 @@ function draw(): void {
|
|||
const elContentIdx = contentElements.value.indexOf(el)
|
||||
const isHighlighted = props.highlightedIndex >= 0 && elContentIdx === props.highlightedIndex
|
||||
|
||||
ctx.strokeStyle = color
|
||||
// Dim non-highlighted elements when a chunk is hovered
|
||||
const dimmed = hasChunkHighlight && !isHighlighted
|
||||
|
||||
ctx.strokeStyle = dimmed ? color + '40' : color
|
||||
ctx.lineWidth = isHighlighted ? 3 : 2
|
||||
ctx.strokeRect(rect.x, rect.y, rect.w, rect.h)
|
||||
|
||||
ctx.fillStyle = color + (isHighlighted ? '40' : '20')
|
||||
ctx.fillStyle = color + (isHighlighted ? '40' : dimmed ? '08' : '20')
|
||||
ctx.fillRect(rect.x, rect.y, rect.w, rect.h)
|
||||
}
|
||||
|
||||
// Draw chunk-highlighted bboxes on top with accent color
|
||||
if (hasChunkHighlight) {
|
||||
const CHUNK_COLOR = '#F59E0B'
|
||||
for (const cb of props.highlightedBboxes) {
|
||||
const rect = bboxToRect(cb.bbox, scale)
|
||||
ctx.strokeStyle = CHUNK_COLOR
|
||||
ctx.lineWidth = 3
|
||||
ctx.strokeRect(rect.x, rect.y, rect.w, rect.h)
|
||||
ctx.fillStyle = CHUNK_COLOR + '30'
|
||||
ctx.fillRect(rect.x, rect.y, rect.w, rect.h)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function onMouseMove(e: MouseEvent): void {
|
||||
|
|
@ -173,7 +195,7 @@ onBeforeUnmount(() => {
|
|||
})
|
||||
|
||||
// Redraw when data or image changes
|
||||
watch([() => props.pageData, () => props.imageEl, () => props.highlightedIndex, hiddenTypes], () => {
|
||||
watch([() => props.pageData, () => props.imageEl, () => props.highlightedIndex, () => props.highlightedBboxes, hiddenTypes], () => {
|
||||
nextTick(draw)
|
||||
})
|
||||
|
||||
|
|
|
|||
|
|
@ -76,6 +76,9 @@
|
|||
class="chunk-card"
|
||||
v-for="(chunk, localIdx) in pagination.paginatedItems.value"
|
||||
:key="globalIndex(localIdx)"
|
||||
:class="{ highlighted: hoveredChunkIdx === globalIndex(localIdx) }"
|
||||
@mouseenter="onChunkHover(chunk, localIdx)"
|
||||
@mouseleave="onChunkLeave"
|
||||
>
|
||||
<div class="chunk-header">
|
||||
<span class="chunk-index">#{{ globalIndex(localIdx) + 1 }}</span>
|
||||
|
|
@ -121,12 +124,16 @@ import { useAnalysisStore } from '../../analysis/store'
|
|||
import { useI18n } from '../../../shared/i18n'
|
||||
import { usePagination } from '../../../shared/composables/usePagination'
|
||||
import { PaginationBar } from '../../../shared/ui'
|
||||
import type { ChunkingOptions } from '../../../shared/types'
|
||||
import type { Chunk, ChunkBbox, ChunkingOptions } from '../../../shared/types'
|
||||
|
||||
const props = defineProps<{
|
||||
currentPage: number
|
||||
}>()
|
||||
|
||||
const emit = defineEmits<{
|
||||
'highlight-bboxes': [bboxes: ChunkBbox[]]
|
||||
}>()
|
||||
|
||||
const analysisStore = useAnalysisStore()
|
||||
const { t } = useI18n()
|
||||
|
||||
|
|
@ -153,6 +160,19 @@ function globalIndex(localIdx: number): number {
|
|||
return (pagination.page.value - 1) * pagination.pageSize.value + localIdx
|
||||
}
|
||||
|
||||
const hoveredChunkIdx = ref(-1)
|
||||
|
||||
function onChunkHover(chunk: Chunk, localIdx: number) {
|
||||
hoveredChunkIdx.value = globalIndex(localIdx)
|
||||
const pageBboxes = chunk.bboxes.filter((b) => b.page === props.currentPage)
|
||||
emit('highlight-bboxes', pageBboxes)
|
||||
}
|
||||
|
||||
function onChunkLeave() {
|
||||
hoveredChunkIdx.value = -1
|
||||
emit('highlight-bboxes', [])
|
||||
}
|
||||
|
||||
async function doRechunk() {
|
||||
if (!analysisStore.currentAnalysis) return
|
||||
await analysisStore.rechunk(analysisStore.currentAnalysis.id, { ...options })
|
||||
|
|
@ -335,6 +355,17 @@ async function doRechunk() {
|
|||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
padding: 10px 12px;
|
||||
cursor: default;
|
||||
transition: border-color 0.15s, background 0.15s;
|
||||
}
|
||||
|
||||
.chunk-card:hover {
|
||||
border-color: #F59E0B;
|
||||
}
|
||||
|
||||
.chunk-card.highlighted {
|
||||
border-color: #F59E0B;
|
||||
background: rgba(245, 158, 11, 0.08);
|
||||
}
|
||||
|
||||
.chunk-header {
|
||||
|
|
|
|||
|
|
@ -141,6 +141,7 @@
|
|||
:image-el="pdfImageRef"
|
||||
:page-data="currentPageData"
|
||||
:highlighted-index="highlightedElementIndex"
|
||||
:highlighted-bboxes="highlightedChunkBboxes"
|
||||
@highlight-element="highlightedElementIndex = $event"
|
||||
/>
|
||||
</div>
|
||||
|
|
@ -292,7 +293,10 @@
|
|||
|
||||
<!-- PREPARER MODE (feature-flipped) -->
|
||||
<div v-if="mode === 'preparer' && chunkingEnabled" class="prepare-panel">
|
||||
<ChunkPanel :current-page="currentPage" />
|
||||
<ChunkPanel
|
||||
:current-page="currentPage"
|
||||
@highlight-bboxes="highlightedChunkBboxes = $event"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -311,7 +315,7 @@ import { ChunkPanel } from '../features/chunking'
|
|||
import { useFeatureFlag } from '../features/feature-flags'
|
||||
import { getPreviewUrl } from '../features/document/api'
|
||||
import { useI18n } from '../shared/i18n'
|
||||
import type { PipelineOptions } from '../shared/types'
|
||||
import type { ChunkBbox, PipelineOptions } from '../shared/types'
|
||||
|
||||
const route = useRoute()
|
||||
const router = useRouter()
|
||||
|
|
@ -324,6 +328,7 @@ const mode = ref('configurer')
|
|||
const currentPage = ref(1)
|
||||
const visualMode = ref(false)
|
||||
const highlightedElementIndex = ref(-1)
|
||||
const highlightedChunkBboxes = ref<ChunkBbox[]>([])
|
||||
const pdfImageRef = ref<HTMLImageElement | null>(null)
|
||||
const bboxOverlayRef = ref<InstanceType<typeof BboxOverlay> | null>(null)
|
||||
|
||||
|
|
@ -410,6 +415,16 @@ function addMore() {
|
|||
documentStore.selectedId = null
|
||||
}
|
||||
|
||||
// Clear highlights when switching modes or pages
|
||||
watch(mode, () => {
|
||||
highlightedElementIndex.value = -1
|
||||
highlightedChunkBboxes.value = []
|
||||
})
|
||||
watch(currentPage, () => {
|
||||
highlightedElementIndex.value = -1
|
||||
highlightedChunkBboxes.value = []
|
||||
})
|
||||
|
||||
// Auto-switch to verifier when analysis completes + refresh document data (pageCount)
|
||||
watch(() => analysisStore.currentAnalysis?.status, (status) => {
|
||||
if (status === 'COMPLETED') {
|
||||
|
|
|
|||
|
|
@ -45,11 +45,17 @@ export interface ChunkingOptions {
|
|||
repeat_table_header?: boolean
|
||||
}
|
||||
|
||||
export interface ChunkBbox {
|
||||
page: number
|
||||
bbox: [number, number, number, number]
|
||||
}
|
||||
|
||||
export interface Chunk {
|
||||
text: string
|
||||
headings: string[]
|
||||
sourcePage: number | null
|
||||
tokenCount: number
|
||||
bboxes: ChunkBbox[]
|
||||
}
|
||||
|
||||
export interface PageElement {
|
||||
|
|
|
|||
Loading…
Reference in a new issue