Add chunk-to-bbox hover highlighting in Prepare mode
Extract bounding boxes from chunk doc_items provenance in the chunker, propagate through domain/service/API layers, and render highlighted bboxes on canvas when hovering a chunk card. Reset highlights on mode and page changes to prevent stale visual state.
This commit is contained in:
parent
ace37429bd
commit
4af6b5b231
9 changed files with 124 additions and 17 deletions
|
|
@ -7,7 +7,13 @@ from typing import Annotated
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, HTTPException, Request
|
from fastapi import APIRouter, Depends, HTTPException, Request
|
||||||
|
|
||||||
from api.schemas import AnalysisResponse, ChunkResponse, CreateAnalysisRequest, RechunkRequest
|
from api.schemas import (
|
||||||
|
AnalysisResponse,
|
||||||
|
ChunkBboxResponse,
|
||||||
|
ChunkResponse,
|
||||||
|
CreateAnalysisRequest,
|
||||||
|
RechunkRequest,
|
||||||
|
)
|
||||||
from services.analysis_service import AnalysisService
|
from services.analysis_service import AnalysisService
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -94,6 +100,7 @@ async def rechunk_analysis(job_id: str, body: RechunkRequest, service: ServiceDe
|
||||||
headings=c.headings,
|
headings=c.headings,
|
||||||
source_page=c.source_page,
|
source_page=c.source_page,
|
||||||
token_count=c.token_count,
|
token_count=c.token_count,
|
||||||
|
bboxes=[ChunkBboxResponse(page=b.page, bbox=b.bbox) for b in c.bboxes],
|
||||||
)
|
)
|
||||||
for c in chunks
|
for c in chunks
|
||||||
]
|
]
|
||||||
|
|
|
||||||
|
|
@ -103,11 +103,17 @@ class ChunkingOptionsRequest(BaseModel):
|
||||||
return v
|
return v
|
||||||
|
|
||||||
|
|
||||||
|
class ChunkBboxResponse(_CamelModel):
|
||||||
|
page: int
|
||||||
|
bbox: list[float]
|
||||||
|
|
||||||
|
|
||||||
class ChunkResponse(_CamelModel):
|
class ChunkResponse(_CamelModel):
|
||||||
text: str
|
text: str
|
||||||
headings: list[str] = []
|
headings: list[str] = []
|
||||||
source_page: int | None = None
|
source_page: int | None = None
|
||||||
token_count: int = 0
|
token_count: int = 0
|
||||||
|
bboxes: list[ChunkBboxResponse] = []
|
||||||
|
|
||||||
|
|
||||||
class CreateAnalysisRequest(BaseModel):
|
class CreateAnalysisRequest(BaseModel):
|
||||||
|
|
|
||||||
|
|
@ -63,9 +63,16 @@ class ChunkingOptions:
|
||||||
return self == ChunkingOptions()
|
return self == ChunkingOptions()
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ChunkBbox:
|
||||||
|
page: int
|
||||||
|
bbox: list[float] # [left, top, right, bottom] in TOPLEFT origin
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ChunkResult:
|
class ChunkResult:
|
||||||
text: str
|
text: str
|
||||||
headings: list[str] = field(default_factory=list)
|
headings: list[str] = field(default_factory=list)
|
||||||
source_page: int | None = None
|
source_page: int | None = None
|
||||||
token_count: int = 0
|
token_count: int = 0
|
||||||
|
bboxes: list[ChunkBbox] = field(default_factory=list)
|
||||||
|
|
|
||||||
|
|
@ -15,7 +15,8 @@ from docling_core.transforms.chunker import HierarchicalChunker
|
||||||
from docling_core.transforms.chunker.hybrid_chunker import HybridChunker
|
from docling_core.transforms.chunker.hybrid_chunker import HybridChunker
|
||||||
from docling_core.types.doc.document import DoclingDocument
|
from docling_core.types.doc.document import DoclingDocument
|
||||||
|
|
||||||
from domain.value_objects import ChunkingOptions, ChunkResult
|
from domain.bbox import EMPTY_BBOX, to_topleft_list
|
||||||
|
from domain.value_objects import ChunkBbox, ChunkingOptions, ChunkResult
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
@ -37,12 +38,22 @@ def _chunk_sync(document_json: str, options: ChunkingOptions) -> list[ChunkResul
|
||||||
for chunk in chunker.chunk(doc):
|
for chunk in chunker.chunk(doc):
|
||||||
source_page = None
|
source_page = None
|
||||||
token_count = 0
|
token_count = 0
|
||||||
|
bboxes: list[ChunkBbox] = []
|
||||||
|
|
||||||
if hasattr(chunk, "meta") and chunk.meta and chunk.meta.doc_items:
|
if hasattr(chunk, "meta") and chunk.meta and chunk.meta.doc_items:
|
||||||
for doc_item in chunk.meta.doc_items:
|
for doc_item in chunk.meta.doc_items:
|
||||||
if hasattr(doc_item, "prov") and doc_item.prov:
|
if not hasattr(doc_item, "prov") or not doc_item.prov:
|
||||||
source_page = doc_item.prov[0].page_no
|
continue
|
||||||
break
|
for prov in doc_item.prov:
|
||||||
|
page_no = prov.page_no
|
||||||
|
if source_page is None:
|
||||||
|
source_page = page_no
|
||||||
|
if prov.bbox:
|
||||||
|
page_obj = doc.pages.get(page_no)
|
||||||
|
if page_obj:
|
||||||
|
bbox = to_topleft_list(prov.bbox, page_obj.size.height)
|
||||||
|
if bbox != EMPTY_BBOX:
|
||||||
|
bboxes.append(ChunkBbox(page=page_no, bbox=bbox))
|
||||||
|
|
||||||
if hasattr(chunker, "tokenizer") and chunker.tokenizer:
|
if hasattr(chunker, "tokenizer") and chunker.tokenizer:
|
||||||
token_count = chunker.tokenizer.count_tokens(chunk.text)
|
token_count = chunker.tokenizer.count_tokens(chunk.text)
|
||||||
|
|
@ -55,6 +66,7 @@ def _chunk_sync(document_json: str, options: ChunkingOptions) -> list[ChunkResul
|
||||||
headings=headings,
|
headings=headings,
|
||||||
source_page=source_page,
|
source_page=source_page,
|
||||||
token_count=token_count,
|
token_count=token_count,
|
||||||
|
bboxes=bboxes,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -30,6 +30,7 @@ def _chunk_to_dict(c: ChunkResult) -> dict:
|
||||||
"headings": c.headings,
|
"headings": c.headings,
|
||||||
"sourcePage": c.source_page,
|
"sourcePage": c.source_page,
|
||||||
"tokenCount": c.token_count,
|
"tokenCount": c.token_count,
|
||||||
|
"bboxes": [{"page": b.page, "bbox": b.bbox} for b in c.bboxes],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -38,7 +38,7 @@
|
||||||
<script setup lang="ts">
|
<script setup lang="ts">
|
||||||
import { ref, computed, watch, nextTick, reactive, onMounted, onBeforeUnmount } from 'vue'
|
import { ref, computed, watch, nextTick, reactive, onMounted, onBeforeUnmount } from 'vue'
|
||||||
import { computeScale, bboxToRect, pointInRect } from '../bboxScaling'
|
import { computeScale, bboxToRect, pointInRect } from '../bboxScaling'
|
||||||
import type { Page, PageElement } from '../../../shared/types'
|
import type { Page, PageElement, ChunkBbox } from '../../../shared/types'
|
||||||
|
|
||||||
const ELEMENT_COLORS: Record<string, string> = {
|
const ELEMENT_COLORS: Record<string, string> = {
|
||||||
title: '#EF4444',
|
title: '#EF4444',
|
||||||
|
|
@ -52,11 +52,15 @@ const ELEMENT_COLORS: Record<string, string> = {
|
||||||
caption: '#EAB308'
|
caption: '#EAB308'
|
||||||
}
|
}
|
||||||
|
|
||||||
const props = defineProps<{
|
const props = withDefaults(
|
||||||
imageEl: HTMLImageElement | null
|
defineProps<{
|
||||||
pageData: Page | null
|
imageEl: HTMLImageElement | null
|
||||||
highlightedIndex: number
|
pageData: Page | null
|
||||||
}>()
|
highlightedIndex: number
|
||||||
|
highlightedBboxes: ChunkBbox[]
|
||||||
|
}>(),
|
||||||
|
{ highlightedBboxes: () => [] },
|
||||||
|
)
|
||||||
|
|
||||||
const emit = defineEmits<{
|
const emit = defineEmits<{
|
||||||
'highlight-element': [index: number]
|
'highlight-element': [index: number]
|
||||||
|
|
@ -114,6 +118,8 @@ function draw(): void {
|
||||||
|
|
||||||
const scale = computeScale(img.clientWidth, img.clientHeight, props.pageData.width, props.pageData.height)
|
const scale = computeScale(img.clientWidth, img.clientHeight, props.pageData.width, props.pageData.height)
|
||||||
|
|
||||||
|
const hasChunkHighlight = props.highlightedBboxes.length > 0
|
||||||
|
|
||||||
for (const el of visibleElements.value) {
|
for (const el of visibleElements.value) {
|
||||||
const rect = bboxToRect(el.bbox, scale)
|
const rect = bboxToRect(el.bbox, scale)
|
||||||
const color = ELEMENT_COLORS[el.type] || ELEMENT_COLORS.text
|
const color = ELEMENT_COLORS[el.type] || ELEMENT_COLORS.text
|
||||||
|
|
@ -121,13 +127,29 @@ function draw(): void {
|
||||||
const elContentIdx = contentElements.value.indexOf(el)
|
const elContentIdx = contentElements.value.indexOf(el)
|
||||||
const isHighlighted = props.highlightedIndex >= 0 && elContentIdx === props.highlightedIndex
|
const isHighlighted = props.highlightedIndex >= 0 && elContentIdx === props.highlightedIndex
|
||||||
|
|
||||||
ctx.strokeStyle = color
|
// Dim non-highlighted elements when a chunk is hovered
|
||||||
|
const dimmed = hasChunkHighlight && !isHighlighted
|
||||||
|
|
||||||
|
ctx.strokeStyle = dimmed ? color + '40' : color
|
||||||
ctx.lineWidth = isHighlighted ? 3 : 2
|
ctx.lineWidth = isHighlighted ? 3 : 2
|
||||||
ctx.strokeRect(rect.x, rect.y, rect.w, rect.h)
|
ctx.strokeRect(rect.x, rect.y, rect.w, rect.h)
|
||||||
|
|
||||||
ctx.fillStyle = color + (isHighlighted ? '40' : '20')
|
ctx.fillStyle = color + (isHighlighted ? '40' : dimmed ? '08' : '20')
|
||||||
ctx.fillRect(rect.x, rect.y, rect.w, rect.h)
|
ctx.fillRect(rect.x, rect.y, rect.w, rect.h)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Draw chunk-highlighted bboxes on top with accent color
|
||||||
|
if (hasChunkHighlight) {
|
||||||
|
const CHUNK_COLOR = '#F59E0B'
|
||||||
|
for (const cb of props.highlightedBboxes) {
|
||||||
|
const rect = bboxToRect(cb.bbox, scale)
|
||||||
|
ctx.strokeStyle = CHUNK_COLOR
|
||||||
|
ctx.lineWidth = 3
|
||||||
|
ctx.strokeRect(rect.x, rect.y, rect.w, rect.h)
|
||||||
|
ctx.fillStyle = CHUNK_COLOR + '30'
|
||||||
|
ctx.fillRect(rect.x, rect.y, rect.w, rect.h)
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function onMouseMove(e: MouseEvent): void {
|
function onMouseMove(e: MouseEvent): void {
|
||||||
|
|
@ -173,7 +195,7 @@ onBeforeUnmount(() => {
|
||||||
})
|
})
|
||||||
|
|
||||||
// Redraw when data or image changes
|
// Redraw when data or image changes
|
||||||
watch([() => props.pageData, () => props.imageEl, () => props.highlightedIndex, hiddenTypes], () => {
|
watch([() => props.pageData, () => props.imageEl, () => props.highlightedIndex, () => props.highlightedBboxes, hiddenTypes], () => {
|
||||||
nextTick(draw)
|
nextTick(draw)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -76,6 +76,9 @@
|
||||||
class="chunk-card"
|
class="chunk-card"
|
||||||
v-for="(chunk, localIdx) in pagination.paginatedItems.value"
|
v-for="(chunk, localIdx) in pagination.paginatedItems.value"
|
||||||
:key="globalIndex(localIdx)"
|
:key="globalIndex(localIdx)"
|
||||||
|
:class="{ highlighted: hoveredChunkIdx === globalIndex(localIdx) }"
|
||||||
|
@mouseenter="onChunkHover(chunk, localIdx)"
|
||||||
|
@mouseleave="onChunkLeave"
|
||||||
>
|
>
|
||||||
<div class="chunk-header">
|
<div class="chunk-header">
|
||||||
<span class="chunk-index">#{{ globalIndex(localIdx) + 1 }}</span>
|
<span class="chunk-index">#{{ globalIndex(localIdx) + 1 }}</span>
|
||||||
|
|
@ -121,12 +124,16 @@ import { useAnalysisStore } from '../../analysis/store'
|
||||||
import { useI18n } from '../../../shared/i18n'
|
import { useI18n } from '../../../shared/i18n'
|
||||||
import { usePagination } from '../../../shared/composables/usePagination'
|
import { usePagination } from '../../../shared/composables/usePagination'
|
||||||
import { PaginationBar } from '../../../shared/ui'
|
import { PaginationBar } from '../../../shared/ui'
|
||||||
import type { ChunkingOptions } from '../../../shared/types'
|
import type { Chunk, ChunkBbox, ChunkingOptions } from '../../../shared/types'
|
||||||
|
|
||||||
const props = defineProps<{
|
const props = defineProps<{
|
||||||
currentPage: number
|
currentPage: number
|
||||||
}>()
|
}>()
|
||||||
|
|
||||||
|
const emit = defineEmits<{
|
||||||
|
'highlight-bboxes': [bboxes: ChunkBbox[]]
|
||||||
|
}>()
|
||||||
|
|
||||||
const analysisStore = useAnalysisStore()
|
const analysisStore = useAnalysisStore()
|
||||||
const { t } = useI18n()
|
const { t } = useI18n()
|
||||||
|
|
||||||
|
|
@ -153,6 +160,19 @@ function globalIndex(localIdx: number): number {
|
||||||
return (pagination.page.value - 1) * pagination.pageSize.value + localIdx
|
return (pagination.page.value - 1) * pagination.pageSize.value + localIdx
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const hoveredChunkIdx = ref(-1)
|
||||||
|
|
||||||
|
function onChunkHover(chunk: Chunk, localIdx: number) {
|
||||||
|
hoveredChunkIdx.value = globalIndex(localIdx)
|
||||||
|
const pageBboxes = chunk.bboxes.filter((b) => b.page === props.currentPage)
|
||||||
|
emit('highlight-bboxes', pageBboxes)
|
||||||
|
}
|
||||||
|
|
||||||
|
function onChunkLeave() {
|
||||||
|
hoveredChunkIdx.value = -1
|
||||||
|
emit('highlight-bboxes', [])
|
||||||
|
}
|
||||||
|
|
||||||
async function doRechunk() {
|
async function doRechunk() {
|
||||||
if (!analysisStore.currentAnalysis) return
|
if (!analysisStore.currentAnalysis) return
|
||||||
await analysisStore.rechunk(analysisStore.currentAnalysis.id, { ...options })
|
await analysisStore.rechunk(analysisStore.currentAnalysis.id, { ...options })
|
||||||
|
|
@ -335,6 +355,17 @@ async function doRechunk() {
|
||||||
border: 1px solid var(--border);
|
border: 1px solid var(--border);
|
||||||
border-radius: var(--radius);
|
border-radius: var(--radius);
|
||||||
padding: 10px 12px;
|
padding: 10px 12px;
|
||||||
|
cursor: default;
|
||||||
|
transition: border-color 0.15s, background 0.15s;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chunk-card:hover {
|
||||||
|
border-color: #F59E0B;
|
||||||
|
}
|
||||||
|
|
||||||
|
.chunk-card.highlighted {
|
||||||
|
border-color: #F59E0B;
|
||||||
|
background: rgba(245, 158, 11, 0.08);
|
||||||
}
|
}
|
||||||
|
|
||||||
.chunk-header {
|
.chunk-header {
|
||||||
|
|
|
||||||
|
|
@ -141,6 +141,7 @@
|
||||||
:image-el="pdfImageRef"
|
:image-el="pdfImageRef"
|
||||||
:page-data="currentPageData"
|
:page-data="currentPageData"
|
||||||
:highlighted-index="highlightedElementIndex"
|
:highlighted-index="highlightedElementIndex"
|
||||||
|
:highlighted-bboxes="highlightedChunkBboxes"
|
||||||
@highlight-element="highlightedElementIndex = $event"
|
@highlight-element="highlightedElementIndex = $event"
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -292,7 +293,10 @@
|
||||||
|
|
||||||
<!-- PREPARER MODE (feature-flipped) -->
|
<!-- PREPARER MODE (feature-flipped) -->
|
||||||
<div v-if="mode === 'preparer' && chunkingEnabled" class="prepare-panel">
|
<div v-if="mode === 'preparer' && chunkingEnabled" class="prepare-panel">
|
||||||
<ChunkPanel :current-page="currentPage" />
|
<ChunkPanel
|
||||||
|
:current-page="currentPage"
|
||||||
|
@highlight-bboxes="highlightedChunkBboxes = $event"
|
||||||
|
/>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -311,7 +315,7 @@ import { ChunkPanel } from '../features/chunking'
|
||||||
import { useFeatureFlag } from '../features/feature-flags'
|
import { useFeatureFlag } from '../features/feature-flags'
|
||||||
import { getPreviewUrl } from '../features/document/api'
|
import { getPreviewUrl } from '../features/document/api'
|
||||||
import { useI18n } from '../shared/i18n'
|
import { useI18n } from '../shared/i18n'
|
||||||
import type { PipelineOptions } from '../shared/types'
|
import type { ChunkBbox, PipelineOptions } from '../shared/types'
|
||||||
|
|
||||||
const route = useRoute()
|
const route = useRoute()
|
||||||
const router = useRouter()
|
const router = useRouter()
|
||||||
|
|
@ -324,6 +328,7 @@ const mode = ref('configurer')
|
||||||
const currentPage = ref(1)
|
const currentPage = ref(1)
|
||||||
const visualMode = ref(false)
|
const visualMode = ref(false)
|
||||||
const highlightedElementIndex = ref(-1)
|
const highlightedElementIndex = ref(-1)
|
||||||
|
const highlightedChunkBboxes = ref<ChunkBbox[]>([])
|
||||||
const pdfImageRef = ref<HTMLImageElement | null>(null)
|
const pdfImageRef = ref<HTMLImageElement | null>(null)
|
||||||
const bboxOverlayRef = ref<InstanceType<typeof BboxOverlay> | null>(null)
|
const bboxOverlayRef = ref<InstanceType<typeof BboxOverlay> | null>(null)
|
||||||
|
|
||||||
|
|
@ -410,6 +415,16 @@ function addMore() {
|
||||||
documentStore.selectedId = null
|
documentStore.selectedId = null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Clear highlights when switching modes or pages
|
||||||
|
watch(mode, () => {
|
||||||
|
highlightedElementIndex.value = -1
|
||||||
|
highlightedChunkBboxes.value = []
|
||||||
|
})
|
||||||
|
watch(currentPage, () => {
|
||||||
|
highlightedElementIndex.value = -1
|
||||||
|
highlightedChunkBboxes.value = []
|
||||||
|
})
|
||||||
|
|
||||||
// Auto-switch to verifier when analysis completes + refresh document data (pageCount)
|
// Auto-switch to verifier when analysis completes + refresh document data (pageCount)
|
||||||
watch(() => analysisStore.currentAnalysis?.status, (status) => {
|
watch(() => analysisStore.currentAnalysis?.status, (status) => {
|
||||||
if (status === 'COMPLETED') {
|
if (status === 'COMPLETED') {
|
||||||
|
|
|
||||||
|
|
@ -45,11 +45,17 @@ export interface ChunkingOptions {
|
||||||
repeat_table_header?: boolean
|
repeat_table_header?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ChunkBbox {
|
||||||
|
page: number
|
||||||
|
bbox: [number, number, number, number]
|
||||||
|
}
|
||||||
|
|
||||||
export interface Chunk {
|
export interface Chunk {
|
||||||
text: string
|
text: string
|
||||||
headings: string[]
|
headings: string[]
|
||||||
sourcePage: number | null
|
sourcePage: number | null
|
||||||
tokenCount: number
|
tokenCount: number
|
||||||
|
bboxes: ChunkBbox[]
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PageElement {
|
export interface PageElement {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue