Add frontend chunking feature with Prepare mode

Chunk/ChunkingOptions types, rechunkAnalysis API, store actions
(currentChunks, rechunk), ChunkPanel UI, Prepare tab in StudioPage
gated by chunking feature flag, i18n keys (FR/EN), and tests.
This commit is contained in:
Pier-Jean Malandrino 2026-04-02 12:05:46 +02:00
parent 05de0cc774
commit ebdf9ae1be
11 changed files with 635 additions and 8 deletions

View file

@ -1,20 +1,34 @@
import type { Analysis, PipelineOptions } from '../../shared/types'
import type { Analysis, Chunk, ChunkingOptions, PipelineOptions } from '../../shared/types'
import { apiFetch } from '../../shared/api/http'
export function createAnalysis(
documentId: string,
pipelineOptions: PipelineOptions | null = null,
chunkingOptions: ChunkingOptions | null = null,
): Promise<Analysis> {
const body: Record<string, unknown> = { documentId }
if (pipelineOptions) {
body.pipelineOptions = pipelineOptions
}
if (chunkingOptions) {
body.chunkingOptions = chunkingOptions
}
return apiFetch<Analysis>('/api/analyses', {
method: 'POST',
body: JSON.stringify(body),
})
}
export function rechunkAnalysis(
jobId: string,
chunkingOptions: ChunkingOptions,
): Promise<Chunk[]> {
return apiFetch<Chunk[]>(`/api/analyses/${jobId}/rechunk`, {
method: 'POST',
body: JSON.stringify({ chunkingOptions }),
})
}
export function fetchAnalyses(): Promise<Analysis[]> {
return apiFetch<Analysis[]>('/api/analyses')
}

View file

@ -131,7 +131,7 @@ describe('useAnalysisStore — pipeline options forwarding', () => {
const store = useAnalysisStore()
await store.run('d1')
expect(api.createAnalysis).toHaveBeenCalledWith('d1', null)
expect(api.createAnalysis).toHaveBeenCalledWith('d1', null, null)
store.stopPolling()
})
@ -155,7 +155,7 @@ describe('useAnalysisStore — pipeline options forwarding', () => {
}
await store.run('d1', opts)
expect(api.createAnalysis).toHaveBeenCalledWith('d1', opts)
expect(api.createAnalysis).toHaveBeenCalledWith('d1', opts, null)
store.stopPolling()
})
@ -168,7 +168,7 @@ describe('useAnalysisStore — pipeline options forwarding', () => {
const opts = { do_ocr: false }
await store.run('d1', opts)
expect(api.createAnalysis).toHaveBeenCalledWith('d1', { do_ocr: false })
expect(api.createAnalysis).toHaveBeenCalledWith('d1', { do_ocr: false }, null)
store.stopPolling()
})

View file

@ -70,7 +70,7 @@ describe('useAnalysisStore', () => {
expect(store.currentAnalysis).toEqual(job)
expect(store.analyses[0]).toEqual(job)
expect(store.running).toBe(true)
expect(api.createAnalysis).toHaveBeenCalledWith('d1', null)
expect(api.createAnalysis).toHaveBeenCalledWith('d1', null, null)
// Advance timer to trigger polling
await vi.advanceTimersByTimeAsync(2000)
@ -90,7 +90,7 @@ describe('useAnalysisStore', () => {
const options = { do_ocr: false, table_mode: 'fast' }
await store.run('d1', options)
expect(api.createAnalysis).toHaveBeenCalledWith('d1', options)
expect(api.createAnalysis).toHaveBeenCalledWith('d1', options, null)
store.stopPolling()
})

View file

@ -1,6 +1,6 @@
import { defineStore } from 'pinia'
import { ref, computed } from 'vue'
import type { Analysis, Page, PipelineOptions } from '../../shared/types'
import type { Analysis, Chunk, ChunkingOptions, Page, PipelineOptions } from '../../shared/types'
import * as api from './api'
export const useAnalysisStore = defineStore('analysis', () => {
@ -35,14 +35,44 @@ export const useAnalysisStore = defineStore('analysis', () => {
}
}
const currentChunks = computed<Chunk[]>(() => {
if (!currentAnalysis.value?.chunksJson) return []
try {
return JSON.parse(currentAnalysis.value.chunksJson) as Chunk[]
} catch {
return []
}
})
const rechunking = ref(false)
async function rechunk(jobId: string, chunkingOptions: ChunkingOptions): Promise<Chunk[]> {
rechunking.value = true
error.value = null
try {
const chunks = await api.rechunkAnalysis(jobId, chunkingOptions)
if (currentAnalysis.value?.id === jobId) {
currentAnalysis.value = await api.fetchAnalysis(jobId)
}
return chunks
} catch (e) {
error.value = (e as Error).message || 'Failed to rechunk'
console.error('Failed to rechunk', e)
throw e
} finally {
rechunking.value = false
}
}
async function run(
documentId: string,
pipelineOptions: PipelineOptions | null = null,
chunkingOptions: ChunkingOptions | null = null,
): Promise<Analysis> {
running.value = true
error.value = null
try {
const analysis = await api.createAnalysis(documentId, pipelineOptions)
const analysis = await api.createAnalysis(documentId, pipelineOptions, chunkingOptions)
currentAnalysis.value = analysis
analyses.value.unshift(analysis)
startPolling(analysis.id)
@ -118,11 +148,14 @@ export const useAnalysisStore = defineStore('analysis', () => {
analyses,
currentAnalysis,
currentPages,
currentChunks,
running,
rechunking,
error,
clearError,
load,
run,
rechunk,
select,
remove,
stopPolling,

View file

@ -0,0 +1,52 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { rechunkAnalysis, createAnalysis } from '../analysis/api'
vi.mock('../../shared/api/http', () => ({
apiFetch: vi.fn(),
}))
import { apiFetch } from '../../shared/api/http'
describe('chunking API', () => {
beforeEach(() => {
vi.clearAllMocks()
})
it('createAnalysis sends chunkingOptions when provided', async () => {
const job = { id: '1', documentId: 'doc-1', status: 'PENDING' }
apiFetch.mockResolvedValue(job)
const chunkingOpts = { chunker_type: 'hybrid' as const, max_tokens: 256 }
await createAnalysis('doc-1', null, chunkingOpts)
expect(apiFetch).toHaveBeenCalledWith('/api/analyses', {
method: 'POST',
body: JSON.stringify({ documentId: 'doc-1', chunkingOptions: chunkingOpts }),
})
})
it('createAnalysis omits chunkingOptions when null', async () => {
apiFetch.mockResolvedValue({ id: '1' })
await createAnalysis('doc-1', null, null)
expect(apiFetch).toHaveBeenCalledWith('/api/analyses', {
method: 'POST',
body: JSON.stringify({ documentId: 'doc-1' }),
})
})
it('rechunkAnalysis sends POST to rechunk endpoint', async () => {
const chunks = [{ text: 'chunk1', headings: [], sourcePage: 1, tokenCount: 10 }]
apiFetch.mockResolvedValue(chunks)
const opts = { chunker_type: 'hybrid' as const, max_tokens: 512 }
const result = await rechunkAnalysis('job-1', opts)
expect(apiFetch).toHaveBeenCalledWith('/api/analyses/job-1/rechunk', {
method: 'POST',
body: JSON.stringify({ chunkingOptions: opts }),
})
expect(result).toEqual(chunks)
})
})

View file

@ -0,0 +1 @@
export { default as ChunkPanel } from './ui/ChunkPanel.vue'

View file

@ -0,0 +1,135 @@
import { describe, it, expect, vi, beforeEach } from 'vitest'
import { setActivePinia, createPinia } from 'pinia'
import { useAnalysisStore } from '../analysis/store'
vi.mock('../analysis/api', () => ({
createAnalysis: vi.fn(),
fetchAnalyses: vi.fn().mockResolvedValue([]),
fetchAnalysis: vi.fn(),
deleteAnalysis: vi.fn(),
rechunkAnalysis: vi.fn(),
}))
import * as api from '../analysis/api'
describe('analysis store — chunking', () => {
beforeEach(() => {
setActivePinia(createPinia())
vi.clearAllMocks()
})
it('currentChunks parses chunksJson from current analysis', () => {
const store = useAnalysisStore()
const chunks = [
{ text: 'chunk1', headings: ['H1'], sourcePage: 1, tokenCount: 10 },
{ text: 'chunk2', headings: [], sourcePage: 2, tokenCount: 20 },
]
store.currentAnalysis = {
id: 'j1',
documentId: 'd1',
documentFilename: null,
status: 'COMPLETED',
contentMarkdown: null,
contentHtml: null,
pagesJson: null,
chunksJson: JSON.stringify(chunks),
hasDocumentJson: true,
errorMessage: null,
startedAt: null,
completedAt: null,
createdAt: '2024-01-01',
}
expect(store.currentChunks).toEqual(chunks)
})
it('currentChunks returns empty array when no chunksJson', () => {
const store = useAnalysisStore()
store.currentAnalysis = {
id: 'j1',
documentId: 'd1',
documentFilename: null,
status: 'COMPLETED',
contentMarkdown: null,
contentHtml: null,
pagesJson: null,
chunksJson: null,
hasDocumentJson: false,
errorMessage: null,
startedAt: null,
completedAt: null,
createdAt: '2024-01-01',
}
expect(store.currentChunks).toEqual([])
})
it('rechunk calls API and refreshes analysis', async () => {
const store = useAnalysisStore()
const chunks = [{ text: 'c1', headings: [], sourcePage: 1, tokenCount: 5 }]
vi.mocked(api.rechunkAnalysis).mockResolvedValue(chunks)
vi.mocked(api.fetchAnalysis).mockResolvedValue({
id: 'j1',
documentId: 'd1',
documentFilename: null,
status: 'COMPLETED',
contentMarkdown: null,
contentHtml: null,
pagesJson: null,
chunksJson: JSON.stringify(chunks),
hasDocumentJson: true,
errorMessage: null,
startedAt: null,
completedAt: null,
createdAt: '2024-01-01',
})
store.currentAnalysis = {
id: 'j1',
documentId: 'd1',
documentFilename: null,
status: 'COMPLETED',
contentMarkdown: null,
contentHtml: null,
pagesJson: null,
chunksJson: null,
hasDocumentJson: true,
errorMessage: null,
startedAt: null,
completedAt: null,
createdAt: '2024-01-01',
}
const result = await store.rechunk('j1', { chunker_type: 'hybrid', max_tokens: 256 })
expect(api.rechunkAnalysis).toHaveBeenCalledWith('j1', {
chunker_type: 'hybrid',
max_tokens: 256,
})
expect(result).toEqual(chunks)
expect(store.rechunking).toBe(false)
})
it('run passes chunkingOptions to API', async () => {
const store = useAnalysisStore()
vi.mocked(api.createAnalysis).mockResolvedValue({
id: 'j1',
documentId: 'd1',
documentFilename: null,
status: 'PENDING',
contentMarkdown: null,
contentHtml: null,
pagesJson: null,
chunksJson: null,
hasDocumentJson: false,
errorMessage: null,
startedAt: null,
completedAt: null,
createdAt: '2024-01-01',
})
await store.run('d1', null, { chunker_type: 'hierarchical' })
expect(api.createAnalysis).toHaveBeenCalledWith('d1', null, {
chunker_type: 'hierarchical',
})
})
})

View file

@ -0,0 +1,336 @@
<template>
<div class="chunk-panel">
<!-- Chunking config -->
<div class="chunk-config">
<div class="config-section">
<label class="config-label">{{ t('chunking.settings') }}</label>
<div class="config-row">
<label class="config-label-sm">{{ t('chunking.chunkerType') }}</label>
<select class="config-select" v-model="options.chunker_type">
<option value="hybrid">Hybrid</option>
<option value="hierarchical">Hierarchical</option>
</select>
</div>
<div class="config-row">
<label class="config-label-sm">{{ t('chunking.maxTokens') }}</label>
<input
type="number"
class="config-input"
v-model.number="options.max_tokens"
min="64"
max="8192"
step="64"
/>
</div>
<div class="config-toggle-row" v-if="options.chunker_type === 'hybrid'">
<label class="toggle-label">
<input type="checkbox" v-model="options.merge_peers" class="toggle-input" />
<span class="toggle-switch" />
<span class="toggle-text">{{ t('chunking.mergePeers') }}</span>
</label>
</div>
<div class="config-toggle-row" v-if="options.chunker_type === 'hybrid'">
<label class="toggle-label">
<input type="checkbox" v-model="options.repeat_table_header" class="toggle-input" />
<span class="toggle-switch" />
<span class="toggle-text">{{ t('chunking.repeatTableHeader') }}</span>
</label>
</div>
</div>
<button
class="chunk-btn primary"
:disabled="!canRechunk || analysisStore.rechunking"
@click="doRechunk"
>
<div v-if="analysisStore.rechunking" class="spinner-sm" />
{{ analysisStore.rechunking ? t('chunking.chunking') : t('chunking.run') }}
</button>
</div>
<!-- Chunks list -->
<div class="chunk-results" v-if="analysisStore.currentChunks.length">
<div class="chunk-summary">
{{ analysisStore.currentChunks.length }} {{ t('chunking.chunks') }}
</div>
<div class="chunk-list">
<div
class="chunk-card"
v-for="(chunk, idx) in analysisStore.currentChunks"
:key="idx"
>
<div class="chunk-header">
<span class="chunk-index">#{{ idx + 1 }}</span>
<span class="chunk-tokens" v-if="chunk.tokenCount">
{{ chunk.tokenCount }} tokens
</span>
<span class="chunk-page" v-if="chunk.sourcePage">
p.{{ chunk.sourcePage }}
</span>
</div>
<div class="chunk-headings" v-if="chunk.headings.length">
<span class="chunk-heading" v-for="h in chunk.headings" :key="h">{{ h }}</span>
</div>
<div class="chunk-text">{{ chunk.text }}</div>
</div>
</div>
</div>
<div class="chunk-empty" v-else-if="!analysisStore.rechunking">
<p>{{ t('chunking.noChunks') }}</p>
</div>
</div>
</template>
<script setup lang="ts">
import { reactive, computed } from 'vue'
import { useAnalysisStore } from '../../analysis/store'
import { useI18n } from '../../../shared/i18n'
import type { ChunkingOptions } from '../../../shared/types'
const analysisStore = useAnalysisStore()
const { t } = useI18n()
const options = reactive<Required<ChunkingOptions>>({
chunker_type: 'hybrid',
max_tokens: 512,
merge_peers: true,
repeat_table_header: true,
})
const canRechunk = computed(() => {
const analysis = analysisStore.currentAnalysis
return analysis?.status === 'COMPLETED' && analysis.hasDocumentJson
})
async function doRechunk() {
if (!analysisStore.currentAnalysis) return
await analysisStore.rechunk(analysisStore.currentAnalysis.id, { ...options })
}
</script>
<style scoped>
.chunk-panel {
display: flex;
flex-direction: column;
height: 100%;
overflow: hidden;
}
.chunk-config {
padding: 16px;
border-bottom: 1px solid var(--border);
display: flex;
flex-direction: column;
gap: 12px;
}
.config-section {
display: flex;
flex-direction: column;
gap: 8px;
}
.config-label {
font-size: 11px;
font-weight: 600;
text-transform: uppercase;
letter-spacing: 0.5px;
color: var(--text-secondary);
}
.config-label-sm {
font-size: 12px;
color: var(--text-secondary);
}
.config-row {
display: flex;
flex-direction: column;
gap: 4px;
}
.config-select,
.config-input {
background: var(--bg-secondary);
border: 1px solid var(--border);
border-radius: var(--radius);
padding: 6px 10px;
font-size: 13px;
color: var(--text);
width: 100%;
}
.config-toggle-row {
display: flex;
align-items: center;
gap: 8px;
}
.toggle-label {
display: flex;
align-items: center;
gap: 8px;
cursor: pointer;
font-size: 13px;
color: var(--text);
}
.toggle-input {
display: none;
}
.toggle-switch {
width: 32px;
height: 18px;
background: var(--bg-tertiary);
border-radius: 9px;
position: relative;
transition: background 0.2s;
flex-shrink: 0;
}
.toggle-switch::after {
content: '';
position: absolute;
top: 2px;
left: 2px;
width: 14px;
height: 14px;
background: white;
border-radius: 50%;
transition: transform 0.2s;
}
.toggle-input:checked + .toggle-switch {
background: var(--accent);
}
.toggle-input:checked + .toggle-switch::after {
transform: translateX(14px);
}
.chunk-btn {
padding: 8px 16px;
border: none;
border-radius: var(--radius);
cursor: pointer;
font-size: 13px;
font-weight: 500;
display: flex;
align-items: center;
justify-content: center;
gap: 6px;
}
.chunk-btn.primary {
background: var(--accent);
color: white;
}
.chunk-btn.primary:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.chunk-results {
flex: 1;
overflow-y: auto;
padding: 12px;
}
.chunk-summary {
font-size: 12px;
font-weight: 600;
color: var(--text-secondary);
text-transform: uppercase;
letter-spacing: 0.5px;
margin-bottom: 8px;
}
.chunk-list {
display: flex;
flex-direction: column;
gap: 8px;
}
.chunk-card {
background: var(--bg-secondary);
border: 1px solid var(--border);
border-radius: var(--radius);
padding: 10px 12px;
}
.chunk-header {
display: flex;
align-items: center;
gap: 8px;
margin-bottom: 6px;
}
.chunk-index {
font-size: 11px;
font-weight: 700;
color: var(--accent);
}
.chunk-tokens,
.chunk-page {
font-size: 11px;
color: var(--text-secondary);
background: var(--bg-tertiary);
padding: 1px 6px;
border-radius: 4px;
}
.chunk-headings {
display: flex;
flex-wrap: wrap;
gap: 4px;
margin-bottom: 6px;
}
.chunk-heading {
font-size: 11px;
color: var(--accent);
background: var(--accent-bg, rgba(99, 102, 241, 0.1));
padding: 1px 6px;
border-radius: 4px;
}
.chunk-text {
font-size: 12px;
color: var(--text);
line-height: 1.5;
white-space: pre-wrap;
word-break: break-word;
max-height: 120px;
overflow-y: auto;
}
.chunk-empty {
display: flex;
align-items: center;
justify-content: center;
flex: 1;
color: var(--text-secondary);
font-size: 13px;
}
.spinner-sm {
width: 14px;
height: 14px;
border: 2px solid rgba(255, 255, 255, 0.3);
border-top-color: white;
border-radius: 50%;
animation: spin 0.6s linear infinite;
}
@keyframes spin {
to { transform: rotate(360deg); }
}
</style>

View file

@ -37,6 +37,15 @@
>
{{ t('studio.verify') }}
</button>
<button
v-if="chunkingEnabled"
class="toggle-btn"
:class="{ active: mode === 'preparer' }"
@click="mode = 'preparer'"
:disabled="!analysisStore.currentAnalysis"
>
{{ t('studio.prepare') }}
</button>
</div>
</div>
<div class="topbar-actions">
@ -280,6 +289,11 @@
@highlight-element="highlightedElementIndex = $event"
/>
</div>
<!-- PREPARER MODE (feature-flipped) -->
<div v-if="mode === 'preparer' && chunkingEnabled" class="prepare-panel">
<ChunkPanel />
</div>
</div>
</div>
</div>
@ -293,6 +307,8 @@ import { useAnalysisStore } from '../features/analysis/store'
import { DocumentUpload, DocumentList } from '../features/document/index'
import { ResultTabs } from '../features/analysis/index'
import BboxOverlay from '../features/analysis/ui/BboxOverlay.vue'
import { ChunkPanel } from '../features/chunking'
import { useFeatureFlag } from '../features/feature-flags'
import { getPreviewUrl } from '../features/document/api'
import { useI18n } from '../shared/i18n'
import type { PipelineOptions } from '../shared/types'
@ -302,6 +318,7 @@ const router = useRouter()
const documentStore = useDocumentStore()
const analysisStore = useAnalysisStore()
const { t } = useI18n()
const chunkingEnabled = useFeatureFlag('chunking')
const mode = ref('configurer')
const currentPage = ref(1)

View file

@ -95,6 +95,18 @@ const messages: Messages = {
'history.emptyDocs': 'Aucun document. Importez un document depuis le Studio.',
'history.open': 'Ouvrir',
// Chunking
'studio.prepare': 'Préparer',
'chunking.settings': 'Chunking',
'chunking.chunkerType': 'Type de chunker',
'chunking.maxTokens': 'Tokens max',
'chunking.mergePeers': 'Fusionner les pairs',
'chunking.repeatTableHeader': 'Répéter en-têtes tableaux',
'chunking.run': 'Chunker',
'chunking.chunking': 'Chunking...',
'chunking.chunks': 'chunks',
'chunking.noChunks': 'Lancez le chunking pour préparer les segments.',
// Settings
'settings.title': 'Paramètres',
'settings.apiUrl': 'API URL',
@ -185,6 +197,17 @@ const messages: Messages = {
'history.emptyDocs': 'No documents yet. Upload a document from the Studio.',
'history.open': 'Open',
'studio.prepare': 'Prepare',
'chunking.settings': 'Chunking',
'chunking.chunkerType': 'Chunker type',
'chunking.maxTokens': 'Max tokens',
'chunking.mergePeers': 'Merge peers',
'chunking.repeatTableHeader': 'Repeat table headers',
'chunking.run': 'Chunk',
'chunking.chunking': 'Chunking...',
'chunking.chunks': 'chunks',
'chunking.noChunks': 'Run chunking to prepare segments.',
'settings.title': 'Settings',
'settings.apiUrl': 'API URL',
'settings.version': 'Version',

View file

@ -30,12 +30,28 @@ export interface Analysis {
contentMarkdown: string | null
contentHtml: string | null
pagesJson: string | null
chunksJson: string | null
hasDocumentJson: boolean
errorMessage: string | null
startedAt: string | null
completedAt: string | null
createdAt: string
}
export interface ChunkingOptions {
chunker_type?: 'hybrid' | 'hierarchical'
max_tokens?: number
merge_peers?: boolean
repeat_table_header?: boolean
}
export interface Chunk {
text: string
headings: string[]
sourcePage: number | null
tokenCount: number
}
export interface PageElement {
type: string
bbox: [number, number, number, number]