From 6ab5c230c225cd30b7d13eda7de9966b1db30b9a Mon Sep 17 00:00:00 2001 From: Richard R Date: Mon, 18 May 2026 13:21:43 -0600 Subject: [PATCH] feat(pdf): improve cross-page block stitching and enforce TTS segment source boundaries Enhance PDF cross-page block stitching to move only sentence continuations and preserve remaining text on the following page. Update TTS segment planning to support strict source boundary enforcement, ensuring segments do not cross block or paragraph-title boundaries. Add new tests for both features and refactor PDF text item normalization to filter out skewed or rotated runs. --- src/components/documents/DocumentSettings.tsx | 55 ++++---- src/components/formPrimitives.tsx | 133 ++++++++++++++++-- src/components/icons/Icons.tsx | 16 +++ src/components/views/PDFViewer.tsx | 31 ++++ src/lib/server/pdf-layout/parsePdf.ts | 18 ++- .../pdf-layout/stitchCrossPageBlocks.ts | 57 +++++++- src/lib/shared/tts-segment-plan.ts | 107 ++++++++++++-- .../pdf-parse-normalize-text-items.spec.ts | 40 ++++++ .../unit/pdf-stitch-cross-page-blocks.spec.ts | 21 +++ tests/unit/tts-segment-plan.spec.ts | 57 ++++++++ 10 files changed, 479 insertions(+), 56 deletions(-) create mode 100644 tests/unit/pdf-parse-normalize-text-items.spec.ts diff --git a/src/components/documents/DocumentSettings.tsx b/src/components/documents/DocumentSettings.tsx index bdb534c..ed6aa92 100644 --- a/src/components/documents/DocumentSettings.tsx +++ b/src/components/documents/DocumentSettings.tsx @@ -16,8 +16,8 @@ import { clampTtsSegmentMaxBlockLength, } from '@/types/config'; import { useFeatureFlag } from '@/contexts/RuntimeConfigContext'; -import { Section, ToggleRow, buttonClass, segmentedButtonClass, segmentedGroupClass } from '@/components/formPrimitives'; -import { RefreshIcon } from '@/components/icons/Icons'; +import { Section, ToggleRow, CheckItem, buttonClass, segmentedButtonClass, segmentedGroupClass } from '@/components/formPrimitives'; +import { RefreshIcon, SparkleIcon } from '@/components/icons/Icons'; import type { ParsedPdfBlockKind, PdfParseStatus } from '@/types/parsed-pdf'; const PDF_SKIP_KIND_OPTIONS: Array<{ kind: ParsedPdfBlockKind; label: string }> = [ @@ -422,19 +422,27 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html, pdf }: { title="PDF Structure" subtitle="Layout-aware parsing controls." variant="flat" + action={ +
+ + + PP-DocLayout-V3 + + + {pdf.parseStatus ?? 'pending'} + + +
+ } > -
- Parse status: {pdf.parseStatus ?? 'pending'} - -
-
-

Skip while reading aloud

- {PDF_SKIP_KIND_OPTIONS.map((option) => { - const checked = pdf.skipBlockKinds.includes(option.kind); - return ( - +

Skip while reading aloud

+
+ {PDF_SKIP_KIND_OPTIONS.map((option) => ( + pdf.onToggleSkipKind(option.kind, enabled)} - variant="flat" /> - ); - })} + ))} +
)} diff --git a/src/components/formPrimitives.tsx b/src/components/formPrimitives.tsx index 73ce83d..6095591 100644 --- a/src/components/formPrimitives.tsx +++ b/src/components/formPrimitives.tsx @@ -1,6 +1,6 @@ 'use client'; -import type { ReactNode } from 'react'; +import { useId, type ReactNode } from 'react'; /** * Shared compact form primitives used by settings-like surfaces across @@ -78,7 +78,7 @@ export function Section({ variant = 'panel', }: { title: string; - subtitle?: string; + subtitle?: ReactNode; children: ReactNode; action?: ReactNode; variant?: 'panel' | 'flat'; @@ -130,6 +130,65 @@ export function Card({ ); } +export type SwitchSize = 'sm' | 'md'; + +const SWITCH_SIZE: Record = { + sm: { + track: 'h-4 w-7', + thumb: 'h-3 w-3', + on: 'translate-x-3', + off: 'translate-x-0.5', + }, + md: { + track: 'h-5 w-9', + thumb: 'h-4 w-4', + on: 'translate-x-4', + off: 'translate-x-0.5', + }, +}; + +export function Switch({ + checked, + onChange, + disabled = false, + size = 'md', + ariaLabel, + ariaLabelledBy, + ariaDescribedBy, +}: { + checked: boolean; + onChange: (checked: boolean) => void; + disabled?: boolean; + size?: SwitchSize; + ariaLabel?: string; + ariaLabelledBy?: string; + ariaDescribedBy?: string; +}) { + const s = SWITCH_SIZE[size]; + return ( + + ); +} + export function ToggleRow({ label, description, @@ -147,32 +206,76 @@ export function ToggleRow({ right?: ReactNode; variant?: 'card' | 'flat'; }) { + const labelId = useId(); + const descId = useId(); const rowClass = variant === 'flat' ? 'px-0.5 pt-1 pb-2 border-b border-offbase last:border-b-0 transition-transform duration-200 ease-out hover:scale-[1.003]' : 'rounded-md border border-offbase bg-background px-2.5 py-1.5 transition-transform duration-200 ease-out hover:scale-[1.005]'; + const handleTextToggle = () => { + if (!disabled) onChange(!checked); + }; return (
- +
+ {label} + {description} +
{right ?
{right}
: null} +
); } +export function CheckItem({ + label, + checked, + onChange, + disabled = false, +}: { + label: string; + checked: boolean; + onChange: (checked: boolean) => void; + disabled?: boolean; +}) { + const labelId = useId(); + const handleTextToggle = () => { + if (!disabled) onChange(!checked); + }; + return ( +
+ + {label} + + +
+ ); +} + export function Field({ label, hint, diff --git a/src/components/icons/Icons.tsx b/src/components/icons/Icons.tsx index b23217d..9e0d10b 100644 --- a/src/components/icons/Icons.tsx +++ b/src/components/icons/Icons.tsx @@ -620,3 +620,19 @@ export function FileSettingsIcon(props: React.SVGProps) { ); } + +export function SparkleIcon(props: React.SVGProps) { + return ( + + + + ); +} diff --git a/src/components/views/PDFViewer.tsx b/src/components/views/PDFViewer.tsx index 483ec55..48c961b 100644 --- a/src/components/views/PDFViewer.tsx +++ b/src/components/views/PDFViewer.tsx @@ -360,6 +360,37 @@ export function PDFViewer({ zoomLevel, pdfState }: PDFViewerProps) { } for (const list of map.values()) { + const geoKey = (entry: { + block: ParsedPdfBlock; + fragment: ParsedPdfBlock['fragments'][number]; + isContinuation: boolean; + }): string => { + const [x0, y0, x1, y1] = entry.fragment.bbox; + const round = (value: number) => Math.round(value * 10) / 10; + return [ + entry.block.kind, + round(x0), + round(y0), + round(x1), + round(y1), + ].join(':'); + }; + + const continuationGeometry = new Set(); + for (const entry of list) { + if (entry.isContinuation) { + continuationGeometry.add(geoKey(entry)); + } + } + + const filtered = list.filter((entry) => { + if (entry.isContinuation) return true; + return !continuationGeometry.has(geoKey(entry)); + }); + + list.length = 0; + list.push(...filtered); + list.sort((a, b) => { if (a.fragment.readingOrder !== b.fragment.readingOrder) { return a.fragment.readingOrder - b.fragment.readingOrder; diff --git a/src/lib/server/pdf-layout/parsePdf.ts b/src/lib/server/pdf-layout/parsePdf.ts index 6939a0a..cbb8f29 100644 --- a/src/lib/server/pdf-layout/parsePdf.ts +++ b/src/lib/server/pdf-layout/parsePdf.ts @@ -15,9 +15,21 @@ interface ParsePdfInput { const LAYOUT_RENDER_SCALE = 1.5; -function normalizeTextItems(items: TextItem[], pageHeight: number): PdfTextItem[] { +export function normalizeTextItemsForLayout(items: TextItem[], pageHeight: number): PdfTextItem[] { return items - .filter((item) => typeof item.str === 'string' && item.str.trim().length > 0) + .filter((item) => { + if (!(typeof item.str === 'string' && item.str.trim().length > 0)) return false; + const transform = item.transform; + if (!Array.isArray(transform) || transform.length < 6) return false; + + // Reject heavily skewed/rotated text runs (e.g. vertical margin labels + // such as arXiv metadata) so they do not get merged into body blocks. + const skewX = Number(transform[1] ?? 0); + const skewY = Number(transform[2] ?? 0); + if (Math.abs(skewX) > 0.5 || Math.abs(skewY) > 0.5) return false; + + return true; + }) .map((item) => { const x = Number(item.transform[4] ?? 0); const width = Math.max(0, Number(item.width ?? 0)); @@ -71,7 +83,7 @@ export async function parsePdf(input: ParsePdfInput): Promise const page = await pdf.getPage(pageNumber); const viewport = page.getViewport({ scale: 1.0 }); const textContent = await page.getTextContent(); - const textItems = normalizeTextItems( + const textItems = normalizeTextItemsForLayout( textContent.items.filter((item): item is TextItem => 'str' in item && 'transform' in item), viewport.height, ); diff --git a/src/lib/server/pdf-layout/stitchCrossPageBlocks.ts b/src/lib/server/pdf-layout/stitchCrossPageBlocks.ts index 2cafdde..0508769 100644 --- a/src/lib/server/pdf-layout/stitchCrossPageBlocks.ts +++ b/src/lib/server/pdf-layout/stitchCrossPageBlocks.ts @@ -28,6 +28,39 @@ function canStitch(a: ParsedPdfBlock, b: ParsedPdfBlock): boolean { return true; } +function splitHeadContinuation(text: string): { continuation: string; remainder: string } { + const normalized = text.replace(/\s+/g, ' ').trim(); + if (!normalized) return { continuation: '', remainder: '' }; + + const CLOSERS = new Set(['"', "'", '”', '’', ')', ']', '}']); + const isTerminal = (ch: string): boolean => ch === '.' || ch === '!' || ch === '?'; + + for (let i = 0; i < normalized.length; i += 1) { + const ch = normalized[i]; + if (!isTerminal(ch)) continue; + + const prev = i > 0 ? normalized[i - 1] : ''; + const next = i + 1 < normalized.length ? normalized[i + 1] : ''; + if (ch === '.' && /\d/.test(prev) && /\d/.test(next)) continue; + + let cut = i + 1; + while (cut < normalized.length && CLOSERS.has(normalized[cut])) cut += 1; + + const after = cut < normalized.length ? normalized[cut] : ''; + if (!after || /\s/.test(after) || /[A-Z]/.test(after)) { + return { + continuation: normalized.slice(0, cut).trim(), + remainder: normalized.slice(cut).trim(), + }; + } + } + + return { + continuation: normalized, + remainder: '', + }; +} + const HARD_BOUNDARY_KINDS = new Set([ 'paragraph_title', 'doc_title', @@ -81,9 +114,27 @@ export function stitchCrossPageBlocks(doc: ParsedPdfDocument): ParsedPdfDocument if (!tail || !head) continue; if (!canStitch(tail, head)) continue; - tail.fragments.push(...head.fragments); - tail.text = `${tail.text} ${head.text}`.replace(/\s+/g, ' ').trim(); - next.blocks.splice(headIndex, 1); + const { continuation, remainder } = splitHeadContinuation(head.text); + if (!continuation) continue; + + const continuationFragment = head.fragments[0] + ? { ...head.fragments[0], text: continuation } + : null; + + if (continuationFragment) { + tail.fragments.push(continuationFragment); + } + tail.text = `${tail.text} ${continuation}`.replace(/\s+/g, ' ').trim(); + + if (!remainder) { + next.blocks.splice(headIndex, 1); + continue; + } + + head.text = remainder; + if (head.fragments[0]) { + head.fragments[0].text = remainder; + } } return { diff --git a/src/lib/shared/tts-segment-plan.ts b/src/lib/shared/tts-segment-plan.ts index de3fd0c..b33ca15 100644 --- a/src/lib/shared/tts-segment-plan.ts +++ b/src/lib/shared/tts-segment-plan.ts @@ -188,6 +188,7 @@ export function planCanonicalTtsSegments( const readerType = options.readerType ?? 'pdf'; const enforceSourceBoundaries = Boolean(options.enforceSourceBoundaries); const keyPrefix = options.keyPrefix ?? TTS_SEGMENT_PLAN_VERSION; + const sourceSeparator = enforceSourceBoundaries ? '\n\n' : ' '; const preparedSources: PreparedSourceUnit[] = []; const textParts: string[] = []; let combinedLength = 0; @@ -197,8 +198,8 @@ export function planCanonicalTtsSegments( if (!text) continue; if (textParts.length > 0) { - textParts.push(' '); - combinedLength += 1; + textParts.push(sourceSeparator); + combinedLength += sourceSeparator.length; } const startOffset = combinedLength; @@ -215,9 +216,73 @@ export function planCanonicalTtsSegments( const canonicalText = textParts.join(''); const splitOptions = { maxBlockLength: options.maxBlockLength }; - const blocks = readerType === 'epub' - ? splitTextToTtsBlocksEPUB(canonicalText, splitOptions) - : splitTextToTtsBlocks(canonicalText, splitOptions); + const splitIntoBlocks = (text: string): string[] => + readerType === 'epub' + ? splitTextToTtsBlocksEPUB(text, splitOptions) + : splitTextToTtsBlocks(text, splitOptions); + + if (enforceSourceBoundaries) { + const segments: CanonicalTtsSegment[] = []; + + for (const source of preparedSources) { + const localBlocks = splitIntoBlocks(source.text); + let localRawCursor = 0; + let localNormalizedCursor = 0; + + for (const block of localBlocks) { + const text = block.trim(); + if (!text) continue; + + const exactStart = source.text.indexOf(text, localRawCursor); + let localStart: number; + let localEnd: number; + + if (exactStart >= 0) { + localStart = exactStart; + localEnd = exactStart + text.length; + localRawCursor = localEnd; + localNormalizedCursor = normalizeWithRawMap(source.text.slice(0, localEnd)).text.length; + } else { + const flexible = findFlexibleOffset(source.text, text, localNormalizedCursor); + if (flexible) { + localStart = flexible.start; + localEnd = flexible.end; + localRawCursor = localEnd; + localNormalizedCursor = flexible.normalizedEnd; + } else { + // Never drop blocks in enforced boundary mode. + localStart = Math.max(0, Math.min(localRawCursor, source.text.length)); + localEnd = Math.max(localStart, Math.min(source.text.length, localStart + text.length)); + localRawCursor = localEnd; + localNormalizedCursor = normalizeWithRawMap(source.text.slice(0, localEnd)).text.length; + } + } + + const absoluteStart = source.startOffset + localStart; + const absoluteEnd = source.startOffset + localEnd; + const ordinal = segments.length; + segments.push({ + key: buildSegmentKey(keyPrefix, text), + ordinal, + text, + ownerSourceKey: source.sourceKey, + ownerLocator: source.locator, + startAnchor: anchorForOffset(source, absoluteStart), + endAnchor: anchorForOffset(source, absoluteEnd), + spansSourceBoundary: false, + }); + } + } + + return { + version: TTS_SEGMENT_PLAN_VERSION, + readerType, + text: canonicalText, + segments, + }; + } + + const blocks = splitIntoBlocks(canonicalText); let rawCursor = 0; let normalizedCursor = 0; @@ -238,11 +303,33 @@ export function planCanonicalTtsSegments( normalizedCursor = normalizeWithRawMap(canonicalText.slice(0, endOffset)).text.length; } else { const flexible = findFlexibleOffset(canonicalText, text, normalizedCursor); - if (!flexible) continue; - startOffset = flexible.start; - endOffset = flexible.end; - rawCursor = endOffset; - normalizedCursor = flexible.normalizedEnd; + if (!flexible) { + if (!enforceSourceBoundaries) continue; + + // In enforced-boundary mode (PDF block source units), never drop a + // split block just because canonical rematching failed. Prefer a + // best-effort anchor inside the source that the cursor currently sits + // in, then emit the segment text as-is. + const fallbackSource = findSourceForOffset(preparedSources, rawCursor, 'start') + ?? preparedSources[preparedSources.length - 1] + ?? null; + if (!fallbackSource) continue; + + const fallbackStart = Math.max(fallbackSource.startOffset, Math.min(rawCursor, fallbackSource.endOffset)); + const fallbackEnd = Math.max( + fallbackStart, + Math.min(fallbackSource.endOffset, fallbackStart + text.length), + ); + startOffset = fallbackStart; + endOffset = fallbackEnd; + rawCursor = fallbackEnd; + normalizedCursor = normalizeWithRawMap(canonicalText.slice(0, fallbackEnd)).text.length; + } else { + startOffset = flexible.start; + endOffset = flexible.end; + rawCursor = endOffset; + normalizedCursor = flexible.normalizedEnd; + } } const ownerSource = findSourceForOffset(preparedSources, startOffset, 'start'); diff --git a/tests/unit/pdf-parse-normalize-text-items.spec.ts b/tests/unit/pdf-parse-normalize-text-items.spec.ts new file mode 100644 index 0000000..d763a12 --- /dev/null +++ b/tests/unit/pdf-parse-normalize-text-items.spec.ts @@ -0,0 +1,40 @@ +import { expect, test } from '@playwright/test'; + +import { normalizeTextItemsForLayout } from '../../src/lib/server/pdf-layout/parsePdf'; +import type { TextItem } from 'pdfjs-dist/types/src/display/api'; + +function makeTextItem( + str: string, + transform: [number, number, number, number, number, number], + width = 100, +): TextItem { + return { + str, + transform, + width, + height: Math.abs(transform[3]), + dir: 'ltr', + fontName: 'test', + hasEOL: false, + } as unknown as TextItem; +} + +test.describe('normalizeTextItemsForLayout', () => { + test('keeps horizontal body text and drops rotated/skewed margin text', () => { + const horizontal = makeTextItem( + 'Powered by large language models', + [10, 0, 0, 10, 100, 600], + ); + + // Typical 90deg-ish rotated/skewed run (like side metadata labels). + const rotated = makeTextItem( + 'arXiv:2407.16741v3 [cs.SE] 18 Apr 2025', + [0, 10, -10, 0, 30, 400], + ); + + const normalized = normalizeTextItemsForLayout([horizontal, rotated], 800); + expect(normalized).toHaveLength(1); + expect(normalized[0]?.text).toBe('Powered by large language models'); + }); +}); + diff --git a/tests/unit/pdf-stitch-cross-page-blocks.spec.ts b/tests/unit/pdf-stitch-cross-page-blocks.spec.ts index cdc3ac2..34902a9 100644 --- a/tests/unit/pdf-stitch-cross-page-blocks.spec.ts +++ b/tests/unit/pdf-stitch-cross-page-blocks.spec.ts @@ -57,6 +57,27 @@ test.describe('stitchCrossPageBlocks', () => { expect(page2?.blocks.map((b) => b.id)).toEqual(['b3']); }); + test('moves only the continuation sentence and keeps remaining text on next page', () => { + const doc = makeDoc( + [ + makeBlock('b1', 'text', 'This sentence continues', 1, 0), + ], + [ + makeBlock('b2', 'text', 'into the next page. This should stay on page two.', 2, 0), + ], + ); + + const stitched = stitchCrossPageBlocks(doc); + const page1 = stitched.pages[0]; + const page2 = stitched.pages[1]; + + expect(page1?.blocks[0]?.text).toBe('This sentence continues into the next page.'); + expect(page1?.blocks[0]?.fragments).toHaveLength(2); + expect(page2?.blocks).toHaveLength(1); + expect(page2?.blocks[0]?.id).toBe('b2'); + expect(page2?.blocks[0]?.text).toBe('This should stay on page two.'); + }); + test('does not stitch across paragraph-title boundary', () => { const doc = makeDoc( [ diff --git a/tests/unit/tts-segment-plan.spec.ts b/tests/unit/tts-segment-plan.spec.ts index 4efbd21..8f03797 100644 --- a/tests/unit/tts-segment-plan.spec.ts +++ b/tests/unit/tts-segment-plan.spec.ts @@ -164,6 +164,63 @@ test.describe('planCanonicalTtsSegments', () => { expect(plan.segments).toHaveLength(1); expect(plan.segments[0].ownerSourceKey).toBe('page:1'); }); + + test('keeps paragraph-title boundaries when source boundaries are enforced', () => { + const plan = planCanonicalTtsSegments([ + { + sourceKey: 'abstract', + locator: { page: 1, readerType: 'pdf', blockId: 'a1' }, + text: 'Released under the permissive MIT license, OpenHands is a community project spanning academia and industry with more than 2.1K contributions.', + }, + { + sourceKey: 'intro-title', + locator: { page: 1, readerType: 'pdf', blockId: 't1' }, + text: '1 INTRODUCTION', + }, + { + sourceKey: 'intro-body', + locator: { page: 1, readerType: 'pdf', blockId: 'p1' }, + text: 'Powered by large language models (LLMs; OpenAI 2024b; Team et al. 2023), user-facing AI systems have become increasingly capable of performing complex tasks such as accurately responding to user queries, solving math problems, and generating code.', + }, + ], { + readerType: 'pdf', + maxBlockLength: 450, + keyPrefix: 'doc:v1', + enforceSourceBoundaries: true, + }); + + expect(plan.segments.some((segment) => segment.ownerSourceKey === 'intro-title' && segment.text === '1 INTRODUCTION')).toBeTruthy(); + expect(plan.segments.some((segment) => segment.ownerSourceKey === 'intro-body' && segment.text.startsWith('Powered by large language models'))).toBeTruthy(); + expect(plan.segments.some((segment) => segment.text.startsWith('1 INTRODUCTION Powered by'))).toBeFalsy(); + }); + + test('does not drop first sentence when canonical rematch fails in enforced boundary mode', () => { + const plan = planCanonicalTtsSegments([ + { + sourceKey: 'title', + locator: { page: 1, readerType: 'pdf', blockId: 't1' }, + text: '1 INTRODUCTION', + }, + { + sourceKey: 'intro', + locator: { page: 1, readerType: 'pdf', blockId: 'p1' }, + // Missing whitespace after sentence terminal is a common PDF extraction artifact. + text: 'Powered by large language models have become increasingly capable of generating code.In particular, AI agents have recently received ever-increasing research focus.', + }, + ], { + readerType: 'pdf', + maxBlockLength: 450, + keyPrefix: 'doc:v1', + enforceSourceBoundaries: true, + }); + + const introSegments = plan.segments + .filter((segment) => segment.ownerSourceKey === 'intro') + .map((segment) => segment.text); + const combinedIntro = introSegments.join(' '); + expect(combinedIntro.includes('Powered by large language models')).toBeTruthy(); + expect(combinedIntro.includes('In particular, AI agents')).toBeTruthy(); + }); }); test.describe('buildSegmentKeyPrefix / buildSegmentKey contract', () => {