From 529f42528a762dbe1179c36473d336f4f450240d Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 2 Jun 2026 22:12:16 -0600 Subject: [PATCH] fix(pdf): handle non-zero viewport origins and improve layout model ordering Update normalizeTextItemsForLayout to correctly apply viewport transforms, including non-zero page origins, ensuring accurate mapping of PDF text items to top-left coordinates. Refactor runLayoutModel to implement a custom order sequence builder for layout regions using model order logits, improving region ordering consistency with model semantics. Update related tests to cover viewport transform edge cases. --- compute/core/src/pdf/normalize-text.ts | 39 +++++-- compute/core/src/pdf/parse.ts | 2 +- compute/core/src/pdf/runLayoutModel.ts | 101 +++++++++++++----- ...-parse-normalize-text-items.vitest.spec.ts | 27 ++++- 4 files changed, 132 insertions(+), 37 deletions(-) diff --git a/compute/core/src/pdf/normalize-text.ts b/compute/core/src/pdf/normalize-text.ts index 2c2eca7..7b1a374 100644 --- a/compute/core/src/pdf/normalize-text.ts +++ b/compute/core/src/pdf/normalize-text.ts @@ -1,7 +1,29 @@ import type { TextItem } from 'pdfjs-dist/types/src/display/api'; import type { PdfTextItem } from './types'; -export function normalizeTextItemsForLayout(items: TextItem[], pageHeight: number): PdfTextItem[] { +interface ViewportLike { + height: number; + transform: readonly number[]; +} + +function applyViewportTransform( + x: number, + y: number, + transform: readonly number[], +): { x: number; y: number } { + const a = Number(transform[0] ?? 1); + const b = Number(transform[1] ?? 0); + const c = Number(transform[2] ?? 0); + const d = Number(transform[3] ?? 1); + const e = Number(transform[4] ?? 0); + const f = Number(transform[5] ?? 0); + return { + x: (a * x) + (c * y) + e, + y: (b * x) + (d * y) + f, + }; +} + +export function normalizeTextItemsForLayout(items: TextItem[], viewport: ViewportLike): PdfTextItem[] { return items .filter((item) => { if (!(typeof item.str === 'string' && item.str.trim().length > 0)) return false; @@ -17,13 +39,18 @@ export function normalizeTextItemsForLayout(items: TextItem[], pageHeight: numbe return true; }) .map((item) => { - const x = Number(item.transform[4] ?? 0); + const origin = applyViewportTransform( + Number(item.transform[4] ?? 0), + Number(item.transform[5] ?? 0), + viewport.transform, + ); const width = Math.max(0, Number(item.width ?? 0)); const height = Math.max(1, Math.abs(Number(item.transform[3] ?? 1))); - const baselineY = Number(item.transform[5] ?? 0); - // pdf.js text transforms are in PDF user-space (origin bottom-left). - // Normalize into top-left page coordinates to match rendered image/model boxes. - const y = Math.max(0, pageHeight - baselineY - height); + // pdf.js text transforms are in PDF user-space and may include non-zero + // page origins via the viewport transform. Map the text baseline into + // viewport coordinates first, then convert baseline to top-left. + const x = Math.max(0, origin.x); + const y = Math.max(0, origin.y - height); return { text: item.str, x, diff --git a/compute/core/src/pdf/parse.ts b/compute/core/src/pdf/parse.ts index a6dbe22..01f9519 100644 --- a/compute/core/src/pdf/parse.ts +++ b/compute/core/src/pdf/parse.ts @@ -72,7 +72,7 @@ export async function parsePdf(input: ParsePdfInput): Promise const textContent = await page.getTextContent(); const textItems = normalizeTextItemsForLayout( textContent.items.filter((item): item is TextItem => 'str' in item && 'transform' in item), - viewport.height, + viewport, ); if (textItems.length > 0) sawText = true; diff --git a/compute/core/src/pdf/runLayoutModel.ts b/compute/core/src/pdf/runLayoutModel.ts index 5b8fb43..93122ac 100644 --- a/compute/core/src/pdf/runLayoutModel.ts +++ b/compute/core/src/pdf/runLayoutModel.ts @@ -196,8 +196,10 @@ function preprocessResized( const ctx = canvas.getContext('2d'); ctx.fillStyle = '#ffffff'; ctx.fillRect(0, 0, preprocessor.inputWidth, preprocessor.inputHeight); - ctx.imageSmoothingEnabled = true; - ctx.imageSmoothingQuality = 'high'; + // Match the upstream image processor more closely. The official + // implementation explicitly disables antialiasing during resize to stay + // close to OpenCV semantics for PP-DocLayoutV3 inputs. + ctx.imageSmoothingEnabled = false; ctx.drawImage(image, 0, 0, preprocessor.inputWidth, preprocessor.inputHeight); const imageData = ctx.getImageData(0, 0, preprocessor.inputWidth, preprocessor.inputHeight); @@ -232,24 +234,40 @@ function clampBox( return [x0, y0, x1, y1]; } -function softmaxMax(logits: Float32Array, offset: number, count: number): { index: number; score: number } { - let maxLogit = Number.NEGATIVE_INFINITY; - let maxIndex = 0; - for (let i = 0; i < count; i += 1) { - const value = logits[offset + i]; - if (value > maxLogit) { - maxLogit = value; - maxIndex = i; +function sigmoid(value: number): number { + if (value >= 0) { + const z = Math.exp(-value); + return 1 / (1 + z); + } + const z = Math.exp(value); + return z / (1 + z); +} + +function buildOrderSequence(orderLogits: Float32Array, numQueries: number): number[] { + const orderVotes = new Float32Array(numQueries); + + for (let col = 0; col < numQueries; col += 1) { + let votes = 0; + + for (let row = 0; row < col; row += 1) { + votes += sigmoid(orderLogits[row * numQueries + col] ?? 0); } + + for (let row = col + 1; row < numQueries; row += 1) { + votes += 1 - sigmoid(orderLogits[col * numQueries + row] ?? 0); + } + + orderVotes[col] = votes; } - let sum = 0; - for (let i = 0; i < count; i += 1) { - sum += Math.exp(logits[offset + i] - maxLogit); + const pointers = Array.from({ length: numQueries }, (_, index) => index) + .sort((a, b) => orderVotes[a]! - orderVotes[b]!); + const orderSeq = new Array(numQueries); + for (let rank = 0; rank < pointers.length; rank += 1) { + const queryIndex = pointers[rank]!; + orderSeq[queryIndex] = rank; } - - const score = sum > 0 ? (1 / sum) : 0; - return { index: maxIndex, score }; + return orderSeq; } function normalizeModelLabel(rawLabel: string): string { @@ -283,25 +301,26 @@ export async function runLayoutModel(input: RunLayoutInput): Promise= numQueries * numQueries + ? buildOrderSequence(orderLogits, numQueries) + : null; - const regions: LayoutRegion[] = []; + const detections: Array<{ + queryIdx: number; + classIdx: number; + score: number; + order: number; + rawBox: [number, number, number, number]; + }> = []; for (let queryIdx = 0; queryIdx < numQueries; queryIdx += 1) { - const cls = softmaxMax(logits, queryIdx * classCount, classCount); - const rawLabel = idToLabel[cls.index]; - if (!rawLabel) continue; - const mapped = LABEL_MAP[normalizeModelLabel(rawLabel)]; - if (!mapped) continue; - - const minScore = CLASS_MIN_SCORE[mapped] ?? MIN_SCORE; - if (!Number.isFinite(cls.score) || cls.score < minScore) continue; - const cx = predBoxes[queryIdx * 4 + 0] * pageWidth; const cy = predBoxes[queryIdx * 4 + 1] * pageHeight; const w = predBoxes[queryIdx * 4 + 2] * pageWidth; @@ -313,7 +332,33 @@ export async function runLayoutModel(input: RunLayoutInput): Promise { + if (Math.abs(b.score - a.score) > 1e-6) return b.score - a.score; + return a.order - b.order; + }); + + for (const detection of detections.slice(0, numQueries)) { + const rawLabel = idToLabel[detection.classIdx]; + if (!rawLabel) continue; + const mapped = LABEL_MAP[normalizeModelLabel(rawLabel)]; + if (!mapped) continue; + + const minScore = CLASS_MIN_SCORE[mapped] ?? MIN_SCORE; + if (!Number.isFinite(detection.score) || detection.score < minScore) continue; + + const clamped = clampBox(detection.rawBox, pageWidth, pageHeight); if (!clamped) continue; const sizeRule = MIN_REGION_SIZE[mapped]; @@ -326,7 +371,7 @@ export async function runLayoutModel(input: RunLayoutInput): Promise { [0, 10, -10, 0, 30, 400], ); - const normalized = normalizeTextItemsForLayout([horizontal, rotated], 800); + const normalized = normalizeTextItemsForLayout([horizontal, rotated], { + height: 800, + transform: [1, 0, 0, -1, 0, 800], + }); expect(normalized).toHaveLength(1); expect(normalized[0]?.text).toBe('Powered by large language models'); }); @@ -41,7 +44,27 @@ describe('normalizeTextItemsForLayout', () => { const vertical = makeTextItem('Side label', [0, 10, -10, 0, 30, 200]); const skewed = makeTextItem('Watermark', [10, 5, 2, 10, 200, 500]); - const normalized = normalizeTextItemsForLayout([vertical, skewed], 800); + const normalized = normalizeTextItemsForLayout([vertical, skewed], { + height: 800, + transform: [1, 0, 0, -1, 0, 800], + }); expect(normalized).toEqual([]); }); + + test('accounts for non-zero page origins in the viewport transform', () => { + const croppedPageLine = makeTextItem( + 'Vasher turned away.', + [11.2, 0, 0, 11.2, 127.5, 644.4128], + 100, + ); + + const normalized = normalizeTextItemsForLayout([croppedPageLine], { + height: 666.0074, + transform: [1, 0, 0, -1, -53.4352, 720.565], + }); + + expect(normalized).toHaveLength(1); + expect(normalized[0]?.x).toBeCloseTo(74.0648, 4); + expect(normalized[0]?.y).toBeCloseTo(64.9522, 4); + }); });