refactor: fix tests + remove deprecated PDF layout processing files and update imports
- Deleted mergeTextWithRegions.ts, LICENSE.txt, manifest.json, parsePdf.ts, renderPage.ts, runLayoutModel.ts, stitchCrossPageBlocks.ts, and types.ts as they are no longer needed. - Updated unit tests to reflect new import paths from @openreader/compute-core. - Adjusted tsconfig.json to include new paths for pdf-layout module.
This commit is contained in:
Richard R
parent
3ebcb3e6fd
commit
384369f48b
21 changed files with 17 additions and 1198 deletions
26
Dockerfile
26
Dockerfile
|
|
@ -45,36 +45,20 @@ FROM node:lts-alpine AS runner
|
|||
# ffmpeg is provided by ffmpeg-static from node_modules.
|
||||
RUN apk add --no-cache ca-certificates libreoffice-writer
|
||||
|
||||
# Install pnpm and production deps from the repository lockfile.
|
||||
# Install pnpm for runtime process commands.
|
||||
RUN npm install -g pnpm@10.33.4
|
||||
|
||||
# App runtime directory
|
||||
WORKDIR /app
|
||||
|
||||
# Copy standalone Next.js server and required static assets.
|
||||
COPY --from=app-builder /app/.next/standalone ./
|
||||
COPY --from=app-builder /app/.next/static ./.next/static
|
||||
COPY --from=app-builder /app/public ./public
|
||||
# Install runtime dependencies from lockfile (no Dockerfile version pin drift).
|
||||
COPY --from=app-builder /app/package.json ./package.json
|
||||
COPY --from=app-builder /app/pnpm-lock.yaml ./pnpm-lock.yaml
|
||||
COPY --from=app-builder /app/pnpm-workspace.yaml ./pnpm-workspace.yaml
|
||||
RUN CI=true pnpm install --prod --frozen-lockfile --config.confirmModulesPurge=false
|
||||
# Copy startup/migration scripts and migration files used by openreader-entrypoint.
|
||||
COPY --from=app-builder /app/scripts/openreader-entrypoint.mjs ./scripts/openreader-entrypoint.mjs
|
||||
COPY --from=app-builder /app/scripts/migrate-fs-v2.mjs ./scripts/migrate-fs-v2.mjs
|
||||
COPY --from=app-builder /app/drizzle/scripts/migrate.mjs ./drizzle/scripts/migrate.mjs
|
||||
COPY --from=app-builder /app/drizzle/sqlite ./drizzle/sqlite
|
||||
COPY --from=app-builder /app/drizzle/postgres ./drizzle/postgres
|
||||
COPY --from=app-builder /app/drizzle.config.sqlite.ts ./drizzle.config.sqlite.ts
|
||||
COPY --from=app-builder /app/drizzle.config.pg.ts ./drizzle.config.pg.ts
|
||||
COPY --from=app-builder /app/src/db ./src/db
|
||||
# Copy built app and runtime files from the builder stage (non-standalone runtime).
|
||||
COPY --from=app-builder /app ./
|
||||
# Include third-party license report and copied license texts at a stable path in the image.
|
||||
COPY --from=app-builder /app/THIRD_PARTY_LICENSES /licenses
|
||||
# Include SeaweedFS license text for the copied weed binary.
|
||||
COPY --from=seaweedfs-builder /tmp/SeaweedFS-LICENSE.txt /licenses/SeaweedFS-LICENSE.txt
|
||||
# Include static model notices for runtime-downloaded assets.
|
||||
COPY --from=app-builder /app/src/lib/server/pdf-layout/model/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt
|
||||
COPY --from=app-builder /app/compute/core/src/pdf-layout/model/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt
|
||||
|
||||
# Copy seaweedfs weed binary for optional embedded local S3.
|
||||
COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed
|
||||
|
|
@ -88,4 +72,4 @@ EXPOSE 3003
|
|||
|
||||
# Start the application
|
||||
ENTRYPOINT ["node", "scripts/openreader-entrypoint.mjs", "--"]
|
||||
CMD ["node", "server.js"]
|
||||
CMD ["pnpm", "start:raw"]
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
".": "./src/index.ts",
|
||||
"./contracts": "./src/contracts.ts",
|
||||
"./local-runtime": "./src/local-runtime.ts",
|
||||
"./runtime/timeout-config": "./src/runtime/timeout-config.ts"
|
||||
"./runtime/timeout-config": "./src/runtime/timeout-config.ts",
|
||||
"./pdf-layout/*": "./src/pdf-layout/*.ts"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import path from 'path';
|
||||
import { fileURLToPath } from 'url';
|
||||
import { createHash } from 'crypto';
|
||||
import { access, mkdir, rename, writeFile, readFile, unlink, copyFile } from 'fs/promises';
|
||||
import { DOCSTORE_DIR } from '../runtime/docstore';
|
||||
|
|
@ -13,8 +12,7 @@ export const MODEL_DATA_PATH = path.join(MODEL_DIR, 'PP-DocLayoutV3.onnx.data');
|
|||
export const MODEL_CONFIG_PATH = path.join(MODEL_DIR, 'pp-doclayoutv3.config.json');
|
||||
export const MODEL_PREPROCESSOR_PATH = path.join(MODEL_DIR, 'pp-doclayoutv3.preprocessor_config.json');
|
||||
const LICENSE_PATH = path.join(MODEL_DIR, 'pp-doclayoutv3.LICENSE.txt');
|
||||
const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
const STATIC_LICENSE_PATH = path.join(MODULE_DIR, 'model', 'LICENSE.txt');
|
||||
const STATIC_LICENSE_PATH = path.join(process.cwd(), 'compute/core/src/pdf-layout/model/LICENSE.txt');
|
||||
|
||||
let inflight: Promise<string> | null = null;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import path from 'path';
|
||||
import { createRequire } from 'node:module';
|
||||
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
||||
import type { ParsedPdfDocument, ParsedPdfPage } from '../types/parsed-pdf';
|
||||
import type { PdfTextItem } from './types';
|
||||
|
|
@ -20,11 +19,9 @@ interface ParsePdfInput {
|
|||
}
|
||||
|
||||
const LAYOUT_RENDER_SCALE = 1.5;
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
function resolvePdfjsStandardFontDataUrl(): string {
|
||||
const pdfjsPackageJson = require.resolve('pdfjs-dist/package.json');
|
||||
const standardFontDir = path.join(path.dirname(pdfjsPackageJson), 'standard_fonts');
|
||||
const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts');
|
||||
return `${standardFontDir.replace(/\/?$/, '/')}`;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import path from 'path';
|
||||
import { createRequire } from 'node:module';
|
||||
import type { Canvas } from '@napi-rs/canvas';
|
||||
|
||||
type CanvasRuntime = {
|
||||
|
|
@ -9,11 +8,9 @@ type CanvasRuntime = {
|
|||
};
|
||||
|
||||
let canvasRuntimePromise: Promise<CanvasRuntime> | null = null;
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
function resolvePdfjsStandardFontDataUrl(): string {
|
||||
const pdfjsPackageJson = require.resolve('pdfjs-dist/package.json');
|
||||
const standardFontDir = path.join(path.dirname(pdfjsPackageJson), 'standard_fonts');
|
||||
const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts');
|
||||
return `${standardFontDir.replace(/\/?$/, '/')}`;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ const computeMode = computeModeRaw === 'none' || computeModeRaw === 'worker' ||
|
|||
? computeModeRaw
|
||||
: 'local';
|
||||
const computeLocal = computeMode === 'local';
|
||||
const isVercel = process.env.VERCEL === '1';
|
||||
const serverExternalPackages = [
|
||||
'@napi-rs/canvas',
|
||||
'better-sqlite3',
|
||||
|
|
@ -30,7 +29,6 @@ const serverExternalPackages = [
|
|||
];
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
...(isVercel ? {} : { output: 'standalone' }),
|
||||
async headers() {
|
||||
return [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ export default defineConfig({
|
|||
/* Run your local dev server before starting the tests */
|
||||
webServer: {
|
||||
// Disable auth rate limiting for tests to support parallel workers creating sessions
|
||||
command: `npm run build && DISABLE_AUTH_RATE_LIMIT=true npm run start`,
|
||||
command: `pnpm build && DISABLE_AUTH_RATE_LIMIT=true pnpm start`,
|
||||
url: 'http://localhost:3003',
|
||||
reuseExistingServer: !process.env.CI,
|
||||
timeout: 120 * 1000,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ import path from 'path';
|
|||
import { createCanvas, loadImage } from '@napi-rs/canvas';
|
||||
import JSZip from 'jszip';
|
||||
import { XMLParser } from 'fast-xml-parser';
|
||||
import { renderPage } from '@/lib/server/pdf-layout/renderPage';
|
||||
import { renderPage } from '@openreader/compute-core/pdf-layout/renderPage';
|
||||
|
||||
export type RenderedDocumentPreview = {
|
||||
bytes: Buffer;
|
||||
|
|
|
|||
|
|
@ -1,130 +0,0 @@
|
|||
import path from 'path';
|
||||
import { createHash } from 'crypto';
|
||||
import { access, mkdir, rename, writeFile, readFile, unlink, copyFile } from 'fs/promises';
|
||||
import { DOCSTORE_DIR } from '@/lib/server/storage/library-mount';
|
||||
import manifest from '@/lib/server/pdf-layout/model/manifest.json';
|
||||
|
||||
const DEFAULT_MODEL_BASE_URL = 'https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main';
|
||||
const PDF_LAYOUT_MODEL_BASE_URL_ENV = 'PDF_LAYOUT_MODEL_BASE_URL';
|
||||
const MODEL_DIR = path.join(DOCSTORE_DIR, 'model');
|
||||
export const MODEL_PATH = path.join(MODEL_DIR, 'PP-DocLayoutV3.onnx');
|
||||
export const MODEL_DATA_PATH = path.join(MODEL_DIR, 'PP-DocLayoutV3.onnx.data');
|
||||
export const MODEL_CONFIG_PATH = path.join(MODEL_DIR, 'pp-doclayoutv3.config.json');
|
||||
export const MODEL_PREPROCESSOR_PATH = path.join(MODEL_DIR, 'pp-doclayoutv3.preprocessor_config.json');
|
||||
const LICENSE_PATH = path.join(MODEL_DIR, 'pp-doclayoutv3.LICENSE.txt');
|
||||
const STATIC_LICENSE_PATH = path.join(process.cwd(), 'src/lib/server/pdf-layout/model/LICENSE.txt');
|
||||
|
||||
let inflight: Promise<string> | null = null;
|
||||
|
||||
async function sha256Hex(filePath: string): Promise<string> {
|
||||
const bytes = await readFile(filePath);
|
||||
return createHash('sha256').update(bytes).digest('hex');
|
||||
}
|
||||
|
||||
async function downloadToFile(url: string, outPath: string): Promise<void> {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) {
|
||||
throw new Error(`Download failed for ${url}: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
const bytes = new Uint8Array(await res.arrayBuffer());
|
||||
await writeFile(outPath, bytes);
|
||||
}
|
||||
|
||||
function joinModelUrl(baseUrl: string, relativePath: string): string {
|
||||
return `${baseUrl.replace(/\/+$/, '')}/${relativePath}`;
|
||||
}
|
||||
|
||||
function manifestEntry(filePath: string): { sha256: string; size: number } | null {
|
||||
const found = manifest.files.find((entry) => entry.path === filePath);
|
||||
if (!found || !found.sha256) return null;
|
||||
return {
|
||||
sha256: found.sha256.toLowerCase(),
|
||||
size: Number(found.size),
|
||||
};
|
||||
}
|
||||
|
||||
async function verifyFile(pathToFile: string, manifestPath: string): Promise<boolean> {
|
||||
const expected = manifestEntry(manifestPath);
|
||||
if (!expected) return true;
|
||||
const bytes = await readFile(pathToFile);
|
||||
if (Number.isFinite(expected.size) && expected.size > 0 && bytes.byteLength !== expected.size) {
|
||||
return false;
|
||||
}
|
||||
const actual = await sha256Hex(pathToFile);
|
||||
return actual === expected.sha256;
|
||||
}
|
||||
|
||||
async function ensureLicense(): Promise<void> {
|
||||
await copyFile(STATIC_LICENSE_PATH, LICENSE_PATH);
|
||||
if (!(await verifyFile(LICENSE_PATH, 'LICENSE.txt'))) {
|
||||
throw new Error('PDF layout model license checksum verification failed');
|
||||
}
|
||||
}
|
||||
|
||||
async function ensureModelInternal(): Promise<string> {
|
||||
try {
|
||||
await access(MODEL_PATH);
|
||||
await access(MODEL_DATA_PATH);
|
||||
await access(MODEL_CONFIG_PATH);
|
||||
await access(MODEL_PREPROCESSOR_PATH);
|
||||
if (
|
||||
await verifyFile(MODEL_PATH, 'model.onnx')
|
||||
&& await verifyFile(MODEL_DATA_PATH, 'model.onnx.data')
|
||||
&& await verifyFile(MODEL_CONFIG_PATH, 'config.json')
|
||||
&& await verifyFile(MODEL_PREPROCESSOR_PATH, 'preprocessor_config.json')
|
||||
) {
|
||||
await ensureLicense();
|
||||
return MODEL_PATH;
|
||||
}
|
||||
} catch {
|
||||
// continue
|
||||
}
|
||||
|
||||
await mkdir(MODEL_DIR, { recursive: true });
|
||||
const modelTmpPath = `${MODEL_PATH}.tmp`;
|
||||
const modelDataTmpPath = `${MODEL_DATA_PATH}.tmp`;
|
||||
const configTmpPath = `${MODEL_CONFIG_PATH}.tmp`;
|
||||
const preprocessorTmpPath = `${MODEL_PREPROCESSOR_PATH}.tmp`;
|
||||
const modelBaseUrl = process.env[PDF_LAYOUT_MODEL_BASE_URL_ENV]?.trim()
|
||||
|| DEFAULT_MODEL_BASE_URL;
|
||||
const modelUrl = joinModelUrl(modelBaseUrl, 'PP-DocLayoutV3.onnx');
|
||||
const modelDataUrl = joinModelUrl(modelBaseUrl, 'PP-DocLayoutV3.onnx.data');
|
||||
const configUrl = joinModelUrl(modelBaseUrl, 'config.json');
|
||||
const preprocessorUrl = joinModelUrl(modelBaseUrl, 'preprocessor_config.json');
|
||||
|
||||
await downloadToFile(modelUrl, modelTmpPath);
|
||||
if (!(await verifyFile(modelTmpPath, 'model.onnx'))) {
|
||||
await unlink(modelTmpPath).catch(() => undefined);
|
||||
throw new Error('PDF layout model checksum verification failed');
|
||||
}
|
||||
await downloadToFile(modelDataUrl, modelDataTmpPath);
|
||||
if (!(await verifyFile(modelDataTmpPath, 'model.onnx.data'))) {
|
||||
await unlink(modelDataTmpPath).catch(() => undefined);
|
||||
throw new Error('PDF layout model external data checksum verification failed');
|
||||
}
|
||||
await downloadToFile(configUrl, configTmpPath);
|
||||
if (!(await verifyFile(configTmpPath, 'config.json'))) {
|
||||
await unlink(configTmpPath).catch(() => undefined);
|
||||
throw new Error('PDF layout model config checksum verification failed');
|
||||
}
|
||||
await downloadToFile(preprocessorUrl, preprocessorTmpPath);
|
||||
if (!(await verifyFile(preprocessorTmpPath, 'preprocessor_config.json'))) {
|
||||
await unlink(preprocessorTmpPath).catch(() => undefined);
|
||||
throw new Error('PDF layout model preprocessor checksum verification failed');
|
||||
}
|
||||
|
||||
await rename(modelTmpPath, MODEL_PATH);
|
||||
await rename(modelDataTmpPath, MODEL_DATA_PATH);
|
||||
await rename(configTmpPath, MODEL_CONFIG_PATH);
|
||||
await rename(preprocessorTmpPath, MODEL_PREPROCESSOR_PATH);
|
||||
await ensureLicense();
|
||||
return MODEL_PATH;
|
||||
}
|
||||
|
||||
export async function ensureModel(): Promise<string> {
|
||||
if (inflight) return inflight;
|
||||
inflight = ensureModelInternal().finally(() => {
|
||||
inflight = null;
|
||||
});
|
||||
return inflight;
|
||||
}
|
||||
|
|
@ -1,153 +0,0 @@
|
|||
import type { LayoutRegion, PdfTextItem } from '@/lib/server/pdf-layout/types';
|
||||
|
||||
const NON_TEXT_REGION_LABELS = new Set<LayoutRegion['label']>(['chart', 'image', 'table', 'seal']);
|
||||
const TEXT_ASSIGNABLE_LABELS = new Set<LayoutRegion['label']>([
|
||||
'abstract',
|
||||
'algorithm',
|
||||
'aside_text',
|
||||
'content',
|
||||
'doc_title',
|
||||
'figure_title',
|
||||
'footer',
|
||||
'footnote',
|
||||
'formula_number',
|
||||
'header',
|
||||
'number',
|
||||
'paragraph_title',
|
||||
'reference',
|
||||
'reference_content',
|
||||
'text',
|
||||
'vision_footnote',
|
||||
'formula',
|
||||
]);
|
||||
|
||||
function centroid(item: PdfTextItem): { x: number; y: number } {
|
||||
return {
|
||||
x: item.x + item.width / 2,
|
||||
y: item.y + item.height / 2,
|
||||
};
|
||||
}
|
||||
|
||||
function contains(region: LayoutRegion, item: PdfTextItem): boolean {
|
||||
const c = centroid(item);
|
||||
return c.x >= region.bbox[0] && c.x <= region.bbox[2] && c.y >= region.bbox[1] && c.y <= region.bbox[3];
|
||||
}
|
||||
|
||||
function sortReadingOrder(items: PdfTextItem[]): PdfTextItem[] {
|
||||
const tolerance = 2;
|
||||
return [...items].sort((a, b) => {
|
||||
if (Math.abs(a.y - b.y) <= tolerance) return a.x - b.x;
|
||||
return a.y - b.y;
|
||||
});
|
||||
}
|
||||
|
||||
function joinText(items: PdfTextItem[]): string {
|
||||
let out = '';
|
||||
let prev: PdfTextItem | null = null;
|
||||
for (const item of items) {
|
||||
if (!prev) {
|
||||
out += item.text;
|
||||
prev = item;
|
||||
continue;
|
||||
}
|
||||
const prevEndX = prev.x + prev.width;
|
||||
const gap = item.x - prevEndX;
|
||||
const lineJump = item.y - prev.y;
|
||||
const lineBreak = lineJump > Math.max(2, Math.min(prev.height, item.height) * 0.6);
|
||||
const avgCharWidth = item.width / Math.max(1, item.text.length);
|
||||
const needsSpace = lineBreak || gap > Math.max(avgCharWidth * 0.3, 2);
|
||||
out += needsSpace ? ` ${item.text}` : item.text;
|
||||
prev = item;
|
||||
}
|
||||
return out.replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
function regionArea(region: LayoutRegion): number {
|
||||
return Math.max(1, (region.bbox[2] - region.bbox[0]) * (region.bbox[3] - region.bbox[1]));
|
||||
}
|
||||
|
||||
function regionScore(region: LayoutRegion): number {
|
||||
return Number.isFinite(region.confidence) ? Number(region.confidence) : 0;
|
||||
}
|
||||
|
||||
export interface RegionTextBlock {
|
||||
region: LayoutRegion;
|
||||
text: string;
|
||||
items: PdfTextItem[];
|
||||
sourceOrder: number;
|
||||
}
|
||||
|
||||
export function mergeTextWithRegions(regions: LayoutRegion[], textItems: PdfTextItem[]): RegionTextBlock[] {
|
||||
const sourceIndex = new Map<PdfTextItem, number>();
|
||||
for (let i = 0; i < textItems.length; i += 1) {
|
||||
sourceIndex.set(textItems[i]!, i);
|
||||
}
|
||||
|
||||
const chunkSourceOrder = (items: PdfTextItem[]): number => {
|
||||
let min = Number.POSITIVE_INFINITY;
|
||||
for (const item of items) {
|
||||
const index = sourceIndex.get(item);
|
||||
if (typeof index === 'number' && index < min) min = index;
|
||||
}
|
||||
return Number.isFinite(min) ? min : Number.MAX_SAFE_INTEGER;
|
||||
};
|
||||
|
||||
const assignableRegions = regions
|
||||
.map((region, index) => ({ region, index }))
|
||||
.filter(({ region }) => TEXT_ASSIGNABLE_LABELS.has(region.label));
|
||||
const assignedByRegion = new Map<number, PdfTextItem[]>();
|
||||
|
||||
for (const item of textItems) {
|
||||
const candidates = assignableRegions.filter(({ region }) => contains(region, item));
|
||||
if (candidates.length === 0) continue;
|
||||
|
||||
candidates.sort((a, b) => {
|
||||
const scoreDelta = regionScore(b.region) - regionScore(a.region);
|
||||
if (Math.abs(scoreDelta) > 1e-6) return scoreDelta;
|
||||
return regionArea(a.region) - regionArea(b.region);
|
||||
});
|
||||
|
||||
const winner = candidates[0];
|
||||
const list = assignedByRegion.get(winner.index) ?? [];
|
||||
list.push(item);
|
||||
assignedByRegion.set(winner.index, list);
|
||||
}
|
||||
|
||||
const out: RegionTextBlock[] = [];
|
||||
|
||||
for (const [regionIndex, assignedItems] of assignedByRegion.entries()) {
|
||||
const region = regions[regionIndex];
|
||||
if (!region) continue;
|
||||
if (assignedItems.length === 0) continue;
|
||||
const ordered = sortReadingOrder(assignedItems);
|
||||
const text = joinText(ordered);
|
||||
if (!text) continue;
|
||||
|
||||
out.push({
|
||||
region,
|
||||
text,
|
||||
items: ordered,
|
||||
sourceOrder: chunkSourceOrder(ordered),
|
||||
});
|
||||
}
|
||||
|
||||
for (const region of regions) {
|
||||
if (!NON_TEXT_REGION_LABELS.has(region.label)) continue;
|
||||
out.push({
|
||||
region,
|
||||
text: '',
|
||||
items: [],
|
||||
sourceOrder: Number.MAX_SAFE_INTEGER,
|
||||
});
|
||||
}
|
||||
|
||||
out.sort((a, b) => {
|
||||
if (a.sourceOrder !== b.sourceOrder) return a.sourceOrder - b.sourceOrder;
|
||||
const ay = a.region.bbox[1];
|
||||
const by = b.region.bbox[1];
|
||||
if (Math.abs(ay - by) <= 2) return a.region.bbox[0] - b.region.bbox[0];
|
||||
return ay - by;
|
||||
});
|
||||
|
||||
return out;
|
||||
}
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
PP-DocLayoutV3 ONNX model assets are distributed by the model publisher under Apache-2.0.
|
||||
See upstream for the authoritative license and terms:
|
||||
https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
{
|
||||
"name": "pp-doclayoutv3",
|
||||
"version": "Bei0001/PP-DocLayoutV3-ONNX@main",
|
||||
"files": [
|
||||
{
|
||||
"path": "model.onnx",
|
||||
"sha256": "c0721928ff08741bb208ebed539c77170db5234a68cb7e546e6cc9bc172a695b",
|
||||
"size": 5088167
|
||||
},
|
||||
{
|
||||
"path": "model.onnx.data",
|
||||
"sha256": "34df3e4b79d7bbbf82abce1b4f3cde3d540fa57ad42ec8905c352b97c408d437",
|
||||
"size": 136774480
|
||||
},
|
||||
{
|
||||
"path": "config.json",
|
||||
"sha256": "3cf834b91d23a756b1519bce4db42c09e852f3e35c35092dd5a3e253a50c071a",
|
||||
"size": 2460
|
||||
},
|
||||
{
|
||||
"path": "preprocessor_config.json",
|
||||
"sha256": "519fe0187a43a1ca429e3ad8317bab8700f0d5e8fb3a6e3a0a413ffac078ba42",
|
||||
"size": 575
|
||||
},
|
||||
{
|
||||
"path": "LICENSE.txt",
|
||||
"sha256": "578a6ba6f86b0692a8f719843f575a3eebf4705768ac5c37d149f441208f601f",
|
||||
"size": 195
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -1,172 +0,0 @@
|
|||
import path from 'path';
|
||||
import { createRequire } from 'node:module';
|
||||
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
||||
import type { ParsedPdfDocument, ParsedPdfPage } from '@/types/parsed-pdf';
|
||||
import type { PdfTextItem } from '@/lib/server/pdf-layout/types';
|
||||
import { ensureModel } from '@/lib/server/pdf-layout/ensureModel';
|
||||
import { runLayoutModel } from '@/lib/server/pdf-layout/runLayoutModel';
|
||||
import { mergeTextWithRegions } from '@/lib/server/pdf-layout/mergeTextWithRegions';
|
||||
import { stitchCrossPageBlocks } from '@/lib/server/pdf-layout/stitchCrossPageBlocks';
|
||||
import { renderPage } from '@/lib/server/pdf-layout/renderPage';
|
||||
|
||||
interface ParsePdfInput {
|
||||
documentId: string;
|
||||
pdfBytes: ArrayBuffer;
|
||||
}
|
||||
|
||||
const LAYOUT_RENDER_SCALE = 1.5;
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
function resolvePdfjsStandardFontDataUrl(): string {
|
||||
const pdfjsPackageJson = require.resolve('pdfjs-dist/package.json');
|
||||
const standardFontDir = path.join(path.dirname(pdfjsPackageJson), 'standard_fonts');
|
||||
return `${standardFontDir.replace(/\/?$/, '/')}`;
|
||||
}
|
||||
|
||||
export function normalizeTextItemsForLayout(items: TextItem[], pageHeight: number): PdfTextItem[] {
|
||||
return items
|
||||
.filter((item) => {
|
||||
if (!(typeof item.str === 'string' && item.str.trim().length > 0)) return false;
|
||||
const transform = item.transform;
|
||||
if (!Array.isArray(transform) || transform.length < 6) return false;
|
||||
|
||||
// Reject heavily skewed/rotated text runs (e.g. vertical margin labels
|
||||
// such as arXiv metadata) so they do not get merged into body blocks.
|
||||
const skewX = Number(transform[1] ?? 0);
|
||||
const skewY = Number(transform[2] ?? 0);
|
||||
if (Math.abs(skewX) > 0.5 || Math.abs(skewY) > 0.5) return false;
|
||||
|
||||
return true;
|
||||
})
|
||||
.map((item) => {
|
||||
const x = Number(item.transform[4] ?? 0);
|
||||
const width = Math.max(0, Number(item.width ?? 0));
|
||||
const height = Math.max(1, Math.abs(Number(item.transform[3] ?? 1)));
|
||||
const baselineY = Number(item.transform[5] ?? 0);
|
||||
// pdf.js text transforms are in PDF user-space (origin bottom-left).
|
||||
// Normalize into top-left page coordinates to match rendered image/model boxes.
|
||||
const y = Math.max(0, pageHeight - baselineY - height);
|
||||
return {
|
||||
text: item.str,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export async function parsePdf(input: ParsePdfInput): Promise<ParsedPdfDocument> {
|
||||
await ensureModel();
|
||||
|
||||
// Keep independent byte copies for text extraction and page rendering. pdf.js
|
||||
// can detach buffers passed to getDocument().
|
||||
const pdfBytesForText = new Uint8Array(input.pdfBytes).slice();
|
||||
const pdfBytesForRender = new Uint8Array(input.pdfBytes).slice();
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
|
||||
if (pdfjs.GlobalWorkerOptions) {
|
||||
pdfjs.GlobalWorkerOptions.workerSrc = 'pdfjs-dist/legacy/build/pdf.worker.mjs';
|
||||
pdfjs.GlobalWorkerOptions.workerPort = null;
|
||||
}
|
||||
const standardFontDataUrl = resolvePdfjsStandardFontDataUrl();
|
||||
|
||||
const loadingTask = pdfjs.getDocument({
|
||||
data: pdfBytesForText,
|
||||
useWorkerFetch: false,
|
||||
standardFontDataUrl,
|
||||
isEvalSupported: false,
|
||||
});
|
||||
const pdf = await loadingTask.promise;
|
||||
|
||||
try {
|
||||
const pages: ParsedPdfPage[] = [];
|
||||
let nextBlockId = 1;
|
||||
let sawText = false;
|
||||
|
||||
for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber += 1) {
|
||||
const page = await pdf.getPage(pageNumber);
|
||||
const viewport = page.getViewport({ scale: 1.0 });
|
||||
const textContent = await page.getTextContent();
|
||||
const textItems = normalizeTextItemsForLayout(
|
||||
textContent.items.filter((item): item is TextItem => 'str' in item && 'transform' in item),
|
||||
viewport.height,
|
||||
);
|
||||
|
||||
if (textItems.length > 0) sawText = true;
|
||||
|
||||
const rendered = await renderPage({
|
||||
pdfBytes: pdfBytesForRender.buffer.slice(
|
||||
pdfBytesForRender.byteOffset,
|
||||
pdfBytesForRender.byteOffset + pdfBytesForRender.byteLength,
|
||||
),
|
||||
pageNumber,
|
||||
scale: LAYOUT_RENDER_SCALE,
|
||||
});
|
||||
const scaleX = rendered.width / Math.max(1, viewport.width);
|
||||
const scaleY = rendered.height / Math.max(1, viewport.height);
|
||||
const layoutTextItems = textItems.map((item) => ({
|
||||
...item,
|
||||
x: item.x * scaleX,
|
||||
y: item.y * scaleY,
|
||||
width: item.width * scaleX,
|
||||
height: item.height * scaleY,
|
||||
}));
|
||||
const regions = await runLayoutModel({
|
||||
pageWidth: rendered.width,
|
||||
pageHeight: rendered.height,
|
||||
textItems: layoutTextItems,
|
||||
pageImage: rendered.image,
|
||||
});
|
||||
const merged = mergeTextWithRegions(regions, layoutTextItems);
|
||||
if (textItems.length > 0 && merged.length === 0) {
|
||||
throw new Error(`layout-merge-empty: page=${pageNumber} regions=${regions.length}`);
|
||||
}
|
||||
|
||||
const blocks = merged
|
||||
.map((entry, readingOrder) => ({
|
||||
id: `b${String(nextBlockId++).padStart(4, '0')}`,
|
||||
kind: entry.region.label,
|
||||
fragments: [{
|
||||
page: pageNumber,
|
||||
bbox: [
|
||||
entry.region.bbox[0] / scaleX,
|
||||
entry.region.bbox[1] / scaleY,
|
||||
entry.region.bbox[2] / scaleX,
|
||||
entry.region.bbox[3] / scaleY,
|
||||
] as [number, number, number, number],
|
||||
text: entry.text,
|
||||
readingOrder,
|
||||
...(typeof entry.region.confidence === 'number' ? { modelConfidence: entry.region.confidence } : {}),
|
||||
}],
|
||||
text: entry.text,
|
||||
}));
|
||||
|
||||
pages.push({
|
||||
pageNumber,
|
||||
width: viewport.width,
|
||||
height: viewport.height,
|
||||
blocks,
|
||||
});
|
||||
}
|
||||
|
||||
if (!sawText) {
|
||||
throw new Error('no-text-layer');
|
||||
}
|
||||
|
||||
const doc: ParsedPdfDocument = {
|
||||
schemaVersion: 1,
|
||||
documentId: input.documentId,
|
||||
parserVersion: 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69',
|
||||
parsedAt: Date.now(),
|
||||
pages,
|
||||
};
|
||||
|
||||
return stitchCrossPageBlocks(doc);
|
||||
} finally {
|
||||
await pdf.destroy().catch(() => undefined);
|
||||
await loadingTask.destroy().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,169 +0,0 @@
|
|||
import path from 'path';
|
||||
import { createRequire } from 'node:module';
|
||||
import type { Canvas } from '@napi-rs/canvas';
|
||||
|
||||
type CanvasRuntime = {
|
||||
DOMMatrixCtor: unknown;
|
||||
Path2DCtor: unknown;
|
||||
createCanvasFn: (width: number, height: number) => Canvas;
|
||||
};
|
||||
|
||||
let canvasRuntimePromise: Promise<CanvasRuntime> | null = null;
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
function resolvePdfjsStandardFontDataUrl(): string {
|
||||
const pdfjsPackageJson = require.resolve('pdfjs-dist/package.json');
|
||||
const standardFontDir = path.join(path.dirname(pdfjsPackageJson), 'standard_fonts');
|
||||
return `${standardFontDir.replace(/\/?$/, '/')}`;
|
||||
}
|
||||
|
||||
async function loadCanvasRuntime(): Promise<CanvasRuntime> {
|
||||
if (!canvasRuntimePromise) {
|
||||
canvasRuntimePromise = (async () => {
|
||||
const mod = await import('@napi-rs/canvas');
|
||||
const namespace = mod as Record<string, unknown>;
|
||||
const fallback = (namespace.default ?? {}) as Record<string, unknown>;
|
||||
|
||||
const createCanvasFn = (namespace.createCanvas ?? fallback.createCanvas) as
|
||||
| ((width: number, height: number) => Canvas)
|
||||
| undefined;
|
||||
const DOMMatrixCtor = namespace.DOMMatrix ?? fallback.DOMMatrix;
|
||||
const Path2DCtor = namespace.Path2D ?? fallback.Path2D;
|
||||
|
||||
if (typeof createCanvasFn !== 'function') {
|
||||
throw new Error(
|
||||
`Canvas runtime missing createCanvas export (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`,
|
||||
);
|
||||
}
|
||||
if (!DOMMatrixCtor || !Path2DCtor) {
|
||||
throw new Error(
|
||||
`Canvas runtime missing DOMMatrix/Path2D exports (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
DOMMatrixCtor,
|
||||
Path2DCtor,
|
||||
createCanvasFn,
|
||||
};
|
||||
})();
|
||||
}
|
||||
return canvasRuntimePromise;
|
||||
}
|
||||
|
||||
function ensureNodeCanvasGlobals(runtime: CanvasRuntime): void {
|
||||
const g = globalThis as Record<string, unknown>;
|
||||
if (typeof g.DOMMatrix === 'undefined') g.DOMMatrix = runtime.DOMMatrixCtor;
|
||||
if (typeof g.Path2D === 'undefined') g.Path2D = runtime.Path2DCtor;
|
||||
}
|
||||
|
||||
interface RenderInput {
|
||||
pdfBytes: ArrayBuffer;
|
||||
pageNumber: number;
|
||||
scale?: number;
|
||||
targetWidth?: number;
|
||||
format?: 'png' | 'jpeg';
|
||||
jpegQuality?: number;
|
||||
}
|
||||
|
||||
function createPdfjsCanvasFactory(runtime: CanvasRuntime) {
|
||||
return class OpenReaderCanvasFactory {
|
||||
create(width: number, height: number) {
|
||||
const canvas = runtime.createCanvasFn(Math.max(1, Math.floor(width)), Math.max(1, Math.floor(height)));
|
||||
return {
|
||||
canvas,
|
||||
context: canvas.getContext('2d') as unknown as CanvasRenderingContext2D,
|
||||
};
|
||||
}
|
||||
|
||||
reset(target: { canvas: Canvas; context: CanvasRenderingContext2D }, width: number, height: number): void {
|
||||
target.canvas.width = Math.max(1, Math.floor(width));
|
||||
target.canvas.height = Math.max(1, Math.floor(height));
|
||||
}
|
||||
|
||||
destroy(target: { canvas: Canvas; context: CanvasRenderingContext2D }): void {
|
||||
target.canvas.width = 0;
|
||||
target.canvas.height = 0;
|
||||
// @ts-expect-error pdf.js expects these nulled on destroy
|
||||
target.canvas = null;
|
||||
// @ts-expect-error pdf.js expects these nulled on destroy
|
||||
target.context = null;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export async function renderPage({
|
||||
pdfBytes,
|
||||
pageNumber,
|
||||
scale = 1.5,
|
||||
targetWidth,
|
||||
format = 'png',
|
||||
jpegQuality = 82,
|
||||
}: RenderInput): Promise<{
|
||||
width: number;
|
||||
height: number;
|
||||
image: Buffer;
|
||||
contentType: 'image/png' | 'image/jpeg';
|
||||
}> {
|
||||
// pdf.js may detach the provided ArrayBuffer. Work with an isolated copy so
|
||||
// callers can safely reuse their original bytes across pages/calls.
|
||||
const isolatedBytes = new Uint8Array(pdfBytes).slice();
|
||||
|
||||
const canvasRuntime = await loadCanvasRuntime();
|
||||
ensureNodeCanvasGlobals(canvasRuntime);
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore
|
||||
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
|
||||
|
||||
if (pdfjs.GlobalWorkerOptions) {
|
||||
pdfjs.GlobalWorkerOptions.workerSrc = 'pdfjs-dist/legacy/build/pdf.worker.mjs';
|
||||
pdfjs.GlobalWorkerOptions.workerPort = null;
|
||||
}
|
||||
|
||||
const standardFontDataUrl = resolvePdfjsStandardFontDataUrl();
|
||||
|
||||
const loadingTask = pdfjs.getDocument({
|
||||
data: isolatedBytes,
|
||||
useWorkerFetch: false,
|
||||
standardFontDataUrl,
|
||||
isEvalSupported: false,
|
||||
// Ensure pdf.js transport uses our canvas backend in Node/Next runtime.
|
||||
CanvasFactory: createPdfjsCanvasFactory(canvasRuntime),
|
||||
});
|
||||
|
||||
const pdf = await loadingTask.promise;
|
||||
try {
|
||||
const page = await pdf.getPage(pageNumber);
|
||||
const baseViewport = page.getViewport({ scale: 1.0 });
|
||||
const effectiveScale = typeof targetWidth === 'number' && Number.isFinite(targetWidth) && targetWidth > 0
|
||||
? (Math.max(1, Math.round(targetWidth)) / Math.max(1, baseViewport.width))
|
||||
: scale;
|
||||
const viewport = page.getViewport({ scale: effectiveScale });
|
||||
const width = Math.max(1, Math.floor(viewport.width));
|
||||
const height = Math.max(1, Math.floor(viewport.height));
|
||||
const canvas = canvasRuntime.createCanvasFn(width, height);
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.fillStyle = '#ffffff';
|
||||
ctx.fillRect(0, 0, width, height);
|
||||
|
||||
const renderTask = page.render({
|
||||
canvasContext: ctx as unknown as CanvasRenderingContext2D,
|
||||
viewport,
|
||||
intent: 'display',
|
||||
});
|
||||
await renderTask.promise;
|
||||
const contentType = format === 'jpeg' ? 'image/jpeg' : 'image/png';
|
||||
const image = format === 'jpeg'
|
||||
? canvas.toBuffer('image/jpeg', jpegQuality)
|
||||
: canvas.toBuffer('image/png');
|
||||
return {
|
||||
width,
|
||||
height,
|
||||
image,
|
||||
contentType,
|
||||
};
|
||||
} finally {
|
||||
await pdf.destroy().catch(() => undefined);
|
||||
await loadingTask.destroy().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,339 +0,0 @@
|
|||
import * as ort from 'onnxruntime-node';
|
||||
import { readFile } from 'fs/promises';
|
||||
import type { LayoutRegion, PdfTextItem } from '@/lib/server/pdf-layout/types';
|
||||
import { ensureModel, MODEL_CONFIG_PATH, MODEL_PREPROCESSOR_PATH } from '@/lib/server/pdf-layout/ensureModel';
|
||||
import { getOnnxThreadsPerJob } from '@/lib/server/compute/cpu-budget';
|
||||
|
||||
interface RunLayoutInput {
|
||||
pageWidth: number;
|
||||
pageHeight: number;
|
||||
textItems: PdfTextItem[];
|
||||
pageImage: Buffer;
|
||||
}
|
||||
|
||||
const DEFAULT_INPUT_SIZE = 800;
|
||||
const MIN_SCORE = 0.5;
|
||||
const CLASS_MIN_SCORE: Partial<Record<LayoutRegion['label'], number>> = {
|
||||
header: 0.4,
|
||||
footer: 0.4,
|
||||
figure_title: 0.45,
|
||||
footnote: 0.45,
|
||||
vision_footnote: 0.45,
|
||||
};
|
||||
|
||||
const LABEL_MAP: Record<string, LayoutRegion['label'] | null> = {
|
||||
// PP-DocLayoutV3 labels
|
||||
abstract: 'abstract',
|
||||
algorithm: 'algorithm',
|
||||
aside_text: 'aside_text',
|
||||
chart: 'chart',
|
||||
content: 'content',
|
||||
display_formula: 'formula',
|
||||
doc_title: 'doc_title',
|
||||
figure_title: 'figure_title',
|
||||
footer: 'footer',
|
||||
footer_image: 'footer',
|
||||
footnote: 'footnote',
|
||||
formula_number: 'formula_number',
|
||||
header: 'header',
|
||||
header_image: 'header',
|
||||
image: 'image',
|
||||
inline_formula: 'formula',
|
||||
number: 'number',
|
||||
paragraph_title: 'paragraph_title',
|
||||
reference: 'reference',
|
||||
reference_content: 'reference_content',
|
||||
seal: 'seal',
|
||||
table: 'table',
|
||||
text: 'text',
|
||||
vertical_text: 'text',
|
||||
vision_footnote: 'vision_footnote',
|
||||
};
|
||||
|
||||
const MIN_REGION_SIZE: Partial<Record<LayoutRegion['label'], { minWidth: number; minHeight: number }>> = {
|
||||
abstract: { minWidth: 24, minHeight: 14 },
|
||||
algorithm: { minWidth: 24, minHeight: 14 },
|
||||
aside_text: { minWidth: 24, minHeight: 14 },
|
||||
content: { minWidth: 24, minHeight: 14 },
|
||||
text: { minWidth: 24, minHeight: 14 },
|
||||
reference: { minWidth: 24, minHeight: 14 },
|
||||
reference_content: { minWidth: 24, minHeight: 14 },
|
||||
paragraph_title: { minWidth: 24, minHeight: 14 },
|
||||
doc_title: { minWidth: 24, minHeight: 14 },
|
||||
number: { minWidth: 18, minHeight: 12 },
|
||||
figure_title: { minWidth: 18, minHeight: 10 },
|
||||
footnote: { minWidth: 18, minHeight: 10 },
|
||||
vision_footnote: { minWidth: 18, minHeight: 10 },
|
||||
header: { minWidth: 18, minHeight: 10 },
|
||||
footer: { minWidth: 18, minHeight: 10 },
|
||||
};
|
||||
|
||||
interface ModelPreprocessor {
|
||||
inputWidth: number;
|
||||
inputHeight: number;
|
||||
rescaleFactor: number;
|
||||
mean: [number, number, number];
|
||||
std: [number, number, number];
|
||||
}
|
||||
|
||||
let sessionPromise: Promise<ort.InferenceSession> | null = null;
|
||||
let idToLabelPromise: Promise<Record<number, string>> | null = null;
|
||||
let preprocessorPromise: Promise<ModelPreprocessor> | null = null;
|
||||
let canvasFnsPromise: Promise<{
|
||||
createCanvasFn: (width: number, height: number) => { getContext: (kind: '2d') => CanvasRenderingContext2D };
|
||||
loadImageFn: (src: Buffer) => Promise<{ width: number; height: number } & CanvasImageSource>;
|
||||
}> | null = null;
|
||||
|
||||
async function getCanvasFns(): Promise<{
|
||||
createCanvasFn: (width: number, height: number) => { getContext: (kind: '2d') => CanvasRenderingContext2D };
|
||||
loadImageFn: (src: Buffer) => Promise<{ width: number; height: number } & CanvasImageSource>;
|
||||
}> {
|
||||
if (!canvasFnsPromise) {
|
||||
canvasFnsPromise = (async () => {
|
||||
const mod = await import('@napi-rs/canvas');
|
||||
const namespace = mod as Record<string, unknown>;
|
||||
const fallback = (namespace.default ?? {}) as Record<string, unknown>;
|
||||
const createCanvasFn = (namespace.createCanvas ?? fallback.createCanvas) as
|
||||
| ((width: number, height: number) => { getContext: (kind: '2d') => CanvasRenderingContext2D })
|
||||
| undefined;
|
||||
const loadImageFn = (namespace.loadImage ?? fallback.loadImage) as
|
||||
| ((src: Buffer) => Promise<{ width: number; height: number } & CanvasImageSource>)
|
||||
| undefined;
|
||||
|
||||
if (typeof createCanvasFn !== 'function' || typeof loadImageFn !== 'function') {
|
||||
throw new Error(
|
||||
`Canvas runtime missing createCanvas/loadImage exports (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`,
|
||||
);
|
||||
}
|
||||
|
||||
return { createCanvasFn, loadImageFn };
|
||||
})();
|
||||
}
|
||||
return canvasFnsPromise;
|
||||
}
|
||||
|
||||
async function getSession(): Promise<ort.InferenceSession> {
|
||||
if (!sessionPromise) {
|
||||
sessionPromise = (async () => {
|
||||
const modelPath = await ensureModel();
|
||||
const onnxThreadsPerJob = getOnnxThreadsPerJob();
|
||||
const stableSessionOptions: ort.InferenceSession.SessionOptions = {
|
||||
executionProviders: ['cpu'],
|
||||
graphOptimizationLevel: 'all',
|
||||
intraOpNumThreads: onnxThreadsPerJob,
|
||||
interOpNumThreads: 1,
|
||||
executionMode: 'sequential',
|
||||
};
|
||||
return ort.InferenceSession.create(modelPath, {
|
||||
...stableSessionOptions,
|
||||
});
|
||||
})();
|
||||
}
|
||||
return sessionPromise;
|
||||
}
|
||||
|
||||
async function getIdToLabel(): Promise<Record<number, string>> {
|
||||
if (!idToLabelPromise) {
|
||||
idToLabelPromise = (async () => {
|
||||
await ensureModel();
|
||||
const raw = await readFile(MODEL_CONFIG_PATH, 'utf8');
|
||||
const parsed = JSON.parse(raw) as { id2label?: Record<string, string> };
|
||||
const out: Record<number, string> = {};
|
||||
for (const [key, value] of Object.entries(parsed.id2label ?? {})) {
|
||||
const n = Number(key);
|
||||
if (Number.isFinite(n)) out[n] = String(value ?? '').trim();
|
||||
}
|
||||
return out;
|
||||
})();
|
||||
}
|
||||
return idToLabelPromise;
|
||||
}
|
||||
|
||||
async function getPreprocessor(): Promise<ModelPreprocessor> {
|
||||
if (!preprocessorPromise) {
|
||||
preprocessorPromise = (async () => {
|
||||
await ensureModel();
|
||||
const raw = await readFile(MODEL_PREPROCESSOR_PATH, 'utf8');
|
||||
const parsed = JSON.parse(raw) as {
|
||||
size?: { width?: number; height?: number };
|
||||
rescale_factor?: number;
|
||||
image_mean?: number[];
|
||||
image_std?: number[];
|
||||
};
|
||||
|
||||
const inputWidth = Math.max(1, Number(parsed.size?.width ?? DEFAULT_INPUT_SIZE));
|
||||
const inputHeight = Math.max(1, Number(parsed.size?.height ?? DEFAULT_INPUT_SIZE));
|
||||
const rescaleFactor = Number.isFinite(parsed.rescale_factor) ? Number(parsed.rescale_factor) : (1 / 255);
|
||||
const mean = [
|
||||
Number(parsed.image_mean?.[0] ?? 0),
|
||||
Number(parsed.image_mean?.[1] ?? 0),
|
||||
Number(parsed.image_mean?.[2] ?? 0),
|
||||
] as [number, number, number];
|
||||
const std = [
|
||||
Number(parsed.image_std?.[0] ?? 1),
|
||||
Number(parsed.image_std?.[1] ?? 1),
|
||||
Number(parsed.image_std?.[2] ?? 1),
|
||||
] as [number, number, number];
|
||||
|
||||
return {
|
||||
inputWidth,
|
||||
inputHeight,
|
||||
rescaleFactor,
|
||||
mean,
|
||||
std,
|
||||
};
|
||||
})();
|
||||
}
|
||||
return preprocessorPromise;
|
||||
}
|
||||
|
||||
function preprocessResized(
|
||||
image: CanvasImageSource,
|
||||
preprocessor: ModelPreprocessor,
|
||||
createCanvasFn: (width: number, height: number) => { getContext: (kind: '2d') => CanvasRenderingContext2D },
|
||||
): ort.Tensor {
|
||||
const canvas = createCanvasFn(preprocessor.inputWidth, preprocessor.inputHeight);
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.fillStyle = '#ffffff';
|
||||
ctx.fillRect(0, 0, preprocessor.inputWidth, preprocessor.inputHeight);
|
||||
ctx.imageSmoothingEnabled = true;
|
||||
ctx.imageSmoothingQuality = 'high';
|
||||
ctx.drawImage(image, 0, 0, preprocessor.inputWidth, preprocessor.inputHeight);
|
||||
|
||||
const imageData = ctx.getImageData(0, 0, preprocessor.inputWidth, preprocessor.inputHeight);
|
||||
const chw = new Float32Array(1 * 3 * preprocessor.inputWidth * preprocessor.inputHeight);
|
||||
const channelSize = preprocessor.inputWidth * preprocessor.inputHeight;
|
||||
for (let y = 0; y < preprocessor.inputHeight; y += 1) {
|
||||
for (let x = 0; x < preprocessor.inputWidth; x += 1) {
|
||||
const pixelIndex = (y * preprocessor.inputWidth + x) * 4;
|
||||
const idx = y * preprocessor.inputWidth + x;
|
||||
const r = imageData.data[pixelIndex] * preprocessor.rescaleFactor;
|
||||
const g = imageData.data[pixelIndex + 1] * preprocessor.rescaleFactor;
|
||||
const b = imageData.data[pixelIndex + 2] * preprocessor.rescaleFactor;
|
||||
|
||||
chw[idx] = (r - preprocessor.mean[0]) / Math.max(1e-8, preprocessor.std[0]);
|
||||
chw[channelSize + idx] = (g - preprocessor.mean[1]) / Math.max(1e-8, preprocessor.std[1]);
|
||||
chw[channelSize * 2 + idx] = (b - preprocessor.mean[2]) / Math.max(1e-8, preprocessor.std[2]);
|
||||
}
|
||||
}
|
||||
return new ort.Tensor('float32', chw, [1, 3, preprocessor.inputHeight, preprocessor.inputWidth]);
|
||||
}
|
||||
|
||||
function clampBox(
|
||||
bbox: [number, number, number, number],
|
||||
pageWidth: number,
|
||||
pageHeight: number,
|
||||
): [number, number, number, number] | null {
|
||||
const x0 = Math.max(0, Math.min(pageWidth, bbox[0]));
|
||||
const y0 = Math.max(0, Math.min(pageHeight, bbox[1]));
|
||||
const x1 = Math.max(0, Math.min(pageWidth, bbox[2]));
|
||||
const y1 = Math.max(0, Math.min(pageHeight, bbox[3]));
|
||||
if (x1 <= x0 || y1 <= y0) return null;
|
||||
return [x0, y0, x1, y1];
|
||||
}
|
||||
|
||||
function softmaxMax(logits: Float32Array, offset: number, count: number): { index: number; score: number } {
|
||||
let maxLogit = Number.NEGATIVE_INFINITY;
|
||||
let maxIndex = 0;
|
||||
for (let i = 0; i < count; i += 1) {
|
||||
const value = logits[offset + i];
|
||||
if (value > maxLogit) {
|
||||
maxLogit = value;
|
||||
maxIndex = i;
|
||||
}
|
||||
}
|
||||
|
||||
let sum = 0;
|
||||
for (let i = 0; i < count; i += 1) {
|
||||
sum += Math.exp(logits[offset + i] - maxLogit);
|
||||
}
|
||||
|
||||
const score = sum > 0 ? (1 / sum) : 0;
|
||||
return { index: maxIndex, score };
|
||||
}
|
||||
|
||||
function normalizeModelLabel(rawLabel: string): string {
|
||||
const normalized = rawLabel.trim().toLowerCase().replace(/[\s-]+/g, '_');
|
||||
if (normalized.endsWith('_image')) {
|
||||
const base = normalized.slice(0, -'_image'.length);
|
||||
if (base === 'header' || base === 'footer') return normalized;
|
||||
}
|
||||
// Some exports suffix duplicate classes (e.g. header_1, footer_1, text_1).
|
||||
return normalized.replace(/_\d+$/g, '');
|
||||
}
|
||||
|
||||
export async function runLayoutModel(input: RunLayoutInput): Promise<LayoutRegion[]> {
|
||||
const { pageWidth, pageHeight, textItems, pageImage } = input;
|
||||
if (!textItems.length) return [];
|
||||
if (!pageImage || pageImage.byteLength === 0) {
|
||||
throw new Error('layout-render-missing-page-image');
|
||||
}
|
||||
|
||||
try {
|
||||
const [session, idToLabel, preprocessor, canvasFns] = await Promise.all([
|
||||
getSession(),
|
||||
getIdToLabel(),
|
||||
getPreprocessor(),
|
||||
getCanvasFns(),
|
||||
]);
|
||||
|
||||
const decodedPageImage = await canvasFns.loadImageFn(pageImage);
|
||||
const pixelValues = preprocessResized(decodedPageImage, preprocessor, canvasFns.createCanvasFn);
|
||||
const output = await session.run({ pixel_values: pixelValues });
|
||||
|
||||
const logits = output.logits?.data as Float32Array | undefined;
|
||||
const predBoxes = output.pred_boxes?.data as Float32Array | undefined;
|
||||
if (!logits || !predBoxes) return [];
|
||||
|
||||
const numQueries = Math.floor(predBoxes.length / 4);
|
||||
if (numQueries <= 0) return [];
|
||||
const classCount = Math.floor(logits.length / numQueries);
|
||||
if (classCount <= 0) return [];
|
||||
|
||||
const regions: LayoutRegion[] = [];
|
||||
|
||||
for (let queryIdx = 0; queryIdx < numQueries; queryIdx += 1) {
|
||||
const cls = softmaxMax(logits, queryIdx * classCount, classCount);
|
||||
const rawLabel = idToLabel[cls.index];
|
||||
if (!rawLabel) continue;
|
||||
const mapped = LABEL_MAP[normalizeModelLabel(rawLabel)];
|
||||
if (!mapped) continue;
|
||||
|
||||
const minScore = CLASS_MIN_SCORE[mapped] ?? MIN_SCORE;
|
||||
if (!Number.isFinite(cls.score) || cls.score < minScore) continue;
|
||||
|
||||
const cx = predBoxes[queryIdx * 4 + 0] * pageWidth;
|
||||
const cy = predBoxes[queryIdx * 4 + 1] * pageHeight;
|
||||
const w = predBoxes[queryIdx * 4 + 2] * pageWidth;
|
||||
const h = predBoxes[queryIdx * 4 + 3] * pageHeight;
|
||||
const rawBox: [number, number, number, number] = [
|
||||
cx - w / 2,
|
||||
cy - h / 2,
|
||||
cx + w / 2,
|
||||
cy + h / 2,
|
||||
];
|
||||
|
||||
const clamped = clampBox(rawBox, pageWidth, pageHeight);
|
||||
if (!clamped) continue;
|
||||
|
||||
const sizeRule = MIN_REGION_SIZE[mapped];
|
||||
if (sizeRule) {
|
||||
const width = clamped[2] - clamped[0];
|
||||
const height = clamped[3] - clamped[1];
|
||||
if (width < sizeRule.minWidth || height < sizeRule.minHeight) continue;
|
||||
}
|
||||
|
||||
regions.push({
|
||||
bbox: clamped,
|
||||
label: mapped,
|
||||
confidence: cls.score,
|
||||
});
|
||||
}
|
||||
|
||||
return regions.sort((a, b) => (b.confidence ?? 0) - (a.confidence ?? 0));
|
||||
} catch (error) {
|
||||
throw new Error(
|
||||
`layout-model-inference-failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,144 +0,0 @@
|
|||
import type { ParsedPdfDocument, ParsedPdfBlock } from '@/types/parsed-pdf';
|
||||
|
||||
const STITCHABLE_KINDS = new Set<ParsedPdfBlock['kind']>([
|
||||
'text',
|
||||
'content',
|
||||
'reference_content',
|
||||
'aside_text',
|
||||
'abstract',
|
||||
'algorithm',
|
||||
'reference',
|
||||
]);
|
||||
|
||||
function stripTrailingClosers(text: string): string {
|
||||
return text.trim().replace(/[\"'”’\]\)]+$/g, '');
|
||||
}
|
||||
|
||||
function isSentenceTerminal(text: string): boolean {
|
||||
return /[.!?]$/.test(stripTrailingClosers(text));
|
||||
}
|
||||
|
||||
function canStitch(a: ParsedPdfBlock, b: ParsedPdfBlock): boolean {
|
||||
if (!STITCHABLE_KINDS.has(a.kind)) return false;
|
||||
if (a.kind !== b.kind) return false;
|
||||
if (isSentenceTerminal(a.text)) return false;
|
||||
const next = b.text.trim();
|
||||
if (!next) return false;
|
||||
if (/^[A-Z]/.test(next)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function splitHeadContinuation(text: string): { continuation: string; remainder: string } {
|
||||
const normalized = text.replace(/\s+/g, ' ').trim();
|
||||
if (!normalized) return { continuation: '', remainder: '' };
|
||||
|
||||
const CLOSERS = new Set(['"', "'", '”', '’', ')', ']', '}']);
|
||||
const isTerminal = (ch: string): boolean => ch === '.' || ch === '!' || ch === '?';
|
||||
|
||||
for (let i = 0; i < normalized.length; i += 1) {
|
||||
const ch = normalized[i];
|
||||
if (!isTerminal(ch)) continue;
|
||||
|
||||
const prev = i > 0 ? normalized[i - 1] : '';
|
||||
const next = i + 1 < normalized.length ? normalized[i + 1] : '';
|
||||
if (ch === '.' && /\d/.test(prev) && /\d/.test(next)) continue;
|
||||
|
||||
let cut = i + 1;
|
||||
while (cut < normalized.length && CLOSERS.has(normalized[cut])) cut += 1;
|
||||
|
||||
const after = cut < normalized.length ? normalized[cut] : '';
|
||||
if (!after || /\s/.test(after) || /[A-Z]/.test(after)) {
|
||||
return {
|
||||
continuation: normalized.slice(0, cut).trim(),
|
||||
remainder: normalized.slice(cut).trim(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
continuation: normalized,
|
||||
remainder: '',
|
||||
};
|
||||
}
|
||||
|
||||
const HARD_BOUNDARY_KINDS = new Set<ParsedPdfBlock['kind']>([
|
||||
'paragraph_title',
|
||||
'doc_title',
|
||||
]);
|
||||
|
||||
function findTailCandidateIndex(blocks: ParsedPdfBlock[]): number {
|
||||
for (let i = blocks.length - 1; i >= 0; i -= 1) {
|
||||
const block = blocks[i];
|
||||
if (!block || !block.text.trim()) continue;
|
||||
if (STITCHABLE_KINDS.has(block.kind)) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
function findHeadCandidateIndex(blocks: ParsedPdfBlock[]): number {
|
||||
for (let i = 0; i < blocks.length; i += 1) {
|
||||
const block = blocks[i];
|
||||
if (!block || !block.text.trim()) continue;
|
||||
if (STITCHABLE_KINDS.has(block.kind)) return i;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
function hasHardBoundaryBetween(
|
||||
pageBlocks: ParsedPdfBlock[],
|
||||
startInclusive: number,
|
||||
endExclusive: number,
|
||||
): boolean {
|
||||
for (let i = startInclusive; i < endExclusive; i += 1) {
|
||||
const block = pageBlocks[i];
|
||||
if (block && HARD_BOUNDARY_KINDS.has(block.kind)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function stitchCrossPageBlocks(doc: ParsedPdfDocument): ParsedPdfDocument {
|
||||
const pages = doc.pages.map((page) => ({ ...page, blocks: page.blocks.map((b) => ({ ...b, fragments: b.fragments.map((f) => ({ ...f })) })) }));
|
||||
|
||||
for (let i = 0; i < pages.length - 1; i += 1) {
|
||||
const page = pages[i];
|
||||
const next = pages[i + 1];
|
||||
const tailIndex = findTailCandidateIndex(page.blocks);
|
||||
const headIndex = findHeadCandidateIndex(next.blocks);
|
||||
if (tailIndex < 0 || headIndex < 0) continue;
|
||||
|
||||
if (hasHardBoundaryBetween(page.blocks, tailIndex + 1, page.blocks.length)) continue;
|
||||
if (hasHardBoundaryBetween(next.blocks, 0, headIndex)) continue;
|
||||
|
||||
const tail = page.blocks[tailIndex];
|
||||
const head = next.blocks[headIndex];
|
||||
if (!tail || !head) continue;
|
||||
if (!canStitch(tail, head)) continue;
|
||||
|
||||
const { continuation, remainder } = splitHeadContinuation(head.text);
|
||||
if (!continuation) continue;
|
||||
|
||||
const continuationFragment = head.fragments[0]
|
||||
? { ...head.fragments[0], text: continuation }
|
||||
: null;
|
||||
|
||||
if (continuationFragment) {
|
||||
tail.fragments.push(continuationFragment);
|
||||
}
|
||||
tail.text = `${tail.text} ${continuation}`.replace(/\s+/g, ' ').trim();
|
||||
|
||||
if (!remainder) {
|
||||
next.blocks.splice(headIndex, 1);
|
||||
continue;
|
||||
}
|
||||
|
||||
head.text = remainder;
|
||||
if (head.fragments[0]) {
|
||||
head.fragments[0].text = remainder;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...doc,
|
||||
pages,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,15 +0,0 @@
|
|||
import type { ParsedPdfBlockKind } from '@/types/parsed-pdf';
|
||||
|
||||
export interface PdfTextItem {
|
||||
text: string;
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface LayoutRegion {
|
||||
bbox: [number, number, number, number];
|
||||
label: ParsedPdfBlockKind;
|
||||
confidence?: number;
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import { mergeTextWithRegions } from '../../src/lib/server/pdf-layout/mergeTextWithRegions';
|
||||
import { mergeTextWithRegions } from '@openreader/compute-core/pdf-layout/mergeTextWithRegions';
|
||||
|
||||
test.describe('mergeTextWithRegions', () => {
|
||||
test('assigns text items to containing regions by centroid and joins in reading order', () => {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
|
||||
import { normalizeTextItemsForLayout } from '../../src/lib/server/pdf-layout/parsePdf';
|
||||
import { normalizeTextItemsForLayout } from '@openreader/compute-core/pdf-layout/parsePdf';
|
||||
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
||||
|
||||
function makeTextItem(
|
||||
|
|
@ -37,4 +37,3 @@ test.describe('normalizeTextItemsForLayout', () => {
|
|||
expect(normalized[0]?.text).toBe('Powered by large language models');
|
||||
});
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { expect, test } from '@playwright/test';
|
||||
import { stitchCrossPageBlocks } from '../../src/lib/server/pdf-layout/stitchCrossPageBlocks';
|
||||
import { stitchCrossPageBlocks } from '@openreader/compute-core/pdf-layout/stitchCrossPageBlocks';
|
||||
import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../src/types/parsed-pdf';
|
||||
|
||||
function makeBlock(
|
||||
|
|
|
|||
|
|
@ -23,7 +23,8 @@
|
|||
"@openreader/compute-core": ["./compute/core/src/index.ts"],
|
||||
"@openreader/compute-core/contracts": ["./compute/core/src/contracts.ts"],
|
||||
"@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"],
|
||||
"@openreader/compute-core/runtime/timeout-config": ["./compute/core/src/runtime/timeout-config.ts"]
|
||||
"@openreader/compute-core/runtime/timeout-config": ["./compute/core/src/runtime/timeout-config.ts"],
|
||||
"@openreader/compute-core/pdf-layout/*": ["./compute/core/src/pdf-layout/*"]
|
||||
}
|
||||
},
|
||||
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
|
||||
|
|
|
|||
Loading…
Reference in a new issue