From 37b6999c2c3dad011642d2ba18dd904511035304 Mon Sep 17 00:00:00 2001 From: Richard R Date: Thu, 21 May 2026 21:30:44 -0600 Subject: [PATCH] refactor(compute-core): collapse to root/local-runtime entrypoints --- compute/core/package.json | 7 +--- compute/core/src/index.ts | 21 ++++++++--- .../pdf-layout/normalizeTextItemsForLayout.ts | 35 +++++++++++++++++++ compute/core/src/pdf-layout/parsePdf.ts | 35 +------------------ compute/worker/src/server.ts | 4 +-- src/lib/server/compute/concurrency-limiter.ts | 2 +- src/lib/server/compute/local.ts | 4 +-- src/lib/server/compute/types.ts | 2 +- src/lib/server/compute/worker-contract.ts | 2 +- src/lib/server/compute/worker.ts | 2 +- src/lib/server/documents/previews-render.ts | 2 +- src/lib/server/jobs/parsePdfJob.ts | 2 +- .../unit/pdf-merge-text-with-regions.spec.ts | 2 +- .../pdf-parse-normalize-text-items.spec.ts | 2 +- .../unit/pdf-stitch-cross-page-blocks.spec.ts | 2 +- tests/unit/whisper-alignment-mapping.spec.ts | 2 +- tests/unit/whisper-alignment-smoke.spec.ts | 2 +- tests/unit/whisper-ensure-model.spec.ts | 2 +- tests/unit/whisper-spectral.spec.ts | 2 +- tests/unit/whisper-token-timestamps.spec.ts | 2 +- tsconfig.json | 7 +--- 21 files changed, 72 insertions(+), 69 deletions(-) create mode 100644 compute/core/src/pdf-layout/normalizeTextItemsForLayout.ts diff --git a/compute/core/package.json b/compute/core/package.json index 0874c4c..90bd875 100644 --- a/compute/core/package.json +++ b/compute/core/package.json @@ -13,11 +13,6 @@ }, "exports": { ".": "./src/index.ts", - "./contracts": "./src/contracts.ts", - "./local-runtime": "./src/local-runtime.ts", - "./runtime": "./src/runtime/index.ts", - "./whisper": "./src/whisper/index.ts", - "./pdf-layout": "./src/pdf-layout/index.ts", - "./pdf-layout/parse": "./src/pdf-layout/parsePdf.ts" + "./local-runtime": "./src/local-runtime.ts" } } diff --git a/compute/core/src/index.ts b/compute/core/src/index.ts index 54397e2..b336427 100644 --- a/compute/core/src/index.ts +++ b/compute/core/src/index.ts @@ -1,6 +1,19 @@ export * from './contracts'; export { - ensureComputeModels, - runPdfLayoutFromPdfBuffer, - runWhisperAlignmentFromAudioBuffer, -} from './local-runtime'; + getComputeJobConcurrency, + getAvailableCpuCores, + getOnnxThreadsPerJob, +} from './runtime/cpu-budget'; +export { + getComputeTimeoutConfig, + getWorkerClientWaitTimeoutMs, + withTimeout, + withIdleTimeoutAndHardCap, + type ComputeTimeoutConfig, + type ComputeOperationKind, + type IdleTimeoutAndHardCapInput, +} from './runtime/timeout-config'; +export { renderPage } from './pdf-layout/renderPage'; +export { mergeTextWithRegions } from './pdf-layout/mergeTextWithRegions'; +export { stitchCrossPageBlocks } from './pdf-layout/stitchCrossPageBlocks'; +export { normalizeTextItemsForLayout } from './pdf-layout/normalizeTextItemsForLayout'; diff --git a/compute/core/src/pdf-layout/normalizeTextItemsForLayout.ts b/compute/core/src/pdf-layout/normalizeTextItemsForLayout.ts new file mode 100644 index 0000000..2c2eca7 --- /dev/null +++ b/compute/core/src/pdf-layout/normalizeTextItemsForLayout.ts @@ -0,0 +1,35 @@ +import type { TextItem } from 'pdfjs-dist/types/src/display/api'; +import type { PdfTextItem } from './types'; + +export function normalizeTextItemsForLayout(items: TextItem[], pageHeight: number): PdfTextItem[] { + return items + .filter((item) => { + if (!(typeof item.str === 'string' && item.str.trim().length > 0)) return false; + const transform = item.transform; + if (!Array.isArray(transform) || transform.length < 6) return false; + + // Reject heavily skewed/rotated text runs (e.g. vertical margin labels + // such as arXiv metadata) so they do not get merged into body blocks. + const skewX = Number(transform[1] ?? 0); + const skewY = Number(transform[2] ?? 0); + if (Math.abs(skewX) > 0.5 || Math.abs(skewY) > 0.5) return false; + + return true; + }) + .map((item) => { + const x = Number(item.transform[4] ?? 0); + const width = Math.max(0, Number(item.width ?? 0)); + const height = Math.max(1, Math.abs(Number(item.transform[3] ?? 1))); + const baselineY = Number(item.transform[5] ?? 0); + // pdf.js text transforms are in PDF user-space (origin bottom-left). + // Normalize into top-left page coordinates to match rendered image/model boxes. + const y = Math.max(0, pageHeight - baselineY - height); + return { + text: item.str, + x, + y, + width, + height, + }; + }); +} diff --git a/compute/core/src/pdf-layout/parsePdf.ts b/compute/core/src/pdf-layout/parsePdf.ts index ccc07dd..1b33ed5 100644 --- a/compute/core/src/pdf-layout/parsePdf.ts +++ b/compute/core/src/pdf-layout/parsePdf.ts @@ -1,12 +1,12 @@ import path from 'path'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; import type { ParsedPdfDocument, ParsedPdfPage } from '../types/parsed-pdf'; -import type { PdfTextItem } from './types'; import { ensureModel } from './ensureModel'; import { runLayoutModel } from './runLayoutModel'; import { mergeTextWithRegions } from './mergeTextWithRegions'; import { stitchCrossPageBlocks } from './stitchCrossPageBlocks'; import { renderPage } from './renderPage'; +import { normalizeTextItemsForLayout } from './normalizeTextItemsForLayout'; interface ParsePdfInput { documentId: string; @@ -25,39 +25,6 @@ function resolvePdfjsStandardFontDataUrl(): string { return `${standardFontDir.replace(/\/?$/, '/')}`; } -export function normalizeTextItemsForLayout(items: TextItem[], pageHeight: number): PdfTextItem[] { - return items - .filter((item) => { - if (!(typeof item.str === 'string' && item.str.trim().length > 0)) return false; - const transform = item.transform; - if (!Array.isArray(transform) || transform.length < 6) return false; - - // Reject heavily skewed/rotated text runs (e.g. vertical margin labels - // such as arXiv metadata) so they do not get merged into body blocks. - const skewX = Number(transform[1] ?? 0); - const skewY = Number(transform[2] ?? 0); - if (Math.abs(skewX) > 0.5 || Math.abs(skewY) > 0.5) return false; - - return true; - }) - .map((item) => { - const x = Number(item.transform[4] ?? 0); - const width = Math.max(0, Number(item.width ?? 0)); - const height = Math.max(1, Math.abs(Number(item.transform[3] ?? 1))); - const baselineY = Number(item.transform[5] ?? 0); - // pdf.js text transforms are in PDF user-space (origin bottom-left). - // Normalize into top-left page coordinates to match rendered image/model boxes. - const y = Math.max(0, pageHeight - baselineY - height); - return { - text: item.str, - x, - y, - width, - height, - }; - }); -} - export async function parsePdf(input: ParsePdfInput): Promise { await ensureModel(); diff --git a/compute/worker/src/server.ts b/compute/worker/src/server.ts index 6ceb6f7..14ce0d7 100644 --- a/compute/worker/src/server.ts +++ b/compute/worker/src/server.ts @@ -32,8 +32,6 @@ import { getOnnxThreadsPerJob, withIdleTimeoutAndHardCap, withTimeout, -} from '@openreader/compute-core/runtime'; -import { type PdfLayoutJobRequest, type PdfLayoutJobResult, type WhisperAlignJobRequest, @@ -45,7 +43,7 @@ import { type WorkerOperationRequest, type WorkerOperationState, type PdfLayoutProgress, -} from '@openreader/compute-core/contracts'; +} from '@openreader/compute-core'; import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; const JOBS_STREAM_NAME = 'compute_jobs'; diff --git a/src/lib/server/compute/concurrency-limiter.ts b/src/lib/server/compute/concurrency-limiter.ts index 1673f21..a67a959 100644 --- a/src/lib/server/compute/concurrency-limiter.ts +++ b/src/lib/server/compute/concurrency-limiter.ts @@ -1,4 +1,4 @@ -import { getComputeJobConcurrency } from '@openreader/compute-core/runtime'; +import { getComputeJobConcurrency } from '@openreader/compute-core'; export class ConcurrencyLimiter { private readonly maxInFlight: number; diff --git a/src/lib/server/compute/local.ts b/src/lib/server/compute/local.ts index 2cf0d6b..2d87965 100644 --- a/src/lib/server/compute/local.ts +++ b/src/lib/server/compute/local.ts @@ -6,12 +6,12 @@ import { getComputeTimeoutConfig, withIdleTimeoutAndHardCap, withTimeout, -} from '@openreader/compute-core/runtime'; + type PdfLayoutProgress, +} from '@openreader/compute-core'; import { runPdfLayoutFromPdfBuffer, runWhisperAlignmentFromAudioBuffer, } from '@openreader/compute-core/local-runtime'; -import type { PdfLayoutProgress } from '@openreader/compute-core/contracts'; export class LocalComputeBackend implements ComputeBackend { readonly mode = 'local' as const; diff --git a/src/lib/server/compute/types.ts b/src/lib/server/compute/types.ts index 524857b..92322a1 100644 --- a/src/lib/server/compute/types.ts +++ b/src/lib/server/compute/types.ts @@ -3,7 +3,7 @@ import type { TTSSentenceAlignment, ParsedPdfDocument, PdfLayoutProgress, -} from '@openreader/compute-core/contracts'; +} from '@openreader/compute-core'; export type ComputeMode = 'local' | 'worker'; diff --git a/src/lib/server/compute/worker-contract.ts b/src/lib/server/compute/worker-contract.ts index 950e62a..83d5a7a 100644 --- a/src/lib/server/compute/worker-contract.ts +++ b/src/lib/server/compute/worker-contract.ts @@ -14,4 +14,4 @@ export { type WorkerJobState, type WorkerJobTiming, type WorkerJobStatusResponse, -} from '@openreader/compute-core/contracts'; +} from '@openreader/compute-core'; diff --git a/src/lib/server/compute/worker.ts b/src/lib/server/compute/worker.ts index b4c8abf..2fa7cdf 100644 --- a/src/lib/server/compute/worker.ts +++ b/src/lib/server/compute/worker.ts @@ -1,6 +1,6 @@ import { createHash, randomUUID } from 'node:crypto'; import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types'; -import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core/runtime'; +import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core'; import type { PdfLayoutJobRequest, PdfLayoutJobResult, diff --git a/src/lib/server/documents/previews-render.ts b/src/lib/server/documents/previews-render.ts index a4efdb0..c32bbce 100644 --- a/src/lib/server/documents/previews-render.ts +++ b/src/lib/server/documents/previews-render.ts @@ -2,7 +2,7 @@ import path from 'path'; import { createCanvas, loadImage } from '@napi-rs/canvas'; import JSZip from 'jszip'; import { XMLParser } from 'fast-xml-parser'; -import { renderPage } from '@openreader/compute-core/pdf-layout'; +import { renderPage } from '@openreader/compute-core'; export type RenderedDocumentPreview = { bytes: Buffer; diff --git a/src/lib/server/jobs/parsePdfJob.ts b/src/lib/server/jobs/parsePdfJob.ts index 87ac52c..a5fbef0 100644 --- a/src/lib/server/jobs/parsePdfJob.ts +++ b/src/lib/server/jobs/parsePdfJob.ts @@ -5,7 +5,7 @@ import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobs import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state'; import { getCompute } from '@/lib/server/compute'; import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache'; -import type { PdfLayoutProgress } from '@openreader/compute-core/contracts'; +import type { PdfLayoutProgress } from '@openreader/compute-core'; interface ParsePdfJobInput { documentId: string; diff --git a/tests/unit/pdf-merge-text-with-regions.spec.ts b/tests/unit/pdf-merge-text-with-regions.spec.ts index c3ca22a..86bd435 100644 --- a/tests/unit/pdf-merge-text-with-regions.spec.ts +++ b/tests/unit/pdf-merge-text-with-regions.spec.ts @@ -1,5 +1,5 @@ import { expect, test } from '@playwright/test'; -import { mergeTextWithRegions } from '@openreader/compute-core/pdf-layout'; +import { mergeTextWithRegions } from '@openreader/compute-core'; test.describe('mergeTextWithRegions', () => { test('assigns text items to containing regions by centroid and joins in reading order', () => { diff --git a/tests/unit/pdf-parse-normalize-text-items.spec.ts b/tests/unit/pdf-parse-normalize-text-items.spec.ts index 5e5f693..aac976c 100644 --- a/tests/unit/pdf-parse-normalize-text-items.spec.ts +++ b/tests/unit/pdf-parse-normalize-text-items.spec.ts @@ -1,6 +1,6 @@ import { expect, test } from '@playwright/test'; -import { normalizeTextItemsForLayout } from '@openreader/compute-core/pdf-layout/parse'; +import { normalizeTextItemsForLayout } from '@openreader/compute-core'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; function makeTextItem( diff --git a/tests/unit/pdf-stitch-cross-page-blocks.spec.ts b/tests/unit/pdf-stitch-cross-page-blocks.spec.ts index ed11f0f..86e6d4a 100644 --- a/tests/unit/pdf-stitch-cross-page-blocks.spec.ts +++ b/tests/unit/pdf-stitch-cross-page-blocks.spec.ts @@ -1,5 +1,5 @@ import { expect, test } from '@playwright/test'; -import { stitchCrossPageBlocks } from '@openreader/compute-core/pdf-layout'; +import { stitchCrossPageBlocks } from '@openreader/compute-core'; import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../src/types/parsed-pdf'; function makeBlock( diff --git a/tests/unit/whisper-alignment-mapping.spec.ts b/tests/unit/whisper-alignment-mapping.spec.ts index 1350d07..d700183 100644 --- a/tests/unit/whisper-alignment-mapping.spec.ts +++ b/tests/unit/whisper-alignment-mapping.spec.ts @@ -1,7 +1,7 @@ import { test, expect } from '@playwright/test'; import { mapWordsToSentenceOffsets, -} from '@openreader/compute-core/whisper'; +} from '../../compute/core/src/whisper/alignment-mapping'; test.describe('whisper alignment mapping', () => { test('maps words to sentence offsets with punctuation and repeated spaces', () => { diff --git a/tests/unit/whisper-alignment-smoke.spec.ts b/tests/unit/whisper-alignment-smoke.spec.ts index 25dac82..83f2e1c 100644 --- a/tests/unit/whisper-alignment-smoke.spec.ts +++ b/tests/unit/whisper-alignment-smoke.spec.ts @@ -1,7 +1,7 @@ import { test, expect } from '@playwright/test'; import { readFile } from 'fs/promises'; import path from 'path'; -import { alignAudioWithText } from '@openreader/compute-core/whisper'; +import { alignAudioWithText } from '../../compute/core/src/whisper/alignment'; test.describe('whisper alignment smoke', () => { test('runs ONNX alignment end-to-end without decoder reshape errors', async () => { diff --git a/tests/unit/whisper-ensure-model.spec.ts b/tests/unit/whisper-ensure-model.spec.ts index 44e7948..613f985 100644 --- a/tests/unit/whisper-ensure-model.spec.ts +++ b/tests/unit/whisper-ensure-model.spec.ts @@ -6,7 +6,7 @@ import path from 'path'; import { createSingleflightRunner, ensureWhisperArtifacts, -} from '@openreader/compute-core/whisper'; +} from '../../compute/core/src/whisper/ensureModel'; function sha256(bytes: Uint8Array): string { return createHash('sha256').update(bytes).digest('hex'); diff --git a/tests/unit/whisper-spectral.spec.ts b/tests/unit/whisper-spectral.spec.ts index 7e9c44f..5b35e2f 100644 --- a/tests/unit/whisper-spectral.spec.ts +++ b/tests/unit/whisper-spectral.spec.ts @@ -1,5 +1,5 @@ import { test, expect } from '@playwright/test'; -import { buildGoertzelCoefficients, goertzelPower } from '@openreader/compute-core/whisper'; +import { buildGoertzelCoefficients, goertzelPower } from '../../compute/core/src/whisper/spectral'; function dftPower(samples: Float32Array, k: number): number { const n = samples.length; diff --git a/tests/unit/whisper-token-timestamps.spec.ts b/tests/unit/whisper-token-timestamps.spec.ts index bce4304..68660b6 100644 --- a/tests/unit/whisper-token-timestamps.spec.ts +++ b/tests/unit/whisper-token-timestamps.spec.ts @@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-node'; import { buildWordsFromTimestampedTokens, extractTokenStartTimestamps, -} from '@openreader/compute-core/whisper'; +} from '../../compute/core/src/whisper/token-timestamps'; test.describe('whisper token timestamp alignment', () => { test('extracts monotonic token timestamps from cross-attention maps', () => { diff --git a/tsconfig.json b/tsconfig.json index 3fba768..47f6fae 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,12 +21,7 @@ "paths": { "@/*": ["./src/*"], "@openreader/compute-core": ["./compute/core/src/index.ts"], - "@openreader/compute-core/contracts": ["./compute/core/src/contracts.ts"], - "@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"], - "@openreader/compute-core/runtime": ["./compute/core/src/runtime/index.ts"], - "@openreader/compute-core/whisper": ["./compute/core/src/whisper/index.ts"], - "@openreader/compute-core/pdf-layout": ["./compute/core/src/pdf-layout/index.ts"], - "@openreader/compute-core/pdf-layout/parse": ["./compute/core/src/pdf-layout/parsePdf.ts"] + "@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"] } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],