refactor(compute-core): collapse to root/local-runtime entrypoints

This commit is contained in:
Richard R 2026-05-21 21:30:44 -06:00
parent 37d11bf9b8
commit 37b6999c2c
21 changed files with 72 additions and 69 deletions

View file

@ -13,11 +13,6 @@
},
"exports": {
".": "./src/index.ts",
"./contracts": "./src/contracts.ts",
"./local-runtime": "./src/local-runtime.ts",
"./runtime": "./src/runtime/index.ts",
"./whisper": "./src/whisper/index.ts",
"./pdf-layout": "./src/pdf-layout/index.ts",
"./pdf-layout/parse": "./src/pdf-layout/parsePdf.ts"
"./local-runtime": "./src/local-runtime.ts"
}
}

View file

@ -1,6 +1,19 @@
export * from './contracts';
export {
ensureComputeModels,
runPdfLayoutFromPdfBuffer,
runWhisperAlignmentFromAudioBuffer,
} from './local-runtime';
getComputeJobConcurrency,
getAvailableCpuCores,
getOnnxThreadsPerJob,
} from './runtime/cpu-budget';
export {
getComputeTimeoutConfig,
getWorkerClientWaitTimeoutMs,
withTimeout,
withIdleTimeoutAndHardCap,
type ComputeTimeoutConfig,
type ComputeOperationKind,
type IdleTimeoutAndHardCapInput,
} from './runtime/timeout-config';
export { renderPage } from './pdf-layout/renderPage';
export { mergeTextWithRegions } from './pdf-layout/mergeTextWithRegions';
export { stitchCrossPageBlocks } from './pdf-layout/stitchCrossPageBlocks';
export { normalizeTextItemsForLayout } from './pdf-layout/normalizeTextItemsForLayout';

View file

@ -0,0 +1,35 @@
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
import type { PdfTextItem } from './types';
export function normalizeTextItemsForLayout(items: TextItem[], pageHeight: number): PdfTextItem[] {
return items
.filter((item) => {
if (!(typeof item.str === 'string' && item.str.trim().length > 0)) return false;
const transform = item.transform;
if (!Array.isArray(transform) || transform.length < 6) return false;
// Reject heavily skewed/rotated text runs (e.g. vertical margin labels
// such as arXiv metadata) so they do not get merged into body blocks.
const skewX = Number(transform[1] ?? 0);
const skewY = Number(transform[2] ?? 0);
if (Math.abs(skewX) > 0.5 || Math.abs(skewY) > 0.5) return false;
return true;
})
.map((item) => {
const x = Number(item.transform[4] ?? 0);
const width = Math.max(0, Number(item.width ?? 0));
const height = Math.max(1, Math.abs(Number(item.transform[3] ?? 1)));
const baselineY = Number(item.transform[5] ?? 0);
// pdf.js text transforms are in PDF user-space (origin bottom-left).
// Normalize into top-left page coordinates to match rendered image/model boxes.
const y = Math.max(0, pageHeight - baselineY - height);
return {
text: item.str,
x,
y,
width,
height,
};
});
}

View file

@ -1,12 +1,12 @@
import path from 'path';
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
import type { ParsedPdfDocument, ParsedPdfPage } from '../types/parsed-pdf';
import type { PdfTextItem } from './types';
import { ensureModel } from './ensureModel';
import { runLayoutModel } from './runLayoutModel';
import { mergeTextWithRegions } from './mergeTextWithRegions';
import { stitchCrossPageBlocks } from './stitchCrossPageBlocks';
import { renderPage } from './renderPage';
import { normalizeTextItemsForLayout } from './normalizeTextItemsForLayout';
interface ParsePdfInput {
documentId: string;
@ -25,39 +25,6 @@ function resolvePdfjsStandardFontDataUrl(): string {
return `${standardFontDir.replace(/\/?$/, '/')}`;
}
export function normalizeTextItemsForLayout(items: TextItem[], pageHeight: number): PdfTextItem[] {
return items
.filter((item) => {
if (!(typeof item.str === 'string' && item.str.trim().length > 0)) return false;
const transform = item.transform;
if (!Array.isArray(transform) || transform.length < 6) return false;
// Reject heavily skewed/rotated text runs (e.g. vertical margin labels
// such as arXiv metadata) so they do not get merged into body blocks.
const skewX = Number(transform[1] ?? 0);
const skewY = Number(transform[2] ?? 0);
if (Math.abs(skewX) > 0.5 || Math.abs(skewY) > 0.5) return false;
return true;
})
.map((item) => {
const x = Number(item.transform[4] ?? 0);
const width = Math.max(0, Number(item.width ?? 0));
const height = Math.max(1, Math.abs(Number(item.transform[3] ?? 1)));
const baselineY = Number(item.transform[5] ?? 0);
// pdf.js text transforms are in PDF user-space (origin bottom-left).
// Normalize into top-left page coordinates to match rendered image/model boxes.
const y = Math.max(0, pageHeight - baselineY - height);
return {
text: item.str,
x,
y,
width,
height,
};
});
}
export async function parsePdf(input: ParsePdfInput): Promise<ParsedPdfDocument> {
await ensureModel();

View file

@ -32,8 +32,6 @@ import {
getOnnxThreadsPerJob,
withIdleTimeoutAndHardCap,
withTimeout,
} from '@openreader/compute-core/runtime';
import {
type PdfLayoutJobRequest,
type PdfLayoutJobResult,
type WhisperAlignJobRequest,
@ -45,7 +43,7 @@ import {
type WorkerOperationRequest,
type WorkerOperationState,
type PdfLayoutProgress,
} from '@openreader/compute-core/contracts';
} from '@openreader/compute-core';
import { GetObjectCommand, PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
const JOBS_STREAM_NAME = 'compute_jobs';

View file

@ -1,4 +1,4 @@
import { getComputeJobConcurrency } from '@openreader/compute-core/runtime';
import { getComputeJobConcurrency } from '@openreader/compute-core';
export class ConcurrencyLimiter {
private readonly maxInFlight: number;

View file

@ -6,12 +6,12 @@ import {
getComputeTimeoutConfig,
withIdleTimeoutAndHardCap,
withTimeout,
} from '@openreader/compute-core/runtime';
type PdfLayoutProgress,
} from '@openreader/compute-core';
import {
runPdfLayoutFromPdfBuffer,
runWhisperAlignmentFromAudioBuffer,
} from '@openreader/compute-core/local-runtime';
import type { PdfLayoutProgress } from '@openreader/compute-core/contracts';
export class LocalComputeBackend implements ComputeBackend {
readonly mode = 'local' as const;

View file

@ -3,7 +3,7 @@ import type {
TTSSentenceAlignment,
ParsedPdfDocument,
PdfLayoutProgress,
} from '@openreader/compute-core/contracts';
} from '@openreader/compute-core';
export type ComputeMode = 'local' | 'worker';

View file

@ -14,4 +14,4 @@ export {
type WorkerJobState,
type WorkerJobTiming,
type WorkerJobStatusResponse,
} from '@openreader/compute-core/contracts';
} from '@openreader/compute-core';

View file

@ -1,6 +1,6 @@
import { createHash, randomUUID } from 'node:crypto';
import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types';
import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core/runtime';
import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core';
import type {
PdfLayoutJobRequest,
PdfLayoutJobResult,

View file

@ -2,7 +2,7 @@ import path from 'path';
import { createCanvas, loadImage } from '@napi-rs/canvas';
import JSZip from 'jszip';
import { XMLParser } from 'fast-xml-parser';
import { renderPage } from '@openreader/compute-core/pdf-layout';
import { renderPage } from '@openreader/compute-core';
export type RenderedDocumentPreview = {
bytes: Buffer;

View file

@ -5,7 +5,7 @@ import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobs
import { stringifyDocumentParseState } from '@/lib/server/documents/parse-state';
import { getCompute } from '@/lib/server/compute';
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
import type { PdfLayoutProgress } from '@openreader/compute-core/contracts';
import type { PdfLayoutProgress } from '@openreader/compute-core';
interface ParsePdfJobInput {
documentId: string;

View file

@ -1,5 +1,5 @@
import { expect, test } from '@playwright/test';
import { mergeTextWithRegions } from '@openreader/compute-core/pdf-layout';
import { mergeTextWithRegions } from '@openreader/compute-core';
test.describe('mergeTextWithRegions', () => {
test('assigns text items to containing regions by centroid and joins in reading order', () => {

View file

@ -1,6 +1,6 @@
import { expect, test } from '@playwright/test';
import { normalizeTextItemsForLayout } from '@openreader/compute-core/pdf-layout/parse';
import { normalizeTextItemsForLayout } from '@openreader/compute-core';
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
function makeTextItem(

View file

@ -1,5 +1,5 @@
import { expect, test } from '@playwright/test';
import { stitchCrossPageBlocks } from '@openreader/compute-core/pdf-layout';
import { stitchCrossPageBlocks } from '@openreader/compute-core';
import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../src/types/parsed-pdf';
function makeBlock(

View file

@ -1,7 +1,7 @@
import { test, expect } from '@playwright/test';
import {
mapWordsToSentenceOffsets,
} from '@openreader/compute-core/whisper';
} from '../../compute/core/src/whisper/alignment-mapping';
test.describe('whisper alignment mapping', () => {
test('maps words to sentence offsets with punctuation and repeated spaces', () => {

View file

@ -1,7 +1,7 @@
import { test, expect } from '@playwright/test';
import { readFile } from 'fs/promises';
import path from 'path';
import { alignAudioWithText } from '@openreader/compute-core/whisper';
import { alignAudioWithText } from '../../compute/core/src/whisper/alignment';
test.describe('whisper alignment smoke', () => {
test('runs ONNX alignment end-to-end without decoder reshape errors', async () => {

View file

@ -6,7 +6,7 @@ import path from 'path';
import {
createSingleflightRunner,
ensureWhisperArtifacts,
} from '@openreader/compute-core/whisper';
} from '../../compute/core/src/whisper/ensureModel';
function sha256(bytes: Uint8Array): string {
return createHash('sha256').update(bytes).digest('hex');

View file

@ -1,5 +1,5 @@
import { test, expect } from '@playwright/test';
import { buildGoertzelCoefficients, goertzelPower } from '@openreader/compute-core/whisper';
import { buildGoertzelCoefficients, goertzelPower } from '../../compute/core/src/whisper/spectral';
function dftPower(samples: Float32Array, k: number): number {
const n = samples.length;

View file

@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-node';
import {
buildWordsFromTimestampedTokens,
extractTokenStartTimestamps,
} from '@openreader/compute-core/whisper';
} from '../../compute/core/src/whisper/token-timestamps';
test.describe('whisper token timestamp alignment', () => {
test('extracts monotonic token timestamps from cross-attention maps', () => {

View file

@ -21,12 +21,7 @@
"paths": {
"@/*": ["./src/*"],
"@openreader/compute-core": ["./compute/core/src/index.ts"],
"@openreader/compute-core/contracts": ["./compute/core/src/contracts.ts"],
"@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"],
"@openreader/compute-core/runtime": ["./compute/core/src/runtime/index.ts"],
"@openreader/compute-core/whisper": ["./compute/core/src/whisper/index.ts"],
"@openreader/compute-core/pdf-layout": ["./compute/core/src/pdf-layout/index.ts"],
"@openreader/compute-core/pdf-layout/parse": ["./compute/core/src/pdf-layout/parsePdf.ts"]
"@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"]
}
},
"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],