refactor(compute-core): rename pdf and whisper modules to clean capability layout

This commit is contained in:
Richard R 2026-05-21 21:41:37 -06:00
parent 3a2c6ffa42
commit 55acf05ad6
25 changed files with 32 additions and 32 deletions

View file

@ -58,14 +58,14 @@ COPY --from=app-builder /app/THIRD_PARTY_LICENSES /licenses
# Include SeaweedFS license text for the copied weed binary.
COPY --from=seaweedfs-builder /tmp/SeaweedFS-LICENSE.txt /licenses/SeaweedFS-LICENSE.txt
# Include static model notices for runtime-downloaded assets.
COPY --from=app-builder /app/compute/core/src/pdf-layout/model/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt
COPY --from=app-builder /app/compute/core/src/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt
# Copy seaweedfs weed binary for optional embedded local S3.
COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed
RUN chmod +x /usr/local/bin/weed
# Include OpenAI Whisper license text for runtime-downloaded ONNX artifacts.
COPY --from=app-builder /app/compute/core/src/whisper/model/LICENSE.txt /licenses/openai-whisper-LICENSE.txt
COPY --from=app-builder /app/compute/core/src/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt
# Expose the port the app runs on
EXPOSE 3003

View file

@ -13,7 +13,7 @@ export {
type ComputeOperationKind,
type IdleTimeoutAndHardCapInput,
} from './config/timeout';
export { renderPage } from './pdf-layout/renderPage';
export { mergeTextWithRegions } from './pdf-layout/mergeTextWithRegions';
export { stitchCrossPageBlocks } from './pdf-layout/stitchCrossPageBlocks';
export { normalizeTextItemsForLayout } from './pdf-layout/normalizeTextItemsForLayout';
export { renderPage } from './pdf/render';
export { mergeTextWithRegions } from './pdf/merge';
export { stitchCrossPageBlocks } from './pdf/stitch';
export { normalizeTextItemsForLayout } from './pdf/normalize-text';

View file

@ -1,7 +1,7 @@
import { ensureWhisperModel } from './whisper/ensureModel';
import { alignAudioWithText } from './whisper/alignment';
import { ensureModel as ensurePdfLayoutModel } from './pdf-layout/ensureModel';
import { parsePdf } from './pdf-layout/parsePdf';
import { ensureWhisperModel } from './whisper/model';
import { alignAudioWithText } from './whisper/align';
import { ensureModel as ensurePdfLayoutModel } from './pdf/model';
import { parsePdf } from './pdf/parse';
export async function ensureComputeModels(): Promise<void> {
await Promise.all([ensureWhisperModel(), ensurePdfLayoutModel()]);

View file

@ -1,3 +0,0 @@
export { renderPage } from './renderPage';
export { mergeTextWithRegions } from './mergeTextWithRegions';
export { stitchCrossPageBlocks } from './stitchCrossPageBlocks';

View file

@ -0,0 +1,3 @@
export { renderPage } from './render';
export { mergeTextWithRegions } from './merge';
export { stitchCrossPageBlocks } from './stitch';

View file

@ -3,7 +3,7 @@ import { fileURLToPath } from 'url';
import { createHash } from 'crypto';
import { access, mkdir, rename, writeFile, readFile, unlink, copyFile } from 'fs/promises';
import { DOCSTORE_DIR } from '../platform/docstore';
import manifest from './model/manifest.json';
import manifest from './assets/manifest.json';
const DEFAULT_MODEL_BASE_URL = 'https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main';
const PDF_LAYOUT_MODEL_BASE_URL_ENV = 'PDF_LAYOUT_MODEL_BASE_URL';
@ -14,7 +14,7 @@ export const MODEL_DATA_PATH = path.join(MODEL_DIR, 'PP-DocLayoutV3.onnx.data');
export const MODEL_CONFIG_PATH = path.join(MODEL_DIR, 'pp-doclayoutv3.config.json');
export const MODEL_PREPROCESSOR_PATH = path.join(MODEL_DIR, 'pp-doclayoutv3.preprocessor_config.json');
const LICENSE_PATH = path.join(MODEL_DIR, 'pp-doclayoutv3.LICENSE.txt');
const STATIC_LICENSE_PATH = path.join(MODULE_DIR, 'model', 'LICENSE.txt');
const STATIC_LICENSE_PATH = path.join(MODULE_DIR, 'assets', 'LICENSE.txt');
let inflight: Promise<string> | null = null;

View file

@ -1,12 +1,12 @@
import path from 'path';
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
import type { ParsedPdfDocument, ParsedPdfPage } from '../types/parsed-pdf';
import { ensureModel } from './ensureModel';
import { ensureModel } from './model';
import { runLayoutModel } from './runLayoutModel';
import { mergeTextWithRegions } from './mergeTextWithRegions';
import { stitchCrossPageBlocks } from './stitchCrossPageBlocks';
import { renderPage } from './renderPage';
import { normalizeTextItemsForLayout } from './normalizeTextItemsForLayout';
import { mergeTextWithRegions } from './merge';
import { stitchCrossPageBlocks } from './stitch';
import { renderPage } from './render';
import { normalizeTextItemsForLayout } from './normalize-text';
interface ParsePdfInput {
documentId: string;

View file

@ -1,7 +1,7 @@
import * as ort from 'onnxruntime-node';
import { readFile } from 'fs/promises';
import type { LayoutRegion, PdfTextItem } from './types';
import { ensureModel, MODEL_CONFIG_PATH, MODEL_PREPROCESSOR_PATH } from './ensureModel';
import { ensureModel, MODEL_CONFIG_PATH, MODEL_PREPROCESSOR_PATH } from './model';
import { getOnnxThreadsPerJob } from '../config/cpu-budget';
interface RunLayoutInput {

View file

@ -14,7 +14,7 @@ import { getComputeTimeoutConfig } from '../config/timeout';
import {
mapWordsToSentenceOffsets,
type WhisperWord,
} from './alignment-mapping';
} from './alignment-map';
import { buildGoertzelCoefficients, goertzelPower } from './spectral';
import {
buildWordsFromTimestampedTokens,
@ -29,7 +29,7 @@ import {
WHISPER_ENCODER_MODEL_PATH,
WHISPER_DECODER_MERGED_MODEL_PATH,
WHISPER_DECODER_WITH_PAST_MODEL_PATH,
} from './ensureModel';
} from './model';
interface WhisperAlignmentOptions {
lang?: string;
@ -114,7 +114,7 @@ const hannWindow = buildHannWindow(N_FFT);
const goertzelCoefficients = buildGoertzelCoefficients(MEL_FILTER_BINS, N_FFT);
const MODULE_DIR = dirname(fileURLToPath(import.meta.url));
const MEL_FILTERS_NPZ_PATH = join(MODULE_DIR, 'model', 'mel_filters.npz');
const MEL_FILTERS_NPZ_PATH = join(MODULE_DIR, 'assets', 'mel_filters.npz');
function buildHannWindow(length: number): Float32Array {
const window = new Float32Array(length);

View file

@ -2,7 +2,7 @@ export {
alignAudioWithText,
makeWhisperCacheKey,
type WhisperRequestBody,
} from './alignment';
} from './align';
export {
ensureWhisperModel,
@ -11,8 +11,8 @@ export {
type WhisperArtifactSpec,
type WhisperStaticArtifactSpec,
type WhisperFetch,
} from './ensureModel';
} from './model';
export { mapWordsToSentenceOffsets, type WhisperWord } from './alignment-mapping';
export { mapWordsToSentenceOffsets, type WhisperWord } from './alignment-map';
export { buildGoertzelCoefficients, goertzelPower } from './spectral';
export { buildWordsFromTimestampedTokens, extractTokenStartTimestamps } from './token-timestamps';

View file

@ -3,11 +3,11 @@ import { fileURLToPath } from 'url';
import { createHash } from 'crypto';
import { access, copyFile, mkdir, readFile, rename, unlink, writeFile } from 'fs/promises';
import { DOCSTORE_DIR } from '../platform/docstore';
import manifest from './model/manifest.json';
import manifest from './assets/manifest.json';
const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
const MODEL_DIR = path.join(DOCSTORE_DIR, 'model', 'whisper-base_timestamped');
const STATIC_LICENSE_PATH = path.join(MODULE_DIR, 'model', 'LICENSE.txt');
const STATIC_LICENSE_PATH = path.join(MODULE_DIR, 'assets', 'LICENSE.txt');
export const WHISPER_CONFIG_PATH = path.join(MODEL_DIR, 'config.json');
export const WHISPER_GENERATION_CONFIG_PATH = path.join(MODEL_DIR, 'generation_config.json');

View file

@ -1,7 +1,7 @@
import { test, expect } from '@playwright/test';
import {
mapWordsToSentenceOffsets,
} from '../../compute/core/src/whisper/alignment-mapping';
} from '../../compute/core/src/whisper/alignment-map';
test.describe('whisper alignment mapping', () => {
test('maps words to sentence offsets with punctuation and repeated spaces', () => {

View file

@ -1,7 +1,7 @@
import { test, expect } from '@playwright/test';
import { readFile } from 'fs/promises';
import path from 'path';
import { alignAudioWithText } from '../../compute/core/src/whisper/alignment';
import { alignAudioWithText } from '../../compute/core/src/whisper/align';
test.describe('whisper alignment smoke', () => {
test('runs ONNX alignment end-to-end without decoder reshape errors', async () => {

View file

@ -6,7 +6,7 @@ import path from 'path';
import {
createSingleflightRunner,
ensureWhisperArtifacts,
} from '../../compute/core/src/whisper/ensureModel';
} from '../../compute/core/src/whisper/model';
function sha256(bytes: Uint8Array): string {
return createHash('sha256').update(bytes).digest('hex');