chore(worker): restructure inference modules and centralize config
- Remove legacy inference and platform files from `src/inference/` and migrate responsibilities to new modular locations under `src/infrastructure/`, `src/inference/pdf/`, and `src/inference/whisper/` - Consolidate environment/config logic into `src/infrastructure/config.ts` - Move docstore and ffmpeg platform utilities to `src/infrastructure/platform.ts` - Refactor PDF and Whisper inference code to use new document layout, layout model, and timestamp utilities - Update API contracts to define and export types and constants previously scattered in inference/types - Adjust all imports in jobs, storage, and tests to reference new module structure and type locations - Inline parser version and encoding logic into API contracts - Remove obsolete files and update test fixtures for new type locations This update improves codebase modularity, maintainability, and separation of concerns. No changes to inference or orchestration functionality. BREAKING CHANGE: inference module structure, type imports, and config utilities have changed; downstream code must update imports and integration points.
This commit is contained in:
parent
3d5dfd2a88
commit
6e69b48103
38 changed files with 163 additions and 248 deletions
|
|
@ -15,13 +15,13 @@ import {
|
|||
import { Kvm } from '@nats-io/kv';
|
||||
import {
|
||||
ensureComputeModels,
|
||||
} from '../inference/local-runtime';
|
||||
} from '../inference/runtime';
|
||||
import {
|
||||
getComputeTimeoutConfig,
|
||||
getComputeOpStaleMs,
|
||||
getAvailableCpuCores,
|
||||
getOnnxThreadsPerJob,
|
||||
} from '../inference';
|
||||
} from '../infrastructure/config';
|
||||
import { encodeSseFrame, OperationOrchestrator } from '../operations';
|
||||
import type {
|
||||
PdfLayoutJobRequest,
|
||||
|
|
|
|||
|
|
@ -1,24 +1,26 @@
|
|||
import type { TTSSentenceAlignment } from '../inference/types/tts';
|
||||
import type { ParsedPdfDocument } from '../inference/types/parsed-pdf';
|
||||
import type { TTSSentenceAlignment, ParsedPdfDocument } from './types';
|
||||
|
||||
export type {
|
||||
TTSAudioBuffer,
|
||||
TTSAudioBytes,
|
||||
TTSSentenceAlignment,
|
||||
TTSSentenceWord,
|
||||
} from '../inference/types/tts';
|
||||
} from './types';
|
||||
export type {
|
||||
ParsedPdfBlockKind,
|
||||
ParsedPdfBlockFragment,
|
||||
ParsedPdfBlock,
|
||||
ParsedPdfPage,
|
||||
ParsedPdfDocument,
|
||||
} from '../inference/types/parsed-pdf';
|
||||
} from './types';
|
||||
|
||||
export const ALIGN_QUEUE_NAME = 'whisper-align';
|
||||
export const PDF_LAYOUT_QUEUE_NAME = 'pdf-layout';
|
||||
export { PDF_PARSER_VERSION } from '../inference/pdf/parser-version';
|
||||
export { encodeParserVersion } from '../inference/pdf/parser-version-key';
|
||||
export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69';
|
||||
|
||||
export function encodeParserVersion(parserVersion: string, defaultVersion = PDF_PARSER_VERSION): string {
|
||||
return encodeURIComponent(parserVersion.trim() || defaultVersion);
|
||||
}
|
||||
|
||||
export interface WhisperAlignJobBase {
|
||||
text: string;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { createHash } from 'node:crypto';
|
||||
import { PDF_PARSER_VERSION } from '../inference/pdf/parser-version';
|
||||
import { PDF_PARSER_VERSION } from '../api/contracts';
|
||||
|
||||
function sha256Hex(input: string): string {
|
||||
return createHash('sha256').update(input).digest('hex');
|
||||
|
|
|
|||
|
|
@ -61,3 +61,20 @@ export interface PdfParseProgress {
|
|||
currentPage?: number;
|
||||
phase: PdfParsePhase;
|
||||
}
|
||||
|
||||
export type TTSAudioBuffer = ArrayBuffer;
|
||||
export type TTSAudioBytes = number[];
|
||||
|
||||
export interface TTSSentenceWord {
|
||||
text: string;
|
||||
startSec: number;
|
||||
endSec: number;
|
||||
charStart: number;
|
||||
charEnd: number;
|
||||
}
|
||||
|
||||
export interface TTSSentenceAlignment {
|
||||
sentence: string;
|
||||
sentenceIndex: number;
|
||||
words: TTSSentenceWord[];
|
||||
}
|
||||
|
|
@ -1,28 +0,0 @@
|
|||
import os from 'node:os';
|
||||
|
||||
function readPositiveInt(name: string, fallback: number): number {
|
||||
const raw = process.env[name]?.trim();
|
||||
if (!raw) return fallback;
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
|
||||
return Math.floor(parsed);
|
||||
}
|
||||
|
||||
export function getComputeJobConcurrency(): number {
|
||||
return readPositiveInt('COMPUTE_JOB_CONCURRENCY', 1);
|
||||
}
|
||||
|
||||
export function getAvailableCpuCores(): number {
|
||||
if (typeof os.availableParallelism === 'function') {
|
||||
const value = os.availableParallelism();
|
||||
if (Number.isFinite(value) && value >= 1) return Math.floor(value);
|
||||
}
|
||||
const fallback = os.cpus().length;
|
||||
return Number.isFinite(fallback) && fallback >= 1 ? Math.floor(fallback) : 1;
|
||||
}
|
||||
|
||||
export function getOnnxThreadsPerJob(): number {
|
||||
const concurrency = getComputeJobConcurrency();
|
||||
const usableCores = Math.max(1, getAvailableCpuCores() - 1);
|
||||
return Math.max(1, Math.floor(usableCores / concurrency));
|
||||
}
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
export * from '../api/contracts';
|
||||
export {
|
||||
getComputeJobConcurrency,
|
||||
getAvailableCpuCores,
|
||||
getOnnxThreadsPerJob,
|
||||
} from './config/cpu-budget';
|
||||
export {
|
||||
getComputeTimeoutConfig,
|
||||
getComputeOpStaleMs,
|
||||
getWorkerClientWaitTimeoutMs,
|
||||
withTimeout,
|
||||
withIdleTimeoutAndHardCap,
|
||||
type ComputeTimeoutConfig,
|
||||
type ComputeOperationKind,
|
||||
type IdleTimeoutAndHardCapInput,
|
||||
} from './config/timeout';
|
||||
export { renderPage } from './pdf/render';
|
||||
export { mergeTextWithRegions } from './pdf/merge';
|
||||
export { PDF_PARSER_VERSION } from './pdf/parser-version';
|
||||
export { encodeParserVersion } from './pdf/parser-version-key';
|
||||
export { stitchCrossPageBlocks } from './pdf/stitch';
|
||||
export { normalizeTextItemsForLayout } from './pdf/normalize-text';
|
||||
export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map';
|
||||
export { buildGoertzelCoefficients, goertzelPower } from './whisper/spectral';
|
||||
export { buildWordsFromTimestampedTokens, extractTokenStartTimestamps } from './whisper/token-timestamps';
|
||||
export * from '../operations';
|
||||
|
|
@ -1,4 +1,18 @@
|
|||
import type { LayoutRegion, PdfTextItem } from './types';
|
||||
import type { ParsedPdfBlockKind } from '../../api/types';
|
||||
|
||||
export interface PdfTextItem {
|
||||
text: string;
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface LayoutRegion {
|
||||
bbox: [number, number, number, number];
|
||||
label: ParsedPdfBlockKind;
|
||||
confidence?: number;
|
||||
}
|
||||
|
||||
const NON_TEXT_REGION_LABELS = new Set<LayoutRegion['label']>(['chart', 'image', 'table', 'seal']);
|
||||
const TEXT_ASSIGNABLE_LABELS = new Set<LayoutRegion['label']>([
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
import * as ort from 'onnxruntime-node';
|
||||
import { readFile } from 'fs/promises';
|
||||
import type { LayoutRegion, PdfTextItem } from './types';
|
||||
import type { LayoutRegion, PdfTextItem } from './document-layout';
|
||||
import { ensureModel, MODEL_CONFIG_PATH, MODEL_PREPROCESSOR_PATH } from './model';
|
||||
import { getOnnxThreadsPerJob } from '../config/cpu-budget';
|
||||
import { getOnnxThreadsPerJob } from '../../infrastructure/config';
|
||||
|
||||
interface RunLayoutInput {
|
||||
pageWidth: number;
|
||||
|
|
@ -3,7 +3,7 @@ import { fileURLToPath } from 'url';
|
|||
import { createHash } from 'crypto';
|
||||
import { readFileSync } from 'fs';
|
||||
import { access, mkdir, rename, writeFile, readFile, unlink, copyFile } from 'fs/promises';
|
||||
import { DOCSTORE_DIR } from '../platform/docstore';
|
||||
import { DOCSTORE_DIR } from '../../infrastructure/platform';
|
||||
|
||||
const DEFAULT_MODEL_BASE_URL = 'https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main';
|
||||
const PDF_LAYOUT_MODEL_BASE_URL_ENV = 'PDF_LAYOUT_MODEL_BASE_URL';
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
||||
import type { PdfTextItem } from './types';
|
||||
import type { PdfTextItem } from './document-layout';
|
||||
|
||||
interface ViewportLike {
|
||||
height: number;
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
||||
import type { ParsedPdfDocument, ParsedPdfPage } from '../types/parsed-pdf';
|
||||
import type { ParsedPdfDocument, ParsedPdfPage } from '../../api/types';
|
||||
import { ensureModel } from './model';
|
||||
import { runLayoutModel } from './runLayoutModel';
|
||||
import { mergeTextWithRegions } from './merge';
|
||||
import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
|
||||
import { PDF_PARSER_VERSION } from './parser-version';
|
||||
import { runLayoutModel } from './layout-model';
|
||||
import { mergeTextWithRegions } from './document-layout';
|
||||
import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs';
|
||||
import { PDF_PARSER_VERSION } from '../../api/contracts';
|
||||
import { stitchCrossPageBlocks } from './stitch';
|
||||
import { renderPage } from './render';
|
||||
import { normalizeTextItemsForLayout } from './normalize-text';
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
import { PDF_PARSER_VERSION } from './parser-version';
|
||||
|
||||
export function encodeParserVersion(
|
||||
parserVersion: string,
|
||||
defaultVersion = PDF_PARSER_VERSION,
|
||||
): string {
|
||||
const normalized = parserVersion.trim() || defaultVersion;
|
||||
return encodeURIComponent(normalized);
|
||||
}
|
||||
|
|
@ -1 +0,0 @@
|
|||
export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69';
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import type { Canvas } from '@napi-rs/canvas';
|
||||
import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
|
||||
import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs';
|
||||
|
||||
type CanvasRuntime = {
|
||||
DOMMatrixCtor: unknown;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import type { ParsedPdfDocument, ParsedPdfBlock } from '../types/parsed-pdf';
|
||||
import type { ParsedPdfDocument, ParsedPdfBlock } from '../../api/types';
|
||||
|
||||
const STITCHABLE_KINDS = new Set<ParsedPdfBlock['kind']>([
|
||||
'text',
|
||||
|
|
|
|||
|
|
@ -1,15 +0,0 @@
|
|||
import type { ParsedPdfBlockKind } from '../types/parsed-pdf';
|
||||
|
||||
export interface PdfTextItem {
|
||||
text: string;
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface LayoutRegion {
|
||||
bbox: [number, number, number, number];
|
||||
label: ParsedPdfBlockKind;
|
||||
confidence?: number;
|
||||
}
|
||||
|
|
@ -1,21 +0,0 @@
|
|||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
|
||||
function findMonorepoRoot(startDir: string): string | null {
|
||||
let current = path.resolve(startDir);
|
||||
for (;;) {
|
||||
const marker = path.join(current, 'pnpm-workspace.yaml');
|
||||
if (fs.existsSync(marker)) return current;
|
||||
const parent = path.dirname(current);
|
||||
if (parent === current) return null;
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveDocstoreDir(): string {
|
||||
const repoRoot = findMonorepoRoot(process.cwd());
|
||||
if (repoRoot) return path.join(repoRoot, 'docstore');
|
||||
return path.join(process.cwd(), 'docstore');
|
||||
}
|
||||
|
||||
export const DOCSTORE_DIR = resolveDocstoreDir();
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
export type {
|
||||
TTSAudioBuffer,
|
||||
TTSAudioBytes,
|
||||
TTSSentenceAlignment,
|
||||
TTSSentenceWord,
|
||||
} from './tts';
|
||||
|
||||
export type {
|
||||
ParsedPdfBlock,
|
||||
ParsedPdfBlockFragment,
|
||||
ParsedPdfBlockKind,
|
||||
ParsedPdfDocument,
|
||||
ParsedPdfPage,
|
||||
PdfParsePhase,
|
||||
PdfParseProgress,
|
||||
PdfParseStatus,
|
||||
} from './parsed-pdf';
|
||||
|
|
@ -1,16 +0,0 @@
|
|||
export type TTSAudioBuffer = ArrayBuffer;
|
||||
export type TTSAudioBytes = number[];
|
||||
|
||||
export interface TTSSentenceWord {
|
||||
text: string;
|
||||
startSec: number;
|
||||
endSec: number;
|
||||
charStart: number;
|
||||
charEnd: number;
|
||||
}
|
||||
|
||||
export interface TTSSentenceAlignment {
|
||||
sentence: string;
|
||||
sentenceIndex: number;
|
||||
words: TTSSentenceWord[];
|
||||
}
|
||||
|
|
@ -7,19 +7,18 @@ import { fileURLToPath } from 'url';
|
|||
import * as ort from 'onnxruntime-node';
|
||||
import { Tokenizer } from '@huggingface/tokenizers';
|
||||
import JSZip from 'jszip';
|
||||
import type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment } from '../types/tts';
|
||||
import { getFFmpegPath } from '../platform/ffmpeg';
|
||||
import { getOnnxThreadsPerJob } from '../config/cpu-budget';
|
||||
import { getComputeTimeoutConfig } from '../config/timeout';
|
||||
import type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment } from '../../api/types';
|
||||
import { getFFmpegPath } from '../../infrastructure/platform';
|
||||
import { getOnnxThreadsPerJob } from '../../infrastructure/config';
|
||||
import { getComputeTimeoutConfig } from '../../infrastructure/config';
|
||||
import {
|
||||
mapWordsToSentenceOffsets,
|
||||
type WhisperWord,
|
||||
} from './alignment-map';
|
||||
import { buildGoertzelCoefficients, goertzelPower } from './spectral';
|
||||
} from './timestamps';
|
||||
import {
|
||||
buildWordsFromTimestampedTokens,
|
||||
extractTokenStartTimestamps,
|
||||
} from './token-timestamps';
|
||||
} from './timestamps';
|
||||
import {
|
||||
ensureWhisperModel,
|
||||
WHISPER_CONFIG_PATH,
|
||||
|
|
@ -31,6 +30,26 @@ import {
|
|||
WHISPER_DECODER_WITH_PAST_MODEL_PATH,
|
||||
} from './model';
|
||||
|
||||
export function buildGoertzelCoefficients(freqBins: number, fftSize: number): Float64Array {
|
||||
const coeffs = new Float64Array(freqBins);
|
||||
for (let k = 0; k < freqBins; k += 1) {
|
||||
coeffs[k] = 2 * Math.cos((2 * Math.PI * k) / fftSize);
|
||||
}
|
||||
return coeffs;
|
||||
}
|
||||
|
||||
export function goertzelPower(samples: Float32Array, coeff: number): number {
|
||||
let s1 = 0;
|
||||
let s2 = 0;
|
||||
for (let i = 0; i < samples.length; i += 1) {
|
||||
const s0 = samples[i] + (coeff * s1) - s2;
|
||||
s2 = s1;
|
||||
s1 = s0;
|
||||
}
|
||||
const power = (s1 * s1) + (s2 * s2) - (coeff * s1 * s2);
|
||||
return !Number.isFinite(power) || power < 0 ? 0 : power;
|
||||
}
|
||||
|
||||
interface WhisperAlignmentOptions {
|
||||
lang?: string;
|
||||
textHint?: string;
|
||||
|
|
|
|||
|
|
@ -1,44 +0,0 @@
|
|||
import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts';
|
||||
|
||||
export interface WhisperWord {
|
||||
start: number;
|
||||
end: number;
|
||||
word: string;
|
||||
}
|
||||
|
||||
export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment {
|
||||
const lowerSentence = sentence.toLowerCase();
|
||||
let cursor = 0;
|
||||
|
||||
const alignedWords: TTSSentenceWord[] = words.map((w) => {
|
||||
const token = w.word.trim();
|
||||
if (!token) {
|
||||
return {
|
||||
text: '',
|
||||
startSec: w.start,
|
||||
endSec: w.end,
|
||||
charStart: cursor,
|
||||
charEnd: cursor,
|
||||
};
|
||||
}
|
||||
|
||||
const idx = lowerSentence.indexOf(token.toLowerCase(), cursor);
|
||||
const start = idx >= 0 ? idx : cursor;
|
||||
const end = Math.min(sentence.length, start + token.length);
|
||||
cursor = Math.max(cursor, end);
|
||||
|
||||
return {
|
||||
text: token,
|
||||
startSec: w.start,
|
||||
endSec: w.end,
|
||||
charStart: start,
|
||||
charEnd: end,
|
||||
};
|
||||
}).filter((word) => word.text.length > 0);
|
||||
|
||||
return {
|
||||
sentence,
|
||||
sentenceIndex: 0,
|
||||
words: alignedWords,
|
||||
};
|
||||
}
|
||||
|
|
@ -3,7 +3,7 @@ import { fileURLToPath } from 'url';
|
|||
import { createHash } from 'crypto';
|
||||
import { readFileSync } from 'fs';
|
||||
import { access, copyFile, mkdir, readFile, rename, unlink, writeFile } from 'fs/promises';
|
||||
import { DOCSTORE_DIR } from '../platform/docstore';
|
||||
import { DOCSTORE_DIR } from '../../infrastructure/platform';
|
||||
|
||||
const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||
const MODEL_DIR = path.join(DOCSTORE_DIR, 'model', 'whisper-base_timestamped');
|
||||
|
|
|
|||
|
|
@ -1,21 +0,0 @@
|
|||
export function buildGoertzelCoefficients(freqBins: number, fftSize: number): Float64Array {
|
||||
const coeffs = new Float64Array(freqBins);
|
||||
for (let k = 0; k < freqBins; k += 1) {
|
||||
coeffs[k] = 2 * Math.cos((2 * Math.PI * k) / fftSize);
|
||||
}
|
||||
return coeffs;
|
||||
}
|
||||
|
||||
export function goertzelPower(samples: Float32Array, coeff: number): number {
|
||||
let s1 = 0;
|
||||
let s2 = 0;
|
||||
for (let i = 0; i < samples.length; i += 1) {
|
||||
const s0 = samples[i] + (coeff * s1) - s2;
|
||||
s2 = s1;
|
||||
s1 = s0;
|
||||
}
|
||||
|
||||
const power = (s1 * s1) + (s2 * s2) - (coeff * s1 * s2);
|
||||
if (!Number.isFinite(power) || power < 0) return 0;
|
||||
return power;
|
||||
}
|
||||
|
|
@ -1,5 +1,6 @@
|
|||
import type { Tokenizer } from '@huggingface/tokenizers';
|
||||
import type * as ort from 'onnxruntime-node';
|
||||
import type { TTSSentenceAlignment, TTSSentenceWord } from '../../api/types';
|
||||
|
||||
const PUNCTUATION_REGEX = '\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E';
|
||||
const PUNCTUATION_ONLY_REGEX = new RegExp(`^[${PUNCTUATION_REGEX}]+$`, 'gu');
|
||||
|
|
@ -12,6 +13,29 @@ export interface WhisperWordTiming {
|
|||
endSec: number;
|
||||
}
|
||||
|
||||
export interface WhisperWord {
|
||||
start: number;
|
||||
end: number;
|
||||
word: string;
|
||||
}
|
||||
|
||||
export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment {
|
||||
const lowerSentence = sentence.toLowerCase();
|
||||
let cursor = 0;
|
||||
const alignedWords: TTSSentenceWord[] = words.map((word) => {
|
||||
const token = word.word.trim();
|
||||
if (!token) {
|
||||
return { text: '', startSec: word.start, endSec: word.end, charStart: cursor, charEnd: cursor };
|
||||
}
|
||||
const index = lowerSentence.indexOf(token.toLowerCase(), cursor);
|
||||
const start = index >= 0 ? index : cursor;
|
||||
const end = Math.min(sentence.length, start + token.length);
|
||||
cursor = Math.max(cursor, end);
|
||||
return { text: token, startSec: word.start, endSec: word.end, charStart: start, charEnd: end };
|
||||
}).filter((word) => word.text.length > 0);
|
||||
return { sentence, sentenceIndex: 0, words: alignedWords };
|
||||
}
|
||||
|
||||
function medianFilter(data: Float32Array, windowSize: number): Float32Array {
|
||||
if (windowSize % 2 === 0 || windowSize <= 0) {
|
||||
throw new Error('Window size must be a positive odd number');
|
||||
|
|
@ -117,3 +117,23 @@ export async function withIdleTimeoutAndHardCap<T>(input: IdleTimeoutAndHardCapI
|
|||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function getComputeJobConcurrency(): number {
|
||||
return readPositiveIntEnv('COMPUTE_JOB_CONCURRENCY', 1);
|
||||
}
|
||||
|
||||
export function getAvailableCpuCores(): number {
|
||||
if (typeof os.availableParallelism === 'function') {
|
||||
const value = os.availableParallelism();
|
||||
if (Number.isFinite(value) && value >= 1) return Math.floor(value);
|
||||
}
|
||||
const fallback = os.cpus().length;
|
||||
return Number.isFinite(fallback) && fallback >= 1 ? Math.floor(fallback) : 1;
|
||||
}
|
||||
|
||||
export function getOnnxThreadsPerJob(): number {
|
||||
const concurrency = getComputeJobConcurrency();
|
||||
const usableCores = Math.max(1, getAvailableCpuCores() - 1);
|
||||
return Math.max(1, Math.floor(usableCores / concurrency));
|
||||
}
|
||||
import os from 'node:os';
|
||||
|
|
@ -1,6 +1,26 @@
|
|||
import { existsSync } from 'fs';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import ffmpegStatic from 'ffmpeg-static';
|
||||
|
||||
function findMonorepoRoot(startDir: string): string | null {
|
||||
let current = path.resolve(startDir);
|
||||
for (;;) {
|
||||
const marker = path.join(current, 'pnpm-workspace.yaml');
|
||||
if (fs.existsSync(marker)) return current;
|
||||
const parent = path.dirname(current);
|
||||
if (parent === current) return null;
|
||||
current = parent;
|
||||
}
|
||||
}
|
||||
|
||||
function resolveDocstoreDir(): string {
|
||||
const repoRoot = findMonorepoRoot(process.cwd());
|
||||
if (repoRoot) return path.join(repoRoot, 'docstore');
|
||||
return path.join(process.cwd(), 'docstore');
|
||||
}
|
||||
|
||||
export const DOCSTORE_DIR = resolveDocstoreDir();
|
||||
|
||||
function normalizePath(value: unknown): string | null {
|
||||
if (typeof value !== 'string') return null;
|
||||
const trimmed = value.trim();
|
||||
|
|
@ -9,20 +29,17 @@ function normalizePath(value: unknown): string | null {
|
|||
|
||||
function resolveBinary(envValue: string | null, bundledValue: string | null, envVarName: string, packageName: string): string {
|
||||
if (envValue) {
|
||||
if ((envValue.includes('/') || envValue.includes('\\')) && !existsSync(envValue)) {
|
||||
if ((envValue.includes('/') || envValue.includes('\\')) && !fs.existsSync(envValue)) {
|
||||
throw new Error(`${envVarName} points to a missing binary: ${envValue}`);
|
||||
}
|
||||
return envValue;
|
||||
}
|
||||
|
||||
if (!bundledValue) {
|
||||
throw new Error(`${packageName} binary is unavailable on this platform. Set ${envVarName} to an installed binary path.`);
|
||||
}
|
||||
|
||||
if ((bundledValue.includes('/') || bundledValue.includes('\\')) && !existsSync(bundledValue)) {
|
||||
if ((bundledValue.includes('/') || bundledValue.includes('\\')) && !fs.existsSync(bundledValue)) {
|
||||
throw new Error(`${packageName} resolved to a missing binary path: ${bundledValue}`);
|
||||
}
|
||||
|
||||
return bundledValue;
|
||||
}
|
||||
|
||||
|
|
@ -2,8 +2,8 @@ import { z } from 'zod';
|
|||
import {
|
||||
runPdfLayoutFromPdfBuffer,
|
||||
runWhisperAlignmentFromAudioBuffer,
|
||||
} from '../inference/local-runtime';
|
||||
import { withIdleTimeoutAndHardCap, withTimeout } from '../inference';
|
||||
} from '../inference/runtime';
|
||||
import { withIdleTimeoutAndHardCap, withTimeout } from '../infrastructure/config';
|
||||
import type {
|
||||
PdfLayoutJobRequest,
|
||||
PdfLayoutJobResult,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { PDF_PARSER_VERSION } from '../inference/pdf/parser-version';
|
||||
import { encodeParserVersion } from '../inference/pdf/parser-version-key';
|
||||
import { PDF_PARSER_VERSION } from '../api/contracts';
|
||||
import { encodeParserVersion } from '../api/contracts';
|
||||
|
||||
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { mergeTextWithRegions } from '../../../src/inference';
|
||||
import { mergeTextWithRegions } from '../../../src/inference/pdf/document-layout';
|
||||
|
||||
describe('mergeTextWithRegions', () => {
|
||||
test('assigns text items to containing regions by centroid and joins in reading order', () => {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
|
||||
import { normalizeTextItemsForLayout } from '../../../src/inference';
|
||||
import { normalizeTextItemsForLayout } from '../../../src/inference/pdf/normalize-text';
|
||||
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
||||
|
||||
function makeTextItem(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { stitchCrossPageBlocks } from '../../../src/inference';
|
||||
import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/inference/types';
|
||||
import { stitchCrossPageBlocks } from '../../../src/inference/pdf/stitch';
|
||||
import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/api/types';
|
||||
import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures';
|
||||
|
||||
function makeBlock(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import type { BaseDocument } from '../../../../../src/types/documents';
|
||||
import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/inference/types';
|
||||
import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/api/types';
|
||||
|
||||
export function makeBaseDocument(overrides: Partial<BaseDocument> = {}): BaseDocument {
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import {
|
||||
mapWordsToSentenceOffsets,
|
||||
} from '../../../src/inference';
|
||||
} from '../../../src/inference/whisper/timestamps';
|
||||
|
||||
describe('whisper alignment mapping', () => {
|
||||
test('maps words to sentence offsets with punctuation and repeated spaces', () => {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
import { describe, expect, test } from 'vitest';
|
||||
import { buildGoertzelCoefficients, goertzelPower } from '../../../src/inference';
|
||||
import { buildGoertzelCoefficients, goertzelPower } from '../../../src/inference/whisper/align';
|
||||
|
||||
function dftPower(samples: Float32Array, k: number): number {
|
||||
const n = samples.length;
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-node';
|
|||
import {
|
||||
buildWordsFromTimestampedTokens,
|
||||
extractTokenStartTimestamps,
|
||||
} from '../../../src/inference';
|
||||
} from '../../../src/inference/whisper/timestamps';
|
||||
|
||||
describe('whisper token timestamp alignment', () => {
|
||||
test('extracts monotonic token timestamps from cross-attention maps', () => {
|
||||
|
|
|
|||
|
|
@ -76,7 +76,7 @@ vi.mock('../../../src/inference/pdf/model', () => ({
|
|||
MODEL_PREPROCESSOR_PATH: '/tmp/model-preprocessor.json',
|
||||
}));
|
||||
|
||||
vi.mock('../../../src/inference/config/cpu-budget', () => ({
|
||||
vi.mock('../../../src/infrastructure/config', () => ({
|
||||
getOnnxThreadsPerJob: vi.fn(() => 1),
|
||||
}));
|
||||
|
||||
|
|
@ -107,7 +107,7 @@ describe('runLayoutModel', () => {
|
|||
},
|
||||
};
|
||||
|
||||
const { runLayoutModel } = await import('../../../src/inference/pdf/runLayoutModel');
|
||||
const { runLayoutModel } = await import('../../../src/inference/pdf/layout-model');
|
||||
const regions = await runLayoutModel({
|
||||
pageWidth: 100,
|
||||
pageHeight: 100,
|
||||
|
|
@ -154,7 +154,7 @@ describe('runLayoutModel', () => {
|
|||
},
|
||||
};
|
||||
|
||||
const { runLayoutModel } = await import('../../../src/inference/pdf/runLayoutModel');
|
||||
const { runLayoutModel } = await import('../../../src/inference/pdf/layout-model');
|
||||
const regions = await runLayoutModel({
|
||||
pageWidth: 100,
|
||||
pageHeight: 100,
|
||||
|
|
|
|||
Loading…
Reference in a new issue