chore(worker): restructure inference modules and centralize config
- Remove legacy inference and platform files from `src/inference/` and migrate responsibilities to new modular locations under `src/infrastructure/`, `src/inference/pdf/`, and `src/inference/whisper/` - Consolidate environment/config logic into `src/infrastructure/config.ts` - Move docstore and ffmpeg platform utilities to `src/infrastructure/platform.ts` - Refactor PDF and Whisper inference code to use new document layout, layout model, and timestamp utilities - Update API contracts to define and export types and constants previously scattered in inference/types - Adjust all imports in jobs, storage, and tests to reference new module structure and type locations - Inline parser version and encoding logic into API contracts - Remove obsolete files and update test fixtures for new type locations This update improves codebase modularity, maintainability, and separation of concerns. No changes to inference or orchestration functionality. BREAKING CHANGE: inference module structure, type imports, and config utilities have changed; downstream code must update imports and integration points.
This commit is contained in:
parent
3d5dfd2a88
commit
6e69b48103
38 changed files with 163 additions and 248 deletions
|
|
@ -15,13 +15,13 @@ import {
|
||||||
import { Kvm } from '@nats-io/kv';
|
import { Kvm } from '@nats-io/kv';
|
||||||
import {
|
import {
|
||||||
ensureComputeModels,
|
ensureComputeModels,
|
||||||
} from '../inference/local-runtime';
|
} from '../inference/runtime';
|
||||||
import {
|
import {
|
||||||
getComputeTimeoutConfig,
|
getComputeTimeoutConfig,
|
||||||
getComputeOpStaleMs,
|
getComputeOpStaleMs,
|
||||||
getAvailableCpuCores,
|
getAvailableCpuCores,
|
||||||
getOnnxThreadsPerJob,
|
getOnnxThreadsPerJob,
|
||||||
} from '../inference';
|
} from '../infrastructure/config';
|
||||||
import { encodeSseFrame, OperationOrchestrator } from '../operations';
|
import { encodeSseFrame, OperationOrchestrator } from '../operations';
|
||||||
import type {
|
import type {
|
||||||
PdfLayoutJobRequest,
|
PdfLayoutJobRequest,
|
||||||
|
|
|
||||||
|
|
@ -1,24 +1,26 @@
|
||||||
import type { TTSSentenceAlignment } from '../inference/types/tts';
|
import type { TTSSentenceAlignment, ParsedPdfDocument } from './types';
|
||||||
import type { ParsedPdfDocument } from '../inference/types/parsed-pdf';
|
|
||||||
|
|
||||||
export type {
|
export type {
|
||||||
TTSAudioBuffer,
|
TTSAudioBuffer,
|
||||||
TTSAudioBytes,
|
TTSAudioBytes,
|
||||||
TTSSentenceAlignment,
|
TTSSentenceAlignment,
|
||||||
TTSSentenceWord,
|
TTSSentenceWord,
|
||||||
} from '../inference/types/tts';
|
} from './types';
|
||||||
export type {
|
export type {
|
||||||
ParsedPdfBlockKind,
|
ParsedPdfBlockKind,
|
||||||
ParsedPdfBlockFragment,
|
ParsedPdfBlockFragment,
|
||||||
ParsedPdfBlock,
|
ParsedPdfBlock,
|
||||||
ParsedPdfPage,
|
ParsedPdfPage,
|
||||||
ParsedPdfDocument,
|
ParsedPdfDocument,
|
||||||
} from '../inference/types/parsed-pdf';
|
} from './types';
|
||||||
|
|
||||||
export const ALIGN_QUEUE_NAME = 'whisper-align';
|
export const ALIGN_QUEUE_NAME = 'whisper-align';
|
||||||
export const PDF_LAYOUT_QUEUE_NAME = 'pdf-layout';
|
export const PDF_LAYOUT_QUEUE_NAME = 'pdf-layout';
|
||||||
export { PDF_PARSER_VERSION } from '../inference/pdf/parser-version';
|
export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69';
|
||||||
export { encodeParserVersion } from '../inference/pdf/parser-version-key';
|
|
||||||
|
export function encodeParserVersion(parserVersion: string, defaultVersion = PDF_PARSER_VERSION): string {
|
||||||
|
return encodeURIComponent(parserVersion.trim() || defaultVersion);
|
||||||
|
}
|
||||||
|
|
||||||
export interface WhisperAlignJobBase {
|
export interface WhisperAlignJobBase {
|
||||||
text: string;
|
text: string;
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { createHash } from 'node:crypto';
|
import { createHash } from 'node:crypto';
|
||||||
import { PDF_PARSER_VERSION } from '../inference/pdf/parser-version';
|
import { PDF_PARSER_VERSION } from '../api/contracts';
|
||||||
|
|
||||||
function sha256Hex(input: string): string {
|
function sha256Hex(input: string): string {
|
||||||
return createHash('sha256').update(input).digest('hex');
|
return createHash('sha256').update(input).digest('hex');
|
||||||
|
|
|
||||||
|
|
@ -61,3 +61,20 @@ export interface PdfParseProgress {
|
||||||
currentPage?: number;
|
currentPage?: number;
|
||||||
phase: PdfParsePhase;
|
phase: PdfParsePhase;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type TTSAudioBuffer = ArrayBuffer;
|
||||||
|
export type TTSAudioBytes = number[];
|
||||||
|
|
||||||
|
export interface TTSSentenceWord {
|
||||||
|
text: string;
|
||||||
|
startSec: number;
|
||||||
|
endSec: number;
|
||||||
|
charStart: number;
|
||||||
|
charEnd: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface TTSSentenceAlignment {
|
||||||
|
sentence: string;
|
||||||
|
sentenceIndex: number;
|
||||||
|
words: TTSSentenceWord[];
|
||||||
|
}
|
||||||
|
|
@ -1,28 +0,0 @@
|
||||||
import os from 'node:os';
|
|
||||||
|
|
||||||
function readPositiveInt(name: string, fallback: number): number {
|
|
||||||
const raw = process.env[name]?.trim();
|
|
||||||
if (!raw) return fallback;
|
|
||||||
const parsed = Number(raw);
|
|
||||||
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
|
|
||||||
return Math.floor(parsed);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getComputeJobConcurrency(): number {
|
|
||||||
return readPositiveInt('COMPUTE_JOB_CONCURRENCY', 1);
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getAvailableCpuCores(): number {
|
|
||||||
if (typeof os.availableParallelism === 'function') {
|
|
||||||
const value = os.availableParallelism();
|
|
||||||
if (Number.isFinite(value) && value >= 1) return Math.floor(value);
|
|
||||||
}
|
|
||||||
const fallback = os.cpus().length;
|
|
||||||
return Number.isFinite(fallback) && fallback >= 1 ? Math.floor(fallback) : 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function getOnnxThreadsPerJob(): number {
|
|
||||||
const concurrency = getComputeJobConcurrency();
|
|
||||||
const usableCores = Math.max(1, getAvailableCpuCores() - 1);
|
|
||||||
return Math.max(1, Math.floor(usableCores / concurrency));
|
|
||||||
}
|
|
||||||
|
|
@ -1,26 +0,0 @@
|
||||||
export * from '../api/contracts';
|
|
||||||
export {
|
|
||||||
getComputeJobConcurrency,
|
|
||||||
getAvailableCpuCores,
|
|
||||||
getOnnxThreadsPerJob,
|
|
||||||
} from './config/cpu-budget';
|
|
||||||
export {
|
|
||||||
getComputeTimeoutConfig,
|
|
||||||
getComputeOpStaleMs,
|
|
||||||
getWorkerClientWaitTimeoutMs,
|
|
||||||
withTimeout,
|
|
||||||
withIdleTimeoutAndHardCap,
|
|
||||||
type ComputeTimeoutConfig,
|
|
||||||
type ComputeOperationKind,
|
|
||||||
type IdleTimeoutAndHardCapInput,
|
|
||||||
} from './config/timeout';
|
|
||||||
export { renderPage } from './pdf/render';
|
|
||||||
export { mergeTextWithRegions } from './pdf/merge';
|
|
||||||
export { PDF_PARSER_VERSION } from './pdf/parser-version';
|
|
||||||
export { encodeParserVersion } from './pdf/parser-version-key';
|
|
||||||
export { stitchCrossPageBlocks } from './pdf/stitch';
|
|
||||||
export { normalizeTextItemsForLayout } from './pdf/normalize-text';
|
|
||||||
export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map';
|
|
||||||
export { buildGoertzelCoefficients, goertzelPower } from './whisper/spectral';
|
|
||||||
export { buildWordsFromTimestampedTokens, extractTokenStartTimestamps } from './whisper/token-timestamps';
|
|
||||||
export * from '../operations';
|
|
||||||
|
|
@ -1,4 +1,18 @@
|
||||||
import type { LayoutRegion, PdfTextItem } from './types';
|
import type { ParsedPdfBlockKind } from '../../api/types';
|
||||||
|
|
||||||
|
export interface PdfTextItem {
|
||||||
|
text: string;
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface LayoutRegion {
|
||||||
|
bbox: [number, number, number, number];
|
||||||
|
label: ParsedPdfBlockKind;
|
||||||
|
confidence?: number;
|
||||||
|
}
|
||||||
|
|
||||||
const NON_TEXT_REGION_LABELS = new Set<LayoutRegion['label']>(['chart', 'image', 'table', 'seal']);
|
const NON_TEXT_REGION_LABELS = new Set<LayoutRegion['label']>(['chart', 'image', 'table', 'seal']);
|
||||||
const TEXT_ASSIGNABLE_LABELS = new Set<LayoutRegion['label']>([
|
const TEXT_ASSIGNABLE_LABELS = new Set<LayoutRegion['label']>([
|
||||||
|
|
@ -1,8 +1,8 @@
|
||||||
import * as ort from 'onnxruntime-node';
|
import * as ort from 'onnxruntime-node';
|
||||||
import { readFile } from 'fs/promises';
|
import { readFile } from 'fs/promises';
|
||||||
import type { LayoutRegion, PdfTextItem } from './types';
|
import type { LayoutRegion, PdfTextItem } from './document-layout';
|
||||||
import { ensureModel, MODEL_CONFIG_PATH, MODEL_PREPROCESSOR_PATH } from './model';
|
import { ensureModel, MODEL_CONFIG_PATH, MODEL_PREPROCESSOR_PATH } from './model';
|
||||||
import { getOnnxThreadsPerJob } from '../config/cpu-budget';
|
import { getOnnxThreadsPerJob } from '../../infrastructure/config';
|
||||||
|
|
||||||
interface RunLayoutInput {
|
interface RunLayoutInput {
|
||||||
pageWidth: number;
|
pageWidth: number;
|
||||||
|
|
@ -3,7 +3,7 @@ import { fileURLToPath } from 'url';
|
||||||
import { createHash } from 'crypto';
|
import { createHash } from 'crypto';
|
||||||
import { readFileSync } from 'fs';
|
import { readFileSync } from 'fs';
|
||||||
import { access, mkdir, rename, writeFile, readFile, unlink, copyFile } from 'fs/promises';
|
import { access, mkdir, rename, writeFile, readFile, unlink, copyFile } from 'fs/promises';
|
||||||
import { DOCSTORE_DIR } from '../platform/docstore';
|
import { DOCSTORE_DIR } from '../../infrastructure/platform';
|
||||||
|
|
||||||
const DEFAULT_MODEL_BASE_URL = 'https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main';
|
const DEFAULT_MODEL_BASE_URL = 'https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main';
|
||||||
const PDF_LAYOUT_MODEL_BASE_URL_ENV = 'PDF_LAYOUT_MODEL_BASE_URL';
|
const PDF_LAYOUT_MODEL_BASE_URL_ENV = 'PDF_LAYOUT_MODEL_BASE_URL';
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
||||||
import type { PdfTextItem } from './types';
|
import type { PdfTextItem } from './document-layout';
|
||||||
|
|
||||||
interface ViewportLike {
|
interface ViewportLike {
|
||||||
height: number;
|
height: number;
|
||||||
|
|
|
||||||
|
|
@ -1,10 +1,10 @@
|
||||||
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
||||||
import type { ParsedPdfDocument, ParsedPdfPage } from '../types/parsed-pdf';
|
import type { ParsedPdfDocument, ParsedPdfPage } from '../../api/types';
|
||||||
import { ensureModel } from './model';
|
import { ensureModel } from './model';
|
||||||
import { runLayoutModel } from './runLayoutModel';
|
import { runLayoutModel } from './layout-model';
|
||||||
import { mergeTextWithRegions } from './merge';
|
import { mergeTextWithRegions } from './document-layout';
|
||||||
import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
|
import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs';
|
||||||
import { PDF_PARSER_VERSION } from './parser-version';
|
import { PDF_PARSER_VERSION } from '../../api/contracts';
|
||||||
import { stitchCrossPageBlocks } from './stitch';
|
import { stitchCrossPageBlocks } from './stitch';
|
||||||
import { renderPage } from './render';
|
import { renderPage } from './render';
|
||||||
import { normalizeTextItemsForLayout } from './normalize-text';
|
import { normalizeTextItemsForLayout } from './normalize-text';
|
||||||
|
|
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
import { PDF_PARSER_VERSION } from './parser-version';
|
|
||||||
|
|
||||||
export function encodeParserVersion(
|
|
||||||
parserVersion: string,
|
|
||||||
defaultVersion = PDF_PARSER_VERSION,
|
|
||||||
): string {
|
|
||||||
const normalized = parserVersion.trim() || defaultVersion;
|
|
||||||
return encodeURIComponent(normalized);
|
|
||||||
}
|
|
||||||
|
|
@ -1 +0,0 @@
|
||||||
export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69';
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import type { Canvas } from '@napi-rs/canvas';
|
import type { Canvas } from '@napi-rs/canvas';
|
||||||
import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
|
import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs';
|
||||||
|
|
||||||
type CanvasRuntime = {
|
type CanvasRuntime = {
|
||||||
DOMMatrixCtor: unknown;
|
DOMMatrixCtor: unknown;
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,4 @@
|
||||||
import type { ParsedPdfDocument, ParsedPdfBlock } from '../types/parsed-pdf';
|
import type { ParsedPdfDocument, ParsedPdfBlock } from '../../api/types';
|
||||||
|
|
||||||
const STITCHABLE_KINDS = new Set<ParsedPdfBlock['kind']>([
|
const STITCHABLE_KINDS = new Set<ParsedPdfBlock['kind']>([
|
||||||
'text',
|
'text',
|
||||||
|
|
|
||||||
|
|
@ -1,15 +0,0 @@
|
||||||
import type { ParsedPdfBlockKind } from '../types/parsed-pdf';
|
|
||||||
|
|
||||||
export interface PdfTextItem {
|
|
||||||
text: string;
|
|
||||||
x: number;
|
|
||||||
y: number;
|
|
||||||
width: number;
|
|
||||||
height: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface LayoutRegion {
|
|
||||||
bbox: [number, number, number, number];
|
|
||||||
label: ParsedPdfBlockKind;
|
|
||||||
confidence?: number;
|
|
||||||
}
|
|
||||||
|
|
@ -1,21 +0,0 @@
|
||||||
import fs from 'fs';
|
|
||||||
import path from 'path';
|
|
||||||
|
|
||||||
function findMonorepoRoot(startDir: string): string | null {
|
|
||||||
let current = path.resolve(startDir);
|
|
||||||
for (;;) {
|
|
||||||
const marker = path.join(current, 'pnpm-workspace.yaml');
|
|
||||||
if (fs.existsSync(marker)) return current;
|
|
||||||
const parent = path.dirname(current);
|
|
||||||
if (parent === current) return null;
|
|
||||||
current = parent;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function resolveDocstoreDir(): string {
|
|
||||||
const repoRoot = findMonorepoRoot(process.cwd());
|
|
||||||
if (repoRoot) return path.join(repoRoot, 'docstore');
|
|
||||||
return path.join(process.cwd(), 'docstore');
|
|
||||||
}
|
|
||||||
|
|
||||||
export const DOCSTORE_DIR = resolveDocstoreDir();
|
|
||||||
|
|
@ -1,17 +0,0 @@
|
||||||
export type {
|
|
||||||
TTSAudioBuffer,
|
|
||||||
TTSAudioBytes,
|
|
||||||
TTSSentenceAlignment,
|
|
||||||
TTSSentenceWord,
|
|
||||||
} from './tts';
|
|
||||||
|
|
||||||
export type {
|
|
||||||
ParsedPdfBlock,
|
|
||||||
ParsedPdfBlockFragment,
|
|
||||||
ParsedPdfBlockKind,
|
|
||||||
ParsedPdfDocument,
|
|
||||||
ParsedPdfPage,
|
|
||||||
PdfParsePhase,
|
|
||||||
PdfParseProgress,
|
|
||||||
PdfParseStatus,
|
|
||||||
} from './parsed-pdf';
|
|
||||||
|
|
@ -1,16 +0,0 @@
|
||||||
export type TTSAudioBuffer = ArrayBuffer;
|
|
||||||
export type TTSAudioBytes = number[];
|
|
||||||
|
|
||||||
export interface TTSSentenceWord {
|
|
||||||
text: string;
|
|
||||||
startSec: number;
|
|
||||||
endSec: number;
|
|
||||||
charStart: number;
|
|
||||||
charEnd: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface TTSSentenceAlignment {
|
|
||||||
sentence: string;
|
|
||||||
sentenceIndex: number;
|
|
||||||
words: TTSSentenceWord[];
|
|
||||||
}
|
|
||||||
|
|
@ -7,19 +7,18 @@ import { fileURLToPath } from 'url';
|
||||||
import * as ort from 'onnxruntime-node';
|
import * as ort from 'onnxruntime-node';
|
||||||
import { Tokenizer } from '@huggingface/tokenizers';
|
import { Tokenizer } from '@huggingface/tokenizers';
|
||||||
import JSZip from 'jszip';
|
import JSZip from 'jszip';
|
||||||
import type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment } from '../types/tts';
|
import type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment } from '../../api/types';
|
||||||
import { getFFmpegPath } from '../platform/ffmpeg';
|
import { getFFmpegPath } from '../../infrastructure/platform';
|
||||||
import { getOnnxThreadsPerJob } from '../config/cpu-budget';
|
import { getOnnxThreadsPerJob } from '../../infrastructure/config';
|
||||||
import { getComputeTimeoutConfig } from '../config/timeout';
|
import { getComputeTimeoutConfig } from '../../infrastructure/config';
|
||||||
import {
|
import {
|
||||||
mapWordsToSentenceOffsets,
|
mapWordsToSentenceOffsets,
|
||||||
type WhisperWord,
|
type WhisperWord,
|
||||||
} from './alignment-map';
|
} from './timestamps';
|
||||||
import { buildGoertzelCoefficients, goertzelPower } from './spectral';
|
|
||||||
import {
|
import {
|
||||||
buildWordsFromTimestampedTokens,
|
buildWordsFromTimestampedTokens,
|
||||||
extractTokenStartTimestamps,
|
extractTokenStartTimestamps,
|
||||||
} from './token-timestamps';
|
} from './timestamps';
|
||||||
import {
|
import {
|
||||||
ensureWhisperModel,
|
ensureWhisperModel,
|
||||||
WHISPER_CONFIG_PATH,
|
WHISPER_CONFIG_PATH,
|
||||||
|
|
@ -31,6 +30,26 @@ import {
|
||||||
WHISPER_DECODER_WITH_PAST_MODEL_PATH,
|
WHISPER_DECODER_WITH_PAST_MODEL_PATH,
|
||||||
} from './model';
|
} from './model';
|
||||||
|
|
||||||
|
export function buildGoertzelCoefficients(freqBins: number, fftSize: number): Float64Array {
|
||||||
|
const coeffs = new Float64Array(freqBins);
|
||||||
|
for (let k = 0; k < freqBins; k += 1) {
|
||||||
|
coeffs[k] = 2 * Math.cos((2 * Math.PI * k) / fftSize);
|
||||||
|
}
|
||||||
|
return coeffs;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function goertzelPower(samples: Float32Array, coeff: number): number {
|
||||||
|
let s1 = 0;
|
||||||
|
let s2 = 0;
|
||||||
|
for (let i = 0; i < samples.length; i += 1) {
|
||||||
|
const s0 = samples[i] + (coeff * s1) - s2;
|
||||||
|
s2 = s1;
|
||||||
|
s1 = s0;
|
||||||
|
}
|
||||||
|
const power = (s1 * s1) + (s2 * s2) - (coeff * s1 * s2);
|
||||||
|
return !Number.isFinite(power) || power < 0 ? 0 : power;
|
||||||
|
}
|
||||||
|
|
||||||
interface WhisperAlignmentOptions {
|
interface WhisperAlignmentOptions {
|
||||||
lang?: string;
|
lang?: string;
|
||||||
textHint?: string;
|
textHint?: string;
|
||||||
|
|
|
||||||
|
|
@ -1,44 +0,0 @@
|
||||||
import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts';
|
|
||||||
|
|
||||||
export interface WhisperWord {
|
|
||||||
start: number;
|
|
||||||
end: number;
|
|
||||||
word: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment {
|
|
||||||
const lowerSentence = sentence.toLowerCase();
|
|
||||||
let cursor = 0;
|
|
||||||
|
|
||||||
const alignedWords: TTSSentenceWord[] = words.map((w) => {
|
|
||||||
const token = w.word.trim();
|
|
||||||
if (!token) {
|
|
||||||
return {
|
|
||||||
text: '',
|
|
||||||
startSec: w.start,
|
|
||||||
endSec: w.end,
|
|
||||||
charStart: cursor,
|
|
||||||
charEnd: cursor,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
const idx = lowerSentence.indexOf(token.toLowerCase(), cursor);
|
|
||||||
const start = idx >= 0 ? idx : cursor;
|
|
||||||
const end = Math.min(sentence.length, start + token.length);
|
|
||||||
cursor = Math.max(cursor, end);
|
|
||||||
|
|
||||||
return {
|
|
||||||
text: token,
|
|
||||||
startSec: w.start,
|
|
||||||
endSec: w.end,
|
|
||||||
charStart: start,
|
|
||||||
charEnd: end,
|
|
||||||
};
|
|
||||||
}).filter((word) => word.text.length > 0);
|
|
||||||
|
|
||||||
return {
|
|
||||||
sentence,
|
|
||||||
sentenceIndex: 0,
|
|
||||||
words: alignedWords,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
@ -3,7 +3,7 @@ import { fileURLToPath } from 'url';
|
||||||
import { createHash } from 'crypto';
|
import { createHash } from 'crypto';
|
||||||
import { readFileSync } from 'fs';
|
import { readFileSync } from 'fs';
|
||||||
import { access, copyFile, mkdir, readFile, rename, unlink, writeFile } from 'fs/promises';
|
import { access, copyFile, mkdir, readFile, rename, unlink, writeFile } from 'fs/promises';
|
||||||
import { DOCSTORE_DIR } from '../platform/docstore';
|
import { DOCSTORE_DIR } from '../../infrastructure/platform';
|
||||||
|
|
||||||
const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
|
const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url));
|
||||||
const MODEL_DIR = path.join(DOCSTORE_DIR, 'model', 'whisper-base_timestamped');
|
const MODEL_DIR = path.join(DOCSTORE_DIR, 'model', 'whisper-base_timestamped');
|
||||||
|
|
|
||||||
|
|
@ -1,21 +0,0 @@
|
||||||
export function buildGoertzelCoefficients(freqBins: number, fftSize: number): Float64Array {
|
|
||||||
const coeffs = new Float64Array(freqBins);
|
|
||||||
for (let k = 0; k < freqBins; k += 1) {
|
|
||||||
coeffs[k] = 2 * Math.cos((2 * Math.PI * k) / fftSize);
|
|
||||||
}
|
|
||||||
return coeffs;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function goertzelPower(samples: Float32Array, coeff: number): number {
|
|
||||||
let s1 = 0;
|
|
||||||
let s2 = 0;
|
|
||||||
for (let i = 0; i < samples.length; i += 1) {
|
|
||||||
const s0 = samples[i] + (coeff * s1) - s2;
|
|
||||||
s2 = s1;
|
|
||||||
s1 = s0;
|
|
||||||
}
|
|
||||||
|
|
||||||
const power = (s1 * s1) + (s2 * s2) - (coeff * s1 * s2);
|
|
||||||
if (!Number.isFinite(power) || power < 0) return 0;
|
|
||||||
return power;
|
|
||||||
}
|
|
||||||
|
|
@ -1,5 +1,6 @@
|
||||||
import type { Tokenizer } from '@huggingface/tokenizers';
|
import type { Tokenizer } from '@huggingface/tokenizers';
|
||||||
import type * as ort from 'onnxruntime-node';
|
import type * as ort from 'onnxruntime-node';
|
||||||
|
import type { TTSSentenceAlignment, TTSSentenceWord } from '../../api/types';
|
||||||
|
|
||||||
const PUNCTUATION_REGEX = '\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E';
|
const PUNCTUATION_REGEX = '\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E';
|
||||||
const PUNCTUATION_ONLY_REGEX = new RegExp(`^[${PUNCTUATION_REGEX}]+$`, 'gu');
|
const PUNCTUATION_ONLY_REGEX = new RegExp(`^[${PUNCTUATION_REGEX}]+$`, 'gu');
|
||||||
|
|
@ -12,6 +13,29 @@ export interface WhisperWordTiming {
|
||||||
endSec: number;
|
endSec: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface WhisperWord {
|
||||||
|
start: number;
|
||||||
|
end: number;
|
||||||
|
word: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment {
|
||||||
|
const lowerSentence = sentence.toLowerCase();
|
||||||
|
let cursor = 0;
|
||||||
|
const alignedWords: TTSSentenceWord[] = words.map((word) => {
|
||||||
|
const token = word.word.trim();
|
||||||
|
if (!token) {
|
||||||
|
return { text: '', startSec: word.start, endSec: word.end, charStart: cursor, charEnd: cursor };
|
||||||
|
}
|
||||||
|
const index = lowerSentence.indexOf(token.toLowerCase(), cursor);
|
||||||
|
const start = index >= 0 ? index : cursor;
|
||||||
|
const end = Math.min(sentence.length, start + token.length);
|
||||||
|
cursor = Math.max(cursor, end);
|
||||||
|
return { text: token, startSec: word.start, endSec: word.end, charStart: start, charEnd: end };
|
||||||
|
}).filter((word) => word.text.length > 0);
|
||||||
|
return { sentence, sentenceIndex: 0, words: alignedWords };
|
||||||
|
}
|
||||||
|
|
||||||
function medianFilter(data: Float32Array, windowSize: number): Float32Array {
|
function medianFilter(data: Float32Array, windowSize: number): Float32Array {
|
||||||
if (windowSize % 2 === 0 || windowSize <= 0) {
|
if (windowSize % 2 === 0 || windowSize <= 0) {
|
||||||
throw new Error('Window size must be a positive odd number');
|
throw new Error('Window size must be a positive odd number');
|
||||||
|
|
@ -117,3 +117,23 @@ export async function withIdleTimeoutAndHardCap<T>(input: IdleTimeoutAndHardCapI
|
||||||
throw error;
|
throw error;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getComputeJobConcurrency(): number {
|
||||||
|
return readPositiveIntEnv('COMPUTE_JOB_CONCURRENCY', 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getAvailableCpuCores(): number {
|
||||||
|
if (typeof os.availableParallelism === 'function') {
|
||||||
|
const value = os.availableParallelism();
|
||||||
|
if (Number.isFinite(value) && value >= 1) return Math.floor(value);
|
||||||
|
}
|
||||||
|
const fallback = os.cpus().length;
|
||||||
|
return Number.isFinite(fallback) && fallback >= 1 ? Math.floor(fallback) : 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getOnnxThreadsPerJob(): number {
|
||||||
|
const concurrency = getComputeJobConcurrency();
|
||||||
|
const usableCores = Math.max(1, getAvailableCpuCores() - 1);
|
||||||
|
return Math.max(1, Math.floor(usableCores / concurrency));
|
||||||
|
}
|
||||||
|
import os from 'node:os';
|
||||||
|
|
@ -1,6 +1,26 @@
|
||||||
import { existsSync } from 'fs';
|
import fs from 'fs';
|
||||||
|
import path from 'path';
|
||||||
import ffmpegStatic from 'ffmpeg-static';
|
import ffmpegStatic from 'ffmpeg-static';
|
||||||
|
|
||||||
|
function findMonorepoRoot(startDir: string): string | null {
|
||||||
|
let current = path.resolve(startDir);
|
||||||
|
for (;;) {
|
||||||
|
const marker = path.join(current, 'pnpm-workspace.yaml');
|
||||||
|
if (fs.existsSync(marker)) return current;
|
||||||
|
const parent = path.dirname(current);
|
||||||
|
if (parent === current) return null;
|
||||||
|
current = parent;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveDocstoreDir(): string {
|
||||||
|
const repoRoot = findMonorepoRoot(process.cwd());
|
||||||
|
if (repoRoot) return path.join(repoRoot, 'docstore');
|
||||||
|
return path.join(process.cwd(), 'docstore');
|
||||||
|
}
|
||||||
|
|
||||||
|
export const DOCSTORE_DIR = resolveDocstoreDir();
|
||||||
|
|
||||||
function normalizePath(value: unknown): string | null {
|
function normalizePath(value: unknown): string | null {
|
||||||
if (typeof value !== 'string') return null;
|
if (typeof value !== 'string') return null;
|
||||||
const trimmed = value.trim();
|
const trimmed = value.trim();
|
||||||
|
|
@ -9,20 +29,17 @@ function normalizePath(value: unknown): string | null {
|
||||||
|
|
||||||
function resolveBinary(envValue: string | null, bundledValue: string | null, envVarName: string, packageName: string): string {
|
function resolveBinary(envValue: string | null, bundledValue: string | null, envVarName: string, packageName: string): string {
|
||||||
if (envValue) {
|
if (envValue) {
|
||||||
if ((envValue.includes('/') || envValue.includes('\\')) && !existsSync(envValue)) {
|
if ((envValue.includes('/') || envValue.includes('\\')) && !fs.existsSync(envValue)) {
|
||||||
throw new Error(`${envVarName} points to a missing binary: ${envValue}`);
|
throw new Error(`${envVarName} points to a missing binary: ${envValue}`);
|
||||||
}
|
}
|
||||||
return envValue;
|
return envValue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!bundledValue) {
|
if (!bundledValue) {
|
||||||
throw new Error(`${packageName} binary is unavailable on this platform. Set ${envVarName} to an installed binary path.`);
|
throw new Error(`${packageName} binary is unavailable on this platform. Set ${envVarName} to an installed binary path.`);
|
||||||
}
|
}
|
||||||
|
if ((bundledValue.includes('/') || bundledValue.includes('\\')) && !fs.existsSync(bundledValue)) {
|
||||||
if ((bundledValue.includes('/') || bundledValue.includes('\\')) && !existsSync(bundledValue)) {
|
|
||||||
throw new Error(`${packageName} resolved to a missing binary path: ${bundledValue}`);
|
throw new Error(`${packageName} resolved to a missing binary path: ${bundledValue}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return bundledValue;
|
return bundledValue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2,8 +2,8 @@ import { z } from 'zod';
|
||||||
import {
|
import {
|
||||||
runPdfLayoutFromPdfBuffer,
|
runPdfLayoutFromPdfBuffer,
|
||||||
runWhisperAlignmentFromAudioBuffer,
|
runWhisperAlignmentFromAudioBuffer,
|
||||||
} from '../inference/local-runtime';
|
} from '../inference/runtime';
|
||||||
import { withIdleTimeoutAndHardCap, withTimeout } from '../inference';
|
import { withIdleTimeoutAndHardCap, withTimeout } from '../infrastructure/config';
|
||||||
import type {
|
import type {
|
||||||
PdfLayoutJobRequest,
|
PdfLayoutJobRequest,
|
||||||
PdfLayoutJobResult,
|
PdfLayoutJobResult,
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { PDF_PARSER_VERSION } from '../inference/pdf/parser-version';
|
import { PDF_PARSER_VERSION } from '../api/contracts';
|
||||||
import { encodeParserVersion } from '../inference/pdf/parser-version-key';
|
import { encodeParserVersion } from '../api/contracts';
|
||||||
|
|
||||||
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i;
|
||||||
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/;
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
import { mergeTextWithRegions } from '../../../src/inference';
|
import { mergeTextWithRegions } from '../../../src/inference/pdf/document-layout';
|
||||||
|
|
||||||
describe('mergeTextWithRegions', () => {
|
describe('mergeTextWithRegions', () => {
|
||||||
test('assigns text items to containing regions by centroid and joins in reading order', () => {
|
test('assigns text items to containing regions by centroid and joins in reading order', () => {
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
|
|
||||||
import { normalizeTextItemsForLayout } from '../../../src/inference';
|
import { normalizeTextItemsForLayout } from '../../../src/inference/pdf/normalize-text';
|
||||||
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
||||||
|
|
||||||
function makeTextItem(
|
function makeTextItem(
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
import { stitchCrossPageBlocks } from '../../../src/inference';
|
import { stitchCrossPageBlocks } from '../../../src/inference/pdf/stitch';
|
||||||
import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/inference/types';
|
import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/api/types';
|
||||||
import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures';
|
import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures';
|
||||||
|
|
||||||
function makeBlock(
|
function makeBlock(
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import type { BaseDocument } from '../../../../../src/types/documents';
|
import type { BaseDocument } from '../../../../../src/types/documents';
|
||||||
import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/inference/types';
|
import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/api/types';
|
||||||
|
|
||||||
export function makeBaseDocument(overrides: Partial<BaseDocument> = {}): BaseDocument {
|
export function makeBaseDocument(overrides: Partial<BaseDocument> = {}): BaseDocument {
|
||||||
return {
|
return {
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,7 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
import {
|
import {
|
||||||
mapWordsToSentenceOffsets,
|
mapWordsToSentenceOffsets,
|
||||||
} from '../../../src/inference';
|
} from '../../../src/inference/whisper/timestamps';
|
||||||
|
|
||||||
describe('whisper alignment mapping', () => {
|
describe('whisper alignment mapping', () => {
|
||||||
test('maps words to sentence offsets with punctuation and repeated spaces', () => {
|
test('maps words to sentence offsets with punctuation and repeated spaces', () => {
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,5 @@
|
||||||
import { describe, expect, test } from 'vitest';
|
import { describe, expect, test } from 'vitest';
|
||||||
import { buildGoertzelCoefficients, goertzelPower } from '../../../src/inference';
|
import { buildGoertzelCoefficients, goertzelPower } from '../../../src/inference/whisper/align';
|
||||||
|
|
||||||
function dftPower(samples: Float32Array, k: number): number {
|
function dftPower(samples: Float32Array, k: number): number {
|
||||||
const n = samples.length;
|
const n = samples.length;
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-node';
|
||||||
import {
|
import {
|
||||||
buildWordsFromTimestampedTokens,
|
buildWordsFromTimestampedTokens,
|
||||||
extractTokenStartTimestamps,
|
extractTokenStartTimestamps,
|
||||||
} from '../../../src/inference';
|
} from '../../../src/inference/whisper/timestamps';
|
||||||
|
|
||||||
describe('whisper token timestamp alignment', () => {
|
describe('whisper token timestamp alignment', () => {
|
||||||
test('extracts monotonic token timestamps from cross-attention maps', () => {
|
test('extracts monotonic token timestamps from cross-attention maps', () => {
|
||||||
|
|
|
||||||
|
|
@ -76,7 +76,7 @@ vi.mock('../../../src/inference/pdf/model', () => ({
|
||||||
MODEL_PREPROCESSOR_PATH: '/tmp/model-preprocessor.json',
|
MODEL_PREPROCESSOR_PATH: '/tmp/model-preprocessor.json',
|
||||||
}));
|
}));
|
||||||
|
|
||||||
vi.mock('../../../src/inference/config/cpu-budget', () => ({
|
vi.mock('../../../src/infrastructure/config', () => ({
|
||||||
getOnnxThreadsPerJob: vi.fn(() => 1),
|
getOnnxThreadsPerJob: vi.fn(() => 1),
|
||||||
}));
|
}));
|
||||||
|
|
||||||
|
|
@ -107,7 +107,7 @@ describe('runLayoutModel', () => {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
const { runLayoutModel } = await import('../../../src/inference/pdf/runLayoutModel');
|
const { runLayoutModel } = await import('../../../src/inference/pdf/layout-model');
|
||||||
const regions = await runLayoutModel({
|
const regions = await runLayoutModel({
|
||||||
pageWidth: 100,
|
pageWidth: 100,
|
||||||
pageHeight: 100,
|
pageHeight: 100,
|
||||||
|
|
@ -154,7 +154,7 @@ describe('runLayoutModel', () => {
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
|
||||||
const { runLayoutModel } = await import('../../../src/inference/pdf/runLayoutModel');
|
const { runLayoutModel } = await import('../../../src/inference/pdf/layout-model');
|
||||||
const regions = await runLayoutModel({
|
const regions = await runLayoutModel({
|
||||||
pageWidth: 100,
|
pageWidth: 100,
|
||||||
pageHeight: 100,
|
pageHeight: 100,
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue