f1aa1c3e3b
Introduce support for external compute worker mode (`COMPUTE_MODE=worker`) using a new `WorkerComputeBackend`. This enables offloading heavy ONNX Whisper alignment and PDF layout parsing to a standalone worker service (Redis + BullMQ), improving scalability and compatibility with serverless/limited environments. - Add `@openreader/compute-core` as a shared package for ONNX inference and PDF parsing logic. - Implement `WorkerComputeBackend` and worker contract/types for remote job execution. - Update compute backend selection logic and remove previous worker mode guards. - Extend `WhisperAlignInput` and `PdfLayoutInput` types to support object keys for remote data access. - Refactor local compute backend to use `@openreader/compute-core` and support both buffer and object key inputs. - Update job runner, TTS segment alignment, and PDF layout parsing flows to use new compute backend APIs. - Add scripts, Docker workflow, and documentation for deploying and running the compute worker. - Update environment variable docs and examples for worker mode, including storage requirements and configuration. - Document published images and stack changes to reflect the new compute worker architecture. BREAKING CHANGE: `COMPUTE_MODE=worker` now requires an external compute worker service and S3-compatible object storage. Embedded SeaweedFS (`weed mini`) is not supported in worker mode. See the new documentation for deployment and configuration details.
36 lines
1.1 KiB
TypeScript
36 lines
1.1 KiB
TypeScript
import { existsSync } from 'fs';
|
|
import ffmpegStatic from 'ffmpeg-static';
|
|
|
|
function normalizePath(value: unknown): string | null {
|
|
if (typeof value !== 'string') return null;
|
|
const trimmed = value.trim();
|
|
return trimmed.length > 0 ? trimmed : null;
|
|
}
|
|
|
|
function resolveBinary(envValue: string | null, bundledValue: string | null, envVarName: string, packageName: string): string {
|
|
if (envValue) {
|
|
if ((envValue.includes('/') || envValue.includes('\\')) && !existsSync(envValue)) {
|
|
throw new Error(`${envVarName} points to a missing binary: ${envValue}`);
|
|
}
|
|
return envValue;
|
|
}
|
|
|
|
if (!bundledValue) {
|
|
throw new Error(`${packageName} binary is unavailable on this platform. Set ${envVarName} to an installed binary path.`);
|
|
}
|
|
|
|
if ((bundledValue.includes('/') || bundledValue.includes('\\')) && !existsSync(bundledValue)) {
|
|
throw new Error(`${packageName} resolved to a missing binary path: ${bundledValue}`);
|
|
}
|
|
|
|
return bundledValue;
|
|
}
|
|
|
|
export function getFFmpegPath(): string {
|
|
return resolveBinary(
|
|
normalizePath(process.env.FFMPEG_BIN),
|
|
normalizePath(ffmpegStatic),
|
|
'FFMPEG_BIN',
|
|
'ffmpeg-static',
|
|
);
|
|
}
|