From f1aa1c3e3ba43bb0706a79084b5737dabcf8eca8 Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 19 May 2026 15:21:25 -0600 Subject: [PATCH] feat(compute): add external compute worker backend and integration Introduce support for external compute worker mode (`COMPUTE_MODE=worker`) using a new `WorkerComputeBackend`. This enables offloading heavy ONNX Whisper alignment and PDF layout parsing to a standalone worker service (Redis + BullMQ), improving scalability and compatibility with serverless/limited environments. - Add `@openreader/compute-core` as a shared package for ONNX inference and PDF parsing logic. - Implement `WorkerComputeBackend` and worker contract/types for remote job execution. - Update compute backend selection logic and remove previous worker mode guards. - Extend `WhisperAlignInput` and `PdfLayoutInput` types to support object keys for remote data access. - Refactor local compute backend to use `@openreader/compute-core` and support both buffer and object key inputs. - Update job runner, TTS segment alignment, and PDF layout parsing flows to use new compute backend APIs. - Add scripts, Docker workflow, and documentation for deploying and running the compute worker. - Update environment variable docs and examples for worker mode, including storage requirements and configuration. - Document published images and stack changes to reflect the new compute worker architecture. BREAKING CHANGE: `COMPUTE_MODE=worker` now requires an external compute worker service and S3-compatible object storage. Embedded SeaweedFS (`weed mini`) is not supported in worker mode. See the new documentation for deployment and configuration details. --- .env.example | 9 +- .github/workflows/docker-publish.yml | 126 +- README.md | 3 +- compute/core/package.json | 17 + compute/core/src/index.ts | 87 ++ compute/core/src/pdf-layout/ensureModel.ts | 132 +++ .../src/pdf-layout/mergeTextWithRegions.ts | 153 +++ compute/core/src/pdf-layout/model/LICENSE.txt | 3 + .../core/src/pdf-layout/model/manifest.json | 31 + compute/core/src/pdf-layout/parsePdf.ts | 165 +++ compute/core/src/pdf-layout/renderPage.ts | 162 +++ compute/core/src/pdf-layout/runLayoutModel.ts | 331 ++++++ .../src/pdf-layout/stitchCrossPageBlocks.ts | 144 +++ compute/core/src/pdf-layout/types.ts | 15 + compute/core/src/runtime/docstore.ts | 7 + compute/core/src/runtime/ffmpeg.ts | 36 + compute/core/src/types/parsed-pdf.ts | 53 + compute/core/src/types/tts.ts | 16 + compute/core/src/whisper/alignment-mapping.ts | 54 + compute/core/src/whisper/alignment.ts | 1012 +++++++++++++++++ compute/core/src/whisper/ensureModel.ts | 242 ++++ compute/core/src/whisper/model/LICENSE.txt | 21 + compute/core/src/whisper/model/manifest.json | 76 ++ .../core/src/whisper/model/mel_filters.npz | Bin 0 -> 4271 bytes compute/core/src/whisper/spectral.ts | 21 + compute/core/src/whisper/token-timestamps.ts | 449 ++++++++ compute/core/tsconfig.json | 7 + compute/worker/.env.example | 25 + compute/worker/Dockerfile | 20 + compute/worker/docker-compose.yml | 41 + compute/worker/package.json | 23 + compute/worker/src/server.ts | 432 +++++++ compute/worker/tsconfig.json | 7 + docs-site/docs/deploy/compute-worker.md | 64 ++ docs-site/docs/deploy/local-development.md | 53 + docs-site/docs/deploy/vercel-deployment.md | 13 +- docs-site/docs/docker-quick-start.md | 7 + .../docs/reference/environment-variables.md | 22 +- docs-site/docs/reference/stack.md | 32 +- docs-site/sidebars.ts | 2 +- next.config.ts | 12 +- package.json | 6 +- pnpm-lock.yaml | 593 ++++++++++ pnpm-workspace.yaml | 1 + src/app/api/tts/segments/ensure/route.ts | 5 +- src/lib/server/compute/index.ts | 7 +- src/lib/server/compute/local.ts | 39 +- src/lib/server/compute/mode.ts | 5 - src/lib/server/compute/types.ts | 10 +- src/lib/server/compute/worker-contract.ts | 11 + src/lib/server/compute/worker.ts | 165 +++ src/lib/server/jobs/parsePdfJob.ts | 6 +- tsconfig.json | 3 +- 53 files changed, 4880 insertions(+), 96 deletions(-) create mode 100644 compute/core/package.json create mode 100644 compute/core/src/index.ts create mode 100644 compute/core/src/pdf-layout/ensureModel.ts create mode 100644 compute/core/src/pdf-layout/mergeTextWithRegions.ts create mode 100644 compute/core/src/pdf-layout/model/LICENSE.txt create mode 100644 compute/core/src/pdf-layout/model/manifest.json create mode 100644 compute/core/src/pdf-layout/parsePdf.ts create mode 100644 compute/core/src/pdf-layout/renderPage.ts create mode 100644 compute/core/src/pdf-layout/runLayoutModel.ts create mode 100644 compute/core/src/pdf-layout/stitchCrossPageBlocks.ts create mode 100644 compute/core/src/pdf-layout/types.ts create mode 100644 compute/core/src/runtime/docstore.ts create mode 100644 compute/core/src/runtime/ffmpeg.ts create mode 100644 compute/core/src/types/parsed-pdf.ts create mode 100644 compute/core/src/types/tts.ts create mode 100644 compute/core/src/whisper/alignment-mapping.ts create mode 100644 compute/core/src/whisper/alignment.ts create mode 100644 compute/core/src/whisper/ensureModel.ts create mode 100644 compute/core/src/whisper/model/LICENSE.txt create mode 100644 compute/core/src/whisper/model/manifest.json create mode 100644 compute/core/src/whisper/model/mel_filters.npz create mode 100644 compute/core/src/whisper/spectral.ts create mode 100644 compute/core/src/whisper/token-timestamps.ts create mode 100644 compute/core/tsconfig.json create mode 100644 compute/worker/.env.example create mode 100644 compute/worker/Dockerfile create mode 100644 compute/worker/docker-compose.yml create mode 100644 compute/worker/package.json create mode 100644 compute/worker/src/server.ts create mode 100644 compute/worker/tsconfig.json create mode 100644 docs-site/docs/deploy/compute-worker.md create mode 100644 src/lib/server/compute/worker-contract.ts create mode 100644 src/lib/server/compute/worker.ts diff --git a/.env.example b/.env.example index 3b38e23..74afbf0 100644 --- a/.env.example +++ b/.env.example @@ -80,10 +80,13 @@ IMPORT_LIBRARY_DIRS= # Heavy compute backend mode for ONNX whisper alignment + PDF layout parsing. # local = run compute in-process (default) # none = disable both capabilities (good for preview/serverless) -# worker = reserved for future external worker mode (not implemented in v1) +# worker = external durable worker mode (requires Redis-backed compute-worker service) COMPUTE_MODE=local -# COMPUTE_WORKER_URL= -# COMPUTE_WORKER_TOKEN= +# Required when COMPUTE_MODE=worker +# COMPUTE_WORKER_URL=http://localhost:8081 +# COMPUTE_WORKER_TOKEN=local-compute-token +# Worker mode requires worker-reachable shared object storage. +# Non-exposed embedded weed mini is not supported in worker mode. # Optional Whisper ONNX base URL override (must contain all expected files) # WHISPER_MODEL_BASE_URL=https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index cae0b8f..2c50173 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -22,30 +22,17 @@ jobs: prepare: runs-on: ubuntu-24.04 outputs: - image_name: ${{ steps.image-name.outputs.image_name }} - legacy_image_name: ${{ steps.image-name.outputs.legacy_image_name }} - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + web_image_name: ${{ steps.image-name.outputs.web_image_name }} + web_legacy_image_name: ${{ steps.image-name.outputs.web_legacy_image_name }} + compute_worker_image_name: ${{ steps.image-name.outputs.compute_worker_image_name }} steps: - name: Compute Docker image names id: image-name run: | owner="${GITHUB_REPOSITORY_OWNER,,}" - echo "image_name=${owner}/openreader" >> "$GITHUB_OUTPUT" - echo "legacy_image_name=${owner}/openreader-webui" >> "$GITHUB_OUTPUT" - - - name: Extract metadata (tags, labels) for Docker - id: meta - uses: docker/metadata-action@v6 - with: - images: | - ${{ env.REGISTRY }}/${{ steps.image-name.outputs.image_name }} - ${{ env.REGISTRY }}/${{ steps.image-name.outputs.legacy_image_name }} - tags: | - type=raw,value=latest,enable=${{ (github.event_name == 'push' && !contains(github.ref, '-pre')) || (github.event_name == 'workflow_dispatch' && inputs.use_latest_tag == true) }},priority=1000 - type=semver,pattern={{version}} - type=ref,event=tag - type=ref,event=branch,enable=${{ github.event_name == 'workflow_dispatch' && inputs.use_latest_tag != true }} + echo "web_image_name=${owner}/openreader" >> "$GITHUB_OUTPUT" + echo "web_legacy_image_name=${owner}/openreader-webui" >> "$GITHUB_OUTPUT" + echo "compute_worker_image_name=${owner}/openreader-compute-worker" >> "$GITHUB_OUTPUT" build: needs: prepare @@ -58,12 +45,30 @@ jobs: fail-fast: false matrix: include: - - arch: amd64 + - image_target: web + arch: amd64 platform: linux/amd64 runner: ubuntu-24.04 - - arch: arm64 + context: . + dockerfile: ./Dockerfile + - image_target: web + arch: arm64 platform: linux/arm64 runner: ubuntu-24.04-arm + context: . + dockerfile: ./Dockerfile + - image_target: compute-worker + arch: amd64 + platform: linux/amd64 + runner: ubuntu-24.04 + context: . + dockerfile: ./compute/worker/Dockerfile + - image_target: compute-worker + arch: arm64 + platform: linux/arm64 + runner: ubuntu-24.04-arm + context: . + dockerfile: ./compute/worker/Dockerfile steps: - name: Checkout repository @@ -82,46 +87,63 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Build and push Docker image (${{ matrix.arch }}) + - name: Select image name + id: image + run: | + if [ "${{ matrix.image_target }}" = "web" ]; then + echo "image_name=${{ needs.prepare.outputs.web_image_name }}" >> "$GITHUB_OUTPUT" + else + echo "image_name=${{ needs.prepare.outputs.compute_worker_image_name }}" >> "$GITHUB_OUTPUT" + fi + + - name: Build and push Docker image (${{ matrix.image_target }} / ${{ matrix.arch }}) id: build uses: docker/build-push-action@v7 with: - context: . + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} platforms: ${{ matrix.platform }} - labels: ${{ needs.prepare.outputs.labels }} - cache-from: type=gha,scope=${{ matrix.arch }} - cache-to: type=gha,mode=max,scope=${{ matrix.arch }} + cache-from: type=gha,scope=${{ matrix.image_target }}-${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.image_target }}-${{ matrix.arch }} provenance: false - outputs: type=image,name=${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true + outputs: type=image,name=${{ env.REGISTRY }}/${{ steps.image.outputs.image_name }},push-by-digest=true,name-canonical=true,push=true - name: Export digest run: | - mkdir -p /tmp/digests + mkdir -p /tmp/digests/${{ matrix.image_target }} digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" + touch "/tmp/digests/${{ matrix.image_target }}/${digest#sha256:}" - name: Upload digest uses: actions/upload-artifact@v7 with: - name: digests-${{ matrix.arch }} - path: /tmp/digests + name: digests-${{ matrix.image_target }}-${{ matrix.arch }} + path: /tmp/digests/${{ matrix.image_target }} if-no-files-found: error retention-days: 1 merge: needs: [prepare, build] - runs-on: ubuntu-24.04 + runs-on: ${{ matrix.runner }} permissions: actions: read contents: read packages: write + strategy: + fail-fast: false + matrix: + include: + - image_target: web + runner: ubuntu-24.04 + - image_target: compute-worker + runner: ubuntu-24.04 steps: - name: Download digests uses: actions/download-artifact@v8 with: - path: /tmp/digests - pattern: digests-* + path: /tmp/digests/${{ matrix.image_target }} + pattern: digests-${{ matrix.image_target }}-* merge-multiple: true - name: Set up Docker Buildx @@ -134,20 +156,46 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Select image names + id: image + run: | + if [ "${{ matrix.image_target }}" = "web" ]; then + echo "image_name=${{ needs.prepare.outputs.web_image_name }}" >> "$GITHUB_OUTPUT" + echo "metadata_images<> "$GITHUB_OUTPUT" + echo "${{ env.REGISTRY }}/${{ needs.prepare.outputs.web_image_name }}" >> "$GITHUB_OUTPUT" + echo "${{ env.REGISTRY }}/${{ needs.prepare.outputs.web_legacy_image_name }}" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + else + echo "image_name=${{ needs.prepare.outputs.compute_worker_image_name }}" >> "$GITHUB_OUTPUT" + echo "metadata_images=${{ env.REGISTRY }}/${{ needs.prepare.outputs.compute_worker_image_name }}" >> "$GITHUB_OUTPUT" + fi + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ steps.image.outputs.metadata_images }} + tags: | + type=raw,value=latest,enable=${{ (github.event_name == 'push' && !contains(github.ref, '-pre')) || (github.event_name == 'workflow_dispatch' && inputs.use_latest_tag == true) }},priority=1000 + type=semver,pattern={{version}} + type=ref,event=tag + type=ref,event=branch,enable=${{ github.event_name == 'workflow_dispatch' && inputs.use_latest_tag != true }} + - name: Create manifest list and push - working-directory: /tmp/digests + working-directory: /tmp/digests/${{ matrix.image_target }} run: | docker buildx imagetools create \ $(echo "$TAGS" | xargs -I {} echo -t {}) \ - $(printf '${{ env.REGISTRY }}/${{ needs.prepare.outputs.image_name }}@sha256:%s ' *) + $(printf '${{ env.REGISTRY }}/${{ steps.image.outputs.image_name }}@sha256:%s ' *) env: - TAGS: ${{ needs.prepare.outputs.tags }} + TAGS: ${{ steps.meta.outputs.tags }} - name: Output build information run: | - echo "✅ Docker images built and pushed successfully!" + echo "✅ Docker image built and pushed successfully!" + echo "🐋 Target: ${{ matrix.image_target }}" echo "🐋 Images:" - echo '${{ needs.prepare.outputs.tags }}' | sed 's/^/ - /' + echo '${{ steps.meta.outputs.tags }}' | sed 's/^/ - /' echo "📝 Event: ${{ github.event_name }}" if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then echo "📝 Triggered by manual workflow dispatch on branch: ${{ github.ref_name }}" diff --git a/README.md b/README.md index 73dfdc9..5fc22dd 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ OpenReader is an open source, self-host-friendly text-to-speech document reader - 🎯 **Multi-provider TTS** with OpenAI-compatible endpoints and cloud providers (Kokoro-FastAPI, KittenTTS-FastAPI, Orpheus-FastAPI or OpenAI, Replicate, DeepInfra). - 📖 **Read-along playback** for PDF/EPUB with sentence-aware narration. -- ⏱️ **Word-by-word highlighting** via built-in ONNX Whisper alignment in local compute mode (`COMPUTE_MODE=local`). +- ⏱️ **Word-by-word highlighting** via ONNX Whisper alignment in local mode (`COMPUTE_MODE=local`) or external worker mode (`COMPUTE_MODE=worker`). - 🧱 **Layout-aware PDF parsing** (PP-DocLayoutV3 ONNX) with structured blocks for cleaner TTS/chaptering. - 🛜 **Sync + library import** to bring docs across devices and from server-mounted folders. - 🗂️ **Flexible storage** with embedded SeaweedFS or external S3-compatible backends. @@ -33,6 +33,7 @@ OpenReader is an open source, self-host-friendly text-to-speech document reader | --- | --- | | Run with Docker | [Docker Quick Start](https://docs.openreader.richardr.dev/docker-quick-start) | | Deploy on Vercel | [Vercel Deployment](https://docs.openreader.richardr.dev/deploy/vercel-deployment) | +| Deploy external compute worker | [Compute Worker (Redis + BullMQ)](https://docs.openreader.richardr.dev/deploy/compute-worker) | | Develop locally | [Local Development](https://docs.openreader.richardr.dev/deploy/local-development) | | Configure auth | [Auth](https://docs.openreader.richardr.dev/configure/auth) | | Configure SQL database | [Database and Migrations](https://docs.openreader.richardr.dev/configure/database) | diff --git a/compute/core/package.json b/compute/core/package.json new file mode 100644 index 0000000..fe9bb45 --- /dev/null +++ b/compute/core/package.json @@ -0,0 +1,17 @@ +{ + "name": "@openreader/compute-core", + "version": "0.0.0", + "private": true, + "type": "module", + "dependencies": { + "@huggingface/tokenizers": "^0.1.3", + "@napi-rs/canvas": "^0.1.100", + "ffmpeg-static": "^5.3.0", + "jszip": "^3.10.1", + "onnxruntime-node": "^1.26.0", + "pdfjs-dist": "4.8.69" + }, + "exports": { + ".": "./src/index.ts" + } +} diff --git a/compute/core/src/index.ts b/compute/core/src/index.ts new file mode 100644 index 0000000..6f9ca15 --- /dev/null +++ b/compute/core/src/index.ts @@ -0,0 +1,87 @@ +import type { TTSSentenceAlignment } from './types/tts'; +import type { ParsedPdfDocument } from './types/parsed-pdf'; +import { ensureWhisperModel } from './whisper/ensureModel'; +import { alignAudioWithText } from './whisper/alignment'; +import { ensureModel as ensurePdfLayoutModel } from './pdf-layout/ensureModel'; +import { parsePdf } from './pdf-layout/parsePdf'; + +export type { + TTSAudioBuffer, + TTSAudioBytes, + TTSSentenceAlignment, + TTSSentenceWord, +} from './types/tts'; +export type { + ParsedPdfBlockKind, + ParsedPdfBlockFragment, + ParsedPdfBlock, + ParsedPdfPage, + ParsedPdfDocument, +} from './types/parsed-pdf'; + +export const ALIGN_QUEUE_NAME = 'whisper-align'; +export const PDF_LAYOUT_QUEUE_NAME = 'pdf-layout'; + +export interface WhisperAlignJobRequest { + text: string; + lang?: string; + cacheKey?: string; + audioObjectKey: string; +} + +export interface WhisperAlignJobResult { + alignments: TTSSentenceAlignment[]; +} + +export interface PdfLayoutJobRequest { + documentId: string; + namespace: string | null; + documentObjectKey: string; +} + +export interface PdfLayoutJobResult { + parsed: ParsedPdfDocument; +} + +export type WorkerJobState = 'queued' | 'running' | 'succeeded' | 'failed'; + +export interface WorkerJobErrorShape { + message: string; + code?: string; +} + +export interface WorkerJobStatusResponse { + status: WorkerJobState; + result?: Result; + error?: WorkerJobErrorShape; +} + +export async function ensureComputeModels(): Promise { + await Promise.all([ensureWhisperModel(), ensurePdfLayoutModel()]); +} + +export async function runWhisperAlignmentFromAudioBuffer(input: { + audioBuffer: ArrayBuffer; + text: string; + cacheKey?: string; + lang?: string; +}): Promise { + const alignments = await alignAudioWithText( + input.audioBuffer, + input.text, + input.cacheKey, + { lang: input.lang }, + ); + return { alignments }; +} + +export async function runPdfLayoutFromPdfBuffer(input: { + documentId: string; + pdfBytes: ArrayBuffer; +}): Promise { + const parsed = await parsePdf({ + documentId: input.documentId, + pdfBytes: input.pdfBytes, + }); + return { parsed }; +} diff --git a/compute/core/src/pdf-layout/ensureModel.ts b/compute/core/src/pdf-layout/ensureModel.ts new file mode 100644 index 0000000..a7a95da --- /dev/null +++ b/compute/core/src/pdf-layout/ensureModel.ts @@ -0,0 +1,132 @@ +import path from 'path'; +import { fileURLToPath } from 'url'; +import { createHash } from 'crypto'; +import { access, mkdir, rename, writeFile, readFile, unlink, copyFile } from 'fs/promises'; +import { DOCSTORE_DIR } from '../runtime/docstore'; +import manifest from './model/manifest.json'; + +const DEFAULT_MODEL_BASE_URL = 'https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main'; +const PDF_LAYOUT_MODEL_BASE_URL_ENV = 'PDF_LAYOUT_MODEL_BASE_URL'; +const MODEL_DIR = path.join(DOCSTORE_DIR, 'model'); +export const MODEL_PATH = path.join(MODEL_DIR, 'PP-DocLayoutV3.onnx'); +export const MODEL_DATA_PATH = path.join(MODEL_DIR, 'PP-DocLayoutV3.onnx.data'); +export const MODEL_CONFIG_PATH = path.join(MODEL_DIR, 'pp-doclayoutv3.config.json'); +export const MODEL_PREPROCESSOR_PATH = path.join(MODEL_DIR, 'pp-doclayoutv3.preprocessor_config.json'); +const LICENSE_PATH = path.join(MODEL_DIR, 'pp-doclayoutv3.LICENSE.txt'); +const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); +const STATIC_LICENSE_PATH = path.join(MODULE_DIR, 'model', 'LICENSE.txt'); + +let inflight: Promise | null = null; + +async function sha256Hex(filePath: string): Promise { + const bytes = await readFile(filePath); + return createHash('sha256').update(bytes).digest('hex'); +} + +async function downloadToFile(url: string, outPath: string): Promise { + const res = await fetch(url); + if (!res.ok) { + throw new Error(`Download failed for ${url}: ${res.status} ${res.statusText}`); + } + const bytes = new Uint8Array(await res.arrayBuffer()); + await writeFile(outPath, bytes); +} + +function joinModelUrl(baseUrl: string, relativePath: string): string { + return `${baseUrl.replace(/\/+$/, '')}/${relativePath}`; +} + +function manifestEntry(filePath: string): { sha256: string; size: number } | null { + const found = manifest.files.find((entry) => entry.path === filePath); + if (!found || !found.sha256) return null; + return { + sha256: found.sha256.toLowerCase(), + size: Number(found.size), + }; +} + +async function verifyFile(pathToFile: string, manifestPath: string): Promise { + const expected = manifestEntry(manifestPath); + if (!expected) return true; + const bytes = await readFile(pathToFile); + if (Number.isFinite(expected.size) && expected.size > 0 && bytes.byteLength !== expected.size) { + return false; + } + const actual = await sha256Hex(pathToFile); + return actual === expected.sha256; +} + +async function ensureLicense(): Promise { + await copyFile(STATIC_LICENSE_PATH, LICENSE_PATH); + if (!(await verifyFile(LICENSE_PATH, 'LICENSE.txt'))) { + throw new Error('PDF layout model license checksum verification failed'); + } +} + +async function ensureModelInternal(): Promise { + try { + await access(MODEL_PATH); + await access(MODEL_DATA_PATH); + await access(MODEL_CONFIG_PATH); + await access(MODEL_PREPROCESSOR_PATH); + if ( + await verifyFile(MODEL_PATH, 'model.onnx') + && await verifyFile(MODEL_DATA_PATH, 'model.onnx.data') + && await verifyFile(MODEL_CONFIG_PATH, 'config.json') + && await verifyFile(MODEL_PREPROCESSOR_PATH, 'preprocessor_config.json') + ) { + await ensureLicense(); + return MODEL_PATH; + } + } catch { + // continue + } + + await mkdir(MODEL_DIR, { recursive: true }); + const modelTmpPath = `${MODEL_PATH}.tmp`; + const modelDataTmpPath = `${MODEL_DATA_PATH}.tmp`; + const configTmpPath = `${MODEL_CONFIG_PATH}.tmp`; + const preprocessorTmpPath = `${MODEL_PREPROCESSOR_PATH}.tmp`; + const modelBaseUrl = process.env[PDF_LAYOUT_MODEL_BASE_URL_ENV]?.trim() + || DEFAULT_MODEL_BASE_URL; + const modelUrl = joinModelUrl(modelBaseUrl, 'PP-DocLayoutV3.onnx'); + const modelDataUrl = joinModelUrl(modelBaseUrl, 'PP-DocLayoutV3.onnx.data'); + const configUrl = joinModelUrl(modelBaseUrl, 'config.json'); + const preprocessorUrl = joinModelUrl(modelBaseUrl, 'preprocessor_config.json'); + + await downloadToFile(modelUrl, modelTmpPath); + if (!(await verifyFile(modelTmpPath, 'model.onnx'))) { + await unlink(modelTmpPath).catch(() => undefined); + throw new Error('PDF layout model checksum verification failed'); + } + await downloadToFile(modelDataUrl, modelDataTmpPath); + if (!(await verifyFile(modelDataTmpPath, 'model.onnx.data'))) { + await unlink(modelDataTmpPath).catch(() => undefined); + throw new Error('PDF layout model external data checksum verification failed'); + } + await downloadToFile(configUrl, configTmpPath); + if (!(await verifyFile(configTmpPath, 'config.json'))) { + await unlink(configTmpPath).catch(() => undefined); + throw new Error('PDF layout model config checksum verification failed'); + } + await downloadToFile(preprocessorUrl, preprocessorTmpPath); + if (!(await verifyFile(preprocessorTmpPath, 'preprocessor_config.json'))) { + await unlink(preprocessorTmpPath).catch(() => undefined); + throw new Error('PDF layout model preprocessor checksum verification failed'); + } + + await rename(modelTmpPath, MODEL_PATH); + await rename(modelDataTmpPath, MODEL_DATA_PATH); + await rename(configTmpPath, MODEL_CONFIG_PATH); + await rename(preprocessorTmpPath, MODEL_PREPROCESSOR_PATH); + await ensureLicense(); + return MODEL_PATH; +} + +export async function ensureModel(): Promise { + if (inflight) return inflight; + inflight = ensureModelInternal().finally(() => { + inflight = null; + }); + return inflight; +} diff --git a/compute/core/src/pdf-layout/mergeTextWithRegions.ts b/compute/core/src/pdf-layout/mergeTextWithRegions.ts new file mode 100644 index 0000000..aeff9ab --- /dev/null +++ b/compute/core/src/pdf-layout/mergeTextWithRegions.ts @@ -0,0 +1,153 @@ +import type { LayoutRegion, PdfTextItem } from './types'; + +const NON_TEXT_REGION_LABELS = new Set(['chart', 'image', 'table', 'seal']); +const TEXT_ASSIGNABLE_LABELS = new Set([ + 'abstract', + 'algorithm', + 'aside_text', + 'content', + 'doc_title', + 'figure_title', + 'footer', + 'footnote', + 'formula_number', + 'header', + 'number', + 'paragraph_title', + 'reference', + 'reference_content', + 'text', + 'vision_footnote', + 'formula', +]); + +function centroid(item: PdfTextItem): { x: number; y: number } { + return { + x: item.x + item.width / 2, + y: item.y + item.height / 2, + }; +} + +function contains(region: LayoutRegion, item: PdfTextItem): boolean { + const c = centroid(item); + return c.x >= region.bbox[0] && c.x <= region.bbox[2] && c.y >= region.bbox[1] && c.y <= region.bbox[3]; +} + +function sortReadingOrder(items: PdfTextItem[]): PdfTextItem[] { + const tolerance = 2; + return [...items].sort((a, b) => { + if (Math.abs(a.y - b.y) <= tolerance) return a.x - b.x; + return a.y - b.y; + }); +} + +function joinText(items: PdfTextItem[]): string { + let out = ''; + let prev: PdfTextItem | null = null; + for (const item of items) { + if (!prev) { + out += item.text; + prev = item; + continue; + } + const prevEndX = prev.x + prev.width; + const gap = item.x - prevEndX; + const lineJump = item.y - prev.y; + const lineBreak = lineJump > Math.max(2, Math.min(prev.height, item.height) * 0.6); + const avgCharWidth = item.width / Math.max(1, item.text.length); + const needsSpace = lineBreak || gap > Math.max(avgCharWidth * 0.3, 2); + out += needsSpace ? ` ${item.text}` : item.text; + prev = item; + } + return out.replace(/\s+/g, ' ').trim(); +} + +function regionArea(region: LayoutRegion): number { + return Math.max(1, (region.bbox[2] - region.bbox[0]) * (region.bbox[3] - region.bbox[1])); +} + +function regionScore(region: LayoutRegion): number { + return Number.isFinite(region.confidence) ? Number(region.confidence) : 0; +} + +export interface RegionTextBlock { + region: LayoutRegion; + text: string; + items: PdfTextItem[]; + sourceOrder: number; +} + +export function mergeTextWithRegions(regions: LayoutRegion[], textItems: PdfTextItem[]): RegionTextBlock[] { + const sourceIndex = new Map(); + for (let i = 0; i < textItems.length; i += 1) { + sourceIndex.set(textItems[i]!, i); + } + + const chunkSourceOrder = (items: PdfTextItem[]): number => { + let min = Number.POSITIVE_INFINITY; + for (const item of items) { + const index = sourceIndex.get(item); + if (typeof index === 'number' && index < min) min = index; + } + return Number.isFinite(min) ? min : Number.MAX_SAFE_INTEGER; + }; + + const assignableRegions = regions + .map((region, index) => ({ region, index })) + .filter(({ region }) => TEXT_ASSIGNABLE_LABELS.has(region.label)); + const assignedByRegion = new Map(); + + for (const item of textItems) { + const candidates = assignableRegions.filter(({ region }) => contains(region, item)); + if (candidates.length === 0) continue; + + candidates.sort((a, b) => { + const scoreDelta = regionScore(b.region) - regionScore(a.region); + if (Math.abs(scoreDelta) > 1e-6) return scoreDelta; + return regionArea(a.region) - regionArea(b.region); + }); + + const winner = candidates[0]; + const list = assignedByRegion.get(winner.index) ?? []; + list.push(item); + assignedByRegion.set(winner.index, list); + } + + const out: RegionTextBlock[] = []; + + for (const [regionIndex, assignedItems] of assignedByRegion.entries()) { + const region = regions[regionIndex]; + if (!region) continue; + if (assignedItems.length === 0) continue; + const ordered = sortReadingOrder(assignedItems); + const text = joinText(ordered); + if (!text) continue; + + out.push({ + region, + text, + items: ordered, + sourceOrder: chunkSourceOrder(ordered), + }); + } + + for (const region of regions) { + if (!NON_TEXT_REGION_LABELS.has(region.label)) continue; + out.push({ + region, + text: '', + items: [], + sourceOrder: Number.MAX_SAFE_INTEGER, + }); + } + + out.sort((a, b) => { + if (a.sourceOrder !== b.sourceOrder) return a.sourceOrder - b.sourceOrder; + const ay = a.region.bbox[1]; + const by = b.region.bbox[1]; + if (Math.abs(ay - by) <= 2) return a.region.bbox[0] - b.region.bbox[0]; + return ay - by; + }); + + return out; +} diff --git a/compute/core/src/pdf-layout/model/LICENSE.txt b/compute/core/src/pdf-layout/model/LICENSE.txt new file mode 100644 index 0000000..cb60fe1 --- /dev/null +++ b/compute/core/src/pdf-layout/model/LICENSE.txt @@ -0,0 +1,3 @@ +PP-DocLayoutV3 ONNX model assets are distributed by the model publisher under Apache-2.0. +See upstream for the authoritative license and terms: +https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX diff --git a/compute/core/src/pdf-layout/model/manifest.json b/compute/core/src/pdf-layout/model/manifest.json new file mode 100644 index 0000000..9f4807d --- /dev/null +++ b/compute/core/src/pdf-layout/model/manifest.json @@ -0,0 +1,31 @@ +{ + "name": "pp-doclayoutv3", + "version": "Bei0001/PP-DocLayoutV3-ONNX@main", + "files": [ + { + "path": "model.onnx", + "sha256": "c0721928ff08741bb208ebed539c77170db5234a68cb7e546e6cc9bc172a695b", + "size": 5088167 + }, + { + "path": "model.onnx.data", + "sha256": "34df3e4b79d7bbbf82abce1b4f3cde3d540fa57ad42ec8905c352b97c408d437", + "size": 136774480 + }, + { + "path": "config.json", + "sha256": "3cf834b91d23a756b1519bce4db42c09e852f3e35c35092dd5a3e253a50c071a", + "size": 2460 + }, + { + "path": "preprocessor_config.json", + "sha256": "519fe0187a43a1ca429e3ad8317bab8700f0d5e8fb3a6e3a0a413ffac078ba42", + "size": 575 + }, + { + "path": "LICENSE.txt", + "sha256": "578a6ba6f86b0692a8f719843f575a3eebf4705768ac5c37d149f441208f601f", + "size": 195 + } + ] +} diff --git a/compute/core/src/pdf-layout/parsePdf.ts b/compute/core/src/pdf-layout/parsePdf.ts new file mode 100644 index 0000000..74c9dfe --- /dev/null +++ b/compute/core/src/pdf-layout/parsePdf.ts @@ -0,0 +1,165 @@ +import path from 'path'; +import type { TextItem } from 'pdfjs-dist/types/src/display/api'; +import type { ParsedPdfDocument, ParsedPdfPage } from '../types/parsed-pdf'; +import type { PdfTextItem } from './types'; +import { ensureModel } from './ensureModel'; +import { runLayoutModel } from './runLayoutModel'; +import { mergeTextWithRegions } from './mergeTextWithRegions'; +import { stitchCrossPageBlocks } from './stitchCrossPageBlocks'; +import { renderPage } from './renderPage'; + +interface ParsePdfInput { + documentId: string; + pdfBytes: ArrayBuffer; +} + +const LAYOUT_RENDER_SCALE = 1.5; + +export function normalizeTextItemsForLayout(items: TextItem[], pageHeight: number): PdfTextItem[] { + return items + .filter((item) => { + if (!(typeof item.str === 'string' && item.str.trim().length > 0)) return false; + const transform = item.transform; + if (!Array.isArray(transform) || transform.length < 6) return false; + + // Reject heavily skewed/rotated text runs (e.g. vertical margin labels + // such as arXiv metadata) so they do not get merged into body blocks. + const skewX = Number(transform[1] ?? 0); + const skewY = Number(transform[2] ?? 0); + if (Math.abs(skewX) > 0.5 || Math.abs(skewY) > 0.5) return false; + + return true; + }) + .map((item) => { + const x = Number(item.transform[4] ?? 0); + const width = Math.max(0, Number(item.width ?? 0)); + const height = Math.max(1, Math.abs(Number(item.transform[3] ?? 1))); + const baselineY = Number(item.transform[5] ?? 0); + // pdf.js text transforms are in PDF user-space (origin bottom-left). + // Normalize into top-left page coordinates to match rendered image/model boxes. + const y = Math.max(0, pageHeight - baselineY - height); + return { + text: item.str, + x, + y, + width, + height, + }; + }); +} + +export async function parsePdf(input: ParsePdfInput): Promise { + await ensureModel(); + + // Keep independent byte copies for text extraction and page rendering. pdf.js + // can detach buffers passed to getDocument(). + const pdfBytesForText = new Uint8Array(input.pdfBytes).slice(); + const pdfBytesForRender = new Uint8Array(input.pdfBytes).slice(); + + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs'); + if (pdfjs.GlobalWorkerOptions) { + pdfjs.GlobalWorkerOptions.workerSrc = 'pdfjs-dist/legacy/build/pdf.worker.mjs'; + pdfjs.GlobalWorkerOptions.workerPort = null; + } + const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts'); + const standardFontDataUrl = `${standardFontDir.replace(/\/?$/, '/')}`; + + const loadingTask = pdfjs.getDocument({ + data: pdfBytesForText, + useWorkerFetch: false, + standardFontDataUrl, + isEvalSupported: false, + }); + const pdf = await loadingTask.promise; + + try { + const pages: ParsedPdfPage[] = []; + let nextBlockId = 1; + let sawText = false; + + for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber += 1) { + const page = await pdf.getPage(pageNumber); + const viewport = page.getViewport({ scale: 1.0 }); + const textContent = await page.getTextContent(); + const textItems = normalizeTextItemsForLayout( + textContent.items.filter((item): item is TextItem => 'str' in item && 'transform' in item), + viewport.height, + ); + + if (textItems.length > 0) sawText = true; + + const rendered = await renderPage({ + pdfBytes: pdfBytesForRender.buffer.slice( + pdfBytesForRender.byteOffset, + pdfBytesForRender.byteOffset + pdfBytesForRender.byteLength, + ), + pageNumber, + scale: LAYOUT_RENDER_SCALE, + }); + const scaleX = rendered.width / Math.max(1, viewport.width); + const scaleY = rendered.height / Math.max(1, viewport.height); + const layoutTextItems = textItems.map((item) => ({ + ...item, + x: item.x * scaleX, + y: item.y * scaleY, + width: item.width * scaleX, + height: item.height * scaleY, + })); + const regions = await runLayoutModel({ + pageWidth: rendered.width, + pageHeight: rendered.height, + textItems: layoutTextItems, + pageImage: rendered.image, + }); + const merged = mergeTextWithRegions(regions, layoutTextItems); + if (textItems.length > 0 && merged.length === 0) { + throw new Error(`layout-merge-empty: page=${pageNumber} regions=${regions.length}`); + } + + const blocks = merged + .map((entry, readingOrder) => ({ + id: `b${String(nextBlockId++).padStart(4, '0')}`, + kind: entry.region.label, + fragments: [{ + page: pageNumber, + bbox: [ + entry.region.bbox[0] / scaleX, + entry.region.bbox[1] / scaleY, + entry.region.bbox[2] / scaleX, + entry.region.bbox[3] / scaleY, + ] as [number, number, number, number], + text: entry.text, + readingOrder, + ...(typeof entry.region.confidence === 'number' ? { modelConfidence: entry.region.confidence } : {}), + }], + text: entry.text, + })); + + pages.push({ + pageNumber, + width: viewport.width, + height: viewport.height, + blocks, + }); + } + + if (!sawText) { + throw new Error('no-text-layer'); + } + + const doc: ParsedPdfDocument = { + schemaVersion: 1, + documentId: input.documentId, + parserVersion: 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69', + parsedAt: Date.now(), + pages, + }; + + return stitchCrossPageBlocks(doc); + } finally { + await pdf.destroy().catch(() => undefined); + await loadingTask.destroy().catch(() => undefined); + } +} diff --git a/compute/core/src/pdf-layout/renderPage.ts b/compute/core/src/pdf-layout/renderPage.ts new file mode 100644 index 0000000..e68fb66 --- /dev/null +++ b/compute/core/src/pdf-layout/renderPage.ts @@ -0,0 +1,162 @@ +import path from 'path'; +import type { Canvas } from '@napi-rs/canvas'; + +type CanvasRuntime = { + DOMMatrixCtor: unknown; + Path2DCtor: unknown; + createCanvasFn: (width: number, height: number) => Canvas; +}; + +let canvasRuntimePromise: Promise | null = null; + +async function loadCanvasRuntime(): Promise { + if (!canvasRuntimePromise) { + canvasRuntimePromise = (async () => { + const mod = await import('@napi-rs/canvas'); + const namespace = mod as Record; + const fallback = (namespace.default ?? {}) as Record; + + const createCanvasFn = (namespace.createCanvas ?? fallback.createCanvas) as + | ((width: number, height: number) => Canvas) + | undefined; + const DOMMatrixCtor = namespace.DOMMatrix ?? fallback.DOMMatrix; + const Path2DCtor = namespace.Path2D ?? fallback.Path2D; + + if (typeof createCanvasFn !== 'function') { + throw new Error( + `Canvas runtime missing createCanvas export (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`, + ); + } + if (!DOMMatrixCtor || !Path2DCtor) { + throw new Error( + `Canvas runtime missing DOMMatrix/Path2D exports (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`, + ); + } + + return { + DOMMatrixCtor, + Path2DCtor, + createCanvasFn, + }; + })(); + } + return canvasRuntimePromise; +} + +function ensureNodeCanvasGlobals(runtime: CanvasRuntime): void { + const g = globalThis as Record; + if (typeof g.DOMMatrix === 'undefined') g.DOMMatrix = runtime.DOMMatrixCtor; + if (typeof g.Path2D === 'undefined') g.Path2D = runtime.Path2DCtor; +} + +interface RenderInput { + pdfBytes: ArrayBuffer; + pageNumber: number; + scale?: number; + targetWidth?: number; + format?: 'png' | 'jpeg'; + jpegQuality?: number; +} + +function createPdfjsCanvasFactory(runtime: CanvasRuntime) { + return class OpenReaderCanvasFactory { + create(width: number, height: number) { + const canvas = runtime.createCanvasFn(Math.max(1, Math.floor(width)), Math.max(1, Math.floor(height))); + return { + canvas, + context: canvas.getContext('2d') as unknown as CanvasRenderingContext2D, + }; + } + + reset(target: { canvas: Canvas; context: CanvasRenderingContext2D }, width: number, height: number): void { + target.canvas.width = Math.max(1, Math.floor(width)); + target.canvas.height = Math.max(1, Math.floor(height)); + } + + destroy(target: { canvas: Canvas; context: CanvasRenderingContext2D }): void { + target.canvas.width = 0; + target.canvas.height = 0; + // @ts-expect-error pdf.js expects these nulled on destroy + target.canvas = null; + // @ts-expect-error pdf.js expects these nulled on destroy + target.context = null; + } + }; +} + +export async function renderPage({ + pdfBytes, + pageNumber, + scale = 1.5, + targetWidth, + format = 'png', + jpegQuality = 82, +}: RenderInput): Promise<{ + width: number; + height: number; + image: Buffer; + contentType: 'image/png' | 'image/jpeg'; +}> { + // pdf.js may detach the provided ArrayBuffer. Work with an isolated copy so + // callers can safely reuse their original bytes across pages/calls. + const isolatedBytes = new Uint8Array(pdfBytes).slice(); + + const canvasRuntime = await loadCanvasRuntime(); + ensureNodeCanvasGlobals(canvasRuntime); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs'); + + if (pdfjs.GlobalWorkerOptions) { + pdfjs.GlobalWorkerOptions.workerSrc = 'pdfjs-dist/legacy/build/pdf.worker.mjs'; + pdfjs.GlobalWorkerOptions.workerPort = null; + } + + const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts'); + const standardFontDataUrl = `${standardFontDir.replace(/\/?$/, '/')}`; + + const loadingTask = pdfjs.getDocument({ + data: isolatedBytes, + useWorkerFetch: false, + standardFontDataUrl, + isEvalSupported: false, + // Ensure pdf.js transport uses our canvas backend in Node/Next runtime. + CanvasFactory: createPdfjsCanvasFactory(canvasRuntime), + }); + + const pdf = await loadingTask.promise; + try { + const page = await pdf.getPage(pageNumber); + const baseViewport = page.getViewport({ scale: 1.0 }); + const effectiveScale = typeof targetWidth === 'number' && Number.isFinite(targetWidth) && targetWidth > 0 + ? (Math.max(1, Math.round(targetWidth)) / Math.max(1, baseViewport.width)) + : scale; + const viewport = page.getViewport({ scale: effectiveScale }); + const width = Math.max(1, Math.floor(viewport.width)); + const height = Math.max(1, Math.floor(viewport.height)); + const canvas = canvasRuntime.createCanvasFn(width, height); + const ctx = canvas.getContext('2d'); + ctx.fillStyle = '#ffffff'; + ctx.fillRect(0, 0, width, height); + + const renderTask = page.render({ + canvasContext: ctx as unknown as CanvasRenderingContext2D, + viewport, + intent: 'display', + }); + await renderTask.promise; + const contentType = format === 'jpeg' ? 'image/jpeg' : 'image/png'; + const image = format === 'jpeg' + ? canvas.toBuffer('image/jpeg', jpegQuality) + : canvas.toBuffer('image/png'); + return { + width, + height, + image, + contentType, + }; + } finally { + await pdf.destroy().catch(() => undefined); + await loadingTask.destroy().catch(() => undefined); + } +} diff --git a/compute/core/src/pdf-layout/runLayoutModel.ts b/compute/core/src/pdf-layout/runLayoutModel.ts new file mode 100644 index 0000000..f7b80eb --- /dev/null +++ b/compute/core/src/pdf-layout/runLayoutModel.ts @@ -0,0 +1,331 @@ +import * as ort from 'onnxruntime-node'; +import { readFile } from 'fs/promises'; +import type { LayoutRegion, PdfTextItem } from './types'; +import { ensureModel, MODEL_CONFIG_PATH, MODEL_PREPROCESSOR_PATH } from './ensureModel'; + +interface RunLayoutInput { + pageWidth: number; + pageHeight: number; + textItems: PdfTextItem[]; + pageImage: Buffer; +} + +const DEFAULT_INPUT_SIZE = 800; +const MIN_SCORE = 0.5; +const CLASS_MIN_SCORE: Partial> = { + header: 0.4, + footer: 0.4, + figure_title: 0.45, + footnote: 0.45, + vision_footnote: 0.45, +}; + +const LABEL_MAP: Record = { + // PP-DocLayoutV3 labels + abstract: 'abstract', + algorithm: 'algorithm', + aside_text: 'aside_text', + chart: 'chart', + content: 'content', + display_formula: 'formula', + doc_title: 'doc_title', + figure_title: 'figure_title', + footer: 'footer', + footer_image: 'footer', + footnote: 'footnote', + formula_number: 'formula_number', + header: 'header', + header_image: 'header', + image: 'image', + inline_formula: 'formula', + number: 'number', + paragraph_title: 'paragraph_title', + reference: 'reference', + reference_content: 'reference_content', + seal: 'seal', + table: 'table', + text: 'text', + vertical_text: 'text', + vision_footnote: 'vision_footnote', +}; + +const MIN_REGION_SIZE: Partial> = { + abstract: { minWidth: 24, minHeight: 14 }, + algorithm: { minWidth: 24, minHeight: 14 }, + aside_text: { minWidth: 24, minHeight: 14 }, + content: { minWidth: 24, minHeight: 14 }, + text: { minWidth: 24, minHeight: 14 }, + reference: { minWidth: 24, minHeight: 14 }, + reference_content: { minWidth: 24, minHeight: 14 }, + paragraph_title: { minWidth: 24, minHeight: 14 }, + doc_title: { minWidth: 24, minHeight: 14 }, + number: { minWidth: 18, minHeight: 12 }, + figure_title: { minWidth: 18, minHeight: 10 }, + footnote: { minWidth: 18, minHeight: 10 }, + vision_footnote: { minWidth: 18, minHeight: 10 }, + header: { minWidth: 18, minHeight: 10 }, + footer: { minWidth: 18, minHeight: 10 }, +}; + +interface ModelPreprocessor { + inputWidth: number; + inputHeight: number; + rescaleFactor: number; + mean: [number, number, number]; + std: [number, number, number]; +} + +let sessionPromise: Promise | null = null; +let idToLabelPromise: Promise> | null = null; +let preprocessorPromise: Promise | null = null; +let canvasFnsPromise: Promise<{ + createCanvasFn: (width: number, height: number) => { getContext: (kind: '2d') => CanvasRenderingContext2D }; + loadImageFn: (src: Buffer) => Promise<{ width: number; height: number } & CanvasImageSource>; +}> | null = null; + +async function getCanvasFns(): Promise<{ + createCanvasFn: (width: number, height: number) => { getContext: (kind: '2d') => CanvasRenderingContext2D }; + loadImageFn: (src: Buffer) => Promise<{ width: number; height: number } & CanvasImageSource>; +}> { + if (!canvasFnsPromise) { + canvasFnsPromise = (async () => { + const mod = await import('@napi-rs/canvas'); + const namespace = mod as Record; + const fallback = (namespace.default ?? {}) as Record; + const createCanvasFn = (namespace.createCanvas ?? fallback.createCanvas) as + | ((width: number, height: number) => { getContext: (kind: '2d') => CanvasRenderingContext2D }) + | undefined; + const loadImageFn = (namespace.loadImage ?? fallback.loadImage) as + | ((src: Buffer) => Promise<{ width: number; height: number } & CanvasImageSource>) + | undefined; + + if (typeof createCanvasFn !== 'function' || typeof loadImageFn !== 'function') { + throw new Error( + `Canvas runtime missing createCanvas/loadImage exports (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`, + ); + } + + return { createCanvasFn, loadImageFn }; + })(); + } + return canvasFnsPromise; +} + +async function getSession(): Promise { + if (!sessionPromise) { + sessionPromise = (async () => { + const modelPath = await ensureModel(); + return ort.InferenceSession.create(modelPath, { + executionProviders: ['cpu'], + graphOptimizationLevel: 'all', + }); + })(); + } + return sessionPromise; +} + +async function getIdToLabel(): Promise> { + if (!idToLabelPromise) { + idToLabelPromise = (async () => { + await ensureModel(); + const raw = await readFile(MODEL_CONFIG_PATH, 'utf8'); + const parsed = JSON.parse(raw) as { id2label?: Record }; + const out: Record = {}; + for (const [key, value] of Object.entries(parsed.id2label ?? {})) { + const n = Number(key); + if (Number.isFinite(n)) out[n] = String(value ?? '').trim(); + } + return out; + })(); + } + return idToLabelPromise; +} + +async function getPreprocessor(): Promise { + if (!preprocessorPromise) { + preprocessorPromise = (async () => { + await ensureModel(); + const raw = await readFile(MODEL_PREPROCESSOR_PATH, 'utf8'); + const parsed = JSON.parse(raw) as { + size?: { width?: number; height?: number }; + rescale_factor?: number; + image_mean?: number[]; + image_std?: number[]; + }; + + const inputWidth = Math.max(1, Number(parsed.size?.width ?? DEFAULT_INPUT_SIZE)); + const inputHeight = Math.max(1, Number(parsed.size?.height ?? DEFAULT_INPUT_SIZE)); + const rescaleFactor = Number.isFinite(parsed.rescale_factor) ? Number(parsed.rescale_factor) : (1 / 255); + const mean = [ + Number(parsed.image_mean?.[0] ?? 0), + Number(parsed.image_mean?.[1] ?? 0), + Number(parsed.image_mean?.[2] ?? 0), + ] as [number, number, number]; + const std = [ + Number(parsed.image_std?.[0] ?? 1), + Number(parsed.image_std?.[1] ?? 1), + Number(parsed.image_std?.[2] ?? 1), + ] as [number, number, number]; + + return { + inputWidth, + inputHeight, + rescaleFactor, + mean, + std, + }; + })(); + } + return preprocessorPromise; +} + +function preprocessResized( + image: CanvasImageSource, + preprocessor: ModelPreprocessor, + createCanvasFn: (width: number, height: number) => { getContext: (kind: '2d') => CanvasRenderingContext2D }, +): ort.Tensor { + const canvas = createCanvasFn(preprocessor.inputWidth, preprocessor.inputHeight); + const ctx = canvas.getContext('2d'); + ctx.fillStyle = '#ffffff'; + ctx.fillRect(0, 0, preprocessor.inputWidth, preprocessor.inputHeight); + ctx.imageSmoothingEnabled = true; + ctx.imageSmoothingQuality = 'high'; + ctx.drawImage(image, 0, 0, preprocessor.inputWidth, preprocessor.inputHeight); + + const imageData = ctx.getImageData(0, 0, preprocessor.inputWidth, preprocessor.inputHeight); + const chw = new Float32Array(1 * 3 * preprocessor.inputWidth * preprocessor.inputHeight); + const channelSize = preprocessor.inputWidth * preprocessor.inputHeight; + for (let y = 0; y < preprocessor.inputHeight; y += 1) { + for (let x = 0; x < preprocessor.inputWidth; x += 1) { + const pixelIndex = (y * preprocessor.inputWidth + x) * 4; + const idx = y * preprocessor.inputWidth + x; + const r = imageData.data[pixelIndex] * preprocessor.rescaleFactor; + const g = imageData.data[pixelIndex + 1] * preprocessor.rescaleFactor; + const b = imageData.data[pixelIndex + 2] * preprocessor.rescaleFactor; + + chw[idx] = (r - preprocessor.mean[0]) / Math.max(1e-8, preprocessor.std[0]); + chw[channelSize + idx] = (g - preprocessor.mean[1]) / Math.max(1e-8, preprocessor.std[1]); + chw[channelSize * 2 + idx] = (b - preprocessor.mean[2]) / Math.max(1e-8, preprocessor.std[2]); + } + } + return new ort.Tensor('float32', chw, [1, 3, preprocessor.inputHeight, preprocessor.inputWidth]); +} + +function clampBox( + bbox: [number, number, number, number], + pageWidth: number, + pageHeight: number, +): [number, number, number, number] | null { + const x0 = Math.max(0, Math.min(pageWidth, bbox[0])); + const y0 = Math.max(0, Math.min(pageHeight, bbox[1])); + const x1 = Math.max(0, Math.min(pageWidth, bbox[2])); + const y1 = Math.max(0, Math.min(pageHeight, bbox[3])); + if (x1 <= x0 || y1 <= y0) return null; + return [x0, y0, x1, y1]; +} + +function softmaxMax(logits: Float32Array, offset: number, count: number): { index: number; score: number } { + let maxLogit = Number.NEGATIVE_INFINITY; + let maxIndex = 0; + for (let i = 0; i < count; i += 1) { + const value = logits[offset + i]; + if (value > maxLogit) { + maxLogit = value; + maxIndex = i; + } + } + + let sum = 0; + for (let i = 0; i < count; i += 1) { + sum += Math.exp(logits[offset + i] - maxLogit); + } + + const score = sum > 0 ? (1 / sum) : 0; + return { index: maxIndex, score }; +} + +function normalizeModelLabel(rawLabel: string): string { + const normalized = rawLabel.trim().toLowerCase().replace(/[\s-]+/g, '_'); + if (normalized.endsWith('_image')) { + const base = normalized.slice(0, -'_image'.length); + if (base === 'header' || base === 'footer') return normalized; + } + // Some exports suffix duplicate classes (e.g. header_1, footer_1, text_1). + return normalized.replace(/_\d+$/g, ''); +} + +export async function runLayoutModel(input: RunLayoutInput): Promise { + const { pageWidth, pageHeight, textItems, pageImage } = input; + if (!textItems.length) return []; + if (!pageImage || pageImage.byteLength === 0) { + throw new Error('layout-render-missing-page-image'); + } + + try { + const [session, idToLabel, preprocessor, canvasFns] = await Promise.all([ + getSession(), + getIdToLabel(), + getPreprocessor(), + getCanvasFns(), + ]); + + const decodedPageImage = await canvasFns.loadImageFn(pageImage); + const pixelValues = preprocessResized(decodedPageImage, preprocessor, canvasFns.createCanvasFn); + const output = await session.run({ pixel_values: pixelValues }); + + const logits = output.logits?.data as Float32Array | undefined; + const predBoxes = output.pred_boxes?.data as Float32Array | undefined; + if (!logits || !predBoxes) return []; + + const numQueries = Math.floor(predBoxes.length / 4); + if (numQueries <= 0) return []; + const classCount = Math.floor(logits.length / numQueries); + if (classCount <= 0) return []; + + const regions: LayoutRegion[] = []; + + for (let queryIdx = 0; queryIdx < numQueries; queryIdx += 1) { + const cls = softmaxMax(logits, queryIdx * classCount, classCount); + const rawLabel = idToLabel[cls.index]; + if (!rawLabel) continue; + const mapped = LABEL_MAP[normalizeModelLabel(rawLabel)]; + if (!mapped) continue; + + const minScore = CLASS_MIN_SCORE[mapped] ?? MIN_SCORE; + if (!Number.isFinite(cls.score) || cls.score < minScore) continue; + + const cx = predBoxes[queryIdx * 4 + 0] * pageWidth; + const cy = predBoxes[queryIdx * 4 + 1] * pageHeight; + const w = predBoxes[queryIdx * 4 + 2] * pageWidth; + const h = predBoxes[queryIdx * 4 + 3] * pageHeight; + const rawBox: [number, number, number, number] = [ + cx - w / 2, + cy - h / 2, + cx + w / 2, + cy + h / 2, + ]; + + const clamped = clampBox(rawBox, pageWidth, pageHeight); + if (!clamped) continue; + + const sizeRule = MIN_REGION_SIZE[mapped]; + if (sizeRule) { + const width = clamped[2] - clamped[0]; + const height = clamped[3] - clamped[1]; + if (width < sizeRule.minWidth || height < sizeRule.minHeight) continue; + } + + regions.push({ + bbox: clamped, + label: mapped, + confidence: cls.score, + }); + } + + return regions.sort((a, b) => (b.confidence ?? 0) - (a.confidence ?? 0)); + } catch (error) { + throw new Error( + `layout-model-inference-failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } +} diff --git a/compute/core/src/pdf-layout/stitchCrossPageBlocks.ts b/compute/core/src/pdf-layout/stitchCrossPageBlocks.ts new file mode 100644 index 0000000..98a3d77 --- /dev/null +++ b/compute/core/src/pdf-layout/stitchCrossPageBlocks.ts @@ -0,0 +1,144 @@ +import type { ParsedPdfDocument, ParsedPdfBlock } from '../types/parsed-pdf'; + +const STITCHABLE_KINDS = new Set([ + 'text', + 'content', + 'reference_content', + 'aside_text', + 'abstract', + 'algorithm', + 'reference', +]); + +function stripTrailingClosers(text: string): string { + return text.trim().replace(/[\"'”’\]\)]+$/g, ''); +} + +function isSentenceTerminal(text: string): boolean { + return /[.!?]$/.test(stripTrailingClosers(text)); +} + +function canStitch(a: ParsedPdfBlock, b: ParsedPdfBlock): boolean { + if (!STITCHABLE_KINDS.has(a.kind)) return false; + if (a.kind !== b.kind) return false; + if (isSentenceTerminal(a.text)) return false; + const next = b.text.trim(); + if (!next) return false; + if (/^[A-Z]/.test(next)) return false; + return true; +} + +function splitHeadContinuation(text: string): { continuation: string; remainder: string } { + const normalized = text.replace(/\s+/g, ' ').trim(); + if (!normalized) return { continuation: '', remainder: '' }; + + const CLOSERS = new Set(['"', "'", '”', '’', ')', ']', '}']); + const isTerminal = (ch: string): boolean => ch === '.' || ch === '!' || ch === '?'; + + for (let i = 0; i < normalized.length; i += 1) { + const ch = normalized[i]; + if (!isTerminal(ch)) continue; + + const prev = i > 0 ? normalized[i - 1] : ''; + const next = i + 1 < normalized.length ? normalized[i + 1] : ''; + if (ch === '.' && /\d/.test(prev) && /\d/.test(next)) continue; + + let cut = i + 1; + while (cut < normalized.length && CLOSERS.has(normalized[cut])) cut += 1; + + const after = cut < normalized.length ? normalized[cut] : ''; + if (!after || /\s/.test(after) || /[A-Z]/.test(after)) { + return { + continuation: normalized.slice(0, cut).trim(), + remainder: normalized.slice(cut).trim(), + }; + } + } + + return { + continuation: normalized, + remainder: '', + }; +} + +const HARD_BOUNDARY_KINDS = new Set([ + 'paragraph_title', + 'doc_title', +]); + +function findTailCandidateIndex(blocks: ParsedPdfBlock[]): number { + for (let i = blocks.length - 1; i >= 0; i -= 1) { + const block = blocks[i]; + if (!block || !block.text.trim()) continue; + if (STITCHABLE_KINDS.has(block.kind)) return i; + } + return -1; +} + +function findHeadCandidateIndex(blocks: ParsedPdfBlock[]): number { + for (let i = 0; i < blocks.length; i += 1) { + const block = blocks[i]; + if (!block || !block.text.trim()) continue; + if (STITCHABLE_KINDS.has(block.kind)) return i; + } + return -1; +} + +function hasHardBoundaryBetween( + pageBlocks: ParsedPdfBlock[], + startInclusive: number, + endExclusive: number, +): boolean { + for (let i = startInclusive; i < endExclusive; i += 1) { + const block = pageBlocks[i]; + if (block && HARD_BOUNDARY_KINDS.has(block.kind)) return true; + } + return false; +} + +export function stitchCrossPageBlocks(doc: ParsedPdfDocument): ParsedPdfDocument { + const pages = doc.pages.map((page) => ({ ...page, blocks: page.blocks.map((b) => ({ ...b, fragments: b.fragments.map((f) => ({ ...f })) })) })); + + for (let i = 0; i < pages.length - 1; i += 1) { + const page = pages[i]; + const next = pages[i + 1]; + const tailIndex = findTailCandidateIndex(page.blocks); + const headIndex = findHeadCandidateIndex(next.blocks); + if (tailIndex < 0 || headIndex < 0) continue; + + if (hasHardBoundaryBetween(page.blocks, tailIndex + 1, page.blocks.length)) continue; + if (hasHardBoundaryBetween(next.blocks, 0, headIndex)) continue; + + const tail = page.blocks[tailIndex]; + const head = next.blocks[headIndex]; + if (!tail || !head) continue; + if (!canStitch(tail, head)) continue; + + const { continuation, remainder } = splitHeadContinuation(head.text); + if (!continuation) continue; + + const continuationFragment = head.fragments[0] + ? { ...head.fragments[0], text: continuation } + : null; + + if (continuationFragment) { + tail.fragments.push(continuationFragment); + } + tail.text = `${tail.text} ${continuation}`.replace(/\s+/g, ' ').trim(); + + if (!remainder) { + next.blocks.splice(headIndex, 1); + continue; + } + + head.text = remainder; + if (head.fragments[0]) { + head.fragments[0].text = remainder; + } + } + + return { + ...doc, + pages, + }; +} diff --git a/compute/core/src/pdf-layout/types.ts b/compute/core/src/pdf-layout/types.ts new file mode 100644 index 0000000..38089a6 --- /dev/null +++ b/compute/core/src/pdf-layout/types.ts @@ -0,0 +1,15 @@ +import type { ParsedPdfBlockKind } from '../types/parsed-pdf'; + +export interface PdfTextItem { + text: string; + x: number; + y: number; + width: number; + height: number; +} + +export interface LayoutRegion { + bbox: [number, number, number, number]; + label: ParsedPdfBlockKind; + confidence?: number; +} diff --git a/compute/core/src/runtime/docstore.ts b/compute/core/src/runtime/docstore.ts new file mode 100644 index 0000000..4e93930 --- /dev/null +++ b/compute/core/src/runtime/docstore.ts @@ -0,0 +1,7 @@ +import path from 'path'; + +export const DOCSTORE_DIR = process.env.COMPUTE_DOCSTORE_DIR?.trim() || path.join(process.cwd(), 'docstore'); + +export function getDocstoreDir(): string { + return DOCSTORE_DIR; +} diff --git a/compute/core/src/runtime/ffmpeg.ts b/compute/core/src/runtime/ffmpeg.ts new file mode 100644 index 0000000..b4ed669 --- /dev/null +++ b/compute/core/src/runtime/ffmpeg.ts @@ -0,0 +1,36 @@ +import { existsSync } from 'fs'; +import ffmpegStatic from 'ffmpeg-static'; + +function normalizePath(value: unknown): string | null { + if (typeof value !== 'string') return null; + const trimmed = value.trim(); + return trimmed.length > 0 ? trimmed : null; +} + +function resolveBinary(envValue: string | null, bundledValue: string | null, envVarName: string, packageName: string): string { + if (envValue) { + if ((envValue.includes('/') || envValue.includes('\\')) && !existsSync(envValue)) { + throw new Error(`${envVarName} points to a missing binary: ${envValue}`); + } + return envValue; + } + + if (!bundledValue) { + throw new Error(`${packageName} binary is unavailable on this platform. Set ${envVarName} to an installed binary path.`); + } + + if ((bundledValue.includes('/') || bundledValue.includes('\\')) && !existsSync(bundledValue)) { + throw new Error(`${packageName} resolved to a missing binary path: ${bundledValue}`); + } + + return bundledValue; +} + +export function getFFmpegPath(): string { + return resolveBinary( + normalizePath(process.env.FFMPEG_BIN), + normalizePath(ffmpegStatic), + 'FFMPEG_BIN', + 'ffmpeg-static', + ); +} diff --git a/compute/core/src/types/parsed-pdf.ts b/compute/core/src/types/parsed-pdf.ts new file mode 100644 index 0000000..137eec4 --- /dev/null +++ b/compute/core/src/types/parsed-pdf.ts @@ -0,0 +1,53 @@ +export type ParsedPdfBlockKind = + | 'abstract' + | 'algorithm' + | 'aside_text' + | 'chart' + | 'content' + | 'formula' + | 'doc_title' + | 'figure_title' + | 'footer' + | 'footnote' + | 'formula_number' + | 'header' + | 'image' + | 'number' + | 'paragraph_title' + | 'reference' + | 'reference_content' + | 'seal' + | 'table' + | 'text' + | 'vision_footnote'; + +export interface ParsedPdfBlockFragment { + page: number; + bbox: [number, number, number, number]; + text: string; + readingOrder: number; + modelConfidence?: number; +} + +export interface ParsedPdfBlock { + id: string; + kind: ParsedPdfBlockKind; + fragments: ParsedPdfBlockFragment[]; + text: string; + parentSectionId?: string; +} + +export interface ParsedPdfPage { + pageNumber: number; + width: number; + height: number; + blocks: ParsedPdfBlock[]; +} + +export interface ParsedPdfDocument { + schemaVersion: 1; + documentId: string; + parserVersion: string; + parsedAt: number; + pages: ParsedPdfPage[]; +} diff --git a/compute/core/src/types/tts.ts b/compute/core/src/types/tts.ts new file mode 100644 index 0000000..73db759 --- /dev/null +++ b/compute/core/src/types/tts.ts @@ -0,0 +1,16 @@ +export type TTSAudioBuffer = ArrayBuffer; +export type TTSAudioBytes = number[]; + +export interface TTSSentenceWord { + text: string; + startSec: number; + endSec: number; + charStart: number; + charEnd: number; +} + +export interface TTSSentenceAlignment { + sentence: string; + sentenceIndex: number; + words: TTSSentenceWord[]; +} diff --git a/compute/core/src/whisper/alignment-mapping.ts b/compute/core/src/whisper/alignment-mapping.ts new file mode 100644 index 0000000..b26ecc8 --- /dev/null +++ b/compute/core/src/whisper/alignment-mapping.ts @@ -0,0 +1,54 @@ +import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts'; + +function preprocessSentenceForAudio(text: string): string { + return text + .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') + .replace(/(\w+)-\s+(\w+)/g, '$1$2') + .replace(/\*/g, '') + .replace(/\s+/g, ' ') + .trim(); +} + +export interface WhisperWord { + start: number; + end: number; + word: string; +} + +export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment { + const normalizedSentence = preprocessSentenceForAudio(sentence); + const lowerSentence = normalizedSentence.toLowerCase(); + let cursor = 0; + + const alignedWords: TTSSentenceWord[] = words.map((w) => { + const token = w.word.trim(); + if (!token) { + return { + text: '', + startSec: w.start, + endSec: w.end, + charStart: cursor, + charEnd: cursor, + }; + } + + const idx = lowerSentence.indexOf(token.toLowerCase(), cursor); + const start = idx >= 0 ? idx : cursor; + const end = Math.min(normalizedSentence.length, start + token.length); + cursor = Math.max(cursor, end); + + return { + text: token, + startSec: w.start, + endSec: w.end, + charStart: start, + charEnd: end, + }; + }).filter((word) => word.text.length > 0); + + return { + sentence, + sentenceIndex: 0, + words: alignedWords, + }; +} diff --git a/compute/core/src/whisper/alignment.ts b/compute/core/src/whisper/alignment.ts new file mode 100644 index 0000000..7f6607f --- /dev/null +++ b/compute/core/src/whisper/alignment.ts @@ -0,0 +1,1012 @@ +import { createHash, randomUUID } from 'crypto'; +import { mkdtemp, readFile, rm, writeFile } from 'fs/promises'; +import { tmpdir } from 'os'; +import { dirname, join } from 'path'; +import { spawn } from 'child_process'; +import { fileURLToPath } from 'url'; +import * as ort from 'onnxruntime-node'; +import { Tokenizer } from '@huggingface/tokenizers'; +import JSZip from 'jszip'; +import type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment } from '../types/tts'; +import { getFFmpegPath } from '../runtime/ffmpeg'; +import { + mapWordsToSentenceOffsets, + type WhisperWord, +} from './alignment-mapping'; +import { buildGoertzelCoefficients, goertzelPower } from './spectral'; +import { + buildWordsFromTimestampedTokens, + extractTokenStartTimestamps, +} from './token-timestamps'; +import { + ensureWhisperModel, + WHISPER_CONFIG_PATH, + WHISPER_GENERATION_CONFIG_PATH, + WHISPER_TOKENIZER_CONFIG_PATH, + WHISPER_TOKENIZER_PATH, + WHISPER_ENCODER_MODEL_PATH, + WHISPER_DECODER_MERGED_MODEL_PATH, + WHISPER_DECODER_WITH_PAST_MODEL_PATH, +} from './ensureModel'; + +interface WhisperAlignmentOptions { + lang?: string; + textHint?: string; +} + +export interface WhisperRequestBody { + text: string; + audio: TTSAudioBytes; + lang?: string; +} + +interface WhisperRuntime { + encoder: ort.InferenceSession; + decoderMerged: ort.InferenceSession; + decoderWithPast: ort.InferenceSession; + tokenizer: Tokenizer; + promptStartToken: number; + defaultLanguageToken: number; + transcribeToken: number; + eosTokenId: number; + noTimestampsTokenId: number; + timestampBeginTokenId: number; + maxInitialTimestampIndex: number; + maxDecodeSteps: number; + suppressTokens: Set; + beginSuppressTokens: Set; + alignmentHeads: Array<[number, number]>; + prefillFetches: string[]; + stepFetches: string[]; +} + +type WhisperAlignmentState = { + alignmentCache: Map; + alignmentInFlight: Map>; + runtimePromise: Promise | null; + alignMutex: Promise; + pendingAlignments: number; + officialMelFilters: Float32Array[] | null; + emptyPastFeedsTemplate: Record | null; +}; + +const WHISPER_ALIGNMENT_STATE_KEY = '__openreaderWhisperAlignmentStateV1'; +const g = globalThis as typeof globalThis & Record; +const state = (() => { + const existing = g[WHISPER_ALIGNMENT_STATE_KEY] as WhisperAlignmentState | undefined; + if (existing) return existing; + const created: WhisperAlignmentState = { + alignmentCache: new Map(), + alignmentInFlight: new Map>(), + runtimePromise: null, + alignMutex: Promise.resolve(), + pendingAlignments: 0, + officialMelFilters: null, + emptyPastFeedsTemplate: null, + }; + g[WHISPER_ALIGNMENT_STATE_KEY] = created; + return created; +})(); +const alignmentCache = state.alignmentCache; +const alignmentInFlight = state.alignmentInFlight; +const ALIGNMENT_CACHE_MAX_ENTRIES = 256; +const MAX_DECODE_STEPS_CAP = 128; +const ALIGNMENT_TIMEOUT_MS = 25000; +const FFMPEG_DECODE_TIMEOUT_MS = 10000; + +const SAMPLE_RATE = 16000; +const N_FFT = 400; +const HOP_LENGTH = 160; +const CHUNK_LENGTH_SECONDS = 30; +const N_SAMPLES = CHUNK_LENGTH_SECONDS * SAMPLE_RATE; +const N_FRAMES = N_SAMPLES / HOP_LENGTH; +const N_MELS = 80; +const WHISPER_NUM_HEADS = 8; +const WHISPER_HEAD_DIM = 64; +const WHISPER_NUM_LAYERS = 6; +const MEL_FILTER_BINS = (N_FFT / 2) + 1; + +const hannWindow = buildHannWindow(N_FFT); +const goertzelCoefficients = buildGoertzelCoefficients(MEL_FILTER_BINS, N_FFT); + +const MODULE_DIR = dirname(fileURLToPath(import.meta.url)); +const MEL_FILTERS_NPZ_PATH = join(MODULE_DIR, 'model', 'mel_filters.npz'); + +function buildHannWindow(length: number): Float32Array { + const window = new Float32Array(length); + for (let i = 0; i < length; i += 1) { + window[i] = 0.5 - 0.5 * Math.cos((2 * Math.PI * i) / length); + } + return window; +} + +function parseNpyFloat32(bytes: Uint8Array): { shape: number[]; data: Float32Array } { + if (bytes.length < 12) { + throw new Error('Invalid NPY payload: too short'); + } + const magic = String.fromCharCode(...bytes.slice(0, 6)); + if (magic !== '\u0093NUMPY') { + throw new Error('Invalid NPY payload: missing magic header'); + } + + const major = bytes[6]; + const headerLength = major <= 1 + ? new DataView(bytes.buffer, bytes.byteOffset + 8, 2).getUint16(0, true) + : new DataView(bytes.buffer, bytes.byteOffset + 8, 4).getUint32(0, true); + const headerOffset = major <= 1 ? 10 : 12; + const header = Buffer.from(bytes.slice(headerOffset, headerOffset + headerLength)).toString('latin1'); + + const descrMatch = header.match(/'descr':\s*'([^']+)'/); + if (!descrMatch || descrMatch[1] !== ' token.trim()) + .filter(Boolean) + .map((token) => Number(token)) + .filter((n) => Number.isFinite(n) && n > 0); + + const dataOffset = headerOffset + headerLength; + const dataBytes = bytes.slice(dataOffset); + const totalFloats = Math.floor(dataBytes.byteLength / 4); + const data = new Float32Array(totalFloats); + const view = new DataView(dataBytes.buffer, dataBytes.byteOffset, dataBytes.byteLength); + for (let i = 0; i < totalFloats; i += 1) { + data[i] = view.getFloat32(i * 4, true); + } + + return { shape, data }; +} + +async function loadOfficialMelFilters(): Promise { + if (state.officialMelFilters) return state.officialMelFilters; + + const npzBytes = await readFile(MEL_FILTERS_NPZ_PATH); + const zip = await JSZip.loadAsync(npzBytes); + const mel80 = zip.file('mel_80.npy'); + if (!mel80) { + throw new Error('OpenAI mel filter asset is missing mel_80.npy'); + } + + const raw = await mel80.async('uint8array'); + const parsed = parseNpyFloat32(raw); + const [rows, cols] = parsed.shape; + if (rows !== N_MELS || cols !== MEL_FILTER_BINS) { + throw new Error(`Unexpected mel filter shape: [${rows}, ${cols}]`); + } + + const filters: Float32Array[] = []; + for (let row = 0; row < rows; row += 1) { + const start = row * cols; + filters.push(parsed.data.slice(start, start + cols)); + } + + state.officialMelFilters = filters; + return filters; +} + +function pcm16ToFloat32(buffer: Buffer): Float32Array { + const view = new Int16Array(buffer.buffer, buffer.byteOffset, Math.floor(buffer.byteLength / 2)); + const out = new Float32Array(view.length); + for (let i = 0; i < view.length; i += 1) { + out[i] = view[i] / 32768; + } + return out; +} + +function padOrTrimAudio(samples: Float32Array): Float32Array { + if (samples.length === N_SAMPLES) return samples; + if (samples.length > N_SAMPLES) return samples.subarray(0, N_SAMPLES); + + const padded = new Float32Array(N_SAMPLES); + padded.set(samples, 0); + return padded; +} + +function reflectPad(audio: Float32Array, pad: number): Float32Array { + const out = new Float32Array(audio.length + (2 * pad)); + out.set(audio, pad); + + // Match PyTorch reflect padding (exclude edge sample). + for (let i = 0; i < pad; i += 1) { + out[pad - 1 - i] = audio[Math.min(audio.length - 1, i + 1)]; + out[pad + audio.length + i] = audio[Math.max(0, audio.length - 2 - i)]; + } + + return out; +} + +function computeLogMelSpectrogram(audioSamples: Float32Array): ort.Tensor { + if (!state.officialMelFilters) { + throw new Error('Whisper mel filters not loaded'); + } + + const paddedAudio = reflectPad(audioSamples, N_FFT / 2); + const stftFrames = N_FRAMES + 1; + const frameCount = N_FRAMES; + const freqBins = MEL_FILTER_BINS; + + const melSpec = Array.from({ length: N_MELS }, () => new Float32Array(frameCount)); + const frame = new Float32Array(N_FFT); + const power = new Float32Array(freqBins); + + for (let frameIndex = 0; frameIndex < stftFrames; frameIndex += 1) { + const offset = frameIndex * HOP_LENGTH; + + for (let i = 0; i < N_FFT; i += 1) { + frame[i] = (paddedAudio[offset + i] ?? 0) * hannWindow[i]; + } + + for (let k = 0; k < freqBins; k += 1) { + power[k] = goertzelPower(frame, goertzelCoefficients[k]); + } + + if (frameIndex === stftFrames - 1) { + continue; + } + + for (let melIndex = 0; melIndex < N_MELS; melIndex += 1) { + const filter = state.officialMelFilters[melIndex]; + let total = 0; + for (let k = 0; k < freqBins; k += 1) { + total += filter[k] * power[k]; + } + melSpec[melIndex][frameIndex] = total; + } + } + + // Whisper normalization from openai/whisper/audio.py + let globalMaxLog = Number.NEGATIVE_INFINITY; + for (let i = 0; i < N_MELS; i += 1) { + for (let j = 0; j < frameCount; j += 1) { + const logVal = Math.log10(Math.max(1e-10, melSpec[i][j])); + if (logVal > globalMaxLog) globalMaxLog = logVal; + melSpec[i][j] = logVal; + } + } + + const floorVal = globalMaxLog - 8.0; + const flattened = new Float32Array(1 * N_MELS * frameCount); + for (let i = 0; i < N_MELS; i += 1) { + for (let j = 0; j < frameCount; j += 1) { + const clamped = Math.max(melSpec[i][j], floorVal); + flattened[(i * frameCount) + j] = (clamped + 4.0) / 4.0; + } + } + + return new ort.Tensor('float32', flattened, [1, N_MELS, frameCount]); +} + +async function decodeToPcm16(inputPath: string, outputPath: string): Promise { + await new Promise((resolve, reject) => { + const ffmpeg = spawn(getFFmpegPath(), [ + '-y', + '-i', + inputPath, + '-f', + 's16le', + '-ar', + String(SAMPLE_RATE), + '-ac', + '1', + outputPath, + ]); + + let stderr = ''; + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + ffmpeg.kill('SIGKILL'); + }, FFMPEG_DECODE_TIMEOUT_MS); + ffmpeg.stderr.on('data', (data) => { + stderr += data.toString(); + }); + + ffmpeg.on('error', (err) => { + clearTimeout(timer); + reject(err); + }); + + ffmpeg.on('close', (code) => { + clearTimeout(timer); + if (timedOut) { + reject(new Error(`ffmpeg decode timed out after ${FFMPEG_DECODE_TIMEOUT_MS}ms`)); + return; + } + if (code === 0) { + resolve(); + } else { + reject(new Error(`ffmpeg decode failed with code ${code}: ${stderr}`)); + } + }); + }); +} + +function parseLanguageCode(lang?: string): string | null { + if (!lang) return null; + const trimmed = lang.trim().toLowerCase(); + if (!trimmed) return null; + if (trimmed.includes('-')) return trimmed.split('-')[0] || null; + if (trimmed.includes('_')) return trimmed.split('_')[0] || null; + return trimmed; +} + +function tensorFromInt64(values: number[]): ort.Tensor { + return new ort.Tensor('int64', BigInt64Array.from(values.map((v) => BigInt(v))), [1, values.length]); +} + +function disposeTensor(tensor: ort.Tensor | undefined | null): void { + if (!tensor) return; + try { + tensor.dispose(); + } catch { + // Best-effort cleanup: ignore disposal errors during fallback path. + } +} + +function disposeTensorMap(tensors: Record): void { + for (const tensor of Object.values(tensors)) { + disposeTensor(tensor); + } +} + +function computeAdaptiveDecodeStepLimit(maxDecodeSteps: number, textHint?: string): number { + const normalized = (textHint ?? '').trim(); + if (!normalized) return Math.min(maxDecodeSteps, 96); + + const chars = normalized.length; + const words = normalized.split(/\s+/).filter(Boolean).length; + const estTokens = Math.max(words * 3, Math.ceil(chars / 2)); + const adaptive = Math.max(64, Math.min(maxDecodeSteps, estTokens + 24)); + return adaptive; +} + +function assertWithinDeadline(deadlineMs: number): void { + if (Date.now() > deadlineMs) { + throw new Error(`Whisper alignment timed out after ${ALIGNMENT_TIMEOUT_MS}ms`); + } +} + +function makeInFlightCoalesceKey(audioBuffer: TTSAudioBuffer, text: string, lang?: string): string { + const bytes = new Uint8Array(audioBuffer); + const span = 4096; + const head = bytes.subarray(0, Math.min(span, bytes.length)); + const tailStart = Math.max(0, bytes.length - span); + const tail = bytes.subarray(tailStart); + return createHash('sha256') + .update(text) + .update('\0') + .update(lang ?? '') + .update('\0') + .update(String(bytes.length)) + .update('\0') + .update(head) + .update('\0') + .update(tail) + .digest('hex'); +} + +function buildEmptyPastFeeds() { + if (state.emptyPastFeedsTemplate) return state.emptyPastFeedsTemplate; + + const feeds: Record = {}; + const emptyDecoderPast = new Float32Array(0); + const emptyEncoderPast = new Float32Array(1 * WHISPER_NUM_HEADS * 1500 * WHISPER_HEAD_DIM); + + for (let i = 0; i < WHISPER_NUM_LAYERS; i += 1) { + feeds[`past_key_values.${i}.decoder.key`] = new ort.Tensor('float32', emptyDecoderPast, [1, WHISPER_NUM_HEADS, 0, WHISPER_HEAD_DIM]); + feeds[`past_key_values.${i}.decoder.value`] = new ort.Tensor('float32', emptyDecoderPast, [1, WHISPER_NUM_HEADS, 0, WHISPER_HEAD_DIM]); + + // First pass still expects encoder KV inputs in the merged decoder graph. + feeds[`past_key_values.${i}.encoder.key`] = new ort.Tensor('float32', emptyEncoderPast, [1, WHISPER_NUM_HEADS, 1500, WHISPER_HEAD_DIM]); + feeds[`past_key_values.${i}.encoder.value`] = new ort.Tensor('float32', emptyEncoderPast, [1, WHISPER_NUM_HEADS, 1500, WHISPER_HEAD_DIM]); + } + + state.emptyPastFeedsTemplate = feeds; + return state.emptyPastFeedsTemplate; +} + +function argmax(values: Float32Array): number | null { + let bestIdx = 0; + let bestScore = Number.NEGATIVE_INFINITY; + + for (let i = 0; i < values.length; i += 1) { + const score = values[i]; + if (score > bestScore) { + bestScore = score; + bestIdx = i; + } + } + + return Number.isFinite(bestScore) ? bestIdx : null; +} + +function applyTokenSuppression(logits: Float32Array, tokens: Set) { + for (const tokenId of tokens) { + if (tokenId >= 0 && tokenId < logits.length) { + logits[tokenId] = Number.NEGATIVE_INFINITY; + } + } +} + +function logSoftmax(input: Float32Array): Float32Array { + let max = Number.NEGATIVE_INFINITY; + for (let i = 0; i < input.length; i += 1) { + if (input[i] > max) max = input[i]; + } + if (!Number.isFinite(max)) { + return new Float32Array(input.length).fill(Number.NEGATIVE_INFINITY); + } + + let sum = 0; + for (let i = 0; i < input.length; i += 1) { + sum += Math.exp(input[i] - max); + } + const logSum = Math.log(sum); + + const out = new Float32Array(input.length); + for (let i = 0; i < input.length; i += 1) { + out[i] = input[i] - max - logSum; + } + return out; +} + +function applyWhisperTimestampLogitsRules(input: { + logits: Float32Array; + generated: number[]; + beginIndex: number; + eosTokenId: number; + noTimestampsTokenId: number; + timestampBeginTokenId: number; + maxInitialTimestampIndex: number; +}) { + const { + logits, + generated, + beginIndex, + eosTokenId, + noTimestampsTokenId, + timestampBeginTokenId, + maxInitialTimestampIndex, + } = input; + + if (noTimestampsTokenId >= 0 && noTimestampsTokenId < logits.length) { + logits[noTimestampsTokenId] = Number.NEGATIVE_INFINITY; + } + + if (generated.length === beginIndex) { + const upper = Math.min(timestampBeginTokenId, logits.length); + for (let i = 0; i < upper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } + + const seq = generated.slice(beginIndex); + const lastWasTimestamp = seq.length >= 1 && seq[seq.length - 1] >= timestampBeginTokenId; + const penultimateWasTimestamp = seq.length < 2 || seq[seq.length - 2] >= timestampBeginTokenId; + + if (lastWasTimestamp) { + if (penultimateWasTimestamp) { + for (let i = timestampBeginTokenId; i < logits.length; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } else { + const upper = Math.min(eosTokenId, logits.length); + for (let i = 0; i < upper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } + } + + if (generated.length === beginIndex && Number.isFinite(maxInitialTimestampIndex)) { + const lastAllowed = timestampBeginTokenId + maxInitialTimestampIndex; + for (let i = lastAllowed + 1; i < logits.length; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } + + const textUpper = Math.min(timestampBeginTokenId, logits.length); + if (textUpper <= 0 || textUpper >= logits.length) return; + + const logprobs = logSoftmax(logits); + + let maxTextTokenLogprob = Number.NEGATIVE_INFINITY; + for (let i = 0; i < textUpper; i += 1) { + if (logprobs[i] > maxTextTokenLogprob) maxTextTokenLogprob = logprobs[i]; + } + + let timestampProbMass = 0; + for (let i = textUpper; i < logprobs.length; i += 1) { + timestampProbMass += Math.exp(logprobs[i]); + } + const timestampLogprob = timestampProbMass > 0 ? Math.log(timestampProbMass) : Number.NEGATIVE_INFINITY; + + if (timestampLogprob > maxTextTokenLogprob) { + for (let i = 0; i < textUpper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } +} + +async function getRuntime(): Promise { + if (state.runtimePromise) return state.runtimePromise; + + state.runtimePromise = (async () => { + await ensureWhisperModel(); + await loadOfficialMelFilters(); + + const [configRaw, generationRaw, tokenizerJsonRaw, tokenizerConfigRaw] = await Promise.all([ + readFile(WHISPER_CONFIG_PATH, 'utf8'), + readFile(WHISPER_GENERATION_CONFIG_PATH, 'utf8'), + readFile(WHISPER_TOKENIZER_PATH, 'utf8'), + readFile(WHISPER_TOKENIZER_CONFIG_PATH, 'utf8'), + ]); + + const config = JSON.parse(configRaw) as { + decoder_start_token_id?: number; + eos_token_id?: number; + forced_decoder_ids?: Array<[number, number | null]>; + }; + + const generationConfig = JSON.parse(generationRaw) as { + no_timestamps_token_id?: number; + max_initial_timestamp_index?: number; + suppress_tokens?: number[]; + begin_suppress_tokens?: number[]; + max_length?: number; + alignment_heads?: Array<[number, number]>; + }; + + const tokenizer = new Tokenizer(JSON.parse(tokenizerJsonRaw), JSON.parse(tokenizerConfigRaw)); + + const promptStartToken = Number(config.decoder_start_token_id ?? 50258); + const eosTokenId = Number(config.eos_token_id ?? 50257); + const noTimestampsTokenId = Number(generationConfig.no_timestamps_token_id ?? 50363); + const timestampBeginTokenId = noTimestampsTokenId + 1; + const maxInitialTimestampIndex = Number(generationConfig.max_initial_timestamp_index ?? 50); + const configuredMaxDecodeSteps = Number(generationConfig.max_length ?? 448); + const maxDecodeSteps = Math.min(configuredMaxDecodeSteps, MAX_DECODE_STEPS_CAP); + const alignmentHeads = Array.isArray(generationConfig.alignment_heads) + ? generationConfig.alignment_heads + .filter((head): head is [number, number] => Array.isArray(head) && head.length === 2) + .map(([layer, head]) => [Number(layer), Number(head)] as [number, number]) + : []; + + const forcedDecoder = Array.isArray(config.forced_decoder_ids) ? config.forced_decoder_ids : []; + const defaultLanguageFromForced = forcedDecoder.find(([index, id]) => index === 1 && typeof id === 'number')?.[1] ?? null; + const transcribeFromForced = forcedDecoder.find(([index, id]) => index === 2 && typeof id === 'number')?.[1] ?? null; + + const defaultLanguageToken = Number(defaultLanguageFromForced ?? tokenizer.token_to_id('<|en|>') ?? 50259); + const transcribeToken = Number(transcribeFromForced ?? tokenizer.token_to_id('<|transcribe|>') ?? 50359); + + const stableSessionOptions: ort.InferenceSession.SessionOptions = { + executionProviders: ['cpu'], + graphOptimizationLevel: 'disabled', + intraOpNumThreads: 1, + interOpNumThreads: 1, + executionMode: 'sequential', + enableCpuMemArena: false, + enableMemPattern: false, + }; + + const encoder = await ort.InferenceSession.create(WHISPER_ENCODER_MODEL_PATH, stableSessionOptions); + const decoderMerged = await ort.InferenceSession.create(WHISPER_DECODER_MERGED_MODEL_PATH, stableSessionOptions); + const decoderWithPast = await ort.InferenceSession.create(WHISPER_DECODER_WITH_PAST_MODEL_PATH, stableSessionOptions); + + const alignmentLayers = [...new Set(alignmentHeads.map(([layer]) => layer))]; + const prefillFetches: string[] = ['logits']; + const stepFetches: string[] = ['logits']; + const mergedOutputNames = new Set(decoderMerged.outputNames); + const withPastOutputNames = new Set(decoderWithPast.outputNames); + + for (let i = 0; i < WHISPER_NUM_LAYERS; i += 1) { + const decoderKey = `present.${i}.decoder.key`; + const decoderValue = `present.${i}.decoder.value`; + if (mergedOutputNames.has(decoderKey)) prefillFetches.push(decoderKey); + if (mergedOutputNames.has(decoderValue)) prefillFetches.push(decoderValue); + if (withPastOutputNames.has(decoderKey)) stepFetches.push(decoderKey); + if (withPastOutputNames.has(decoderValue)) stepFetches.push(decoderValue); + + const encoderKey = `present.${i}.encoder.key`; + const encoderValue = `present.${i}.encoder.value`; + if (mergedOutputNames.has(encoderKey)) prefillFetches.push(encoderKey); + if (mergedOutputNames.has(encoderValue)) prefillFetches.push(encoderValue); + } + + for (const layer of alignmentLayers) { + const key = `cross_attentions.${layer}`; + if (mergedOutputNames.has(key)) prefillFetches.push(key); + if (withPastOutputNames.has(key)) stepFetches.push(key); + } + + return { + encoder, + decoderMerged, + decoderWithPast, + tokenizer, + promptStartToken, + defaultLanguageToken, + transcribeToken, + eosTokenId, + noTimestampsTokenId, + timestampBeginTokenId, + maxInitialTimestampIndex, + maxDecodeSteps, + suppressTokens: new Set((generationConfig.suppress_tokens ?? []).map((v) => Number(v))), + beginSuppressTokens: new Set((generationConfig.begin_suppress_tokens ?? []).map((v) => Number(v))), + alignmentHeads, + prefillFetches, + stepFetches, + }; + })().catch((error) => { + state.runtimePromise = null; + throw error; + }); + + return state.runtimePromise; +} + +function resolveLanguageToken(runtime: WhisperRuntime, lang?: string): number { + const parsed = parseLanguageCode(lang); + if (!parsed) return runtime.defaultLanguageToken; + + const candidate = runtime.tokenizer.token_to_id(`<|${parsed}|>`); + return typeof candidate === 'number' ? candidate : runtime.defaultLanguageToken; +} + +async function runWhisperOnnx( + audioSamples: Float32Array, + opts: WhisperAlignmentOptions, + numFrames: number, + deadlineMs: number, +): Promise { + assertWithinDeadline(deadlineMs); + const runtime = await getRuntime(); + const decodeStepLimit = computeAdaptiveDecodeStepLimit(runtime.maxDecodeSteps, opts.textHint); + const mel = computeLogMelSpectrogram(audioSamples); + const encoderPast: Record = {}; + const decoderPast: Record = {}; + const crossAttentions: Record = {}; + let encoderHidden: ort.Tensor | null = null; + let outputs: Record | null = null; + + try { + const encoderOutputs = await runtime.encoder.run({ + input_features: mel, + }, ['last_hidden_state']); + encoderHidden = encoderOutputs.last_hidden_state; + + const languageToken = resolveLanguageToken(runtime, opts.lang); + const promptTokens = [ + runtime.promptStartToken, + languageToken, + runtime.transcribeToken, + ]; + + const generated: number[] = [...promptTokens]; + const emptyPastFeeds = buildEmptyPastFeeds(); + type LayerChunk = { + data: Float32Array; + heads: number; + seqLen: number; + frames: number; + }; + const selectedHeadsByLayer = new Map(); + for (const [layer, head] of runtime.alignmentHeads) { + const existing = selectedHeadsByLayer.get(layer) ?? []; + if (!existing.includes(head)) existing.push(head); + selectedHeadsByLayer.set(layer, existing); + } + for (const [layer, heads] of selectedHeadsByLayer) { + heads.sort((a, b) => a - b); + selectedHeadsByLayer.set(layer, heads); + } + const crossAttentionChunks = new Map(); + + const captureCrossAttentions = (stepOutputs: Record, prefill = false) => { + for (const [layer, selectedHeads] of selectedHeadsByLayer) { + const key = `cross_attentions.${layer}`; + const tensor = stepOutputs[key]; + if (!tensor) continue; + const [, , seqLen, frames] = tensor.dims; + const data = tensor.data as Float32Array; + const rowsToKeep = prefill ? seqLen : 1; + const seqStart = prefill ? 0 : Math.max(0, seqLen - 1); + const copied = new Float32Array(selectedHeads.length * rowsToKeep * frames); + for (let h = 0; h < selectedHeads.length; h += 1) { + const sourceHead = selectedHeads[h]!; + for (let s = 0; s < rowsToKeep; s += 1) { + const sourceSeq = seqStart + s; + for (let f = 0; f < frames; f += 1) { + const src = (((sourceHead * seqLen) + sourceSeq) * frames) + f; + const dst = (((h * rowsToKeep) + s) * frames) + f; + copied[dst] = data[src] ?? 0; + } + } + } + const list = crossAttentionChunks.get(layer) ?? []; + list.push({ data: copied, heads: selectedHeads.length, seqLen: rowsToKeep, frames }); + crossAttentionChunks.set(layer, list); + } + }; + const beginIndex = promptTokens.length; + + // Prefill: run prompt in merged decoder (non-cache branch), identical to first + // forward pass in transformers.js/transformers generation. + const prefillInputIds = tensorFromInt64(generated); + const prefillUseCacheBranch = new ort.Tensor('bool', Uint8Array.from([0]), [1]); + const prefillFeeds: Record = { + input_ids: prefillInputIds, + encoder_hidden_states: encoderHidden, + use_cache_branch: prefillUseCacheBranch, + ...emptyPastFeeds, + }; + try { + assertWithinDeadline(deadlineMs); + outputs = await runtime.decoderMerged.run(prefillFeeds, runtime.prefillFetches); + } finally { + disposeTensor(prefillInputIds); + disposeTensor(prefillUseCacheBranch); + } + captureCrossAttentions(outputs, true); + + for (let i = 0; i < WHISPER_NUM_LAYERS; i += 1) { + encoderPast[`past_key_values.${i}.encoder.key`] = outputs[`present.${i}.encoder.key`]; + encoderPast[`past_key_values.${i}.encoder.value`] = outputs[`present.${i}.encoder.value`]; + decoderPast[`past_key_values.${i}.decoder.key`] = outputs[`present.${i}.decoder.key`]; + decoderPast[`past_key_values.${i}.decoder.value`] = outputs[`present.${i}.decoder.value`]; + } + + for (let step = 0; step < decodeStepLimit; step += 1) { + assertWithinDeadline(deadlineMs); + if (!outputs) break; + const logits = outputs.logits; + const logitsData = logits.data as Float32Array; + const vocabSize = logits.dims[2] ?? 0; + const offset = logitsData.length - vocabSize; + const lastLogits = logitsData.subarray(offset); + + applyTokenSuppression(lastLogits, runtime.suppressTokens); + if (generated.length === beginIndex) { + applyTokenSuppression(lastLogits, runtime.beginSuppressTokens); + } + applyWhisperTimestampLogitsRules({ + logits: lastLogits, + generated, + beginIndex, + eosTokenId: runtime.eosTokenId, + noTimestampsTokenId: runtime.noTimestampsTokenId, + timestampBeginTokenId: runtime.timestampBeginTokenId, + maxInitialTimestampIndex: runtime.maxInitialTimestampIndex, + }); + + const nextToken = argmax(lastLogits) ?? runtime.eosTokenId; + generated.push(nextToken); + if (nextToken === runtime.eosTokenId) break; + + const previousDecoderPast = { ...decoderPast }; + const stepInputIds = tensorFromInt64([nextToken]); + const stepFeeds: Record = { + input_ids: stepInputIds, + ...previousDecoderPast, + ...encoderPast, + }; + let nextOutputs: Record; + try { + assertWithinDeadline(deadlineMs); + nextOutputs = await runtime.decoderWithPast.run(stepFeeds, runtime.stepFetches); + } finally { + disposeTensor(stepInputIds); + } + captureCrossAttentions(nextOutputs, false); + + for (let i = 0; i < WHISPER_NUM_LAYERS; i += 1) { + decoderPast[`past_key_values.${i}.decoder.key`] = nextOutputs[`present.${i}.decoder.key`]; + decoderPast[`past_key_values.${i}.decoder.value`] = nextOutputs[`present.${i}.decoder.value`]; + } + + disposeTensorMap(previousDecoderPast); + disposeTensor(outputs.logits); + for (const [name, tensor] of Object.entries(outputs)) { + if (name.startsWith('cross_attentions.')) { + disposeTensor(tensor); + } + } + outputs = nextOutputs; + } + + if (crossAttentionChunks.size === 0) { + return []; + } + + const remappedAlignmentHeads: Array<[number, number]> = runtime.alignmentHeads + .map(([layer, head]) => { + const selectedHeads = selectedHeadsByLayer.get(layer) ?? []; + const remappedHead = selectedHeads.indexOf(head); + if (remappedHead < 0) return null; + return [layer, remappedHead] as [number, number]; + }) + .filter((pair): pair is [number, number] => pair !== null); + + for (let layer = 0; layer < WHISPER_NUM_LAYERS; layer += 1) { + const chunks = crossAttentionChunks.get(layer); + if (!chunks || !chunks.length) continue; + + const heads = chunks[0].heads; + const frames = chunks[0].frames; + const concatSeqLen = chunks.reduce((sum, chunk) => sum + chunk.seqLen, 0); + const merged = new Float32Array(1 * heads * concatSeqLen * frames); + let seqOffset = 0; + + for (const chunk of chunks) { + const { data, seqLen, frames: tensorFrames } = chunk; + const copyFrames = Math.min(frames, tensorFrames); + + for (let h = 0; h < heads; h += 1) { + for (let s = 0; s < seqLen; s += 1) { + for (let f = 0; f < copyFrames; f += 1) { + const src = (((h * seqLen) + s) * tensorFrames) + f; + const dst = (((h * concatSeqLen) + (seqOffset + s)) * frames) + f; + merged[dst] = data[src] ?? 0; + } + } + } + seqOffset += seqLen; + } + + crossAttentions[`cross_attentions.${layer}`] = new ort.Tensor('float32', merged, [1, heads, concatSeqLen, frames]); + } + + const tokenStartTimestamps = extractTokenStartTimestamps({ + crossAttentions, + decoderLayers: WHISPER_NUM_LAYERS, + alignmentHeads: remappedAlignmentHeads, + numFrames, + numInputIds: promptTokens.length, + timePrecision: 0.02, + sequenceLength: generated.length, + }); + + const timedWords = buildWordsFromTimestampedTokens({ + tokens: generated, + tokenStartTimestamps, + tokenizer: runtime.tokenizer, + eosTokenId: runtime.eosTokenId, + promptLength: promptTokens.length, + timestampBeginTokenId: runtime.timestampBeginTokenId, + timePrecision: 0.02, + language: parseLanguageCode(opts.lang) ?? 'english', + }); + + const maxSec = Math.max(0, numFrames * 0.02); + return timedWords.map((word) => ({ + word: word.word, + start: Math.min(maxSec, Math.max(0, word.startSec)), + end: Math.min(maxSec, Math.max(0, word.endSec)), + })); + } finally { + disposeTensor(mel); + if (outputs?.logits) disposeTensor(outputs.logits); + if (outputs) { + for (const [name, tensor] of Object.entries(outputs)) { + if (name.startsWith('cross_attentions.')) { + disposeTensor(tensor); + } + } + } + disposeTensorMap(crossAttentions); + disposeTensorMap(decoderPast); + disposeTensorMap(encoderPast); + disposeTensor(encoderHidden); + } +} + +export async function alignAudioWithText( + audioBuffer: TTSAudioBuffer, + text: string, + cacheKey?: string, + opts: WhisperAlignmentOptions = {}, +): Promise { + if (!text.trim()) return []; + + if (cacheKey && alignmentCache.has(cacheKey)) { + const cached = alignmentCache.get(cacheKey)!; + alignmentCache.delete(cacheKey); + alignmentCache.set(cacheKey, cached); + return cached; + } + + if (cacheKey) { + const inFlight = alignmentInFlight.get(cacheKey); + if (inFlight) return inFlight; + } + const inFlightKey = cacheKey ?? makeInFlightCoalesceKey(audioBuffer, text, opts.lang); + const shared = alignmentInFlight.get(inFlightKey); + if (shared) return shared; + + state.pendingAlignments += 1; + const run = (async (): Promise => { + const deadlineMs = Date.now() + ALIGNMENT_TIMEOUT_MS; + const previous = state.alignMutex; + let release!: () => void; + state.alignMutex = new Promise((resolve) => { + release = resolve; + }); + + await previous; + + // Another request with the same cache key may have completed while this one + // was waiting on the mutex. + if (cacheKey && alignmentCache.has(cacheKey)) { + const cached = alignmentCache.get(cacheKey)!; + alignmentCache.delete(cacheKey); + alignmentCache.set(cacheKey, cached); + release(); + return cached; + } + + let tmpBase = ''; + let inputPath = ''; + let pcmPath = ''; + + try { + tmpBase = await mkdtemp(join(tmpdir(), 'openreader-whisper-')); + inputPath = join(tmpBase, `${randomUUID()}-input.bin`); + pcmPath = join(tmpBase, `${randomUUID()}-input.pcm16`); + + await writeFile(inputPath, Buffer.from(new Uint8Array(audioBuffer))); + await decodeToPcm16(inputPath, pcmPath); + + const pcmBytes = await readFile(pcmPath); + const decodedSamples = pcm16ToFloat32(pcmBytes); + const effectiveSampleLength = Math.min(decodedSamples.length, N_SAMPLES); + const effectiveFrameCount = Math.max(1, Math.floor((effectiveSampleLength / HOP_LENGTH) / 2)); + const normalizedAudio = padOrTrimAudio(decodedSamples); + + const words = await runWhisperOnnx( + normalizedAudio, + { ...opts, textHint: text }, + effectiveFrameCount, + deadlineMs, + ); + const alignment = mapWordsToSentenceOffsets(text, words); + const result: TTSSentenceAlignment[] = [alignment]; + + if (cacheKey) { + if (alignmentCache.has(cacheKey)) { + alignmentCache.delete(cacheKey); + } + alignmentCache.set(cacheKey, result); + while (alignmentCache.size > ALIGNMENT_CACHE_MAX_ENTRIES) { + const oldest = alignmentCache.keys().next().value; + if (!oldest) break; + alignmentCache.delete(oldest); + } + } + + return result; + } finally { + if (tmpBase) { + await rm(tmpBase, { recursive: true, force: true }).catch(() => {}); + } + release(); + state.pendingAlignments = Math.max(0, state.pendingAlignments - 1); + } + })(); + + alignmentInFlight.set(inFlightKey, run); + run.finally(() => { + if (alignmentInFlight.get(inFlightKey) === run) { + alignmentInFlight.delete(inFlightKey); + } + }); + return run; +} + +export function makeWhisperCacheKey(input: WhisperRequestBody): string { + return createHash('sha256') + .update( + JSON.stringify({ + text: input.text, + lang: input.lang || '', + audioLen: input.audio?.length || 0, + }), + ) + .digest('hex'); +} diff --git a/compute/core/src/whisper/ensureModel.ts b/compute/core/src/whisper/ensureModel.ts new file mode 100644 index 0000000..4d2a05c --- /dev/null +++ b/compute/core/src/whisper/ensureModel.ts @@ -0,0 +1,242 @@ +import path from 'path'; +import { fileURLToPath } from 'url'; +import { createHash } from 'crypto'; +import { access, copyFile, mkdir, readFile, rename, unlink, writeFile } from 'fs/promises'; +import { DOCSTORE_DIR } from '../runtime/docstore'; +import manifest from './model/manifest.json'; + +const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); +const MODEL_DIR = path.join(DOCSTORE_DIR, 'model', 'whisper-base_timestamped'); +const STATIC_LICENSE_PATH = path.join(MODULE_DIR, 'model', 'LICENSE.txt'); + +export const WHISPER_CONFIG_PATH = path.join(MODEL_DIR, 'config.json'); +export const WHISPER_GENERATION_CONFIG_PATH = path.join(MODEL_DIR, 'generation_config.json'); +export const WHISPER_TOKENIZER_PATH = path.join(MODEL_DIR, 'tokenizer.json'); +export const WHISPER_TOKENIZER_CONFIG_PATH = path.join(MODEL_DIR, 'tokenizer_config.json'); +export const WHISPER_ENCODER_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'encoder_model_int8.onnx'); +export const WHISPER_DECODER_MERGED_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'decoder_model_merged_int8.onnx'); +export const WHISPER_DECODER_WITH_PAST_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'decoder_with_past_model_int8.onnx'); + +const BASE_MODEL_URL = 'https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main'; +const WHISPER_MODEL_BASE_URL_ENV = 'WHISPER_MODEL_BASE_URL'; + +const MODEL_RELATIVE_PATHS: string[] = [ + 'config.json', + 'generation_config.json', + 'tokenizer.json', + 'tokenizer_config.json', + 'merges.txt', + 'vocab.json', + 'normalizer.json', + 'added_tokens.json', + 'preprocessor_config.json', + 'special_tokens_map.json', + 'onnx/encoder_model_int8.onnx', + 'onnx/decoder_model_merged_int8.onnx', + 'onnx/decoder_with_past_model_int8.onnx', +]; + +const DEFAULT_URLS: Record = { + 'config.json': `${BASE_MODEL_URL}/config.json`, + 'generation_config.json': `${BASE_MODEL_URL}/generation_config.json`, + 'tokenizer.json': `${BASE_MODEL_URL}/tokenizer.json`, + 'tokenizer_config.json': `${BASE_MODEL_URL}/tokenizer_config.json`, + 'merges.txt': `${BASE_MODEL_URL}/merges.txt`, + 'vocab.json': `${BASE_MODEL_URL}/vocab.json`, + 'normalizer.json': `${BASE_MODEL_URL}/normalizer.json`, + 'added_tokens.json': `${BASE_MODEL_URL}/added_tokens.json`, + 'preprocessor_config.json': `${BASE_MODEL_URL}/preprocessor_config.json`, + 'special_tokens_map.json': `${BASE_MODEL_URL}/special_tokens_map.json`, + 'onnx/encoder_model_int8.onnx': `${BASE_MODEL_URL}/onnx/encoder_model_int8.onnx`, + 'onnx/decoder_model_merged_int8.onnx': `${BASE_MODEL_URL}/onnx/decoder_model_merged_int8.onnx`, + 'onnx/decoder_with_past_model_int8.onnx': `${BASE_MODEL_URL}/onnx/decoder_with_past_model_int8.onnx`, +}; + +type ManifestEntry = { path: string; sha256?: string; size?: number }; + +export interface WhisperArtifactSpec { + path: string; + sha256?: string; + size?: number; + url: string; +} + +export interface WhisperStaticArtifactSpec { + path: string; + sha256?: string; + size?: number; + sourcePath: string; +} + +export type WhisperFetch = (input: RequestInfo | URL, init?: RequestInit) => Promise; + +const MANIFEST_FILES = manifest.files as ManifestEntry[]; +const MODEL_FILES = MANIFEST_FILES.filter((entry) => entry.path !== 'LICENSE.txt'); +const LICENSE_FILE = MANIFEST_FILES.find((entry) => entry.path === 'LICENSE.txt'); + +function normalizeExpected(entry: { sha256?: string; size?: number }): { sha256: string | null; size: number } { + return { + sha256: typeof entry.sha256 === 'string' ? entry.sha256.toLowerCase() : null, + size: Number(entry.size ?? 0), + }; +} + +function resolvePath(relativePath: string, modelDir: string): string { + return path.join(modelDir, relativePath); +} + +function joinModelUrl(baseUrl: string, relativePath: string): string { + return `${baseUrl.replace(/\/+$/, '')}/${relativePath}`; +} + +function resolveUrl(relativePath: string): string { + const overrideBase = process.env[WHISPER_MODEL_BASE_URL_ENV]?.trim(); + if (overrideBase) { + return joinModelUrl(overrideBase, relativePath); + } + const fallback = DEFAULT_URLS[relativePath]; + if (!fallback) { + throw new Error(`No default URL configured for Whisper model artifact: ${relativePath}`); + } + return fallback; +} + +function sha256OfBytes(bytes: Uint8Array): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function verifyBytes(bytes: Uint8Array, expected: { sha256?: string; size?: number }): boolean { + const normalized = normalizeExpected(expected); + if (Number.isFinite(normalized.size) && normalized.size > 0 && bytes.byteLength !== normalized.size) { + return false; + } + if (!normalized.sha256) return true; + return sha256OfBytes(bytes) === normalized.sha256; +} + +async function verifyFile(filePath: string, expected: { sha256?: string; size?: number }): Promise { + const bytes = await readFile(filePath); + return verifyBytes(bytes, expected); +} + +async function downloadToFile(fetchImpl: WhisperFetch, url: string, outPath: string): Promise { + const res = await fetchImpl(url); + if (!res.ok) { + throw new Error(`Download failed for ${url}: ${res.status} ${res.statusText}`); + } + const bytes = new Uint8Array(await res.arrayBuffer()); + await writeFile(outPath, bytes); +} + +export async function ensureWhisperArtifacts(options: { + modelDir: string; + artifacts: WhisperArtifactSpec[]; + staticArtifacts?: WhisperStaticArtifactSpec[]; + fetchImpl?: WhisperFetch; +}): Promise { + const { + modelDir, + artifacts, + staticArtifacts = [], + fetchImpl = fetch, + } = options; + + try { + await Promise.all(artifacts.map(async (artifact) => { + const target = resolvePath(artifact.path, modelDir); + await access(target); + const valid = await verifyFile(target, artifact); + if (!valid) { + throw new Error(`Checksum mismatch for existing Whisper artifact: ${artifact.path}`); + } + })); + + await Promise.all(staticArtifacts.map(async (artifact) => { + const target = resolvePath(artifact.path, modelDir); + await access(target); + const valid = await verifyFile(target, artifact); + if (!valid) { + throw new Error(`Checksum mismatch for existing Whisper static artifact: ${artifact.path}`); + } + })); + + return; + } catch { + // Continue to repair/download. + } + + for (const artifact of artifacts) { + const target = resolvePath(artifact.path, modelDir); + const targetDir = path.dirname(target); + const tmp = `${target}.tmp`; + + await mkdir(targetDir, { recursive: true }); + await downloadToFile(fetchImpl, artifact.url, tmp); + if (!(await verifyFile(tmp, artifact))) { + await unlink(tmp).catch(() => undefined); + throw new Error(`Whisper artifact checksum verification failed: ${artifact.path}`); + } + await rename(tmp, target); + } + + for (const artifact of staticArtifacts) { + const target = resolvePath(artifact.path, modelDir); + const targetDir = path.dirname(target); + await mkdir(targetDir, { recursive: true }); + await copyFile(artifact.sourcePath, target); + if (!(await verifyFile(target, artifact))) { + throw new Error(`Whisper static artifact checksum verification failed: ${artifact.path}`); + } + } +} + +export function createSingleflightRunner(work: () => Promise): () => Promise { + let inflight: Promise | null = null; + return async () => { + if (inflight) return inflight; + inflight = work().finally(() => { + inflight = null; + }); + return inflight; + }; +} + +async function ensureModelInternal(): Promise { + if (process.env[WHISPER_MODEL_BASE_URL_ENV]?.trim()) { + for (const relativePath of MODEL_RELATIVE_PATHS) { + if (!(relativePath in DEFAULT_URLS)) { + throw new Error(`Missing default URL path mapping for Whisper artifact: ${relativePath}`); + } + } + } + + const artifacts: WhisperArtifactSpec[] = MODEL_FILES.map((entry) => ({ + path: entry.path, + sha256: entry.sha256, + size: entry.size, + url: resolveUrl(entry.path), + })); + + const staticArtifacts: WhisperStaticArtifactSpec[] = LICENSE_FILE + ? [{ + path: LICENSE_FILE.path, + sha256: LICENSE_FILE.sha256, + size: LICENSE_FILE.size, + sourcePath: STATIC_LICENSE_PATH, + }] + : []; + + await ensureWhisperArtifacts({ + modelDir: MODEL_DIR, + artifacts, + staticArtifacts, + }); + + return WHISPER_ENCODER_MODEL_PATH; +} + +const ensureWhisperModelSingleflight = createSingleflightRunner(ensureModelInternal); + +export async function ensureWhisperModel(): Promise { + return ensureWhisperModelSingleflight(); +} diff --git a/compute/core/src/whisper/model/LICENSE.txt b/compute/core/src/whisper/model/LICENSE.txt new file mode 100644 index 0000000..d255525 --- /dev/null +++ b/compute/core/src/whisper/model/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 OpenAI + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/compute/core/src/whisper/model/manifest.json b/compute/core/src/whisper/model/manifest.json new file mode 100644 index 0000000..2fffd3d --- /dev/null +++ b/compute/core/src/whisper/model/manifest.json @@ -0,0 +1,76 @@ +{ + "name": "whisper-base_timestamped-int8", + "version": "onnx-community/whisper-base_timestamped@608c49e61301901684bc36cac8f74b95ff6b5a8e", + "files": [ + { + "path": "config.json", + "sha256": "f4d0608f7d918166da7edb3e188de5ef1bfe70d9802e785d271fd88111e9cf4b", + "size": 2243 + }, + { + "path": "generation_config.json", + "sha256": "61070cf8de25b1e9256e8e102ded49d8d24a8369ed36ef84fdf21549e68125a0", + "size": 3832 + }, + { + "path": "tokenizer.json", + "sha256": "27fc476bfe7f17299480be2273fc0608e4d5a99aba2ab5dec5374b4482d1a566", + "size": 2480466 + }, + { + "path": "tokenizer_config.json", + "sha256": "2e036e4dbacfdeb7242c7d4ec4149f4a16e86026048f94d1637e3a8ee9c6a573", + "size": 282682 + }, + { + "path": "merges.txt", + "sha256": "2df2990a395e35e8dfbc7511e08c12d56018d8d04691e0133e5d63b21e154dc6", + "size": 493869 + }, + { + "path": "vocab.json", + "sha256": "50d6a919f0a0601d56a04eb583c780d18553aa388254ba3158eb6a00f13e2c1a", + "size": 1036584 + }, + { + "path": "normalizer.json", + "sha256": "bf1c507dc8724ca9cf9903640dacfb69dae2f00edee4f21ceba106a7392f26dd", + "size": 52666 + }, + { + "path": "added_tokens.json", + "sha256": "9715fd2243b6f06a5858b5e32950d2853f73dd5bc201aafcf76f5082a2d8acd1", + "size": 34604 + }, + { + "path": "preprocessor_config.json", + "sha256": "a6a76d28c93edb273669eb9e0b0636a2bddbb1272c3261e47b7ca6dfdbac1b8d", + "size": 339 + }, + { + "path": "special_tokens_map.json", + "sha256": "e67ae3a0aaa99abcd9f187138e12db1f65c16a14761c50ef10eef2c174a7a691", + "size": 2194 + }, + { + "path": "onnx/encoder_model_int8.onnx", + "sha256": "152da96dd8ff3f28f3fadabc2e8960405a277846453ff94ed411fe935a72917f", + "size": 23159150 + }, + { + "path": "onnx/decoder_model_merged_int8.onnx", + "sha256": "cf9a8d5bcddc0917a0078135b484cedcaf44f28909cd91910abd29dced9171db", + "size": 53712708 + }, + { + "path": "onnx/decoder_with_past_model_int8.onnx", + "sha256": "bdd92860d0ed7dff2aca623963378cbba1b617bfae127356db1c8aa8baa930ef", + "size": 50131672 + }, + { + "path": "LICENSE.txt", + "sha256": "b5d65a59060e68c4ff940e1eddfa6f94b2d68fdf58ed7f4dd57721c997e35e9d", + "size": 1063 + } + ] +} diff --git a/compute/core/src/whisper/model/mel_filters.npz b/compute/core/src/whisper/model/mel_filters.npz new file mode 100644 index 0000000000000000000000000000000000000000..28ea26909dbdfd608aef67afc4d74d7961ae4bb6 GIT binary patch literal 4271 zcmZ`-cQjmYw;lx1g6JcN7QKe3LG%_Oh!VX=^k~teM-XGQ(Mu4$_Y%?jkm$lFBkB+( z3yfKIgF zxGiAhze`A@t->QRNVV!%P+W=o}VHkB) z%g>qyRHfN1IQ4-=`Y@0T9qE#o+;4E3VQ!epW1Xt=ZG`I3U|62t?<>5h*W|9VvJc`KZ+)ghnA**Z~ET21Tjf_f8oe`vy zZQNtlOx?dDhS71hnOus5cqj)hfyF@H&4y?@9z{I#&cf>A+s2~~(I>TQF}SaR3_tqa z(7&ZdN^vR*t<~?{9DEoI>0PL@Sl?wa?Z{rGX`*eEx9Nh=z*J3HZL1*Py4z$TD#+;m zSSW(kcOTe(4hqgib_W6&xx+j~-u(p)Nn6?>a%wHk=h7Ay$%lcGoo;gAY zmVV7|!Nb;w(PlH@c24{ple2Y3<*9J@jE=sfLzwu_BiAFPE$0Axp`^Nq!H}eG0?r-X zFj@Pwp^al*p>K{@_Cz`q#(N0Y=OpZy^ z{P$KjLJuk_Y%I)$mh`b{uOW5C5Xcmxk!gt_Zg zw>}6fkD4zRK9!#ems~H%U$>V;_wK38Zf-baU$S!#i;7!HWsi}GuC>%@?lMdgkUGC& zh9gC?O-5BlS2#}?7x0?eP#bOL(cqE{M%LJD$CZnplD)CgQR#KCttD=dZK+Ck5R52; z*%5hZ+SXU7)8k%Y^_1U>yI*By(INn&+ir-_4$#dUwTlMNyR@iGQIaZ+eiYqucu)CB z#i{Ru1w+aU#}DHSyzjG_9c?ToB_YjU#f;N=qel98WBIjIc1!#ePwRR+(go&-by#}@ z+M+klVke5b@lWfZ+O&|c??YvRe)&W)qAgtc>t-IZtbRTG#X}49_Q$>P%-)=0W_QY-x%DPep2Vm9#ci zyQcCc4p2&dLtV1@rPe!%>Y^#9W8#ZH&}^@wJKT7N;R9A7cEq&;Y2CYvd@R+Mn&b5O zVyfS^*H#kD74=J5uhD)o`TXoX>>Si$!cT?TXRxj2pB)w_ljjhTby&Je;X|BESZZT= zC%G5!-$BJf&a~U78d_3zBjrvrkJ0CCl@Rfcf7I(`VTNPnI^B#B$zOfPW zG&mEd?R0+W<`l08O1dkcWKS8wB!Z*Cs%I1nMs-EeB-uu5?t@PuD3|z>je8DKi#X(B z{Z=Rz{4X%?-UnxnHQtkELIZ&=J;fK_t}yu8|IxG0(85e&K>H3!!~zlhyJrgti~o1i zzBS*jTgdG~Exp#B-T)6A+PB ztD-e`j^@XAx}|L&JSEFkRvS_%3b%m86z02#Hfn{Y+qIqQ_muywgt?roUA7oiS1xBD zFxmDMsj_cbBcn*^rn^KIMP{AlHM`NiVm*D&`z~7FH#hf<$L3HmJ+=NdiY5>W?nKD? z8Ox6{9dKyI1o8a-j9BtV-|=lm`<`v>tR^Cln&x1dMYzu{@wq5KW!#K14_QMnpH5K%Pavag+g6(i8i-#Eq zguc}rH3?BxH4SOqZW#7m*aT(U9-n#_Xn^Q19(}eH!xG`nI!GYziVQNcA0)`FDHD%~ zz2$HnxW4BQ{#*@u`dssbAa`|fESn$8i8FdxGZh48_Uf~_Q@tv?4in)6fwSed)k&ITqu|){^(WL~J z?Lb|0ro06J^>f>^2}^e-+$u5bU4IZNfO?75v8lstS15%XYw2ac^pkU34{QhDR(umt zPu~`w2?FP|nn3!RWZ3{?=77@teulahD9*S*k5KmY3*adlM)%{SR~bkZYlx1q@fkE= zI$7+kiw5!ha=dYlO>Z5KgxnZEJsaBm%v#nkX0MN-h%n&KA?N}xU3K3o-3Jpk?ANq2n9&Lh%K_CTvfiN ze>6w~NSSl8$#NEZ^t7h9YOxI=zcAG|a+m6AWei`3Jw7K;b;T${pJa^4RwRt%F>?>M zBmoQqm1`<_W7i!5P~THp-II)Ka^u;=z;}d{;SVj{G_4`9^HaEb!=@Pa;Dw)CH^DjsGxFqmb%o$Bkop$KnH8 zDYN)Bh)5=5!-*|f0Gh4)oZG=TEBr()g^DCtSQhmT3!ZN`Qd-E%@1cE}hm8&Vq5B+C zVF2_O)9IiZ(v(xzTwJIg5|}KVuE(;}|7dVIrT`$d=q_OG|3PY}x*URYkMXXJ6PT1$IFkNyvY_(9UglDi6TaeikPS(!Bnij z;Szn+)I_oxnRz7(WTYTp+IHSWQ?Xd~tQn(Q1r)kThM?NM< z?d6LaBG!H}R$zRy!Ij(}1?xe^+o+!;tqWJ3NgjHl1XNxzusxQ0I#6qzM(_00UPMw* zF*GWW_q&fqAN=uimSKgBu_@jD%MX3hpNY|*4r=e=k1lw2r**IyD(hcq?A+HtUgUy4Dqh5D7|G9q{)TsUj{g~c!xy>9wk^(LiXA4VKGz_zMvJMX#AgsR z34T3hhJ)#&sUaQ1+0PML(?YA~{5?=(MT}X^Vib%};uoI{qGW@wgJ&_M+8S8clsNz2 zPQkxMi`#3+Khwtl>>K>wxc{71{&!qGu&Zzz_wU(7TLTyG){PAu?!cXs?Dp-y0Ekcn AQvd(} literal 0 HcmV?d00001 diff --git a/compute/core/src/whisper/spectral.ts b/compute/core/src/whisper/spectral.ts new file mode 100644 index 0000000..b7223cc --- /dev/null +++ b/compute/core/src/whisper/spectral.ts @@ -0,0 +1,21 @@ +export function buildGoertzelCoefficients(freqBins: number, fftSize: number): Float64Array { + const coeffs = new Float64Array(freqBins); + for (let k = 0; k < freqBins; k += 1) { + coeffs[k] = 2 * Math.cos((2 * Math.PI * k) / fftSize); + } + return coeffs; +} + +export function goertzelPower(samples: Float32Array, coeff: number): number { + let s1 = 0; + let s2 = 0; + for (let i = 0; i < samples.length; i += 1) { + const s0 = samples[i] + (coeff * s1) - s2; + s2 = s1; + s1 = s0; + } + + const power = (s1 * s1) + (s2 * s2) - (coeff * s1 * s2); + if (!Number.isFinite(power) || power < 0) return 0; + return power; +} diff --git a/compute/core/src/whisper/token-timestamps.ts b/compute/core/src/whisper/token-timestamps.ts new file mode 100644 index 0000000..47edc1d --- /dev/null +++ b/compute/core/src/whisper/token-timestamps.ts @@ -0,0 +1,449 @@ +import type { Tokenizer } from '@huggingface/tokenizers'; +import type * as ort from 'onnxruntime-node'; + +const PUNCTUATION_REGEX = '\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E'; +const PUNCTUATION_ONLY_REGEX = new RegExp(`^[${PUNCTUATION_REGEX}]+$`, 'gu'); + +type TokenTimestamp = [start: number, end: number]; + +export interface WhisperWordTiming { + word: string; + startSec: number; + endSec: number; +} + +function medianFilter(data: Float32Array, windowSize: number): Float32Array { + if (windowSize % 2 === 0 || windowSize <= 0) { + throw new Error('Window size must be a positive odd number'); + } + + const output = new Float32Array(data.length); + const buffer = new Float32Array(windowSize); + const halfWindow = Math.floor(windowSize / 2); + + for (let i = 0; i < data.length; i += 1) { + let valuesIndex = 0; + for (let j = -halfWindow; j <= halfWindow; j += 1) { + let index = i + j; + if (index < 0) { + index = Math.abs(index); + } else if (index >= data.length) { + index = (2 * (data.length - 1)) - index; + } + buffer[valuesIndex] = data[index]; + valuesIndex += 1; + } + + const sortable = Array.from(buffer); + sortable.sort((a, b) => a - b); + output[i] = sortable[halfWindow] ?? 0; + } + + return output; +} + +function dynamicTimeWarping(matrix: Float32Array[], rows: number, cols: number): [number[], number[]] { + const cost: number[][] = Array.from({ length: rows + 1 }, () => Array(cols + 1).fill(Number.POSITIVE_INFINITY)); + const trace: number[][] = Array.from({ length: rows + 1 }, () => Array(cols + 1).fill(-1)); + cost[0][0] = 0; + + for (let j = 1; j <= cols; j += 1) { + for (let i = 1; i <= rows; i += 1) { + const c0 = cost[i - 1][j - 1]; + const c1 = cost[i - 1][j]; + const c2 = cost[i][j - 1]; + let c: number; + let t: number; + if (c0 < c1 && c0 < c2) { + c = c0; + t = 0; + } else if (c1 < c0 && c1 < c2) { + c = c1; + t = 1; + } else { + c = c2; + t = 2; + } + cost[i][j] = matrix[i - 1][j - 1] + c; + trace[i][j] = t; + } + } + + for (let i = 0; i <= cols; i += 1) trace[0][i] = 2; + for (let i = 0; i <= rows; i += 1) trace[i][0] = 1; + + let i = rows; + let j = cols; + const textIndices: number[] = []; + const timeIndices: number[] = []; + while (i > 0 || j > 0) { + textIndices.push(i - 1); + timeIndices.push(j - 1); + const step = trace[i][j]; + if (step === 0) { + i -= 1; + j -= 1; + } else if (step === 1) { + i -= 1; + } else if (step === 2) { + j -= 1; + } else { + throw new Error(`Unexpected DTW trace state at [${i}, ${j}]`); + } + } + + textIndices.reverse(); + timeIndices.reverse(); + return [textIndices, timeIndices]; +} + +function round2(value: number): number { + return Math.round(value * 100) / 100; +} + +function decodeTokens(tokenizer: Pick, tokens: number[]): string { + return tokenizer.decode(tokens, { skip_special_tokens: false }); +} + +function splitTokensOnUnicode( + tokenizer: Pick, + tokens: number[], +): [string[], number[][], number[][]] { + const decodedFull = decodeTokens(tokenizer, tokens); + const replacementChar = '\uFFFD'; + const words: string[] = []; + const wordTokens: number[][] = []; + const tokenIndices: number[][] = []; + let currentTokens: number[] = []; + let currentIndices: number[] = []; + let unicodeOffset = 0; + + for (let i = 0; i < tokens.length; i += 1) { + currentTokens.push(tokens[i]); + currentIndices.push(i); + + const decoded = decodeTokens(tokenizer, currentTokens); + if ( + !decoded.includes(replacementChar) + || decodedFull[unicodeOffset + decoded.indexOf(replacementChar)] === replacementChar + ) { + words.push(decoded); + wordTokens.push(currentTokens); + tokenIndices.push(currentIndices); + currentTokens = []; + currentIndices = []; + unicodeOffset += decoded.length; + } + } + + return [words, wordTokens, tokenIndices]; +} + +function splitTokensOnSpaces( + tokenizer: Pick, + tokens: number[], + eosTokenId: number, +): [string[], number[][], number[][]] { + const [subwords, subwordTokens, subwordIndices] = splitTokensOnUnicode(tokenizer, tokens); + const words: string[] = []; + const wordTokens: number[][] = []; + const tokenIndices: number[][] = []; + + for (let i = 0; i < subwords.length; i += 1) { + const subword = subwords[i]; + const tokenList = subwordTokens[i]; + const indices = subwordIndices[i]; + const special = tokenList[0] >= eosTokenId; + const withSpace = subword.startsWith(' '); + const trimmed = subword.trim(); + const punctuation = PUNCTUATION_ONLY_REGEX.test(trimmed); + + if (special || withSpace || punctuation || words.length === 0) { + words.push(subword); + wordTokens.push([...tokenList]); + tokenIndices.push([...indices]); + } else { + const ix = words.length - 1; + words[ix] += subword; + wordTokens[ix].push(...tokenList); + tokenIndices[ix].push(...indices); + } + } + + return [words, wordTokens, tokenIndices]; +} + +function mergePunctuations( + words: string[], + tokens: number[][], + indices: number[][], + prependPunctuations = '"\'“¡¿([{-', + appendPunctuations = '"\'.。,，!！?？:：”)]}、', +): [string[], number[][], number[][]] { + const newWords = words.map((w) => `${w}`); + const newTokens = tokens.map((t) => [...t]); + const newIndices = indices.map((idx) => [...idx]); + + let i = newWords.length - 2; + let j = newWords.length - 1; + while (i >= 0) { + if (newWords[i].startsWith(' ') && prependPunctuations.includes(newWords[i].trim())) { + newWords[j] = newWords[i] + newWords[j]; + newTokens[j] = [...newTokens[i], ...newTokens[j]]; + newIndices[j] = [...newIndices[i], ...newIndices[j]]; + newWords[i] = ''; + newTokens[i] = []; + newIndices[i] = []; + } else { + j = i; + } + i -= 1; + } + + i = 0; + j = 1; + while (j < newWords.length) { + if (!newWords[i].endsWith(' ') && appendPunctuations.includes(newWords[j])) { + newWords[i] += newWords[j]; + newTokens[i] = [...newTokens[i], ...newTokens[j]]; + newIndices[i] = [...newIndices[i], ...newIndices[j]]; + newWords[j] = ''; + newTokens[j] = []; + newIndices[j] = []; + } else { + i = j; + } + j += 1; + } + + return [ + newWords.filter((w) => w.length > 0), + newTokens.filter((t) => t.length > 0), + newIndices.filter((t) => t.length > 0), + ]; +} + +function combineTokensIntoWords( + tokenizer: Pick, + tokens: number[], + eosTokenId: number, + language = 'english', +): [string[], number[][], number[][]] { + let words: string[]; + let wordTokens: number[][]; + let tokenIndices: number[][]; + + if (['chinese', 'japanese', 'thai', 'lao', 'myanmar', 'zh', 'ja', 'th', 'lo', 'my'].includes(language)) { + [words, wordTokens, tokenIndices] = splitTokensOnUnicode(tokenizer, tokens); + } else { + [words, wordTokens, tokenIndices] = splitTokensOnSpaces(tokenizer, tokens, eosTokenId); + } + + return mergePunctuations(words, wordTokens, tokenIndices); +} + +export function extractTokenStartTimestamps(input: { + crossAttentions: Record; + decoderLayers: number; + alignmentHeads: Array<[number, number]>; + numFrames: number; + numInputIds: number; + timePrecision?: number; + sequenceLength: number; +}): number[] { + const { + crossAttentions, + decoderLayers, + alignmentHeads, + numFrames, + numInputIds, + timePrecision = 0.02, + sequenceLength, + } = input; + + const frameCount = Math.max(1, numFrames); + const perLayer: Float32Array[] = []; + for (let layer = 0; layer < decoderLayers; layer += 1) { + const key = `cross_attentions.${layer}`; + const tensor = crossAttentions[key]; + if (!tensor) continue; + perLayer[layer] = tensor.data as Float32Array; + } + + const selected: Float32Array[] = []; + let seqLen = 0; + let attnFrames = 0; + for (const [layer, head] of alignmentHeads) { + const flat = perLayer[layer]; + if (!flat) continue; + const layerTensor = crossAttentions[`cross_attentions.${layer}`]; + if (!layerTensor || layerTensor.dims.length < 4) continue; + const [, numHeads, currentSeqLen, currentFrames] = layerTensor.dims; + if (head >= numHeads) continue; + seqLen = currentSeqLen; + attnFrames = Math.min(currentFrames, frameCount); + const headSlice = new Float32Array(seqLen * attnFrames); + for (let s = 0; s < seqLen; s += 1) { + for (let f = 0; f < attnFrames; f += 1) { + const flatIndex = (((head * currentSeqLen) + s) * currentFrames) + f; + headSlice[(s * attnFrames) + f] = flat[flatIndex] ?? 0; + } + } + selected.push(headSlice); + } + + if (!selected.length || seqLen === 0 || attnFrames === 0) { + return new Array(sequenceLength).fill(0); + } + + const normalizedHeads = selected.map((headData) => { + const means = new Float32Array(attnFrames); + const stds = new Float32Array(attnFrames); + + for (let f = 0; f < attnFrames; f += 1) { + let sum = 0; + for (let s = 0; s < seqLen; s += 1) sum += headData[(s * attnFrames) + f]; + const mean = sum / seqLen; + means[f] = mean; + let varSum = 0; + for (let s = 0; s < seqLen; s += 1) { + const d = headData[(s * attnFrames) + f] - mean; + varSum += d * d; + } + stds[f] = Math.sqrt(varSum / seqLen) || 1; + } + + const out = new Float32Array(headData.length); + for (let s = 0; s < seqLen; s += 1) { + const row = new Float32Array(attnFrames); + for (let f = 0; f < attnFrames; f += 1) { + row[f] = (headData[(s * attnFrames) + f] - means[f]) / stds[f]; + } + const filtered = medianFilter(row, 7); + out.set(filtered, s * attnFrames); + } + return out; + }); + + const croppedRows = Math.max(0, seqLen - numInputIds); + if (croppedRows === 0) return new Array(sequenceLength).fill(0); + + const matrix: Float32Array[] = Array.from({ length: croppedRows }, () => new Float32Array(attnFrames)); + for (const headData of normalizedHeads) { + for (let r = 0; r < croppedRows; r += 1) { + const srcRow = r + numInputIds; + for (let f = 0; f < attnFrames; f += 1) { + matrix[r][f] += headData[(srcRow * attnFrames) + f]; + } + } + } + + const scale = 1 / normalizedHeads.length; + for (let r = 0; r < croppedRows; r += 1) { + for (let f = 0; f < attnFrames; f += 1) { + matrix[r][f] = -matrix[r][f] * scale; + } + } + + const [textIndices, timeIndices] = dynamicTimeWarping(matrix, croppedRows, attnFrames); + const jumps = new Array(textIndices.length).fill(false); + for (let i = 0; i < textIndices.length; i += 1) { + jumps[i] = i === 0 ? true : textIndices[i] !== textIndices[i - 1]; + } + + const jumpTimes: number[] = []; + for (let i = 0; i < jumps.length; i += 1) { + if (jumps[i]) jumpTimes.push(timeIndices[i] * timePrecision); + } + + const timestamps = new Array(sequenceLength).fill(0); + for (let i = 0; i < numInputIds && i < timestamps.length; i += 1) timestamps[i] = 0; + for (let i = 0; i < jumpTimes.length && (numInputIds + i) < timestamps.length; i += 1) { + timestamps[numInputIds + i] = jumpTimes[i]; + } + if (timestamps.length > 0 && jumpTimes.length > 0) { + timestamps[timestamps.length - 1] = jumpTimes[jumpTimes.length - 1]; + } + return timestamps; +} + +export function buildWordsFromTimestampedTokens(input: { + tokens: number[]; + tokenStartTimestamps: number[]; + tokenizer: Pick; + eosTokenId: number; + promptLength: number; + timestampBeginTokenId: number; + timePrecision?: number; + language?: string; +}): WhisperWordTiming[] { + const { + tokens, + tokenStartTimestamps, + tokenizer, + eosTokenId, + promptLength, + timestampBeginTokenId, + timePrecision = 0.02, + language = 'english', + } = input; + + const limit = Math.min(tokens.length, tokenStartTimestamps.length); + const tokenRanges: TokenTimestamp[] = []; + for (let i = 0; i < limit; i += 1) { + const start = tokenStartTimestamps[i] ?? 0; + const end = i + 1 < limit ? (tokenStartTimestamps[i + 1] ?? (start + timePrecision)) : (start + timePrecision); + tokenRanges.push([start, Math.max(start, end)]); + } + + const words: WhisperWordTiming[] = []; + let segmentStart: number | null = null; + let textTokens: number[] = []; + let textRanges: TokenTimestamp[] = []; + + const flushSegment = (segmentEnd: number | null) => { + if (!textTokens.length) return; + const [wordTexts, , tokenIndices] = combineTokensIntoWords(tokenizer, textTokens, eosTokenId, language); + for (let i = 0; i < wordTexts.length; i += 1) { + const indices = tokenIndices[i]; + if (!indices.length) continue; + const start = textRanges[indices[0]]?.[0] ?? segmentStart ?? 0; + const end = textRanges[indices[indices.length - 1]]?.[1] ?? segmentEnd ?? start; + const clampedStart = segmentStart == null ? start : Math.max(segmentStart, start); + const clampedEndBase = segmentEnd == null ? end : Math.min(segmentEnd, end); + const clampedEnd = Math.max( + clampedStart + (clampedEndBase <= clampedStart ? timePrecision : 0), + clampedEndBase, + ); + words.push({ + word: wordTexts[i].trim(), + startSec: round2(clampedStart), + endSec: round2(clampedEnd), + }); + } + textTokens = []; + textRanges = []; + }; + + for (let i = promptLength; i < limit; i += 1) { + const token = tokens[i]; + if (token === eosTokenId) break; + + if (token >= timestampBeginTokenId) { + const ts = (token - timestampBeginTokenId) * timePrecision; + if (segmentStart == null) { + segmentStart = ts; + } else { + flushSegment(ts); + segmentStart = ts; + } + continue; + } + + textTokens.push(token); + textRanges.push(tokenRanges[i]); + } + + flushSegment(null); + return words.filter((w) => w.word.length > 0); +} diff --git a/compute/core/tsconfig.json b/compute/core/tsconfig.json new file mode 100644 index 0000000..3e68ae5 --- /dev/null +++ b/compute/core/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "noEmit": true + }, + "include": ["src/**/*.ts"] +} diff --git a/compute/worker/.env.example b/compute/worker/.env.example new file mode 100644 index 0000000..4809e98 --- /dev/null +++ b/compute/worker/.env.example @@ -0,0 +1,25 @@ +# Compute worker bind +COMPUTE_WORKER_HOST=0.0.0.0 +COMPUTE_WORKER_PORT=8081 +COMPUTE_LOG_FORMAT=pretty +# COMPUTE_LOG_LEVEL=info + +# App <-> worker auth +COMPUTE_WORKER_TOKEN=local-compute-token + +# Redis/BullMQ +REDIS_URL=redis://redis:6379 + +# Shared object storage (must be reachable from worker) +S3_BUCKET=openreader-documents +S3_REGION=us-east-1 +S3_ACCESS_KEY_ID=devkey +S3_SECRET_ACCESS_KEY=devsecret +S3_PREFIX=openreader +# Optional for non-AWS S3-compatible endpoints: +S3_ENDPOINT=http://host.docker.internal:8333 +S3_FORCE_PATH_STYLE=true + +# Queue + execution tuning +COMPUTE_QUEUE_MAX_DEPTH=64 +COMPUTE_PREWARM_MODELS=true diff --git a/compute/worker/Dockerfile b/compute/worker/Dockerfile new file mode 100644 index 0000000..cb0c4a5 --- /dev/null +++ b/compute/worker/Dockerfile @@ -0,0 +1,20 @@ +FROM node:lts + +RUN npm install -g pnpm@11.1.2 + +WORKDIR /workspace + +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY compute/worker/package.json compute/worker/package.json +COPY compute/core/package.json compute/core/package.json + +RUN pnpm install --frozen-lockfile + +COPY . . + +ENV COMPUTE_WORKER_HOST=0.0.0.0 +ENV COMPUTE_WORKER_PORT=8081 + +EXPOSE 8081 + +CMD ["pnpm", "--filter", "@openreader/compute-worker", "dev"] diff --git a/compute/worker/docker-compose.yml b/compute/worker/docker-compose.yml new file mode 100644 index 0000000..067b2c6 --- /dev/null +++ b/compute/worker/docker-compose.yml @@ -0,0 +1,41 @@ +services: + redis: + image: redis:7-alpine + container_name: openreader-compute-redis + ports: + - "6379:6379" + + compute-worker: + build: + context: ../.. + dockerfile: compute/worker/Dockerfile + container_name: openreader-compute-worker + depends_on: + - redis + env_file: + - ./.env + environment: + REDIS_URL: ${REDIS_URL:-redis://redis:6379} + COMPUTE_WORKER_HOST: ${COMPUTE_WORKER_HOST:-0.0.0.0} + COMPUTE_WORKER_PORT: ${COMPUTE_WORKER_PORT:-8081} + COMPUTE_WORKER_TOKEN: ${COMPUTE_WORKER_TOKEN:-local-compute-token} + S3_PREFIX: ${S3_PREFIX:-openreader} + COMPUTE_PREWARM_MODELS: ${COMPUTE_PREWARM_MODELS:-true} + ports: + - "8081:8081" + develop: + watch: + - action: sync+restart + path: . + target: /workspace/compute/worker + - action: sync+restart + path: ../../compute/core + target: /workspace/compute/core + - action: rebuild + path: ./package.json + - action: rebuild + path: ../../compute/core/package.json + - action: rebuild + path: ../../pnpm-lock.yaml + - action: rebuild + path: ../../pnpm-workspace.yaml diff --git a/compute/worker/package.json b/compute/worker/package.json new file mode 100644 index 0000000..a904a29 --- /dev/null +++ b/compute/worker/package.json @@ -0,0 +1,23 @@ +{ + "name": "@openreader/compute-worker", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "tsx watch src/server.ts", + "start": "tsx src/server.ts" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.1045.0", + "@openreader/compute-core": "workspace:*", + "bullmq": "^5.61.2", + "fastify": "^5.6.2", + "ioredis": "^5.8.2", + "pino": "^9.14.0", + "pino-pretty": "^13.1.2", + "zod": "^4.1.12" + }, + "devDependencies": { + "tsx": "^4.20.6" + } +} diff --git a/compute/worker/src/server.ts b/compute/worker/src/server.ts new file mode 100644 index 0000000..8226dc3 --- /dev/null +++ b/compute/worker/src/server.ts @@ -0,0 +1,432 @@ +import Fastify, { type FastifyReply, type FastifyRequest } from 'fastify'; +import { Queue, Worker, type Job, type JobsOptions } from 'bullmq'; +import IORedis from 'ioredis'; +import { z } from 'zod'; +import { + ALIGN_QUEUE_NAME, + PDF_LAYOUT_QUEUE_NAME, + ensureComputeModels, + runPdfLayoutFromPdfBuffer, + runWhisperAlignmentFromAudioBuffer, + type PdfLayoutJobRequest, + type PdfLayoutJobResult, + type WhisperAlignJobRequest, + type WhisperAlignJobResult, + type WorkerJobStatusResponse, +} from '@openreader/compute-core'; +import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; + +function requireEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +function readIntEnv(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return fallback; + return Math.floor(parsed); +} + +function parseBoolEnv(name: string, fallback: boolean): boolean { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const normalized = raw.toLowerCase(); + return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on'; +} + +function buildLoggerConfig(): boolean | Record { + const format = (process.env.COMPUTE_LOG_FORMAT?.trim().toLowerCase() || 'pretty'); + if (format === 'json') return true; + return { + level: process.env.COMPUTE_LOG_LEVEL?.trim() || 'info', + transport: { + target: 'pino-pretty', + options: { + colorize: true, + translateTime: 'SYS:standard', + ignore: 'pid,hostname', + }, + }, + }; +} + +function normalizeS3Prefix(prefix: string | undefined): string { + const value = (prefix || 'openreader').trim(); + return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; +} + +function buildS3Client(): S3Client { + const bucket = requireEnv('S3_BUCKET'); + const region = requireEnv('S3_REGION'); + const accessKeyId = requireEnv('S3_ACCESS_KEY_ID'); + const secretAccessKey = requireEnv('S3_SECRET_ACCESS_KEY'); + const endpoint = process.env.S3_ENDPOINT?.trim() || undefined; + const forcePathStyle = parseBoolEnv('S3_FORCE_PATH_STYLE', false); + + void bucket; + + return new S3Client({ + region, + endpoint, + forcePathStyle, + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + credentials: { + accessKeyId, + secretAccessKey, + }, + }); +} + +async function bodyToBuffer(body: unknown): Promise { + if (!body) return Buffer.alloc(0); + if (body instanceof Uint8Array) return Buffer.from(body); + if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); + if (body instanceof ArrayBuffer) return Buffer.from(body); + if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { + const maybe = body as { transformToByteArray?: () => Promise }; + if (typeof maybe.transformToByteArray === 'function') { + return Buffer.from(await maybe.transformToByteArray()); + } + } + if (typeof body === 'object' && body !== null && 'on' in body) { + const stream = body as NodeJS.ReadableStream; + const chunks: Buffer[] = []; + for await (const chunk of stream) { + if (Buffer.isBuffer(chunk)) chunks.push(chunk); + else if (typeof chunk === 'string') chunks.push(Buffer.from(chunk)); + else chunks.push(Buffer.from(chunk as Uint8Array)); + } + return Buffer.concat(chunks); + } + throw new Error('Unsupported S3 response body type'); +} + +function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +} + +async function withTimeout(promise: Promise, timeoutMs: number, label: string): Promise { + let timer: NodeJS.Timeout | null = null; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +function parseRetryAfterSeconds(raw: string | undefined): number { + const parsed = Number(raw ?? ''); + if (!Number.isFinite(parsed) || parsed <= 0) return 2; + return Math.max(1, Math.floor(parsed)); +} + +function isAuthed(request: FastifyRequest, expectedToken: string): boolean { + const auth = request.headers.authorization; + if (!auth?.startsWith('Bearer ')) return false; + const token = auth.slice('Bearer '.length).trim(); + return token === expectedToken; +} + +const alignSchema = z.object({ + text: z.string().trim().min(1), + lang: z.string().trim().min(1).max(16).optional(), + cacheKey: z.string().trim().min(1).max(256).optional(), + audioObjectKey: z.string().trim().min(1).max(2048), +}); + +const layoutSchema = z.object({ + documentId: z.string().trim().min(1), + namespace: z.string().trim().min(1).max(128).nullable(), + documentObjectKey: z.string().trim().min(1).max(2048), +}); + +function mapJobState(job: Job): WorkerJobStatusResponse { + if (job.failedReason) { + return { + status: 'failed', + error: { + message: job.failedReason || 'Worker job failed', + }, + }; + } + if (typeof job.returnvalue !== 'undefined' && job.finishedOn) { + return { + status: 'succeeded', + result: job.returnvalue as Result, + }; + } + + if (job.processedOn) return { status: 'running' }; + return { status: 'queued' }; +} + +async function getQueueDepth(queue: Queue): Promise { + const counts = await queue.getJobCounts('waiting', 'active', 'prioritized', 'delayed'); + return (counts.waiting ?? 0) + (counts.active ?? 0) + (counts.prioritized ?? 0) + (counts.delayed ?? 0); +} + +async function main(): Promise { + const port = readIntEnv('COMPUTE_WORKER_PORT', 8081); + const host = process.env.COMPUTE_WORKER_HOST?.trim() || '0.0.0.0'; + const workerToken = requireEnv('COMPUTE_WORKER_TOKEN'); + const redisUrl = requireEnv('REDIS_URL'); + const queueMaxDepth = readIntEnv('COMPUTE_QUEUE_MAX_DEPTH', 64); + const retryAfterSec = parseRetryAfterSeconds(process.env.COMPUTE_QUEUE_RETRY_AFTER_SEC); + + const whisperConcurrency = readIntEnv('COMPUTE_WHISPER_CONCURRENCY', 1); + const pdfConcurrency = readIntEnv('COMPUTE_PDF_CONCURRENCY', 2); + const whisperTimeoutMs = readIntEnv('COMPUTE_WHISPER_TIMEOUT_MS', 30_000); + const pdfTimeoutMs = readIntEnv('COMPUTE_PDF_TIMEOUT_MS', 90_000); + const attempts = readIntEnv('COMPUTE_JOB_ATTEMPTS', 2); + const prewarmModels = parseBoolEnv('COMPUTE_PREWARM_MODELS', true); + + const redis = new IORedis(redisUrl, { + maxRetriesPerRequest: null, + enableReadyCheck: true, + }); + + const queueDefaults: JobsOptions = { + attempts, + backoff: { + type: 'exponential', + delay: 500, + }, + removeOnComplete: { + age: 60 * 60, + count: 1000, + }, + removeOnFail: { + age: 24 * 60 * 60, + count: 5000, + }, + }; + + const alignQueue = new Queue(ALIGN_QUEUE_NAME, { + connection: redis, + defaultJobOptions: queueDefaults, + }); + const layoutQueue = new Queue(PDF_LAYOUT_QUEUE_NAME, { + connection: redis, + defaultJobOptions: queueDefaults, + }); + + const s3 = buildS3Client(); + const s3Bucket = requireEnv('S3_BUCKET'); + const s3Prefix = normalizeS3Prefix(process.env.S3_PREFIX); + + const ensureSafeKey = (key: string): string => { + const trimmed = key.trim(); + if (!trimmed.startsWith(`${s3Prefix}/`)) { + throw new Error('Object key prefix mismatch'); + } + return trimmed; + }; + + const readObjectByKey = async (key: string): Promise => { + const safeKey = ensureSafeKey(key); + const response = await s3.send(new GetObjectCommand({ + Bucket: s3Bucket, + Key: safeKey, + })); + const bytes = await bodyToBuffer(response.Body); + return toArrayBuffer(new Uint8Array(bytes)); + }; + + const alignWorker = new Worker( + ALIGN_QUEUE_NAME, + async (job) => { + const parsed = alignSchema.parse(job.data); + const audioBuffer = await readObjectByKey(parsed.audioObjectKey); + return withTimeout( + runWhisperAlignmentFromAudioBuffer({ + audioBuffer, + text: parsed.text, + cacheKey: parsed.cacheKey, + lang: parsed.lang, + }), + whisperTimeoutMs, + 'whisper alignment job', + ); + }, + { + connection: redis, + concurrency: whisperConcurrency, + }, + ); + + const layoutWorker = new Worker( + PDF_LAYOUT_QUEUE_NAME, + async (job) => { + const parsed = layoutSchema.parse(job.data); + const pdfBytes = await readObjectByKey(parsed.documentObjectKey); + return withTimeout( + runPdfLayoutFromPdfBuffer({ + documentId: parsed.documentId, + pdfBytes, + }), + pdfTimeoutMs, + 'pdf layout job', + ); + }, + { + connection: redis, + concurrency: pdfConcurrency, + }, + ); + + alignWorker.on('failed', (job, err) => { + console.error('[compute-worker] align job failed', { + jobId: job?.id, + error: err.message, + }); + }); + + layoutWorker.on('failed', (job, err) => { + console.error('[compute-worker] layout job failed', { + jobId: job?.id, + error: err.message, + }); + }); + + if (prewarmModels) { + await ensureComputeModels(); + } + + const app = Fastify({ + logger: buildLoggerConfig(), + }); + + app.addHook('onRequest', async (request, reply) => { + const path = request.url.split('?')[0] ?? request.url; + if (path === '/health/live' || path === '/health/ready') return; + if (!isAuthed(request, workerToken)) { + return reply.code(401).send({ error: 'Unauthorized' }); + } + return; + }); + + app.get('/health/live', async () => ({ ok: true })); + + app.get('/health/ready', async (_request, reply) => { + try { + await redis.ping(); + return { ok: true }; + } catch (error) { + reply.code(503); + return { + ok: false, + error: error instanceof Error ? error.message : String(error), + }; + } + }); + + const rejectIfSaturated = async (queue: Queue, reply: FastifyReply): Promise => { + const depth = await getQueueDepth(queue); + if (depth < queueMaxDepth) return false; + reply.header('Retry-After', String(retryAfterSec)); + reply.code(429).send({ + error: 'Queue is saturated', + retryAfterSeconds: retryAfterSec, + queueDepth: depth, + queueMaxDepth, + }); + return true; + }; + + app.post('/align/whisper/jobs', async (request, reply) => { + const parsed = alignSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { + error: 'Invalid request body', + issues: parsed.error.issues, + }; + } + if (await rejectIfSaturated(alignQueue, reply)) return; + + const job = await alignQueue.add('align', parsed.data); + reply.code(202); + return { jobId: String(job.id) }; + }); + + app.get('/align/whisper/jobs/:jobId', async (request, reply) => { + const params = z.object({ jobId: z.string().trim().min(1) }).safeParse(request.params); + if (!params.success) { + reply.code(400); + return { error: 'Invalid job id' }; + } + const job = await alignQueue.getJob(params.data.jobId); + if (!job) { + reply.code(404); + return { error: 'Job not found' }; + } + return mapJobState(job); + }); + + app.post('/layout/pdf/jobs', async (request, reply) => { + const parsed = layoutSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { + error: 'Invalid request body', + issues: parsed.error.issues, + }; + } + if (await rejectIfSaturated(layoutQueue, reply)) return; + + const job = await layoutQueue.add('layout', parsed.data); + reply.code(202); + return { jobId: String(job.id) }; + }); + + app.get('/layout/pdf/jobs/:jobId', async (request, reply) => { + const params = z.object({ jobId: z.string().trim().min(1) }).safeParse(request.params); + if (!params.success) { + reply.code(400); + return { error: 'Invalid job id' }; + } + const job = await layoutQueue.getJob(params.data.jobId); + if (!job) { + reply.code(404); + return { error: 'Job not found' }; + } + return mapJobState(job); + }); + + const close = async (): Promise => { + await app.close(); + await Promise.allSettled([ + alignWorker.close(), + layoutWorker.close(), + alignQueue.close(), + layoutQueue.close(), + redis.quit(), + ]); + }; + + process.once('SIGINT', () => { + void close().finally(() => process.exit(0)); + }); + process.once('SIGTERM', () => { + void close().finally(() => process.exit(0)); + }); + + await app.listen({ host, port }); + app.log.info({ host, port }, 'compute worker listening'); +} + +void main().catch((error) => { + console.error('[compute-worker] fatal startup error', error); + process.exit(1); +}); diff --git a/compute/worker/tsconfig.json b/compute/worker/tsconfig.json new file mode 100644 index 0000000..3e68ae5 --- /dev/null +++ b/compute/worker/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "noEmit": true + }, + "include": ["src/**/*.ts"] +} diff --git a/docs-site/docs/deploy/compute-worker.md b/docs-site/docs/deploy/compute-worker.md new file mode 100644 index 0000000..af00ab2 --- /dev/null +++ b/docs-site/docs/deploy/compute-worker.md @@ -0,0 +1,64 @@ +--- +title: Compute Worker (Redis + BullMQ) +--- + +Use this guide for `COMPUTE_MODE=worker` deployments where heavy compute runs outside the Next.js app server. + +## Overview + +The compute worker handles: + +- Whisper word alignment (`/align/whisper/jobs`) +- PDF layout parsing (`/layout/pdf/jobs`) + +The app server enqueues jobs and polls status. Queue durability and retries are backed by Redis + BullMQ. + +## Published image + +- App server image: `ghcr.io/richardr1126/openreader` +- Compute worker image: `ghcr.io/richardr1126/openreader-compute-worker` + +## Worker environment variables + +Required: + +- `COMPUTE_WORKER_TOKEN`: bearer token expected by worker routes +- `REDIS_URL`: BullMQ Redis connection string +- `S3_BUCKET` +- `S3_REGION` +- `S3_ACCESS_KEY_ID` +- `S3_SECRET_ACCESS_KEY` + +Common optional: + +- `S3_ENDPOINT` (for non-AWS S3-compatible storage) +- `S3_FORCE_PATH_STYLE=true` (for many S3-compatible providers) +- `S3_PREFIX=openreader` +- `COMPUTE_WORKER_HOST=0.0.0.0` +- `COMPUTE_WORKER_PORT=8081` +- `COMPUTE_LOG_FORMAT=pretty` (default) or `json` +- `COMPUTE_QUEUE_MAX_DEPTH=64` +- `COMPUTE_PREWARM_MODELS=true` + +## App server environment variables (worker mode) + +Set on the Next.js app server: + +```env +COMPUTE_MODE=worker +COMPUTE_WORKER_URL=http://:8081 +COMPUTE_WORKER_TOKEN= +``` + +`COMPUTE_MODE=worker` has no local fallback. If worker is unavailable, affected requests fail. + +## Production notes + +- Worker mode assumes shared object storage is reachable by both app server and worker. +- Non-exposed embedded `weed mini` is not supported with external worker mode. +- Protect `COMPUTE_WORKER_TOKEN` and avoid exposing worker routes publicly without auth. + +## Health endpoints + +- `GET /health/live` +- `GET /health/ready` diff --git a/docs-site/docs/deploy/local-development.md b/docs-site/docs/deploy/local-development.md index 7ab46c5..80c07c7 100644 --- a/docs-site/docs/deploy/local-development.md +++ b/docs-site/docs/deploy/local-development.md @@ -124,6 +124,40 @@ If you need mirrors or pinned artifact locations, set `WHISPER_MODEL_BASE_URL` i +
+External compute worker dev stack (optional) + +Use this when you want durable compute with Redis/BullMQ while keeping Next.js on native host `pnpm dev`. +Full worker deployment details are in [Compute Worker (Redis + BullMQ)](./compute-worker). + +Start only Redis + compute-worker via compose watch: + +```bash +docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --watch +# or: pnpm compute:dev:watch +``` + +`compute/worker/.env.example` contains a starter config. Copy it to `compute/worker/.env` and adjust values for your environment. + +Run the main app separately on the host: + +```bash +pnpm dev +``` + +For app server worker mode, set: + +```env +COMPUTE_MODE=worker +COMPUTE_WORKER_URL=http://localhost:8081 +COMPUTE_WORKER_TOKEN= +``` + +Worker mode requires worker-reachable shared object storage (S3-compatible endpoint). +Non-exposed embedded `weed mini` is not a supported topology for external worker mode. + +
+ ## Steps ### Required flow @@ -206,6 +240,25 @@ S3_SECRET_ACCESS_KEY=your-secret-key # Optional for non-AWS providers: # S3_ENDPOINT=https://your-s3-compatible-endpoint # S3_FORCE_PATH_STYLE=true +``` + + + + +```env +API_BASE=http://host.docker.internal:8880/v1 +API_KEY=none +COMPUTE_MODE=worker +COMPUTE_WORKER_URL=http://localhost:8081 +COMPUTE_WORKER_TOKEN= +USE_EMBEDDED_WEED_MINI=false +S3_BUCKET=your-bucket +S3_REGION=us-east-1 +S3_ACCESS_KEY_ID=your-access-key +S3_SECRET_ACCESS_KEY=your-secret-key +# Optional for non-AWS providers: +# S3_ENDPOINT=https://your-s3-compatible-endpoint +# S3_FORCE_PATH_STYLE=true ``` diff --git a/docs-site/docs/deploy/vercel-deployment.md b/docs-site/docs/deploy/vercel-deployment.md index e6053fd..e9a1162 100644 --- a/docs-site/docs/deploy/vercel-deployment.md +++ b/docs-site/docs/deploy/vercel-deployment.md @@ -8,6 +8,8 @@ This guide covers deploying OpenReader to Vercel with external Postgres and S3-c - Documents (PDF/EPUB/TXT/MD) work with `POSTGRES_URL` + external S3 storage. - Audiobook routes work on Node.js serverless functions using `ffmpeg-static`. +- Heavy compute features (Whisper alignment + PDF layout parsing) work through `COMPUTE_MODE=worker` with an external compute worker service. +- For worker setup details and worker-specific env vars, see [Compute Worker (Redis + BullMQ)](./compute-worker). :::warning DOCX Conversion Limitation `docx` conversion requires `soffice` (LibreOffice), which is not available in a standard Vercel runtime. @@ -36,12 +38,15 @@ AUTH_SECRET=... ADMIN_EMAILS=you@example.com # comma-separated; admins manage TTS + features in-app # Heavy compute (recommended on Vercel in v1) -# local = requires native binaries/models in-process +# local = requires native binaries/models in-process (not recommended on Vercel) +# worker = external durable compute worker (recommended) # none = disable ONNX whisper alignment + PDF layout parsing -COMPUTE_MODE=none +COMPUTE_MODE=worker +COMPUTE_WORKER_URL=https://your-compute-worker.example.com +COMPUTE_WORKER_TOKEN=... # First-boot seed for the TTS shared provider (optional; manage in-app afterwards) -API_KEY=your_replicate_key +# API_KEY=your_replicate_key # API_BASE only needed for OpenAI-compatible self-hosted providers ``` @@ -129,4 +134,4 @@ Adjust memory per route if your files are larger or your plan differs. 1. Upload and read a PDF/EPUB document. 2. Confirm sync/blob fetch works across refreshes/devices. 3. Generate at least one audiobook chapter and play/download it. -4. If you run with local compute (`COMPUTE_MODE=local`) outside Vercel, verify word highlighting timestamps on a TTS run. +4. Verify worker-backed word highlighting and PDF parsing in `COMPUTE_MODE=worker`. diff --git a/docs-site/docs/docker-quick-start.md b/docs-site/docs/docker-quick-start.md index 933cc29..97a555a 100644 --- a/docs-site/docs/docker-quick-start.md +++ b/docs-site/docs/docker-quick-start.md @@ -22,6 +22,12 @@ OpenReader currently pins embedded SeaweedFS to `4.18` in CI and Docker builds. `4.19` introduced intermittent `InternalError` responses on S3 `PutObject` in our upload flow. ::: +## Published images + +- App server: `ghcr.io/richardr1126/openreader:latest` +- Compute worker (Optional): `ghcr.io/richardr1126/openreader-compute-worker:latest` +- Legacy app alias: `ghcr.io/richardr1126/openreader-webui:latest` + ## 1. Start the Docker container @@ -135,6 +141,7 @@ Visit [http://localhost:3003](http://localhost:3003) after startup. ## 3. Update Docker image Legacy image compatibility: `ghcr.io/richardr1126/openreader-webui:latest` remains available as an alias. +For external compute mode image details, see [Compute Worker (Redis + BullMQ)](./deploy/compute-worker). ```bash docker stop openreader || true && \ diff --git a/docs-site/docs/reference/environment-variables.md b/docs-site/docs/reference/environment-variables.md index f04133b..3053452 100644 --- a/docs-site/docs/reference/environment-variables.md +++ b/docs-site/docs/reference/environment-variables.md @@ -54,8 +54,8 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `IMPORT_LIBRARY_DIR` | Library import | `docstore/library` fallback | Set a single server library root | | `IMPORT_LIBRARY_DIRS` | Library import | unset | Set multiple roots (comma/colon/semicolon separated) | | `COMPUTE_MODE` | Heavy compute backend | `local` | Set to `none` to disable ONNX word alignment + PDF layout parsing | -| `COMPUTE_WORKER_URL` | Heavy compute backend | unset | Reserved for future worker backend mode (`worker`) | -| `COMPUTE_WORKER_TOKEN` | Heavy compute backend | unset | Reserved for future worker backend mode (`worker`) | +| `COMPUTE_WORKER_URL` | Heavy compute backend | unset | Required when `COMPUTE_MODE=worker`; base URL for external compute worker | +| `COMPUTE_WORKER_TOKEN` | Heavy compute backend | unset | Required bearer token for external compute worker auth | | `PDF_LAYOUT_MODEL_BASE_URL` | PDF layout model | PP-DocLayoutV3 ONNX base URL | Optional base URL override for `ensureModel()` | | `WHISPER_MODEL_BASE_URL` | Whisper ONNX model | onnx-community defaults | Optional base URL override for ONNX whisper-base_timestamped int8 downloads | | `FFMPEG_BIN` | Audio runtime | auto-detected (`ffmpeg-static`) | Override ffmpeg binary path | @@ -357,22 +357,26 @@ Selects the backend for heavy compute features (ONNX word alignment + PDF layout - Default: `local` - Supported in v1: - `local`: run compute in-process on the app server + - `worker`: enqueue async jobs in an external durable compute worker (Redis + BullMQ) - `none`: disable these features cleanly -- `worker` is reserved for a future external worker backend and currently fails fast at startup if selected +- `worker` requires `COMPUTE_WORKER_URL` and `COMPUTE_WORKER_TOKEN` +- `worker` assumes the external worker can directly reach shared object storage (S3-compatible endpoint) +- `worker` is not compatible with non-exposed embedded `weed mini` storage topologies +- Worker service env vars are documented in [Compute Worker (Redis + BullMQ)](../deploy/compute-worker) ### COMPUTE_WORKER_URL -Reserved for future external compute worker mode. +Base URL for external compute worker mode. -- Used only when `COMPUTE_MODE=worker` (not implemented in v1) -- Leave unset in v1 +- Used only when `COMPUTE_MODE=worker` +- Example: `http://localhost:8081` ### COMPUTE_WORKER_TOKEN -Reserved bearer token for future external compute worker mode. +Bearer token for external compute worker auth. -- Used only when `COMPUTE_MODE=worker` (not implemented in v1) -- Leave unset in v1 +- Used only when `COMPUTE_MODE=worker` +- Must match worker service `COMPUTE_WORKER_TOKEN` ### PDF_LAYOUT_MODEL_BASE_URL diff --git a/docs-site/docs/reference/stack.md b/docs-site/docs/reference/stack.md index b6dc596..7534a86 100644 --- a/docs-site/docs/reference/stack.md +++ b/docs-site/docs/reference/stack.md @@ -4,9 +4,10 @@ title: Stack ## Framework -- [Next.js](https://nextjs.org/) 15 (App Router) +- [Next.js](https://nextjs.org/) 15 (App Router, Turbopack in dev) - [React](https://react.dev/) 19 - [TypeScript](https://www.typescriptlang.org/) +- [pnpm](https://pnpm.io/) workspaces monorepo ## Containerization and runtime @@ -17,24 +18,48 @@ title: Stack - UI: [Tailwind CSS](https://tailwindcss.com), [Headless UI](https://headlessui.com), [@tailwindcss/typography](https://tailwindcss.com/docs/typography-plugin) - Interactions: `react-dnd`, `react-dropzone` +- Server state: [TanStack Query](https://tanstack.com/query) (React Query v5) - Authentication: [Better Auth](https://www.better-auth.com/) client SDK - Local storage/cache: [Dexie.js](https://dexie.org/) (IndexedDB) +- Audio playback: [Howler.js](https://howlerjs.com/) +- Notifications: `react-hot-toast` - Document rendering: - PDF: [react-pdf](https://github.com/wojtekmaj/react-pdf), [pdf.js](https://mozilla.github.io/pdf.js/) - EPUB: [react-reader](https://github.com/gerhardsletten/react-reader), [epubjs](https://github.com/futurepress/epub.js/) - Markdown/Text: [react-markdown](https://github.com/remarkjs/react-markdown), [remark-gfm](https://github.com/remarkjs/remark-gfm) - Text preprocessing/matching: [compromise](https://github.com/spencermountain/compromise), [cmpstr](https://github.com/remsky/cmpstr) +- Analytics: [Vercel Analytics](https://vercel.com/analytics) ## Next.js server - APIs: Route Handlers for sync, blob/content access, migrations, audiobook export, TTS/Whisper proxying - State sync: request-based today (not realtime push updates) -- Authentication: [Better Auth](https://www.better-auth.com/) server handlers/adapters +- Authentication: [Better Auth](https://www.better-auth.com/) server handlers/adapters with anonymous session support - Metadata DB: [Drizzle ORM](https://orm.drizzle.team/) with SQLite (`better-sqlite3`) by default and optional Postgres (`pg`) - App tables are manually maintained in Drizzle schema files - Auth tables are auto-generated by the [Better Auth](https://www.better-auth.com/) CLI and migrated alongside app tables via Drizzle - Blob storage: embedded [SeaweedFS](https://github.com/seaweedfs/seaweedfs) (`weed mini`) by default, or external S3-compatible storage via AWS SDK v3 -- Audio/processing pipeline: OpenAI-compatible TTS providers, [ffmpeg](https://ffmpeg.org/) for audiobook assembly, built-in ONNX Whisper (`onnx-community/whisper-base_timestamped` int8) for word timestamps +- TTS providers: OpenAI-compatible API (`openai` SDK), [Replicate](https://replicate.com/) (`replicate` client), DeepInfra, and custom OpenAI-compatible endpoints — credentials are encrypted at rest +- Audio pipeline: [ffmpeg](https://ffmpeg.org/) (`ffmpeg-static`) for audiobook assembly, `archiver` for export packaging +- Utilities: `lru-cache` for in-process caching, `fast-xml-parser` for EPUB/XML parsing, `uuid` for identifier generation, `zod` for schema validation + +## External compute worker (optional) + +Monorepo packages under `compute/`: + +- **`@openreader/compute-core`** — ONNX runtime lifecycle, model management, and inference logic shared between local and worker modes + - ONNX runtime: `onnxruntime-node` with `@huggingface/tokenizers` + - Whisper alignment: `onnx-community/whisper-base_timestamped` (int8) for word-level timestamps + - PDF layout: `Bei0001/PP-DocLayoutV3-ONNX` for document block detection and layout parsing + - PDF rendering: `pdfjs-dist`, `@napi-rs/canvas` for server-side page rasterization + - Utilities: `jszip`, `ffmpeg-static` +- **`@openreader/compute-worker`** — standalone Node.js worker service + - HTTP server: [Fastify](https://fastify.dev/) v5 + - Job queue: [BullMQ](https://bullmq.io/) + [ioredis](https://github.com/redis/ioredis) (queues: `whisper-align`, `pdf-layout`) + - Storage: AWS SDK v3 S3 client for reading/writing blobs + - Logging: [Pino](https://getpino.io/) + - Validation: [Zod](https://zod.dev/) +- Compute mode is controlled by `COMPUTE_MODE` env var: `local` (in-process), `worker` (remote queue via HTTP + Redis), or disabled ## Tooling and testing @@ -42,3 +67,4 @@ title: Stack - TypeScript - [Playwright](https://playwright.dev/) end-to-end tests - Drizzle migration/generation scripts +- [Docusaurus](https://docusaurus.io/) documentation site (`docs-site/`) diff --git a/docs-site/sidebars.ts b/docs-site/sidebars.ts index b433750..fe5d1a3 100644 --- a/docs-site/sidebars.ts +++ b/docs-site/sidebars.ts @@ -53,7 +53,7 @@ const sidebars: SidebarsConfig = { { type: 'category', label: '🚀 Deploy', - items: ['deploy/local-development', 'deploy/vercel-deployment'], + items: ['deploy/local-development', 'deploy/compute-worker', 'deploy/vercel-deployment'], }, { type: 'category', diff --git a/next.config.ts b/next.config.ts index 0773b73..7e54e19 100644 --- a/next.config.ts +++ b/next.config.ts @@ -15,13 +15,16 @@ const securityHeaders = [ }, ]; -const computeMode = (process.env.COMPUTE_MODE || 'local').trim().toLowerCase(); -const computeDisabled = computeMode === 'none'; +const computeModeRaw = (process.env.COMPUTE_MODE || 'local').trim().toLowerCase(); +const computeMode = computeModeRaw === 'none' || computeModeRaw === 'worker' || computeModeRaw === 'local' + ? computeModeRaw + : 'local'; +const computeLocal = computeMode === 'local'; const serverExternalPackages = [ '@napi-rs/canvas', 'ffmpeg-static', 'better-sqlite3', - ...(computeDisabled ? [] : ['onnxruntime-node', '@huggingface/tokenizers']), + ...(computeLocal ? ['onnxruntime-node', '@huggingface/tokenizers'] : []), ]; const nextConfig: NextConfig = { @@ -39,6 +42,7 @@ const nextConfig: NextConfig = { canvas: '@napi-rs/canvas', }, }, + transpilePackages: ['@openreader/compute-core'], serverExternalPackages, outputFileTracingIncludes: { '/api/audiobook': [ @@ -60,7 +64,7 @@ const nextConfig: NextConfig = { './node_modules/pdfjs-dist/legacy/build/pdf.worker.mjs', ], }, - ...(computeDisabled + ...(!computeLocal ? { outputFileTracingExcludes: { '/*': [ diff --git a/package.json b/package.json index 115a93e..838d117 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,11 @@ "docs:build": "pnpm --dir docs-site build", "docs:serve": "pnpm --dir docs-site serve", "docs:version": "pnpm --dir docs-site docusaurus docs:version", - "docs:clear": "pnpm --dir docs-site clear" + "docs:clear": "pnpm --dir docs-site clear", + "compute:worker:dev": "pnpm --filter @openreader/compute-worker dev", + "compute:worker:start": "pnpm --filter @openreader/compute-worker start", + "compute:dev:compose": "docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --build", + "compute:dev:watch": "docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --watch" }, "dependencies": { "@aws-sdk/client-s3": "^3.1045.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a2c265a..49947fd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -186,6 +186,58 @@ importers: specifier: ^5.9.3 version: 5.9.3 + compute/core: + dependencies: + '@huggingface/tokenizers': + specifier: ^0.1.3 + version: 0.1.3 + '@napi-rs/canvas': + specifier: ^0.1.100 + version: 0.1.100 + ffmpeg-static: + specifier: ^5.3.0 + version: 5.3.0 + jszip: + specifier: ^3.10.1 + version: 3.10.1 + onnxruntime-node: + specifier: ^1.26.0 + version: 1.26.0 + pdfjs-dist: + specifier: 4.8.69 + version: 4.8.69 + + compute/worker: + dependencies: + '@aws-sdk/client-s3': + specifier: ^3.1045.0 + version: 3.1045.0 + '@openreader/compute-core': + specifier: workspace:* + version: link:../core + bullmq: + specifier: ^5.61.2 + version: 5.76.10 + fastify: + specifier: ^5.6.2 + version: 5.8.5 + ioredis: + specifier: ^5.8.2 + version: 5.10.1 + pino: + specifier: ^9.14.0 + version: 9.14.0 + pino-pretty: + specifier: ^13.1.2 + version: 13.1.3 + zod: + specifier: ^4.1.12 + version: 4.4.3 + devDependencies: + tsx: + specifier: ^4.20.6 + version: 4.21.0 + packages: '@alloc/quick-lru@5.2.0': @@ -948,6 +1000,24 @@ packages: resolution: {integrity: sha512-43/qtrDUokr7LJqoF2c3+RInu/t4zfrpYdoSDfYyhg52rwLV6TnOvdG4fXm7IkSB3wErkcmJS9iEhjVtOSEjjA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@fastify/ajv-compiler@4.0.5': + resolution: {integrity: sha512-KoWKW+MhvfTRWL4qrhUwAAZoaChluo0m0vbiJlGMt2GXvL4LVPQEjt8kSpHI3IBq5Rez8fg+XeH3cneztq+C7A==} + + '@fastify/error@4.2.0': + resolution: {integrity: sha512-RSo3sVDXfHskiBZKBPRgnQTtIqpi/7zhJOEmAxCiBcM7d0uwdGdxLlsCaLzGs8v8NnxIRlfG0N51p5yFaOentQ==} + + '@fastify/fast-json-stringify-compiler@5.0.3': + resolution: {integrity: sha512-uik7yYHkLr6fxd8hJSZ8c+xF4WafPK+XzneQDPU+D10r5X19GW8lJcom2YijX2+qtFF1ENJlHXKFM9ouXNJYgQ==} + + '@fastify/forwarded@3.0.1': + resolution: {integrity: sha512-JqDochHFqXs3C3Ml3gOY58zM7OqO9ENqPo0UqAjAjH8L01fRZqwX9iLeX34//kiJubF7r2ZQHtBRU36vONbLlw==} + + '@fastify/merge-json-schemas@0.2.1': + resolution: {integrity: sha512-OA3KGBCy6KtIvLf8DINC5880o5iBlDX4SxzLQS8HorJAbqluzLRn80UXU0bxZn7UOFhFgpRJDasfwn9nG4FG4A==} + + '@fastify/proxy-addr@5.1.0': + resolution: {integrity: sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==} + '@floating-ui/core@1.7.5': resolution: {integrity: sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==} @@ -1161,6 +1231,9 @@ packages: '@internationalized/string@3.2.8': resolution: {integrity: sha512-NdbMQUSfXLYIQol5VyMtinm9pZDciiMfN7RtmSuSB78io1hqwJ0naYfxyW6vgxWBkzWymQa/3uLDlbfmshtCaA==} + '@ioredis/commands@1.5.1': + resolution: {integrity: sha512-JH8ZL/ywcJyR9MmJ5BNqZllXNZQqQbnVZOqpPQqE1vHiFgAw4NHbvE0FOduNU8IX9babitBT46571OnPTT0Zcw==} + '@isaacs/cliui@8.0.2': resolution: {integrity: sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==} engines: {node: '>=12'} @@ -1178,6 +1251,36 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + resolution: {integrity: sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==} + cpu: [arm64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + resolution: {integrity: sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==} + cpu: [x64] + os: [darwin] + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + resolution: {integrity: sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==} + cpu: [arm64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + resolution: {integrity: sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==} + cpu: [arm] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + resolution: {integrity: sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==} + cpu: [x64] + os: [linux] + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + resolution: {integrity: sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==} + cpu: [x64] + os: [win32] + '@napi-rs/canvas-android-arm64@0.1.100': resolution: {integrity: sha512-hjhCKhntPv9+t4ckHymdx0phYNcVW+GKQR6Lzw2zE+pOVjOplSmtx9nNNknTjbEDLcuLZqA1y8ufKg1XfgftzQ==} engines: {node: '>= 10'} @@ -1345,6 +1448,9 @@ packages: resolution: {integrity: sha512-/UhIkaZgPutTFmQ7RnIJGgDXZmtEJ7Dvi86xNTFWcnRxVRNk/aotsqDJYeEvDP+FSMB2SdW+pQzNMcWP0rwuNA==} engines: {node: '>=14'} + '@pinojs/redact@0.4.0': + resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -1393,6 +1499,7 @@ packages: '@smithy/core@3.24.1': resolution: {integrity: sha512-3mT7o4qQyUWttYnVK3A0Z/u3Xha3E81tXn32Tz6vjZiUXhBrkEivpw1hBYfh84iFF9CSzkBU9Y1DJ3Q6RQ231g==} engines: {node: '>=18.0.0'} + deprecated: Deprecated due to bug in browser bundling instructions https://github.com/smithy-lang/smithy-typescript/issues/2025 '@smithy/credential-provider-imds@4.3.1': resolution: {integrity: sha512-0S/acwHnqX4WrjXzhdiDRxsG2s9SC0cpPIK9nZ1R6UOHd+j7uL28+4bHu22urbLk2TVw3fkp6na/+fkUt/pLNQ==} @@ -1853,6 +1960,9 @@ packages: resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} engines: {node: '>=6.5'} + abstract-logging@2.0.1: + resolution: {integrity: sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA==} + acorn-jsx@5.3.2: resolution: {integrity: sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ==} peerDependencies: @@ -1871,9 +1981,20 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} + ajv-formats@3.0.1: + resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} + peerDependencies: + ajv: ^8.0.0 + peerDependenciesMeta: + ajv: + optional: true + ajv@6.15.0: resolution: {integrity: sha512-fgFx7Hfoq60ytK2c7DhnF8jIvzYgOMxfugjLOSMHjLIPgenqa7S7oaagATUq99mV6IYvN2tRmC0wnTYX6iPbMw==} + ajv@8.20.0: + resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -1961,6 +2082,10 @@ packages: async@3.2.6: resolution: {integrity: sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==} + atomic-sleep@1.0.0: + resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} + engines: {node: '>=8.0.0'} + attr-accept@2.2.5: resolution: {integrity: sha512-0bDNnY/u6pPwHDMoF0FieU354oBi0a8rD9FcsLwzcGWbc8KS8KPIi7y+s13OlVY+gMWc/9xEMUgNE6Qm8ZllYQ==} engines: {node: '>=4'} @@ -1969,6 +2094,9 @@ packages: resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==} engines: {node: '>= 0.4'} + avvio@9.2.0: + resolution: {integrity: sha512-2t/sy01ArdHHE0vRH5Hsay+RtCZt3dLPji7W7/MMOCEgze5b7SNDC4j5H6FnVgPkI1MTNFGzHdHrVXDDl7QSSQ==} + axe-core@4.11.4: resolution: {integrity: sha512-KunSNx+TVpkAw/6ULfhnx+HWRecjqZGTOyquAoWHYLRSdK1tB5Ihce1ZW+UY3fj33bYAFWPu7W/GRSmmrCGuxA==} engines: {node: '>=4'} @@ -2153,6 +2281,10 @@ packages: buffer@6.0.3: resolution: {integrity: sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==} + bullmq@5.76.10: + resolution: {integrity: sha512-LWve7SpQjYSpCP2GEsWmoyzTz2H37L8HRmSTu3YihYsTOr5kJxrfEX6aEV7m6eskEMWXSHZYTMZepX6qNaH6CQ==} + engines: {node: '>=12.22.0'} + call-bind-apply-helpers@1.0.2: resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} engines: {node: '>= 0.4'} @@ -2216,6 +2348,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + cmpstr@3.3.0: resolution: {integrity: sha512-of06NSLK24YPV89qYmlJQxENZF43BtIGlXmrFaUIdkuBMayIckFX9NjRB6PPfkhxpLIOb6LhiTe/P5QoBu6r3w==} engines: {node: '>=18.0.0'} @@ -2227,6 +2363,9 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + colorette@2.0.20: + resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} + comma-separated-tokens@2.0.3: resolution: {integrity: sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==} @@ -2249,6 +2388,10 @@ packages: resolution: {integrity: sha512-MWufYdFw53ccGjCA+Ol7XJYpAlW6/prSMzuPOTRnJGcGzuhLn4Scrz7qf6o8bROZ514ltazcIFJZevcfbo0x7A==} engines: {'0': node >= 6.0} + cookie@1.1.1: + resolution: {integrity: sha512-ei8Aos7ja0weRpFzJnEA9UHJ/7XQmqglbRwnf2ATjcB9Wq874VKH9kfjjirM6UhU2/E5fFYadylyhFldcqSidQ==} + engines: {node: '>=18'} + core-js@3.49.0: resolution: {integrity: sha512-es1U2+YTtzpwkxVLwAFdSpaIMyQaq0PBgm3YD1W3Qpsn1NAmO3KSgZfu+oGSWVu6NvLHoHCV/aYcsE5wiB7ALg==} @@ -2264,6 +2407,10 @@ packages: resolution: {integrity: sha512-piICUB6ei4IlTv1+653yq5+KoqfBYmj9bw6LqXoOneTMDXk5nM1qt12mFW1caG3LlJXEKW1Bp0WggEmIfQB34g==} engines: {node: '>= 14'} + cron-parser@4.9.0: + resolution: {integrity: sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q==} + engines: {node: '>=12.0.0'} + cross-spawn@7.0.6: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} @@ -2295,6 +2442,9 @@ packages: resolution: {integrity: sha512-BS8PfmtDGnrgYdOonGZQdLZslWIeCGFP9tpan0hi1Co2Zr2NKADsvGYA8XxuG/4UWgJ6Cjtv+YJnB6MM69QGlQ==} engines: {node: '>= 0.4'} + dateformat@4.6.3: + resolution: {integrity: sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA==} + debug@3.2.7: resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} peerDependencies: @@ -2337,6 +2487,10 @@ packages: defu@6.1.7: resolution: {integrity: sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==} + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} @@ -2717,6 +2871,12 @@ packages: extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + fast-copy@4.0.3: + resolution: {integrity: sha512-58apWr0GUiDFM8+3afrO6eYwJBn9ZAhDOzG3L+/9llab/haCARS2UIfffmOurYLwbgDRs8n0rfr6qAAPEAuAQw==} + + fast-decode-uri-component@1.0.1: + resolution: {integrity: sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -2734,9 +2894,21 @@ packages: fast-json-stable-stringify@2.1.0: resolution: {integrity: sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==} + fast-json-stringify@6.4.0: + resolution: {integrity: sha512-ibRCQ0GZKJIQ+P3Et1h0LhPgp3PMTYk0MH8O+kW3lNYsvmaQww5Nn3f1jf73Q0jR1Yz3a1CDP4/NZD3vOajWJQ==} + fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + fast-querystring@1.1.2: + resolution: {integrity: sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==} + + fast-safe-stringify@2.1.1: + resolution: {integrity: sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA==} + + fast-uri@3.1.2: + resolution: {integrity: sha512-rVjf7ArG3LTk+FS6Yw81V1DLuZl1bRbNrev6Tmd/9RaroeeRRJhAt7jg/6YFxbvAQXUCavSoZhPPj6oOx+5KjQ==} + fast-xml-builder@1.2.0: resolution: {integrity: sha512-00aAWieqff+ZJhsXA4g1g7M8k+7AYoMUUHF+/zFb5U6Uv/P0Vl4QZo84/IcufzYalLuEj9928bXN9PbbFzMF0Q==} @@ -2748,6 +2920,9 @@ packages: resolution: {integrity: sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==} hasBin: true + fastify@5.8.5: + resolution: {integrity: sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==} + fastq@1.20.1: resolution: {integrity: sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==} @@ -2779,6 +2954,10 @@ packages: resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} engines: {node: '>=8'} + find-my-way@9.6.0: + resolution: {integrity: sha512-Zf4Xve4RymLl7NgaavNebZ01joJ8MfVerOG43wy7SHLO+r+K0C6d/SE0BiR7AV5V1VOCFlOP7ecdo+I4qmiHrQ==} + engines: {node: '>=20'} + find-up@5.0.0: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} @@ -2917,6 +3096,9 @@ packages: hast-util-whitespace@3.0.0: resolution: {integrity: sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==} + help-me@5.0.0: + resolution: {integrity: sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==} + hoist-non-react-statics@3.3.2: resolution: {integrity: sha512-/gGivxi8JPKWNm/W0jSmzcMPpfpPLc3dY/6GxhX2hQ9iGj3aDfklV4ET7NjKpSinLpJ5vafa9iiGIEZg10SfBw==} @@ -2968,6 +3150,14 @@ packages: resolution: {integrity: sha512-4gd7VpWNQNB4UKKCFFVcp1AVv+FMOgs9NKzjHKusc8jTMhd5eL1NqQqOpE0KzMds804/yHlglp3uxgluOqAPLw==} engines: {node: '>= 0.4'} + ioredis@5.10.1: + resolution: {integrity: sha512-HuEDBTI70aYdx1v6U97SbNx9F1+svQKBDo30o0b9fw055LMepzpOOd0Ccg9Q6tbqmBSJaMuY0fB7yw9/vjBYCA==} + engines: {node: '>=12.22.0'} + + ipaddr.js@2.4.0: + resolution: {integrity: sha512-9VGk3HGanVE6JoZXHiCpnGy5X0jYDnN4EA4lntFPj+1vIWlFhIylq2CrrCOJH9EAhc5CYhq18F2Av2tgoAPsYQ==} + engines: {node: '>= 10'} + is-alphabetical@2.0.1: resolution: {integrity: sha512-FWyyY60MeTNyeSRpkM2Iry0G9hpr7/9kD40mD/cGQEuilcZYS4okz8SN2Q6rLCJ8gbCt6fN+rC+6tMGS99LaxQ==} @@ -3122,6 +3312,10 @@ packages: jose@6.2.3: resolution: {integrity: sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==} + joycon@3.1.1: + resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} + engines: {node: '>=10'} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} @@ -3132,9 +3326,15 @@ packages: json-buffer@3.0.1: resolution: {integrity: sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==} + json-schema-ref-resolver@3.0.0: + resolution: {integrity: sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==} + json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + json-schema-traverse@1.0.0: + resolution: {integrity: sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==} + json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} @@ -3177,6 +3377,9 @@ packages: lie@3.3.0: resolution: {integrity: sha512-UaiMJzeWRlEujzAuw5LokY1L5ecNQYZKfmyZ9L7wDHb/p5etKaxXhohBcrw0EYby+G/NA52vRSN4N39dxHAIwQ==} + light-my-request@6.6.0: + resolution: {integrity: sha512-CHYbu8RtboSIoVsHZ6Ye4cj4Aw/yg2oAFimlF7mNvfDV192LR7nDiKtSIfCuLT7KokPSTn/9kfVLm5OGN0A28A==} + lilconfig@3.1.3: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} engines: {node: '>=14'} @@ -3191,6 +3394,12 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + lodash.merge@4.6.2: resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} @@ -3211,6 +3420,10 @@ packages: resolution: {integrity: sha512-Gf/KoL3C/MlI7Bt0PGI9I+TeTC/I6r/csU58N4BSNc4lppLBeKsOdFYkK+dX0ABDUMJNfCHTyPpzwwO21Awd3A==} engines: {node: 20 || >=22} + luxon@3.7.2: + resolution: {integrity: sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==} + engines: {node: '>=12'} + make-cancellable-promise@1.3.2: resolution: {integrity: sha512-GCXh3bq/WuMbS+Ky4JBPW1hYTOU+znU+Q5m9Pu+pI8EoUqIHk9+tviOKC6/qhHh8C4/As3tzJ69IF32kdz85ww==} @@ -3408,6 +3621,13 @@ packages: ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + msgpackr-extract@3.0.3: + resolution: {integrity: sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==} + hasBin: true + + msgpackr@2.0.1: + resolution: {integrity: sha512-9J+tqTEsbHqY8YohazYgty7LgerFIWxvMLpUjqETSmjHojtJm2WnX2kK/2a1fLI7CO7ERP1YSEUXMucz4j+yBA==} + mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} @@ -3459,6 +3679,9 @@ packages: resolution: {integrity: sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ==} engines: {node: '>=10'} + node-abort-controller@3.1.1: + resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} + node-addon-api@7.1.1: resolution: {integrity: sha512-5m3bsyrjFWE1xf7nz7YXdN4udnVtXK6/Yfgn5qnahL6bCkf2yKt4k3nuTKAtT4r3IG8JNR2ncsIMdZuAzJjHQQ==} @@ -3466,6 +3689,10 @@ packages: resolution: {integrity: sha512-pyFS63ptit/P5WqUkt+UUfe+4oevH+bFeIiPPdfb0pFeYEu/1ELnJu5l+5EcTKYL5M7zaAa7S8ddywgXypqKCw==} engines: {node: '>= 0.4'} + node-gyp-build-optional-packages@5.2.2: + resolution: {integrity: sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==} + hasBin: true + normalize-path@3.0.0: resolution: {integrity: sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==} engines: {node: '>=0.10.0'} @@ -3506,6 +3733,10 @@ packages: resolution: {integrity: sha512-gXah6aZrcUxjWg2zR2MwouP2eHlCBzdV4pygudehaKXSGW4v2AsRQUK+lwwXhii6KFZcunEnmSUoYp5CXibxtA==} engines: {node: '>= 0.4'} + on-exit-leak-free@2.1.2: + resolution: {integrity: sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==} + engines: {node: '>=14.0.0'} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -3639,6 +3870,23 @@ packages: resolution: {integrity: sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog==} engines: {node: '>=0.10.0'} + pino-abstract-transport@2.0.0: + resolution: {integrity: sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==} + + pino-abstract-transport@3.0.0: + resolution: {integrity: sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==} + + pino-pretty@13.1.3: + resolution: {integrity: sha512-ttXRkkOz6WWC95KeY9+xxWL6AtImwbyMHrL1mSwqwW9u+vLp/WIElvHvCSDg0xO/Dzrggz1zv3rN5ovTRVowKg==} + hasBin: true + + pino-std-serializers@7.1.0: + resolution: {integrity: sha512-BndPH67/JxGExRgiX1dX0w1FvZck5Wa4aal9198SrRhZjH3GxKQUKIBnYJTdj2HDN3UQAS06HlfcSbQj2OHmaw==} + + pino@9.14.0: + resolution: {integrity: sha512-8OEwKp5juEvb/MjpIc4hjqfgCNysrS94RIOMXYvpYCdm/jglrKEiAYmiumbmGhCvs+IcInsphYDFwqrjr7398w==} + hasBin: true + pirates@4.0.7: resolution: {integrity: sha512-TfySrs/5nm8fQJDcBDuUng3VOUKsd7S+zqvbOTiGXHfxX4wK31ard+hoNuvkicM/2YFzlpDgABOevKSsB4G/FA==} engines: {node: '>= 6'} @@ -3741,6 +3989,12 @@ packages: process-nextick-args@2.0.1: resolution: {integrity: sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==} + process-warning@4.0.1: + resolution: {integrity: sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==} + + process-warning@5.0.0: + resolution: {integrity: sha512-a39t9ApHNx2L4+HBnQKqxxHNs1r7KF+Intd8Q/g1bUh6q0WIp9voPXJ/x0j+ZL45KF1pJd9+q2jLIRMfvEshkA==} + process@0.11.10: resolution: {integrity: sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==} engines: {node: '>= 0.6.0'} @@ -3765,6 +4019,9 @@ packages: queue-microtask@1.2.3: resolution: {integrity: sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==} + quick-format-unescaped@4.0.4: + resolution: {integrity: sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==} + rc@1.2.8: resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} hasBin: true @@ -3868,6 +4125,18 @@ packages: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} + real-require@0.2.0: + resolution: {integrity: sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==} + engines: {node: '>= 12.13.0'} + + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + redux@4.2.1: resolution: {integrity: sha512-LAUYz4lc+Do8/g7aeRa8JkyDErK6ekstQaqWQrNRW//MY1TvCEpMtpTWvlQ+FPbWCx+Xixu/6SHt5N0HR+SB4w==} @@ -3895,6 +4164,10 @@ packages: resolution: {integrity: sha512-1ufKejfUVz/azy+5TnzQP7U1+MHVWZ6psnQ06az8byUUnRhT+DZ/MvewzB1NQYBVMgNKR7xPDtTwlcP5nv/5+w==} engines: {git: '>=2.11.0', node: '>=18.0.0', npm: '>=7.19.0', yarn: '>=1.7.0'} + require-from-string@2.0.2: + resolution: {integrity: sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==} + engines: {node: '>=0.10.0'} + resolve-from@4.0.0: resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==} engines: {node: '>=4'} @@ -3912,10 +4185,17 @@ packages: engines: {node: '>= 0.4'} hasBin: true + ret@0.5.0: + resolution: {integrity: sha512-I1XxrZSQ+oErkRR4jYbAyEEu2I0avBvvMM5JN+6EBprOGRCs63ENqZ3vjavq8fBw2+62G5LF5XelKwuJpcvcxw==} + engines: {node: '>=10'} + reusify@1.1.0: resolution: {integrity: sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} + rfdc@1.4.1: + resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + rou3@0.7.12: resolution: {integrity: sha512-iFE4hLDuloSWcD7mjdCDhx2bKcIsYbtOTpfH5MHHLSKMOUyjqQXTeZVa289uuwEGEKFoE/BAPbhaU4B774nceg==} @@ -3940,9 +4220,20 @@ packages: resolution: {integrity: sha512-x/+Cz4YrimQxQccJf5mKEbIa1NzeCRNI5Ecl/ekmlYaampdNLPalVyIcCZNNH3MvmqBugV5TMYZXv0ljslUlaw==} engines: {node: '>= 0.4'} + safe-regex2@5.1.1: + resolution: {integrity: sha512-mOSBvHGDZMuIEZMdOz/aCEYDCv0E7nfcNsIhUF+/P+xC7Hyf3FkvymqgPbg9D1EdSGu+uKbJgy09K/RKKc7kJA==} + hasBin: true + + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + scheduler@0.27.0: resolution: {integrity: sha512-eNv+WrVbKu1f3vbYJT/xtiF5syA5HPIMtf9IgY/nKg0sWqzAUEvqY/xm7OcZc/qafLx/iO9FgOmeSAp4v5ti/Q==} + secure-json-parse@4.1.0: + resolution: {integrity: sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA==} + semver@6.3.1: resolution: {integrity: sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==} hasBin: true @@ -3956,6 +4247,9 @@ packages: resolution: {integrity: sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==} engines: {node: '>=10'} + set-cookie-parser@2.7.2: + resolution: {integrity: sha512-oeM1lpU/UvhTxw+g3cIfxXHyJRc/uidd3yK1P242gzHds0udQBYzs3y8j4gCCW+ZJ7ad0yctld8RYO+bdurlvw==} + set-cookie-parser@3.1.0: resolution: {integrity: sha512-kjnC1DXBHcxaOaOXBHBeRtltsDG2nUiUni+jP92M9gYdW12rsmx92UsfpH7o5tDRs7I1ZZPSQJQGv3UaRfCiuw==} @@ -4012,6 +4306,9 @@ packages: simple-get@4.0.1: resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + sonic-boom@4.2.1: + resolution: {integrity: sha512-w6AxtubXa2wTXAUsZMMWERrsIRAdrK0Sc+FUytWvYAhBJLyuI4llrMIC1DtlNSdI99EI86KZum2MMq3EAZlF9Q==} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -4033,6 +4330,9 @@ packages: stable-hash@0.0.5: resolution: {integrity: sha512-+L3ccpzibovGXFK+Ap/f8LOS0ahMrHTf3xu7mMLSpEGU0EO9ucaysSylKo9eRDFNhWve/y275iPmIZ4z39a9iA==} + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + stop-iteration-iterator@1.1.0: resolution: {integrity: sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==} engines: {node: '>= 0.4'} @@ -4100,6 +4400,10 @@ packages: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} + strip-json-comments@5.0.3: + resolution: {integrity: sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw==} + engines: {node: '>=14.16'} + strnum@2.3.0: resolution: {integrity: sha512-ums3KNd42PGyx5xaoVTO1mjU1bH3NpY4vsrVlnv9PNGqQj8wd7rJ6nEypLrJ7z5vxK5RP0yMLo6J/Gsm62DI5Q==} @@ -4169,6 +4473,9 @@ packages: thenify@3.3.1: resolution: {integrity: sha512-RVZSIV5IG10Hk3enotrhvz0T9em6cyHBLkH/YAZuKqd8hRkKhSfCGIcP2KUY0EPxndzANBmNllzWPwak+bheSw==} + thread-stream@3.1.0: + resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==} + tiny-invariant@1.3.3: resolution: {integrity: sha512-+FbBPE1o9QAYvviau/qC5SE3caw21q3xkvWKBtja5vgqOWIHHJ3ioaq1VPfn/Szqctz2bU/oYeKd9/z5BL+PVg==} @@ -4180,6 +4487,10 @@ packages: resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} engines: {node: '>=8.0'} + toad-cache@3.7.1: + resolution: {integrity: sha512-5DXWzE4Vz7xNHsv+xQ+MGfJYyC78Aok3tEr0MNwHoRf7vZnga1mQXZ4/Nsodld4VR6Wd+VhfmqnNrsRJyYPfrQ==} + engines: {node: '>=20'} + trim-lines@3.0.1: resolution: {integrity: sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==} @@ -5181,6 +5492,29 @@ snapshots: '@eslint/core': 0.17.0 levn: 0.4.1 + '@fastify/ajv-compiler@4.0.5': + dependencies: + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + fast-uri: 3.1.2 + + '@fastify/error@4.2.0': {} + + '@fastify/fast-json-stringify-compiler@5.0.3': + dependencies: + fast-json-stringify: 6.4.0 + + '@fastify/forwarded@3.0.1': {} + + '@fastify/merge-json-schemas@0.2.1': + dependencies: + dequal: 2.0.3 + + '@fastify/proxy-addr@5.1.0': + dependencies: + '@fastify/forwarded': 3.0.1 + ipaddr.js: 2.4.0 + '@floating-ui/core@1.7.5': dependencies: '@floating-ui/utils': 0.2.11 @@ -5343,6 +5677,8 @@ snapshots: dependencies: '@swc/helpers': 0.5.21 + '@ioredis/commands@1.5.1': {} + '@isaacs/cliui@8.0.2': dependencies: string-width: 5.1.2 @@ -5366,6 +5702,24 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-arm@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-linux-x64@3.0.3': + optional: true + + '@msgpackr-extract/msgpackr-extract-win32-x64@3.0.3': + optional: true + '@napi-rs/canvas-android-arm64@0.1.100': optional: true @@ -5472,6 +5826,8 @@ snapshots: '@opentelemetry/semantic-conventions@1.41.1': {} + '@pinojs/redact@0.4.0': {} + '@pkgjs/parseargs@0.11.0': optional: true @@ -5989,6 +6345,8 @@ snapshots: dependencies: event-target-shim: 5.0.1 + abstract-logging@2.0.1: {} + acorn-jsx@5.3.2(acorn@8.16.0): dependencies: acorn: 8.16.0 @@ -6003,6 +6361,10 @@ snapshots: transitivePeerDependencies: - supports-color + ajv-formats@3.0.1(ajv@8.20.0): + optionalDependencies: + ajv: 8.20.0 + ajv@6.15.0: dependencies: fast-deep-equal: 3.1.3 @@ -6010,6 +6372,13 @@ snapshots: json-schema-traverse: 0.4.1 uri-js: 4.4.1 + ajv@8.20.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-uri: 3.1.2 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + ansi-regex@5.0.1: {} ansi-regex@6.2.2: {} @@ -6134,12 +6503,19 @@ snapshots: async@3.2.6: {} + atomic-sleep@1.0.0: {} + attr-accept@2.2.5: {} available-typed-arrays@1.0.7: dependencies: possible-typed-array-names: 1.1.0 + avvio@9.2.0: + dependencies: + '@fastify/error': 4.2.0 + fastq: 1.20.1 + axe-core@4.11.4: {} axobject-query@4.1.0: {} @@ -6276,6 +6652,17 @@ snapshots: base64-js: 1.5.1 ieee754: 1.2.1 + bullmq@5.76.10: + dependencies: + cron-parser: 4.9.0 + ioredis: 5.10.1 + msgpackr: 2.0.1 + node-abort-controller: 3.1.1 + semver: 7.8.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + call-bind-apply-helpers@1.0.2: dependencies: es-errors: 1.3.0 @@ -6340,6 +6727,8 @@ snapshots: clsx@2.1.1: {} + cluster-key-slot@1.1.2: {} + cmpstr@3.3.0: {} color-convert@2.0.1: @@ -6348,6 +6737,8 @@ snapshots: color-name@1.1.4: {} + colorette@2.0.20: {} + comma-separated-tokens@2.0.3: {} commander@4.1.1: {} @@ -6375,6 +6766,8 @@ snapshots: readable-stream: 3.6.2 typedarray: 0.0.6 + cookie@1.1.1: {} + core-js@3.49.0: {} core-util-is@1.0.3: {} @@ -6386,6 +6779,10 @@ snapshots: crc-32: 1.2.2 readable-stream: 4.7.0 + cron-parser@4.9.0: + dependencies: + luxon: 3.7.2 + cross-spawn@7.0.6: dependencies: path-key: 3.1.1 @@ -6421,6 +6818,8 @@ snapshots: es-errors: 1.3.0 is-data-view: 1.0.2 + dateformat@4.6.3: {} + debug@3.2.7: dependencies: ms: 2.1.3 @@ -6455,6 +6854,8 @@ snapshots: defu@6.1.7: {} + denque@2.1.0: {} + dequal@2.0.3: {} detect-libc@2.1.2: {} @@ -6971,6 +7372,10 @@ snapshots: extend@3.0.2: {} + fast-copy@4.0.3: {} + + fast-decode-uri-component@1.0.1: {} + fast-deep-equal@3.1.3: {} fast-fifo@1.3.2: {} @@ -6993,8 +7398,25 @@ snapshots: fast-json-stable-stringify@2.1.0: {} + fast-json-stringify@6.4.0: + dependencies: + '@fastify/merge-json-schemas': 0.2.1 + ajv: 8.20.0 + ajv-formats: 3.0.1(ajv@8.20.0) + fast-uri: 3.1.2 + json-schema-ref-resolver: 3.0.0 + rfdc: 1.4.1 + fast-levenshtein@2.0.6: {} + fast-querystring@1.1.2: + dependencies: + fast-decode-uri-component: 1.0.1 + + fast-safe-stringify@2.1.1: {} + + fast-uri@3.1.2: {} + fast-xml-builder@1.2.0: dependencies: path-expression-matcher: 1.5.0 @@ -7015,6 +7437,24 @@ snapshots: strnum: 2.3.0 xml-naming: 0.1.0 + fastify@5.8.5: + dependencies: + '@fastify/ajv-compiler': 4.0.5 + '@fastify/error': 4.2.0 + '@fastify/fast-json-stringify-compiler': 5.0.3 + '@fastify/proxy-addr': 5.1.0 + abstract-logging: 2.0.1 + avvio: 9.2.0 + fast-json-stringify: 6.4.0 + find-my-way: 9.6.0 + light-my-request: 6.6.0 + pino: 9.14.0 + process-warning: 5.0.0 + rfdc: 1.4.1 + secure-json-parse: 4.1.0 + semver: 7.8.0 + toad-cache: 3.7.1 + fastq@1.20.1: dependencies: reusify: 1.1.0 @@ -7046,6 +7486,12 @@ snapshots: dependencies: to-regex-range: 5.0.1 + find-my-way@9.6.0: + dependencies: + fast-deep-equal: 3.1.3 + fast-querystring: 1.1.2 + safe-regex2: 5.1.1 + find-up@5.0.0: dependencies: locate-path: 6.0.0 @@ -7207,6 +7653,8 @@ snapshots: dependencies: '@types/hast': 3.0.4 + help-me@5.0.0: {} + hoist-non-react-statics@3.3.2: dependencies: react-is: 16.13.1 @@ -7253,6 +7701,22 @@ snapshots: hasown: 2.0.3 side-channel: 1.1.0 + ioredis@5.10.1: + dependencies: + '@ioredis/commands': 1.5.1 + cluster-key-slot: 1.1.2 + debug: 4.4.3 + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + + ipaddr.js@2.4.0: {} + is-alphabetical@2.0.1: {} is-alphanumerical@2.0.1: @@ -7411,6 +7875,8 @@ snapshots: jose@6.2.3: {} + joycon@3.1.1: {} + js-tokens@4.0.0: {} js-yaml@4.1.1: @@ -7419,8 +7885,14 @@ snapshots: json-buffer@3.0.1: {} + json-schema-ref-resolver@3.0.0: + dependencies: + dequal: 2.0.3 + json-schema-traverse@0.4.1: {} + json-schema-traverse@1.0.0: {} + json-stable-stringify-without-jsonify@1.0.1: {} json5@1.0.2: @@ -7470,6 +7942,12 @@ snapshots: dependencies: immediate: 3.0.6 + light-my-request@6.6.0: + dependencies: + cookie: 1.1.1 + process-warning: 4.0.1 + set-cookie-parser: 2.7.2 + lilconfig@3.1.3: {} lines-and-columns@1.2.4: {} @@ -7482,6 +7960,10 @@ snapshots: dependencies: p-locate: 5.0.0 + lodash.defaults@4.2.0: {} + + lodash.isarguments@3.1.0: {} + lodash.merge@4.6.2: {} lodash@4.18.1: {} @@ -7496,6 +7978,8 @@ snapshots: lru-cache@11.3.6: {} + luxon@3.7.2: {} + make-cancellable-promise@1.3.2: {} make-event-props@1.6.2: {} @@ -7891,6 +8375,22 @@ snapshots: ms@2.1.3: {} + msgpackr-extract@3.0.3: + dependencies: + node-gyp-build-optional-packages: 5.2.2 + optionalDependencies: + '@msgpackr-extract/msgpackr-extract-darwin-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-darwin-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-arm64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-linux-x64': 3.0.3 + '@msgpackr-extract/msgpackr-extract-win32-x64': 3.0.3 + optional: true + + msgpackr@2.0.1: + optionalDependencies: + msgpackr-extract: 3.0.3 + mz@2.7.0: dependencies: any-promise: 1.3.0 @@ -7937,6 +8437,8 @@ snapshots: dependencies: semver: 7.8.0 + node-abort-controller@3.1.1: {} + node-addon-api@7.1.1: optional: true @@ -7947,6 +8449,11 @@ snapshots: object.entries: 1.1.9 semver: 6.3.1 + node-gyp-build-optional-packages@5.2.2: + dependencies: + detect-libc: 2.1.2 + optional: true + normalize-path@3.0.0: {} object-assign@4.1.1: {} @@ -7993,6 +8500,8 @@ snapshots: define-properties: 1.2.1 es-object-atoms: 1.1.1 + on-exit-leak-free@2.1.2: {} + once@1.4.0: dependencies: wrappy: 1.0.2 @@ -8118,6 +8627,46 @@ snapshots: pify@2.3.0: {} + pino-abstract-transport@2.0.0: + dependencies: + split2: 4.2.0 + + pino-abstract-transport@3.0.0: + dependencies: + split2: 4.2.0 + + pino-pretty@13.1.3: + dependencies: + colorette: 2.0.20 + dateformat: 4.6.3 + fast-copy: 4.0.3 + fast-safe-stringify: 2.1.1 + help-me: 5.0.0 + joycon: 3.1.1 + minimist: 1.2.8 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 3.0.0 + pump: 3.0.4 + secure-json-parse: 4.1.0 + sonic-boom: 4.2.1 + strip-json-comments: 5.0.3 + + pino-std-serializers@7.1.0: {} + + pino@9.14.0: + dependencies: + '@pinojs/redact': 0.4.0 + atomic-sleep: 1.0.0 + on-exit-leak-free: 2.1.2 + pino-abstract-transport: 2.0.0 + pino-std-serializers: 7.1.0 + process-warning: 5.0.0 + quick-format-unescaped: 4.0.4 + real-require: 0.2.0 + safe-stable-stringify: 2.5.0 + sonic-boom: 4.2.1 + thread-stream: 3.1.0 + pirates@4.0.7: {} playwright-core@1.60.0: {} @@ -8208,6 +8757,10 @@ snapshots: process-nextick-args@2.0.1: {} + process-warning@4.0.1: {} + + process-warning@5.0.0: {} + process@0.11.10: {} progress@2.0.3: {} @@ -8229,6 +8782,8 @@ snapshots: queue-microtask@1.2.3: {} + quick-format-unescaped@4.0.4: {} + rc@1.2.8: dependencies: deep-extend: 0.6.0 @@ -8379,6 +8934,14 @@ snapshots: dependencies: picomatch: 2.3.2 + real-require@0.2.0: {} + + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + redux@4.2.1: dependencies: '@babel/runtime': 7.29.2 @@ -8441,6 +9004,8 @@ snapshots: optionalDependencies: readable-stream: 4.7.0 + require-from-string@2.0.2: {} + resolve-from@4.0.0: {} resolve-pkg-maps@1.0.0: {} @@ -8461,8 +9026,12 @@ snapshots: path-parse: 1.0.7 supports-preserve-symlinks-flag: 1.0.0 + ret@0.5.0: {} + reusify@1.1.0: {} + rfdc@1.4.1: {} + rou3@0.7.12: {} run-parallel@1.2.0: @@ -8492,8 +9061,16 @@ snapshots: es-errors: 1.3.0 is-regex: 1.2.1 + safe-regex2@5.1.1: + dependencies: + ret: 0.5.0 + + safe-stable-stringify@2.5.0: {} + scheduler@0.27.0: {} + secure-json-parse@4.1.0: {} + semver@6.3.1: {} semver@7.8.0: {} @@ -8502,6 +9079,8 @@ snapshots: dependencies: type-fest: 0.20.2 + set-cookie-parser@2.7.2: {} + set-cookie-parser@3.1.0: {} set-function-length@1.2.2: @@ -8604,6 +9183,10 @@ snapshots: once: 1.4.0 simple-concat: 1.0.1 + sonic-boom@4.2.1: + dependencies: + atomic-sleep: 1.0.0 + source-map-js@1.2.1: {} source-map-support@0.5.21: @@ -8619,6 +9202,8 @@ snapshots: stable-hash@0.0.5: {} + standard-as-callback@2.1.0: {} + stop-iteration-iterator@1.1.0: dependencies: es-errors: 1.3.0 @@ -8722,6 +9307,8 @@ snapshots: strip-json-comments@3.1.1: {} + strip-json-comments@5.0.3: {} + strnum@2.3.0: {} style-to-js@1.1.21: @@ -8832,6 +9419,10 @@ snapshots: dependencies: any-promise: 1.3.0 + thread-stream@3.1.0: + dependencies: + real-require: 0.2.0 + tiny-invariant@1.3.3: {} tinyglobby@0.2.16: @@ -8843,6 +9434,8 @@ snapshots: dependencies: is-number: 7.0.0 + toad-cache@3.7.1: {} + trim-lines@3.0.1: {} trough@2.2.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index caae007..67ae185 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,5 +1,6 @@ packages: - . + - compute/* allowBuilds: '@napi-rs/canvas': true diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts index ad5bffa..aa80c1e 100644 --- a/src/app/api/tts/segments/ensure/route.ts +++ b/src/app/api/tts/segments/ensure/route.ts @@ -372,10 +372,8 @@ export async function POST(request: NextRequest) { // previously unavailable, retry alignment using the current segment text. if (!alignment) { try { - const audioBuffer = await getTtsSegmentAudioObject(existing.audioKey); - const whisperBytes = Uint8Array.from(audioBuffer); const aligned = (await getCompute().alignWords({ - audioBuffer: whisperBytes.buffer, + audioObjectKey: existing.audioKey, text: segment.text, })).alignments; alignment = aligned[0] ? { ...aligned[0], sentenceIndex: segment.original.segmentIndex } : null; @@ -527,6 +525,7 @@ export async function POST(request: NextRequest) { const whisperBytes = Uint8Array.from(persistedBuffer); const aligned = (await getCompute().alignWords({ audioBuffer: whisperBytes.buffer, + audioObjectKey: audioKey, text: segment.text, })).alignments; alignment = aligned[0] ? { ...aligned[0], sentenceIndex: segment.original.segmentIndex } : null; diff --git a/src/lib/server/compute/index.ts b/src/lib/server/compute/index.ts index 8b211b2..d9f847d 100644 --- a/src/lib/server/compute/index.ts +++ b/src/lib/server/compute/index.ts @@ -1,17 +1,14 @@ import type { ComputeBackend, ComputeMode } from '@/lib/server/compute/types'; import { NoneComputeBackend } from '@/lib/server/compute/none'; import { isComputeModeAvailable, readComputeMode } from '@/lib/server/compute/mode'; +import { WorkerComputeBackend } from '@/lib/server/compute/worker'; let backend: ComputeBackend | null = null; function createBackend(): ComputeBackend { const mode: ComputeMode = readComputeMode(); if (mode === 'none') return new NoneComputeBackend(); - if (mode === 'worker') { - throw new Error( - 'COMPUTE_MODE=worker is not implemented yet in v1. Switch to local/none or implement WorkerComputeBackend (v2).', - ); - } + if (mode === 'worker') return new WorkerComputeBackend(); // Intentionally lazy-load local compute so COMPUTE_MODE=none builds // can avoid tracing heavy ONNX dependencies. // eslint-disable-next-line @typescript-eslint/no-require-imports diff --git a/src/lib/server/compute/local.ts b/src/lib/server/compute/local.ts index a38bf6a..c1f1475 100644 --- a/src/lib/server/compute/local.ts +++ b/src/lib/server/compute/local.ts @@ -1,21 +1,40 @@ import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types'; +import { getDocumentBlob } from '@/lib/server/documents/blobstore'; +import { getTtsSegmentAudioObject } from '@/lib/server/tts/segments-blobstore'; +import { + runPdfLayoutFromPdfBuffer, + runWhisperAlignmentFromAudioBuffer, +} from '@openreader/compute-core'; export class LocalComputeBackend implements ComputeBackend { readonly mode = 'local' as const; async alignWords(input: WhisperAlignInput): Promise { - const { alignAudioWithText } = await import('@/lib/server/whisper/alignment'); - const alignments = await alignAudioWithText( - input.audioBuffer, - input.text, - input.cacheKey, - { lang: input.lang }, - ); - return { alignments }; + let audioBuffer = input.audioBuffer ?? null; + if (!audioBuffer && input.audioObjectKey) { + const bytes = new Uint8Array(await getTtsSegmentAudioObject(input.audioObjectKey)); + audioBuffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + } + if (!audioBuffer) { + throw new Error('Local compute alignment requires audioBuffer or audioObjectKey'); + } + return runWhisperAlignmentFromAudioBuffer({ + audioBuffer, + text: input.text, + cacheKey: input.cacheKey, + lang: input.lang, + }); } async parsePdfLayout(input: PdfLayoutInput) { - const { parsePdf } = await import('@/lib/server/pdf-layout/parsePdf'); - return parsePdf({ documentId: input.documentId, pdfBytes: input.pdfBytes }); + let pdfBytes = input.pdfBytes ?? null; + if (!pdfBytes && input.documentId && typeof input.namespace !== 'undefined') { + const bytes = new Uint8Array(await getDocumentBlob(input.documentId, input.namespace)); + pdfBytes = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + } + if (!pdfBytes) { + throw new Error('Local compute PDF layout requires pdfBytes or (documentId + namespace)'); + } + return (await runPdfLayoutFromPdfBuffer({ documentId: input.documentId, pdfBytes })).parsed; } } diff --git a/src/lib/server/compute/mode.ts b/src/lib/server/compute/mode.ts index 5ebd349..42b066e 100644 --- a/src/lib/server/compute/mode.ts +++ b/src/lib/server/compute/mode.ts @@ -7,10 +7,5 @@ export function readComputeMode(): ComputeMode { } export function isComputeModeAvailable(mode: ComputeMode): boolean { - if (mode === 'worker') { - throw new Error( - 'COMPUTE_MODE=worker is not implemented yet in v1. Switch to local/none or implement WorkerComputeBackend (v2).', - ); - } return mode !== 'none'; } diff --git a/src/lib/server/compute/types.ts b/src/lib/server/compute/types.ts index 3a411eb..2acea80 100644 --- a/src/lib/server/compute/types.ts +++ b/src/lib/server/compute/types.ts @@ -1,10 +1,10 @@ -import type { TTSAudioBuffer, TTSSentenceAlignment } from '@/types/tts'; -import type { ParsedPdfDocument } from '@/types/parsed-pdf'; +import type { TTSAudioBuffer, TTSSentenceAlignment, ParsedPdfDocument } from '@openreader/compute-core'; export type ComputeMode = 'local' | 'worker' | 'none'; export interface WhisperAlignInput { - audioBuffer: TTSAudioBuffer; + audioBuffer?: TTSAudioBuffer; + audioObjectKey?: string; text: string; cacheKey?: string; lang?: string; @@ -16,7 +16,9 @@ export interface WhisperAlignResult { export interface PdfLayoutInput { documentId: string; - pdfBytes: ArrayBuffer; + namespace?: string | null; + documentObjectKey?: string; + pdfBytes?: ArrayBuffer; } export interface ComputeBackend { diff --git a/src/lib/server/compute/worker-contract.ts b/src/lib/server/compute/worker-contract.ts new file mode 100644 index 0000000..ec4df20 --- /dev/null +++ b/src/lib/server/compute/worker-contract.ts @@ -0,0 +1,11 @@ +export { + ALIGN_QUEUE_NAME, + PDF_LAYOUT_QUEUE_NAME, + type PdfLayoutJobRequest, + type PdfLayoutJobResult, + type WhisperAlignJobRequest, + type WhisperAlignJobResult, + type WorkerJobErrorShape, + type WorkerJobState, + type WorkerJobStatusResponse, +} from '@openreader/compute-core'; diff --git a/src/lib/server/compute/worker.ts b/src/lib/server/compute/worker.ts new file mode 100644 index 0000000..79af3ba --- /dev/null +++ b/src/lib/server/compute/worker.ts @@ -0,0 +1,165 @@ +import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types'; +import type { + PdfLayoutJobRequest, + PdfLayoutJobResult, + WhisperAlignJobRequest, + WhisperAlignJobResult, + WorkerJobStatusResponse, +} from '@/lib/server/compute/worker-contract'; + +class WorkerHttpError extends Error { + status: number; + retryAfterMs: number | null; + + constructor(message: string, status: number, retryAfterMs: number | null = null) { + super(message); + this.name = 'WorkerHttpError'; + this.status = status; + this.retryAfterMs = retryAfterMs; + } +} + +const DEFAULT_WAIT_TIMEOUT_MS = 45_000; +const DEFAULT_RETRIES = 2; +const POLL_INTERVAL_MS = 400; +const POLL_MAX_INTERVAL_MS = 1_500; + +function readRequiredEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required when COMPUTE_MODE=worker`); + return value; +} + +function parseRetryAfterMs(value: string | null): number | null { + if (!value) return null; + const asNum = Number(value); + if (Number.isFinite(asNum)) { + return Math.max(0, Math.floor(asNum * 1000)); + } + const when = Date.parse(value); + if (Number.isNaN(when)) return null; + return Math.max(0, when - Date.now()); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function shouldRetry(error: unknown): boolean { + if (error instanceof WorkerHttpError) { + return error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504; + } + if (error instanceof Error) { + const msg = error.message.toLowerCase(); + return msg.includes('network') || msg.includes('timeout') || msg.includes('fetch failed'); + } + return false; +} + +async function withRetries(attempts: number, operation: () => Promise): Promise { + let lastError: unknown = null; + for (let attempt = 0; attempt < attempts; attempt += 1) { + try { + return await operation(); + } catch (error) { + lastError = error; + if (attempt === attempts - 1 || !shouldRetry(error)) break; + if (error instanceof WorkerHttpError && typeof error.retryAfterMs === 'number') { + await sleep(error.retryAfterMs); + } else { + await sleep((attempt + 1) * 250); + } + } + } + throw lastError instanceof Error ? lastError : new Error('Unknown worker compute failure'); +} + +export class WorkerComputeBackend implements ComputeBackend { + readonly mode = 'worker' as const; + private readonly baseUrl: string; + private readonly token: string; + private readonly waitTimeoutMs: number; + private readonly retries: number; + + constructor() { + this.baseUrl = readRequiredEnv('COMPUTE_WORKER_URL').replace(/\/+$/, ''); + this.token = readRequiredEnv('COMPUTE_WORKER_TOKEN'); + this.waitTimeoutMs = DEFAULT_WAIT_TIMEOUT_MS; + this.retries = DEFAULT_RETRIES; + } + + async alignWords(input: WhisperAlignInput): Promise { + if (!input.audioObjectKey) { + throw new Error('Worker compute alignment requires audioObjectKey'); + } + const payload: WhisperAlignJobRequest = { + text: input.text, + lang: input.lang, + cacheKey: input.cacheKey, + audioObjectKey: input.audioObjectKey, + }; + + return withRetries(this.retries, async () => { + const { jobId } = await this.requestJson<{ jobId: string }>('POST', '/align/whisper/jobs', payload); + const status = await this.waitForJob(`/align/whisper/jobs/${encodeURIComponent(jobId)}`); + if (status.status !== 'succeeded' || !status.result) { + throw new Error(status.error?.message || 'Whisper worker job did not complete'); + } + return { alignments: status.result.alignments }; + }); + } + + async parsePdfLayout(input: PdfLayoutInput) { + if (!input.documentObjectKey) { + throw new Error('Worker compute PDF layout requires documentObjectKey'); + } + const payload: PdfLayoutJobRequest = { + documentId: input.documentId, + namespace: input.namespace ?? null, + documentObjectKey: input.documentObjectKey, + }; + return withRetries(this.retries, async () => { + const { jobId } = await this.requestJson<{ jobId: string }>('POST', '/layout/pdf/jobs', payload); + const status = await this.waitForJob(`/layout/pdf/jobs/${encodeURIComponent(jobId)}`); + if (status.status !== 'succeeded' || !status.result) { + throw new Error(status.error?.message || 'PDF layout worker job did not complete'); + } + return status.result.parsed; + }); + } + + private async requestJson(method: 'GET' | 'POST', path: string, body?: unknown): Promise { + const res = await fetch(`${this.baseUrl}${path}`, { + method, + headers: { + Authorization: `Bearer ${this.token}`, + ...(method === 'POST' ? { 'Content-Type': 'application/json' } : {}), + }, + ...(method === 'POST' ? { body: JSON.stringify(body ?? {}) } : {}), + }); + + if (!res.ok) { + const retryAfterMs = parseRetryAfterMs(res.headers.get('retry-after')); + const detail = await res.text().catch(() => ''); + throw new WorkerHttpError( + `Worker request failed (${method} ${path}): ${res.status}${detail ? ` ${detail}` : ''}`, + res.status, + retryAfterMs, + ); + } + + return res.json() as Promise; + } + + private async waitForJob(path: string): Promise> { + const started = Date.now(); + let interval = POLL_INTERVAL_MS; + while ((Date.now() - started) < this.waitTimeoutMs) { + const status = await this.requestJson>('GET', path); + if (status.status === 'succeeded' || status.status === 'failed') return status; + await sleep(interval); + interval = Math.min(POLL_MAX_INTERVAL_MS, Math.floor(interval * 1.5)); + } + throw new Error(`Timed out waiting for worker job after ${this.waitTimeoutMs}ms`); + } +} diff --git a/src/lib/server/jobs/parsePdfJob.ts b/src/lib/server/jobs/parsePdfJob.ts index eab9f02..fba48e4 100644 --- a/src/lib/server/jobs/parsePdfJob.ts +++ b/src/lib/server/jobs/parsePdfJob.ts @@ -2,7 +2,7 @@ import { and, eq } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; import { UnsupportedComputeError } from '@/lib/server/compute/types'; -import { getDocumentBlob, putParsedDocumentBlob } from '@/lib/server/documents/blobstore'; +import { documentKey, putParsedDocumentBlob } from '@/lib/server/documents/blobstore'; import { getCompute } from '@/lib/server/compute'; import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache'; @@ -29,10 +29,10 @@ export async function parsePdfJob(input: ParsePdfJobInput): Promise { .set({ parseStatus: 'running' }) .where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId))); - const pdfBytes = await getDocumentBlob(input.documentId, input.namespace); const parsed = await getCompute().parsePdfLayout({ documentId: input.documentId, - pdfBytes: new Uint8Array(pdfBytes).buffer, + namespace: input.namespace, + documentObjectKey: documentKey(input.documentId, input.namespace), }); const parsedJson = Buffer.from(JSON.stringify(parsed)); diff --git a/tsconfig.json b/tsconfig.json index 3dfc4fb..00d1d04 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -19,7 +19,8 @@ } ], "paths": { - "@/*": ["./src/*"] + "@/*": ["./src/*"], + "@openreader/compute-core": ["./compute/core/src/index.ts"] } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],