From cd530a365d7dddc5f72a297c499cbe9ab6b07b14 Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 26 May 2026 19:22:07 -0600 Subject: [PATCH] refactor(whisper): migrate default ONNX model references to q4 variant Update all configuration, manifest, and documentation references to use the q4 Whisper ONNX model files instead of int8. Adjust expected file names, hashes, and environment variable descriptions to reflect this new default. This aligns runtime, deployment, and developer documentation with the updated model artifact expectations for improved consistency. --- .env.example | 4 ++++ compute/core/src/whisper/assets/manifest.json | 20 +++++++++---------- compute/core/src/whisper/model.ts | 18 ++++++++--------- compute/worker/.env.example | 4 ++++ docs-site/docs/deploy/compute-worker.md | 8 ++++++-- docs-site/docs/deploy/local-development.md | 2 +- .../docs/reference/environment-variables.md | 4 ++-- docs-site/docs/reference/stack.md | 2 +- 8 files changed, 37 insertions(+), 25 deletions(-) diff --git a/.env.example b/.env.example index 36cc589..7ed67d1 100644 --- a/.env.example +++ b/.env.example @@ -96,6 +96,10 @@ IMPORT_LIBRARY_DIRS= # COMPUTE_OP_STALE_MS=1800000 # Optional Whisper ONNX base URL override (must contain all expected files) +# Default expected Whisper variant is q4: +# - onnx/encoder_model_q4.onnx +# - onnx/decoder_model_merged_q4.onnx +# - onnx/decoder_with_past_model_q4.onnx # WHISPER_MODEL_BASE_URL=https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main # Optional PDF layout ONNX base URL override (must contain all expected files) diff --git a/compute/core/src/whisper/assets/manifest.json b/compute/core/src/whisper/assets/manifest.json index 2fffd3d..b1adda1 100644 --- a/compute/core/src/whisper/assets/manifest.json +++ b/compute/core/src/whisper/assets/manifest.json @@ -1,5 +1,5 @@ { - "name": "whisper-base_timestamped-int8", + "name": "whisper-base_timestamped-q4", "version": "onnx-community/whisper-base_timestamped@608c49e61301901684bc36cac8f74b95ff6b5a8e", "files": [ { @@ -53,19 +53,19 @@ "size": 2194 }, { - "path": "onnx/encoder_model_int8.onnx", - "sha256": "152da96dd8ff3f28f3fadabc2e8960405a277846453ff94ed411fe935a72917f", - "size": 23159150 + "path": "onnx/encoder_model_q4.onnx", + "sha256": "0df94e35822653ba16e23c3f19f05b9b7fe7aae97884d1b277e02044f33c4880", + "size": 18771046 }, { - "path": "onnx/decoder_model_merged_int8.onnx", - "sha256": "cf9a8d5bcddc0917a0078135b484cedcaf44f28909cd91910abd29dced9171db", - "size": 53712708 + "path": "onnx/decoder_model_merged_q4.onnx", + "sha256": "fc1902ce2e42c69b2346d8e2a98898c60c01da1e6a64ae90f41d22350ac7db13", + "size": 123738327 }, { - "path": "onnx/decoder_with_past_model_int8.onnx", - "sha256": "bdd92860d0ed7dff2aca623963378cbba1b617bfae127356db1c8aa8baa930ef", - "size": 50131672 + "path": "onnx/decoder_with_past_model_q4.onnx", + "sha256": "d864ca26509968b00d92e6823c4db7ac460c106f9228a21bc5199e9893fe4126", + "size": 121378741 }, { "path": "LICENSE.txt", diff --git a/compute/core/src/whisper/model.ts b/compute/core/src/whisper/model.ts index 100bc6f..4c5a021 100644 --- a/compute/core/src/whisper/model.ts +++ b/compute/core/src/whisper/model.ts @@ -14,9 +14,9 @@ export const WHISPER_CONFIG_PATH = path.join(MODEL_DIR, 'config.json'); export const WHISPER_GENERATION_CONFIG_PATH = path.join(MODEL_DIR, 'generation_config.json'); export const WHISPER_TOKENIZER_PATH = path.join(MODEL_DIR, 'tokenizer.json'); export const WHISPER_TOKENIZER_CONFIG_PATH = path.join(MODEL_DIR, 'tokenizer_config.json'); -export const WHISPER_ENCODER_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'encoder_model_int8.onnx'); -export const WHISPER_DECODER_MERGED_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'decoder_model_merged_int8.onnx'); -export const WHISPER_DECODER_WITH_PAST_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'decoder_with_past_model_int8.onnx'); +export const WHISPER_ENCODER_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'encoder_model_q4.onnx'); +export const WHISPER_DECODER_MERGED_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'decoder_model_merged_q4.onnx'); +export const WHISPER_DECODER_WITH_PAST_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'decoder_with_past_model_q4.onnx'); const BASE_MODEL_URL = 'https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main'; const WHISPER_MODEL_BASE_URL_ENV = 'WHISPER_MODEL_BASE_URL'; @@ -32,9 +32,9 @@ const MODEL_RELATIVE_PATHS: string[] = [ 'added_tokens.json', 'preprocessor_config.json', 'special_tokens_map.json', - 'onnx/encoder_model_int8.onnx', - 'onnx/decoder_model_merged_int8.onnx', - 'onnx/decoder_with_past_model_int8.onnx', + 'onnx/encoder_model_q4.onnx', + 'onnx/decoder_model_merged_q4.onnx', + 'onnx/decoder_with_past_model_q4.onnx', ]; const DEFAULT_URLS: Record = { @@ -48,9 +48,9 @@ const DEFAULT_URLS: Record = { 'added_tokens.json': `${BASE_MODEL_URL}/added_tokens.json`, 'preprocessor_config.json': `${BASE_MODEL_URL}/preprocessor_config.json`, 'special_tokens_map.json': `${BASE_MODEL_URL}/special_tokens_map.json`, - 'onnx/encoder_model_int8.onnx': `${BASE_MODEL_URL}/onnx/encoder_model_int8.onnx`, - 'onnx/decoder_model_merged_int8.onnx': `${BASE_MODEL_URL}/onnx/decoder_model_merged_int8.onnx`, - 'onnx/decoder_with_past_model_int8.onnx': `${BASE_MODEL_URL}/onnx/decoder_with_past_model_int8.onnx`, + 'onnx/encoder_model_q4.onnx': `${BASE_MODEL_URL}/onnx/encoder_model_q4.onnx`, + 'onnx/decoder_model_merged_q4.onnx': `${BASE_MODEL_URL}/onnx/decoder_model_merged_q4.onnx`, + 'onnx/decoder_with_past_model_q4.onnx': `${BASE_MODEL_URL}/onnx/decoder_with_past_model_q4.onnx`, }; type ManifestEntry = { path: string; sha256?: string; size?: number }; diff --git a/compute/worker/.env.example b/compute/worker/.env.example index 3c38f5d..12bc936 100644 --- a/compute/worker/.env.example +++ b/compute/worker/.env.example @@ -44,5 +44,9 @@ S3_FORCE_PATH_STYLE=true # COMPUTE_NATS_REPLICAS=1 # COMPUTE_OP_STALE_MS=1800000 # Optional model mirrors +# Default expected Whisper variant is q4: +# - onnx/encoder_model_q4.onnx +# - onnx/decoder_model_merged_q4.onnx +# - onnx/decoder_with_past_model_q4.onnx # WHISPER_MODEL_BASE_URL=https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main # PDF_LAYOUT_MODEL_BASE_URL=https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main diff --git a/docs-site/docs/deploy/compute-worker.md b/docs-site/docs/deploy/compute-worker.md index 5f027ee..4b220bd 100644 --- a/docs-site/docs/deploy/compute-worker.md +++ b/docs-site/docs/deploy/compute-worker.md @@ -55,7 +55,7 @@ Advanced tuning (usually leave unset unless you need overrides): - `COMPUTE_JOB_CONCURRENCY=1` (shared total compute jobs across whisper + PDF) - `COMPUTE_WHISPER_TIMEOUT_MS=30000` - `COMPUTE_PDF_TIMEOUT_MS=300000` -- `WHISPER_MODEL_BASE_URL=https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main` (optional override) +- `WHISPER_MODEL_BASE_URL=https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main` (optional override, q4 defaults) - `PDF_LAYOUT_MODEL_BASE_URL=https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main` (optional override) - `COMPUTE_PDF_JOB_ATTEMPTS=1` (PDF layout retry attempts) - `COMPUTE_JOBS_STREAM_MAX_BYTES=268435456` (256MB JetStream jobs stream cap) @@ -79,7 +79,7 @@ COMPUTE_WORKER_TOKEN= # COMPUTE_OP_STALE_MS=1800000 ``` -Model artifact overrides (`WHISPER_MODEL_BASE_URL`, `PDF_LAYOUT_MODEL_BASE_URL`) are worker runtime variables and should be set on the compute worker service environment. +Model artifact overrides (`WHISPER_MODEL_BASE_URL`, `PDF_LAYOUT_MODEL_BASE_URL`) are worker runtime variables and should be set on the compute worker service environment. Current Whisper defaults expect q4 artifacts (`encoder_model_q4.onnx`, `decoder_model_merged_q4.onnx`, `decoder_with_past_model_q4.onnx`) under that base URL. `COMPUTE_OP_STALE_MS` is shared by both services in worker mode: @@ -168,6 +168,10 @@ COMPUTE_WORKER_TOKEN= # COMPUTE_WHISPER_TIMEOUT_MS=30000 # COMPUTE_PDF_TIMEOUT_MS=300000 # WHISPER_MODEL_BASE_URL=https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main +# # Expects q4 files at that base: +# # - onnx/encoder_model_q4.onnx +# # - onnx/decoder_model_merged_q4.onnx +# # - onnx/decoder_with_past_model_q4.onnx # PDF_LAYOUT_MODEL_BASE_URL=https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main # COMPUTE_PDF_JOB_ATTEMPTS=1 # COMPUTE_JOBS_STREAM_MAX_BYTES=268435456 diff --git a/docs-site/docs/deploy/local-development.md b/docs-site/docs/deploy/local-development.md index 14ca0a4..8498144 100644 --- a/docs-site/docs/deploy/local-development.md +++ b/docs-site/docs/deploy/local-development.md @@ -155,7 +155,7 @@ No extra native Whisper CLI build step is required. Word-by-word highlighting and PDF layout parsing are worker-backed in current releases. -If you need mirrors or pinned artifact locations, set `WHISPER_MODEL_BASE_URL` in `.env`. +If you need mirrors or pinned artifact locations, set `WHISPER_MODEL_BASE_URL` in `.env` (current defaults expect q4 Whisper files at that base URL). diff --git a/docs-site/docs/reference/environment-variables.md b/docs-site/docs/reference/environment-variables.md index 005ef32..f479496 100644 --- a/docs-site/docs/reference/environment-variables.md +++ b/docs-site/docs/reference/environment-variables.md @@ -65,7 +65,7 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `COMPUTE_PDF_TIMEOUT_MS` | Heavy compute backend | `300000` | Shared PDF idle-timeout budget (worker + worker client wait budget) | | `COMPUTE_OP_STALE_MS` | Heavy compute backend | `max(30m, 4x max compute timeout)` | Shared stale window for worker op replacement and app-side stale PDF parse-state healing | | `PDF_LAYOUT_MODEL_BASE_URL` | PDF layout model | PP-DocLayoutV3 ONNX base URL | Optional base URL override for `ensureModel()` | -| `WHISPER_MODEL_BASE_URL` | Whisper ONNX model | onnx-community defaults | Optional base URL override for ONNX whisper-base_timestamped int8 downloads | +| `WHISPER_MODEL_BASE_URL` | Whisper ONNX model | onnx-community defaults | Optional base URL override for ONNX whisper-base_timestamped q4 downloads | | `FFMPEG_BIN` | Audio runtime | auto-detected (`ffmpeg-static`) | Override ffmpeg binary path | @@ -460,7 +460,7 @@ Optional base URL override for PP-DocLayoutV3 artifacts downloaded by `ensureMod Optional base URL override for the built-in ONNX Whisper alignment model downloader. - Default: `https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main` -- Default model variant: int8 (`encoder_model_int8.onnx`, `decoder_model_merged_int8.onnx`, `decoder_with_past_model_int8.onnx`) +- Default model variant: q4 (`encoder_model_q4.onnx`, `decoder_model_merged_q4.onnx`, `decoder_with_past_model_q4.onnx`) - The base URL must host all expected manifest files under the same relative paths. - Configure this on the worker service env (not only the app server env) diff --git a/docs-site/docs/reference/stack.md b/docs-site/docs/reference/stack.md index 49f5f09..ea847e6 100644 --- a/docs-site/docs/reference/stack.md +++ b/docs-site/docs/reference/stack.md @@ -49,7 +49,7 @@ Monorepo packages under `compute/`: - **`@openreader/compute-core`** — ONNX runtime lifecycle, model management, and inference logic shared by compute worker runtime + app/worker contracts - ONNX runtime: `onnxruntime-node` with `@huggingface/tokenizers` - - Whisper alignment: `onnx-community/whisper-base_timestamped` (int8) for word-level timestamps + - Whisper alignment: `onnx-community/whisper-base_timestamped` (q4) for word-level timestamps - PDF layout: `Bei0001/PP-DocLayoutV3-ONNX` for document block detection and layout parsing - PDF rendering: `pdfjs-dist`, `@napi-rs/canvas` for server-side page rasterization - Utilities: `jszip`, `ffmpeg-static`