diff --git a/.env.example b/.env.example index 36cc589..7ed67d1 100644 --- a/.env.example +++ b/.env.example @@ -96,6 +96,10 @@ IMPORT_LIBRARY_DIRS= # COMPUTE_OP_STALE_MS=1800000 # Optional Whisper ONNX base URL override (must contain all expected files) +# Default expected Whisper variant is q4: +# - onnx/encoder_model_q4.onnx +# - onnx/decoder_model_merged_q4.onnx +# - onnx/decoder_with_past_model_q4.onnx # WHISPER_MODEL_BASE_URL=https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main # Optional PDF layout ONNX base URL override (must contain all expected files) diff --git a/compute/core/src/whisper/assets/manifest.json b/compute/core/src/whisper/assets/manifest.json index 2fffd3d..b1adda1 100644 --- a/compute/core/src/whisper/assets/manifest.json +++ b/compute/core/src/whisper/assets/manifest.json @@ -1,5 +1,5 @@ { - "name": "whisper-base_timestamped-int8", + "name": "whisper-base_timestamped-q4", "version": "onnx-community/whisper-base_timestamped@608c49e61301901684bc36cac8f74b95ff6b5a8e", "files": [ { @@ -53,19 +53,19 @@ "size": 2194 }, { - "path": "onnx/encoder_model_int8.onnx", - "sha256": "152da96dd8ff3f28f3fadabc2e8960405a277846453ff94ed411fe935a72917f", - "size": 23159150 + "path": "onnx/encoder_model_q4.onnx", + "sha256": "0df94e35822653ba16e23c3f19f05b9b7fe7aae97884d1b277e02044f33c4880", + "size": 18771046 }, { - "path": "onnx/decoder_model_merged_int8.onnx", - "sha256": "cf9a8d5bcddc0917a0078135b484cedcaf44f28909cd91910abd29dced9171db", - "size": 53712708 + "path": "onnx/decoder_model_merged_q4.onnx", + "sha256": "fc1902ce2e42c69b2346d8e2a98898c60c01da1e6a64ae90f41d22350ac7db13", + "size": 123738327 }, { - "path": "onnx/decoder_with_past_model_int8.onnx", - "sha256": "bdd92860d0ed7dff2aca623963378cbba1b617bfae127356db1c8aa8baa930ef", - "size": 50131672 + "path": "onnx/decoder_with_past_model_q4.onnx", + "sha256": "d864ca26509968b00d92e6823c4db7ac460c106f9228a21bc5199e9893fe4126", + "size": 121378741 }, { "path": "LICENSE.txt", diff --git a/compute/core/src/whisper/model.ts b/compute/core/src/whisper/model.ts index 100bc6f..4c5a021 100644 --- a/compute/core/src/whisper/model.ts +++ b/compute/core/src/whisper/model.ts @@ -14,9 +14,9 @@ export const WHISPER_CONFIG_PATH = path.join(MODEL_DIR, 'config.json'); export const WHISPER_GENERATION_CONFIG_PATH = path.join(MODEL_DIR, 'generation_config.json'); export const WHISPER_TOKENIZER_PATH = path.join(MODEL_DIR, 'tokenizer.json'); export const WHISPER_TOKENIZER_CONFIG_PATH = path.join(MODEL_DIR, 'tokenizer_config.json'); -export const WHISPER_ENCODER_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'encoder_model_int8.onnx'); -export const WHISPER_DECODER_MERGED_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'decoder_model_merged_int8.onnx'); -export const WHISPER_DECODER_WITH_PAST_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'decoder_with_past_model_int8.onnx'); +export const WHISPER_ENCODER_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'encoder_model_q4.onnx'); +export const WHISPER_DECODER_MERGED_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'decoder_model_merged_q4.onnx'); +export const WHISPER_DECODER_WITH_PAST_MODEL_PATH = path.join(MODEL_DIR, 'onnx', 'decoder_with_past_model_q4.onnx'); const BASE_MODEL_URL = 'https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main'; const WHISPER_MODEL_BASE_URL_ENV = 'WHISPER_MODEL_BASE_URL'; @@ -32,9 +32,9 @@ const MODEL_RELATIVE_PATHS: string[] = [ 'added_tokens.json', 'preprocessor_config.json', 'special_tokens_map.json', - 'onnx/encoder_model_int8.onnx', - 'onnx/decoder_model_merged_int8.onnx', - 'onnx/decoder_with_past_model_int8.onnx', + 'onnx/encoder_model_q4.onnx', + 'onnx/decoder_model_merged_q4.onnx', + 'onnx/decoder_with_past_model_q4.onnx', ]; const DEFAULT_URLS: Record = { @@ -48,9 +48,9 @@ const DEFAULT_URLS: Record = { 'added_tokens.json': `${BASE_MODEL_URL}/added_tokens.json`, 'preprocessor_config.json': `${BASE_MODEL_URL}/preprocessor_config.json`, 'special_tokens_map.json': `${BASE_MODEL_URL}/special_tokens_map.json`, - 'onnx/encoder_model_int8.onnx': `${BASE_MODEL_URL}/onnx/encoder_model_int8.onnx`, - 'onnx/decoder_model_merged_int8.onnx': `${BASE_MODEL_URL}/onnx/decoder_model_merged_int8.onnx`, - 'onnx/decoder_with_past_model_int8.onnx': `${BASE_MODEL_URL}/onnx/decoder_with_past_model_int8.onnx`, + 'onnx/encoder_model_q4.onnx': `${BASE_MODEL_URL}/onnx/encoder_model_q4.onnx`, + 'onnx/decoder_model_merged_q4.onnx': `${BASE_MODEL_URL}/onnx/decoder_model_merged_q4.onnx`, + 'onnx/decoder_with_past_model_q4.onnx': `${BASE_MODEL_URL}/onnx/decoder_with_past_model_q4.onnx`, }; type ManifestEntry = { path: string; sha256?: string; size?: number }; diff --git a/compute/worker/.env.example b/compute/worker/.env.example index 3c38f5d..12bc936 100644 --- a/compute/worker/.env.example +++ b/compute/worker/.env.example @@ -44,5 +44,9 @@ S3_FORCE_PATH_STYLE=true # COMPUTE_NATS_REPLICAS=1 # COMPUTE_OP_STALE_MS=1800000 # Optional model mirrors +# Default expected Whisper variant is q4: +# - onnx/encoder_model_q4.onnx +# - onnx/decoder_model_merged_q4.onnx +# - onnx/decoder_with_past_model_q4.onnx # WHISPER_MODEL_BASE_URL=https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main # PDF_LAYOUT_MODEL_BASE_URL=https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main diff --git a/docs-site/docs/deploy/compute-worker.md b/docs-site/docs/deploy/compute-worker.md index 5f027ee..4b220bd 100644 --- a/docs-site/docs/deploy/compute-worker.md +++ b/docs-site/docs/deploy/compute-worker.md @@ -55,7 +55,7 @@ Advanced tuning (usually leave unset unless you need overrides): - `COMPUTE_JOB_CONCURRENCY=1` (shared total compute jobs across whisper + PDF) - `COMPUTE_WHISPER_TIMEOUT_MS=30000` - `COMPUTE_PDF_TIMEOUT_MS=300000` -- `WHISPER_MODEL_BASE_URL=https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main` (optional override) +- `WHISPER_MODEL_BASE_URL=https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main` (optional override, q4 defaults) - `PDF_LAYOUT_MODEL_BASE_URL=https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main` (optional override) - `COMPUTE_PDF_JOB_ATTEMPTS=1` (PDF layout retry attempts) - `COMPUTE_JOBS_STREAM_MAX_BYTES=268435456` (256MB JetStream jobs stream cap) @@ -79,7 +79,7 @@ COMPUTE_WORKER_TOKEN= # COMPUTE_OP_STALE_MS=1800000 ``` -Model artifact overrides (`WHISPER_MODEL_BASE_URL`, `PDF_LAYOUT_MODEL_BASE_URL`) are worker runtime variables and should be set on the compute worker service environment. +Model artifact overrides (`WHISPER_MODEL_BASE_URL`, `PDF_LAYOUT_MODEL_BASE_URL`) are worker runtime variables and should be set on the compute worker service environment. Current Whisper defaults expect q4 artifacts (`encoder_model_q4.onnx`, `decoder_model_merged_q4.onnx`, `decoder_with_past_model_q4.onnx`) under that base URL. `COMPUTE_OP_STALE_MS` is shared by both services in worker mode: @@ -168,6 +168,10 @@ COMPUTE_WORKER_TOKEN= # COMPUTE_WHISPER_TIMEOUT_MS=30000 # COMPUTE_PDF_TIMEOUT_MS=300000 # WHISPER_MODEL_BASE_URL=https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main +# # Expects q4 files at that base: +# # - onnx/encoder_model_q4.onnx +# # - onnx/decoder_model_merged_q4.onnx +# # - onnx/decoder_with_past_model_q4.onnx # PDF_LAYOUT_MODEL_BASE_URL=https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main # COMPUTE_PDF_JOB_ATTEMPTS=1 # COMPUTE_JOBS_STREAM_MAX_BYTES=268435456 diff --git a/docs-site/docs/deploy/local-development.md b/docs-site/docs/deploy/local-development.md index 14ca0a4..8498144 100644 --- a/docs-site/docs/deploy/local-development.md +++ b/docs-site/docs/deploy/local-development.md @@ -155,7 +155,7 @@ No extra native Whisper CLI build step is required. Word-by-word highlighting and PDF layout parsing are worker-backed in current releases. -If you need mirrors or pinned artifact locations, set `WHISPER_MODEL_BASE_URL` in `.env`. +If you need mirrors or pinned artifact locations, set `WHISPER_MODEL_BASE_URL` in `.env` (current defaults expect q4 Whisper files at that base URL). diff --git a/docs-site/docs/reference/environment-variables.md b/docs-site/docs/reference/environment-variables.md index 005ef32..f479496 100644 --- a/docs-site/docs/reference/environment-variables.md +++ b/docs-site/docs/reference/environment-variables.md @@ -65,7 +65,7 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `COMPUTE_PDF_TIMEOUT_MS` | Heavy compute backend | `300000` | Shared PDF idle-timeout budget (worker + worker client wait budget) | | `COMPUTE_OP_STALE_MS` | Heavy compute backend | `max(30m, 4x max compute timeout)` | Shared stale window for worker op replacement and app-side stale PDF parse-state healing | | `PDF_LAYOUT_MODEL_BASE_URL` | PDF layout model | PP-DocLayoutV3 ONNX base URL | Optional base URL override for `ensureModel()` | -| `WHISPER_MODEL_BASE_URL` | Whisper ONNX model | onnx-community defaults | Optional base URL override for ONNX whisper-base_timestamped int8 downloads | +| `WHISPER_MODEL_BASE_URL` | Whisper ONNX model | onnx-community defaults | Optional base URL override for ONNX whisper-base_timestamped q4 downloads | | `FFMPEG_BIN` | Audio runtime | auto-detected (`ffmpeg-static`) | Override ffmpeg binary path | @@ -460,7 +460,7 @@ Optional base URL override for PP-DocLayoutV3 artifacts downloaded by `ensureMod Optional base URL override for the built-in ONNX Whisper alignment model downloader. - Default: `https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main` -- Default model variant: int8 (`encoder_model_int8.onnx`, `decoder_model_merged_int8.onnx`, `decoder_with_past_model_int8.onnx`) +- Default model variant: q4 (`encoder_model_q4.onnx`, `decoder_model_merged_q4.onnx`, `decoder_with_past_model_q4.onnx`) - The base URL must host all expected manifest files under the same relative paths. - Configure this on the worker service env (not only the app server env) diff --git a/docs-site/docs/reference/stack.md b/docs-site/docs/reference/stack.md index 49f5f09..ea847e6 100644 --- a/docs-site/docs/reference/stack.md +++ b/docs-site/docs/reference/stack.md @@ -49,7 +49,7 @@ Monorepo packages under `compute/`: - **`@openreader/compute-core`** — ONNX runtime lifecycle, model management, and inference logic shared by compute worker runtime + app/worker contracts - ONNX runtime: `onnxruntime-node` with `@huggingface/tokenizers` - - Whisper alignment: `onnx-community/whisper-base_timestamped` (int8) for word-level timestamps + - Whisper alignment: `onnx-community/whisper-base_timestamped` (q4) for word-level timestamps - PDF layout: `Bei0001/PP-DocLayoutV3-ONNX` for document block detection and layout parsing - PDF rendering: `pdfjs-dist`, `@napi-rs/canvas` for server-side page rasterization - Utilities: `jszip`, `ffmpeg-static`