diff --git a/.env.example b/.env.example index 69c8ed0..e316a0f 100644 --- a/.env.example +++ b/.env.example @@ -15,15 +15,6 @@ API_BASE=http://localhost:8880/v1 API_KEY=api_key_optional -# (Optional) TTS request/cache tuning (defaults shown below; leave unset to use defaults) -# TTS_CACHE_MAX_SIZE_BYTES=268435456 # 256MB -# TTS_CACHE_TTL_MS=1800000 # 30 minutes -# TTS_MAX_RETRIES=2 -# TTS_RETRY_INITIAL_MS=250 -# TTS_RETRY_MAX_MS=2000 -# TTS_RETRY_BACKOFF=2 -# TTS_UPSTREAM_TIMEOUT_MS=285000 # 285 seconds - # Auth configuration (recommended; required for admin features) # (Optional) Auth is only enabled when AUTH_SECRET and BASE_URL are set BASE_URL=http://localhost:3003 # Externally facing URL for this app (set to LAN IP for access from other devices on the network) @@ -34,19 +25,8 @@ AUTH_TRUSTED_ORIGINS=http://localhost:3003,http://127.0.0.1:3003 # Additional tr # (Optional) Sign in w/ GitHub Configuration # GITHUB_CLIENT_ID= # GITHUB_CLIENT_SECRET= -# (Optional) Disable Better Auth built-in rate limiting (useful for testing; default: `false`) -# DISABLE_AUTH_RATE_LIMIT=false - -# (Optional) Honor the x-openreader-test-namespace header on a production build. -# This is test/CI scaffolding and MUST stay unset on real deployments; it is set -# automatically by the Playwright web server. Non-production builds honor it -# without this flag. -# ENABLE_TEST_NAMESPACE=false # (Optional) Comma-separated list of emails that are auto-promoted to admin. -# Admins see the "Admin" tab in Settings (TTS shared providers + site features). -# Demotion is automatic: removing an email here demotes the user on next login. -# Requires auth to be enabled (AUTH_SECRET + BASE_URL). ADMIN_EMAILS= # (Optional) Backend DB used for server-side metadata (documents/audiobooks) and, when auth is enabled, auth tables. @@ -74,47 +54,28 @@ S3_BUCKET= # IMPORT_LIBRARY_DIR= # IMPORT_LIBRARY_DIRS= -# Compute -# Embedded/local (default): leave COMPUTE_WORKER_URL empty. -# External worker: set COMPUTE_WORKER_URL + COMPUTE_WORKER_TOKEN. -# External worker split: -# - App side (this root `.env`): set only routing/auth + shared timeout/stale overrides. -# - Worker side (`compute/worker/.env*` or worker platform env): set NATS_*, S3_*, model base URLs, and worker tuning. -# Details: docs/deploy/compute-worker -# COMPUTE_WORKER_URL=http://localhost:8081 -# COMPUTE_WORKER_TOKEN=local-compute-token -# Optional embedded startup controls: +# Compute worker configuration +# (Optional) Embedded compute worker (automatic startup) for local compute tasks (defaults shown below) # EMBEDDED_COMPUTE_WORKER_PORT=8081 # EMBEDDED_NATS_PORT=4222 # EMBEDDED_NATS_MONITOR_PORT=8222 # EMBEDDED_NATS_STORE_DIR=docstore/nats/jetstream -# `NATS_URL` here is for embedded startup only. In external worker mode, set `NATS_URL` on the worker service env. # NATS_URL=nats://127.0.0.1:4222 -# Optional worker log level: # COMPUTE_LOG_LEVEL=info -# Optional shared compute tuning: # COMPUTE_JOB_CONCURRENCY=1 # COMPUTE_WHISPER_TIMEOUT_MS=30000 # COMPUTE_PDF_TIMEOUT_MS=300000 # COMPUTE_OP_STALE_MS=1800000 - -# Optional Whisper ONNX base URL override (must contain all expected files) -# Default expected Whisper variant is q4: -# - onnx/encoder_model_q4.onnx -# - onnx/decoder_model_merged_q4.onnx -# - onnx/decoder_with_past_model_q4.onnx # WHISPER_MODEL_BASE_URL=https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main - -# Optional PDF layout ONNX base URL override (must contain all expected files) # PDF_LAYOUT_MODEL_BASE_URL=https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main +# (Optional) External compute worker config (see docs/deploy/compute-worker) +# COMPUTE_WORKER_URL=http://localhost:8081 +# COMPUTE_WORKER_TOKEN=local-compute-token # (Optional) Override ffmpeg binary path used for audiobook processing # FFMPEG_BIN= -# (Optional) Client feature flags — seeded into the admin-managed runtime -# config on first boot, then ignored. Edit values from Settings → Admin → -# Site features instead of redeploying. SSR-injected so they take effect -# without rebuilding (unlike the old build-time public-env pattern). +# (Optional) Values seeded into the admin-managed runtime config on first boot, then ignored. # RUNTIME_SEED_ENABLE_DOCX_CONVERSION=true # RUNTIME_SEED_ENABLE_DESTRUCTIVE_DELETE_ACTIONS=true # RUNTIME_SEED_ENABLE_TTS_PROVIDERS_TAB=true @@ -126,7 +87,10 @@ S3_BUCKET= # RUNTIME_SEED_DISABLE_TTS_LIMIT=true # RUNTIME_SEED_DISABLE_COMPUTE_LIMIT=true -# Migrations configuration +# (Optional) Test/dev overrides +# DISABLE_AUTH_RATE_LIMIT=false +# ENABLE_TEST_NAMESPACE=false + # (Optional) Skip automatic startup migrations when set to `false` (default: `true`) # RUN_DRIZZLE_MIGRATIONS=true # (Optional) Skip automatic filesystem->S3/DB migration pass when set to `false` (default: `true`) diff --git a/docs-site/docs/configure/admin-panel.md b/docs-site/docs/configure/admin-panel.md index f8d32df..706115b 100644 --- a/docs-site/docs/configure/admin-panel.md +++ b/docs-site/docs/configure/admin-panel.md @@ -105,6 +105,19 @@ The **Disable TTS daily rate limiting** and **Disable PDF parsing rate limiting* - TTS: anonymous/authenticated per-user daily limits and anonymous/authenticated IP daily backstops. - PDF parsing: burst limit + window (seconds) and sustained limit + window (seconds). The sustained window doubles as a concurrency cap. +## TTS upstream + +At the end of the **Site features** tab, a dedicated **TTS upstream** group controls server-side request and cache tuning (DB-backed runtime settings, not env vars): + +| Key | What it controls | +| --- | --- | +| `ttsUpstreamMaxRetries` | Maximum retry attempts for upstream TTS 429/5xx responses. | +| `ttsUpstreamTimeoutMs` | Upstream request timeout for OpenAI-compatible TTS calls. | +| `ttsCacheMaxSizeBytes` | Maximum size of the in-memory TTS audio cache. | +| `ttsCacheTtlMs` | Time-to-live for cached TTS audio buffers. | + +In v4 these settings are admin-only and are no longer configurable through environment variables. + ## Migrating off env vars The future-direction goal is to remove `RUNTIME_SEED_*` / `API_KEY` / `API_BASE` from your `.env` entirely. To do that safely: diff --git a/docs-site/docs/reference/environment-variables.md b/docs-site/docs/reference/environment-variables.md index 2fc71f9..cc1c680 100644 --- a/docs-site/docs/reference/environment-variables.md +++ b/docs-site/docs/reference/environment-variables.md @@ -17,21 +17,12 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `LOG_LEVEL` | Runtime logging | `info` | Set app server log level | | `API_BASE` | Legacy bootstrap seed | none | Optional first-boot seed into `default-openai`; then manage in Settings → Admin → Shared providers | | `API_KEY` | Legacy bootstrap seed | none | Optional first-boot seed into `default-openai`; then manage in Settings → Admin → Shared providers | -| `TTS_CACHE_MAX_SIZE_BYTES` | TTS caching | `268435456` (256 MB) | Tune in-memory TTS cache size | -| `TTS_CACHE_TTL_MS` | TTS caching | `1800000` (30 min) | Tune in-memory TTS cache TTL | -| `TTS_MAX_RETRIES` | TTS retry | `2` | Tune retry attempts for upstream 429/5xx | -| `TTS_RETRY_INITIAL_MS` | TTS retry | `250` | Tune initial retry delay | -| `TTS_RETRY_MAX_MS` | TTS retry | `2000` | Tune max retry delay | -| `TTS_RETRY_BACKOFF` | TTS retry | `2` | Tune exponential backoff factor | -| `TTS_UPSTREAM_TIMEOUT_MS` | TTS request timeout | `285000` | Set max upstream TTS request duration before fail-fast | | `BASE_URL` | Auth | unset | Required (with `AUTH_SECRET`) to enable auth | | `AUTH_SECRET` | Auth | unset | Required (with `BASE_URL`) to enable auth | | `AUTH_TRUSTED_ORIGINS` | Auth | empty | Add extra allowed origins | | `USE_ANONYMOUS_AUTH_SESSIONS` | Auth | `false` | Set `true` to enable anonymous auth sessions | | `GITHUB_CLIENT_ID` | Auth/OAuth | unset | Set with `GITHUB_CLIENT_SECRET` to enable GitHub sign-in | | `GITHUB_CLIENT_SECRET` | Auth/OAuth | unset | Set with `GITHUB_CLIENT_ID` to enable GitHub sign-in | -| `DISABLE_AUTH_RATE_LIMIT` | Rate limiting | `false` | Set `true` to disable auth-layer rate limiting | -| `ENABLE_TEST_NAMESPACE` | Testing/CI | unset | Honor the `x-openreader-test-namespace` header on a production build; leave unset on real deployments | | `ADMIN_EMAILS` | Auth/Admin | empty | Comma-separated emails auto-promoted to admin (requires auth enabled) | | `POSTGRES_URL` | Database | unset (SQLite mode) | Set to switch metadata/auth DB to Postgres | | `USE_EMBEDDED_WEED_MINI` | Storage | `true` when unset | Set `false` to use external S3-compatible storage only | @@ -46,8 +37,6 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `S3_PREFIX` | Storage | `openreader` | Customize object key prefix | | `IMPORT_LIBRARY_DIR` | Library import | `docstore/library` fallback | Set a single server library root | | `IMPORT_LIBRARY_DIRS` | Library import | unset | Set multiple roots (comma/colon/semicolon separated) | -| `COMPUTE_WORKER_URL` | Heavy compute backend | unset | Set only for standalone external compute worker; leave unset for embedded worker startup | -| `COMPUTE_WORKER_TOKEN` | Heavy compute backend | unset (auto-generated in embedded startup) | Required for standalone external compute worker auth; must match worker | | `EMBEDDED_COMPUTE_WORKER_PORT` | Heavy compute backend | `8081` | Override embedded worker bind port | | `EMBEDDED_NATS_PORT` | Heavy compute backend | `4222` | Override embedded NATS client port | | `EMBEDDED_NATS_MONITOR_PORT` | Heavy compute backend | `8222` | Override embedded NATS monitor port | @@ -58,8 +47,10 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `COMPUTE_WHISPER_TIMEOUT_MS` | Heavy compute backend | `30000` | Shared whisper alignment timeout budget (worker + worker client wait budget) | | `COMPUTE_PDF_TIMEOUT_MS` | Heavy compute backend | `300000` | Shared PDF idle-timeout budget (worker + worker client wait budget) | | `COMPUTE_OP_STALE_MS` | Heavy compute backend | `max(30m, 4x max compute timeout)` | Shared stale window for worker op replacement and app-side stale PDF parse-state healing | -| `PDF_LAYOUT_MODEL_BASE_URL` | PDF layout model | PP-DocLayoutV3 ONNX base URL | Optional base URL override for `ensureModel()` | | `WHISPER_MODEL_BASE_URL` | Whisper ONNX model | onnx-community defaults | Optional base URL override for ONNX whisper-base_timestamped q4 downloads | +| `PDF_LAYOUT_MODEL_BASE_URL` | PDF layout model | PP-DocLayoutV3 ONNX base URL | Optional base URL override for `ensureModel()` | +| `COMPUTE_WORKER_URL` | Heavy compute backend | unset | Set only for standalone external compute worker; leave unset for embedded worker startup | +| `COMPUTE_WORKER_TOKEN` | Heavy compute backend | unset (auto-generated in embedded startup) | Required for standalone external compute worker auth; must match worker | | `FFMPEG_BIN` | Audio runtime | auto-detected (`ffmpeg-static`) | Override ffmpeg binary path | | `RUNTIME_SEED_*` runtime seeds | Legacy bootstrap seed | varies | Optional first-boot seeds for site features; then manage in Settings → Admin → Site features | | `RUNTIME_SEED_ENABLE_DOCX_CONVERSION` | Legacy bootstrap seed | `true` | Optional first-boot seed to enable/disable DOCX conversion UI | @@ -72,6 +63,8 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o | `RUNTIME_SEED_ENABLE_AUDIOBOOK_EXPORT` | Legacy bootstrap seed | `true` | Optional first-boot seed to enable audiobook export UI | | `RUNTIME_SEED_DISABLE_TTS_LIMIT` | Legacy bootstrap seed | `true` | Optional first-boot seed that keeps TTS daily rate limiting disabled | | `RUNTIME_SEED_DISABLE_COMPUTE_LIMIT` | Legacy bootstrap seed | `true` | Optional first-boot seed that keeps PDF parsing rate limiting disabled (other compute-limit values + max upload size are admin-only) | +| `DISABLE_AUTH_RATE_LIMIT` | Rate limiting | `false` | Set `true` to disable auth-layer rate limiting | +| `ENABLE_TEST_NAMESPACE` | Testing/CI | unset | Honor the `x-openreader-test-namespace` header on a production build; leave unset on real deployments | | `RUN_DRIZZLE_MIGRATIONS` | Database migrations | `true` | Set `false` to skip startup Drizzle schema migrations | | `RUN_FS_MIGRATIONS` | Storage migrations | `true` | Set `false` to skip startup filesystem -> S3/DB migration pass | @@ -112,49 +105,16 @@ Bootstrap API key for the legacy OpenAI-compatible TTS endpoint. - **Seeded on first boot** into the auto-created `default-openai` shared provider (encrypted at rest), then no longer read by the running app. Manage in **Settings → Admin → Shared providers** afterwards. - Related docs: [Admin Panel](../configure/admin-panel), [TTS Providers](../configure/tts-providers) -### TTS_CACHE_MAX_SIZE_BYTES +### TTS Upstream Settings (Runtime Settings) -Maximum in-memory TTS audio cache size in bytes. +TTS upstream request behavior and cache settings are now managed from **Settings → Admin → Site features → TTS upstream**. -- Default: `268435456` (256 MB) +- `ttsUpstreamMaxRetries` default: `2` +- `ttsUpstreamTimeoutMs` default: `285000` +- `ttsCacheMaxSizeBytes` default: `268435456` (256 MB) +- `ttsCacheTtlMs` default: `1800000` (30 minutes) -### TTS_CACHE_TTL_MS - -In-memory TTS audio cache TTL in milliseconds. - -- Default: `1800000` (30 minutes) - -### TTS_MAX_RETRIES - -Maximum retries for upstream TTS failures (429/5xx). - -- Default: `2` - -### TTS_RETRY_INITIAL_MS - -Initial retry delay in milliseconds for TTS upstream requests. - -- Default: `250` - -### TTS_RETRY_MAX_MS - -Maximum retry delay in milliseconds. - -- Default: `2000` - -### TTS_RETRY_BACKOFF - -Exponential backoff multiplier between retries. - -- Default: `2` - -### TTS_UPSTREAM_TIMEOUT_MS - -Maximum upstream TTS request timeout in milliseconds. - -- Default: `285000` (285 seconds) -- Applies to outbound provider calls from server routes using shared TTS generation -- Increase for slower providers/models; decrease to fail fast and surface retryable errors sooner +There are no environment variables for these settings in v4. ### TTS Daily Rate Limiting (Runtime Settings) @@ -221,24 +181,6 @@ GitHub OAuth client secret. - Enable only with `GITHUB_CLIENT_ID` -### DISABLE_AUTH_RATE_LIMIT - -Controls Better Auth rate limiting. - -- Default behavior: auth-layer rate limiting enabled -- Set to `true` to disable auth-layer rate limiting -- This does not affect TTS character rate limiting -- Related docs: [Auth](../configure/auth) - -### ENABLE_TEST_NAMESPACE - -Honors the `x-openreader-test-namespace` request header, which scopes documents/storage into an isolated namespace for end-to-end tests. - -- Default: unset (header ignored on production builds) -- Non-production builds (`NODE_ENV !== 'production'`) honor the header without this flag. -- Production builds (`pnpm build && pnpm start`) honor it only when `ENABLE_TEST_NAMESPACE=true`. The Playwright web server sets this automatically. -- **Leave unset on real deployments.** It is test/CI scaffolding only. - ### ADMIN_EMAILS Comma-separated list of email addresses that are auto-promoted to admin. @@ -369,25 +311,6 @@ Multiple library roots for server library import. ## Audio Tooling and Alignment -### COMPUTE_WORKER_URL - -Base URL for standalone external compute worker mode. - -- Leave unset for embedded/local startup (`pnpm dev` / `pnpm start`) so entrypoint can start embedded worker+NATS. -- Embedded startup requires `nats-server` available on host PATH. -- Required only when using a standalone external worker service. -- App-side only: set on app server/root `.env` (routing target), not worker-only env files. -- Example: `http://localhost:8081` - -### COMPUTE_WORKER_TOKEN - -Bearer token for compute-worker auth. - -- Required for standalone external worker service mode. -- Must match worker service `COMPUTE_WORKER_TOKEN`. -- In embedded startup, entrypoint auto-generates one if unset. -- In external worker mode, set this on both app server/root `.env` and worker service env (`compute/worker/.env*` or platform env). - ### EMBEDDED_COMPUTE_WORKER_PORT Embedded compute-worker HTTP port. @@ -464,6 +387,15 @@ Shared stale window in milliseconds. - If a parse row is stuck in `pending`/`running` past this window, app routes mark it failed so retries/reparse can proceed. - Keep this value aligned on both app-server and worker service envs. +### WHISPER_MODEL_BASE_URL + +Optional base URL override for the built-in ONNX Whisper alignment model downloader. + +- Default: `https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main` +- Default model variant: q4 (`encoder_model_q4.onnx`, `decoder_model_merged_q4.onnx`, `decoder_with_past_model_q4.onnx`) +- The base URL must host all expected manifest files under the same relative paths. +- Configure this on the worker service env (not only the app server env) + ### PDF_LAYOUT_MODEL_BASE_URL Optional base URL override for PP-DocLayoutV3 artifacts downloaded by `ensureModel()`. @@ -476,14 +408,24 @@ Optional base URL override for PP-DocLayoutV3 artifacts downloaded by `ensureMod - `preprocessor_config.json` - Configure this on the worker service env (not only the app server env) -### WHISPER_MODEL_BASE_URL +### COMPUTE_WORKER_URL -Optional base URL override for the built-in ONNX Whisper alignment model downloader. +Base URL for standalone external compute worker mode. -- Default: `https://huggingface.co/onnx-community/whisper-base_timestamped/resolve/main` -- Default model variant: q4 (`encoder_model_q4.onnx`, `decoder_model_merged_q4.onnx`, `decoder_with_past_model_q4.onnx`) -- The base URL must host all expected manifest files under the same relative paths. -- Configure this on the worker service env (not only the app server env) +- Leave unset for embedded/local startup (`pnpm dev` / `pnpm start`) so entrypoint can start embedded worker+NATS. +- Embedded startup requires `nats-server` available on host PATH. +- Required only when using a standalone external compute worker service. +- App-side only: set on app server/root `.env` (routing target), not worker-only env files. +- Example: `http://localhost:8081` + +### COMPUTE_WORKER_TOKEN + +Bearer token for compute-worker auth. + +- Required for standalone external worker service mode. +- Must match worker service `COMPUTE_WORKER_TOKEN`. +- In embedded startup, entrypoint auto-generates one if unset. +- In external worker mode, set this on both app server/root `.env` and worker service env (`compute/worker/.env*` or platform env). ### FFMPEG_BIN @@ -594,6 +536,26 @@ Seeds the PDF parsing rate-limit on/off state on first boot. - Runtime key: `disableComputeRateLimit` - The burst/sustained limits, their windows, and the max upload size are admin-only runtime settings (see [Compute (PDF Parsing) Rate Limiting](#compute-pdf-parsing-rate-limiting-runtime-settings)). +## Test and Dev Overrides + +### DISABLE_AUTH_RATE_LIMIT + +Controls Better Auth rate limiting. + +- Default behavior: auth-layer rate limiting enabled +- Set to `true` to disable auth-layer rate limiting +- This does not affect TTS character rate limiting +- Related docs: [Auth](../configure/auth) + +### ENABLE_TEST_NAMESPACE + +Honors the `x-openreader-test-namespace` request header, which scopes documents/storage into an isolated namespace for end-to-end tests. + +- Default: unset (header ignored on production builds) +- Non-production builds (`NODE_ENV !== 'production'`) honor the header without this flag. +- Production builds (`pnpm build && pnpm start`) honor it only when `ENABLE_TEST_NAMESPACE=true`. The Playwright web server sets this automatically. +- **Leave unset on real deployments.** It is test/CI scaffolding only. + ## Migration Controls ### RUN_DRIZZLE_MIGRATIONS diff --git a/src/app/api/audiobook/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts index f0875de..8ec30b2 100644 --- a/src/app/api/audiobook/chapter/route.ts +++ b/src/app/api/audiobook/chapter/route.ts @@ -614,6 +614,12 @@ export async function POST(request: NextRequest) { testNamespace, }, request.signal, + { + ttsCacheMaxSizeBytes: runtimeConfig.ttsCacheMaxSizeBytes, + ttsCacheTtlMs: runtimeConfig.ttsCacheTtlMs, + ttsUpstreamMaxRetries: runtimeConfig.ttsUpstreamMaxRetries, + ttsUpstreamTimeoutMs: runtimeConfig.ttsUpstreamTimeoutMs, + }, ); workDir = await mkdtemp(join(tmpdir(), 'openreader-audiobook-')); diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts index 80178fb..191a5b0 100644 --- a/src/app/api/tts/segments/ensure/route.ts +++ b/src/app/api/tts/segments/ensure/route.ts @@ -641,7 +641,12 @@ export async function POST(request: NextRequest) { apiKey: requestCreds.apiKey || 'none', baseUrl: requestCreds.baseUrl, testNamespace: scope.testNamespace, - }, request.signal); + }, request.signal, { + ttsCacheMaxSizeBytes: runtimeConfig.ttsCacheMaxSizeBytes, + ttsCacheTtlMs: runtimeConfig.ttsCacheTtlMs, + ttsUpstreamMaxRetries: runtimeConfig.ttsUpstreamMaxRetries, + ttsUpstreamTimeoutMs: runtimeConfig.ttsUpstreamTimeoutMs, + }); stageTimings.generateTtsMs = Date.now() - ttsStartedAt; failedStage = 's3.put_audio'; diff --git a/src/components/admin/AdminFeaturesPanel.tsx b/src/components/admin/AdminFeaturesPanel.tsx index 2facf72..73618ae 100644 --- a/src/components/admin/AdminFeaturesPanel.tsx +++ b/src/components/admin/AdminFeaturesPanel.tsx @@ -506,6 +506,71 @@ export function AdminFeaturesPanel() { /> +
Upstream} + > +
+
+
+ + {renderSource('ttsUpstreamMaxRetries')} +
+ updatePositiveIntDraft('ttsUpstreamMaxRetries', event.target.value)} + /> +
+
+
+ + {renderSource('ttsUpstreamTimeoutMs')} +
+ updatePositiveIntDraft('ttsUpstreamTimeoutMs', event.target.value)} + /> +
+
+
+ + {renderSource('ttsCacheMaxSizeBytes')} +
+ updatePositiveIntDraft('ttsCacheMaxSizeBytes', event.target.value)} + /> +
+
+
+ + {renderSource('ttsCacheTtlMs')} +
+ updatePositiveIntDraft('ttsCacheTtlMs', event.target.value)} + /> +
+
+
+

{dirty.size > 0 diff --git a/src/contexts/RuntimeConfigContext.tsx b/src/contexts/RuntimeConfigContext.tsx index 017bec5..a67754a 100644 --- a/src/contexts/RuntimeConfigContext.tsx +++ b/src/contexts/RuntimeConfigContext.tsx @@ -27,6 +27,10 @@ export interface RuntimeConfig { ttsDailyLimitAuthenticated: number; ttsIpDailyLimitAnonymous: number; ttsIpDailyLimitAuthenticated: number; + ttsCacheMaxSizeBytes: number; + ttsCacheTtlMs: number; + ttsUpstreamMaxRetries: number; + ttsUpstreamTimeoutMs: number; computeAvailable: boolean; } @@ -46,6 +50,10 @@ const RUNTIME_DEFAULTS: RuntimeConfig = { ttsDailyLimitAuthenticated: 500_000, ttsIpDailyLimitAnonymous: 100_000, ttsIpDailyLimitAuthenticated: 1_000_000, + ttsCacheMaxSizeBytes: 256 * 1024 * 1024, + ttsCacheTtlMs: 1000 * 60 * 30, + ttsUpstreamMaxRetries: 2, + ttsUpstreamTimeoutMs: 285_000, computeAvailable: true, }; diff --git a/src/lib/server/admin/settings.ts b/src/lib/server/admin/settings.ts index 1abbd34..24058e5 100644 --- a/src/lib/server/admin/settings.ts +++ b/src/lib/server/admin/settings.ts @@ -111,6 +111,10 @@ export const RUNTIME_CONFIG_SCHEMA = { ttsDailyLimitAuthenticated: positiveIntValue(500_000), ttsIpDailyLimitAnonymous: positiveIntValue(100_000), ttsIpDailyLimitAuthenticated: positiveIntValue(1_000_000), + ttsCacheMaxSizeBytes: positiveIntValue(256 * 1024 * 1024), + ttsCacheTtlMs: positiveIntValue(1000 * 60 * 30), + ttsUpstreamMaxRetries: positiveIntValue(2), + ttsUpstreamTimeoutMs: positiveIntValue(285_000), // Per-user throttle for expensive PDF-layout parsing. Disabled by default // (admins enable it in Settings → Admin), mirroring disableTtsRateLimit. // When enabled, the sub-limits below apply (admin-tunable, no env seed): diff --git a/src/lib/server/tts/generate.ts b/src/lib/server/tts/generate.ts index ce82ed1..01a7978 100644 --- a/src/lib/server/tts/generate.ts +++ b/src/lib/server/tts/generate.ts @@ -56,14 +56,83 @@ const replicateBlockedUntilByScope = new LRUCache({ max: REPLICATE_COOLDOWN_SCOPE_CACHE_MAX_ENTRIES, }); -const TTS_CACHE_MAX_SIZE_BYTES = Number(process.env.TTS_CACHE_MAX_SIZE_BYTES || 256 * 1024 * 1024); // 256MB -const TTS_CACHE_TTL_MS = Number(process.env.TTS_CACHE_TTL_MS || 1000 * 60 * 30); // 30 minutes +const DEFAULT_TTS_CACHE_MAX_SIZE_BYTES = 256 * 1024 * 1024; +const DEFAULT_TTS_CACHE_TTL_MS = 1000 * 60 * 30; +const DEFAULT_TTS_UPSTREAM_MAX_RETRIES = 2; +const DEFAULT_TTS_UPSTREAM_TIMEOUT_MS = 285_000; +const OPENAI_RETRY_INITIAL_DELAY_MS = 250; +const OPENAI_RETRY_MAX_DELAY_MS = 2000; +const OPENAI_RETRY_BACKOFF = 2; -const ttsAudioCache = new LRUCache({ - maxSize: TTS_CACHE_MAX_SIZE_BYTES, - sizeCalculation: (value) => value.byteLength, - ttl: TTS_CACHE_TTL_MS, -}); +export interface TtsUpstreamRuntimeSettings { + ttsCacheMaxSizeBytes?: number; + ttsCacheTtlMs?: number; + ttsUpstreamMaxRetries?: number; + ttsUpstreamTimeoutMs?: number; +} + +interface ResolvedTtsUpstreamRuntimeSettings { + ttsCacheMaxSizeBytes: number; + ttsCacheTtlMs: number; + ttsUpstreamMaxRetries: number; + ttsUpstreamTimeoutMs: number; +} + +function clampPositiveInteger(value: unknown, fallback: number): number { + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed < 1) return fallback; + return Math.floor(parsed); +} + +function resolveTtsUpstreamSettings( + settings?: TtsUpstreamRuntimeSettings, +): ResolvedTtsUpstreamRuntimeSettings { + return { + ttsCacheMaxSizeBytes: clampPositiveInteger( + settings?.ttsCacheMaxSizeBytes, + DEFAULT_TTS_CACHE_MAX_SIZE_BYTES, + ), + ttsCacheTtlMs: clampPositiveInteger(settings?.ttsCacheTtlMs, DEFAULT_TTS_CACHE_TTL_MS), + ttsUpstreamMaxRetries: clampPositiveInteger( + settings?.ttsUpstreamMaxRetries, + DEFAULT_TTS_UPSTREAM_MAX_RETRIES, + ), + ttsUpstreamTimeoutMs: clampPositiveInteger( + settings?.ttsUpstreamTimeoutMs, + DEFAULT_TTS_UPSTREAM_TIMEOUT_MS, + ), + }; +} + +function createTtsAudioCache(maxSize: number, ttlMs: number): LRUCache { + return new LRUCache({ + maxSize, + sizeCalculation: (value) => value.byteLength, + ttl: ttlMs, + }); +} + +let activeCacheConfig = { + maxSize: DEFAULT_TTS_CACHE_MAX_SIZE_BYTES, + ttlMs: DEFAULT_TTS_CACHE_TTL_MS, +}; + +let ttsAudioCache = createTtsAudioCache(activeCacheConfig.maxSize, activeCacheConfig.ttlMs); + +function ensureTtsAudioCache(settings: ResolvedTtsUpstreamRuntimeSettings): void { + if ( + activeCacheConfig.maxSize === settings.ttsCacheMaxSizeBytes + && activeCacheConfig.ttlMs === settings.ttsCacheTtlMs + ) { + return; + } + + activeCacheConfig = { + maxSize: settings.ttsCacheMaxSizeBytes, + ttlMs: settings.ttsCacheTtlMs, + }; + ttsAudioCache = createTtsAudioCache(activeCacheConfig.maxSize, activeCacheConfig.ttlMs); +} const inflightRequests = new Map(); @@ -324,13 +393,11 @@ async function getTestMockTtsBuffer(testNamespace?: string | null): Promise { let attempt = 0; - const maxRetries = Number(process.env.TTS_MAX_RETRIES ?? 2); - let delay = Number(process.env.TTS_RETRY_INITIAL_MS ?? 250); - const maxDelay = Number(process.env.TTS_RETRY_MAX_MS ?? 2000); - const backoff = Number(process.env.TTS_RETRY_BACKOFF ?? 2); + let delay = OPENAI_RETRY_INITIAL_DELAY_MS; for (; ;) { try { @@ -348,8 +415,8 @@ async function fetchTTSBufferWithRetry( throw error; } - await sleep(Math.min(delay, maxDelay)); - delay = Math.min(maxDelay, delay * backoff); + await sleep(Math.min(delay, OPENAI_RETRY_MAX_DELAY_MS)); + delay = Math.min(OPENAI_RETRY_MAX_DELAY_MS, delay * OPENAI_RETRY_BACKOFF); attempt += 1; } } @@ -434,14 +501,17 @@ async function buildReplicateInput(request: ResolvedServerTTSRequest): Promise { +async function runReplicateRequest( + request: ResolvedServerTTSRequest, + signal: AbortSignal, + maxRetries: number, +): Promise { const replicate = new Replicate({ auth: request.apiKey }); const input = await buildReplicateInput(request); const modelId = request.model as `${string}/${string}`; const cooldownScopeKey = getReplicateCooldownScopeKey(request); return runWithReplicateGate(cooldownScopeKey, signal, async () => { - const maxRetries = Number(process.env.TTS_MAX_RETRIES ?? 2); let attempt = 0; for (; ;) { @@ -493,19 +563,23 @@ async function runReplicateRequest(request: ResolvedServerTTSRequest, signal: Ab }); } -async function runProviderRequest(request: ResolvedServerTTSRequest, signal: AbortSignal): Promise { +async function runProviderRequest( + request: ResolvedServerTTSRequest, + signal: AbortSignal, + upstreamSettings: ResolvedTtsUpstreamRuntimeSettings, +): Promise { const mockBuffer = await getTestMockTtsBuffer(request.testNamespace); if (mockBuffer) return mockBuffer; if (request.provider === 'replicate') { - return runReplicateRequest(request, signal); + return runReplicateRequest(request, signal, upstreamSettings.ttsUpstreamMaxRetries); } const openai = new OpenAI({ apiKey: request.apiKey, baseURL: request.baseUrl, maxRetries: 0, - timeout: Number(process.env.TTS_UPSTREAM_TIMEOUT_MS ?? 285_000), + timeout: upstreamSettings.ttsUpstreamTimeoutMs, }); const createParams: ExtendedSpeechParams = { @@ -520,13 +594,17 @@ async function runProviderRequest(request: ResolvedServerTTSRequest, signal: Abo createParams.instructions = request.instructions; } - return fetchTTSBufferWithRetry(openai, createParams, signal); + return fetchTTSBufferWithRetry(openai, createParams, signal, upstreamSettings.ttsUpstreamMaxRetries); } export async function generateTTSBuffer( request: ServerTTSRequest, - signal?: AbortSignal + signal?: AbortSignal, + runtimeSettings?: TtsUpstreamRuntimeSettings, ): Promise { + const upstreamSettings = resolveTtsUpstreamSettings(runtimeSettings); + ensureTtsAudioCache(upstreamSettings); + const resolved = resolveTTSRequest(request); const cacheKey = makeCacheKey({ provider: resolved.provider, @@ -569,7 +647,7 @@ export async function generateTTSBuffer( consumers: 1, promise: (async () => { try { - const buffer = await runProviderRequest(resolved, controller.signal); + const buffer = await runProviderRequest(resolved, controller.signal, upstreamSettings); ttsAudioCache.set(cacheKey, buffer); return buffer; } finally {