From 893f74f03874f8299aba045c3869fd3c880abfcc Mon Sep 17 00:00:00 2001 From: Richard R Date: Thu, 16 Apr 2026 12:52:08 -0600 Subject: [PATCH] refactor(tts): update Replicate model defaults and voice resolution Align default Replicate model to the versioned Kokoro model across UI, docs, and server logic. Refactor TTS settings merging for audiobooks to ensure consistent normalization. Improve Retry-After header handling for upstream rate limits. Expand Replicate voice resolution to use model schemas when available, with test coverage for custom and built-in models. Update documentation to reflect new Replicate defaults, model selection, and configuration guidance. --- README.md | 2 +- .../tts-provider-guides/replicate.md | 5 +- docs-site/docs/configure/tts-providers.md | 8 +- docs-site/docs/deploy/vercel-deployment.md | 9 +- docs-site/docs/introduction.md | 2 +- src/app/api/audiobook/chapter/route.ts | 28 +- src/app/api/tts/route.ts | 10 +- src/components/SettingsModal.tsx | 50 ++- src/lib/server/tts/generate.ts | 86 +++-- src/lib/shared/kokoro.ts | 4 +- src/lib/shared/tts-provider-catalog.ts | 346 +++++++++++++++--- tests/unit/kokoro.spec.ts | 21 ++ tests/unit/tts-provider-catalog.spec.ts | 245 +++++++++++++ 13 files changed, 692 insertions(+), 124 deletions(-) create mode 100644 tests/unit/kokoro.spec.ts diff --git a/README.md b/README.md index 06704d0..f233826 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ OpenReader is an open source, self-host-friendly text-to-speech document reader ## ✨ Highlights -- 🎯 **Multi-provider TTS** with OpenAI-compatible endpoints and cloud providers (OpenAI, Replicate, DeepInfra, Kokoro, KittenTTS-FastAPI, Orpheus, custom). +- 🎯 **Multi-provider TTS** with OpenAI-compatible endpoints and cloud providers (Kokoro-FastAPI, KittenTTS-FastAPI, Orpheus-FastAPI or OpenAI, Replicate, DeepInfra). - 📖 **Read-along playback** for PDF/EPUB with sentence-aware narration. - ⏱️ **Word-by-word highlighting** via optional `whisper.cpp` timestamps. - 🛜 **Sync + library import** to bring docs across devices and from server-mounted folders. diff --git a/docs-site/docs/configure/tts-provider-guides/replicate.md b/docs-site/docs/configure/tts-provider-guides/replicate.md index b1e2c5c..14cdaf6 100644 --- a/docs-site/docs/configure/tts-provider-guides/replicate.md +++ b/docs-site/docs/configure/tts-provider-guides/replicate.md @@ -11,7 +11,7 @@ Use Replicate's hosted TTS models as your provider. ```env API_KEY=r8_... NEXT_PUBLIC_DEFAULT_TTS_PROVIDER=replicate -NEXT_PUBLIC_DEFAULT_TTS_MODEL=google/gemini-3.1-flash-tts +NEXT_PUBLIC_DEFAULT_TTS_MODEL=alphanumericuser/kokoro-82m:89b6fa84e4fa2dd6bd3a96be3e1f12827a3516c9fda8fddbac7a0be131c9a6f5 ``` **Or in-app via Settings -> TTS Provider:** @@ -25,11 +25,12 @@ Settings modal values override env vars. See [TTS Providers](../tts-providers) f ## Notes - Built-in Replicate models: + - `alphanumericuser/kokoro-82m:89b6fa84e4fa2dd6bd3a96be3e1f12827a3516c9fda8fddbac7a0be131c9a6f5` - `google/gemini-3.1-flash-tts` - `minimax/speech-2.8-turbo` - `qwen/qwen3-tts` - `inworld/tts-1.5-mini` -- You can also choose `Other` and enter any Replicate model ID (for example `owner/model-name`). +- You can also choose `Other` and enter any Replicate model ID (for example `owner/model-name` or `owner/model-name:version`). - Native model speed is not available on all Replicate models; OpenReader hides/disables native speed controls where unsupported. - TTS requests are sent from the server, not the browser. The API key is never exposed to clients. diff --git a/docs-site/docs/configure/tts-providers.md b/docs-site/docs/configure/tts-providers.md index ef50dd0..89bd942 100644 --- a/docs-site/docs/configure/tts-providers.md +++ b/docs-site/docs/configure/tts-providers.md @@ -16,16 +16,16 @@ Set env vars as deployment-level defaults. Users (or you, in a single-user setup - **OpenAI**: Cloud. Base URL pre-filled (`https://api.openai.com/v1`). API key required. - **Replicate**: Cloud. Base URL managed internally by OpenReader. API key required. -- **Deepinfra**: Cloud. Base URL pre-filled (`https://api.deepinfra.com/v1/openai`). API key required. +- **DeepInfra**: Cloud. Base URL pre-filled (`https://api.deepinfra.com/v1/openai`). API key required. - **Custom OpenAI-Like**: Self-hosted or any custom endpoint. `API_BASE` must be set manually (typically ending in `/v1`). API key optional. -For `OpenAI`, `Deepinfra`, and `Replicate` you only need to supply an API key. For `Custom OpenAI-Like` you must also set `API_BASE`. +For `OpenAI`, `DeepInfra`, and `Replicate` you only need to supply an API key. For `Custom OpenAI-Like` you must also set `API_BASE`. ## Built-in model catalogs -- **Replicate** models: `google/gemini-3.1-flash-tts`, `minimax/speech-2.8-turbo`, `qwen/qwen3-tts`, `inworld/tts-1.5-mini` (or choose `Other` and enter any Replicate model ID) +- **Replicate** models: `alphanumericuser/kokoro-82m:89b6fa84e4fa2dd6bd3a96be3e1f12827a3516c9fda8fddbac7a0be131c9a6f5`, `google/gemini-3.1-flash-tts`, `minimax/speech-2.8-turbo`, `qwen/qwen3-tts`, `inworld/tts-1.5-mini` (or choose `Other` and enter any Replicate model ID, such as `owner/model` or `owner/model:version`) - **OpenAI** models: `tts-1`, `tts-1-hd`, `gpt-4o-mini-tts` -- **Deepinfra** models: includes `hexgrad/Kokoro-82M` and additional hosted models (depending on API key / feature flags) +- **DeepInfra** models: includes `hexgrad/Kokoro-82M` and additional hosted models (depending on API key / feature flags) ## Custom provider requirements diff --git a/docs-site/docs/deploy/vercel-deployment.md b/docs-site/docs/deploy/vercel-deployment.md index 0d63344..c5a7e0c 100644 --- a/docs-site/docs/deploy/vercel-deployment.md +++ b/docs-site/docs/deploy/vercel-deployment.md @@ -18,8 +18,7 @@ This guide covers deploying OpenReader to Vercel with external Postgres and S3-c Recommended production setup (auth enabled): ```bash -API_BASE=https://api.deepinfra.com/v1/openai -API_KEY=your_deepinfra_key +API_KEY=your_replicate_key POSTGRES_URL=postgres://... USE_EMBEDDED_WEED_MINI=false S3_ACCESS_KEY_ID=... @@ -33,8 +32,8 @@ AUTH_SECRET=... NEXT_PUBLIC_ENABLE_DOCX_CONVERSION=false NEXT_PUBLIC_ENABLE_DESTRUCTIVE_DELETE_ACTIONS=false NEXT_PUBLIC_ENABLE_TTS_PROVIDERS_TAB=false -NEXT_PUBLIC_DEFAULT_TTS_PROVIDER=deepinfra -NEXT_PUBLIC_DEFAULT_TTS_MODEL=hexgrad/Kokoro-82M +NEXT_PUBLIC_DEFAULT_TTS_PROVIDER=replicate +NEXT_PUBLIC_DEFAULT_TTS_MODEL=alphanumericuser/kokoro-82m:89b6fa84e4fa2dd6bd3a96be3e1f12827a3516c9fda8fddbac7a0be131c9a6f5 NEXT_PUBLIC_SHOW_ALL_DEEPINFRA_MODELS=false NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT=true NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT=false @@ -50,7 +49,7 @@ We recommend setting these defaults for a production-like environment: - `NEXT_PUBLIC_ENABLE_DESTRUCTIVE_DELETE_ACTIONS=false`: Hides destructive "Delete All" actions - `NEXT_PUBLIC_ENABLE_TTS_PROVIDERS_TAB=false`: Hides the Settings -> TTS Provider section - `NEXT_PUBLIC_DEFAULT_TTS_PROVIDER=replicate`: Points default TTS to a scalable provider -- `NEXT_PUBLIC_DEFAULT_TTS_MODEL=google/gemini-3.1-flash-tts`: Uses a high-quality default model +- `NEXT_PUBLIC_DEFAULT_TTS_MODEL=alphanumericuser/kokoro-82m:89b6fa84e4fa2dd6bd3a96be3e1f12827a3516c9fda8fddbac7a0be131c9a6f5`: Uses a low-cost default model - `NEXT_PUBLIC_SHOW_ALL_DEEPINFRA_MODELS=false`: Restricts usage to free models if no key is provided - `NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT=true`: (Optional) Controls audiobook export UI - `NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT=false`: (Optional) Controls word highlighting UI (requires timestamp backend) diff --git a/docs-site/docs/introduction.md b/docs-site/docs/introduction.md index b7f4a6f..79a617d 100644 --- a/docs-site/docs/introduction.md +++ b/docs-site/docs/introduction.md @@ -18,7 +18,7 @@ It supports multiple TTS providers including OpenAI, Replicate, DeepInfra, and c - [**Orpheus-FastAPI**](https://github.com/Lex-au/Orpheus-FastAPI) - **Custom OpenAI-compatible**: any TTS API with `/v1/audio/voices` and `/v1/audio/speech` endpoints - **Cloud TTS providers**: - - [**Replicate**](https://replicate.com/explore): `google/gemini-3.1-flash-tts`, `minimax/speech-2.8-turbo`, `qwen/qwen3-tts`, and `inworld/tts-1.5-mini` + - [**Replicate**](https://replicate.com/explore): includes a built-in catalog and supports any Replicate model ID via `Other` - [**DeepInfra**](https://deepinfra.com/models/text-to-speech): Kokoro-82M and other hosted models - [**OpenAI API**](https://platform.openai.com/docs/pricing#transcription-and-speech): `tts-1`, `tts-1-hd`, and `gpt-4o-mini-tts` - 🛜 **Server-side Document Storage** diff --git a/src/app/api/audiobook/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts index 61374bf..24e157a 100644 --- a/src/app/api/audiobook/chapter/route.ts +++ b/src/app/api/audiobook/chapter/route.ts @@ -307,9 +307,16 @@ export async function POST(request: NextRequest) { existingSettings = null; } + const normalizedExistingSettings = existingSettings ? normalizeNativeSpeedForSettings(existingSettings) : undefined; const incomingSettings = data.settings ? normalizeNativeSpeedForSettings(data.settings) : undefined; - if (existingSettings && hasChapters && incomingSettings) { - const normalizedExistingSettings = normalizeNativeSpeedForSettings(existingSettings); + const mergedSettings = (normalizedExistingSettings || incomingSettings) + ? normalizeNativeSpeedForSettings({ + ...(normalizedExistingSettings || {}), + ...(incomingSettings || {}), + } as AudiobookGenerationSettings) + : undefined; + + if (normalizedExistingSettings && hasChapters && incomingSettings) { const mismatch = normalizedExistingSettings.ttsProvider !== incomingSettings.ttsProvider || normalizedExistingSettings.ttsModel !== incomingSettings.ttsModel || @@ -330,10 +337,9 @@ export async function POST(request: NextRequest) { const format: TTSAudiobookFormat = (existingFormats.values().next().value as TTSAudiobookFormat | undefined) ?? - existingSettings?.format ?? - incomingSettings?.format ?? + mergedSettings?.format ?? requestedFormat; - const rawPostSpeed = incomingSettings?.postSpeed ?? existingSettings?.postSpeed ?? 1; + const rawPostSpeed = mergedSettings?.postSpeed ?? 1; const postSpeed = Number.isFinite(Number(rawPostSpeed)) ? Number(rawPostSpeed) : 1; let chapterIndex: number; @@ -357,22 +363,20 @@ export async function POST(request: NextRequest) { } const provider = request.headers.get('x-tts-provider') - || incomingSettings?.ttsProvider - || existingSettings?.ttsProvider + || mergedSettings?.ttsProvider || 'openai'; const openApiKey = request.headers.get('x-openai-key') || process.env.API_KEY || 'none'; const openApiBaseUrl = request.headers.get('x-openai-base-url') || process.env.API_BASE; - const model = incomingSettings?.ttsModel ?? existingSettings?.ttsModel; - const voice = incomingSettings?.voice - || existingSettings?.voice + const model = mergedSettings?.ttsModel; + const voice = mergedSettings?.voice || (provider === 'openai' ? 'alloy' : provider === 'deepinfra' ? 'af_bella' : 'af_sarah'); - const rawNativeSpeed = incomingSettings?.nativeSpeed ?? existingSettings?.nativeSpeed ?? 1; + const rawNativeSpeed = mergedSettings?.nativeSpeed ?? 1; const nativeSpeed = Number.isFinite(Number(rawNativeSpeed)) ? Number(rawNativeSpeed) : 1; - const instructions = incomingSettings?.ttsInstructions ?? existingSettings?.ttsInstructions; + const instructions = mergedSettings?.ttsInstructions; if (authEnabled && userId && isTtsRateLimitEnabled()) { const isAnonymous = Boolean(user?.isAnonymous); diff --git a/src/app/api/tts/route.ts b/src/app/api/tts/route.ts index 7983393..0843e1e 100644 --- a/src/app/api/tts/route.ts +++ b/src/app/api/tts/route.ts @@ -66,8 +66,14 @@ function getUpstreamRetryAfterSeconds(error: unknown): number | undefined { const retryAfterHeader = response?.headers?.get?.('retry-after'); if (!retryAfterHeader) return undefined; const parsed = Number(retryAfterHeader); - if (!Number.isFinite(parsed) || parsed <= 0) return undefined; - return Math.ceil(parsed); + if (Number.isFinite(parsed) && parsed > 0) { + return Math.ceil(parsed); + } + const parsedDateMs = Date.parse(retryAfterHeader); + if (!Number.isFinite(parsedDateMs)) return undefined; + const seconds = (parsedDateMs - Date.now()) / 1000; + if (seconds <= 0) return undefined; + return Math.ceil(seconds); } export async function POST(req: NextRequest) { diff --git a/src/components/SettingsModal.tsx b/src/components/SettingsModal.tsx index bf01a68..6a8d560 100644 --- a/src/components/SettingsModal.tsx +++ b/src/components/SettingsModal.tsx @@ -36,7 +36,7 @@ import { deleteDocuments, mimeTypeForDoc, uploadDocuments } from '@/lib/client/a import { cacheStoredDocumentFromBytes, clearDocumentCache } from '@/lib/client/cache/documents'; import { clearAllDocumentPreviewCaches, clearInMemoryDocumentPreviewCache } from '@/lib/client/cache/previews'; import { resolveTtsSettingsViewModel } from '@/lib/client/settings/tts-settings'; -import { supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog'; +import { REPLICATE_KOKORO_82M_VERSIONED_MODEL, supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog'; const enableDestructiveDelete = process.env.NEXT_PUBLIC_ENABLE_DESTRUCTIVE_DELETE_ACTIONS !== 'false'; const showAllDeepInfra = process.env.NEXT_PUBLIC_SHOW_ALL_DEEPINFRA_MODELS !== 'false'; @@ -392,6 +392,12 @@ export function SettingsModal({ className = '' }: { className?: string }) { const btnPrimary = `${btnBase} bg-accent text-background hover:bg-secondary-accent hover:scale-[1.04]`; const btnSecondary = `${btnBase} bg-background text-foreground hover:bg-offbase hover:text-accent hover:scale-[1.04]`; const btnOutline = `${btnBase} bg-background border border-offbase text-foreground hover:bg-offbase hover:text-accent hover:scale-[1.02]`; + const shouldShowBaseUrl = localTTSProvider !== 'replicate' + && (localTTSProvider === 'custom-openai' || !localBaseUrl || localBaseUrl === ''); + const selectedModel = ttsModels.find(m => m.id === selectedModelId) || ttsModels[0]; + const selectedModelVersion = selectedModel?.id?.includes(':') + ? selectedModel.id.slice(selectedModel.id.indexOf(':')) + : ''; return ( <> @@ -508,7 +514,7 @@ export function SettingsModal({ className = '' }: { className?: string }) { setModelValue('kokoro'); setLocalBaseUrl(''); } else if (provider.id === 'replicate') { - setModelValue('google/gemini-3.1-flash-tts'); + setModelValue(REPLICATE_KOKORO_82M_VERSIONED_MODEL); setLocalBaseUrl(''); } else if (provider.id === 'deepinfra') { setModelValue('hexgrad/Kokoro-82M'); @@ -562,7 +568,7 @@ export function SettingsModal({ className = '' }: { className?: string }) { - {localTTSProvider !== 'replicate' && (localTTSProvider === 'custom-openai' || !localBaseUrl || localBaseUrl === '') && ( + {shouldShowBaseUrl && (