feat: add support for Replicate TTS provider and models
- Updated environment variables documentation to include Replicate as a TTS provider option. - Added Replicate to the sidebar for TTS provider guides. - Included Replicate as a dependency in package.json and pnpm-lock.yaml. - Enhanced audiobook chapter generation to normalize native speed settings based on the TTS provider. - Improved error handling in TTS API routes to provide retry information for rate-limited responses. - Updated AudiobookExportModal to reflect native speed support for Replicate models. - Modified SettingsModal to set default model for Replicate. - Enhanced SpeedControl component to conditionally render native speed controls based on provider support. - Updated TTSContext to utilize effective native speed for TTS requests. - Implemented Replicate request handling in the TTS generation logic. - Added new documentation for configuring Replicate as a TTS provider.
This commit is contained in:
parent
6606b0ee4a
commit
9da9232d39
18 changed files with 461 additions and 94 deletions
|
|
@ -18,7 +18,7 @@ OpenReader is an open source, self-host-friendly text-to-speech document reader
|
||||||
|
|
||||||
## ✨ Highlights
|
## ✨ Highlights
|
||||||
|
|
||||||
- 🎯 **Multi-provider TTS** with OpenAI-compatible endpoints (OpenAI, DeepInfra, Kokoro, KittenTTS-FastAPI, Orpheus, custom).
|
- 🎯 **Multi-provider TTS** with OpenAI-compatible endpoints and cloud providers (OpenAI, Replicate, DeepInfra, Kokoro, KittenTTS-FastAPI, Orpheus, custom).
|
||||||
- 📖 **Read-along playback** for PDF/EPUB with sentence-aware narration.
|
- 📖 **Read-along playback** for PDF/EPUB with sentence-aware narration.
|
||||||
- ⏱️ **Word-by-word highlighting** via optional `whisper.cpp` timestamps.
|
- ⏱️ **Word-by-word highlighting** via optional `whisper.cpp` timestamps.
|
||||||
- 🛜 **Sync + library import** to bring docs across devices and from server-mounted folders.
|
- 🛜 **Sync + library import** to bring docs across devices and from server-mounted folders.
|
||||||
|
|
|
||||||
39
docs-site/docs/configure/tts-provider-guides/replicate.md
Normal file
39
docs-site/docs/configure/tts-provider-guides/replicate.md
Normal file
|
|
@ -0,0 +1,39 @@
|
||||||
|
---
|
||||||
|
title: Replicate
|
||||||
|
---
|
||||||
|
|
||||||
|
Use Replicate's hosted TTS models as your provider.
|
||||||
|
|
||||||
|
## Setup
|
||||||
|
|
||||||
|
**Environment variables (recommended for deployment):**
|
||||||
|
|
||||||
|
```env
|
||||||
|
API_KEY=r8_...
|
||||||
|
NEXT_PUBLIC_DEFAULT_TTS_PROVIDER=replicate
|
||||||
|
NEXT_PUBLIC_DEFAULT_TTS_MODEL=google/gemini-3.1-flash-tts
|
||||||
|
```
|
||||||
|
|
||||||
|
**Or in-app via Settings -> TTS Provider:**
|
||||||
|
|
||||||
|
1. Set provider to `Replicate`.
|
||||||
|
2. Enter your `API_KEY`.
|
||||||
|
3. Choose a model and voice.
|
||||||
|
|
||||||
|
Settings modal values override env vars. See [TTS Providers](../tts-providers) for how the two layers interact.
|
||||||
|
|
||||||
|
## Notes
|
||||||
|
|
||||||
|
- Built-in Replicate models:
|
||||||
|
- `google/gemini-3.1-flash-tts`
|
||||||
|
- `minimax/speech-2.8-turbo`
|
||||||
|
- `qwen/qwen3-tts`
|
||||||
|
- `inworld/tts-1.5-mini`
|
||||||
|
- Native model speed is not available on all Replicate models; OpenReader hides/disables native speed controls where unsupported.
|
||||||
|
- TTS requests are sent from the server, not the browser. The API key is never exposed to clients.
|
||||||
|
|
||||||
|
## References
|
||||||
|
|
||||||
|
- [Replicate](https://replicate.com/explore)
|
||||||
|
- [TTS Providers](../tts-providers)
|
||||||
|
- [TTS Environment Variables](../../reference/environment-variables#tts-provider-and-request-behavior)
|
||||||
|
|
@ -15,10 +15,17 @@ Set env vars as deployment-level defaults. Users (or you, in a single-user setup
|
||||||
## Providers
|
## Providers
|
||||||
|
|
||||||
- **OpenAI**: Cloud. Base URL pre-filled (`https://api.openai.com/v1`). API key required.
|
- **OpenAI**: Cloud. Base URL pre-filled (`https://api.openai.com/v1`). API key required.
|
||||||
|
- **Replicate**: Cloud. Base URL managed internally by OpenReader. API key required.
|
||||||
- **Deepinfra**: Cloud. Base URL pre-filled (`https://api.deepinfra.com/v1/openai`). API key required.
|
- **Deepinfra**: Cloud. Base URL pre-filled (`https://api.deepinfra.com/v1/openai`). API key required.
|
||||||
- **Custom OpenAI-Like**: Self-hosted or any custom endpoint. `API_BASE` must be set manually (typically ending in `/v1`). API key optional.
|
- **Custom OpenAI-Like**: Self-hosted or any custom endpoint. `API_BASE` must be set manually (typically ending in `/v1`). API key optional.
|
||||||
|
|
||||||
For `OpenAI` and `Deepinfra` you only need to supply an API key. For `Custom OpenAI-Like` you must also set `API_BASE`.
|
For `OpenAI`, `Deepinfra`, and `Replicate` you only need to supply an API key. For `Custom OpenAI-Like` you must also set `API_BASE`.
|
||||||
|
|
||||||
|
## Built-in model catalogs
|
||||||
|
|
||||||
|
- **Replicate** models: `google/gemini-3.1-flash-tts`, `minimax/speech-2.8-turbo`, `qwen/qwen3-tts`, `inworld/tts-1.5-mini`
|
||||||
|
- **OpenAI** models: `tts-1`, `tts-1-hd`, `gpt-4o-mini-tts`
|
||||||
|
- **Deepinfra** models: includes `hexgrad/Kokoro-82M` and additional hosted models (depending on API key / feature flags)
|
||||||
|
|
||||||
## Custom provider requirements
|
## Custom provider requirements
|
||||||
|
|
||||||
|
|
@ -36,6 +43,7 @@ TTS requests originate from the **Next.js server**, not the browser. `API_BASE`
|
||||||
- [Kokoro-FastAPI](./tts-provider-guides/kokoro-fastapi)
|
- [Kokoro-FastAPI](./tts-provider-guides/kokoro-fastapi)
|
||||||
- [KittenTTS-FastAPI](./tts-provider-guides/kitten-tts-fastapi)
|
- [KittenTTS-FastAPI](./tts-provider-guides/kitten-tts-fastapi)
|
||||||
- [Orpheus-FastAPI](./tts-provider-guides/orpheus-fastapi)
|
- [Orpheus-FastAPI](./tts-provider-guides/orpheus-fastapi)
|
||||||
|
- [Replicate](./tts-provider-guides/replicate)
|
||||||
- [DeepInfra](./tts-provider-guides/deepinfra)
|
- [DeepInfra](./tts-provider-guides/deepinfra)
|
||||||
- [OpenAI](./tts-provider-guides/openai)
|
- [OpenAI](./tts-provider-guides/openai)
|
||||||
- [Other](./tts-provider-guides/other)
|
- [Other](./tts-provider-guides/other)
|
||||||
|
|
|
||||||
|
|
@ -49,8 +49,8 @@ We recommend setting these defaults for a production-like environment:
|
||||||
- `NEXT_PUBLIC_ENABLE_DOCX_CONVERSION=false`: Disables DOCX upload (requires external tools anyway)
|
- `NEXT_PUBLIC_ENABLE_DOCX_CONVERSION=false`: Disables DOCX upload (requires external tools anyway)
|
||||||
- `NEXT_PUBLIC_ENABLE_DESTRUCTIVE_DELETE_ACTIONS=false`: Hides destructive "Delete All" actions
|
- `NEXT_PUBLIC_ENABLE_DESTRUCTIVE_DELETE_ACTIONS=false`: Hides destructive "Delete All" actions
|
||||||
- `NEXT_PUBLIC_ENABLE_TTS_PROVIDERS_TAB=false`: Hides the Settings -> TTS Provider section
|
- `NEXT_PUBLIC_ENABLE_TTS_PROVIDERS_TAB=false`: Hides the Settings -> TTS Provider section
|
||||||
- `NEXT_PUBLIC_DEFAULT_TTS_PROVIDER=deepinfra`: Points default TTS to a scalable provider
|
- `NEXT_PUBLIC_DEFAULT_TTS_PROVIDER=replicate`: Points default TTS to a scalable provider
|
||||||
- `NEXT_PUBLIC_DEFAULT_TTS_MODEL=hexgrad/Kokoro-82M`: Uses a high-quality default model
|
- `NEXT_PUBLIC_DEFAULT_TTS_MODEL=google/gemini-3.1-flash-tts`: Uses a high-quality default model
|
||||||
- `NEXT_PUBLIC_SHOW_ALL_DEEPINFRA_MODELS=false`: Restricts usage to free models if no key is provided
|
- `NEXT_PUBLIC_SHOW_ALL_DEEPINFRA_MODELS=false`: Restricts usage to free models if no key is provided
|
||||||
- `NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT=true`: (Optional) Controls audiobook export UI
|
- `NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT=true`: (Optional) Controls audiobook export UI
|
||||||
- `NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT=false`: (Optional) Controls word highlighting UI (requires timestamp backend)
|
- `NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT=false`: (Optional) Controls word highlighting UI (requires timestamp backend)
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem';
|
||||||
## Prerequisites
|
## Prerequisites
|
||||||
|
|
||||||
- A recent Docker version installed
|
- A recent Docker version installed
|
||||||
- A TTS API server that OpenReader can reach (Kokoro-FastAPI, KittenTTS-FastAPI, Orpheus-FastAPI, DeepInfra, OpenAI, or equivalent)
|
- A TTS API server that OpenReader can reach (Kokoro-FastAPI, KittenTTS-FastAPI, Orpheus-FastAPI, Replicate, DeepInfra, OpenAI, or equivalent)
|
||||||
|
|
||||||
:::note
|
:::note
|
||||||
If you have suitable hardware, you can run Kokoro locally with Docker. See [Kokoro-FastAPI](./configure/tts-provider-guides/kokoro-fastapi).
|
If you have suitable hardware, you can run Kokoro locally with Docker. See [Kokoro-FastAPI](./configure/tts-provider-guides/kokoro-fastapi).
|
||||||
|
|
|
||||||
|
|
@ -8,7 +8,7 @@ OpenReader is an open source text-to-speech document reader built with Next.js.
|
||||||
|
|
||||||
> Previously named **OpenReader-WebUI**.
|
> Previously named **OpenReader-WebUI**.
|
||||||
|
|
||||||
It supports multiple TTS providers including OpenAI, DeepInfra, and custom OpenAI-compatible endpoints such as [Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI), [KittenTTS-FastAPI](https://github.com/richardr1126/KittenTTS-FastAPI), and [Orpheus-FastAPI](https://github.com/Lex-au/Orpheus-FastAPI).
|
It supports multiple TTS providers including OpenAI, Replicate, DeepInfra, and custom OpenAI-compatible endpoints such as [Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI), [KittenTTS-FastAPI](https://github.com/richardr1126/KittenTTS-FastAPI), and [Orpheus-FastAPI](https://github.com/Lex-au/Orpheus-FastAPI).
|
||||||
|
|
||||||
## ✨ Highlights
|
## ✨ Highlights
|
||||||
|
|
||||||
|
|
@ -18,6 +18,7 @@ It supports multiple TTS providers including OpenAI, DeepInfra, and custom OpenA
|
||||||
- [**Orpheus-FastAPI**](https://github.com/Lex-au/Orpheus-FastAPI)
|
- [**Orpheus-FastAPI**](https://github.com/Lex-au/Orpheus-FastAPI)
|
||||||
- **Custom OpenAI-compatible**: any TTS API with `/v1/audio/voices` and `/v1/audio/speech` endpoints
|
- **Custom OpenAI-compatible**: any TTS API with `/v1/audio/voices` and `/v1/audio/speech` endpoints
|
||||||
- **Cloud TTS providers**:
|
- **Cloud TTS providers**:
|
||||||
|
- [**Replicate**](https://replicate.com/explore): `google/gemini-3.1-flash-tts`, `minimax/speech-2.8-turbo`, `qwen/qwen3-tts`, and `inworld/tts-1.5-mini`
|
||||||
- [**DeepInfra**](https://deepinfra.com/models/text-to-speech): Kokoro-82M and other hosted models
|
- [**DeepInfra**](https://deepinfra.com/models/text-to-speech): Kokoro-82M and other hosted models
|
||||||
- [**OpenAI API**](https://platform.openai.com/docs/pricing#transcription-and-speech): `tts-1`, `tts-1-hd`, and `gpt-4o-mini-tts`
|
- [**OpenAI API**](https://platform.openai.com/docs/pricing#transcription-and-speech): `tts-1`, `tts-1-hd`, and `gpt-4o-mini-tts`
|
||||||
- 🛜 **Server-side Document Storage**
|
- 🛜 **Server-side Document Storage**
|
||||||
|
|
|
||||||
|
|
@ -379,7 +379,7 @@ Controls whether the **TTS Provider** section appears in the Settings modal.
|
||||||
Sets the default TTS provider for new users.
|
Sets the default TTS provider for new users.
|
||||||
|
|
||||||
- Default: `custom-openai`
|
- Default: `custom-openai`
|
||||||
- Example values: `deepinfra`, `openai`, `custom-openai`
|
- Example values: `replicate`, `deepinfra`, `openai`, `custom-openai`
|
||||||
|
|
||||||
### NEXT_PUBLIC_DEFAULT_TTS_MODEL
|
### NEXT_PUBLIC_DEFAULT_TTS_MODEL
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,7 @@ const sidebars: SidebarsConfig = {
|
||||||
'configure/tts-provider-guides/kokoro-fastapi',
|
'configure/tts-provider-guides/kokoro-fastapi',
|
||||||
'configure/tts-provider-guides/kitten-tts-fastapi',
|
'configure/tts-provider-guides/kitten-tts-fastapi',
|
||||||
'configure/tts-provider-guides/orpheus-fastapi',
|
'configure/tts-provider-guides/orpheus-fastapi',
|
||||||
|
'configure/tts-provider-guides/replicate',
|
||||||
'configure/tts-provider-guides/deepinfra',
|
'configure/tts-provider-guides/deepinfra',
|
||||||
'configure/tts-provider-guides/openai',
|
'configure/tts-provider-guides/openai',
|
||||||
'configure/tts-provider-guides/other',
|
'configure/tts-provider-guides/other',
|
||||||
|
|
|
||||||
|
|
@ -59,6 +59,7 @@
|
||||||
"react-pdf": "^9.2.1",
|
"react-pdf": "^9.2.1",
|
||||||
"react-reader": "^2.0.15",
|
"react-reader": "^2.0.15",
|
||||||
"remark-gfm": "^4.0.1",
|
"remark-gfm": "^4.0.1",
|
||||||
|
"replicate": "^1.4.0",
|
||||||
"uuid": "^11.1.0"
|
"uuid": "^11.1.0"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
|
|
||||||
|
|
@ -124,6 +124,9 @@ importers:
|
||||||
remark-gfm:
|
remark-gfm:
|
||||||
specifier: ^4.0.1
|
specifier: ^4.0.1
|
||||||
version: 4.0.1
|
version: 4.0.1
|
||||||
|
replicate:
|
||||||
|
specifier: ^1.4.0
|
||||||
|
version: 1.4.0
|
||||||
uuid:
|
uuid:
|
||||||
specifier: ^11.1.0
|
specifier: ^11.1.0
|
||||||
version: 11.1.0
|
version: 11.1.0
|
||||||
|
|
@ -3588,6 +3591,10 @@ packages:
|
||||||
remark-stringify@11.0.0:
|
remark-stringify@11.0.0:
|
||||||
resolution: {integrity: sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==}
|
resolution: {integrity: sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==}
|
||||||
|
|
||||||
|
replicate@1.4.0:
|
||||||
|
resolution: {integrity: sha512-1ufKejfUVz/azy+5TnzQP7U1+MHVWZ6psnQ06az8byUUnRhT+DZ/MvewzB1NQYBVMgNKR7xPDtTwlcP5nv/5+w==}
|
||||||
|
engines: {git: '>=2.11.0', node: '>=18.0.0', npm: '>=7.19.0', yarn: '>=1.7.0'}
|
||||||
|
|
||||||
resolve-from@4.0.0:
|
resolve-from@4.0.0:
|
||||||
resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==}
|
resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==}
|
||||||
engines: {node: '>=4'}
|
engines: {node: '>=4'}
|
||||||
|
|
@ -8028,6 +8035,10 @@ snapshots:
|
||||||
mdast-util-to-markdown: 2.1.2
|
mdast-util-to-markdown: 2.1.2
|
||||||
unified: 11.0.5
|
unified: 11.0.5
|
||||||
|
|
||||||
|
replicate@1.4.0:
|
||||||
|
optionalDependencies:
|
||||||
|
readable-stream: 4.7.0
|
||||||
|
|
||||||
resolve-from@4.0.0: {}
|
resolve-from@4.0.0: {}
|
||||||
|
|
||||||
resolve-pkg-maps@1.0.0: {}
|
resolve-pkg-maps@1.0.0: {}
|
||||||
|
|
|
||||||
|
|
@ -29,6 +29,7 @@ import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/li
|
||||||
import { buildAllowedAudiobookUserIds, pickAudiobookOwner } from '@/lib/server/audiobooks/user-scope';
|
import { buildAllowedAudiobookUserIds, pickAudiobookOwner } from '@/lib/server/audiobooks/user-scope';
|
||||||
import { getFFmpegPath } from '@/lib/server/audiobooks/ffmpeg-bin';
|
import { getFFmpegPath } from '@/lib/server/audiobooks/ffmpeg-bin';
|
||||||
import { generateTTSBuffer } from '@/lib/server/tts/generate';
|
import { generateTTSBuffer } from '@/lib/server/tts/generate';
|
||||||
|
import { supportsNativeModelSpeed } from '@/lib/shared/tts-provider-catalog';
|
||||||
import type { AudiobookGenerationSettings } from '@/types/client';
|
import type { AudiobookGenerationSettings } from '@/types/client';
|
||||||
import type { TTSAudiobookFormat } from '@/types/tts';
|
import type { TTSAudiobookFormat } from '@/types/tts';
|
||||||
|
|
||||||
|
|
@ -92,6 +93,12 @@ function s3NotConfiguredResponse(): NextResponse {
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeNativeSpeedForSettings(settings: AudiobookGenerationSettings): AudiobookGenerationSettings {
|
||||||
|
return supportsNativeModelSpeed(settings.ttsProvider, settings.ttsModel)
|
||||||
|
? settings
|
||||||
|
: { ...settings, nativeSpeed: 1 };
|
||||||
|
}
|
||||||
|
|
||||||
function chapterFileMimeType(format: TTSAudiobookFormat): string {
|
function chapterFileMimeType(format: TTSAudiobookFormat): string {
|
||||||
return format === 'mp3' ? 'audio/mpeg' : 'audio/mp4';
|
return format === 'mp3' ? 'audio/mpeg' : 'audio/mp4';
|
||||||
}
|
}
|
||||||
|
|
@ -300,18 +307,19 @@ export async function POST(request: NextRequest) {
|
||||||
existingSettings = null;
|
existingSettings = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const incomingSettings = data.settings;
|
const incomingSettings = data.settings ? normalizeNativeSpeedForSettings(data.settings) : undefined;
|
||||||
if (existingSettings && hasChapters && incomingSettings) {
|
if (existingSettings && hasChapters && incomingSettings) {
|
||||||
|
const normalizedExistingSettings = normalizeNativeSpeedForSettings(existingSettings);
|
||||||
const mismatch =
|
const mismatch =
|
||||||
existingSettings.ttsProvider !== incomingSettings.ttsProvider ||
|
normalizedExistingSettings.ttsProvider !== incomingSettings.ttsProvider ||
|
||||||
existingSettings.ttsModel !== incomingSettings.ttsModel ||
|
normalizedExistingSettings.ttsModel !== incomingSettings.ttsModel ||
|
||||||
existingSettings.voice !== incomingSettings.voice ||
|
normalizedExistingSettings.voice !== incomingSettings.voice ||
|
||||||
existingSettings.nativeSpeed !== incomingSettings.nativeSpeed ||
|
normalizedExistingSettings.nativeSpeed !== incomingSettings.nativeSpeed ||
|
||||||
existingSettings.postSpeed !== incomingSettings.postSpeed ||
|
normalizedExistingSettings.postSpeed !== incomingSettings.postSpeed ||
|
||||||
existingSettings.format !== incomingSettings.format ||
|
normalizedExistingSettings.format !== incomingSettings.format ||
|
||||||
(existingSettings.ttsInstructions || '') !== (incomingSettings.ttsInstructions || '');
|
(normalizedExistingSettings.ttsInstructions || '') !== (incomingSettings.ttsInstructions || '');
|
||||||
if (mismatch) {
|
if (mismatch) {
|
||||||
return NextResponse.json({ error: 'Audiobook settings mismatch', settings: existingSettings }, { status: 409 });
|
return NextResponse.json({ error: 'Audiobook settings mismatch', settings: normalizedExistingSettings }, { status: 409 });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -54,9 +54,22 @@ function getUpstreamStatus(error: unknown): number | undefined {
|
||||||
const rec = error as Record<string, unknown>;
|
const rec = error as Record<string, unknown>;
|
||||||
if (typeof rec.status === 'number') return rec.status;
|
if (typeof rec.status === 'number') return rec.status;
|
||||||
if (typeof rec.statusCode === 'number') return rec.statusCode;
|
if (typeof rec.statusCode === 'number') return rec.statusCode;
|
||||||
|
const response = rec.response as { status?: unknown } | undefined;
|
||||||
|
if (response && typeof response.status === 'number') return response.status;
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getUpstreamRetryAfterSeconds(error: unknown): number | undefined {
|
||||||
|
if (typeof error !== 'object' || error === null) return undefined;
|
||||||
|
const rec = error as Record<string, unknown>;
|
||||||
|
const response = rec.response as { headers?: { get?: (name: string) => string | null } } | undefined;
|
||||||
|
const retryAfterHeader = response?.headers?.get?.('retry-after');
|
||||||
|
if (!retryAfterHeader) return undefined;
|
||||||
|
const parsed = Number(retryAfterHeader);
|
||||||
|
if (!Number.isFinite(parsed) || parsed <= 0) return undefined;
|
||||||
|
return Math.ceil(parsed);
|
||||||
|
}
|
||||||
|
|
||||||
export async function POST(req: NextRequest) {
|
export async function POST(req: NextRequest) {
|
||||||
let providerForError: string | null = null;
|
let providerForError: string | null = null;
|
||||||
let didCreateDeviceIdCookie = false;
|
let didCreateDeviceIdCookie = false;
|
||||||
|
|
@ -221,14 +234,18 @@ export async function POST(req: NextRequest) {
|
||||||
|
|
||||||
const upstreamStatus = getUpstreamStatus(error);
|
const upstreamStatus = getUpstreamStatus(error);
|
||||||
if (upstreamStatus === 429) {
|
if (upstreamStatus === 429) {
|
||||||
|
const retryAfterSeconds = getUpstreamRetryAfterSeconds(error);
|
||||||
const problem: ProblemDetails = {
|
const problem: ProblemDetails = {
|
||||||
type: PROBLEM_TYPES.upstreamRateLimited,
|
type: PROBLEM_TYPES.upstreamRateLimited,
|
||||||
title: 'Upstream rate limited',
|
title: 'Upstream rate limited',
|
||||||
status: 429,
|
status: 429,
|
||||||
detail: 'The TTS provider is rate limiting requests. Please try again shortly.',
|
detail: retryAfterSeconds
|
||||||
|
? `The TTS provider is rate limiting requests. Please retry in about ${retryAfterSeconds}s.`
|
||||||
|
: 'The TTS provider is rate limiting requests. Please try again shortly.',
|
||||||
code: 'UPSTREAM_RATE_LIMIT',
|
code: 'UPSTREAM_RATE_LIMIT',
|
||||||
provider: providerForError ?? undefined,
|
provider: providerForError ?? undefined,
|
||||||
upstreamStatus,
|
upstreamStatus,
|
||||||
|
retryAfterSeconds,
|
||||||
instance: req.nextUrl.pathname,
|
instance: req.nextUrl.pathname,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
@ -236,11 +253,16 @@ export async function POST(req: NextRequest) {
|
||||||
status: 429,
|
status: 429,
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': 'application/problem+json',
|
'Content-Type': 'application/problem+json',
|
||||||
|
...(retryAfterSeconds ? { 'Retry-After': String(retryAfterSeconds) } : {}),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
console.warn('Error generating TTS:', error);
|
const statusHint = getUpstreamStatus(error);
|
||||||
|
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||||
|
console.warn(
|
||||||
|
`Error generating TTS${typeof statusHint === 'number' ? ` (upstream ${statusHint})` : ''}: ${errorMessage}`,
|
||||||
|
);
|
||||||
const errorBody: TTSError = {
|
const errorBody: TTSError = {
|
||||||
code: 'TTS_GENERATION_FAILED',
|
code: 'TTS_GENERATION_FAILED',
|
||||||
message: 'Failed to generate audio',
|
message: 'Failed to generate audio',
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ import { LoadingSpinner } from '@/components/Spinner';
|
||||||
import { useConfig } from '@/contexts/ConfigContext';
|
import { useConfig } from '@/contexts/ConfigContext';
|
||||||
import { useTTS } from '@/contexts/TTSContext';
|
import { useTTS } from '@/contexts/TTSContext';
|
||||||
import { VoicesControlBase } from '@/components/player/VoicesControlBase';
|
import { VoicesControlBase } from '@/components/player/VoicesControlBase';
|
||||||
import { supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog';
|
import { supportsNativeModelSpeed, supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog';
|
||||||
import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts';
|
import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts';
|
||||||
import {
|
import {
|
||||||
getAudiobookStatus,
|
getAudiobookStatus,
|
||||||
|
|
@ -73,6 +73,8 @@ export function AudiobookExportModal({
|
||||||
const formatSpeed = useCallback((speed: number) => {
|
const formatSpeed = useCallback((speed: number) => {
|
||||||
return Number.isInteger(speed) ? speed.toString() : speed.toFixed(1);
|
return Number.isInteger(speed) ? speed.toString() : speed.toFixed(1);
|
||||||
}, []);
|
}, []);
|
||||||
|
const nativeSpeedSupported = useMemo(() => supportsNativeModelSpeed(ttsProvider, ttsModel), [ttsProvider, ttsModel]);
|
||||||
|
const effectiveNativeSpeed = nativeSpeedSupported ? nativeSpeed : 1;
|
||||||
|
|
||||||
const hasExistingAudiobook = Boolean(bookId) || chapters.length > 0;
|
const hasExistingAudiobook = Boolean(bookId) || chapters.length > 0;
|
||||||
const isLegacyAudiobookMissingSettings = hasExistingAudiobook && savedSettings === null;
|
const isLegacyAudiobookMissingSettings = hasExistingAudiobook && savedSettings === null;
|
||||||
|
|
@ -113,12 +115,12 @@ export function AudiobookExportModal({
|
||||||
ttsProvider,
|
ttsProvider,
|
||||||
ttsModel,
|
ttsModel,
|
||||||
voice: nextVoice,
|
voice: nextVoice,
|
||||||
nativeSpeed,
|
nativeSpeed: effectiveNativeSpeed,
|
||||||
postSpeed,
|
postSpeed,
|
||||||
format,
|
format,
|
||||||
ttsInstructions: supportsTtsInstructions(ttsModel) ? ttsInstructions : undefined,
|
ttsInstructions: supportsTtsInstructions(ttsModel) ? ttsInstructions : undefined,
|
||||||
};
|
};
|
||||||
}, [savedSettings, audiobookVoice, configVoice, availableVoices, ttsProvider, ttsModel, ttsInstructions, nativeSpeed, postSpeed, format]);
|
}, [savedSettings, audiobookVoice, configVoice, availableVoices, ttsProvider, ttsModel, ttsInstructions, effectiveNativeSpeed, postSpeed, format]);
|
||||||
|
|
||||||
const fetchExistingChapters = useCallback(async (soft: boolean = false) => {
|
const fetchExistingChapters = useCallback(async (soft: boolean = false) => {
|
||||||
if (soft) {
|
if (soft) {
|
||||||
|
|
@ -529,11 +531,15 @@ export function AudiobookExportModal({
|
||||||
<div className="text-sm font-medium text-foreground">{savedSettings.format.toUpperCase()}</div>
|
<div className="text-sm font-medium text-foreground">{savedSettings.format.toUpperCase()}</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div className="grid grid-cols-2 gap-3">
|
<div className="grid grid-cols-2 gap-3">
|
||||||
<div className="rounded-lg bg-base p-3">
|
<div className="rounded-lg bg-base p-3">
|
||||||
<div className="text-[11px] uppercase tracking-wider text-muted mb-1">Native speed</div>
|
<div className="text-[11px] uppercase tracking-wider text-muted mb-1">Native speed</div>
|
||||||
<div className="text-sm font-medium text-foreground">{formatSpeed(savedSettings.nativeSpeed)}x</div>
|
<div className="text-sm font-medium text-foreground">
|
||||||
</div>
|
{supportsNativeModelSpeed(savedSettings.ttsProvider, savedSettings.ttsModel)
|
||||||
|
? `${formatSpeed(savedSettings.nativeSpeed)}x`
|
||||||
|
: 'Not supported'}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<div className="rounded-lg bg-base p-3">
|
<div className="rounded-lg bg-base p-3">
|
||||||
<div className="text-[11px] uppercase tracking-wider text-muted mb-1">Post speed</div>
|
<div className="text-[11px] uppercase tracking-wider text-muted mb-1">Post speed</div>
|
||||||
<div className="text-sm font-medium text-foreground">{formatSpeed(savedSettings.postSpeed)}x</div>
|
<div className="text-sm font-medium text-foreground">{formatSpeed(savedSettings.postSpeed)}x</div>
|
||||||
|
|
@ -616,29 +622,39 @@ export function AudiobookExportModal({
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Speed controls */}
|
{/* Speed controls */}
|
||||||
<div className="rounded-lg bg-base p-3 space-y-3">
|
<div className="rounded-lg bg-base p-3 space-y-3">
|
||||||
<div className="space-y-2">
|
{!nativeSpeedSupported && (
|
||||||
<div className="flex items-center justify-between">
|
<div className="rounded-md border border-offbase bg-background px-2 py-1.5 text-[11px] text-muted">
|
||||||
<label className="text-[11px] uppercase tracking-wider font-medium text-muted">Native model speed</label>
|
Native model speed is not available for this model.
|
||||||
<span className="text-xs font-medium text-accent tabular-nums">{formatSpeed(nativeSpeed)}x</span>
|
</div>
|
||||||
</div>
|
)}
|
||||||
<input
|
|
||||||
type="range"
|
|
||||||
min="0.5"
|
|
||||||
max="3"
|
|
||||||
step="0.1"
|
|
||||||
value={nativeSpeed}
|
|
||||||
onChange={(e) => setNativeSpeed(parseFloat(e.target.value))}
|
|
||||||
className="w-full h-1.5 bg-offbase rounded-full appearance-none cursor-pointer [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-full [&::-webkit-slider-runnable-track]:h-1.5 [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-webkit-slider-thumb]:-mt-[5px] [&::-webkit-slider-thumb]:shadow-sm [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-full [&::-moz-range-track]:h-1.5 [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent [&::-moz-range-thumb]:border-0"
|
|
||||||
/>
|
|
||||||
<div className="flex justify-between text-[10px] text-muted">
|
|
||||||
<span>0.5x</span>
|
|
||||||
<span>3x</span>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="border-t border-offbase" />
|
{nativeSpeedSupported && (
|
||||||
|
<>
|
||||||
|
<div className="space-y-2">
|
||||||
|
<div className="flex items-center justify-between">
|
||||||
|
<label className="text-[11px] uppercase tracking-wider font-medium text-muted">Native model speed</label>
|
||||||
|
<span className="text-xs font-medium text-accent tabular-nums">{formatSpeed(nativeSpeed)}x</span>
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
type="range"
|
||||||
|
min="0.5"
|
||||||
|
max="3"
|
||||||
|
step="0.1"
|
||||||
|
value={nativeSpeed}
|
||||||
|
onChange={(e) => setNativeSpeed(parseFloat(e.target.value))}
|
||||||
|
className="w-full h-1.5 bg-offbase rounded-full appearance-none cursor-pointer [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-full [&::-webkit-slider-runnable-track]:h-1.5 [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-webkit-slider-thumb]:-mt-[5px] [&::-webkit-slider-thumb]:shadow-sm [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-full [&::-moz-range-track]:h-1.5 [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent [&::-moz-range-thumb]:border-0"
|
||||||
|
/>
|
||||||
|
<div className="flex justify-between text-[10px] text-muted">
|
||||||
|
<span>0.5x</span>
|
||||||
|
<span>3x</span>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="border-t border-offbase" />
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
|
||||||
<div className="space-y-2">
|
<div className="space-y-2">
|
||||||
<div className="flex items-center justify-between">
|
<div className="flex items-center justify-between">
|
||||||
|
|
|
||||||
|
|
@ -491,7 +491,7 @@ export function SettingsModal({ className = '' }: { className?: string }) {
|
||||||
</nav>
|
</nav>
|
||||||
|
|
||||||
{/* Content */}
|
{/* Content */}
|
||||||
<div className="flex-1 p-4 overflow-y-auto">
|
<div className="flex-1 min-w-0 p-4 overflow-y-auto">
|
||||||
{/* API Section */}
|
{/* API Section */}
|
||||||
{activeSection === 'api' && (
|
{activeSection === 'api' && (
|
||||||
<div className="space-y-4">
|
<div className="space-y-4">
|
||||||
|
|
@ -507,6 +507,9 @@ export function SettingsModal({ className = '' }: { className?: string }) {
|
||||||
} else if (provider.id === 'custom-openai') {
|
} else if (provider.id === 'custom-openai') {
|
||||||
setModelValue('kokoro');
|
setModelValue('kokoro');
|
||||||
setLocalBaseUrl('');
|
setLocalBaseUrl('');
|
||||||
|
} else if (provider.id === 'replicate') {
|
||||||
|
setModelValue('google/gemini-3.1-flash-tts');
|
||||||
|
setLocalBaseUrl('');
|
||||||
} else if (provider.id === 'deepinfra') {
|
} else if (provider.id === 'deepinfra') {
|
||||||
setModelValue('hexgrad/Kokoro-82M');
|
setModelValue('hexgrad/Kokoro-82M');
|
||||||
setLocalBaseUrl('https://api.deepinfra.com/v1/openai');
|
setLocalBaseUrl('https://api.deepinfra.com/v1/openai');
|
||||||
|
|
@ -528,7 +531,10 @@ export function SettingsModal({ className = '' }: { className?: string }) {
|
||||||
leaveFrom="opacity-100"
|
leaveFrom="opacity-100"
|
||||||
leaveTo="opacity-0"
|
leaveTo="opacity-0"
|
||||||
>
|
>
|
||||||
<ListboxOptions className="absolute mt-1 w-full overflow-auto rounded-md bg-background py-1 shadow-lg ring-1 ring-offbase focus:outline-none z-50">
|
<ListboxOptions
|
||||||
|
anchor="bottom start"
|
||||||
|
className="z-50 w-[var(--button-width)] max-h-60 overflow-y-auto overscroll-contain rounded-md bg-background py-1 shadow-lg ring-1 ring-offbase focus:outline-none [--anchor-gap:0.25rem]"
|
||||||
|
>
|
||||||
{ttsProviders.map((provider) => (
|
{ttsProviders.map((provider) => (
|
||||||
<ListboxOption
|
<ListboxOption
|
||||||
key={provider.id}
|
key={provider.id}
|
||||||
|
|
@ -556,7 +562,7 @@ export function SettingsModal({ className = '' }: { className?: string }) {
|
||||||
</Listbox>
|
</Listbox>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{(localTTSProvider === 'custom-openai' || !localBaseUrl || localBaseUrl === '') && (
|
{localTTSProvider !== 'replicate' && (localTTSProvider === 'custom-openai' || !localBaseUrl || localBaseUrl === '') && (
|
||||||
<div className="space-y-1.5">
|
<div className="space-y-1.5">
|
||||||
<label className="block text-sm font-medium text-foreground">
|
<label className="block text-sm font-medium text-foreground">
|
||||||
API Base URL
|
API Base URL
|
||||||
|
|
@ -614,7 +620,10 @@ export function SettingsModal({ className = '' }: { className?: string }) {
|
||||||
leaveFrom="opacity-100"
|
leaveFrom="opacity-100"
|
||||||
leaveTo="opacity-0"
|
leaveTo="opacity-0"
|
||||||
>
|
>
|
||||||
<ListboxOptions className="absolute mt-1 w-full overflow-auto rounded-md bg-background py-1 shadow-lg ring-1 ring-offbase focus:outline-none z-50">
|
<ListboxOptions
|
||||||
|
anchor="bottom start"
|
||||||
|
className="z-50 w-[var(--button-width)] max-h-60 overflow-y-auto overscroll-contain rounded-md bg-background py-1 shadow-lg ring-1 ring-offbase focus:outline-none [--anchor-gap:0.25rem]"
|
||||||
|
>
|
||||||
{ttsModels.map((model) => (
|
{ttsModels.map((model) => (
|
||||||
<ListboxOption
|
<ListboxOption
|
||||||
key={model.id}
|
key={model.id}
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,7 @@ import { Input, Popover, PopoverButton, PopoverPanel } from '@headlessui/react';
|
||||||
import { ChevronUpDownIcon, SpeedometerIcon } from '@/components/icons/Icons';
|
import { ChevronUpDownIcon, SpeedometerIcon } from '@/components/icons/Icons';
|
||||||
import { useConfig } from '@/contexts/ConfigContext';
|
import { useConfig } from '@/contexts/ConfigContext';
|
||||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||||
|
import { supportsNativeModelSpeed } from '@/lib/shared/tts-provider-catalog';
|
||||||
|
|
||||||
export const SpeedControl = ({
|
export const SpeedControl = ({
|
||||||
setSpeedAndRestart,
|
setSpeedAndRestart,
|
||||||
|
|
@ -12,7 +13,8 @@ export const SpeedControl = ({
|
||||||
setSpeedAndRestart: (speed: number) => void;
|
setSpeedAndRestart: (speed: number) => void;
|
||||||
setAudioPlayerSpeedAndRestart: (speed: number) => void;
|
setAudioPlayerSpeedAndRestart: (speed: number) => void;
|
||||||
}) => {
|
}) => {
|
||||||
const { voiceSpeed, audioPlayerSpeed } = useConfig();
|
const { voiceSpeed, audioPlayerSpeed, ttsProvider, ttsModel } = useConfig();
|
||||||
|
const nativeSpeedSupported = supportsNativeModelSpeed(ttsProvider, ttsModel);
|
||||||
|
|
||||||
const [localVoiceSpeed, setLocalVoiceSpeed] = useState(voiceSpeed);
|
const [localVoiceSpeed, setLocalVoiceSpeed] = useState(voiceSpeed);
|
||||||
const [localAudioSpeed, setLocalAudioSpeed] = useState(audioPlayerSpeed);
|
const [localAudioSpeed, setLocalAudioSpeed] = useState(audioPlayerSpeed);
|
||||||
|
|
@ -53,28 +55,28 @@ export const SpeedControl = ({
|
||||||
const triggerLabel = useMemo(
|
const triggerLabel = useMemo(
|
||||||
() => {
|
() => {
|
||||||
const parts: string[] = [];
|
const parts: string[] = [];
|
||||||
if (localVoiceSpeed !== 1.0) parts.push(`${formatSpeed(localVoiceSpeed, 1)}x`);
|
if (nativeSpeedSupported && localVoiceSpeed !== 1.0) parts.push(`${formatSpeed(localVoiceSpeed, 1)}x`);
|
||||||
if (localAudioSpeed !== 1.0) parts.push(`${formatSpeed(localAudioSpeed, 1)}x`);
|
if (localAudioSpeed !== 1.0) parts.push(`${formatSpeed(localAudioSpeed, 1)}x`);
|
||||||
return parts.length > 0 ? parts.join(' • ') : '1x';
|
return parts.length > 0 ? parts.join(' • ') : '1x';
|
||||||
},
|
},
|
||||||
[formatSpeed, localVoiceSpeed, localAudioSpeed]
|
[formatSpeed, localVoiceSpeed, localAudioSpeed, nativeSpeedSupported]
|
||||||
);
|
);
|
||||||
|
|
||||||
const compactTriggerLabel = useMemo(() => {
|
const compactTriggerLabel = useMemo(() => {
|
||||||
const voiceIsDefault = localVoiceSpeed === 1.0;
|
const voiceIsDefault = !nativeSpeedSupported || localVoiceSpeed === 1.0;
|
||||||
const audioIsDefault = localAudioSpeed === 1.0;
|
const audioIsDefault = localAudioSpeed === 1.0;
|
||||||
|
|
||||||
let combined = 1.0;
|
let combined = 1.0;
|
||||||
if (!voiceIsDefault && !audioIsDefault) {
|
if (nativeSpeedSupported && !voiceIsDefault && !audioIsDefault) {
|
||||||
combined = (localVoiceSpeed + localAudioSpeed) / 2;
|
combined = (localVoiceSpeed + localAudioSpeed) / 2;
|
||||||
} else if (!voiceIsDefault) {
|
} else if (nativeSpeedSupported && !voiceIsDefault) {
|
||||||
combined = localVoiceSpeed;
|
combined = localVoiceSpeed;
|
||||||
} else if (!audioIsDefault) {
|
} else if (!audioIsDefault) {
|
||||||
combined = localAudioSpeed;
|
combined = localAudioSpeed;
|
||||||
}
|
}
|
||||||
|
|
||||||
return `${formatSpeed(combined, 2)}x`;
|
return `${formatSpeed(combined, 2)}x`;
|
||||||
}, [formatSpeed, localVoiceSpeed, localAudioSpeed]);
|
}, [formatSpeed, localVoiceSpeed, localAudioSpeed, nativeSpeedSupported]);
|
||||||
|
|
||||||
const min = 0.5;
|
const min = 0.5;
|
||||||
const max = 3;
|
const max = 3;
|
||||||
|
|
@ -90,28 +92,36 @@ export const SpeedControl = ({
|
||||||
</PopoverButton>
|
</PopoverButton>
|
||||||
<PopoverPanel anchor="top" className="absolute z-50 bg-base p-3 rounded-md shadow-lg border border-offbase">
|
<PopoverPanel anchor="top" className="absolute z-50 bg-base p-3 rounded-md shadow-lg border border-offbase">
|
||||||
<div className="flex flex-col space-y-4">
|
<div className="flex flex-col space-y-4">
|
||||||
<div className="flex flex-col space-y-2">
|
{!nativeSpeedSupported && (
|
||||||
<div className="text-xs font-medium text-foreground">Native model speed</div>
|
<div className="rounded-md border border-offbase bg-background px-2 py-1.5 text-[11px] text-muted">
|
||||||
<div className="flex justify-between">
|
Native model speed is not available for this model.
|
||||||
<span className="text-xs">{min.toFixed(1)}x</span>
|
|
||||||
<span className="text-xs font-bold">
|
|
||||||
{Number.isInteger(localVoiceSpeed) ? localVoiceSpeed.toString() : localVoiceSpeed.toFixed(1)}x
|
|
||||||
</span>
|
|
||||||
<span className="text-xs">{max.toFixed(1)}x</span>
|
|
||||||
</div>
|
</div>
|
||||||
<Input
|
)}
|
||||||
type="range"
|
|
||||||
min={min}
|
{nativeSpeedSupported && (
|
||||||
max={max}
|
<div className="flex flex-col space-y-2">
|
||||||
step={step}
|
<div className="text-xs font-medium text-foreground">Native model speed</div>
|
||||||
value={localVoiceSpeed}
|
<div className="flex justify-between">
|
||||||
onChange={handleVoiceSpeedChange}
|
<span className="text-xs">{min.toFixed(1)}x</span>
|
||||||
onMouseUp={handleVoiceSpeedChangeComplete}
|
<span className="text-xs font-bold">
|
||||||
onKeyUp={handleVoiceSpeedChangeComplete}
|
{Number.isInteger(localVoiceSpeed) ? localVoiceSpeed.toString() : localVoiceSpeed.toFixed(1)}x
|
||||||
onTouchEnd={handleVoiceSpeedChangeComplete}
|
</span>
|
||||||
className="w-full bg-offbase rounded-lg appearance-none cursor-pointer accent-accent [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-lg [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-lg [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent"
|
<span className="text-xs">{max.toFixed(1)}x</span>
|
||||||
/>
|
</div>
|
||||||
</div>
|
<Input
|
||||||
|
type="range"
|
||||||
|
min={min}
|
||||||
|
max={max}
|
||||||
|
step={step}
|
||||||
|
value={localVoiceSpeed}
|
||||||
|
onChange={handleVoiceSpeedChange}
|
||||||
|
onMouseUp={handleVoiceSpeedChangeComplete}
|
||||||
|
onKeyUp={handleVoiceSpeedChangeComplete}
|
||||||
|
onTouchEnd={handleVoiceSpeedChangeComplete}
|
||||||
|
className="w-full bg-offbase rounded-lg appearance-none cursor-pointer accent-accent [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-lg [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-lg [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent"
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
<div className="flex flex-col space-y-2">
|
<div className="flex flex-col space-y-2">
|
||||||
<div className="text-xs font-medium text-foreground">Audio player speed</div>
|
<div className="text-xs font-medium text-foreground">Audio player speed</div>
|
||||||
|
|
|
||||||
|
|
@ -39,7 +39,7 @@ import { getDocumentProgress, scheduleDocumentProgressSync } from '@/lib/client/
|
||||||
import { withRetry, generateTTS, alignAudio } from '@/lib/client/api/audiobooks';
|
import { withRetry, generateTTS, alignAudio } from '@/lib/client/api/audiobooks';
|
||||||
import { preprocessSentenceForAudio, splitTextToTtsBlocks, splitTextToTtsBlocksEPUB } from '@/lib/shared/nlp';
|
import { preprocessSentenceForAudio, splitTextToTtsBlocks, splitTextToTtsBlocksEPUB } from '@/lib/shared/nlp';
|
||||||
import { isKokoroModel } from '@/lib/shared/kokoro';
|
import { isKokoroModel } from '@/lib/shared/kokoro';
|
||||||
import { supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog';
|
import { supportsNativeModelSpeed, supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog';
|
||||||
import { useAuthRateLimit } from '@/contexts/AuthRateLimitContext';
|
import { useAuthRateLimit } from '@/contexts/AuthRateLimitContext';
|
||||||
import type {
|
import type {
|
||||||
TTSLocation,
|
TTSLocation,
|
||||||
|
|
@ -386,6 +386,10 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const [voice, setVoice] = useState(configVoice);
|
const [voice, setVoice] = useState(configVoice);
|
||||||
const [ttsModel, setTTSModel] = useState(configTTSModel);
|
const [ttsModel, setTTSModel] = useState(configTTSModel);
|
||||||
const [ttsInstructions, setTTSInstructions] = useState(configTTSInstructions);
|
const [ttsInstructions, setTTSInstructions] = useState(configTTSInstructions);
|
||||||
|
const effectiveNativeSpeed = useMemo(
|
||||||
|
() => (supportsNativeModelSpeed(configTTSProvider, ttsModel) ? speed : 1),
|
||||||
|
[configTTSProvider, ttsModel, speed],
|
||||||
|
);
|
||||||
|
|
||||||
// Track pending preload requests
|
// Track pending preload requests
|
||||||
const preloadRequests = useRef<Map<string, Promise<string>>>(new Map());
|
const preloadRequests = useRef<Map<string, Promise<string>>>(new Map());
|
||||||
|
|
@ -1015,7 +1019,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const alignmentKey = buildCacheKey(
|
const alignmentKey = buildCacheKey(
|
||||||
sentence,
|
sentence,
|
||||||
voice,
|
voice,
|
||||||
speed,
|
effectiveNativeSpeed,
|
||||||
configTTSProvider,
|
configTTSProvider,
|
||||||
ttsModel,
|
ttsModel,
|
||||||
);
|
);
|
||||||
|
|
@ -1053,7 +1057,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const audioCacheKey = buildCacheKey(
|
const audioCacheKey = buildCacheKey(
|
||||||
sentence,
|
sentence,
|
||||||
voice,
|
voice,
|
||||||
speed,
|
effectiveNativeSpeed,
|
||||||
configTTSProvider,
|
configTTSProvider,
|
||||||
ttsModel,
|
ttsModel,
|
||||||
);
|
);
|
||||||
|
|
@ -1097,7 +1101,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const reqBody: TTSRequestPayload = {
|
const reqBody: TTSRequestPayload = {
|
||||||
text: sentence,
|
text: sentence,
|
||||||
voice,
|
voice,
|
||||||
speed,
|
speed: effectiveNativeSpeed,
|
||||||
format: 'mp3',
|
format: 'mp3',
|
||||||
model: ttsModel,
|
model: ttsModel,
|
||||||
instructions: supportsTtsInstructions(ttsModel) ? ttsInstructions : undefined,
|
instructions: supportsTtsInstructions(ttsModel) ? ttsInstructions : undefined,
|
||||||
|
|
@ -1190,7 +1194,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
}
|
}
|
||||||
}, [
|
}, [
|
||||||
voice,
|
voice,
|
||||||
speed,
|
effectiveNativeSpeed,
|
||||||
ttsModel,
|
ttsModel,
|
||||||
ttsInstructions,
|
ttsInstructions,
|
||||||
audioCache,
|
audioCache,
|
||||||
|
|
@ -1532,7 +1536,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const alignmentKey = buildCacheKey(
|
const alignmentKey = buildCacheKey(
|
||||||
sentence,
|
sentence,
|
||||||
voice,
|
voice,
|
||||||
speed,
|
effectiveNativeSpeed,
|
||||||
configTTSProvider,
|
configTTSProvider,
|
||||||
ttsModel,
|
ttsModel,
|
||||||
);
|
);
|
||||||
|
|
@ -1549,7 +1553,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
if (howl) {
|
if (howl) {
|
||||||
howl.play();
|
howl.play();
|
||||||
}
|
}
|
||||||
}, [sentences, currentIndex, playSentenceWithHowl, voice, speed, configTTSProvider, ttsModel]);
|
}, [sentences, currentIndex, playSentenceWithHowl, voice, effectiveNativeSpeed, configTTSProvider, ttsModel]);
|
||||||
|
|
||||||
// Keep the current playback rate applied to the active Howl. Some browsers (notably
|
// Keep the current playback rate applied to the active Howl. Some browsers (notably
|
||||||
// iOS Safari with HTML5 audio) can reset playbackRate after initial load/play.
|
// iOS Safari with HTML5 audio) can reset playbackRate after initial load/play.
|
||||||
|
|
@ -1620,7 +1624,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const nextKey = buildCacheKey(
|
const nextKey = buildCacheKey(
|
||||||
nextSentence,
|
nextSentence,
|
||||||
voice,
|
voice,
|
||||||
speed,
|
effectiveNativeSpeed,
|
||||||
configTTSProvider,
|
configTTSProvider,
|
||||||
ttsModel,
|
ttsModel,
|
||||||
);
|
);
|
||||||
|
|
@ -1673,7 +1677,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error('Error initiating preload:', error);
|
console.error('Error initiating preload:', error);
|
||||||
}
|
}
|
||||||
}, [isAtLimit, currentIndex, sentences, audioCache, processSentence, voice, speed, configTTSProvider, ttsModel]);
|
}, [isAtLimit, currentIndex, sentences, audioCache, processSentence, voice, effectiveNativeSpeed, configTTSProvider, ttsModel]);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Main Playback Driver
|
* Main Playback Driver
|
||||||
|
|
|
||||||
|
|
@ -1,7 +1,8 @@
|
||||||
import OpenAI from 'openai';
|
import OpenAI from 'openai';
|
||||||
|
import Replicate from 'replicate';
|
||||||
import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs';
|
import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs';
|
||||||
import { isKokoroModel } from '@/lib/shared/kokoro';
|
import { isKokoroModel } from '@/lib/shared/kokoro';
|
||||||
import { supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog';
|
import { supportsNativeModelSpeed, supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog';
|
||||||
import { LRUCache } from 'lru-cache';
|
import { LRUCache } from 'lru-cache';
|
||||||
import { createHash } from 'crypto';
|
import { createHash } from 'crypto';
|
||||||
import { access, readFile } from 'fs/promises';
|
import { access, readFile } from 'fs/promises';
|
||||||
|
|
@ -45,6 +46,9 @@ type InflightEntry = {
|
||||||
consumers: number;
|
consumers: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let replicateQueue: Promise<void> = Promise.resolve();
|
||||||
|
let replicateBlockedUntilMs = 0;
|
||||||
|
|
||||||
const TTS_CACHE_MAX_SIZE_BYTES = Number(process.env.TTS_CACHE_MAX_SIZE_BYTES || 256 * 1024 * 1024); // 256MB
|
const TTS_CACHE_MAX_SIZE_BYTES = Number(process.env.TTS_CACHE_MAX_SIZE_BYTES || 256 * 1024 * 1024); // 256MB
|
||||||
const TTS_CACHE_TTL_MS = Number(process.env.TTS_CACHE_TTL_MS || 1000 * 60 * 30); // 30 minutes
|
const TTS_CACHE_TTL_MS = Number(process.env.TTS_CACHE_TTL_MS || 1000 * 60 * 30); // 30 minutes
|
||||||
|
|
||||||
|
|
@ -63,17 +67,79 @@ function sleep(ms: number) {
|
||||||
return new Promise((res) => setTimeout(res, ms));
|
return new Promise((res) => setTimeout(res, ms));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function sleepWithSignal(ms: number, signal: AbortSignal): Promise<void> {
|
||||||
|
if (ms <= 0) return Promise.resolve();
|
||||||
|
if (signal.aborted) {
|
||||||
|
throw new DOMException('The operation was aborted.', 'AbortError');
|
||||||
|
}
|
||||||
|
|
||||||
|
return new Promise<void>((resolve, reject) => {
|
||||||
|
const timer = setTimeout(() => {
|
||||||
|
signal.removeEventListener('abort', onAbort);
|
||||||
|
resolve();
|
||||||
|
}, ms);
|
||||||
|
|
||||||
|
const onAbort = () => {
|
||||||
|
clearTimeout(timer);
|
||||||
|
signal.removeEventListener('abort', onAbort);
|
||||||
|
reject(new DOMException('The operation was aborted.', 'AbortError'));
|
||||||
|
};
|
||||||
|
|
||||||
|
signal.addEventListener('abort', onAbort, { once: true });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
function getUpstreamStatus(error: unknown): number | undefined {
|
function getUpstreamStatus(error: unknown): number | undefined {
|
||||||
if (typeof error !== 'object' || error === null) return undefined;
|
if (typeof error !== 'object' || error === null) return undefined;
|
||||||
const rec = error as Record<string, unknown>;
|
const rec = error as Record<string, unknown>;
|
||||||
if (typeof rec.status === 'number') return rec.status;
|
if (typeof rec.status === 'number') return rec.status;
|
||||||
if (typeof rec.statusCode === 'number') return rec.statusCode;
|
if (typeof rec.statusCode === 'number') return rec.statusCode;
|
||||||
|
const response = rec.response as { status?: unknown } | undefined;
|
||||||
|
if (response && typeof response.status === 'number') return response.status;
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function getUpstreamRetryAfterSeconds(error: unknown): number | undefined {
|
||||||
|
if (typeof error !== 'object' || error === null) return undefined;
|
||||||
|
const rec = error as Record<string, unknown>;
|
||||||
|
const response = rec.response as { headers?: { get?: (name: string) => string | null } } | undefined;
|
||||||
|
const retryAfterHeader = response?.headers?.get?.('retry-after');
|
||||||
|
if (!retryAfterHeader) return undefined;
|
||||||
|
const parsed = Number(retryAfterHeader);
|
||||||
|
if (!Number.isFinite(parsed) || parsed <= 0) return undefined;
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
|
function applyReplicateCooldown(cooldownMs: number) {
|
||||||
|
if (!Number.isFinite(cooldownMs) || cooldownMs <= 0) return;
|
||||||
|
const next = Date.now() + cooldownMs;
|
||||||
|
replicateBlockedUntilMs = Math.max(replicateBlockedUntilMs, next);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runWithReplicateGate<T>(signal: AbortSignal, operation: () => Promise<T>): Promise<T> {
|
||||||
|
let release: (() => void) | undefined;
|
||||||
|
const prev = replicateQueue;
|
||||||
|
replicateQueue = new Promise<void>((resolve) => {
|
||||||
|
release = resolve;
|
||||||
|
});
|
||||||
|
|
||||||
|
await prev;
|
||||||
|
try {
|
||||||
|
const waitMs = Math.max(0, replicateBlockedUntilMs - Date.now());
|
||||||
|
if (waitMs > 0) {
|
||||||
|
await sleepWithSignal(waitMs, signal);
|
||||||
|
}
|
||||||
|
return await operation();
|
||||||
|
} finally {
|
||||||
|
release?.();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function resolveTTSRequest(input: ServerTTSRequest): ResolvedServerTTSRequest {
|
function resolveTTSRequest(input: ServerTTSRequest): ResolvedServerTTSRequest {
|
||||||
const provider = input.provider || 'openai';
|
const provider = input.provider || 'openai';
|
||||||
const rawModel = provider === 'deepinfra' && !input.model ? 'hexgrad/Kokoro-82M' : input.model;
|
const rawModel = provider === 'deepinfra' && !input.model ? 'hexgrad/Kokoro-82M'
|
||||||
|
: provider === 'replicate' && !input.model ? 'google/gemini-3.1-flash-tts'
|
||||||
|
: input.model;
|
||||||
const model = (rawModel ?? 'gpt-4o-mini-tts') as SpeechCreateParams['model'];
|
const model = (rawModel ?? 'gpt-4o-mini-tts') as SpeechCreateParams['model'];
|
||||||
|
|
||||||
const normalizedVoice = (
|
const normalizedVoice = (
|
||||||
|
|
@ -83,7 +149,8 @@ function resolveTTSRequest(input: ServerTTSRequest): ResolvedServerTTSRequest {
|
||||||
) as string;
|
) as string;
|
||||||
|
|
||||||
const format = input.format || 'mp3';
|
const format = input.format || 'mp3';
|
||||||
const speed = Number.isFinite(Number(input.speed)) ? Number(input.speed) : 1;
|
const requestedSpeed = Number.isFinite(Number(input.speed)) ? Number(input.speed) : 1;
|
||||||
|
const speed = supportsNativeModelSpeed(provider, model as string) ? requestedSpeed : 1;
|
||||||
const instructions = supportsTtsInstructions(model as string) && input.instructions
|
const instructions = supportsTtsInstructions(model as string) && input.instructions
|
||||||
? input.instructions
|
? input.instructions
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
@ -196,10 +263,112 @@ async function fetchTTSBufferWithRetry(
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function buildReplicateInput(request: ResolvedServerTTSRequest): Record<string, unknown> {
|
||||||
|
const model = request.model as string;
|
||||||
|
|
||||||
|
if (model === 'google/gemini-3.1-flash-tts') {
|
||||||
|
const input: Record<string, unknown> = {
|
||||||
|
text: request.text,
|
||||||
|
voice: request.voice,
|
||||||
|
};
|
||||||
|
if (request.instructions) {
|
||||||
|
input.prompt = request.instructions;
|
||||||
|
}
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (model === 'minimax/speech-2.8-turbo') {
|
||||||
|
const input: Record<string, unknown> = {
|
||||||
|
text: request.text,
|
||||||
|
voice_id: request.voice,
|
||||||
|
audio_format: request.format === 'mp3' ? 'mp3' : 'wav',
|
||||||
|
};
|
||||||
|
if (request.speed !== 1) {
|
||||||
|
input.speed = Math.max(0.5, Math.min(2.0, request.speed));
|
||||||
|
}
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (model === 'qwen/qwen3-tts') {
|
||||||
|
const input: Record<string, unknown> = {
|
||||||
|
text: request.text,
|
||||||
|
mode: 'custom_voice',
|
||||||
|
speaker: request.voice,
|
||||||
|
};
|
||||||
|
if (request.instructions) {
|
||||||
|
input.style_instruction = request.instructions;
|
||||||
|
}
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (model === 'inworld/tts-1.5-mini') {
|
||||||
|
const input: Record<string, unknown> = {
|
||||||
|
text: request.text,
|
||||||
|
voice_id: request.voice,
|
||||||
|
audio_format: request.format === 'mp3' ? 'mp3' : 'wav',
|
||||||
|
};
|
||||||
|
if (request.speed !== 1) {
|
||||||
|
input.speaking_rate = request.speed;
|
||||||
|
}
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
|
||||||
|
return { text: request.text };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function runReplicateRequest(request: ResolvedServerTTSRequest, signal: AbortSignal): Promise<Buffer> {
|
||||||
|
const replicate = new Replicate({ auth: request.apiKey });
|
||||||
|
const input = buildReplicateInput(request);
|
||||||
|
const modelId = request.model as `${string}/${string}`;
|
||||||
|
|
||||||
|
return runWithReplicateGate(signal, async () => {
|
||||||
|
const maxRetries = Number(process.env.TTS_MAX_RETRIES ?? 2);
|
||||||
|
let attempt = 0;
|
||||||
|
|
||||||
|
for (; ;) {
|
||||||
|
try {
|
||||||
|
const output = await replicate.run(modelId, { input, signal }) as unknown;
|
||||||
|
|
||||||
|
// Output is a URI string pointing to the generated audio file
|
||||||
|
const audioUrl = typeof output === 'string' ? output : String(output);
|
||||||
|
const audioResponse = await fetch(audioUrl, { signal });
|
||||||
|
if (!audioResponse.ok) {
|
||||||
|
throw new Error(`Failed to fetch Replicate audio: ${audioResponse.status}`);
|
||||||
|
}
|
||||||
|
const buffer = await audioResponse.arrayBuffer();
|
||||||
|
return Buffer.from(buffer);
|
||||||
|
} catch (error) {
|
||||||
|
if (signal.aborted || (error instanceof Error && error.name === 'AbortError')) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
const status = getUpstreamStatus(error) ?? 0;
|
||||||
|
const retryable = status === 429 || status >= 500;
|
||||||
|
const retryAfterSeconds = status === 429 ? getUpstreamRetryAfterSeconds(error) : undefined;
|
||||||
|
const delay = retryAfterSeconds ? Math.max(retryAfterSeconds * 1000, 1000) : 10_000;
|
||||||
|
if (status === 429) {
|
||||||
|
applyReplicateCooldown(delay);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!retryable || attempt >= maxRetries) {
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
|
|
||||||
|
await sleepWithSignal(delay, signal);
|
||||||
|
attempt += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async function runProviderRequest(request: ResolvedServerTTSRequest, signal: AbortSignal): Promise<Buffer> {
|
async function runProviderRequest(request: ResolvedServerTTSRequest, signal: AbortSignal): Promise<Buffer> {
|
||||||
const mockBuffer = await getTestMockTtsBuffer(request.testNamespace);
|
const mockBuffer = await getTestMockTtsBuffer(request.testNamespace);
|
||||||
if (mockBuffer) return mockBuffer;
|
if (mockBuffer) return mockBuffer;
|
||||||
|
|
||||||
|
if (request.provider === 'replicate') {
|
||||||
|
return runReplicateRequest(request, signal);
|
||||||
|
}
|
||||||
|
|
||||||
const openai = new OpenAI({
|
const openai = new OpenAI({
|
||||||
apiKey: request.apiKey,
|
apiKey: request.apiKey,
|
||||||
baseURL: request.baseUrl,
|
baseURL: request.baseUrl,
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
import { isKokoroModel } from '@/lib/shared/kokoro';
|
import { isKokoroModel } from '@/lib/shared/kokoro';
|
||||||
|
|
||||||
export type TtsProviderId = 'custom-openai' | 'deepinfra' | 'openai';
|
export type TtsProviderId = 'custom-openai' | 'replicate' | 'deepinfra' | 'openai';
|
||||||
export type TtsVoiceSource = 'static' | 'deepinfra-api' | 'custom-openai-api';
|
export type TtsVoiceSource = 'static' | 'deepinfra-api' | 'custom-openai-api';
|
||||||
|
|
||||||
export interface TtsModelDefinition {
|
export interface TtsModelDefinition {
|
||||||
|
|
@ -54,6 +54,13 @@ const DEEPINFRA_MODELS_LIMITED: TtsModelDefinition[] = [
|
||||||
{ id: 'hexgrad/Kokoro-82M', name: 'hexgrad/Kokoro-82M' },
|
{ id: 'hexgrad/Kokoro-82M', name: 'hexgrad/Kokoro-82M' },
|
||||||
];
|
];
|
||||||
|
|
||||||
|
const REPLICATE_MODELS: TtsModelDefinition[] = [
|
||||||
|
{ id: 'google/gemini-3.1-flash-tts', name: 'Gemini 3.1 Flash TTS' },
|
||||||
|
{ id: 'minimax/speech-2.8-turbo', name: 'MiniMax Speech 2.8 Turbo' },
|
||||||
|
{ id: 'qwen/qwen3-tts', name: 'Qwen3 TTS' },
|
||||||
|
{ id: 'inworld/tts-1.5-mini', name: 'Inworld TTS 1.5 Mini' },
|
||||||
|
];
|
||||||
|
|
||||||
const DEFAULT_MODELS: TtsModelDefinition[] = [{ id: 'tts-1', name: 'TTS-1' }];
|
const DEFAULT_MODELS: TtsModelDefinition[] = [{ id: 'tts-1', name: 'TTS-1' }];
|
||||||
|
|
||||||
export const OPENAI_DEFAULT_VOICES = ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer'] as const;
|
export const OPENAI_DEFAULT_VOICES = ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer'] as const;
|
||||||
|
|
@ -71,6 +78,22 @@ export const KOKORO_DEFAULT_VOICES = [
|
||||||
export const ORPHEUS_DEFAULT_VOICES = ['tara', 'leah', 'jess', 'leo', 'dan', 'mia', 'zac'] as const;
|
export const ORPHEUS_DEFAULT_VOICES = ['tara', 'leah', 'jess', 'leo', 'dan', 'mia', 'zac'] as const;
|
||||||
export const SESAME_DEFAULT_VOICES = ['conversational_a', 'conversational_b', 'read_speech_a', 'read_speech_b', 'read_speech_c', 'read_speech_d', 'none'] as const;
|
export const SESAME_DEFAULT_VOICES = ['conversational_a', 'conversational_b', 'read_speech_a', 'read_speech_b', 'read_speech_c', 'read_speech_d', 'none'] as const;
|
||||||
|
|
||||||
|
// Replicate model voices
|
||||||
|
export const GEMINI_FLASH_TTS_VOICES = [
|
||||||
|
'Zephyr', 'Puck', 'Charon', 'Kore', 'Fenrir', 'Leda', 'Orus', 'Aoede',
|
||||||
|
'Callirrhoe', 'Autonoe', 'Enceladus', 'Iapetus', 'Umbriel', 'Algenib',
|
||||||
|
'Despina', 'Erinome', 'Laomedeia', 'Achernar', 'Algieba', 'Schedar',
|
||||||
|
'Gacrux', 'Pulcherrima', 'Achird', 'Zubenelgenubi', 'Vindemiatrix',
|
||||||
|
'Sadachbia', 'Sadaltager', 'Sulafat', 'Alnilam', 'Rasalgethi',
|
||||||
|
] as const;
|
||||||
|
export const MINIMAX_SPEECH_VOICES = [
|
||||||
|
'Deep_Voice_Man', 'Imposing_Manner', 'Elegant_Man', 'Casual_Guy',
|
||||||
|
'Friendly_Person', 'Decent_Boy', 'Lively_Girl', 'Exuberant_Girl',
|
||||||
|
'Inspirational_girl', 'Young_Knight', 'Abbess', 'Wise_Woman',
|
||||||
|
] as const;
|
||||||
|
export const QWEN3_TTS_VOICES = ['Aiden', 'Dylan'] as const;
|
||||||
|
export const INWORLD_TTS_VOICES = ['Ashley', 'Dennis', 'Alex', 'Darlene'] as const;
|
||||||
|
|
||||||
export const TTS_PROVIDER_DEFINITIONS: TtsProviderDefinition[] = [
|
export const TTS_PROVIDER_DEFINITIONS: TtsProviderDefinition[] = [
|
||||||
{
|
{
|
||||||
id: 'custom-openai',
|
id: 'custom-openai',
|
||||||
|
|
@ -78,6 +101,12 @@ export const TTS_PROVIDER_DEFINITIONS: TtsProviderDefinition[] = [
|
||||||
supportsCustomModel: true,
|
supportsCustomModel: true,
|
||||||
models: () => CUSTOM_OPENAI_MODELS,
|
models: () => CUSTOM_OPENAI_MODELS,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
id: 'replicate',
|
||||||
|
name: 'Replicate',
|
||||||
|
supportsCustomModel: false,
|
||||||
|
models: () => REPLICATE_MODELS,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
id: 'deepinfra',
|
id: 'deepinfra',
|
||||||
name: 'Deepinfra',
|
name: 'Deepinfra',
|
||||||
|
|
@ -97,8 +126,31 @@ export const TTS_PROVIDER_DEFINITIONS: TtsProviderDefinition[] = [
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
const MODELS_WITH_INSTRUCTIONS = new Set([
|
||||||
|
'gpt-4o-mini-tts',
|
||||||
|
'google/gemini-3.1-flash-tts',
|
||||||
|
'qwen/qwen3-tts',
|
||||||
|
]);
|
||||||
|
|
||||||
|
const REPLICATE_MODELS_WITHOUT_NATIVE_SPEED = new Set([
|
||||||
|
'google/gemini-3.1-flash-tts',
|
||||||
|
'qwen/qwen3-tts',
|
||||||
|
]);
|
||||||
|
|
||||||
export function supportsTtsInstructions(model: string | null | undefined): boolean {
|
export function supportsTtsInstructions(model: string | null | undefined): boolean {
|
||||||
return model === 'gpt-4o-mini-tts';
|
return !!model && MODELS_WITH_INSTRUCTIONS.has(model);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function supportsNativeModelSpeed(provider: string | null | undefined, model: string | null | undefined): boolean {
|
||||||
|
if (!model) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (provider === 'replicate' && REPLICATE_MODELS_WITHOUT_NATIVE_SPEED.has(model)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function getProviderDefinition(provider: string | null | undefined): TtsProviderDefinition | undefined {
|
export function getProviderDefinition(provider: string | null | undefined): TtsProviderDefinition | undefined {
|
||||||
|
|
@ -128,6 +180,22 @@ export function getDefaultVoices(provider: string, model: string): string[] {
|
||||||
return [...CUSTOM_OPENAI_DEFAULT_VOICES];
|
return [...CUSTOM_OPENAI_DEFAULT_VOICES];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (provider === 'replicate') {
|
||||||
|
if (model === 'google/gemini-3.1-flash-tts') {
|
||||||
|
return [...GEMINI_FLASH_TTS_VOICES];
|
||||||
|
}
|
||||||
|
if (model === 'minimax/speech-2.8-turbo') {
|
||||||
|
return [...MINIMAX_SPEECH_VOICES];
|
||||||
|
}
|
||||||
|
if (model === 'qwen/qwen3-tts') {
|
||||||
|
return [...QWEN3_TTS_VOICES];
|
||||||
|
}
|
||||||
|
if (model === 'inworld/tts-1.5-mini') {
|
||||||
|
return [...INWORLD_TTS_VOICES];
|
||||||
|
}
|
||||||
|
return [...GEMINI_FLASH_TTS_VOICES];
|
||||||
|
}
|
||||||
|
|
||||||
if (provider === 'deepinfra') {
|
if (provider === 'deepinfra') {
|
||||||
if (model === 'hexgrad/Kokoro-82M') {
|
if (model === 'hexgrad/Kokoro-82M') {
|
||||||
return [...KOKORO_DEFAULT_VOICES];
|
return [...KOKORO_DEFAULT_VOICES];
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue