feat: add support for Replicate TTS provider and models
- Updated environment variables documentation to include Replicate as a TTS provider option. - Added Replicate to the sidebar for TTS provider guides. - Included Replicate as a dependency in package.json and pnpm-lock.yaml. - Enhanced audiobook chapter generation to normalize native speed settings based on the TTS provider. - Improved error handling in TTS API routes to provide retry information for rate-limited responses. - Updated AudiobookExportModal to reflect native speed support for Replicate models. - Modified SettingsModal to set default model for Replicate. - Enhanced SpeedControl component to conditionally render native speed controls based on provider support. - Updated TTSContext to utilize effective native speed for TTS requests. - Implemented Replicate request handling in the TTS generation logic. - Added new documentation for configuring Replicate as a TTS provider.
This commit is contained in:
Richard R
parent
6606b0ee4a
commit
9da9232d39
18 changed files with 461 additions and 94 deletions
|
|
@ -18,7 +18,7 @@ OpenReader is an open source, self-host-friendly text-to-speech document reader
|
|||
|
||||
## ✨ Highlights
|
||||
|
||||
- 🎯 **Multi-provider TTS** with OpenAI-compatible endpoints (OpenAI, DeepInfra, Kokoro, KittenTTS-FastAPI, Orpheus, custom).
|
||||
- 🎯 **Multi-provider TTS** with OpenAI-compatible endpoints and cloud providers (OpenAI, Replicate, DeepInfra, Kokoro, KittenTTS-FastAPI, Orpheus, custom).
|
||||
- 📖 **Read-along playback** for PDF/EPUB with sentence-aware narration.
|
||||
- ⏱️ **Word-by-word highlighting** via optional `whisper.cpp` timestamps.
|
||||
- 🛜 **Sync + library import** to bring docs across devices and from server-mounted folders.
|
||||
|
|
|
|||
39
docs-site/docs/configure/tts-provider-guides/replicate.md
Normal file
39
docs-site/docs/configure/tts-provider-guides/replicate.md
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
---
|
||||
title: Replicate
|
||||
---
|
||||
|
||||
Use Replicate's hosted TTS models as your provider.
|
||||
|
||||
## Setup
|
||||
|
||||
**Environment variables (recommended for deployment):**
|
||||
|
||||
```env
|
||||
API_KEY=r8_...
|
||||
NEXT_PUBLIC_DEFAULT_TTS_PROVIDER=replicate
|
||||
NEXT_PUBLIC_DEFAULT_TTS_MODEL=google/gemini-3.1-flash-tts
|
||||
```
|
||||
|
||||
**Or in-app via Settings -> TTS Provider:**
|
||||
|
||||
1. Set provider to `Replicate`.
|
||||
2. Enter your `API_KEY`.
|
||||
3. Choose a model and voice.
|
||||
|
||||
Settings modal values override env vars. See [TTS Providers](../tts-providers) for how the two layers interact.
|
||||
|
||||
## Notes
|
||||
|
||||
- Built-in Replicate models:
|
||||
- `google/gemini-3.1-flash-tts`
|
||||
- `minimax/speech-2.8-turbo`
|
||||
- `qwen/qwen3-tts`
|
||||
- `inworld/tts-1.5-mini`
|
||||
- Native model speed is not available on all Replicate models; OpenReader hides/disables native speed controls where unsupported.
|
||||
- TTS requests are sent from the server, not the browser. The API key is never exposed to clients.
|
||||
|
||||
## References
|
||||
|
||||
- [Replicate](https://replicate.com/explore)
|
||||
- [TTS Providers](../tts-providers)
|
||||
- [TTS Environment Variables](../../reference/environment-variables#tts-provider-and-request-behavior)
|
||||
|
|
@ -15,10 +15,17 @@ Set env vars as deployment-level defaults. Users (or you, in a single-user setup
|
|||
## Providers
|
||||
|
||||
- **OpenAI**: Cloud. Base URL pre-filled (`https://api.openai.com/v1`). API key required.
|
||||
- **Replicate**: Cloud. Base URL managed internally by OpenReader. API key required.
|
||||
- **Deepinfra**: Cloud. Base URL pre-filled (`https://api.deepinfra.com/v1/openai`). API key required.
|
||||
- **Custom OpenAI-Like**: Self-hosted or any custom endpoint. `API_BASE` must be set manually (typically ending in `/v1`). API key optional.
|
||||
|
||||
For `OpenAI` and `Deepinfra` you only need to supply an API key. For `Custom OpenAI-Like` you must also set `API_BASE`.
|
||||
For `OpenAI`, `Deepinfra`, and `Replicate` you only need to supply an API key. For `Custom OpenAI-Like` you must also set `API_BASE`.
|
||||
|
||||
## Built-in model catalogs
|
||||
|
||||
- **Replicate** models: `google/gemini-3.1-flash-tts`, `minimax/speech-2.8-turbo`, `qwen/qwen3-tts`, `inworld/tts-1.5-mini`
|
||||
- **OpenAI** models: `tts-1`, `tts-1-hd`, `gpt-4o-mini-tts`
|
||||
- **Deepinfra** models: includes `hexgrad/Kokoro-82M` and additional hosted models (depending on API key / feature flags)
|
||||
|
||||
## Custom provider requirements
|
||||
|
||||
|
|
@ -36,6 +43,7 @@ TTS requests originate from the **Next.js server**, not the browser. `API_BASE`
|
|||
- [Kokoro-FastAPI](./tts-provider-guides/kokoro-fastapi)
|
||||
- [KittenTTS-FastAPI](./tts-provider-guides/kitten-tts-fastapi)
|
||||
- [Orpheus-FastAPI](./tts-provider-guides/orpheus-fastapi)
|
||||
- [Replicate](./tts-provider-guides/replicate)
|
||||
- [DeepInfra](./tts-provider-guides/deepinfra)
|
||||
- [OpenAI](./tts-provider-guides/openai)
|
||||
- [Other](./tts-provider-guides/other)
|
||||
|
|
|
|||
|
|
@ -49,8 +49,8 @@ We recommend setting these defaults for a production-like environment:
|
|||
- `NEXT_PUBLIC_ENABLE_DOCX_CONVERSION=false`: Disables DOCX upload (requires external tools anyway)
|
||||
- `NEXT_PUBLIC_ENABLE_DESTRUCTIVE_DELETE_ACTIONS=false`: Hides destructive "Delete All" actions
|
||||
- `NEXT_PUBLIC_ENABLE_TTS_PROVIDERS_TAB=false`: Hides the Settings -> TTS Provider section
|
||||
- `NEXT_PUBLIC_DEFAULT_TTS_PROVIDER=deepinfra`: Points default TTS to a scalable provider
|
||||
- `NEXT_PUBLIC_DEFAULT_TTS_MODEL=hexgrad/Kokoro-82M`: Uses a high-quality default model
|
||||
- `NEXT_PUBLIC_DEFAULT_TTS_PROVIDER=replicate`: Points default TTS to a scalable provider
|
||||
- `NEXT_PUBLIC_DEFAULT_TTS_MODEL=google/gemini-3.1-flash-tts`: Uses a high-quality default model
|
||||
- `NEXT_PUBLIC_SHOW_ALL_DEEPINFRA_MODELS=false`: Restricts usage to free models if no key is provided
|
||||
- `NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT=true`: (Optional) Controls audiobook export UI
|
||||
- `NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT=false`: (Optional) Controls word highlighting UI (requires timestamp backend)
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import TabItem from '@theme/TabItem';
|
|||
## Prerequisites
|
||||
|
||||
- A recent Docker version installed
|
||||
- A TTS API server that OpenReader can reach (Kokoro-FastAPI, KittenTTS-FastAPI, Orpheus-FastAPI, DeepInfra, OpenAI, or equivalent)
|
||||
- A TTS API server that OpenReader can reach (Kokoro-FastAPI, KittenTTS-FastAPI, Orpheus-FastAPI, Replicate, DeepInfra, OpenAI, or equivalent)
|
||||
|
||||
:::note
|
||||
If you have suitable hardware, you can run Kokoro locally with Docker. See [Kokoro-FastAPI](./configure/tts-provider-guides/kokoro-fastapi).
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ OpenReader is an open source text-to-speech document reader built with Next.js.
|
|||
|
||||
> Previously named **OpenReader-WebUI**.
|
||||
|
||||
It supports multiple TTS providers including OpenAI, DeepInfra, and custom OpenAI-compatible endpoints such as [Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI), [KittenTTS-FastAPI](https://github.com/richardr1126/KittenTTS-FastAPI), and [Orpheus-FastAPI](https://github.com/Lex-au/Orpheus-FastAPI).
|
||||
It supports multiple TTS providers including OpenAI, Replicate, DeepInfra, and custom OpenAI-compatible endpoints such as [Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI), [KittenTTS-FastAPI](https://github.com/richardr1126/KittenTTS-FastAPI), and [Orpheus-FastAPI](https://github.com/Lex-au/Orpheus-FastAPI).
|
||||
|
||||
## ✨ Highlights
|
||||
|
||||
|
|
@ -18,6 +18,7 @@ It supports multiple TTS providers including OpenAI, DeepInfra, and custom OpenA
|
|||
- [**Orpheus-FastAPI**](https://github.com/Lex-au/Orpheus-FastAPI)
|
||||
- **Custom OpenAI-compatible**: any TTS API with `/v1/audio/voices` and `/v1/audio/speech` endpoints
|
||||
- **Cloud TTS providers**:
|
||||
- [**Replicate**](https://replicate.com/explore): `google/gemini-3.1-flash-tts`, `minimax/speech-2.8-turbo`, `qwen/qwen3-tts`, and `inworld/tts-1.5-mini`
|
||||
- [**DeepInfra**](https://deepinfra.com/models/text-to-speech): Kokoro-82M and other hosted models
|
||||
- [**OpenAI API**](https://platform.openai.com/docs/pricing#transcription-and-speech): `tts-1`, `tts-1-hd`, and `gpt-4o-mini-tts`
|
||||
- 🛜 **Server-side Document Storage**
|
||||
|
|
|
|||
|
|
@ -379,7 +379,7 @@ Controls whether the **TTS Provider** section appears in the Settings modal.
|
|||
Sets the default TTS provider for new users.
|
||||
|
||||
- Default: `custom-openai`
|
||||
- Example values: `deepinfra`, `openai`, `custom-openai`
|
||||
- Example values: `replicate`, `deepinfra`, `openai`, `custom-openai`
|
||||
|
||||
### NEXT_PUBLIC_DEFAULT_TTS_MODEL
|
||||
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ const sidebars: SidebarsConfig = {
|
|||
'configure/tts-provider-guides/kokoro-fastapi',
|
||||
'configure/tts-provider-guides/kitten-tts-fastapi',
|
||||
'configure/tts-provider-guides/orpheus-fastapi',
|
||||
'configure/tts-provider-guides/replicate',
|
||||
'configure/tts-provider-guides/deepinfra',
|
||||
'configure/tts-provider-guides/openai',
|
||||
'configure/tts-provider-guides/other',
|
||||
|
|
|
|||
|
|
@ -59,6 +59,7 @@
|
|||
"react-pdf": "^9.2.1",
|
||||
"react-reader": "^2.0.15",
|
||||
"remark-gfm": "^4.0.1",
|
||||
"replicate": "^1.4.0",
|
||||
"uuid": "^11.1.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
|
|
|
|||
|
|
@ -124,6 +124,9 @@ importers:
|
|||
remark-gfm:
|
||||
specifier: ^4.0.1
|
||||
version: 4.0.1
|
||||
replicate:
|
||||
specifier: ^1.4.0
|
||||
version: 1.4.0
|
||||
uuid:
|
||||
specifier: ^11.1.0
|
||||
version: 11.1.0
|
||||
|
|
@ -3588,6 +3591,10 @@ packages:
|
|||
remark-stringify@11.0.0:
|
||||
resolution: {integrity: sha512-1OSmLd3awB/t8qdoEOMazZkNsfVTeY4fTsgzcQFdXNq8ToTN4ZGwrMnlda4K6smTFKD+GRV6O48i6Z4iKgPPpw==}
|
||||
|
||||
replicate@1.4.0:
|
||||
resolution: {integrity: sha512-1ufKejfUVz/azy+5TnzQP7U1+MHVWZ6psnQ06az8byUUnRhT+DZ/MvewzB1NQYBVMgNKR7xPDtTwlcP5nv/5+w==}
|
||||
engines: {git: '>=2.11.0', node: '>=18.0.0', npm: '>=7.19.0', yarn: '>=1.7.0'}
|
||||
|
||||
resolve-from@4.0.0:
|
||||
resolution: {integrity: sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==}
|
||||
engines: {node: '>=4'}
|
||||
|
|
@ -8028,6 +8035,10 @@ snapshots:
|
|||
mdast-util-to-markdown: 2.1.2
|
||||
unified: 11.0.5
|
||||
|
||||
replicate@1.4.0:
|
||||
optionalDependencies:
|
||||
readable-stream: 4.7.0
|
||||
|
||||
resolve-from@4.0.0: {}
|
||||
|
||||
resolve-pkg-maps@1.0.0: {}
|
||||
|
|
|
|||
|
|
@ -29,6 +29,7 @@ import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/li
|
|||
import { buildAllowedAudiobookUserIds, pickAudiobookOwner } from '@/lib/server/audiobooks/user-scope';
|
||||
import { getFFmpegPath } from '@/lib/server/audiobooks/ffmpeg-bin';
|
||||
import { generateTTSBuffer } from '@/lib/server/tts/generate';
|
||||
import { supportsNativeModelSpeed } from '@/lib/shared/tts-provider-catalog';
|
||||
import type { AudiobookGenerationSettings } from '@/types/client';
|
||||
import type { TTSAudiobookFormat } from '@/types/tts';
|
||||
|
||||
|
|
@ -92,6 +93,12 @@ function s3NotConfiguredResponse(): NextResponse {
|
|||
);
|
||||
}
|
||||
|
||||
function normalizeNativeSpeedForSettings(settings: AudiobookGenerationSettings): AudiobookGenerationSettings {
|
||||
return supportsNativeModelSpeed(settings.ttsProvider, settings.ttsModel)
|
||||
? settings
|
||||
: { ...settings, nativeSpeed: 1 };
|
||||
}
|
||||
|
||||
function chapterFileMimeType(format: TTSAudiobookFormat): string {
|
||||
return format === 'mp3' ? 'audio/mpeg' : 'audio/mp4';
|
||||
}
|
||||
|
|
@ -300,18 +307,19 @@ export async function POST(request: NextRequest) {
|
|||
existingSettings = null;
|
||||
}
|
||||
|
||||
const incomingSettings = data.settings;
|
||||
const incomingSettings = data.settings ? normalizeNativeSpeedForSettings(data.settings) : undefined;
|
||||
if (existingSettings && hasChapters && incomingSettings) {
|
||||
const normalizedExistingSettings = normalizeNativeSpeedForSettings(existingSettings);
|
||||
const mismatch =
|
||||
existingSettings.ttsProvider !== incomingSettings.ttsProvider ||
|
||||
existingSettings.ttsModel !== incomingSettings.ttsModel ||
|
||||
existingSettings.voice !== incomingSettings.voice ||
|
||||
existingSettings.nativeSpeed !== incomingSettings.nativeSpeed ||
|
||||
existingSettings.postSpeed !== incomingSettings.postSpeed ||
|
||||
existingSettings.format !== incomingSettings.format ||
|
||||
(existingSettings.ttsInstructions || '') !== (incomingSettings.ttsInstructions || '');
|
||||
normalizedExistingSettings.ttsProvider !== incomingSettings.ttsProvider ||
|
||||
normalizedExistingSettings.ttsModel !== incomingSettings.ttsModel ||
|
||||
normalizedExistingSettings.voice !== incomingSettings.voice ||
|
||||
normalizedExistingSettings.nativeSpeed !== incomingSettings.nativeSpeed ||
|
||||
normalizedExistingSettings.postSpeed !== incomingSettings.postSpeed ||
|
||||
normalizedExistingSettings.format !== incomingSettings.format ||
|
||||
(normalizedExistingSettings.ttsInstructions || '') !== (incomingSettings.ttsInstructions || '');
|
||||
if (mismatch) {
|
||||
return NextResponse.json({ error: 'Audiobook settings mismatch', settings: existingSettings }, { status: 409 });
|
||||
return NextResponse.json({ error: 'Audiobook settings mismatch', settings: normalizedExistingSettings }, { status: 409 });
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -54,9 +54,22 @@ function getUpstreamStatus(error: unknown): number | undefined {
|
|||
const rec = error as Record<string, unknown>;
|
||||
if (typeof rec.status === 'number') return rec.status;
|
||||
if (typeof rec.statusCode === 'number') return rec.statusCode;
|
||||
const response = rec.response as { status?: unknown } | undefined;
|
||||
if (response && typeof response.status === 'number') return response.status;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function getUpstreamRetryAfterSeconds(error: unknown): number | undefined {
|
||||
if (typeof error !== 'object' || error === null) return undefined;
|
||||
const rec = error as Record<string, unknown>;
|
||||
const response = rec.response as { headers?: { get?: (name: string) => string | null } } | undefined;
|
||||
const retryAfterHeader = response?.headers?.get?.('retry-after');
|
||||
if (!retryAfterHeader) return undefined;
|
||||
const parsed = Number(retryAfterHeader);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return undefined;
|
||||
return Math.ceil(parsed);
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
let providerForError: string | null = null;
|
||||
let didCreateDeviceIdCookie = false;
|
||||
|
|
@ -221,14 +234,18 @@ export async function POST(req: NextRequest) {
|
|||
|
||||
const upstreamStatus = getUpstreamStatus(error);
|
||||
if (upstreamStatus === 429) {
|
||||
const retryAfterSeconds = getUpstreamRetryAfterSeconds(error);
|
||||
const problem: ProblemDetails = {
|
||||
type: PROBLEM_TYPES.upstreamRateLimited,
|
||||
title: 'Upstream rate limited',
|
||||
status: 429,
|
||||
detail: 'The TTS provider is rate limiting requests. Please try again shortly.',
|
||||
detail: retryAfterSeconds
|
||||
? `The TTS provider is rate limiting requests. Please retry in about ${retryAfterSeconds}s.`
|
||||
: 'The TTS provider is rate limiting requests. Please try again shortly.',
|
||||
code: 'UPSTREAM_RATE_LIMIT',
|
||||
provider: providerForError ?? undefined,
|
||||
upstreamStatus,
|
||||
retryAfterSeconds,
|
||||
instance: req.nextUrl.pathname,
|
||||
};
|
||||
|
||||
|
|
@ -236,11 +253,16 @@ export async function POST(req: NextRequest) {
|
|||
status: 429,
|
||||
headers: {
|
||||
'Content-Type': 'application/problem+json',
|
||||
...(retryAfterSeconds ? { 'Retry-After': String(retryAfterSeconds) } : {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
console.warn('Error generating TTS:', error);
|
||||
const statusHint = getUpstreamStatus(error);
|
||||
const errorMessage = error instanceof Error ? error.message : String(error);
|
||||
console.warn(
|
||||
`Error generating TTS${typeof statusHint === 'number' ? ` (upstream ${statusHint})` : ''}: ${errorMessage}`,
|
||||
);
|
||||
const errorBody: TTSError = {
|
||||
code: 'TTS_GENERATION_FAILED',
|
||||
message: 'Failed to generate audio',
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ import { LoadingSpinner } from '@/components/Spinner';
|
|||
import { useConfig } from '@/contexts/ConfigContext';
|
||||
import { useTTS } from '@/contexts/TTSContext';
|
||||
import { VoicesControlBase } from '@/components/player/VoicesControlBase';
|
||||
import { supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog';
|
||||
import { supportsNativeModelSpeed, supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog';
|
||||
import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts';
|
||||
import {
|
||||
getAudiobookStatus,
|
||||
|
|
@ -73,6 +73,8 @@ export function AudiobookExportModal({
|
|||
const formatSpeed = useCallback((speed: number) => {
|
||||
return Number.isInteger(speed) ? speed.toString() : speed.toFixed(1);
|
||||
}, []);
|
||||
const nativeSpeedSupported = useMemo(() => supportsNativeModelSpeed(ttsProvider, ttsModel), [ttsProvider, ttsModel]);
|
||||
const effectiveNativeSpeed = nativeSpeedSupported ? nativeSpeed : 1;
|
||||
|
||||
const hasExistingAudiobook = Boolean(bookId) || chapters.length > 0;
|
||||
const isLegacyAudiobookMissingSettings = hasExistingAudiobook && savedSettings === null;
|
||||
|
|
@ -113,12 +115,12 @@ export function AudiobookExportModal({
|
|||
ttsProvider,
|
||||
ttsModel,
|
||||
voice: nextVoice,
|
||||
nativeSpeed,
|
||||
nativeSpeed: effectiveNativeSpeed,
|
||||
postSpeed,
|
||||
format,
|
||||
ttsInstructions: supportsTtsInstructions(ttsModel) ? ttsInstructions : undefined,
|
||||
};
|
||||
}, [savedSettings, audiobookVoice, configVoice, availableVoices, ttsProvider, ttsModel, ttsInstructions, nativeSpeed, postSpeed, format]);
|
||||
}, [savedSettings, audiobookVoice, configVoice, availableVoices, ttsProvider, ttsModel, ttsInstructions, effectiveNativeSpeed, postSpeed, format]);
|
||||
|
||||
const fetchExistingChapters = useCallback(async (soft: boolean = false) => {
|
||||
if (soft) {
|
||||
|
|
@ -529,11 +531,15 @@ export function AudiobookExportModal({
|
|||
<div className="text-sm font-medium text-foreground">{savedSettings.format.toUpperCase()}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<div className="rounded-lg bg-base p-3">
|
||||
<div className="text-[11px] uppercase tracking-wider text-muted mb-1">Native speed</div>
|
||||
<div className="text-sm font-medium text-foreground">{formatSpeed(savedSettings.nativeSpeed)}x</div>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-3">
|
||||
<div className="rounded-lg bg-base p-3">
|
||||
<div className="text-[11px] uppercase tracking-wider text-muted mb-1">Native speed</div>
|
||||
<div className="text-sm font-medium text-foreground">
|
||||
{supportsNativeModelSpeed(savedSettings.ttsProvider, savedSettings.ttsModel)
|
||||
? `${formatSpeed(savedSettings.nativeSpeed)}x`
|
||||
: 'Not supported'}
|
||||
</div>
|
||||
</div>
|
||||
<div className="rounded-lg bg-base p-3">
|
||||
<div className="text-[11px] uppercase tracking-wider text-muted mb-1">Post speed</div>
|
||||
<div className="text-sm font-medium text-foreground">{formatSpeed(savedSettings.postSpeed)}x</div>
|
||||
|
|
@ -616,29 +622,39 @@ export function AudiobookExportModal({
|
|||
</div>
|
||||
</div>
|
||||
|
||||
{/* Speed controls */}
|
||||
<div className="rounded-lg bg-base p-3 space-y-3">
|
||||
<div className="space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<label className="text-[11px] uppercase tracking-wider font-medium text-muted">Native model speed</label>
|
||||
<span className="text-xs font-medium text-accent tabular-nums">{formatSpeed(nativeSpeed)}x</span>
|
||||
</div>
|
||||
<input
|
||||
type="range"
|
||||
min="0.5"
|
||||
max="3"
|
||||
step="0.1"
|
||||
value={nativeSpeed}
|
||||
onChange={(e) => setNativeSpeed(parseFloat(e.target.value))}
|
||||
className="w-full h-1.5 bg-offbase rounded-full appearance-none cursor-pointer [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-full [&::-webkit-slider-runnable-track]:h-1.5 [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-webkit-slider-thumb]:-mt-[5px] [&::-webkit-slider-thumb]:shadow-sm [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-full [&::-moz-range-track]:h-1.5 [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent [&::-moz-range-thumb]:border-0"
|
||||
/>
|
||||
<div className="flex justify-between text-[10px] text-muted">
|
||||
<span>0.5x</span>
|
||||
<span>3x</span>
|
||||
</div>
|
||||
</div>
|
||||
{/* Speed controls */}
|
||||
<div className="rounded-lg bg-base p-3 space-y-3">
|
||||
{!nativeSpeedSupported && (
|
||||
<div className="rounded-md border border-offbase bg-background px-2 py-1.5 text-[11px] text-muted">
|
||||
Native model speed is not available for this model.
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="border-t border-offbase" />
|
||||
{nativeSpeedSupported && (
|
||||
<>
|
||||
<div className="space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<label className="text-[11px] uppercase tracking-wider font-medium text-muted">Native model speed</label>
|
||||
<span className="text-xs font-medium text-accent tabular-nums">{formatSpeed(nativeSpeed)}x</span>
|
||||
</div>
|
||||
<input
|
||||
type="range"
|
||||
min="0.5"
|
||||
max="3"
|
||||
step="0.1"
|
||||
value={nativeSpeed}
|
||||
onChange={(e) => setNativeSpeed(parseFloat(e.target.value))}
|
||||
className="w-full h-1.5 bg-offbase rounded-full appearance-none cursor-pointer [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-full [&::-webkit-slider-runnable-track]:h-1.5 [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-webkit-slider-thumb]:-mt-[5px] [&::-webkit-slider-thumb]:shadow-sm [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-full [&::-moz-range-track]:h-1.5 [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent [&::-moz-range-thumb]:border-0"
|
||||
/>
|
||||
<div className="flex justify-between text-[10px] text-muted">
|
||||
<span>0.5x</span>
|
||||
<span>3x</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="border-t border-offbase" />
|
||||
</>
|
||||
)}
|
||||
|
||||
<div className="space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
|
|
|
|||
|
|
@ -491,7 +491,7 @@ export function SettingsModal({ className = '' }: { className?: string }) {
|
|||
</nav>
|
||||
|
||||
{/* Content */}
|
||||
<div className="flex-1 p-4 overflow-y-auto">
|
||||
<div className="flex-1 min-w-0 p-4 overflow-y-auto">
|
||||
{/* API Section */}
|
||||
{activeSection === 'api' && (
|
||||
<div className="space-y-4">
|
||||
|
|
@ -507,6 +507,9 @@ export function SettingsModal({ className = '' }: { className?: string }) {
|
|||
} else if (provider.id === 'custom-openai') {
|
||||
setModelValue('kokoro');
|
||||
setLocalBaseUrl('');
|
||||
} else if (provider.id === 'replicate') {
|
||||
setModelValue('google/gemini-3.1-flash-tts');
|
||||
setLocalBaseUrl('');
|
||||
} else if (provider.id === 'deepinfra') {
|
||||
setModelValue('hexgrad/Kokoro-82M');
|
||||
setLocalBaseUrl('https://api.deepinfra.com/v1/openai');
|
||||
|
|
@ -528,7 +531,10 @@ export function SettingsModal({ className = '' }: { className?: string }) {
|
|||
leaveFrom="opacity-100"
|
||||
leaveTo="opacity-0"
|
||||
>
|
||||
<ListboxOptions className="absolute mt-1 w-full overflow-auto rounded-md bg-background py-1 shadow-lg ring-1 ring-offbase focus:outline-none z-50">
|
||||
<ListboxOptions
|
||||
anchor="bottom start"
|
||||
className="z-50 w-[var(--button-width)] max-h-60 overflow-y-auto overscroll-contain rounded-md bg-background py-1 shadow-lg ring-1 ring-offbase focus:outline-none [--anchor-gap:0.25rem]"
|
||||
>
|
||||
{ttsProviders.map((provider) => (
|
||||
<ListboxOption
|
||||
key={provider.id}
|
||||
|
|
@ -556,7 +562,7 @@ export function SettingsModal({ className = '' }: { className?: string }) {
|
|||
</Listbox>
|
||||
</div>
|
||||
|
||||
{(localTTSProvider === 'custom-openai' || !localBaseUrl || localBaseUrl === '') && (
|
||||
{localTTSProvider !== 'replicate' && (localTTSProvider === 'custom-openai' || !localBaseUrl || localBaseUrl === '') && (
|
||||
<div className="space-y-1.5">
|
||||
<label className="block text-sm font-medium text-foreground">
|
||||
API Base URL
|
||||
|
|
@ -614,7 +620,10 @@ export function SettingsModal({ className = '' }: { className?: string }) {
|
|||
leaveFrom="opacity-100"
|
||||
leaveTo="opacity-0"
|
||||
>
|
||||
<ListboxOptions className="absolute mt-1 w-full overflow-auto rounded-md bg-background py-1 shadow-lg ring-1 ring-offbase focus:outline-none z-50">
|
||||
<ListboxOptions
|
||||
anchor="bottom start"
|
||||
className="z-50 w-[var(--button-width)] max-h-60 overflow-y-auto overscroll-contain rounded-md bg-background py-1 shadow-lg ring-1 ring-offbase focus:outline-none [--anchor-gap:0.25rem]"
|
||||
>
|
||||
{ttsModels.map((model) => (
|
||||
<ListboxOption
|
||||
key={model.id}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import { Input, Popover, PopoverButton, PopoverPanel } from '@headlessui/react';
|
|||
import { ChevronUpDownIcon, SpeedometerIcon } from '@/components/icons/Icons';
|
||||
import { useConfig } from '@/contexts/ConfigContext';
|
||||
import { useCallback, useEffect, useMemo, useState } from 'react';
|
||||
import { supportsNativeModelSpeed } from '@/lib/shared/tts-provider-catalog';
|
||||
|
||||
export const SpeedControl = ({
|
||||
setSpeedAndRestart,
|
||||
|
|
@ -12,7 +13,8 @@ export const SpeedControl = ({
|
|||
setSpeedAndRestart: (speed: number) => void;
|
||||
setAudioPlayerSpeedAndRestart: (speed: number) => void;
|
||||
}) => {
|
||||
const { voiceSpeed, audioPlayerSpeed } = useConfig();
|
||||
const { voiceSpeed, audioPlayerSpeed, ttsProvider, ttsModel } = useConfig();
|
||||
const nativeSpeedSupported = supportsNativeModelSpeed(ttsProvider, ttsModel);
|
||||
|
||||
const [localVoiceSpeed, setLocalVoiceSpeed] = useState(voiceSpeed);
|
||||
const [localAudioSpeed, setLocalAudioSpeed] = useState(audioPlayerSpeed);
|
||||
|
|
@ -53,28 +55,28 @@ export const SpeedControl = ({
|
|||
const triggerLabel = useMemo(
|
||||
() => {
|
||||
const parts: string[] = [];
|
||||
if (localVoiceSpeed !== 1.0) parts.push(`${formatSpeed(localVoiceSpeed, 1)}x`);
|
||||
if (nativeSpeedSupported && localVoiceSpeed !== 1.0) parts.push(`${formatSpeed(localVoiceSpeed, 1)}x`);
|
||||
if (localAudioSpeed !== 1.0) parts.push(`${formatSpeed(localAudioSpeed, 1)}x`);
|
||||
return parts.length > 0 ? parts.join(' • ') : '1x';
|
||||
},
|
||||
[formatSpeed, localVoiceSpeed, localAudioSpeed]
|
||||
[formatSpeed, localVoiceSpeed, localAudioSpeed, nativeSpeedSupported]
|
||||
);
|
||||
|
||||
const compactTriggerLabel = useMemo(() => {
|
||||
const voiceIsDefault = localVoiceSpeed === 1.0;
|
||||
const voiceIsDefault = !nativeSpeedSupported || localVoiceSpeed === 1.0;
|
||||
const audioIsDefault = localAudioSpeed === 1.0;
|
||||
|
||||
let combined = 1.0;
|
||||
if (!voiceIsDefault && !audioIsDefault) {
|
||||
if (nativeSpeedSupported && !voiceIsDefault && !audioIsDefault) {
|
||||
combined = (localVoiceSpeed + localAudioSpeed) / 2;
|
||||
} else if (!voiceIsDefault) {
|
||||
} else if (nativeSpeedSupported && !voiceIsDefault) {
|
||||
combined = localVoiceSpeed;
|
||||
} else if (!audioIsDefault) {
|
||||
combined = localAudioSpeed;
|
||||
}
|
||||
|
||||
return `${formatSpeed(combined, 2)}x`;
|
||||
}, [formatSpeed, localVoiceSpeed, localAudioSpeed]);
|
||||
}, [formatSpeed, localVoiceSpeed, localAudioSpeed, nativeSpeedSupported]);
|
||||
|
||||
const min = 0.5;
|
||||
const max = 3;
|
||||
|
|
@ -90,28 +92,36 @@ export const SpeedControl = ({
|
|||
</PopoverButton>
|
||||
<PopoverPanel anchor="top" className="absolute z-50 bg-base p-3 rounded-md shadow-lg border border-offbase">
|
||||
<div className="flex flex-col space-y-4">
|
||||
<div className="flex flex-col space-y-2">
|
||||
<div className="text-xs font-medium text-foreground">Native model speed</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-xs">{min.toFixed(1)}x</span>
|
||||
<span className="text-xs font-bold">
|
||||
{Number.isInteger(localVoiceSpeed) ? localVoiceSpeed.toString() : localVoiceSpeed.toFixed(1)}x
|
||||
</span>
|
||||
<span className="text-xs">{max.toFixed(1)}x</span>
|
||||
{!nativeSpeedSupported && (
|
||||
<div className="rounded-md border border-offbase bg-background px-2 py-1.5 text-[11px] text-muted">
|
||||
Native model speed is not available for this model.
|
||||
</div>
|
||||
<Input
|
||||
type="range"
|
||||
min={min}
|
||||
max={max}
|
||||
step={step}
|
||||
value={localVoiceSpeed}
|
||||
onChange={handleVoiceSpeedChange}
|
||||
onMouseUp={handleVoiceSpeedChangeComplete}
|
||||
onKeyUp={handleVoiceSpeedChangeComplete}
|
||||
onTouchEnd={handleVoiceSpeedChangeComplete}
|
||||
className="w-full bg-offbase rounded-lg appearance-none cursor-pointer accent-accent [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-lg [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-lg [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{nativeSpeedSupported && (
|
||||
<div className="flex flex-col space-y-2">
|
||||
<div className="text-xs font-medium text-foreground">Native model speed</div>
|
||||
<div className="flex justify-between">
|
||||
<span className="text-xs">{min.toFixed(1)}x</span>
|
||||
<span className="text-xs font-bold">
|
||||
{Number.isInteger(localVoiceSpeed) ? localVoiceSpeed.toString() : localVoiceSpeed.toFixed(1)}x
|
||||
</span>
|
||||
<span className="text-xs">{max.toFixed(1)}x</span>
|
||||
</div>
|
||||
<Input
|
||||
type="range"
|
||||
min={min}
|
||||
max={max}
|
||||
step={step}
|
||||
value={localVoiceSpeed}
|
||||
onChange={handleVoiceSpeedChange}
|
||||
onMouseUp={handleVoiceSpeedChangeComplete}
|
||||
onKeyUp={handleVoiceSpeedChangeComplete}
|
||||
onTouchEnd={handleVoiceSpeedChangeComplete}
|
||||
className="w-full bg-offbase rounded-lg appearance-none cursor-pointer accent-accent [&::-webkit-slider-runnable-track]:bg-offbase [&::-webkit-slider-runnable-track]:rounded-lg [&::-webkit-slider-thumb]:appearance-none [&::-webkit-slider-thumb]:h-4 [&::-webkit-slider-thumb]:w-4 [&::-webkit-slider-thumb]:rounded-full [&::-webkit-slider-thumb]:bg-accent [&::-moz-range-track]:bg-offbase [&::-moz-range-track]:rounded-lg [&::-moz-range-thumb]:appearance-none [&::-moz-range-thumb]:h-4 [&::-moz-range-thumb]:w-4 [&::-moz-range-thumb]:rounded-full [&::-moz-range-thumb]:bg-accent"
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex flex-col space-y-2">
|
||||
<div className="text-xs font-medium text-foreground">Audio player speed</div>
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ import { getDocumentProgress, scheduleDocumentProgressSync } from '@/lib/client/
|
|||
import { withRetry, generateTTS, alignAudio } from '@/lib/client/api/audiobooks';
|
||||
import { preprocessSentenceForAudio, splitTextToTtsBlocks, splitTextToTtsBlocksEPUB } from '@/lib/shared/nlp';
|
||||
import { isKokoroModel } from '@/lib/shared/kokoro';
|
||||
import { supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog';
|
||||
import { supportsNativeModelSpeed, supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog';
|
||||
import { useAuthRateLimit } from '@/contexts/AuthRateLimitContext';
|
||||
import type {
|
||||
TTSLocation,
|
||||
|
|
@ -386,6 +386,10 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const [voice, setVoice] = useState(configVoice);
|
||||
const [ttsModel, setTTSModel] = useState(configTTSModel);
|
||||
const [ttsInstructions, setTTSInstructions] = useState(configTTSInstructions);
|
||||
const effectiveNativeSpeed = useMemo(
|
||||
() => (supportsNativeModelSpeed(configTTSProvider, ttsModel) ? speed : 1),
|
||||
[configTTSProvider, ttsModel, speed],
|
||||
);
|
||||
|
||||
// Track pending preload requests
|
||||
const preloadRequests = useRef<Map<string, Promise<string>>>(new Map());
|
||||
|
|
@ -1015,7 +1019,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const alignmentKey = buildCacheKey(
|
||||
sentence,
|
||||
voice,
|
||||
speed,
|
||||
effectiveNativeSpeed,
|
||||
configTTSProvider,
|
||||
ttsModel,
|
||||
);
|
||||
|
|
@ -1053,7 +1057,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const audioCacheKey = buildCacheKey(
|
||||
sentence,
|
||||
voice,
|
||||
speed,
|
||||
effectiveNativeSpeed,
|
||||
configTTSProvider,
|
||||
ttsModel,
|
||||
);
|
||||
|
|
@ -1097,7 +1101,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const reqBody: TTSRequestPayload = {
|
||||
text: sentence,
|
||||
voice,
|
||||
speed,
|
||||
speed: effectiveNativeSpeed,
|
||||
format: 'mp3',
|
||||
model: ttsModel,
|
||||
instructions: supportsTtsInstructions(ttsModel) ? ttsInstructions : undefined,
|
||||
|
|
@ -1190,7 +1194,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
}
|
||||
}, [
|
||||
voice,
|
||||
speed,
|
||||
effectiveNativeSpeed,
|
||||
ttsModel,
|
||||
ttsInstructions,
|
||||
audioCache,
|
||||
|
|
@ -1532,7 +1536,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const alignmentKey = buildCacheKey(
|
||||
sentence,
|
||||
voice,
|
||||
speed,
|
||||
effectiveNativeSpeed,
|
||||
configTTSProvider,
|
||||
ttsModel,
|
||||
);
|
||||
|
|
@ -1549,7 +1553,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
if (howl) {
|
||||
howl.play();
|
||||
}
|
||||
}, [sentences, currentIndex, playSentenceWithHowl, voice, speed, configTTSProvider, ttsModel]);
|
||||
}, [sentences, currentIndex, playSentenceWithHowl, voice, effectiveNativeSpeed, configTTSProvider, ttsModel]);
|
||||
|
||||
// Keep the current playback rate applied to the active Howl. Some browsers (notably
|
||||
// iOS Safari with HTML5 audio) can reset playbackRate after initial load/play.
|
||||
|
|
@ -1620,7 +1624,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const nextKey = buildCacheKey(
|
||||
nextSentence,
|
||||
voice,
|
||||
speed,
|
||||
effectiveNativeSpeed,
|
||||
configTTSProvider,
|
||||
ttsModel,
|
||||
);
|
||||
|
|
@ -1673,7 +1677,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
} catch (error) {
|
||||
console.error('Error initiating preload:', error);
|
||||
}
|
||||
}, [isAtLimit, currentIndex, sentences, audioCache, processSentence, voice, speed, configTTSProvider, ttsModel]);
|
||||
}, [isAtLimit, currentIndex, sentences, audioCache, processSentence, voice, effectiveNativeSpeed, configTTSProvider, ttsModel]);
|
||||
|
||||
/**
|
||||
* Main Playback Driver
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
import OpenAI from 'openai';
|
||||
import Replicate from 'replicate';
|
||||
import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs';
|
||||
import { isKokoroModel } from '@/lib/shared/kokoro';
|
||||
import { supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog';
|
||||
import { supportsNativeModelSpeed, supportsTtsInstructions } from '@/lib/shared/tts-provider-catalog';
|
||||
import { LRUCache } from 'lru-cache';
|
||||
import { createHash } from 'crypto';
|
||||
import { access, readFile } from 'fs/promises';
|
||||
|
|
@ -45,6 +46,9 @@ type InflightEntry = {
|
|||
consumers: number;
|
||||
};
|
||||
|
||||
let replicateQueue: Promise<void> = Promise.resolve();
|
||||
let replicateBlockedUntilMs = 0;
|
||||
|
||||
const TTS_CACHE_MAX_SIZE_BYTES = Number(process.env.TTS_CACHE_MAX_SIZE_BYTES || 256 * 1024 * 1024); // 256MB
|
||||
const TTS_CACHE_TTL_MS = Number(process.env.TTS_CACHE_TTL_MS || 1000 * 60 * 30); // 30 minutes
|
||||
|
||||
|
|
@ -63,17 +67,79 @@ function sleep(ms: number) {
|
|||
return new Promise((res) => setTimeout(res, ms));
|
||||
}
|
||||
|
||||
function sleepWithSignal(ms: number, signal: AbortSignal): Promise<void> {
|
||||
if (ms <= 0) return Promise.resolve();
|
||||
if (signal.aborted) {
|
||||
throw new DOMException('The operation was aborted.', 'AbortError');
|
||||
}
|
||||
|
||||
return new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
resolve();
|
||||
}, ms);
|
||||
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
signal.removeEventListener('abort', onAbort);
|
||||
reject(new DOMException('The operation was aborted.', 'AbortError'));
|
||||
};
|
||||
|
||||
signal.addEventListener('abort', onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
function getUpstreamStatus(error: unknown): number | undefined {
|
||||
if (typeof error !== 'object' || error === null) return undefined;
|
||||
const rec = error as Record<string, unknown>;
|
||||
if (typeof rec.status === 'number') return rec.status;
|
||||
if (typeof rec.statusCode === 'number') return rec.statusCode;
|
||||
const response = rec.response as { status?: unknown } | undefined;
|
||||
if (response && typeof response.status === 'number') return response.status;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function getUpstreamRetryAfterSeconds(error: unknown): number | undefined {
|
||||
if (typeof error !== 'object' || error === null) return undefined;
|
||||
const rec = error as Record<string, unknown>;
|
||||
const response = rec.response as { headers?: { get?: (name: string) => string | null } } | undefined;
|
||||
const retryAfterHeader = response?.headers?.get?.('retry-after');
|
||||
if (!retryAfterHeader) return undefined;
|
||||
const parsed = Number(retryAfterHeader);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return undefined;
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function applyReplicateCooldown(cooldownMs: number) {
|
||||
if (!Number.isFinite(cooldownMs) || cooldownMs <= 0) return;
|
||||
const next = Date.now() + cooldownMs;
|
||||
replicateBlockedUntilMs = Math.max(replicateBlockedUntilMs, next);
|
||||
}
|
||||
|
||||
async function runWithReplicateGate<T>(signal: AbortSignal, operation: () => Promise<T>): Promise<T> {
|
||||
let release: (() => void) | undefined;
|
||||
const prev = replicateQueue;
|
||||
replicateQueue = new Promise<void>((resolve) => {
|
||||
release = resolve;
|
||||
});
|
||||
|
||||
await prev;
|
||||
try {
|
||||
const waitMs = Math.max(0, replicateBlockedUntilMs - Date.now());
|
||||
if (waitMs > 0) {
|
||||
await sleepWithSignal(waitMs, signal);
|
||||
}
|
||||
return await operation();
|
||||
} finally {
|
||||
release?.();
|
||||
}
|
||||
}
|
||||
|
||||
function resolveTTSRequest(input: ServerTTSRequest): ResolvedServerTTSRequest {
|
||||
const provider = input.provider || 'openai';
|
||||
const rawModel = provider === 'deepinfra' && !input.model ? 'hexgrad/Kokoro-82M' : input.model;
|
||||
const rawModel = provider === 'deepinfra' && !input.model ? 'hexgrad/Kokoro-82M'
|
||||
: provider === 'replicate' && !input.model ? 'google/gemini-3.1-flash-tts'
|
||||
: input.model;
|
||||
const model = (rawModel ?? 'gpt-4o-mini-tts') as SpeechCreateParams['model'];
|
||||
|
||||
const normalizedVoice = (
|
||||
|
|
@ -83,7 +149,8 @@ function resolveTTSRequest(input: ServerTTSRequest): ResolvedServerTTSRequest {
|
|||
) as string;
|
||||
|
||||
const format = input.format || 'mp3';
|
||||
const speed = Number.isFinite(Number(input.speed)) ? Number(input.speed) : 1;
|
||||
const requestedSpeed = Number.isFinite(Number(input.speed)) ? Number(input.speed) : 1;
|
||||
const speed = supportsNativeModelSpeed(provider, model as string) ? requestedSpeed : 1;
|
||||
const instructions = supportsTtsInstructions(model as string) && input.instructions
|
||||
? input.instructions
|
||||
: undefined;
|
||||
|
|
@ -196,10 +263,112 @@ async function fetchTTSBufferWithRetry(
|
|||
}
|
||||
}
|
||||
|
||||
function buildReplicateInput(request: ResolvedServerTTSRequest): Record<string, unknown> {
|
||||
const model = request.model as string;
|
||||
|
||||
if (model === 'google/gemini-3.1-flash-tts') {
|
||||
const input: Record<string, unknown> = {
|
||||
text: request.text,
|
||||
voice: request.voice,
|
||||
};
|
||||
if (request.instructions) {
|
||||
input.prompt = request.instructions;
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
if (model === 'minimax/speech-2.8-turbo') {
|
||||
const input: Record<string, unknown> = {
|
||||
text: request.text,
|
||||
voice_id: request.voice,
|
||||
audio_format: request.format === 'mp3' ? 'mp3' : 'wav',
|
||||
};
|
||||
if (request.speed !== 1) {
|
||||
input.speed = Math.max(0.5, Math.min(2.0, request.speed));
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
if (model === 'qwen/qwen3-tts') {
|
||||
const input: Record<string, unknown> = {
|
||||
text: request.text,
|
||||
mode: 'custom_voice',
|
||||
speaker: request.voice,
|
||||
};
|
||||
if (request.instructions) {
|
||||
input.style_instruction = request.instructions;
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
if (model === 'inworld/tts-1.5-mini') {
|
||||
const input: Record<string, unknown> = {
|
||||
text: request.text,
|
||||
voice_id: request.voice,
|
||||
audio_format: request.format === 'mp3' ? 'mp3' : 'wav',
|
||||
};
|
||||
if (request.speed !== 1) {
|
||||
input.speaking_rate = request.speed;
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
return { text: request.text };
|
||||
}
|
||||
|
||||
async function runReplicateRequest(request: ResolvedServerTTSRequest, signal: AbortSignal): Promise<Buffer> {
|
||||
const replicate = new Replicate({ auth: request.apiKey });
|
||||
const input = buildReplicateInput(request);
|
||||
const modelId = request.model as `${string}/${string}`;
|
||||
|
||||
return runWithReplicateGate(signal, async () => {
|
||||
const maxRetries = Number(process.env.TTS_MAX_RETRIES ?? 2);
|
||||
let attempt = 0;
|
||||
|
||||
for (; ;) {
|
||||
try {
|
||||
const output = await replicate.run(modelId, { input, signal }) as unknown;
|
||||
|
||||
// Output is a URI string pointing to the generated audio file
|
||||
const audioUrl = typeof output === 'string' ? output : String(output);
|
||||
const audioResponse = await fetch(audioUrl, { signal });
|
||||
if (!audioResponse.ok) {
|
||||
throw new Error(`Failed to fetch Replicate audio: ${audioResponse.status}`);
|
||||
}
|
||||
const buffer = await audioResponse.arrayBuffer();
|
||||
return Buffer.from(buffer);
|
||||
} catch (error) {
|
||||
if (signal.aborted || (error instanceof Error && error.name === 'AbortError')) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
const status = getUpstreamStatus(error) ?? 0;
|
||||
const retryable = status === 429 || status >= 500;
|
||||
const retryAfterSeconds = status === 429 ? getUpstreamRetryAfterSeconds(error) : undefined;
|
||||
const delay = retryAfterSeconds ? Math.max(retryAfterSeconds * 1000, 1000) : 10_000;
|
||||
if (status === 429) {
|
||||
applyReplicateCooldown(delay);
|
||||
}
|
||||
|
||||
if (!retryable || attempt >= maxRetries) {
|
||||
throw error;
|
||||
}
|
||||
|
||||
await sleepWithSignal(delay, signal);
|
||||
attempt += 1;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async function runProviderRequest(request: ResolvedServerTTSRequest, signal: AbortSignal): Promise<Buffer> {
|
||||
const mockBuffer = await getTestMockTtsBuffer(request.testNamespace);
|
||||
if (mockBuffer) return mockBuffer;
|
||||
|
||||
if (request.provider === 'replicate') {
|
||||
return runReplicateRequest(request, signal);
|
||||
}
|
||||
|
||||
const openai = new OpenAI({
|
||||
apiKey: request.apiKey,
|
||||
baseURL: request.baseUrl,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
import { isKokoroModel } from '@/lib/shared/kokoro';
|
||||
|
||||
export type TtsProviderId = 'custom-openai' | 'deepinfra' | 'openai';
|
||||
export type TtsProviderId = 'custom-openai' | 'replicate' | 'deepinfra' | 'openai';
|
||||
export type TtsVoiceSource = 'static' | 'deepinfra-api' | 'custom-openai-api';
|
||||
|
||||
export interface TtsModelDefinition {
|
||||
|
|
@ -54,6 +54,13 @@ const DEEPINFRA_MODELS_LIMITED: TtsModelDefinition[] = [
|
|||
{ id: 'hexgrad/Kokoro-82M', name: 'hexgrad/Kokoro-82M' },
|
||||
];
|
||||
|
||||
const REPLICATE_MODELS: TtsModelDefinition[] = [
|
||||
{ id: 'google/gemini-3.1-flash-tts', name: 'Gemini 3.1 Flash TTS' },
|
||||
{ id: 'minimax/speech-2.8-turbo', name: 'MiniMax Speech 2.8 Turbo' },
|
||||
{ id: 'qwen/qwen3-tts', name: 'Qwen3 TTS' },
|
||||
{ id: 'inworld/tts-1.5-mini', name: 'Inworld TTS 1.5 Mini' },
|
||||
];
|
||||
|
||||
const DEFAULT_MODELS: TtsModelDefinition[] = [{ id: 'tts-1', name: 'TTS-1' }];
|
||||
|
||||
export const OPENAI_DEFAULT_VOICES = ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer'] as const;
|
||||
|
|
@ -71,6 +78,22 @@ export const KOKORO_DEFAULT_VOICES = [
|
|||
export const ORPHEUS_DEFAULT_VOICES = ['tara', 'leah', 'jess', 'leo', 'dan', 'mia', 'zac'] as const;
|
||||
export const SESAME_DEFAULT_VOICES = ['conversational_a', 'conversational_b', 'read_speech_a', 'read_speech_b', 'read_speech_c', 'read_speech_d', 'none'] as const;
|
||||
|
||||
// Replicate model voices
|
||||
export const GEMINI_FLASH_TTS_VOICES = [
|
||||
'Zephyr', 'Puck', 'Charon', 'Kore', 'Fenrir', 'Leda', 'Orus', 'Aoede',
|
||||
'Callirrhoe', 'Autonoe', 'Enceladus', 'Iapetus', 'Umbriel', 'Algenib',
|
||||
'Despina', 'Erinome', 'Laomedeia', 'Achernar', 'Algieba', 'Schedar',
|
||||
'Gacrux', 'Pulcherrima', 'Achird', 'Zubenelgenubi', 'Vindemiatrix',
|
||||
'Sadachbia', 'Sadaltager', 'Sulafat', 'Alnilam', 'Rasalgethi',
|
||||
] as const;
|
||||
export const MINIMAX_SPEECH_VOICES = [
|
||||
'Deep_Voice_Man', 'Imposing_Manner', 'Elegant_Man', 'Casual_Guy',
|
||||
'Friendly_Person', 'Decent_Boy', 'Lively_Girl', 'Exuberant_Girl',
|
||||
'Inspirational_girl', 'Young_Knight', 'Abbess', 'Wise_Woman',
|
||||
] as const;
|
||||
export const QWEN3_TTS_VOICES = ['Aiden', 'Dylan'] as const;
|
||||
export const INWORLD_TTS_VOICES = ['Ashley', 'Dennis', 'Alex', 'Darlene'] as const;
|
||||
|
||||
export const TTS_PROVIDER_DEFINITIONS: TtsProviderDefinition[] = [
|
||||
{
|
||||
id: 'custom-openai',
|
||||
|
|
@ -78,6 +101,12 @@ export const TTS_PROVIDER_DEFINITIONS: TtsProviderDefinition[] = [
|
|||
supportsCustomModel: true,
|
||||
models: () => CUSTOM_OPENAI_MODELS,
|
||||
},
|
||||
{
|
||||
id: 'replicate',
|
||||
name: 'Replicate',
|
||||
supportsCustomModel: false,
|
||||
models: () => REPLICATE_MODELS,
|
||||
},
|
||||
{
|
||||
id: 'deepinfra',
|
||||
name: 'Deepinfra',
|
||||
|
|
@ -97,8 +126,31 @@ export const TTS_PROVIDER_DEFINITIONS: TtsProviderDefinition[] = [
|
|||
},
|
||||
];
|
||||
|
||||
const MODELS_WITH_INSTRUCTIONS = new Set([
|
||||
'gpt-4o-mini-tts',
|
||||
'google/gemini-3.1-flash-tts',
|
||||
'qwen/qwen3-tts',
|
||||
]);
|
||||
|
||||
const REPLICATE_MODELS_WITHOUT_NATIVE_SPEED = new Set([
|
||||
'google/gemini-3.1-flash-tts',
|
||||
'qwen/qwen3-tts',
|
||||
]);
|
||||
|
||||
export function supportsTtsInstructions(model: string | null | undefined): boolean {
|
||||
return model === 'gpt-4o-mini-tts';
|
||||
return !!model && MODELS_WITH_INSTRUCTIONS.has(model);
|
||||
}
|
||||
|
||||
export function supportsNativeModelSpeed(provider: string | null | undefined, model: string | null | undefined): boolean {
|
||||
if (!model) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (provider === 'replicate' && REPLICATE_MODELS_WITHOUT_NATIVE_SPEED.has(model)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
export function getProviderDefinition(provider: string | null | undefined): TtsProviderDefinition | undefined {
|
||||
|
|
@ -128,6 +180,22 @@ export function getDefaultVoices(provider: string, model: string): string[] {
|
|||
return [...CUSTOM_OPENAI_DEFAULT_VOICES];
|
||||
}
|
||||
|
||||
if (provider === 'replicate') {
|
||||
if (model === 'google/gemini-3.1-flash-tts') {
|
||||
return [...GEMINI_FLASH_TTS_VOICES];
|
||||
}
|
||||
if (model === 'minimax/speech-2.8-turbo') {
|
||||
return [...MINIMAX_SPEECH_VOICES];
|
||||
}
|
||||
if (model === 'qwen/qwen3-tts') {
|
||||
return [...QWEN3_TTS_VOICES];
|
||||
}
|
||||
if (model === 'inworld/tts-1.5-mini') {
|
||||
return [...INWORLD_TTS_VOICES];
|
||||
}
|
||||
return [...GEMINI_FLASH_TTS_VOICES];
|
||||
}
|
||||
|
||||
if (provider === 'deepinfra') {
|
||||
if (model === 'hexgrad/Kokoro-82M') {
|
||||
return [...KOKORO_DEFAULT_VOICES];
|
||||
|
|
|
|||
Loading…
Reference in a new issue