feat(tts,ui,api): add Kokoro multi-voice selection and SDK support
- Introduce voice utils (model detection, voice parsing/weights, limits) - Enable Kokoro multi-voice strings across OpenAI/Deepinfra/custom providers - Normalize non-Kokoro voices to single token for SDK calls - Expose full Kokoro voice list for custom-openai Kokoro models - Update TTS API to return audio as ArrayBuffer and improve logging - Add multi-select UI for Kokoro voices with provider-based clamping - Preserve Kokoro voice strings in TTSContext and coalesce restarts - Merge multi-sentence quoted dialogue in NLP sentence splitter - Update tests for single and multi-voice selection flows
This commit is contained in:
parent
21870ed576
commit
1dcffc82d8
8 changed files with 368 additions and 54 deletions
|
|
@ -1,6 +1,7 @@
|
||||||
import { NextRequest, NextResponse } from 'next/server';
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
import OpenAI from 'openai';
|
import OpenAI from 'openai';
|
||||||
import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs';
|
import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs';
|
||||||
|
import { isKokoroModel, stripVoiceWeights } from '@/utils/voice';
|
||||||
|
|
||||||
type CustomVoice = string;
|
type CustomVoice = string;
|
||||||
type ExtendedSpeechParams = Omit<SpeechCreateParams, 'voice'> & {
|
type ExtendedSpeechParams = Omit<SpeechCreateParams, 'voice'> & {
|
||||||
|
|
@ -15,11 +16,7 @@ export async function POST(req: NextRequest) {
|
||||||
const openApiBaseUrl = req.headers.get('x-openai-base-url') || process.env.API_BASE;
|
const openApiBaseUrl = req.headers.get('x-openai-base-url') || process.env.API_BASE;
|
||||||
const provider = req.headers.get('x-tts-provider') || 'openai';
|
const provider = req.headers.get('x-tts-provider') || 'openai';
|
||||||
const { text, voice, speed, format, model, instructions } = await req.json();
|
const { text, voice, speed, format, model, instructions } = await req.json();
|
||||||
console.log('Received TTS request:', text, voice, speed, format, model);
|
console.log('Received TTS request:', { provider, model, voice, speed, format, hasInstructions: Boolean(instructions) });
|
||||||
|
|
||||||
if (!openApiKey) {
|
|
||||||
return NextResponse.json({ error: 'Missing OpenAI API key' }, { status: 401 });
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!text || !voice || !speed) {
|
if (!text || !voice || !speed) {
|
||||||
return NextResponse.json({ error: 'Missing required parameters' }, { status: 400 });
|
return NextResponse.json({ error: 'Missing required parameters' }, { status: 400 });
|
||||||
|
|
@ -27,18 +24,28 @@ export async function POST(req: NextRequest) {
|
||||||
|
|
||||||
// Apply Deepinfra defaults if provider is deepinfra
|
// Apply Deepinfra defaults if provider is deepinfra
|
||||||
const finalModel = provider === 'deepinfra' && !model ? 'hexgrad/Kokoro-82M' : model;
|
const finalModel = provider === 'deepinfra' && !model ? 'hexgrad/Kokoro-82M' : model;
|
||||||
const finalVoice = provider === 'deepinfra' && !voice ? 'af_bella' : voice;
|
const initialVoice = provider === 'deepinfra' && !voice ? 'af_bella' : voice;
|
||||||
|
|
||||||
// Initialize OpenAI client with abort signal
|
// For SDK providers (OpenAI/Deepinfra), preserve multi-voice for Kokoro models, otherwise normalize to first token
|
||||||
|
const isKokoro = isKokoroModel(finalModel);
|
||||||
|
let normalizedVoice = initialVoice;
|
||||||
|
if (!isKokoro && typeof normalizedVoice === 'string' && normalizedVoice.includes('+')) {
|
||||||
|
normalizedVoice = stripVoiceWeights(normalizedVoice.split('+')[0]);
|
||||||
|
console.log('Normalized multi-voice to single for non-Kokoro SDK provider:', normalizedVoice);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize OpenAI client with abort signal (OpenAI/deepinfra)
|
||||||
const openai = new OpenAI({
|
const openai = new OpenAI({
|
||||||
apiKey: openApiKey,
|
apiKey: openApiKey,
|
||||||
baseURL: openApiBaseUrl,
|
baseURL: openApiBaseUrl,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Unified path: all providers (openai, deepinfra, custom-openai) go through the SDK below.
|
||||||
|
|
||||||
// Request audio from OpenAI and pass along the abort signal
|
// Request audio from OpenAI and pass along the abort signal
|
||||||
const createParams: ExtendedSpeechParams = {
|
const createParams: ExtendedSpeechParams = {
|
||||||
model: finalModel || 'tts-1',
|
model: finalModel || 'tts-1',
|
||||||
voice: finalVoice as "alloy",
|
voice: normalizedVoice as SpeechCreateParams['voice'],
|
||||||
input: text,
|
input: text,
|
||||||
speed: speed,
|
speed: speed,
|
||||||
response_format: format === 'aac' ? 'aac' : 'mp3',
|
response_format: format === 'aac' ? 'aac' : 'mp3',
|
||||||
|
|
@ -51,13 +58,11 @@ export async function POST(req: NextRequest) {
|
||||||
|
|
||||||
const response = await openai.audio.speech.create(createParams as SpeechCreateParams, { signal: req.signal });
|
const response = await openai.audio.speech.create(createParams as SpeechCreateParams, { signal: req.signal });
|
||||||
|
|
||||||
// Get the audio data as array buffer
|
// Read the audio data as an ArrayBuffer and return it with appropriate headers
|
||||||
// This will also be aborted if the client cancels
|
// This will also be aborted if the client cancels
|
||||||
const stream = response.body;
|
const buffer = await response.arrayBuffer();
|
||||||
|
|
||||||
// Return audio data with appropriate headers
|
|
||||||
const contentType = format === 'aac' ? 'audio/aac' : 'audio/mpeg';
|
const contentType = format === 'aac' ? 'audio/aac' : 'audio/mpeg';
|
||||||
return new NextResponse(stream, {
|
return new NextResponse(buffer, {
|
||||||
headers: {
|
headers: {
|
||||||
'Content-Type': contentType
|
'Content-Type': contentType
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,4 +1,5 @@
|
||||||
import { NextRequest, NextResponse } from 'next/server';
|
import { NextRequest, NextResponse } from 'next/server';
|
||||||
|
import { isKokoroModel } from '@/utils/voice';
|
||||||
|
|
||||||
const OPENAI_VOICES = ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer'];
|
const OPENAI_VOICES = ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer'];
|
||||||
const GPT4O_MINI_VOICES = ['alloy', 'ash', 'coral', 'echo', 'fable', 'onyx', 'nova', 'sage', 'shimmer'];
|
const GPT4O_MINI_VOICES = ['alloy', 'ash', 'coral', 'echo', 'fable', 'onyx', 'nova', 'sage', 'shimmer'];
|
||||||
|
|
@ -29,6 +30,10 @@ function getDefaultVoices(provider: string, model: string): string[] {
|
||||||
|
|
||||||
// For Custom OpenAI-Like provider
|
// For Custom OpenAI-Like provider
|
||||||
if (provider === 'custom-openai') {
|
if (provider === 'custom-openai') {
|
||||||
|
// If using Kokoro-FastAPI (model string contains 'kokoro'), expose full Kokoro voices
|
||||||
|
if (isKokoroModel(model)) {
|
||||||
|
return KOKORO_VOICES;
|
||||||
|
}
|
||||||
return CUSTOM_OPENAI_VOICES;
|
return CUSTOM_OPENAI_VOICES;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -72,7 +77,7 @@ async function fetchDeepinfraVoices(apiKey: string): Promise<string[]> {
|
||||||
}
|
}
|
||||||
|
|
||||||
const data = await response.json();
|
const data = await response.json();
|
||||||
//console.log('Deepinfra voices response:', data);
|
|
||||||
// Extract voice names from the response, excluding preset voices
|
// Extract voice names from the response, excluding preset voices
|
||||||
if (data.voices && Array.isArray(data.voices)) {
|
if (data.voices && Array.isArray(data.voices)) {
|
||||||
return data.voices
|
return data.voices
|
||||||
|
|
|
||||||
|
|
@ -8,40 +8,135 @@ import {
|
||||||
} from '@headlessui/react';
|
} from '@headlessui/react';
|
||||||
import { ChevronUpDownIcon, AudioWaveIcon } from '@/components/icons/Icons';
|
import { ChevronUpDownIcon, AudioWaveIcon } from '@/components/icons/Icons';
|
||||||
import { useConfig } from '@/contexts/ConfigContext';
|
import { useConfig } from '@/contexts/ConfigContext';
|
||||||
|
import { useEffect, useMemo, useState, useCallback } from 'react';
|
||||||
|
import {
|
||||||
|
parseKokoroVoiceNames,
|
||||||
|
buildKokoroVoiceString,
|
||||||
|
getMaxVoicesForProvider,
|
||||||
|
isKokoroModel
|
||||||
|
} from '@/utils/voice';
|
||||||
|
|
||||||
export const VoicesControl = ({ availableVoices, setVoiceAndRestart }: {
|
export const VoicesControl = ({ availableVoices, setVoiceAndRestart }: {
|
||||||
availableVoices: string[];
|
availableVoices: string[];
|
||||||
setVoiceAndRestart: (voice: string) => void;
|
setVoiceAndRestart: (voice: string) => void;
|
||||||
}) => {
|
}) => {
|
||||||
const { voice: configVoice } = useConfig();
|
const { voice: configVoice, ttsModel, ttsProvider } = useConfig();
|
||||||
|
|
||||||
// If the saved voice is not in the available list, use the first available voice
|
const isKokoro = isKokoroModel(ttsModel);
|
||||||
const currentVoice = (configVoice && availableVoices.includes(configVoice))
|
const maxVoices = getMaxVoicesForProvider(ttsProvider, ttsModel || '');
|
||||||
? configVoice
|
|
||||||
: availableVoices[0] || '';
|
const clampToLimit = useCallback((names: string[]): string[] => {
|
||||||
|
if (maxVoices === Infinity) return names;
|
||||||
|
if (names.length <= maxVoices) return names;
|
||||||
|
// For initial clamp, keep the first up to max allowed
|
||||||
|
return names.slice(0, maxVoices);
|
||||||
|
}, [maxVoices]);
|
||||||
|
|
||||||
|
// Local selection state for Kokoro multi-select
|
||||||
|
const [selectedVoices, setSelectedVoices] = useState<string[]>([]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!isKokoro) return;
|
||||||
|
let initial: string[] = [];
|
||||||
|
if (configVoice && configVoice.includes('+')) {
|
||||||
|
initial = parseKokoroVoiceNames(configVoice);
|
||||||
|
} else if (configVoice && availableVoices.includes(configVoice)) {
|
||||||
|
initial = [configVoice];
|
||||||
|
} else if (availableVoices.length > 0) {
|
||||||
|
initial = [availableVoices[0]];
|
||||||
|
}
|
||||||
|
setSelectedVoices(clampToLimit(initial));
|
||||||
|
}, [isKokoro, configVoice, availableVoices, maxVoices, clampToLimit]);
|
||||||
|
|
||||||
|
// If the saved voice is not in the available list, use the first available voice (non-Kokoro)
|
||||||
|
const currentVoice = useMemo(() => {
|
||||||
|
if (isKokoro) {
|
||||||
|
const combined = buildKokoroVoiceString(selectedVoices);
|
||||||
|
return combined || (availableVoices[0] || '');
|
||||||
|
}
|
||||||
|
return (configVoice && availableVoices.includes(configVoice))
|
||||||
|
? configVoice
|
||||||
|
: availableVoices[0] || '';
|
||||||
|
}, [isKokoro, selectedVoices, availableVoices, configVoice]);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="relative">
|
<div className="relative">
|
||||||
<Listbox value={currentVoice} onChange={setVoiceAndRestart}>
|
{isKokoro ? (
|
||||||
<ListboxButton className="flex items-center space-x-0.5 sm:space-x-1 bg-transparent text-foreground text-xs sm:text-sm focus:outline-none cursor-pointer hover:bg-offbase rounded pl-1.5 sm:pl-2 pr-0.5 sm:pr-1 py-0.5 sm:py-1 transform transition-transform duration-200 ease-in-out hover:scale-[1.04] hover:text-accent">
|
<Listbox
|
||||||
<AudioWaveIcon className="h-3 w-3 sm:h-3.5 sm:w-3.5" />
|
multiple
|
||||||
<span>{currentVoice}</span>
|
value={selectedVoices}
|
||||||
<ChevronUpDownIcon className="h-2.5 w-2.5 sm:h-3 sm:w-3" />
|
onChange={(vals: string[]) => {
|
||||||
</ListboxButton>
|
if (!vals || vals.length === 0) return; // prevent empty selection
|
||||||
<ListboxOptions anchor='top end' className="absolute z-50 w-28 sm:w-32 max-h-64 overflow-auto rounded-lg bg-base shadow-lg ring-1 ring-black ring-opacity-5 focus:outline-none">
|
|
||||||
{availableVoices.map((voiceId) => (
|
let next = vals;
|
||||||
<ListboxOption
|
|
||||||
key={voiceId}
|
// Enforce deepinfra max selection of 2 voices
|
||||||
value={voiceId}
|
if (maxVoices !== Infinity && vals.length > maxVoices) {
|
||||||
className={({ active, selected }) =>
|
// Determine the newly added voice
|
||||||
`relative cursor-pointer select-none py-0.5 px-1.5 sm:py-2 sm:px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium' : ''}`
|
const newlyAdded = vals.find(v => !selectedVoices.includes(v));
|
||||||
|
if (newlyAdded) {
|
||||||
|
const lastPrev = selectedVoices[selectedVoices.length - 1] ?? selectedVoices[0] ?? '';
|
||||||
|
// Build next as [last previously selected, newly added], deduped, limited to max
|
||||||
|
const pair = Array.from(new Set([lastPrev, newlyAdded])).filter(Boolean);
|
||||||
|
next = pair.slice(0, maxVoices);
|
||||||
|
} else {
|
||||||
|
// Fallback: keep the last maxVoices options
|
||||||
|
next = vals.slice(-maxVoices);
|
||||||
}
|
}
|
||||||
>
|
}
|
||||||
<span className='text-xs sm:text-sm'>{voiceId}</span>
|
|
||||||
</ListboxOption>
|
setSelectedVoices(next);
|
||||||
))}
|
const combined = buildKokoroVoiceString(next);
|
||||||
</ListboxOptions>
|
if (combined) {
|
||||||
</Listbox>
|
setVoiceAndRestart(combined);
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<ListboxButton className="flex items-center space-x-0.5 sm:space-x-1 bg-transparent text-foreground text-xs sm:text-sm focus:outline-none cursor-pointer hover:bg-offbase rounded pl-1.5 sm:pl-2 pr-0.5 sm:pr-1 py-0.5 sm:py-1 transform transition-transform duration-200 ease-in-out hover:scale-[1.04] hover:text-accent">
|
||||||
|
<AudioWaveIcon className="h-3 w-3 sm:h-3.5 sm:w-3.5" />
|
||||||
|
<span>
|
||||||
|
{selectedVoices.length > 1
|
||||||
|
? selectedVoices.join(' + ')
|
||||||
|
: selectedVoices[0] || currentVoice}
|
||||||
|
</span>
|
||||||
|
<ChevronUpDownIcon className="h-2.5 w-2.5 sm:h-3 sm:w-3" />
|
||||||
|
</ListboxButton>
|
||||||
|
<ListboxOptions anchor='top end' className="absolute z-50 w-40 sm:w-44 max-h-64 overflow-auto rounded-lg bg-base shadow-lg ring-1 ring-black ring-opacity-5 focus:outline-none">
|
||||||
|
{availableVoices.map((voiceId) => (
|
||||||
|
<ListboxOption
|
||||||
|
key={voiceId}
|
||||||
|
value={voiceId}
|
||||||
|
className={({ active, selected }) =>
|
||||||
|
`relative cursor-pointer select-none py-1 px-2 sm:py-2 sm:px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium bg-accent text-background' : ''} ${selected && active ? 'text-foreground' : ''}`
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<span className='text-xs sm:text-sm'>{voiceId}</span>
|
||||||
|
</ListboxOption>
|
||||||
|
))}
|
||||||
|
</ListboxOptions>
|
||||||
|
</Listbox>
|
||||||
|
) : (
|
||||||
|
<Listbox value={currentVoice} onChange={setVoiceAndRestart}>
|
||||||
|
<ListboxButton className="flex items-center space-x-0.5 sm:space-x-1 bg-transparent text-foreground text-xs sm:text-sm focus:outline-none cursor-pointer hover:bg-offbase rounded pl-1.5 sm:pl-2 pr-0.5 sm:pr-1 py-0.5 sm:py-1 transform transition-transform duration-200 ease-in-out hover:scale-[1.04] hover:text-accent">
|
||||||
|
<AudioWaveIcon className="h-3 w-3 sm:h-3.5 sm:w-3.5" />
|
||||||
|
<span>{currentVoice}</span>
|
||||||
|
<ChevronUpDownIcon className="h-2.5 w-2.5 sm:h-3 sm:w-3" />
|
||||||
|
</ListboxButton>
|
||||||
|
<ListboxOptions anchor='top end' className="absolute z-50 w-28 sm:w-32 max-h-64 overflow-auto rounded-lg bg-base shadow-lg ring-1 ring-black ring-opacity-5 focus:outline-none">
|
||||||
|
{availableVoices.map((voiceId) => (
|
||||||
|
<ListboxOption
|
||||||
|
key={voiceId}
|
||||||
|
value={voiceId}
|
||||||
|
className={({ active, selected }) =>
|
||||||
|
`relative cursor-pointer select-none py-1 px-2 sm:py-2 sm:px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium bg-accent text-background' : ''} ${selected && active ? 'text-foreground' : ''}`
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<span className='text-xs sm:text-sm'>{voiceId}</span>
|
||||||
|
</ListboxOption>
|
||||||
|
))}
|
||||||
|
</ListboxOptions>
|
||||||
|
</Listbox>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
@ -38,6 +38,7 @@ import { getLastDocumentLocation, setLastDocumentLocation } from '@/utils/indexe
|
||||||
import { useBackgroundState } from '@/hooks/audio/useBackgroundState';
|
import { useBackgroundState } from '@/hooks/audio/useBackgroundState';
|
||||||
import { withRetry } from '@/utils/audio';
|
import { withRetry } from '@/utils/audio';
|
||||||
import { processTextToSentences } from '@/utils/nlp';
|
import { processTextToSentences } from '@/utils/nlp';
|
||||||
|
import { isKokoroModel } from '@/utils/voice';
|
||||||
|
|
||||||
// Media globals
|
// Media globals
|
||||||
declare global {
|
declare global {
|
||||||
|
|
@ -155,6 +156,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const activeAbortControllers = useRef<Set<AbortController>>(new Set());
|
const activeAbortControllers = useRef<Set<AbortController>>(new Set());
|
||||||
// Track if we're restoring from a saved position
|
// Track if we're restoring from a saved position
|
||||||
const [pendingRestoreIndex, setPendingRestoreIndex] = useState<number | null>(null);
|
const [pendingRestoreIndex, setPendingRestoreIndex] = useState<number | null>(null);
|
||||||
|
// Guard to coalesce rapid restarts and only resume the latest change
|
||||||
|
const restartSeqRef = useRef(0);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Processes text into sentences using the shared NLP utility
|
* Processes text into sentences using the shared NLP utility
|
||||||
|
|
@ -412,19 +415,31 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
*/
|
*/
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (availableVoices.length > 0) {
|
if (availableVoices.length > 0) {
|
||||||
|
// Allow Kokoro multi-voice strings (e.g., "voice1(0.5)+voice2(0.5)") for any provider
|
||||||
|
const isKokoro = isKokoroModel(configTTSModel);
|
||||||
|
|
||||||
|
if (isKokoro) {
|
||||||
|
// If Kokoro and we have any voice string (including plus/weights), don't override it.
|
||||||
|
// Only default when voice is empty.
|
||||||
|
if (!voice) {
|
||||||
|
setVoice(availableVoices[0]);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (!voice || !availableVoices.includes(voice)) {
|
if (!voice || !availableVoices.includes(voice)) {
|
||||||
console.log(`Voice "${voice || '(empty)'}" not found in available voices. Using "${availableVoices[0]}"`);
|
console.log(`Voice "${voice || '(empty)'}" not found in available voices. Using "${availableVoices[0]}"`);
|
||||||
setVoice(availableVoices[0]);
|
setVoice(availableVoices[0]);
|
||||||
// Don't save to config - just use it temporarily until user explicitly selects one
|
// Don't save to config - just use it temporarily until user explicitly selects one
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}, [availableVoices, voice]);
|
}, [availableVoices, voice, configTTSModel]);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generates and plays audio for the current sentence
|
* Generates and plays audio for the current sentence
|
||||||
*
|
*
|
||||||
* @param {string} sentence - The sentence to generate audio for
|
* @param {string} sentence - The sentence to generate audio for
|
||||||
* @returns {Promise<AudioBuffer | undefined>} The generated audio buffer
|
* @returns {Promise<ArrayBuffer | undefined>} The generated audio buffer
|
||||||
*/
|
*/
|
||||||
const getAudio = useCallback(async (sentence: string): Promise<ArrayBuffer | undefined> => {
|
const getAudio = useCallback(async (sentence: string): Promise<ArrayBuffer | undefined> => {
|
||||||
// Check if the audio is already cached
|
// Check if the audio is already cached
|
||||||
|
|
@ -791,6 +806,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const setSpeedAndRestart = useCallback((newSpeed: number) => {
|
const setSpeedAndRestart = useCallback((newSpeed: number) => {
|
||||||
const wasPlaying = isPlaying;
|
const wasPlaying = isPlaying;
|
||||||
|
|
||||||
|
// Bump restart sequence to invalidate older restarts
|
||||||
|
const mySeq = ++restartSeqRef.current;
|
||||||
|
|
||||||
// Set a flag to prevent double audio requests during config update
|
// Set a flag to prevent double audio requests during config update
|
||||||
setIsProcessing(true);
|
setIsProcessing(true);
|
||||||
|
|
||||||
|
|
@ -806,8 +824,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
// Update config after state changes
|
// Update config after state changes
|
||||||
updateConfigKey('voiceSpeed', newSpeed).then(() => {
|
updateConfigKey('voiceSpeed', newSpeed).then(() => {
|
||||||
setIsProcessing(false);
|
setIsProcessing(false);
|
||||||
// Resume playback if it was playing before
|
// Resume playback if it was playing before and this is the latest restart
|
||||||
if (wasPlaying) {
|
if (wasPlaying && mySeq === restartSeqRef.current) {
|
||||||
setIsPlaying(true);
|
setIsPlaying(true);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
@ -821,6 +839,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const setVoiceAndRestart = useCallback((newVoice: string) => {
|
const setVoiceAndRestart = useCallback((newVoice: string) => {
|
||||||
const wasPlaying = isPlaying;
|
const wasPlaying = isPlaying;
|
||||||
|
|
||||||
|
// Bump restart sequence to invalidate older restarts
|
||||||
|
const mySeq = ++restartSeqRef.current;
|
||||||
|
|
||||||
// Set a flag to prevent double audio requests during config update
|
// Set a flag to prevent double audio requests during config update
|
||||||
setIsProcessing(true);
|
setIsProcessing(true);
|
||||||
|
|
||||||
|
|
@ -836,8 +857,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
// Update config after state changes
|
// Update config after state changes
|
||||||
updateConfigKey('voice', newVoice).then(() => {
|
updateConfigKey('voice', newVoice).then(() => {
|
||||||
setIsProcessing(false);
|
setIsProcessing(false);
|
||||||
// Resume playback if it was playing before
|
// Resume playback if it was playing before and this is the latest restart
|
||||||
if (wasPlaying) {
|
if (wasPlaying && mySeq === restartSeqRef.current) {
|
||||||
setIsPlaying(true);
|
setIsPlaying(true);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
@ -851,6 +872,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const setAudioPlayerSpeedAndRestart = useCallback((newSpeed: number) => {
|
const setAudioPlayerSpeedAndRestart = useCallback((newSpeed: number) => {
|
||||||
const wasPlaying = isPlaying;
|
const wasPlaying = isPlaying;
|
||||||
|
|
||||||
|
// Bump restart sequence to invalidate older restarts
|
||||||
|
const mySeq = ++restartSeqRef.current;
|
||||||
|
|
||||||
// Set a flag to prevent double audio requests during config update
|
// Set a flag to prevent double audio requests during config update
|
||||||
setIsProcessing(true);
|
setIsProcessing(true);
|
||||||
|
|
||||||
|
|
@ -865,8 +889,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
// Update config after state changes
|
// Update config after state changes
|
||||||
updateConfigKey('audioPlayerSpeed', newSpeed).then(() => {
|
updateConfigKey('audioPlayerSpeed', newSpeed).then(() => {
|
||||||
setIsProcessing(false);
|
setIsProcessing(false);
|
||||||
// Resume playback if it was playing before
|
// Resume playback if it was playing before and this is the latest restart
|
||||||
if (wasPlaying) {
|
if (wasPlaying && mySeq === restartSeqRef.current) {
|
||||||
setIsPlaying(true);
|
setIsPlaying(true);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -42,11 +42,14 @@ export const splitIntoSentences = (text: string): string[] => {
|
||||||
const doc = nlp(cleanedText);
|
const doc = nlp(cleanedText);
|
||||||
const rawSentences = doc.sentences().out('array') as string[];
|
const rawSentences = doc.sentences().out('array') as string[];
|
||||||
|
|
||||||
|
// Merge multi-sentence dialogue enclosed in quotes into single items
|
||||||
|
const mergedSentences = mergeQuotedDialogue(rawSentences);
|
||||||
|
|
||||||
let currentBlock = '';
|
let currentBlock = '';
|
||||||
|
|
||||||
for (const sentence of rawSentences) {
|
for (const sentence of mergedSentences) {
|
||||||
const trimmedSentence = sentence.trim();
|
const trimmedSentence = sentence.trim();
|
||||||
|
|
||||||
if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) {
|
if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) {
|
||||||
blocks.push(currentBlock.trim());
|
blocks.push(currentBlock.trim());
|
||||||
currentBlock = trimmedSentence;
|
currentBlock = trimmedSentence;
|
||||||
|
|
@ -150,4 +153,71 @@ export const processTextWithMapping = (text: string): {
|
||||||
rawSentences,
|
rawSentences,
|
||||||
sentenceMapping
|
sentenceMapping
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
// Helper functions to merge quoted dialogue across sentences
|
||||||
|
const countDoubleQuotes = (s: string): number => {
|
||||||
|
const matches = s.match(/["“”]/g);
|
||||||
|
return matches ? matches.length : 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
const countCurlySingleQuotes = (s: string): number => {
|
||||||
|
const matches = s.match(/[‘’]/g);
|
||||||
|
return matches ? matches.length : 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
const countStandaloneStraightSingles = (s: string): number => {
|
||||||
|
let count = 0;
|
||||||
|
for (let i = 0; i < s.length; i++) {
|
||||||
|
if (s[i] === "'") {
|
||||||
|
const prev = i > 0 ? s[i - 1] : '';
|
||||||
|
const next = i + 1 < s.length ? s[i + 1] : '';
|
||||||
|
const isPrevAlphaNum = /[A-Za-z0-9]/.test(prev);
|
||||||
|
const isNextAlphaNum = /[A-Za-z0-9]/.test(next);
|
||||||
|
// Count only when not clearly an apostrophe inside a word (e.g., don't)
|
||||||
|
if (!(isPrevAlphaNum && isNextAlphaNum)) {
|
||||||
|
count++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return count;
|
||||||
|
};
|
||||||
|
|
||||||
|
const mergeQuotedDialogue = (rawSentences: string[]): string[] => {
|
||||||
|
const result: string[] = [];
|
||||||
|
let buffer = '';
|
||||||
|
let insideDouble = false;
|
||||||
|
let insideSingle = false;
|
||||||
|
|
||||||
|
for (const s of rawSentences) {
|
||||||
|
const t = s.trim();
|
||||||
|
const dblCount = countDoubleQuotes(t);
|
||||||
|
const singleCount = countCurlySingleQuotes(t) + countStandaloneStraightSingles(t);
|
||||||
|
|
||||||
|
if (insideDouble || insideSingle) {
|
||||||
|
buffer = buffer ? `${buffer} ${t}` : t;
|
||||||
|
} else {
|
||||||
|
// Start buffering if this sentence opens an unclosed quote
|
||||||
|
if ((dblCount % 2 === 1) || (singleCount % 2 === 1)) {
|
||||||
|
buffer = t;
|
||||||
|
} else {
|
||||||
|
result.push(t);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Toggle quote states after processing this sentence
|
||||||
|
if (dblCount % 2 === 1) insideDouble = !insideDouble;
|
||||||
|
if (singleCount % 2 === 1) insideSingle = !insideSingle;
|
||||||
|
|
||||||
|
// If all open quotes are closed, flush buffer
|
||||||
|
if (!(insideDouble || insideSingle) && buffer) {
|
||||||
|
result.push(buffer);
|
||||||
|
buffer = '';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (buffer) {
|
||||||
|
result.push(buffer);
|
||||||
|
}
|
||||||
|
|
||||||
|
return result;
|
||||||
|
};
|
||||||
74
src/utils/voice.ts
Normal file
74
src/utils/voice.ts
Normal file
|
|
@ -0,0 +1,74 @@
|
||||||
|
/**
|
||||||
|
* Voice Utilities
|
||||||
|
*
|
||||||
|
* This module provides utilities for handling voice selection and management,
|
||||||
|
* particularly for Kokoro multi-voice syntax.
|
||||||
|
*/
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Parses a Kokoro voice string into individual voice names
|
||||||
|
* Strips weights like "af_heart(0.5)" -> "af_heart"
|
||||||
|
*
|
||||||
|
* @param voiceString - Voice string to parse (e.g., "af_heart(0.5)+bf_emma(0.5)")
|
||||||
|
* @returns Array of voice names without weights
|
||||||
|
*/
|
||||||
|
export const parseKokoroVoiceNames = (voiceString: string): string[] =>
|
||||||
|
voiceString
|
||||||
|
.split('+')
|
||||||
|
.map(s => s.trim())
|
||||||
|
.filter(Boolean)
|
||||||
|
.map(s => s.replace(/\([^)]*\)/g, '').trim());
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Builds a Kokoro voice string from an array of voice names
|
||||||
|
* Automatically calculates equal weights for multiple voices
|
||||||
|
*
|
||||||
|
* @param names - Array of voice names
|
||||||
|
* @returns Formatted voice string with weights or single voice name
|
||||||
|
*/
|
||||||
|
export const buildKokoroVoiceString = (names: string[]): string => {
|
||||||
|
const n = names.length;
|
||||||
|
if (n === 0) return '';
|
||||||
|
if (n === 1) return names[0];
|
||||||
|
|
||||||
|
const weight = 1 / n;
|
||||||
|
const weightString = weight.toFixed(2).replace(/0+$/, '').replace(/\.$/, '');
|
||||||
|
return names.map(name => `${name}(${weightString})`).join('+');
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Checks if a model name is a Kokoro model
|
||||||
|
*
|
||||||
|
* @param modelName - TTS model name
|
||||||
|
* @returns True if the model is a Kokoro model
|
||||||
|
*/
|
||||||
|
export const isKokoroModel = (modelName: string | undefined): boolean => {
|
||||||
|
return (modelName || '').toLowerCase().includes('kokoro');
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Strips weight annotations from a voice string
|
||||||
|
*
|
||||||
|
* @param voiceString - Voice string with or without weights
|
||||||
|
* @returns Voice string without weights
|
||||||
|
*/
|
||||||
|
export const stripVoiceWeights = (voiceString: string): string => {
|
||||||
|
return voiceString.replace(/\([^)]*\)/g, '').trim();
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Determines the maximum number of voices allowed for a provider/model combination
|
||||||
|
*
|
||||||
|
* @param provider - TTS provider name
|
||||||
|
* @param model - TTS model name
|
||||||
|
* @returns Maximum number of voices (Infinity for unlimited)
|
||||||
|
*/
|
||||||
|
export const getMaxVoicesForProvider = (provider: string, model: string): number => {
|
||||||
|
if (!isKokoroModel(model)) return 1;
|
||||||
|
|
||||||
|
// Deepinfra Kokoro supports up to 2 voices
|
||||||
|
if (provider === 'deepinfra') return 2;
|
||||||
|
|
||||||
|
// Other providers with Kokoro support unlimited voices
|
||||||
|
return Infinity;
|
||||||
|
};
|
||||||
|
|
@ -93,13 +93,15 @@ export async function pauseTTSAndVerify(page: Page) {
|
||||||
export async function setupTest(page: Page) {
|
export async function setupTest(page: Page) {
|
||||||
// Navigate to the home page before each test
|
// Navigate to the home page before each test
|
||||||
await page.goto('/');
|
await page.goto('/');
|
||||||
await page.waitForLoadState('networkidle');
|
//await page.waitForLoadState('networkidle');
|
||||||
|
|
||||||
|
// If running in CI, select the "Custom OpenAI-Like" model and "Deepinfra" provider
|
||||||
|
//if (process.env.CI) {
|
||||||
await page.getByRole('button', { name: 'Custom OpenAI-Like' }).click();
|
await page.getByRole('button', { name: 'Custom OpenAI-Like' }).click();
|
||||||
await page.getByText('Deepinfra').click();
|
await page.getByText('Deepinfra').click();
|
||||||
|
//}
|
||||||
|
|
||||||
// Click the "done" button to dismiss the welcome message
|
// Click the "done" button to dismiss the welcome message
|
||||||
await page.getByRole('tab', { name: '🔑 API' }).click();
|
|
||||||
await page.getByRole('button', { name: 'Save' }).click();
|
await page.getByRole('button', { name: 'Save' }).click();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -7,6 +7,7 @@ import {
|
||||||
selectVoiceAndAssertPlayback,
|
selectVoiceAndAssertPlayback,
|
||||||
changeNativeSpeedAndAssert,
|
changeNativeSpeedAndAssert,
|
||||||
expectMediaState,
|
expectMediaState,
|
||||||
|
expectProcessingTransition,
|
||||||
} from './helpers';
|
} from './helpers';
|
||||||
|
|
||||||
test.describe('Play/Pause Tests', () => {
|
test.describe('Play/Pause Tests', () => {
|
||||||
|
|
@ -48,7 +49,7 @@ test.describe('Play/Pause Tests', () => {
|
||||||
await pauseTTSAndVerify(page);
|
await pauseTTSAndVerify(page);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('loads voices and switches voice with processing state then resumes play', async ({ page }) => {
|
test('switches to a single voice and resumes playing', async ({ page }) => {
|
||||||
// Start playback
|
// Start playback
|
||||||
await playTTSAndWaitForASecond(page, 'sample.pdf');
|
await playTTSAndWaitForASecond(page, 'sample.pdf');
|
||||||
|
|
||||||
|
|
@ -61,8 +62,46 @@ test.describe('Play/Pause Tests', () => {
|
||||||
const options = page.getByRole('option');
|
const options = page.getByRole('option');
|
||||||
expect(await options.count()).toBeGreaterThan(0);
|
expect(await options.count()).toBeGreaterThan(0);
|
||||||
|
|
||||||
// Switch to the first available voice and assert processing -> playing
|
// Step 1: Select af_bella (adds it to the multi-select list)
|
||||||
await selectVoiceAndAssertPlayback(page, /.*/);
|
await selectVoiceAndAssertPlayback(page, 'af_bella');
|
||||||
|
|
||||||
|
// Step 2: Deselect the first (initially selected) voice so that only af_bella remains
|
||||||
|
await openVoicesMenu(page);
|
||||||
|
const selected = page.locator('[role="option"][aria-selected="true"]');
|
||||||
|
const count = await selected.count();
|
||||||
|
for (let i = 0; i < count; i++) {
|
||||||
|
const opt = selected.nth(i);
|
||||||
|
const name = (await opt.textContent())?.trim() ?? '';
|
||||||
|
// Deselect the first selected option that is not af_bella
|
||||||
|
if (!/af_bella/i.test(name)) {
|
||||||
|
await opt.click();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await expectProcessingTransition(page);
|
||||||
|
|
||||||
|
// Final state should be playing
|
||||||
|
await expectMediaState(page, 'playing');
|
||||||
|
});
|
||||||
|
|
||||||
|
test('selects multiple Kokoro voices and resumes playing', async ({ page }) => {
|
||||||
|
// Start playback
|
||||||
|
await playTTSAndWaitForASecond(page, 'sample.pdf');
|
||||||
|
|
||||||
|
// Ensure TTS controls are present
|
||||||
|
await expect(page.getByRole('button', { name: 'Skip backward' })).toBeVisible();
|
||||||
|
await expect(page.getByRole('button', { name: 'Skip forward' })).toBeVisible();
|
||||||
|
|
||||||
|
// Select first voice (e.g., bf_emma) and assert processing -> playing
|
||||||
|
await openVoicesMenu(page);
|
||||||
|
await selectVoiceAndAssertPlayback(page, 'bf_emma');
|
||||||
|
|
||||||
|
// Select second voice (e.g., af_heart) to create a multi-voice mix and assert again
|
||||||
|
await openVoicesMenu(page);
|
||||||
|
await selectVoiceAndAssertPlayback(page, 'af_heart');
|
||||||
|
|
||||||
|
// Final state should be playing
|
||||||
|
await expectMediaState(page, 'playing');
|
||||||
});
|
});
|
||||||
|
|
||||||
test('changing TTS native speed toggles processing and returns to playing', async ({ page }) => {
|
test('changing TTS native speed toggles processing and returns to playing', async ({ page }) => {
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue