feat(tts,ui,api): add Kokoro multi-voice selection and SDK support
- Introduce voice utils (model detection, voice parsing/weights, limits) - Enable Kokoro multi-voice strings across OpenAI/Deepinfra/custom providers - Normalize non-Kokoro voices to single token for SDK calls - Expose full Kokoro voice list for custom-openai Kokoro models - Update TTS API to return audio as ArrayBuffer and improve logging - Add multi-select UI for Kokoro voices with provider-based clamping - Preserve Kokoro voice strings in TTSContext and coalesce restarts - Merge multi-sentence quoted dialogue in NLP sentence splitter - Update tests for single and multi-voice selection flows
This commit is contained in:
Richard Roberson
parent
21870ed576
commit
1dcffc82d8
8 changed files with 368 additions and 54 deletions
|
|
@ -1,6 +1,7 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import OpenAI from 'openai';
|
||||
import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs';
|
||||
import { isKokoroModel, stripVoiceWeights } from '@/utils/voice';
|
||||
|
||||
type CustomVoice = string;
|
||||
type ExtendedSpeechParams = Omit<SpeechCreateParams, 'voice'> & {
|
||||
|
|
@ -15,11 +16,7 @@ export async function POST(req: NextRequest) {
|
|||
const openApiBaseUrl = req.headers.get('x-openai-base-url') || process.env.API_BASE;
|
||||
const provider = req.headers.get('x-tts-provider') || 'openai';
|
||||
const { text, voice, speed, format, model, instructions } = await req.json();
|
||||
console.log('Received TTS request:', text, voice, speed, format, model);
|
||||
|
||||
if (!openApiKey) {
|
||||
return NextResponse.json({ error: 'Missing OpenAI API key' }, { status: 401 });
|
||||
}
|
||||
console.log('Received TTS request:', { provider, model, voice, speed, format, hasInstructions: Boolean(instructions) });
|
||||
|
||||
if (!text || !voice || !speed) {
|
||||
return NextResponse.json({ error: 'Missing required parameters' }, { status: 400 });
|
||||
|
|
@ -27,18 +24,28 @@ export async function POST(req: NextRequest) {
|
|||
|
||||
// Apply Deepinfra defaults if provider is deepinfra
|
||||
const finalModel = provider === 'deepinfra' && !model ? 'hexgrad/Kokoro-82M' : model;
|
||||
const finalVoice = provider === 'deepinfra' && !voice ? 'af_bella' : voice;
|
||||
const initialVoice = provider === 'deepinfra' && !voice ? 'af_bella' : voice;
|
||||
|
||||
// Initialize OpenAI client with abort signal
|
||||
// For SDK providers (OpenAI/Deepinfra), preserve multi-voice for Kokoro models, otherwise normalize to first token
|
||||
const isKokoro = isKokoroModel(finalModel);
|
||||
let normalizedVoice = initialVoice;
|
||||
if (!isKokoro && typeof normalizedVoice === 'string' && normalizedVoice.includes('+')) {
|
||||
normalizedVoice = stripVoiceWeights(normalizedVoice.split('+')[0]);
|
||||
console.log('Normalized multi-voice to single for non-Kokoro SDK provider:', normalizedVoice);
|
||||
}
|
||||
|
||||
// Initialize OpenAI client with abort signal (OpenAI/deepinfra)
|
||||
const openai = new OpenAI({
|
||||
apiKey: openApiKey,
|
||||
baseURL: openApiBaseUrl,
|
||||
});
|
||||
|
||||
// Unified path: all providers (openai, deepinfra, custom-openai) go through the SDK below.
|
||||
|
||||
// Request audio from OpenAI and pass along the abort signal
|
||||
const createParams: ExtendedSpeechParams = {
|
||||
model: finalModel || 'tts-1',
|
||||
voice: finalVoice as "alloy",
|
||||
voice: normalizedVoice as SpeechCreateParams['voice'],
|
||||
input: text,
|
||||
speed: speed,
|
||||
response_format: format === 'aac' ? 'aac' : 'mp3',
|
||||
|
|
@ -51,13 +58,11 @@ export async function POST(req: NextRequest) {
|
|||
|
||||
const response = await openai.audio.speech.create(createParams as SpeechCreateParams, { signal: req.signal });
|
||||
|
||||
// Get the audio data as array buffer
|
||||
// Read the audio data as an ArrayBuffer and return it with appropriate headers
|
||||
// This will also be aborted if the client cancels
|
||||
const stream = response.body;
|
||||
|
||||
// Return audio data with appropriate headers
|
||||
const buffer = await response.arrayBuffer();
|
||||
const contentType = format === 'aac' ? 'audio/aac' : 'audio/mpeg';
|
||||
return new NextResponse(stream, {
|
||||
return new NextResponse(buffer, {
|
||||
headers: {
|
||||
'Content-Type': contentType
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { isKokoroModel } from '@/utils/voice';
|
||||
|
||||
const OPENAI_VOICES = ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer'];
|
||||
const GPT4O_MINI_VOICES = ['alloy', 'ash', 'coral', 'echo', 'fable', 'onyx', 'nova', 'sage', 'shimmer'];
|
||||
|
|
@ -29,6 +30,10 @@ function getDefaultVoices(provider: string, model: string): string[] {
|
|||
|
||||
// For Custom OpenAI-Like provider
|
||||
if (provider === 'custom-openai') {
|
||||
// If using Kokoro-FastAPI (model string contains 'kokoro'), expose full Kokoro voices
|
||||
if (isKokoroModel(model)) {
|
||||
return KOKORO_VOICES;
|
||||
}
|
||||
return CUSTOM_OPENAI_VOICES;
|
||||
}
|
||||
|
||||
|
|
@ -72,7 +77,7 @@ async function fetchDeepinfraVoices(apiKey: string): Promise<string[]> {
|
|||
}
|
||||
|
||||
const data = await response.json();
|
||||
//console.log('Deepinfra voices response:', data);
|
||||
|
||||
// Extract voice names from the response, excluding preset voices
|
||||
if (data.voices && Array.isArray(data.voices)) {
|
||||
return data.voices
|
||||
|
|
|
|||
|
|
@ -8,40 +8,135 @@ import {
|
|||
} from '@headlessui/react';
|
||||
import { ChevronUpDownIcon, AudioWaveIcon } from '@/components/icons/Icons';
|
||||
import { useConfig } from '@/contexts/ConfigContext';
|
||||
import { useEffect, useMemo, useState, useCallback } from 'react';
|
||||
import {
|
||||
parseKokoroVoiceNames,
|
||||
buildKokoroVoiceString,
|
||||
getMaxVoicesForProvider,
|
||||
isKokoroModel
|
||||
} from '@/utils/voice';
|
||||
|
||||
export const VoicesControl = ({ availableVoices, setVoiceAndRestart }: {
|
||||
availableVoices: string[];
|
||||
setVoiceAndRestart: (voice: string) => void;
|
||||
}) => {
|
||||
const { voice: configVoice } = useConfig();
|
||||
const { voice: configVoice, ttsModel, ttsProvider } = useConfig();
|
||||
|
||||
// If the saved voice is not in the available list, use the first available voice
|
||||
const currentVoice = (configVoice && availableVoices.includes(configVoice))
|
||||
? configVoice
|
||||
: availableVoices[0] || '';
|
||||
const isKokoro = isKokoroModel(ttsModel);
|
||||
const maxVoices = getMaxVoicesForProvider(ttsProvider, ttsModel || '');
|
||||
|
||||
const clampToLimit = useCallback((names: string[]): string[] => {
|
||||
if (maxVoices === Infinity) return names;
|
||||
if (names.length <= maxVoices) return names;
|
||||
// For initial clamp, keep the first up to max allowed
|
||||
return names.slice(0, maxVoices);
|
||||
}, [maxVoices]);
|
||||
|
||||
// Local selection state for Kokoro multi-select
|
||||
const [selectedVoices, setSelectedVoices] = useState<string[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!isKokoro) return;
|
||||
let initial: string[] = [];
|
||||
if (configVoice && configVoice.includes('+')) {
|
||||
initial = parseKokoroVoiceNames(configVoice);
|
||||
} else if (configVoice && availableVoices.includes(configVoice)) {
|
||||
initial = [configVoice];
|
||||
} else if (availableVoices.length > 0) {
|
||||
initial = [availableVoices[0]];
|
||||
}
|
||||
setSelectedVoices(clampToLimit(initial));
|
||||
}, [isKokoro, configVoice, availableVoices, maxVoices, clampToLimit]);
|
||||
|
||||
// If the saved voice is not in the available list, use the first available voice (non-Kokoro)
|
||||
const currentVoice = useMemo(() => {
|
||||
if (isKokoro) {
|
||||
const combined = buildKokoroVoiceString(selectedVoices);
|
||||
return combined || (availableVoices[0] || '');
|
||||
}
|
||||
return (configVoice && availableVoices.includes(configVoice))
|
||||
? configVoice
|
||||
: availableVoices[0] || '';
|
||||
}, [isKokoro, selectedVoices, availableVoices, configVoice]);
|
||||
|
||||
return (
|
||||
<div className="relative">
|
||||
<Listbox value={currentVoice} onChange={setVoiceAndRestart}>
|
||||
<ListboxButton className="flex items-center space-x-0.5 sm:space-x-1 bg-transparent text-foreground text-xs sm:text-sm focus:outline-none cursor-pointer hover:bg-offbase rounded pl-1.5 sm:pl-2 pr-0.5 sm:pr-1 py-0.5 sm:py-1 transform transition-transform duration-200 ease-in-out hover:scale-[1.04] hover:text-accent">
|
||||
<AudioWaveIcon className="h-3 w-3 sm:h-3.5 sm:w-3.5" />
|
||||
<span>{currentVoice}</span>
|
||||
<ChevronUpDownIcon className="h-2.5 w-2.5 sm:h-3 sm:w-3" />
|
||||
</ListboxButton>
|
||||
<ListboxOptions anchor='top end' className="absolute z-50 w-28 sm:w-32 max-h-64 overflow-auto rounded-lg bg-base shadow-lg ring-1 ring-black ring-opacity-5 focus:outline-none">
|
||||
{availableVoices.map((voiceId) => (
|
||||
<ListboxOption
|
||||
key={voiceId}
|
||||
value={voiceId}
|
||||
className={({ active, selected }) =>
|
||||
`relative cursor-pointer select-none py-0.5 px-1.5 sm:py-2 sm:px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium' : ''}`
|
||||
{isKokoro ? (
|
||||
<Listbox
|
||||
multiple
|
||||
value={selectedVoices}
|
||||
onChange={(vals: string[]) => {
|
||||
if (!vals || vals.length === 0) return; // prevent empty selection
|
||||
|
||||
let next = vals;
|
||||
|
||||
// Enforce deepinfra max selection of 2 voices
|
||||
if (maxVoices !== Infinity && vals.length > maxVoices) {
|
||||
// Determine the newly added voice
|
||||
const newlyAdded = vals.find(v => !selectedVoices.includes(v));
|
||||
if (newlyAdded) {
|
||||
const lastPrev = selectedVoices[selectedVoices.length - 1] ?? selectedVoices[0] ?? '';
|
||||
// Build next as [last previously selected, newly added], deduped, limited to max
|
||||
const pair = Array.from(new Set([lastPrev, newlyAdded])).filter(Boolean);
|
||||
next = pair.slice(0, maxVoices);
|
||||
} else {
|
||||
// Fallback: keep the last maxVoices options
|
||||
next = vals.slice(-maxVoices);
|
||||
}
|
||||
>
|
||||
<span className='text-xs sm:text-sm'>{voiceId}</span>
|
||||
</ListboxOption>
|
||||
))}
|
||||
</ListboxOptions>
|
||||
</Listbox>
|
||||
}
|
||||
|
||||
setSelectedVoices(next);
|
||||
const combined = buildKokoroVoiceString(next);
|
||||
if (combined) {
|
||||
setVoiceAndRestart(combined);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<ListboxButton className="flex items-center space-x-0.5 sm:space-x-1 bg-transparent text-foreground text-xs sm:text-sm focus:outline-none cursor-pointer hover:bg-offbase rounded pl-1.5 sm:pl-2 pr-0.5 sm:pr-1 py-0.5 sm:py-1 transform transition-transform duration-200 ease-in-out hover:scale-[1.04] hover:text-accent">
|
||||
<AudioWaveIcon className="h-3 w-3 sm:h-3.5 sm:w-3.5" />
|
||||
<span>
|
||||
{selectedVoices.length > 1
|
||||
? selectedVoices.join(' + ')
|
||||
: selectedVoices[0] || currentVoice}
|
||||
</span>
|
||||
<ChevronUpDownIcon className="h-2.5 w-2.5 sm:h-3 sm:w-3" />
|
||||
</ListboxButton>
|
||||
<ListboxOptions anchor='top end' className="absolute z-50 w-40 sm:w-44 max-h-64 overflow-auto rounded-lg bg-base shadow-lg ring-1 ring-black ring-opacity-5 focus:outline-none">
|
||||
{availableVoices.map((voiceId) => (
|
||||
<ListboxOption
|
||||
key={voiceId}
|
||||
value={voiceId}
|
||||
className={({ active, selected }) =>
|
||||
`relative cursor-pointer select-none py-1 px-2 sm:py-2 sm:px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium bg-accent text-background' : ''} ${selected && active ? 'text-foreground' : ''}`
|
||||
}
|
||||
>
|
||||
<span className='text-xs sm:text-sm'>{voiceId}</span>
|
||||
</ListboxOption>
|
||||
))}
|
||||
</ListboxOptions>
|
||||
</Listbox>
|
||||
) : (
|
||||
<Listbox value={currentVoice} onChange={setVoiceAndRestart}>
|
||||
<ListboxButton className="flex items-center space-x-0.5 sm:space-x-1 bg-transparent text-foreground text-xs sm:text-sm focus:outline-none cursor-pointer hover:bg-offbase rounded pl-1.5 sm:pl-2 pr-0.5 sm:pr-1 py-0.5 sm:py-1 transform transition-transform duration-200 ease-in-out hover:scale-[1.04] hover:text-accent">
|
||||
<AudioWaveIcon className="h-3 w-3 sm:h-3.5 sm:w-3.5" />
|
||||
<span>{currentVoice}</span>
|
||||
<ChevronUpDownIcon className="h-2.5 w-2.5 sm:h-3 sm:w-3" />
|
||||
</ListboxButton>
|
||||
<ListboxOptions anchor='top end' className="absolute z-50 w-28 sm:w-32 max-h-64 overflow-auto rounded-lg bg-base shadow-lg ring-1 ring-black ring-opacity-5 focus:outline-none">
|
||||
{availableVoices.map((voiceId) => (
|
||||
<ListboxOption
|
||||
key={voiceId}
|
||||
value={voiceId}
|
||||
className={({ active, selected }) =>
|
||||
`relative cursor-pointer select-none py-1 px-2 sm:py-2 sm:px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium bg-accent text-background' : ''} ${selected && active ? 'text-foreground' : ''}`
|
||||
}
|
||||
>
|
||||
<span className='text-xs sm:text-sm'>{voiceId}</span>
|
||||
</ListboxOption>
|
||||
))}
|
||||
</ListboxOptions>
|
||||
</Listbox>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
|
@ -38,6 +38,7 @@ import { getLastDocumentLocation, setLastDocumentLocation } from '@/utils/indexe
|
|||
import { useBackgroundState } from '@/hooks/audio/useBackgroundState';
|
||||
import { withRetry } from '@/utils/audio';
|
||||
import { processTextToSentences } from '@/utils/nlp';
|
||||
import { isKokoroModel } from '@/utils/voice';
|
||||
|
||||
// Media globals
|
||||
declare global {
|
||||
|
|
@ -155,6 +156,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const activeAbortControllers = useRef<Set<AbortController>>(new Set());
|
||||
// Track if we're restoring from a saved position
|
||||
const [pendingRestoreIndex, setPendingRestoreIndex] = useState<number | null>(null);
|
||||
// Guard to coalesce rapid restarts and only resume the latest change
|
||||
const restartSeqRef = useRef(0);
|
||||
|
||||
/**
|
||||
* Processes text into sentences using the shared NLP utility
|
||||
|
|
@ -412,19 +415,31 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
*/
|
||||
useEffect(() => {
|
||||
if (availableVoices.length > 0) {
|
||||
// Allow Kokoro multi-voice strings (e.g., "voice1(0.5)+voice2(0.5)") for any provider
|
||||
const isKokoro = isKokoroModel(configTTSModel);
|
||||
|
||||
if (isKokoro) {
|
||||
// If Kokoro and we have any voice string (including plus/weights), don't override it.
|
||||
// Only default when voice is empty.
|
||||
if (!voice) {
|
||||
setVoice(availableVoices[0]);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (!voice || !availableVoices.includes(voice)) {
|
||||
console.log(`Voice "${voice || '(empty)'}" not found in available voices. Using "${availableVoices[0]}"`);
|
||||
setVoice(availableVoices[0]);
|
||||
// Don't save to config - just use it temporarily until user explicitly selects one
|
||||
}
|
||||
}
|
||||
}, [availableVoices, voice]);
|
||||
}, [availableVoices, voice, configTTSModel]);
|
||||
|
||||
/**
|
||||
* Generates and plays audio for the current sentence
|
||||
*
|
||||
* @param {string} sentence - The sentence to generate audio for
|
||||
* @returns {Promise<AudioBuffer | undefined>} The generated audio buffer
|
||||
* @returns {Promise<ArrayBuffer | undefined>} The generated audio buffer
|
||||
*/
|
||||
const getAudio = useCallback(async (sentence: string): Promise<ArrayBuffer | undefined> => {
|
||||
// Check if the audio is already cached
|
||||
|
|
@ -791,6 +806,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const setSpeedAndRestart = useCallback((newSpeed: number) => {
|
||||
const wasPlaying = isPlaying;
|
||||
|
||||
// Bump restart sequence to invalidate older restarts
|
||||
const mySeq = ++restartSeqRef.current;
|
||||
|
||||
// Set a flag to prevent double audio requests during config update
|
||||
setIsProcessing(true);
|
||||
|
||||
|
|
@ -806,8 +824,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
// Update config after state changes
|
||||
updateConfigKey('voiceSpeed', newSpeed).then(() => {
|
||||
setIsProcessing(false);
|
||||
// Resume playback if it was playing before
|
||||
if (wasPlaying) {
|
||||
// Resume playback if it was playing before and this is the latest restart
|
||||
if (wasPlaying && mySeq === restartSeqRef.current) {
|
||||
setIsPlaying(true);
|
||||
}
|
||||
});
|
||||
|
|
@ -821,6 +839,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const setVoiceAndRestart = useCallback((newVoice: string) => {
|
||||
const wasPlaying = isPlaying;
|
||||
|
||||
// Bump restart sequence to invalidate older restarts
|
||||
const mySeq = ++restartSeqRef.current;
|
||||
|
||||
// Set a flag to prevent double audio requests during config update
|
||||
setIsProcessing(true);
|
||||
|
||||
|
|
@ -836,8 +857,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
// Update config after state changes
|
||||
updateConfigKey('voice', newVoice).then(() => {
|
||||
setIsProcessing(false);
|
||||
// Resume playback if it was playing before
|
||||
if (wasPlaying) {
|
||||
// Resume playback if it was playing before and this is the latest restart
|
||||
if (wasPlaying && mySeq === restartSeqRef.current) {
|
||||
setIsPlaying(true);
|
||||
}
|
||||
});
|
||||
|
|
@ -851,6 +872,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const setAudioPlayerSpeedAndRestart = useCallback((newSpeed: number) => {
|
||||
const wasPlaying = isPlaying;
|
||||
|
||||
// Bump restart sequence to invalidate older restarts
|
||||
const mySeq = ++restartSeqRef.current;
|
||||
|
||||
// Set a flag to prevent double audio requests during config update
|
||||
setIsProcessing(true);
|
||||
|
||||
|
|
@ -865,8 +889,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
// Update config after state changes
|
||||
updateConfigKey('audioPlayerSpeed', newSpeed).then(() => {
|
||||
setIsProcessing(false);
|
||||
// Resume playback if it was playing before
|
||||
if (wasPlaying) {
|
||||
// Resume playback if it was playing before and this is the latest restart
|
||||
if (wasPlaying && mySeq === restartSeqRef.current) {
|
||||
setIsPlaying(true);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -42,11 +42,14 @@ export const splitIntoSentences = (text: string): string[] => {
|
|||
const doc = nlp(cleanedText);
|
||||
const rawSentences = doc.sentences().out('array') as string[];
|
||||
|
||||
// Merge multi-sentence dialogue enclosed in quotes into single items
|
||||
const mergedSentences = mergeQuotedDialogue(rawSentences);
|
||||
|
||||
let currentBlock = '';
|
||||
|
||||
for (const sentence of rawSentences) {
|
||||
for (const sentence of mergedSentences) {
|
||||
const trimmedSentence = sentence.trim();
|
||||
|
||||
|
||||
if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) {
|
||||
blocks.push(currentBlock.trim());
|
||||
currentBlock = trimmedSentence;
|
||||
|
|
@ -150,4 +153,71 @@ export const processTextWithMapping = (text: string): {
|
|||
rawSentences,
|
||||
sentenceMapping
|
||||
};
|
||||
};
|
||||
};
|
||||
// Helper functions to merge quoted dialogue across sentences
|
||||
const countDoubleQuotes = (s: string): number => {
|
||||
const matches = s.match(/["“”]/g);
|
||||
return matches ? matches.length : 0;
|
||||
};
|
||||
|
||||
const countCurlySingleQuotes = (s: string): number => {
|
||||
const matches = s.match(/[‘’]/g);
|
||||
return matches ? matches.length : 0;
|
||||
};
|
||||
|
||||
const countStandaloneStraightSingles = (s: string): number => {
|
||||
let count = 0;
|
||||
for (let i = 0; i < s.length; i++) {
|
||||
if (s[i] === "'") {
|
||||
const prev = i > 0 ? s[i - 1] : '';
|
||||
const next = i + 1 < s.length ? s[i + 1] : '';
|
||||
const isPrevAlphaNum = /[A-Za-z0-9]/.test(prev);
|
||||
const isNextAlphaNum = /[A-Za-z0-9]/.test(next);
|
||||
// Count only when not clearly an apostrophe inside a word (e.g., don't)
|
||||
if (!(isPrevAlphaNum && isNextAlphaNum)) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return count;
|
||||
};
|
||||
|
||||
const mergeQuotedDialogue = (rawSentences: string[]): string[] => {
|
||||
const result: string[] = [];
|
||||
let buffer = '';
|
||||
let insideDouble = false;
|
||||
let insideSingle = false;
|
||||
|
||||
for (const s of rawSentences) {
|
||||
const t = s.trim();
|
||||
const dblCount = countDoubleQuotes(t);
|
||||
const singleCount = countCurlySingleQuotes(t) + countStandaloneStraightSingles(t);
|
||||
|
||||
if (insideDouble || insideSingle) {
|
||||
buffer = buffer ? `${buffer} ${t}` : t;
|
||||
} else {
|
||||
// Start buffering if this sentence opens an unclosed quote
|
||||
if ((dblCount % 2 === 1) || (singleCount % 2 === 1)) {
|
||||
buffer = t;
|
||||
} else {
|
||||
result.push(t);
|
||||
}
|
||||
}
|
||||
|
||||
// Toggle quote states after processing this sentence
|
||||
if (dblCount % 2 === 1) insideDouble = !insideDouble;
|
||||
if (singleCount % 2 === 1) insideSingle = !insideSingle;
|
||||
|
||||
// If all open quotes are closed, flush buffer
|
||||
if (!(insideDouble || insideSingle) && buffer) {
|
||||
result.push(buffer);
|
||||
buffer = '';
|
||||
}
|
||||
}
|
||||
|
||||
if (buffer) {
|
||||
result.push(buffer);
|
||||
}
|
||||
|
||||
return result;
|
||||
};
|
||||
74
src/utils/voice.ts
Normal file
74
src/utils/voice.ts
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
/**
|
||||
* Voice Utilities
|
||||
*
|
||||
* This module provides utilities for handling voice selection and management,
|
||||
* particularly for Kokoro multi-voice syntax.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Parses a Kokoro voice string into individual voice names
|
||||
* Strips weights like "af_heart(0.5)" -> "af_heart"
|
||||
*
|
||||
* @param voiceString - Voice string to parse (e.g., "af_heart(0.5)+bf_emma(0.5)")
|
||||
* @returns Array of voice names without weights
|
||||
*/
|
||||
export const parseKokoroVoiceNames = (voiceString: string): string[] =>
|
||||
voiceString
|
||||
.split('+')
|
||||
.map(s => s.trim())
|
||||
.filter(Boolean)
|
||||
.map(s => s.replace(/\([^)]*\)/g, '').trim());
|
||||
|
||||
/**
|
||||
* Builds a Kokoro voice string from an array of voice names
|
||||
* Automatically calculates equal weights for multiple voices
|
||||
*
|
||||
* @param names - Array of voice names
|
||||
* @returns Formatted voice string with weights or single voice name
|
||||
*/
|
||||
export const buildKokoroVoiceString = (names: string[]): string => {
|
||||
const n = names.length;
|
||||
if (n === 0) return '';
|
||||
if (n === 1) return names[0];
|
||||
|
||||
const weight = 1 / n;
|
||||
const weightString = weight.toFixed(2).replace(/0+$/, '').replace(/\.$/, '');
|
||||
return names.map(name => `${name}(${weightString})`).join('+');
|
||||
};
|
||||
|
||||
/**
|
||||
* Checks if a model name is a Kokoro model
|
||||
*
|
||||
* @param modelName - TTS model name
|
||||
* @returns True if the model is a Kokoro model
|
||||
*/
|
||||
export const isKokoroModel = (modelName: string | undefined): boolean => {
|
||||
return (modelName || '').toLowerCase().includes('kokoro');
|
||||
};
|
||||
|
||||
/**
|
||||
* Strips weight annotations from a voice string
|
||||
*
|
||||
* @param voiceString - Voice string with or without weights
|
||||
* @returns Voice string without weights
|
||||
*/
|
||||
export const stripVoiceWeights = (voiceString: string): string => {
|
||||
return voiceString.replace(/\([^)]*\)/g, '').trim();
|
||||
};
|
||||
|
||||
/**
|
||||
* Determines the maximum number of voices allowed for a provider/model combination
|
||||
*
|
||||
* @param provider - TTS provider name
|
||||
* @param model - TTS model name
|
||||
* @returns Maximum number of voices (Infinity for unlimited)
|
||||
*/
|
||||
export const getMaxVoicesForProvider = (provider: string, model: string): number => {
|
||||
if (!isKokoroModel(model)) return 1;
|
||||
|
||||
// Deepinfra Kokoro supports up to 2 voices
|
||||
if (provider === 'deepinfra') return 2;
|
||||
|
||||
// Other providers with Kokoro support unlimited voices
|
||||
return Infinity;
|
||||
};
|
||||
|
|
@ -93,13 +93,15 @@ export async function pauseTTSAndVerify(page: Page) {
|
|||
export async function setupTest(page: Page) {
|
||||
// Navigate to the home page before each test
|
||||
await page.goto('/');
|
||||
await page.waitForLoadState('networkidle');
|
||||
//await page.waitForLoadState('networkidle');
|
||||
|
||||
// If running in CI, select the "Custom OpenAI-Like" model and "Deepinfra" provider
|
||||
//if (process.env.CI) {
|
||||
await page.getByRole('button', { name: 'Custom OpenAI-Like' }).click();
|
||||
await page.getByText('Deepinfra').click();
|
||||
//}
|
||||
|
||||
// Click the "done" button to dismiss the welcome message
|
||||
await page.getByRole('tab', { name: '🔑 API' }).click();
|
||||
await page.getByRole('button', { name: 'Save' }).click();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ import {
|
|||
selectVoiceAndAssertPlayback,
|
||||
changeNativeSpeedAndAssert,
|
||||
expectMediaState,
|
||||
expectProcessingTransition,
|
||||
} from './helpers';
|
||||
|
||||
test.describe('Play/Pause Tests', () => {
|
||||
|
|
@ -48,7 +49,7 @@ test.describe('Play/Pause Tests', () => {
|
|||
await pauseTTSAndVerify(page);
|
||||
});
|
||||
|
||||
test('loads voices and switches voice with processing state then resumes play', async ({ page }) => {
|
||||
test('switches to a single voice and resumes playing', async ({ page }) => {
|
||||
// Start playback
|
||||
await playTTSAndWaitForASecond(page, 'sample.pdf');
|
||||
|
||||
|
|
@ -61,8 +62,46 @@ test.describe('Play/Pause Tests', () => {
|
|||
const options = page.getByRole('option');
|
||||
expect(await options.count()).toBeGreaterThan(0);
|
||||
|
||||
// Switch to the first available voice and assert processing -> playing
|
||||
await selectVoiceAndAssertPlayback(page, /.*/);
|
||||
// Step 1: Select af_bella (adds it to the multi-select list)
|
||||
await selectVoiceAndAssertPlayback(page, 'af_bella');
|
||||
|
||||
// Step 2: Deselect the first (initially selected) voice so that only af_bella remains
|
||||
await openVoicesMenu(page);
|
||||
const selected = page.locator('[role="option"][aria-selected="true"]');
|
||||
const count = await selected.count();
|
||||
for (let i = 0; i < count; i++) {
|
||||
const opt = selected.nth(i);
|
||||
const name = (await opt.textContent())?.trim() ?? '';
|
||||
// Deselect the first selected option that is not af_bella
|
||||
if (!/af_bella/i.test(name)) {
|
||||
await opt.click();
|
||||
break;
|
||||
}
|
||||
}
|
||||
await expectProcessingTransition(page);
|
||||
|
||||
// Final state should be playing
|
||||
await expectMediaState(page, 'playing');
|
||||
});
|
||||
|
||||
test('selects multiple Kokoro voices and resumes playing', async ({ page }) => {
|
||||
// Start playback
|
||||
await playTTSAndWaitForASecond(page, 'sample.pdf');
|
||||
|
||||
// Ensure TTS controls are present
|
||||
await expect(page.getByRole('button', { name: 'Skip backward' })).toBeVisible();
|
||||
await expect(page.getByRole('button', { name: 'Skip forward' })).toBeVisible();
|
||||
|
||||
// Select first voice (e.g., bf_emma) and assert processing -> playing
|
||||
await openVoicesMenu(page);
|
||||
await selectVoiceAndAssertPlayback(page, 'bf_emma');
|
||||
|
||||
// Select second voice (e.g., af_heart) to create a multi-voice mix and assert again
|
||||
await openVoicesMenu(page);
|
||||
await selectVoiceAndAssertPlayback(page, 'af_heart');
|
||||
|
||||
// Final state should be playing
|
||||
await expectMediaState(page, 'playing');
|
||||
});
|
||||
|
||||
test('changing TTS native speed toggles processing and returns to playing', async ({ page }) => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue