diff --git a/src/app/api/tts/route.ts b/src/app/api/tts/route.ts index d15d164..3be509a 100644 --- a/src/app/api/tts/route.ts +++ b/src/app/api/tts/route.ts @@ -1,6 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import OpenAI from 'openai'; import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs'; +import { isKokoroModel, stripVoiceWeights } from '@/utils/voice'; type CustomVoice = string; type ExtendedSpeechParams = Omit & { @@ -15,11 +16,7 @@ export async function POST(req: NextRequest) { const openApiBaseUrl = req.headers.get('x-openai-base-url') || process.env.API_BASE; const provider = req.headers.get('x-tts-provider') || 'openai'; const { text, voice, speed, format, model, instructions } = await req.json(); - console.log('Received TTS request:', text, voice, speed, format, model); - - if (!openApiKey) { - return NextResponse.json({ error: 'Missing OpenAI API key' }, { status: 401 }); - } + console.log('Received TTS request:', { provider, model, voice, speed, format, hasInstructions: Boolean(instructions) }); if (!text || !voice || !speed) { return NextResponse.json({ error: 'Missing required parameters' }, { status: 400 }); @@ -27,18 +24,28 @@ export async function POST(req: NextRequest) { // Apply Deepinfra defaults if provider is deepinfra const finalModel = provider === 'deepinfra' && !model ? 'hexgrad/Kokoro-82M' : model; - const finalVoice = provider === 'deepinfra' && !voice ? 'af_bella' : voice; + const initialVoice = provider === 'deepinfra' && !voice ? 'af_bella' : voice; - // Initialize OpenAI client with abort signal + // For SDK providers (OpenAI/Deepinfra), preserve multi-voice for Kokoro models, otherwise normalize to first token + const isKokoro = isKokoroModel(finalModel); + let normalizedVoice = initialVoice; + if (!isKokoro && typeof normalizedVoice === 'string' && normalizedVoice.includes('+')) { + normalizedVoice = stripVoiceWeights(normalizedVoice.split('+')[0]); + console.log('Normalized multi-voice to single for non-Kokoro SDK provider:', normalizedVoice); + } + + // Initialize OpenAI client with abort signal (OpenAI/deepinfra) const openai = new OpenAI({ apiKey: openApiKey, baseURL: openApiBaseUrl, }); + // Unified path: all providers (openai, deepinfra, custom-openai) go through the SDK below. + // Request audio from OpenAI and pass along the abort signal const createParams: ExtendedSpeechParams = { model: finalModel || 'tts-1', - voice: finalVoice as "alloy", + voice: normalizedVoice as SpeechCreateParams['voice'], input: text, speed: speed, response_format: format === 'aac' ? 'aac' : 'mp3', @@ -51,13 +58,11 @@ export async function POST(req: NextRequest) { const response = await openai.audio.speech.create(createParams as SpeechCreateParams, { signal: req.signal }); - // Get the audio data as array buffer + // Read the audio data as an ArrayBuffer and return it with appropriate headers // This will also be aborted if the client cancels - const stream = response.body; - - // Return audio data with appropriate headers + const buffer = await response.arrayBuffer(); const contentType = format === 'aac' ? 'audio/aac' : 'audio/mpeg'; - return new NextResponse(stream, { + return new NextResponse(buffer, { headers: { 'Content-Type': contentType } diff --git a/src/app/api/tts/voices/route.ts b/src/app/api/tts/voices/route.ts index 6dcbeac..60a7252 100644 --- a/src/app/api/tts/voices/route.ts +++ b/src/app/api/tts/voices/route.ts @@ -1,4 +1,5 @@ import { NextRequest, NextResponse } from 'next/server'; +import { isKokoroModel } from '@/utils/voice'; const OPENAI_VOICES = ['alloy', 'echo', 'fable', 'onyx', 'nova', 'shimmer']; const GPT4O_MINI_VOICES = ['alloy', 'ash', 'coral', 'echo', 'fable', 'onyx', 'nova', 'sage', 'shimmer']; @@ -29,6 +30,10 @@ function getDefaultVoices(provider: string, model: string): string[] { // For Custom OpenAI-Like provider if (provider === 'custom-openai') { + // If using Kokoro-FastAPI (model string contains 'kokoro'), expose full Kokoro voices + if (isKokoroModel(model)) { + return KOKORO_VOICES; + } return CUSTOM_OPENAI_VOICES; } @@ -72,7 +77,7 @@ async function fetchDeepinfraVoices(apiKey: string): Promise { } const data = await response.json(); - //console.log('Deepinfra voices response:', data); + // Extract voice names from the response, excluding preset voices if (data.voices && Array.isArray(data.voices)) { return data.voices diff --git a/src/components/player/VoicesControl.tsx b/src/components/player/VoicesControl.tsx index ebe35f5..a3645a2 100644 --- a/src/components/player/VoicesControl.tsx +++ b/src/components/player/VoicesControl.tsx @@ -8,40 +8,135 @@ import { } from '@headlessui/react'; import { ChevronUpDownIcon, AudioWaveIcon } from '@/components/icons/Icons'; import { useConfig } from '@/contexts/ConfigContext'; +import { useEffect, useMemo, useState, useCallback } from 'react'; +import { + parseKokoroVoiceNames, + buildKokoroVoiceString, + getMaxVoicesForProvider, + isKokoroModel +} from '@/utils/voice'; export const VoicesControl = ({ availableVoices, setVoiceAndRestart }: { availableVoices: string[]; setVoiceAndRestart: (voice: string) => void; }) => { - const { voice: configVoice } = useConfig(); + const { voice: configVoice, ttsModel, ttsProvider } = useConfig(); - // If the saved voice is not in the available list, use the first available voice - const currentVoice = (configVoice && availableVoices.includes(configVoice)) - ? configVoice - : availableVoices[0] || ''; + const isKokoro = isKokoroModel(ttsModel); + const maxVoices = getMaxVoicesForProvider(ttsProvider, ttsModel || ''); + + const clampToLimit = useCallback((names: string[]): string[] => { + if (maxVoices === Infinity) return names; + if (names.length <= maxVoices) return names; + // For initial clamp, keep the first up to max allowed + return names.slice(0, maxVoices); + }, [maxVoices]); + + // Local selection state for Kokoro multi-select + const [selectedVoices, setSelectedVoices] = useState([]); + + useEffect(() => { + if (!isKokoro) return; + let initial: string[] = []; + if (configVoice && configVoice.includes('+')) { + initial = parseKokoroVoiceNames(configVoice); + } else if (configVoice && availableVoices.includes(configVoice)) { + initial = [configVoice]; + } else if (availableVoices.length > 0) { + initial = [availableVoices[0]]; + } + setSelectedVoices(clampToLimit(initial)); + }, [isKokoro, configVoice, availableVoices, maxVoices, clampToLimit]); + + // If the saved voice is not in the available list, use the first available voice (non-Kokoro) + const currentVoice = useMemo(() => { + if (isKokoro) { + const combined = buildKokoroVoiceString(selectedVoices); + return combined || (availableVoices[0] || ''); + } + return (configVoice && availableVoices.includes(configVoice)) + ? configVoice + : availableVoices[0] || ''; + }, [isKokoro, selectedVoices, availableVoices, configVoice]); return (
- - - - {currentVoice} - - - - {availableVoices.map((voiceId) => ( - - `relative cursor-pointer select-none py-0.5 px-1.5 sm:py-2 sm:px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium' : ''}` + {isKokoro ? ( + { + if (!vals || vals.length === 0) return; // prevent empty selection + + let next = vals; + + // Enforce deepinfra max selection of 2 voices + if (maxVoices !== Infinity && vals.length > maxVoices) { + // Determine the newly added voice + const newlyAdded = vals.find(v => !selectedVoices.includes(v)); + if (newlyAdded) { + const lastPrev = selectedVoices[selectedVoices.length - 1] ?? selectedVoices[0] ?? ''; + // Build next as [last previously selected, newly added], deduped, limited to max + const pair = Array.from(new Set([lastPrev, newlyAdded])).filter(Boolean); + next = pair.slice(0, maxVoices); + } else { + // Fallback: keep the last maxVoices options + next = vals.slice(-maxVoices); } - > - {voiceId} - - ))} - - + } + + setSelectedVoices(next); + const combined = buildKokoroVoiceString(next); + if (combined) { + setVoiceAndRestart(combined); + } + }} + > + + + + {selectedVoices.length > 1 + ? selectedVoices.join(' + ') + : selectedVoices[0] || currentVoice} + + + + + {availableVoices.map((voiceId) => ( + + `relative cursor-pointer select-none py-1 px-2 sm:py-2 sm:px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium bg-accent text-background' : ''} ${selected && active ? 'text-foreground' : ''}` + } + > + {voiceId} + + ))} + + + ) : ( + + + + {currentVoice} + + + + {availableVoices.map((voiceId) => ( + + `relative cursor-pointer select-none py-1 px-2 sm:py-2 sm:px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium bg-accent text-background' : ''} ${selected && active ? 'text-foreground' : ''}` + } + > + {voiceId} + + ))} + + + )}
); } \ No newline at end of file diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index 5e659f8..38d2798 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -38,6 +38,7 @@ import { getLastDocumentLocation, setLastDocumentLocation } from '@/utils/indexe import { useBackgroundState } from '@/hooks/audio/useBackgroundState'; import { withRetry } from '@/utils/audio'; import { processTextToSentences } from '@/utils/nlp'; +import { isKokoroModel } from '@/utils/voice'; // Media globals declare global { @@ -155,6 +156,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const activeAbortControllers = useRef>(new Set()); // Track if we're restoring from a saved position const [pendingRestoreIndex, setPendingRestoreIndex] = useState(null); + // Guard to coalesce rapid restarts and only resume the latest change + const restartSeqRef = useRef(0); /** * Processes text into sentences using the shared NLP utility @@ -412,19 +415,31 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement */ useEffect(() => { if (availableVoices.length > 0) { + // Allow Kokoro multi-voice strings (e.g., "voice1(0.5)+voice2(0.5)") for any provider + const isKokoro = isKokoroModel(configTTSModel); + + if (isKokoro) { + // If Kokoro and we have any voice string (including plus/weights), don't override it. + // Only default when voice is empty. + if (!voice) { + setVoice(availableVoices[0]); + } + return; + } + if (!voice || !availableVoices.includes(voice)) { console.log(`Voice "${voice || '(empty)'}" not found in available voices. Using "${availableVoices[0]}"`); setVoice(availableVoices[0]); // Don't save to config - just use it temporarily until user explicitly selects one } } - }, [availableVoices, voice]); + }, [availableVoices, voice, configTTSModel]); /** * Generates and plays audio for the current sentence * * @param {string} sentence - The sentence to generate audio for - * @returns {Promise} The generated audio buffer + * @returns {Promise} The generated audio buffer */ const getAudio = useCallback(async (sentence: string): Promise => { // Check if the audio is already cached @@ -791,6 +806,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const setSpeedAndRestart = useCallback((newSpeed: number) => { const wasPlaying = isPlaying; + // Bump restart sequence to invalidate older restarts + const mySeq = ++restartSeqRef.current; + // Set a flag to prevent double audio requests during config update setIsProcessing(true); @@ -806,8 +824,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Update config after state changes updateConfigKey('voiceSpeed', newSpeed).then(() => { setIsProcessing(false); - // Resume playback if it was playing before - if (wasPlaying) { + // Resume playback if it was playing before and this is the latest restart + if (wasPlaying && mySeq === restartSeqRef.current) { setIsPlaying(true); } }); @@ -821,6 +839,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const setVoiceAndRestart = useCallback((newVoice: string) => { const wasPlaying = isPlaying; + // Bump restart sequence to invalidate older restarts + const mySeq = ++restartSeqRef.current; + // Set a flag to prevent double audio requests during config update setIsProcessing(true); @@ -836,8 +857,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Update config after state changes updateConfigKey('voice', newVoice).then(() => { setIsProcessing(false); - // Resume playback if it was playing before - if (wasPlaying) { + // Resume playback if it was playing before and this is the latest restart + if (wasPlaying && mySeq === restartSeqRef.current) { setIsPlaying(true); } }); @@ -851,6 +872,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const setAudioPlayerSpeedAndRestart = useCallback((newSpeed: number) => { const wasPlaying = isPlaying; + // Bump restart sequence to invalidate older restarts + const mySeq = ++restartSeqRef.current; + // Set a flag to prevent double audio requests during config update setIsProcessing(true); @@ -865,8 +889,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement // Update config after state changes updateConfigKey('audioPlayerSpeed', newSpeed).then(() => { setIsProcessing(false); - // Resume playback if it was playing before - if (wasPlaying) { + // Resume playback if it was playing before and this is the latest restart + if (wasPlaying && mySeq === restartSeqRef.current) { setIsPlaying(true); } }); diff --git a/src/utils/nlp.ts b/src/utils/nlp.ts index d009e98..3727820 100644 --- a/src/utils/nlp.ts +++ b/src/utils/nlp.ts @@ -42,11 +42,14 @@ export const splitIntoSentences = (text: string): string[] => { const doc = nlp(cleanedText); const rawSentences = doc.sentences().out('array') as string[]; + // Merge multi-sentence dialogue enclosed in quotes into single items + const mergedSentences = mergeQuotedDialogue(rawSentences); + let currentBlock = ''; - for (const sentence of rawSentences) { + for (const sentence of mergedSentences) { const trimmedSentence = sentence.trim(); - + if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) { blocks.push(currentBlock.trim()); currentBlock = trimmedSentence; @@ -150,4 +153,71 @@ export const processTextWithMapping = (text: string): { rawSentences, sentenceMapping }; -}; \ No newline at end of file +}; +// Helper functions to merge quoted dialogue across sentences +const countDoubleQuotes = (s: string): number => { + const matches = s.match(/["“”]/g); + return matches ? matches.length : 0; +}; + +const countCurlySingleQuotes = (s: string): number => { + const matches = s.match(/[‘’]/g); + return matches ? matches.length : 0; +}; + +const countStandaloneStraightSingles = (s: string): number => { + let count = 0; + for (let i = 0; i < s.length; i++) { + if (s[i] === "'") { + const prev = i > 0 ? s[i - 1] : ''; + const next = i + 1 < s.length ? s[i + 1] : ''; + const isPrevAlphaNum = /[A-Za-z0-9]/.test(prev); + const isNextAlphaNum = /[A-Za-z0-9]/.test(next); + // Count only when not clearly an apostrophe inside a word (e.g., don't) + if (!(isPrevAlphaNum && isNextAlphaNum)) { + count++; + } + } + } + return count; +}; + +const mergeQuotedDialogue = (rawSentences: string[]): string[] => { + const result: string[] = []; + let buffer = ''; + let insideDouble = false; + let insideSingle = false; + + for (const s of rawSentences) { + const t = s.trim(); + const dblCount = countDoubleQuotes(t); + const singleCount = countCurlySingleQuotes(t) + countStandaloneStraightSingles(t); + + if (insideDouble || insideSingle) { + buffer = buffer ? `${buffer} ${t}` : t; + } else { + // Start buffering if this sentence opens an unclosed quote + if ((dblCount % 2 === 1) || (singleCount % 2 === 1)) { + buffer = t; + } else { + result.push(t); + } + } + + // Toggle quote states after processing this sentence + if (dblCount % 2 === 1) insideDouble = !insideDouble; + if (singleCount % 2 === 1) insideSingle = !insideSingle; + + // If all open quotes are closed, flush buffer + if (!(insideDouble || insideSingle) && buffer) { + result.push(buffer); + buffer = ''; + } + } + + if (buffer) { + result.push(buffer); + } + + return result; +}; \ No newline at end of file diff --git a/src/utils/voice.ts b/src/utils/voice.ts new file mode 100644 index 0000000..52ff20e --- /dev/null +++ b/src/utils/voice.ts @@ -0,0 +1,74 @@ +/** + * Voice Utilities + * + * This module provides utilities for handling voice selection and management, + * particularly for Kokoro multi-voice syntax. + */ + +/** + * Parses a Kokoro voice string into individual voice names + * Strips weights like "af_heart(0.5)" -> "af_heart" + * + * @param voiceString - Voice string to parse (e.g., "af_heart(0.5)+bf_emma(0.5)") + * @returns Array of voice names without weights + */ +export const parseKokoroVoiceNames = (voiceString: string): string[] => + voiceString + .split('+') + .map(s => s.trim()) + .filter(Boolean) + .map(s => s.replace(/\([^)]*\)/g, '').trim()); + +/** + * Builds a Kokoro voice string from an array of voice names + * Automatically calculates equal weights for multiple voices + * + * @param names - Array of voice names + * @returns Formatted voice string with weights or single voice name + */ +export const buildKokoroVoiceString = (names: string[]): string => { + const n = names.length; + if (n === 0) return ''; + if (n === 1) return names[0]; + + const weight = 1 / n; + const weightString = weight.toFixed(2).replace(/0+$/, '').replace(/\.$/, ''); + return names.map(name => `${name}(${weightString})`).join('+'); +}; + +/** + * Checks if a model name is a Kokoro model + * + * @param modelName - TTS model name + * @returns True if the model is a Kokoro model + */ +export const isKokoroModel = (modelName: string | undefined): boolean => { + return (modelName || '').toLowerCase().includes('kokoro'); +}; + +/** + * Strips weight annotations from a voice string + * + * @param voiceString - Voice string with or without weights + * @returns Voice string without weights + */ +export const stripVoiceWeights = (voiceString: string): string => { + return voiceString.replace(/\([^)]*\)/g, '').trim(); +}; + +/** + * Determines the maximum number of voices allowed for a provider/model combination + * + * @param provider - TTS provider name + * @param model - TTS model name + * @returns Maximum number of voices (Infinity for unlimited) + */ +export const getMaxVoicesForProvider = (provider: string, model: string): number => { + if (!isKokoroModel(model)) return 1; + + // Deepinfra Kokoro supports up to 2 voices + if (provider === 'deepinfra') return 2; + + // Other providers with Kokoro support unlimited voices + return Infinity; +}; diff --git a/tests/helpers.ts b/tests/helpers.ts index 14148a8..327ec07 100644 --- a/tests/helpers.ts +++ b/tests/helpers.ts @@ -93,13 +93,15 @@ export async function pauseTTSAndVerify(page: Page) { export async function setupTest(page: Page) { // Navigate to the home page before each test await page.goto('/'); - await page.waitForLoadState('networkidle'); + //await page.waitForLoadState('networkidle'); + // If running in CI, select the "Custom OpenAI-Like" model and "Deepinfra" provider + //if (process.env.CI) { await page.getByRole('button', { name: 'Custom OpenAI-Like' }).click(); await page.getByText('Deepinfra').click(); + //} // Click the "done" button to dismiss the welcome message - await page.getByRole('tab', { name: '🔑 API' }).click(); await page.getByRole('button', { name: 'Save' }).click(); } diff --git a/tests/play.spec.ts b/tests/play.spec.ts index 2ca689a..189d9d1 100644 --- a/tests/play.spec.ts +++ b/tests/play.spec.ts @@ -7,6 +7,7 @@ import { selectVoiceAndAssertPlayback, changeNativeSpeedAndAssert, expectMediaState, + expectProcessingTransition, } from './helpers'; test.describe('Play/Pause Tests', () => { @@ -48,7 +49,7 @@ test.describe('Play/Pause Tests', () => { await pauseTTSAndVerify(page); }); - test('loads voices and switches voice with processing state then resumes play', async ({ page }) => { + test('switches to a single voice and resumes playing', async ({ page }) => { // Start playback await playTTSAndWaitForASecond(page, 'sample.pdf'); @@ -61,8 +62,46 @@ test.describe('Play/Pause Tests', () => { const options = page.getByRole('option'); expect(await options.count()).toBeGreaterThan(0); - // Switch to the first available voice and assert processing -> playing - await selectVoiceAndAssertPlayback(page, /.*/); + // Step 1: Select af_bella (adds it to the multi-select list) + await selectVoiceAndAssertPlayback(page, 'af_bella'); + + // Step 2: Deselect the first (initially selected) voice so that only af_bella remains + await openVoicesMenu(page); + const selected = page.locator('[role="option"][aria-selected="true"]'); + const count = await selected.count(); + for (let i = 0; i < count; i++) { + const opt = selected.nth(i); + const name = (await opt.textContent())?.trim() ?? ''; + // Deselect the first selected option that is not af_bella + if (!/af_bella/i.test(name)) { + await opt.click(); + break; + } + } + await expectProcessingTransition(page); + + // Final state should be playing + await expectMediaState(page, 'playing'); + }); + + test('selects multiple Kokoro voices and resumes playing', async ({ page }) => { + // Start playback + await playTTSAndWaitForASecond(page, 'sample.pdf'); + + // Ensure TTS controls are present + await expect(page.getByRole('button', { name: 'Skip backward' })).toBeVisible(); + await expect(page.getByRole('button', { name: 'Skip forward' })).toBeVisible(); + + // Select first voice (e.g., bf_emma) and assert processing -> playing + await openVoicesMenu(page); + await selectVoiceAndAssertPlayback(page, 'bf_emma'); + + // Select second voice (e.g., af_heart) to create a multi-voice mix and assert again + await openVoicesMenu(page); + await selectVoiceAndAssertPlayback(page, 'af_heart'); + + // Final state should be playing + await expectMediaState(page, 'playing'); }); test('changing TTS native speed toggles processing and returns to playing', async ({ page }) => {