diff --git a/src/app/pdf/[id]/page.tsx b/src/app/pdf/[id]/page.tsx index f9df8ca..a404dac 100644 --- a/src/app/pdf/[id]/page.tsx +++ b/src/app/pdf/[id]/page.tsx @@ -6,7 +6,6 @@ import { useParams } from 'next/navigation'; import Link from 'next/link'; import { useEffect, useState } from 'react'; import { PDFSkeleton } from '@/components/PDFSkeleton'; -import TTSPlayer from '@/components/TTSPlayer'; import { useTTS } from '@/contexts/TTSContext'; // Dynamic import for client-side rendering only @@ -73,7 +72,6 @@ export default function PDFViewerPage() { return ( <> -
diff --git a/src/components/PDFViewer.tsx b/src/components/PDFViewer.tsx index dcbf43c..7bc5f35 100644 --- a/src/components/PDFViewer.tsx +++ b/src/components/PDFViewer.tsx @@ -10,6 +10,7 @@ import { useTTS } from '@/contexts/TTSContext'; import { usePDF } from '@/contexts/PDFContext'; import { pdfjs } from 'react-pdf'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; +import TTSPlayer from '@/components/player/TTSPlayer'; interface PDFViewerProps { pdfData: Blob | undefined; @@ -19,7 +20,7 @@ interface PDFViewerProps { export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) { const [numPages, setNumPages] = useState(); const [containerWidth, setContainerWidth] = useState(0); - const { setText, currentSentence, stopAndPlayFromIndex, isProcessing, isPlaying, currentIndex, sentences } = useTTS(); + const { setText, currentSentence, stopAndPlayFromIndex, isProcessing, isPlaying, currentIndex, sentences, stop } = useTTS(); const [pdfText, setPdfText] = useState(''); const [pdfDataUrl, setPdfDataUrl] = useState(); const [loadingError, setLoadingError] = useState(); @@ -179,6 +180,14 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) { const handlePageChange = async (pageNumber: number) => { if (pageNumber < 1 || pageNumber > (numPages || 1)) return; + + const wasPlaying = isPlaying; + + // Stop current playback and reset states + if (isPlaying) { + stop(); + } + setCurrentPage(pageNumber); // Extract text from the new page @@ -240,6 +249,13 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) { setPageText(fullText.trim()); setText(fullText.trim()); // Update TTS with current page text + + // // Wait for text processing before resuming playback + // await new Promise(resolve => setTimeout(resolve, 300)); + + // if (wasPlaying) { + // stopAndPlayFromIndex(0); + // } } catch (error) { console.error('Error extracting page text:', error); } @@ -248,17 +264,16 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) { // Auto-advance to next page when TTS reaches the end useEffect(() => { - if (!isPlaying && currentIndex >= sentences.length - 1) { - handlePageChange(currentPage + 1); + if (!isPlaying && currentIndex >= sentences.length - 1 && currentPage < (numPages || 1)) { + const timer = setTimeout(() => { + handlePageChange(currentPage + 1); + }, 500); // Longer delay to ensure states are settled + return () => clearTimeout(timer); } - }, [isPlaying, currentIndex, sentences.length, currentPage]); + }, [isPlaying, currentIndex, sentences.length, currentPage, numPages]); return ( -
+
{loadingError ? (
{loadingError}
) : null} @@ -273,27 +288,6 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) { className="flex flex-col items-center m-0" >
-
- -
-

- {currentPage} / {numPages} -

-
- -
+
); } \ No newline at end of file diff --git a/src/components/Spinner.tsx b/src/components/Spinner.tsx new file mode 100644 index 0000000..1c4e0b2 --- /dev/null +++ b/src/components/Spinner.tsx @@ -0,0 +1,8 @@ +// Loading spinner component +export function LoadingSpinner() { + return ( +
+
+
+ ); +} \ No newline at end of file diff --git a/src/components/TTSPlayer.tsx b/src/components/TTSPlayer.tsx deleted file mode 100644 index e8b6ad5..0000000 --- a/src/components/TTSPlayer.tsx +++ /dev/null @@ -1,129 +0,0 @@ -'use client'; - -import { useTTS } from '@/contexts/TTSContext'; -import { Button, Listbox, ListboxButton, ListboxOptions, ListboxOption } from '@headlessui/react'; -import { - PlayIcon, - PauseIcon, - SkipForwardIcon, - SkipBackwardIcon, - ChevronUpDownIcon, -} from './icons/Icons'; - -// Loading spinner component -function LoadingSpinner() { - return ( -
-
-
- ); -} - -const speedOptions = [ - { value: 0.5, label: '0.5x' }, - { value: 0.75, label: '0.75x' }, - { value: 1, label: '1x' }, - { value: 1.25, label: '1.25x' }, - { value: 1.5, label: '1.5x' }, - { value: 1.75, label: '1.75x' }, - { value: 2, label: '2x' }, - { value: 2.5, label: '2.5x' }, - { value: 3, label: '3x' }, -]; - -export default function TTSPlayer() { - const { - isPlaying, - togglePlay, - skipForward, - skipBackward, - isProcessing, - speed, - setSpeedAndRestart, - voice, - setVoiceAndRestart, - availableVoices, - } = useTTS(); - - //console.log(availableVoices); - - return ( -
-
-
- - - {speed}x - - - - {speedOptions.map((option) => ( - - `relative cursor-pointer select-none py-2 px-3 ${active ? 'bg-offbase' : '' - } ${selected ? 'font-medium' : ''}` - } - > - {option.label} - - ))} - - -
- - - - - - - - -
- - - {voice} - - - - {availableVoices.map((voiceId) => ( - - `relative cursor-pointer select-none py-2 px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium' : ''}` - } - > - {voiceId} - - ))} - - -
-
-
- ); -} diff --git a/src/components/player/Navigator.tsx b/src/components/player/Navigator.tsx new file mode 100644 index 0000000..95cdba5 --- /dev/null +++ b/src/components/player/Navigator.tsx @@ -0,0 +1,42 @@ +import { Button } from '@headlessui/react'; + +export const Navigator = ({ currentPage, numPages, onPageChange }: { + currentPage: number; + numPages: number | undefined; + onPageChange: (page: number) => void; +}) => { + return ( +
+ {/* Page back */} + + + {/* Page number */} +
+

+ {currentPage} / {numPages || 1} +

+
+ + {/* Page forward */} + +
+ ); +} \ No newline at end of file diff --git a/src/components/player/SpeedControl.tsx b/src/components/player/SpeedControl.tsx new file mode 100644 index 0000000..e544e14 --- /dev/null +++ b/src/components/player/SpeedControl.tsx @@ -0,0 +1,49 @@ +import { + Listbox, + ListboxButton, + ListboxOption, + ListboxOptions, +} from '@headlessui/react'; +import { ChevronUpDownIcon } from '@/components/icons/Icons'; + +const speedOptions = [ + { value: 0.5, label: '0.5x' }, + { value: 0.75, label: '0.75x' }, + { value: 1, label: '1x' }, + { value: 1.25, label: '1.25x' }, + { value: 1.5, label: '1.5x' }, + { value: 1.75, label: '1.75x' }, + { value: 2, label: '2x' }, + { value: 2.5, label: '2.5x' }, + { value: 3, label: '3x' }, +]; + +export const SpeedControl = ({ speed, setSpeedAndRestart }: { + speed: number; + setSpeedAndRestart: (speed: number) => void; +}) => { + return ( +
+ + + {speed}x + + + + {speedOptions.map((option) => ( + + `relative cursor-pointer select-none py-2 px-3 ${active ? 'bg-offbase' : '' + } ${selected ? 'font-medium' : ''}` + } + > + {option.label} + + ))} + + +
+ ); +} \ No newline at end of file diff --git a/src/components/player/TTSPlayer.tsx b/src/components/player/TTSPlayer.tsx new file mode 100644 index 0000000..e178554 --- /dev/null +++ b/src/components/player/TTSPlayer.tsx @@ -0,0 +1,75 @@ +'use client'; + +import { useTTS } from '@/contexts/TTSContext'; +import { Button } from '@headlessui/react'; +import { + PlayIcon, + PauseIcon, + SkipForwardIcon, + SkipBackwardIcon, +} from '@/components/icons/Icons'; +import { LoadingSpinner } from '@/components/Spinner'; +import { VoicesControl } from '@/components/player/VoicesControl'; +import { SpeedControl } from '@/components/player/SpeedControl'; +import { Navigator } from '@/components/player/Navigator'; + +export default function TTSPlayer({ currentPage, numPages, onPageChange }: { + currentPage: number; + numPages: number | undefined; + onPageChange: (page: number) => void; +}) { + const { + isPlaying, + togglePlay, + skipForward, + skipBackward, + isProcessing, + speed, + setSpeedAndRestart, + voice, + setVoiceAndRestart, + availableVoices, + } = useTTS(); + + return ( +
+
+ {/* Speed control */} + + + {/* Page Navigation */} + + + {/* Playback Controls */} + + + + + + + {/* Voice control */} + +
+
+ ); +} diff --git a/src/components/player/VoicesControl.tsx b/src/components/player/VoicesControl.tsx new file mode 100644 index 0000000..a2e8366 --- /dev/null +++ b/src/components/player/VoicesControl.tsx @@ -0,0 +1,37 @@ +import { + Listbox, + ListboxButton, + ListboxOption, + ListboxOptions, +} from '@headlessui/react'; +import { ChevronUpDownIcon } from '@/components/icons/Icons'; + +export const VoicesControl = ({ voice, availableVoices, setVoiceAndRestart }: { + voice: string; + availableVoices: string[]; + setVoiceAndRestart: (voice: string) => void; +}) => { + return ( +
+ + + {voice} + + + + {availableVoices.map((voiceId) => ( + + `relative cursor-pointer select-none py-2 px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium' : ''}` + } + > + {voiceId} + + ))} + + +
+ ); +} \ No newline at end of file diff --git a/src/contexts/PDFContext.tsx b/src/contexts/PDFContext.tsx index 734048e..227d114 100644 --- a/src/contexts/PDFContext.tsx +++ b/src/contexts/PDFContext.tsx @@ -324,12 +324,12 @@ export function PDFProvider({ children }: { children: ReactNode }) { // Handle text click events in the PDF viewer const handleTextClick = useCallback(( event: MouseEvent, - pdfText: string, + pageText: string, // Renamed from pdfText to pageText for clarity containerRef: React.RefObject, stopAndPlayFromIndex: (index: number) => void, isProcessing: boolean ) => { - if (isProcessing) return; // Don't process clicks while TTS is processing + if (isProcessing) return; const target = event.target as HTMLElement; if (!target.matches('.react-pdf__Page__textContent span')) return; @@ -361,7 +361,8 @@ export function PDFProvider({ children }: { children: ReactNode }) { if (bestMatch.rating >= similarityThreshold) { const matchText = bestMatch.text; - const sentences = nlp(pdfText).sentences().out('array') as string[]; + // Use pageText instead of full PDF text for sentence splitting + const sentences = nlp(pageText).sentences().out('array') as string[]; let bestSentenceMatch = { sentence: '', rating: 0 }; for (const sentence of sentences) { @@ -375,7 +376,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { const sentenceIndex = sentences.findIndex((sentence) => sentence === bestSentenceMatch.sentence); if (sentenceIndex !== -1) { stopAndPlayFromIndex(sentenceIndex); - highlightPattern(pdfText, bestSentenceMatch.sentence, containerRef); + highlightPattern(pageText, bestSentenceMatch.sentence, containerRef); } } } diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index d090af4..b06f619 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -8,13 +8,15 @@ import React, { useEffect, useRef, } from 'react'; -import nlp from 'compromise'; import OpenAI from 'openai'; import { LRUCache } from 'lru-cache'; // Import LRUCache directly -import { useConfig } from './ConfigContext'; import { Howl } from 'howler'; -// Add type declarations +import { useConfig } from '@/contexts/ConfigContext'; +import { splitIntoSentences, preprocessSentenceForAudio } from '@/services/nlp'; +import { audioBufferToURL } from '@/services/audio'; + +// Media globals declare global { interface Window { webkitAudioContext: typeof AudioContext; @@ -34,12 +36,12 @@ interface TTSContextType { setText: (text: string) => void; currentSentence: string; audioQueue: AudioBuffer[]; - currentAudioIndex: number; stop: () => void; - setCurrentIndex: (index: number) => void; stopAndPlayFromIndex: (index: number) => void; sentences: string[]; isProcessing: boolean; + setIsProcessing: (value: boolean) => void; + setIsPlaying: (value: boolean) => void; speed: number; setSpeed: (speed: number) => void; setSpeedAndRestart: (speed: number) => void; @@ -47,6 +49,8 @@ interface TTSContextType { setVoice: (voice: string) => void; setVoiceAndRestart: (voice: string) => void; availableVoices: string[]; + currentIndex: number; + setCurrentIndex: (index: number) => void; } const TTSContext = createContext(undefined); @@ -66,10 +70,7 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { const currentRequestRef = useRef(null); const [activeHowl, setActiveHowl] = useState(null); const [audioQueue] = useState([]); - const [currentAudioIndex] = useState(0); const [isProcessing, setIsProcessing] = useState(false); - const skipTriggeredRef = useRef(false); - const skipTimeoutRef = useRef(null); const isPausingRef = useRef(false); const [speed, setSpeed] = useState(1); const [voice, setVoice] = useState('alloy'); @@ -78,31 +79,32 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { // Audio cache using LRUCache with a maximum size of 50 entries const audioCacheRef = useRef(new LRUCache({ max: 50 })); - // Move these function declarations up before they are used + const setText = useCallback((text: string) => { + setCurrentText(text); + const newSentences = splitIntoSentences(text); + setSentences(newSentences); + setCurrentIndex(0); + setIsPlaying(false); + + // Clear audio cache + audioCacheRef.current.clear(); + }, []); + const togglePlay = useCallback(() => { setIsPlaying((prev) => { if (!prev) { isPausingRef.current = false; return true; } else { - if (activeHowl) { - isPausingRef.current = true; - activeHowl.stop(); - setActiveHowl(null); - } + isPausingRef.current = true; + abortAudio(); return false; } }); }, [activeHowl]); + - const skipForward = useCallback(() => { - if (skipTimeoutRef.current) { - clearTimeout(skipTimeoutRef.current); - } - - skipTriggeredRef.current = true; - setIsProcessing(true); - + const abortAudio = useCallback(() => { if (currentRequestRef.current) { currentRequestRef.current.abort(); currentRequestRef.current = null; @@ -112,6 +114,12 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { activeHowl.stop(); setActiveHowl(null); } + }, [currentRequestRef, activeHowl]); + + const skipForward = useCallback(() => { + setIsProcessing(true); + + abortAudio(); setCurrentIndex((prev) => { const nextIndex = Math.min(prev + 1, sentences.length - 1); @@ -119,40 +127,22 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { return nextIndex; }); - skipTimeoutRef.current = setTimeout(() => { - skipTriggeredRef.current = false; - setIsProcessing(false); - }, 100); + setIsProcessing(false); }, [sentences, activeHowl]); const skipBackward = useCallback(() => { - if (skipTimeoutRef.current) { - clearTimeout(skipTimeoutRef.current); - } - - skipTriggeredRef.current = true; setIsProcessing(true); - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - currentRequestRef.current = null; - } - - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); - } + abortAudio(); setCurrentIndex((prev) => { const nextIndex = Math.max(prev - 1, 0); console.log('Skipping backward to:', sentences[nextIndex]); return nextIndex; }); + - skipTimeoutRef.current = setTimeout(() => { - skipTriggeredRef.current = false; - setIsProcessing(false); - }, 100); + setIsProcessing(false); }, [sentences, activeHowl]); // Initialize OpenAI instance when config loads @@ -230,31 +220,8 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { } }, [togglePlay, skipForward, skipBackward]); - // Text preprocessing function to clean and normalize text - const preprocessSentenceForAudio = (text: string): string => { - return text - // Replace URLs with descriptive text including domain - .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') - // Remove special characters except basic punctuation - //.replace(/[^\w\s.,!?;:'"()-]/g, ' ') - // Fix hyphenated words at line breaks (word- word -> wordword) - .replace(/(\w+)-\s+(\w+)/g, '$1$2') - // Replace multiple spaces with single space - .replace(/\s+/g, ' ') - // Trim whitespace - .trim(); - }; - - const splitIntoSentences = (text: string): string[] => { - // Preprocess the text before splitting into sentences - const cleanedText = preprocessSentenceForAudio(text); - const doc = nlp(cleanedText); - return doc.sentences().out('array') as string[]; - }; - - const processNextSentence = useCallback(async () => { - if (!isPlaying || currentIndex >= sentences.length - 1 || skipTriggeredRef.current) { - setIsPlaying(false); + const advance = useCallback(async () => { + if (currentIndex >= sentences.length - 1) { return; } @@ -266,74 +233,67 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { } return prev; }); - }, [isPlaying, currentIndex, sentences]); + }, [currentIndex, sentences]); - const processAndPlaySentence = async (sentence: string) => { - if (!audioContext || isProcessing || !openaiRef.current) return; + //new function to return audio buffer with caching + const getAudio = useCallback(async (sentence: string): Promise => { + // Check if the audio is already cached + const cachedAudio = audioCacheRef.current.get(sentence); + if (cachedAudio) { + console.log('Using cached audio for sentence:', sentence); + return cachedAudio; + } + + // If not cached, fetch the audio from OpenAI API + if (openaiRef.current) { + const response = await openaiRef.current.audio.speech.create({ + model: 'tts-1', + voice: voice as "alloy", + input: sentence, + speed: speed, + }); + + const arrayBuffer = await response.arrayBuffer(); + const audioBuffer = await audioContext!.decodeAudioData(arrayBuffer); + + // Cache the audio buffer + audioCacheRef.current.set(sentence, audioBuffer); + + return audioBuffer; + } + }, [audioContext, voice, speed]); + + const processSentence = async (sentence: string, preload: boolean = false): Promise => { + if (!audioContext || !openaiRef.current) throw new Error('Audio context not initialized'); + if (!preload) setIsProcessing(true); try { - // Only set processing if we need to fetch from API const cleanedSentence = preprocessSentenceForAudio(sentence); - if (!audioCacheRef.current.has(cleanedSentence)) { - setIsProcessing(true); - } - - // Cancel any existing request - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - } - - // Create new abort controller for this request currentRequestRef.current = new AbortController(); - // Stop any currently playing audio - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); + const audioBuffer = await getAudio(cleanedSentence); + if (!currentRequestRef.current) throw new Error('Request cancelled'); + + return audioBufferToURL(audioBuffer!); + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + console.log('Request was cancelled'); } + throw error; + } finally { + currentRequestRef.current = null; + } + }; - let audioBuffer = audioCacheRef.current.get(cleanedSentence); - - if (!audioBuffer) { - console.log(' Processing TTS for sentence:', cleanedSentence.substring(0, 50) + '...'); - const startTime = Date.now(); - const response = await openaiRef.current.audio.speech.create({ - model: 'tts-1', - voice: voice as "alloy", - input: cleanedSentence, - speed: speed, - }); - - const duration = Date.now() - startTime; - console.log(` TTS processing completed in ${duration}ms`); - - const arrayBuffer = await response.arrayBuffer(); - audioBuffer = await audioContext.decodeAudioData(arrayBuffer); - - // Store in cache - audioCacheRef.current.set(cleanedSentence, audioBuffer); - setIsProcessing(false); - } - - // If the request was aborted or component unmounted, do not proceed - if (!currentRequestRef.current) return; - - // Convert AudioBuffer to URL for Howler - const audioUrl = audioBufferToURL(audioBuffer!); + const playSentence = async (sentence: string) => { + try { + const audioUrl = await processSentence(sentence); + setIsProcessing(false); const howl = new Howl({ src: [audioUrl], format: ['wav'], html5: true, - onend: () => { - setActiveHowl(null); - // Cleanup the URL when audio ends - URL.revokeObjectURL(audioUrl); - if (isPlaying && !skipTriggeredRef.current && !isPausingRef.current) { - processNextSentence(); - } - isPausingRef.current = false; - }, onplay: () => { if ('mediaSession' in navigator) { navigator.mediaSession.playbackState = 'playing'; @@ -344,117 +304,39 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { navigator.mediaSession.playbackState = 'paused'; } }, + onend: () => { + URL.revokeObjectURL(audioUrl); + setActiveHowl(null); + if (isPlaying && !isPausingRef.current) { + advance(); + } + isPausingRef.current = false; + }, + onloaderror: (id, error) => { + console.error('Error loading audio:', error); + setIsProcessing(false); + setActiveHowl(null); + URL.revokeObjectURL(audioUrl); + advance(); + }, }); setActiveHowl(howl); howl.play(); - } catch (error: unknown) { - if (error instanceof Error && error.name === 'AbortError') { - console.log('Request was cancelled'); - } else { - console.error('Error processing TTS:', error); - } + } catch (error) { + console.error('Error playing TTS:', error); setActiveHowl(null); setIsProcessing(false); - } finally { - currentRequestRef.current = null; - } - }; - - // Add utility function to convert AudioBuffer to URL - const audioBufferToURL = (audioBuffer: AudioBuffer): string => { - // Get WAV file bytes - const wavBytes = getWavBytes(audioBuffer.getChannelData(0), { - isFloat: true, // floating point or 16-bit integer - numChannels: 1, // number of channels - sampleRate: audioBuffer.sampleRate, // audio sample rate - }); - - // Create blob and URL - const blob = new Blob([wavBytes], { type: 'audio/wav' }); - return URL.createObjectURL(blob); - }; - - // Add helper function for WAV conversion - const getWavBytes = (samples: Float32Array, opts: { - isFloat?: boolean, - numChannels?: number, - sampleRate?: number, - }) => { - const { - isFloat = true, - numChannels = 1, - sampleRate = 44100, - } = opts; - - const bytesPerSample = isFloat ? 4 : 2; - const numSamples = samples.length; - - // WAV header size is 44 bytes - const buffer = new ArrayBuffer(44 + numSamples * bytesPerSample); - const dv = new DataView(buffer); - - let pos = 0; - - // Write WAV header - writeString(dv, pos, 'RIFF'); pos += 4; - dv.setUint32(pos, 36 + numSamples * bytesPerSample, true); pos += 4; - writeString(dv, pos, 'WAVE'); pos += 4; - writeString(dv, pos, 'fmt '); pos += 4; - dv.setUint32(pos, 16, true); pos += 4; - dv.setUint16(pos, isFloat ? 3 : 1, true); pos += 2; - dv.setUint16(pos, numChannels, true); pos += 2; - dv.setUint32(pos, sampleRate, true); pos += 4; - dv.setUint32(pos, sampleRate * numChannels * bytesPerSample, true); pos += 4; - dv.setUint16(pos, numChannels * bytesPerSample, true); pos += 2; - dv.setUint16(pos, bytesPerSample * 8, true); pos += 2; - writeString(dv, pos, 'data'); pos += 4; - dv.setUint32(pos, numSamples * bytesPerSample, true); pos += 4; - - if (isFloat) { - for (let i = 0; i < numSamples; i++) { - dv.setFloat32(pos, samples[i], true); - pos += bytesPerSample; - } - } else { - for (let i = 0; i < numSamples; i++) { - const s = Math.max(-1, Math.min(1, samples[i])); - dv.setInt16(pos, s < 0 ? s * 0x8000 : s * 0x7FFF, true); - pos += bytesPerSample; + if (error instanceof Error && + (error.message.includes('Unable to decode audio data') || + error.message.includes('EncodingError'))) { + console.log('Skipping problematic sentence:', sentence.substring(0, 50) + '...'); + advance(); } } - - return buffer; }; - const writeString = (view: DataView, offset: number, string: string): void => { - for (let i = 0; i < string.length; i++) { - view.setUint8(offset + i, string.charCodeAt(i)); - } - }; - - const setText = useCallback((text: string) => { - setCurrentText(text); - const newSentences = splitIntoSentences(text); - setSentences(newSentences); - setCurrentIndex(0); - setIsPlaying(false); - - // Clear audio cache - audioCacheRef.current.clear(); - - // Preload the first sentence immediately - if (newSentences.length > 0) { - preloadSentence(newSentences[0]).then(() => { - // Preload the second sentence after a small delay - if (newSentences[1]) { - setTimeout(() => preloadSentence(newSentences[1]), 200); - } - }); - } - }, []); - // Preload adjacent sentences when currentIndex changes useEffect(() => { /* @@ -470,7 +352,7 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { // Only preload next sentence to reduce API load if (sentences[currentIndex + 1] && !audioCacheRef.current.has(sentences[currentIndex + 1])) { await new Promise((resolve) => setTimeout(resolve, 200)); // Add small delay - await preloadSentence(sentences[currentIndex + 1]); + await processSentence(sentences[currentIndex + 1], true); // True indicates preloading } } catch (error) { console.error('Error preloading adjacent sentences:', error); @@ -479,94 +361,26 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { preloadAdjacentSentences(); }, [currentIndex, sentences]); - const isMounted = useRef(false); - + // Add this useEffect before the return statement in TTSProvider useEffect(() => { - /* - * Plays the current sentence when the component is mounted or the currentIndex changes. - * Handles audio playback and auto-advances to the next sentence when finished. - * - * Dependencies: - * - isPlaying: Re-runs when the isPlaying state changes - * - currentIndex: Re-runs when the currentIndex changes - * - sentences: Re-runs when the sentences array changes - * - isProcessing: Re-runs when the isProcessing state changes - */ - // Skip the first mount in development - if (process.env.NODE_ENV === 'development') { - if (!isMounted.current) { - isMounted.current = true; - return; - } - } - - let isEffectActive = true; - const playAudio = async () => { - if (isPlaying && sentences[currentIndex] && !isProcessing && isEffectActive) { - await processAndPlaySentence(sentences[currentIndex]); + if (isPlaying && sentences[currentIndex] && !isProcessing) { + await playSentence(sentences[currentIndex]); } }; playAudio(); - - return () => { - isEffectActive = false; - // Clean up any playing audio when the effect is cleaned up - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); - } - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - currentRequestRef.current = null; - } - }; }, [isPlaying, currentIndex, sentences, isProcessing]); - const preloadSentence = async (sentence: string) => { - if (!audioContext || !openaiRef.current) return; - if (audioCacheRef.current.has(sentence)) return; // Already cached - - try { - console.log(' Preloading TTS for sentence:', sentence.substring(0, 50) + '...'); - const startTime = Date.now(); - const response = await openaiRef.current.audio.speech.create({ - model: 'tts-1', - voice: voice as "alloy", - input: sentence, - speed: speed, - }); - - const duration = Date.now() - startTime; - console.log(` Preload TTS completed in ${duration}ms`); - - const arrayBuffer = await response.arrayBuffer(); - const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); - - // Store in cache - audioCacheRef.current.set(sentence, audioBuffer); - } catch (error: unknown) { - if (error instanceof Error && error.name === 'AbortError') { - console.log('Request was cancelled'); - } else { - console.error('Error preloading TTS:', error); - } - } - }; + // const playAudio = useCallback(async () => { + // if (isPlaying && sentences[currentIndex] && !isProcessing) { + // await processAndPlaySentence(sentences[currentIndex]); + // } + // }, [isPlaying, sentences, currentIndex, isProcessing]); const stop = useCallback(() => { // Cancel any ongoing request - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - currentRequestRef.current = null; - } - - // Stop current audio - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); - } + abortAudio(); setIsPlaying(false); setCurrentIndex(0); @@ -575,49 +389,19 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { }, [activeHowl]); const stopAndPlayFromIndex = useCallback((index: number) => { - // Cancel any ongoing request - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - currentRequestRef.current = null; - } - - // Stop current audio - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); - } - - // Set skip flag to prevent immediate auto-advance - skipTriggeredRef.current = true; - - // Set new index and start playing - setCurrentIndex(index); - setIsPlaying(true); - - // Reset skip flag after a short delay to allow future auto-advance - if (skipTimeoutRef.current) { - clearTimeout(skipTimeoutRef.current); - } - skipTimeoutRef.current = setTimeout(() => { - skipTriggeredRef.current = false; - }, 100); - }, [activeHowl]); + abortAudio(); + + // Set the states in the next tick to ensure clean state + setTimeout(() => { + setCurrentIndex(index); + setIsPlaying(true); + }, 50); + }, []); const setCurrentIndexWithoutPlay = useCallback((index: number) => { - // Cancel any ongoing request - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - currentRequestRef.current = null; - } - - // Stop current audio - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); - } + abortAudio(); setCurrentIndex(index); - skipTriggeredRef.current = false; }, [activeHowl]); const setSpeedAndRestart = useCallback((newSpeed: number) => { @@ -661,12 +445,13 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { setText, currentSentence: sentences[currentIndex] || '', audioQueue, - currentAudioIndex, stop, setCurrentIndex: setCurrentIndexWithoutPlay, stopAndPlayFromIndex, sentences, isProcessing, + setIsProcessing, + setIsPlaying, speed, setSpeed, setSpeedAndRestart, @@ -674,6 +459,7 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { setVoice, setVoiceAndRestart, availableVoices, + currentIndex, }; if (configIsLoading) { diff --git a/src/services/audio.ts b/src/services/audio.ts new file mode 100644 index 0000000..ba4d097 --- /dev/null +++ b/src/services/audio.ts @@ -0,0 +1,71 @@ +// Add utility function to convert AudioBuffer to URL +export const audioBufferToURL = (audioBuffer: AudioBuffer): string => { + // Get WAV file bytes + const wavBytes = getWavBytes(audioBuffer.getChannelData(0), { + isFloat: true, // floating point or 16-bit integer + numChannels: 1, // number of channels + sampleRate: audioBuffer.sampleRate, // audio sample rate + }); + + // Create blob and URL + const blob = new Blob([wavBytes], { type: 'audio/wav' }); + return URL.createObjectURL(blob); +}; + +// Add helper function for WAV conversion +export const getWavBytes = (samples: Float32Array, opts: { + isFloat?: boolean, + numChannels?: number, + sampleRate?: number, +}) => { + const { + isFloat = true, + numChannels = 1, + sampleRate = 44100, + } = opts; + + const bytesPerSample = isFloat ? 4 : 2; + const numSamples = samples.length; + + // WAV header size is 44 bytes + const buffer = new ArrayBuffer(44 + numSamples * bytesPerSample); + const dv = new DataView(buffer); + + let pos = 0; + + // Write WAV header + writeString(dv, pos, 'RIFF'); pos += 4; + dv.setUint32(pos, 36 + numSamples * bytesPerSample, true); pos += 4; + writeString(dv, pos, 'WAVE'); pos += 4; + writeString(dv, pos, 'fmt '); pos += 4; + dv.setUint32(pos, 16, true); pos += 4; + dv.setUint16(pos, isFloat ? 3 : 1, true); pos += 2; + dv.setUint16(pos, numChannels, true); pos += 2; + dv.setUint32(pos, sampleRate, true); pos += 4; + dv.setUint32(pos, sampleRate * numChannels * bytesPerSample, true); pos += 4; + dv.setUint16(pos, numChannels * bytesPerSample, true); pos += 2; + dv.setUint16(pos, bytesPerSample * 8, true); pos += 2; + writeString(dv, pos, 'data'); pos += 4; + dv.setUint32(pos, numSamples * bytesPerSample, true); pos += 4; + + if (isFloat) { + for (let i = 0; i < numSamples; i++) { + dv.setFloat32(pos, samples[i], true); + pos += bytesPerSample; + } + } else { + for (let i = 0; i < numSamples; i++) { + const s = Math.max(-1, Math.min(1, samples[i])); + dv.setInt16(pos, s < 0 ? s * 0x8000 : s * 0x7FFF, true); + pos += bytesPerSample; + } + } + + return buffer; +}; + +export const writeString = (view: DataView, offset: number, string: string): void => { + for (let i = 0; i < string.length; i++) { + view.setUint8(offset + i, string.charCodeAt(i)); + } +}; \ No newline at end of file diff --git a/src/services/nlp.ts b/src/services/nlp.ts new file mode 100644 index 0000000..d702be9 --- /dev/null +++ b/src/services/nlp.ts @@ -0,0 +1,23 @@ +import nlp from 'compromise'; + +// Text preprocessing function to clean and normalize text +export const preprocessSentenceForAudio = (text: string): string => { + return text + // Replace URLs with descriptive text including domain + .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') + // Remove special characters except basic punctuation + //.replace(/[^\w\s.,!?;:'"()-]/g, ' ') + // Fix hyphenated words at line breaks (word- word -> wordword) + .replace(/(\w+)-\s+(\w+)/g, '$1$2') + // Replace multiple spaces with single space + .replace(/\s+/g, ' ') + // Trim whitespace + .trim(); +}; + +export const splitIntoSentences = (text: string): string[] => { + // Preprocess the text before splitting into sentences + const cleanedText = preprocessSentenceForAudio(text); + const doc = nlp(cleanedText); + return doc.sentences().out('array') as string[]; +}; \ No newline at end of file