fix(audio): improve Firefox compatibility and audio caching

- Replace base64 data URIs with Blob URLs for better Firefox compatibility
  - Fixes audio playback issues in Mozilla Firefox
  - More efficient memory usage and better cross-browser support
  - Add proper Blob URL cleanup to prevent memory leaks

- Enhance audio preloading from 1 to 3 sentences ahead
  - Reduces waiting time between sentences during TTS playback
  - Improves overall user experience with smoother audio transitions
  - Maintains existing cache management and abort control logic

Changes:
- Add activeBlobUrls ref to track and cleanup Blob URLs
- Convert ArrayBuffer to Blob URL instead of base64 data URI in processSentence
- Revoke Blob URLs in onend/onstop handlers and abortAudio
- Update preloadNextAudio to buffer 3 sentences ahead instead of 1
This commit is contained in:
Claude 2025-12-31 10:50:56 +00:00
parent d5ec96d395
commit 453765076c
No known key found for this signature in database

View file

@ -350,6 +350,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
const preloadRequests = useRef<Map<string, Promise<string>>>(new Map()); const preloadRequests = useRef<Map<string, Promise<string>>>(new Map());
// Track active abort controllers for TTS requests // Track active abort controllers for TTS requests
const activeAbortControllers = useRef<Set<AbortController>>(new Set()); const activeAbortControllers = useRef<Set<AbortController>>(new Set());
// Track active Blob URLs for cleanup
const activeBlobUrls = useRef<Set<string>>(new Set());
// Track if we're restoring from a saved position // Track if we're restoring from a saved position
const [pendingRestoreIndex, setPendingRestoreIndex] = useState<number | null>(null); const [pendingRestoreIndex, setPendingRestoreIndex] = useState<number | null>(null);
// Guard to coalesce rapid restarts and only resume the latest change // Guard to coalesce rapid restarts and only resume the latest change
@ -405,6 +407,12 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
}); });
activeAbortControllers.current.clear(); activeAbortControllers.current.clear();
preloadRequests.current.clear(); preloadRequests.current.clear();
// Revoke all active Blob URLs to prevent memory leaks
activeBlobUrls.current.forEach(url => {
URL.revokeObjectURL(url);
});
activeBlobUrls.current.clear();
} }
if (pageTurnTimeoutRef.current) { if (pageTurnTimeoutRef.current) {
@ -931,11 +939,14 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
const audioBuffer = await getAudio(sentence); const audioBuffer = await getAudio(sentence);
if (!audioBuffer) throw new Error('No audio data generated'); if (!audioBuffer) throw new Error('No audio data generated');
// Convert to base64 data URI // Convert to Blob URL (better browser compatibility, especially Firefox)
const bytes = new Uint8Array(audioBuffer); const blob = new Blob([audioBuffer], { type: 'audio/mp3' });
const binaryString = bytes.reduce((acc, byte) => acc + String.fromCharCode(byte), ''); const blobUrl = URL.createObjectURL(blob);
const base64String = btoa(binaryString);
return `data:audio/mp3;base64,${base64String}`; // Track Blob URL for cleanup
activeBlobUrls.current.add(blobUrl);
return blobUrl;
} catch (error) { } catch (error) {
setIsProcessing(false); setIsProcessing(false);
throw error; throw error;
@ -971,9 +982,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
const createHowl = async (retryCount = 0): Promise<Howl | null> => { const createHowl = async (retryCount = 0): Promise<Howl | null> => {
try { try {
// Get the processed audio data URI directly from processSentence // Get the processed audio Blob URL directly from processSentence
const audioDataUri = await processSentence(sentence); const audioBlobUrl = await processSentence(sentence);
if (!audioDataUri) { if (!audioBlobUrl) {
throw new Error('No audio data generated'); throw new Error('No audio data generated');
} }
@ -983,7 +994,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
} }
return new Howl({ return new Howl({
src: [audioDataUri], src: [audioBlobUrl],
format: ['mp3', 'mpeg'], format: ['mp3', 'mpeg'],
html5: true, html5: true,
preload: true, preload: true,
@ -1067,6 +1078,13 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
onend: function (this: Howl) { onend: function (this: Howl) {
this.unload(); this.unload();
setActiveHowl(null); setActiveHowl(null);
// Revoke the Blob URL to free memory
if (activeBlobUrls.current.has(audioBlobUrl)) {
URL.revokeObjectURL(audioBlobUrl);
activeBlobUrls.current.delete(audioBlobUrl);
}
if (pageTurnTimeoutRef.current) { if (pageTurnTimeoutRef.current) {
clearTimeout(pageTurnTimeoutRef.current); clearTimeout(pageTurnTimeoutRef.current);
pageTurnTimeoutRef.current = null; pageTurnTimeoutRef.current = null;
@ -1078,6 +1096,12 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
onstop: function (this: Howl) { onstop: function (this: Howl) {
setIsProcessing(false); setIsProcessing(false);
this.unload(); this.unload();
// Revoke the Blob URL to free memory
if (activeBlobUrls.current.has(audioBlobUrl)) {
URL.revokeObjectURL(audioBlobUrl);
activeBlobUrls.current.delete(audioBlobUrl);
}
} }
}); });
} catch (error) { } catch (error) {
@ -1186,12 +1210,17 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
}, [activeHowl, isPlaying, currentSentenceAlignment]); }, [activeHowl, isPlaying, currentSentenceAlignment]);
/** /**
* Preloads the next sentence's audio * Preloads the next 3 sentences' audio for smoother playback
*/ */
const preloadNextAudio = useCallback(async () => { const preloadNextAudio = useCallback(async () => {
try { try {
const nextSentence = sentences[currentIndex + 1]; // Preload the next 3 sentences to ensure smooth playback
if (nextSentence) { const PRELOAD_AHEAD_COUNT = 3;
for (let i = 1; i <= PRELOAD_AHEAD_COUNT; i++) {
const nextSentence = sentences[currentIndex + i];
if (!nextSentence) break; // Stop if we've reached the end
const nextKey = buildCacheKey( const nextKey = buildCacheKey(
nextSentence, nextSentence,
voice, voice,
@ -1201,9 +1230,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
); );
if (!audioCache.has(nextKey) && !preloadRequests.current.has(nextSentence)) { if (!audioCache.has(nextKey) && !preloadRequests.current.has(nextSentence)) {
// Start preloading but don't wait for it to complete // Start preloading but don't wait for it to complete
processSentence(nextSentence, true).catch(error => { processSentence(nextSentence, true).catch(error => {
console.error('Error preloading next sentence:', error); console.error(`Error preloading sentence ${i} ahead:`, error);
}); });
} }
} }