use howl for audio playback

package-lock.json

@@ -9,9 +9,11 @@
       "version": "0.1.0",
       "dependencies": {
         "@headlessui/react": "^2.2.0",
+        "@types/howler": "^2.2.12",
         "@types/string-similarity": "^4.0.2",
         "@types/uuid": "^10.0.0",
         "compromise": "^14.14.4",
+        "howler": "^2.2.4",
         "lru-cache": "^11.0.2",
         "next": "15.1.5",
         "openai": "^4.79.1",
@@ -1108,6 +1110,12 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/@types/howler": {
+      "version": "2.2.12",
+      "resolved": "https://registry.npmjs.org/@types/howler/-/howler-2.2.12.tgz",
+      "integrity": "sha512-hy769UICzOSdK0Kn1FBk4gN+lswcj1EKRkmiDtMkUGvFfYJzgaDXmVXkSShS2m89ERAatGIPnTUlp2HhfkVo5g==",
+      "license": "MIT"
+    },
     "node_modules/@types/json-schema": {
       "version": "7.0.15",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
@@ -3659,6 +3667,12 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/howler": {
+      "version": "2.2.4",
+      "resolved": "https://registry.npmjs.org/howler/-/howler-2.2.4.tgz",
+      "integrity": "sha512-iARIBPgcQrwtEr+tALF+rapJ8qSc+Set2GJQl7xT1MQzWaVkFebdJhR3alVlSiUf5U7nAANKuj3aWpwerocD5w==",
+      "license": "MIT"
+    },
     "node_modules/humanize-ms": {
       "version": "1.2.1",
       "resolved": "https://registry.npmjs.org/humanize-ms/-/humanize-ms-1.2.1.tgz",

package.json

@@ -5,14 +5,16 @@
   "scripts": {
     "dev": "next dev --turbopack",
     "build": "next build",
-    "start": "next start -p 3002",
+    "start": "next start -p 3003",
     "lint": "next lint"
   },
   "dependencies": {
     "@headlessui/react": "^2.2.0",
+    "@types/howler": "^2.2.12",
     "@types/string-similarity": "^4.0.2",
     "@types/uuid": "^10.0.0",
     "compromise": "^14.14.4",
+    "howler": "^2.2.4",
     "lru-cache": "^11.0.2",
     "next": "15.1.5",
     "openai": "^4.79.1",

src/contexts/TTSContext.tsx

@@ -12,6 +12,7 @@ import nlp from 'compromise';
 import OpenAI from 'openai';
 import { LRUCache } from 'lru-cache'; // Import LRUCache directly
 import { useConfig } from './ConfigContext';
+import { Howl } from 'howler';
 
 // Add type declarations
 declare global {
@@ -22,7 +23,7 @@ declare global {
 
 type AudioContextType = typeof window extends undefined
   ? never
-  : AudioContext | null;
+  : (AudioContext | null);
 
 interface TTSContextType {
   isPlaying: boolean;
@@ -63,7 +64,7 @@ export function TTSProvider({ children }: { children: React.ReactNode }) {
   const [currentIndex, setCurrentIndex] = useState(0);
   const [audioContext, setAudioContext] = useState<AudioContextType>(null);
   const currentRequestRef = useRef<AbortController | null>(null);
-  const [activeSource, setActiveSource] = useState<AudioBufferSourceNode | null>(null);
+  const [activeHowl, setActiveHowl] = useState<Howl | null>(null);
   const [audioQueue] = useState<AudioBuffer[]>([]);
   const [currentAudioIndex] = useState(0);
   const [isProcessing, setIsProcessing] = useState(false);
@@ -77,6 +78,83 @@ export function TTSProvider({ children }: { children: React.ReactNode }) {
   // Audio cache using LRUCache with a maximum size of 50 entries
   const audioCacheRef = useRef(new LRUCache<string, AudioBuffer>({ max: 50 }));
 
+  // Move these function declarations up before they are used
+  const togglePlay = useCallback(() => {
+    setIsPlaying((prev) => {
+      if (!prev) {
+        isPausingRef.current = false;
+        return true;
+      } else {
+        if (activeHowl) {
+          isPausingRef.current = true;
+          activeHowl.stop();
+          setActiveHowl(null);
+        }
+        return false;
+      }
+    });
+  }, [activeHowl]);
+
+  const skipForward = useCallback(() => {
+    if (skipTimeoutRef.current) {
+      clearTimeout(skipTimeoutRef.current);
+    }
+
+    skipTriggeredRef.current = true;
+    setIsProcessing(true);
+
+    if (currentRequestRef.current) {
+      currentRequestRef.current.abort();
+      currentRequestRef.current = null;
+    }
+
+    if (activeHowl) {
+      activeHowl.stop();
+      setActiveHowl(null);
+    }
+
+    setCurrentIndex((prev) => {
+      const nextIndex = Math.min(prev + 1, sentences.length - 1);
+      console.log('Skipping forward to:', sentences[nextIndex]);
+      return nextIndex;
+    });
+
+    skipTimeoutRef.current = setTimeout(() => {
+      skipTriggeredRef.current = false;
+      setIsProcessing(false);
+    }, 100);
+  }, [sentences, activeHowl]);
+
+  const skipBackward = useCallback(() => {
+    if (skipTimeoutRef.current) {
+      clearTimeout(skipTimeoutRef.current);
+    }
+
+    skipTriggeredRef.current = true;
+    setIsProcessing(true);
+
+    if (currentRequestRef.current) {
+      currentRequestRef.current.abort();
+      currentRequestRef.current = null;
+    }
+
+    if (activeHowl) {
+      activeHowl.stop();
+      setActiveHowl(null);
+    }
+
+    setCurrentIndex((prev) => {
+      const nextIndex = Math.max(prev - 1, 0);
+      console.log('Skipping backward to:', sentences[nextIndex]);
+      return nextIndex;
+    });
+
+    skipTimeoutRef.current = setTimeout(() => {
+      skipTriggeredRef.current = false;
+      setIsProcessing(false);
+    }, 100);
+  }, [sentences, activeHowl]);
+
   // Initialize OpenAI instance when config loads
   useEffect(() => {
     const fetchVoices = async () => {
@@ -130,6 +208,28 @@ export function TTSProvider({ children }: { children: React.ReactNode }) {
     }
   }, [audioContext]);
 
+  // Now the MediaSession effect can use these functions
+  useEffect(() => {
+    if ('mediaSession' in navigator) {
+      navigator.mediaSession.metadata = new MediaMetadata({
+        title: 'Text to Speech',
+        artist: 'OpenReader WebUI',
+        album: 'Current Reading',
+        artwork: [
+          {
+            src: 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=',
+            type: 'image/png',
+          },
+        ],
+      });
+
+      navigator.mediaSession.setActionHandler('play', () => togglePlay());
+      navigator.mediaSession.setActionHandler('pause', () => togglePlay());
+      navigator.mediaSession.setActionHandler('nexttrack', () => skipForward());
+      navigator.mediaSession.setActionHandler('previoustrack', () => skipBackward());
+    }
+  }, [togglePlay, skipForward, skipBackward]);
+
   // Text preprocessing function to clean and normalize text
   const preprocessSentenceForAudio = (text: string): string => {
     return text
@@ -187,9 +287,9 @@ export function TTSProvider({ children }: { children: React.ReactNode }) {
       currentRequestRef.current = new AbortController();
 
       // Stop any currently playing audio
-      if (activeSource) {
+      if (activeHowl) {
-        activeSource.stop();
+        activeHowl.stop();
-        setActiveSource(null);
+        setActiveHowl(null);
       }
 
       let audioBuffer = audioCacheRef.current.get(cleanedSentence);
@@ -218,114 +318,121 @@ export function TTSProvider({ children }: { children: React.ReactNode }) {
       // If the request was aborted or component unmounted, do not proceed
       if (!currentRequestRef.current) return;
 
-      const source = audioContext.createBufferSource();
+      // Convert AudioBuffer to URL for Howler
-      source.buffer = audioBuffer;
+      const audioUrl = audioBufferToURL(audioBuffer!);
-      source.connect(audioContext.destination);
-      setActiveSource(source);
 
-      // Set up onended handler before starting
+      const howl = new Howl({
-      source.onended = () => {
+        src: [audioUrl],
-        setActiveSource(null);
+        format: ['wav'],
-        // Only advance if we're playing and not pausing or skipping
+        html5: true,
-        if (isPlaying && !skipTriggeredRef.current && !isPausingRef.current) {
+        onend: () => {
-          processNextSentence();
+          setActiveHowl(null);
-        }
+          // Cleanup the URL when audio ends
-        isPausingRef.current = false; // Reset pause flag
+          URL.revokeObjectURL(audioUrl);
-      };
+          if (isPlaying && !skipTriggeredRef.current && !isPausingRef.current) {
+            processNextSentence();
+          }
+          isPausingRef.current = false;
+        },
+        onplay: () => {
+          if ('mediaSession' in navigator) {
+            navigator.mediaSession.playbackState = 'playing';
+          }
+        },
+        onpause: () => {
+          if ('mediaSession' in navigator) {
+            navigator.mediaSession.playbackState = 'paused';
+          }
+        },
+      });
+
+      setActiveHowl(howl);
+      howl.play();
 
-      source.start(0);
     } catch (error: unknown) {
       if (error instanceof Error && error.name === 'AbortError') {
         console.log('Request was cancelled');
       } else {
         console.error('Error processing TTS:', error);
       }
-      setActiveSource(null);
+      setActiveHowl(null);
       setIsProcessing(false);
     } finally {
       currentRequestRef.current = null;
     }
   };
 
-  const togglePlay = useCallback(() => {
+  // Add utility function to convert AudioBuffer to URL
-    setIsPlaying((prev) => {
+  const audioBufferToURL = (audioBuffer: AudioBuffer): string => {
-      if (!prev) {
+    // Get WAV file bytes
-        isPausingRef.current = false;
+    const wavBytes = getWavBytes(audioBuffer.getChannelData(0), {
-        return true;
+      isFloat: true,       // floating point or 16-bit integer
-      } else {
+      numChannels: 1,      // number of channels
-        if (activeSource) {
+      sampleRate: audioBuffer.sampleRate,    // audio sample rate
-          isPausingRef.current = true;
+    });
-          activeSource.stop();
+
-          setActiveSource(null);
+    // Create blob and URL
-        }
+    const blob = new Blob([wavBytes], { type: 'audio/wav' });
-        return false;
+    return URL.createObjectURL(blob);
+  };
+
+  // Add helper function for WAV conversion
+  const getWavBytes = (samples: Float32Array, opts: {
+    isFloat?: boolean,
+    numChannels?: number,
+    sampleRate?: number,
+  }) => {
+    const {
+      isFloat = true,
+      numChannels = 1,
+      sampleRate = 44100,
+    } = opts;
+
+    const bytesPerSample = isFloat ? 4 : 2;
+    const numSamples = samples.length;
+
+    // WAV header size is 44 bytes
+    const buffer = new ArrayBuffer(44 + numSamples * bytesPerSample);
+    const dv = new DataView(buffer);
+
+    let pos = 0;
+
+    // Write WAV header
+    writeString(dv, pos, 'RIFF'); pos += 4;
+    dv.setUint32(pos, 36 + numSamples * bytesPerSample, true); pos += 4;
+    writeString(dv, pos, 'WAVE'); pos += 4;
+    writeString(dv, pos, 'fmt '); pos += 4;
+    dv.setUint32(pos, 16, true); pos += 4;
+    dv.setUint16(pos, isFloat ? 3 : 1, true); pos += 2;
+    dv.setUint16(pos, numChannels, true); pos += 2;
+    dv.setUint32(pos, sampleRate, true); pos += 4;
+    dv.setUint32(pos, sampleRate * numChannels * bytesPerSample, true); pos += 4;
+    dv.setUint16(pos, numChannels * bytesPerSample, true); pos += 2;
+    dv.setUint16(pos, bytesPerSample * 8, true); pos += 2;
+    writeString(dv, pos, 'data'); pos += 4;
+    dv.setUint32(pos, numSamples * bytesPerSample, true); pos += 4;
+
+    if (isFloat) {
+      for (let i = 0; i < numSamples; i++) {
+        dv.setFloat32(pos, samples[i], true);
+        pos += bytesPerSample;
+      }
+    } else {
+      for (let i = 0; i < numSamples; i++) {
+        const s = Math.max(-1, Math.min(1, samples[i]));
+        dv.setInt16(pos, s < 0 ? s * 0x8000 : s * 0x7FFF, true);
+        pos += bytesPerSample;
       }
-    });
-  }, [activeSource]);
-
-  const skipForward = useCallback(() => {
-    if (skipTimeoutRef.current) {
-      clearTimeout(skipTimeoutRef.current);
     }
 
-    skipTriggeredRef.current = true;
+    return buffer;
-    setIsProcessing(true);
+  };
-    // Cancel any ongoing request
+
-    if (currentRequestRef.current) {
+  const writeString = (view: DataView, offset: number, string: string): void => {
-      currentRequestRef.current.abort();
+    for (let i = 0; i < string.length; i++) {
-      currentRequestRef.current = null;
+      view.setUint8(offset + i, string.charCodeAt(i));
     }
-
+  };
-    // Stop current audio
-    if (activeSource) {
-      activeSource.stop();
-      setActiveSource(null);
-    }
-
-    setCurrentIndex((prev) => {
-      const nextIndex = Math.min(prev + 1, sentences.length - 1);
-      console.log('Skipping forward to:', sentences[nextIndex]);
-      return nextIndex;
-    });
-
-    // Reset skip flag after a short delay
-    skipTimeoutRef.current = setTimeout(() => {
-      skipTriggeredRef.current = false;
-      setIsProcessing(false);
-    }, 100);
-  }, [sentences, activeSource]);
-
-  const skipBackward = useCallback(() => {
-    if (skipTimeoutRef.current) {
-      clearTimeout(skipTimeoutRef.current);
-    }
-
-    skipTriggeredRef.current = true;
-    setIsProcessing(true);
-    // Cancel any ongoing request
-    if (currentRequestRef.current) {
-      currentRequestRef.current.abort();
-      currentRequestRef.current = null;
-    }
-
-    // Stop current audio
-    if (activeSource) {
-      activeSource.stop();
-      setActiveSource(null);
-    }
-
-    setCurrentIndex((prev) => {
-      const nextIndex = Math.max(prev - 1, 0);
-      console.log('Skipping backward to:', sentences[nextIndex]);
-      return nextIndex;
-    });
-
-    // Reset skip flag after a short delay
-    skipTimeoutRef.current = setTimeout(() => {
-      skipTriggeredRef.current = false;
-      setIsProcessing(false);
-    }, 100);
-  }, [sentences, activeSource]);
 
   const setText = useCallback((text: string) => {
     setCurrentText(text);
@@ -406,9 +513,9 @@ export function TTSProvider({ children }: { children: React.ReactNode }) {
     return () => {
       isEffectActive = false;
       // Clean up any playing audio when the effect is cleaned up
-      if (activeSource) {
+      if (activeHowl) {
-        activeSource.stop();
+        activeHowl.stop();
-        setActiveSource(null);
+        setActiveHowl(null);
       }
       if (currentRequestRef.current) {
         currentRequestRef.current.abort();
@@ -456,16 +563,16 @@ export function TTSProvider({ children }: { children: React.ReactNode }) {
     }
 
     // Stop current audio
-    if (activeSource) {
+    if (activeHowl) {
-      activeSource.stop();
+      activeHowl.stop();
-      setActiveSource(null);
+      setActiveHowl(null);
     }
 
     setIsPlaying(false);
     setCurrentIndex(0);
     setCurrentText('');
     setIsProcessing(false);
-  }, [activeSource]);
+  }, [activeHowl]);
 
   const stopAndPlayFromIndex = useCallback((index: number) => {
     // Cancel any ongoing request
@@ -475,9 +582,9 @@ export function TTSProvider({ children }: { children: React.ReactNode }) {
     }
 
     // Stop current audio
-    if (activeSource) {
+    if (activeHowl) {
-      activeSource.stop();
+      activeHowl.stop();
-      setActiveSource(null);
+      setActiveHowl(null);
     }
 
     // Set skip flag to prevent immediate auto-advance
@@ -494,7 +601,7 @@ export function TTSProvider({ children }: { children: React.ReactNode }) {
     skipTimeoutRef.current = setTimeout(() => {
       skipTriggeredRef.current = false;
     }, 100);
-  }, [activeSource]);
+  }, [activeHowl]);
 
   const setCurrentIndexWithoutPlay = useCallback((index: number) => {
     // Cancel any ongoing request
@@ -504,14 +611,14 @@ export function TTSProvider({ children }: { children: React.ReactNode }) {
     }
 
     // Stop current audio
-    if (activeSource) {
+    if (activeHowl) {
-      activeSource.stop();
+      activeHowl.stop();
-      setActiveSource(null);
+      setActiveHowl(null);
     }
 
     setCurrentIndex(index);
     skipTriggeredRef.current = false;
-  }, [activeSource]);
+  }, [activeHowl]);
 
   const setSpeedAndRestart = useCallback((newSpeed: number) => {
     setSpeed(newSpeed);