Support gpt-4o-mini-tts with instructions

README.md

@@ -13,7 +13,11 @@
 
 OpenReader WebUI is a document reader with Text-to-Speech capabilities, offering a TTS read along experience with narration for both PDF and EPUB documents. It can use any OpenAI compatible TTS endpoint, including [Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI).
 
-- 🎯 **TTS API Integration**: Compatible with OpenAI text to speech API, Kokoro FastAPI TTS, or any other compatible service; enabling high-quality voice narration
+- 🎯 **TTS API Integration**: 
+  - Compatible with OpenAI text to speech API and GPT-4o Mini TTS, Kokoro-FastAPI TTS, or any other compatible service
+  - Support for multiple TTS models (tts-1, tts-1-hd, gpt-4o-mini-tts, kokoro)
+  - Custom model support for experimental or self-hosted models
+  - Model-specific instructions support (for gpt-4o-mini-tts)
 - 💾 **Local-First Architecture**: Uses IndexedDB browser storage for documents
 - 🛜 **Optional Server-side documents**: Manually upload documents to the next backend for all users to download
 - 📖 **Read Along Experience**: Follow along with highlighted text as the TTS narrates
@@ -22,6 +26,8 @@ OpenReader WebUI is a document reader with Text-to-Speech capabilities, offering
 - 📲 **Mobile Support**: Works on mobile devices, and can be added as a PWA web app
 - 🎨 **Customizable Experience**: 
   - 🔑 Set TTS API base URL (and optional API key)
+  - 🤖 Choose from multiple TTS models or use custom models
+  - 🎯 Set model-specific instructions for GPT-4o Mini TTS
   - 🏎️ Adjustable playback speed
   - 📐 Customize PDF text extraction margins
   - 🗣️ Multiple voice options (checks `/v1/audio/voices` endpoint)
@@ -31,6 +37,7 @@ OpenReader WebUI is a document reader with Text-to-Speech capabilities, offering
 ### 🛠️ Work in progress
 - [x] **Audiobook creation and download** (m4b format)
 - [x] **Get PDFs on iOS 17 and below working 🤞**
+- [x] **Support for GPT-4o Mini TTS with instructions**
 - [ ] **End-to-end Testing**: More playwright tests (in progress)
 - [ ] **More document formats**: .txt, .md
 - [ ] **Support more TTS APIs**: ElevenLabs, etc.

package.json

@@ -1,6 +1,6 @@
 {
   "name": "openreader-webui",
-  "version": "0.2.4-patch.4",
+  "version": "0.2.5",
   "private": true,
   "scripts": {
     "dev": "next dev --turbopack -p 3003",

src/app/api/tts/route.ts

@@ -1,12 +1,19 @@
 import { NextRequest, NextResponse } from 'next/server';
 import OpenAI from 'openai';
+import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs';
+
+type CustomVoice = string;
+type ExtendedSpeechParams = Omit<SpeechCreateParams, 'voice'> & {
+  voice: SpeechCreateParams['voice'] | CustomVoice;
+  instructions?: string;
+};
 
 export async function POST(req: NextRequest) {
   try {
     // Get API credentials from headers or fall back to environment variables
     const openApiKey = req.headers.get('x-openai-key') || process.env.API_KEY || 'none';
     const openApiBaseUrl = req.headers.get('x-openai-base-url') || process.env.API_BASE;
-    const { text, voice, speed, format, model } = await req.json();
+    const { text, voice, speed, format, model, instructions } = await req.json();
     console.log('Received TTS request:', text, voice, speed, format, model);
 
     if (!openApiKey) {
@@ -24,13 +31,20 @@ export async function POST(req: NextRequest) {
     });
 
     // Request audio from OpenAI and pass along the abort signal
-    const response = await openai.audio.speech.create({
+    const createParams: ExtendedSpeechParams = {
       model: model || 'tts-1',
       voice: voice as "alloy",
       input: text,
       speed: speed,
       response_format: format === 'aac' ? 'aac' : 'mp3',
-    }, { signal: req.signal }); // Pass the abort signal to OpenAI client
+    };
+
+    // Only add instructions if model is gpt-4o-mini-tts and instructions are provided
+    if (model === 'gpt-4o-mini-tts' && instructions) {
+      createParams.instructions = instructions;
+    }
+
+    const response = await openai.audio.speech.create(createParams as SpeechCreateParams, { signal: req.signal });
 
     // Get the audio data as array buffer
     // This will also be aborted if the client cancels

src/components/SettingsModal.tsx

@@ -1,6 +1,6 @@
 'use client';
 
-import { Fragment, useState, useEffect, useCallback } from 'react';
+import { Fragment, useState, useEffect, useCallback, useMemo } from 'react';
 import {
   Dialog,
   DialogPanel,
@@ -39,17 +39,27 @@ export function SettingsModal() {
   const [isOpen, setIsOpen] = useState(false);
 
   const { theme, setTheme } = useTheme();
-  const { apiKey, baseUrl, ttsModel, updateConfig, updateConfigKey } = useConfig();
+  const { apiKey, baseUrl, ttsModel, ttsInstructions, updateConfig, updateConfigKey } = useConfig();
   const { refreshPDFs, refreshEPUBs, clearPDFs, clearEPUBs } = useDocuments();
   const [localApiKey, setLocalApiKey] = useState(apiKey);
   const [localBaseUrl, setLocalBaseUrl] = useState(baseUrl);
   const [localTTSModel, setLocalTTSModel] = useState(ttsModel);
+  const [customModel, setCustomModel] = useState('');
+  const [localTTSInstructions, setLocalTTSInstructions] = useState(ttsInstructions);
   const [isSyncing, setIsSyncing] = useState(false);
   const [isLoading, setIsLoading] = useState(false);
   const selectedTheme = themes.find(t => t.id === theme) || themes[0];
   const [showClearLocalConfirm, setShowClearLocalConfirm] = useState(false);
   const [showClearServerConfirm, setShowClearServerConfirm] = useState(false);
 
+  const ttsModels = useMemo(() => [
+    { id: 'tts-1', name: 'TTS-1' },
+    { id: 'tts-1-hd', name: '($$) TTS-1-HD' },
+    { id: 'gpt-4o-mini-tts', name: '($$$) GPT-4o Mini TTS' },
+    { id: 'kokoro', name: 'Kokoro' },
+    { id: 'custom', name: 'Custom Model' }
+  ], []);
+
   // set firstVisit on initial load
   const checkFirstVist = useCallback(async () => {
     if (!isDev) return;
@@ -65,7 +75,13 @@ export function SettingsModal() {
     setLocalApiKey(apiKey);
     setLocalBaseUrl(baseUrl);
     setLocalTTSModel(ttsModel);
-  }, [apiKey, baseUrl, ttsModel, checkFirstVist]);
+    setLocalTTSInstructions(ttsInstructions);
+    // Set custom model if current model is not in predefined list
+    if (!ttsModels.some(m => m.id === ttsModel) && ttsModel !== '') {
+      setCustomModel(ttsModel);
+      setLocalTTSModel('custom');
+    }
+  }, [apiKey, baseUrl, ttsModel, ttsModels, ttsInstructions, checkFirstVist]);
 
   const handleSync = async () => {
     try {
@@ -120,6 +136,18 @@ export function SettingsModal() {
     }
   };
 
+  const resetToCurrent = useCallback(() => {
+    setIsOpen(false);
+    setLocalApiKey(apiKey);
+    setLocalBaseUrl(baseUrl);
+    setLocalTTSModel(ttsModel);
+    setLocalTTSInstructions(ttsInstructions);
+    if (!ttsModels.some(m => m.id === ttsModel) && ttsModel !== '') {
+      setCustomModel(ttsModel);
+      setLocalTTSModel('custom');
+    }
+  }, [apiKey, baseUrl, ttsModel, ttsInstructions, ttsModels]);
+
   const tabs = [
     { name: 'Appearance', icon: '✨' },
     { name: 'API', icon: '🔑' },
@@ -136,11 +164,7 @@ export function SettingsModal() {
       <SettingsIcon className="w-4 h-4 sm:w-5 sm:h-5 hover:animate-spin-slow" />
 
       <Transition appear show={isOpen} as={Fragment}>
-        <Dialog as="div" className="relative z-50" onClose={() => {
+        <Dialog as="div" className="relative z-50" onClose={resetToCurrent}>
-          setIsOpen(false);
-          setLocalApiKey(apiKey);
-          setLocalBaseUrl(baseUrl);
-        }}>
           <TransitionChild
             as={Fragment}
             enter="ease-out duration-300"
@@ -173,7 +197,7 @@ export function SettingsModal() {
                   </DialogTitle>
 
                   <TabGroup>
-                    <TabList className="flex space-x-1 rounded-xl bg-background p-1 mb-4">
+                    <TabList className="flex flex-col sm:flex-col-none sm:flex-row gap-1 rounded-xl bg-background p-1 mb-4">
                       {tabs.map((tab) => (
                         <Tab
                           key={tab.name}
@@ -274,21 +298,84 @@ export function SettingsModal() {
                         </div>
 
                         <div className="space-y-2">
-                          <label className="block text-sm font-medium text-foreground">
+                          <label className="block text-sm font-medium text-foreground">TTS Model</label>
-                            OpenAI TTS Model
+                          <div className="flex flex-col gap-2">
-                            {localTTSModel !== 'tts-1' && <span className="ml-2 text-xs text-accent">(Custom model)</span>}
+                            <Listbox 
-                          </label>
+                              value={ttsModels.find(m => m.id === localTTSModel) || ttsModels[0]} 
-                          <div className="flex gap-2">
+                              onChange={(model) => {
-                            <Input
+                                setLocalTTSModel(model.id);
-                              type="text"
+                                if (model.id !== 'custom') {
-                              value={localTTSModel}
+                                  setCustomModel('');
-                              onChange={(e) => handleInputChange('ttsModel', e.target.value)}
+                                }
-                              placeholder="tts-1"
+                              }}
-                              className="w-full rounded-lg bg-background py-2 px-3 text-foreground shadow-sm focus:outline-none focus:ring-2 focus:ring-accent"
+                            >
-                            />
+                              <ListboxButton className="relative w-full cursor-pointer rounded-lg bg-background py-2 pl-3 pr-10 text-left text-foreground shadow-sm focus:outline-none focus:ring-2 focus:ring-accent transform transition-transform duration-200 ease-in-out hover:scale-[1.01] hover:text-accent">
+                                <span className="block truncate">
+                                  {ttsModels.find(m => m.id === localTTSModel)?.name || 'Select Model'}
+                                </span>
+                                <span className="pointer-events-none absolute inset-y-0 right-0 flex items-center pr-2">
+                                  <ChevronUpDownIcon className="h-5 w-5 text-muted" />
+                                </span>
+                              </ListboxButton>
+                              <Transition
+                                as={Fragment}
+                                leave="transition ease-in duration-100"
+                                leaveFrom="opacity-100"
+                                leaveTo="opacity-0"
+                              >
+                                <ListboxOptions className="absolute mt-1 w-full overflow-auto rounded-md bg-background py-1 shadow-lg ring-1 ring-black/5 focus:outline-none z-50">
+                                  {ttsModels.map((model) => (
+                                    <ListboxOption
+                                      key={model.id}
+                                      className={({ active }) =>
+                                        `relative cursor-pointer select-none py-2 pl-10 pr-4 ${
+                                          active ? 'bg-accent/10 text-accent' : 'text-foreground'
+                                        }`
+                                      }
+                                      value={model}
+                                    >
+                                      {({ selected }) => (
+                                        <>
+                                          <span className={`block truncate ${selected ? 'font-medium' : 'font-normal'}`}>
+                                            {model.name}
+                                          </span>
+                                          {selected ? (
+                                            <span className="absolute inset-y-0 left-0 flex items-center pl-3 text-accent">
+                                              <CheckIcon className="h-5 w-5" />
+                                            </span>
+                                          ) : null}
+                                        </>
+                                      )}
+                                    </ListboxOption>
+                                  ))}
+                                </ListboxOptions>
+                              </Transition>
+                            </Listbox>
+
+                            {localTTSModel === 'custom' && (
+                              <Input
+                                type="text"
+                                value={customModel}
+                                onChange={(e) => setCustomModel(e.target.value)}
+                                placeholder="Enter custom model name"
+                                className="w-full rounded-lg bg-background py-2 px-3 text-foreground shadow-sm focus:outline-none focus:ring-2 focus:ring-accent"
+                              />
+                            )}
                           </div>
                         </div>
 
+                        {localTTSModel === 'gpt-4o-mini-tts' && (
+                          <div className="space-y-2">
+                            <label className="block text-sm font-medium text-foreground">TTS Instructions</label>
+                            <textarea
+                              value={localTTSInstructions}
+                              onChange={(e) => setLocalTTSInstructions(e.target.value)}
+                              placeholder="Enter instructions for the TTS model"
+                              className="w-full h-24 rounded-lg bg-background py-2 px-3 text-foreground shadow-sm focus:outline-none focus:ring-2 focus:ring-accent"
+                            />
+                          </div>
+                        )}
+
                         <div className="mt-6 flex justify-end gap-2">
                           <Button
                             type="button"
@@ -300,6 +387,8 @@ export function SettingsModal() {
                               setLocalApiKey('');
                               setLocalBaseUrl('');
                               setLocalTTSModel('tts-1');
+                              setCustomModel('');
+                              setLocalTTSInstructions('');
                             }}
                           >
                             Reset
@@ -315,7 +404,9 @@ export function SettingsModal() {
                                 apiKey: localApiKey || '',
                                 baseUrl: localBaseUrl || '',
                               });
-                              await updateConfigKey('ttsModel', localTTSModel || 'tts-1');
+                              const finalModel = localTTSModel === 'custom' ? customModel : localTTSModel;
+                              await updateConfigKey('ttsModel', finalModel);
+                              await updateConfigKey('ttsInstructions', localTTSInstructions);
                               setIsOpen(false);
                             }}
                           >

src/contexts/ConfigContext.tsx

@@ -20,6 +20,7 @@ type ConfigValues = {
   leftMargin: number;
   rightMargin: number;
   ttsModel: string;
+  ttsInstructions: string;
 };
 
 /** Interface defining the configuration context shape and functionality */
@@ -36,6 +37,7 @@ interface ConfigContextType {
   leftMargin: number;
   rightMargin: number;
   ttsModel: string;
+  ttsInstructions: string;
   updateConfig: (newConfig: Partial<{ apiKey: string; baseUrl: string; viewType: ViewType }>) => Promise<void>;
   updateConfigKey: <K extends keyof ConfigValues>(key: K, value: ConfigValues[K]) => Promise<void>;
   isLoading: boolean;
@@ -64,6 +66,7 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
   const [leftMargin, setLeftMargin] = useState<number>(0.07);
   const [rightMargin, setRightMargin] = useState<number>(0.07);
   const [ttsModel, setTTSModel] = useState<string>('tts-1');
+  const [ttsInstructions, setTTSInstructions] = useState<string>('');
 
   const [isLoading, setIsLoading] = useState(true);
   const [isDBReady, setIsDBReady] = useState(false);
@@ -88,6 +91,7 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
         const cachedLeftMargin = await getItem('leftMargin');
         const cachedRightMargin = await getItem('rightMargin');
         const cachedTTSModel = await getItem('ttsModel');
+        const cachedTTSInstructions = await getItem('ttsInstructions');
 
         // Only set API key and base URL if they were explicitly saved by the user
         if (cachedApiKey) {
@@ -110,6 +114,7 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
         setLeftMargin(parseFloat(cachedLeftMargin || '0.07'));
         setRightMargin(parseFloat(cachedRightMargin || '0.07'));
         setTTSModel(cachedTTSModel || 'tts-1');
+        setTTSInstructions(cachedTTSInstructions || '');
 
         // Only save non-sensitive settings by default
         if (!cachedViewType) {
@@ -128,6 +133,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
         if (cachedTTSModel === null) {
           await setItem('ttsModel', 'tts-1');
         }
+        if (cachedTTSInstructions === null) {
+          await setItem('ttsInstructions', '');
+        }
         
       } catch (error) {
         console.error('Error initializing:', error);
@@ -221,6 +229,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
         case 'ttsModel':
           setTTSModel(value as string);
           break;
+        case 'ttsInstructions':
+          setTTSInstructions(value as string);
+          break;
       }
     } catch (error) {
       console.error(`Error updating config key ${key}:`, error);
@@ -244,6 +255,7 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
       leftMargin,
       rightMargin,
       ttsModel,
+      ttsInstructions,
       updateConfig, 
       updateConfigKey,
       isLoading, 

src/contexts/TTSContext.tsx

@@ -98,6 +98,7 @@ export function TTSProvider({ children }: { children: ReactNode }) {
     voiceSpeed,
     voice: configVoice,
     ttsModel: configTTSModel,
+    ttsInstructions: configTTSInstructions,
     updateConfigKey,
     skipBlank,
   } = useConfig();
@@ -140,6 +141,7 @@ export function TTSProvider({ children }: { children: ReactNode }) {
   const [speed, setSpeed] = useState(voiceSpeed);
   const [voice, setVoice] = useState(configVoice);
   const [ttsModel, setTTSModel] = useState(configTTSModel);
+  const [ttsInstructions, setTTSInstructions] = useState(configTTSInstructions);
 
   // Track pending preload requests
   const preloadRequests = useRef<Map<string, Promise<string>>>(new Map());
@@ -389,8 +391,9 @@ export function TTSProvider({ children }: { children: ReactNode }) {
       fetchVoices();
       updateVoiceAndSpeed();
       setTTSModel(configTTSModel);
+      setTTSInstructions(configTTSInstructions);
     }
-  }, [configIsLoading, openApiKey, openApiBaseUrl, updateVoiceAndSpeed, fetchVoices, configTTSModel]);
+  }, [configIsLoading, openApiKey, openApiBaseUrl, updateVoiceAndSpeed, fetchVoices, configTTSModel, configTTSInstructions]);
 
   /**
    * Generates and plays audio for the current sentence
@@ -426,7 +429,8 @@ export function TTSProvider({ children }: { children: ReactNode }) {
               text: sentence,
               voice: voice,
               speed: speed,
-              model: ttsModel
+              model: ttsModel,
+              instructions: ttsModel === 'gpt-4o-mini-tts' ? ttsInstructions : undefined
             }),
             signal: controller.signal,
           });
@@ -470,7 +474,7 @@ export function TTSProvider({ children }: { children: ReactNode }) {
       });
       throw error;
     }
-  }, [voice, speed, ttsModel, audioCache, openApiKey, openApiBaseUrl]);
+  }, [voice, speed, ttsModel, ttsInstructions, audioCache, openApiKey, openApiBaseUrl]);
 
   /**
    * Processes and plays the current sentence