danvics/openreader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Support gpt-4o-mini-tts with instructions

Browse source
This commit is contained in:
Richard Roberson 2025-03-21 11:32:42 -06:00
parent f464b059e4
commit 02cd288fc9
6 changed files with 158 additions and 30 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
README.md
View file

@ -13,7 +13,11 @@
OpenReader WebUI is a document reader with Text-to-Speech capabilities, offering a TTS read along experience with narration for both PDF and EPUB documents. It can use any OpenAI compatible TTS endpoint, including [Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI).
- 🎯 **TTS API Integration**: Compatible with OpenAI text to speech API, Kokoro FastAPI TTS, or any other compatible service; enabling high-quality voice narration
- 🎯 **TTS API Integration**:
- Compatible with OpenAI text to speech API and GPT-4o Mini TTS, Kokoro-FastAPI TTS, or any other compatible service
- Support for multiple TTS models (tts-1, tts-1-hd, gpt-4o-mini-tts, kokoro)
- Custom model support for experimental or self-hosted models
- Model-specific instructions support (for gpt-4o-mini-tts)
- 💾 **Local-First Architecture**: Uses IndexedDB browser storage for documents
- 🛜 **Optional Server-side documents**: Manually upload documents to the next backend for all users to download
- 📖 **Read Along Experience**: Follow along with highlighted text as the TTS narrates
@ -22,6 +26,8 @@ OpenReader WebUI is a document reader with Text-to-Speech capabilities, offering
- 📲 **Mobile Support**: Works on mobile devices, and can be added as a PWA web app
- 🎨 **Customizable Experience**:
- 🔑 Set TTS API base URL (and optional API key)
- 🤖 Choose from multiple TTS models or use custom models
- 🎯 Set model-specific instructions for GPT-4o Mini TTS
- 🏎️ Adjustable playback speed
- 📐 Customize PDF text extraction margins
- 🗣️ Multiple voice options (checks `/v1/audio/voices` endpoint)
@ -31,6 +37,7 @@ OpenReader WebUI is a document reader with Text-to-Speech capabilities, offering
### 🛠️ Work in progress
- [x] **Audiobook creation and download** (m4b format)
- [x] **Get PDFs on iOS 17 and below working 🤞**
- [x] **Support for GPT-4o Mini TTS with instructions**
- [ ] **End-to-end Testing**: More playwright tests (in progress)
- [ ] **More document formats**: .txt, .md
- [ ] **Support more TTS APIs**: ElevenLabs, etc.

2
package.json
View file

@ -1,6 +1,6 @@
{
"name": "openreader-webui",
"version": "0.2.4-patch.4",
"version": "0.2.5",
"private": true,
"scripts": {
"dev": "next dev --turbopack -p 3003",

20
src/app/api/tts/route.ts
View file

@ -1,12 +1,19 @@
import { NextRequest, NextResponse } from 'next/server';
import OpenAI from 'openai';
import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs';
type CustomVoice = string;
type ExtendedSpeechParams = Omit<SpeechCreateParams, 'voice'> & {
voice: SpeechCreateParams['voice'] | CustomVoice;
instructions?: string;
};
export async function POST(req: NextRequest) {
try {
// Get API credentials from headers or fall back to environment variables
const openApiKey = req.headers.get('x-openai-key') || process.env.API_KEY || 'none';
const openApiBaseUrl = req.headers.get('x-openai-base-url') || process.env.API_BASE;
const { text, voice, speed, format, model } = await req.json();
const { text, voice, speed, format, model, instructions } = await req.json();
console.log('Received TTS request:', text, voice, speed, format, model);
if (!openApiKey) {
@ -24,13 +31,20 @@ export async function POST(req: NextRequest) {
});
// Request audio from OpenAI and pass along the abort signal
const response = await openai.audio.speech.create({
const createParams: ExtendedSpeechParams = {
model: model || 'tts-1',
voice: voice as "alloy",
input: text,
speed: speed,
response_format: format === 'aac' ? 'aac' : 'mp3',
}, { signal: req.signal }); // Pass the abort signal to OpenAI client
};
// Only add instructions if model is gpt-4o-mini-tts and instructions are provided
if (model === 'gpt-4o-mini-tts' && instructions) {
createParams.instructions = instructions;
}
const response = await openai.audio.speech.create(createParams as SpeechCreateParams, { signal: req.signal });
// Get the audio data as array buffer
// This will also be aborted if the client cancels

135
src/components/SettingsModal.tsx
View file

@ -1,6 +1,6 @@
'use client';
import { Fragment, useState, useEffect, useCallback } from 'react';
import { Fragment, useState, useEffect, useCallback, useMemo } from 'react';
import {
Dialog,
DialogPanel,
@ -39,17 +39,27 @@ export function SettingsModal() {
const [isOpen, setIsOpen] = useState(false);
const { theme, setTheme } = useTheme();
const { apiKey, baseUrl, ttsModel, updateConfig, updateConfigKey } = useConfig();
const { apiKey, baseUrl, ttsModel, ttsInstructions, updateConfig, updateConfigKey } = useConfig();
const { refreshPDFs, refreshEPUBs, clearPDFs, clearEPUBs } = useDocuments();
const [localApiKey, setLocalApiKey] = useState(apiKey);
const [localBaseUrl, setLocalBaseUrl] = useState(baseUrl);
const [localTTSModel, setLocalTTSModel] = useState(ttsModel);
const [customModel, setCustomModel] = useState('');
const [localTTSInstructions, setLocalTTSInstructions] = useState(ttsInstructions);
const [isSyncing, setIsSyncing] = useState(false);
const [isLoading, setIsLoading] = useState(false);
const selectedTheme = themes.find(t => t.id === theme) || themes[0];
const [showClearLocalConfirm, setShowClearLocalConfirm] = useState(false);
const [showClearServerConfirm, setShowClearServerConfirm] = useState(false);
const ttsModels = useMemo(() => [
{ id: 'tts-1', name: 'TTS-1' },
{ id: 'tts-1-hd', name: '($$) TTS-1-HD' },
{ id: 'gpt-4o-mini-tts', name: '($$$) GPT-4o Mini TTS' },
{ id: 'kokoro', name: 'Kokoro' },
{ id: 'custom', name: 'Custom Model' }
], []);
// set firstVisit on initial load
const checkFirstVist = useCallback(async () => {
if (!isDev) return;
@ -65,7 +75,13 @@ export function SettingsModal() {
setLocalApiKey(apiKey);
setLocalBaseUrl(baseUrl);
setLocalTTSModel(ttsModel);
}, [apiKey, baseUrl, ttsModel, checkFirstVist]);
setLocalTTSInstructions(ttsInstructions);
// Set custom model if current model is not in predefined list
if (!ttsModels.some(m => m.id === ttsModel) && ttsModel !== '') {
setCustomModel(ttsModel);
setLocalTTSModel('custom');
}
}, [apiKey, baseUrl, ttsModel, ttsModels, ttsInstructions, checkFirstVist]);
const handleSync = async () => {
try {
@ -120,6 +136,18 @@ export function SettingsModal() {
}
};
const resetToCurrent = useCallback(() => {
setIsOpen(false);
setLocalApiKey(apiKey);
setLocalBaseUrl(baseUrl);
setLocalTTSModel(ttsModel);
setLocalTTSInstructions(ttsInstructions);
if (!ttsModels.some(m => m.id === ttsModel) && ttsModel !== '') {
setCustomModel(ttsModel);
setLocalTTSModel('custom');
}
}, [apiKey, baseUrl, ttsModel, ttsInstructions, ttsModels]);
const tabs = [
{ name: 'Appearance', icon: '✨' },
{ name: 'API', icon: '🔑' },
@ -136,11 +164,7 @@ export function SettingsModal() {
<SettingsIcon className="w-4 h-4 sm:w-5 sm:h-5 hover:animate-spin-slow" />
<Transition appear show={isOpen} as={Fragment}>
<Dialog as="div" className="relative z-50" onClose={() => {
setIsOpen(false);
setLocalApiKey(apiKey);
setLocalBaseUrl(baseUrl);
}}>
<Dialog as="div" className="relative z-50" onClose={resetToCurrent}>
<TransitionChild
as={Fragment}
enter="ease-out duration-300"
@ -173,7 +197,7 @@ export function SettingsModal() {
</DialogTitle>
<TabGroup>
<TabList className="flex space-x-1 rounded-xl bg-background p-1 mb-4">
<TabList className="flex flex-col sm:flex-col-none sm:flex-row gap-1 rounded-xl bg-background p-1 mb-4">
{tabs.map((tab) => (
<Tab
key={tab.name}
@ -274,21 +298,84 @@ export function SettingsModal() {
</div>
<div className="space-y-2">
<label className="block text-sm font-medium text-foreground">
OpenAI TTS Model
{localTTSModel !== 'tts-1' && <span className="ml-2 text-xs text-accent">(Custom model)</span>}
</label>
<div className="flex gap-2">
<Input
type="text"
value={localTTSModel}
onChange={(e) => handleInputChange('ttsModel', e.target.value)}
placeholder="tts-1"
className="w-full rounded-lg bg-background py-2 px-3 text-foreground shadow-sm focus:outline-none focus:ring-2 focus:ring-accent"
/>
<label className="block text-sm font-medium text-foreground">TTS Model</label>
<div className="flex flex-col gap-2">
<Listbox
value={ttsModels.find(m => m.id === localTTSModel) || ttsModels[0]}
onChange={(model) => {
setLocalTTSModel(model.id);
if (model.id !== 'custom') {
setCustomModel('');
}
}}
>
<ListboxButton className="relative w-full cursor-pointer rounded-lg bg-background py-2 pl-3 pr-10 text-left text-foreground shadow-sm focus:outline-none focus:ring-2 focus:ring-accent transform transition-transform duration-200 ease-in-out hover:scale-[1.01] hover:text-accent">
<span className="block truncate">
{ttsModels.find(m => m.id === localTTSModel)?.name || 'Select Model'}
</span>
<span className="pointer-events-none absolute inset-y-0 right-0 flex items-center pr-2">
<ChevronUpDownIcon className="h-5 w-5 text-muted" />
</span>
</ListboxButton>
<Transition
as={Fragment}
leave="transition ease-in duration-100"
leaveFrom="opacity-100"
leaveTo="opacity-0"
>
<ListboxOptions className="absolute mt-1 w-full overflow-auto rounded-md bg-background py-1 shadow-lg ring-1 ring-black/5 focus:outline-none z-50">
{ttsModels.map((model) => (
<ListboxOption
key={model.id}
className={({ active }) =>
`relative cursor-pointer select-none py-2 pl-10 pr-4 ${
active ? 'bg-accent/10 text-accent' : 'text-foreground'
}`
}
value={model}
>
{({ selected }) => (
<>
<span className={`block truncate ${selected ? 'font-medium' : 'font-normal'}`}>
{model.name}
</span>
{selected ? (
<span className="absolute inset-y-0 left-0 flex items-center pl-3 text-accent">
<CheckIcon className="h-5 w-5" />
</span>
) : null}
</>
)}
</ListboxOption>
))}
</ListboxOptions>
</Transition>
</Listbox>
{localTTSModel === 'custom' && (
<Input
type="text"
value={customModel}
onChange={(e) => setCustomModel(e.target.value)}
placeholder="Enter custom model name"
className="w-full rounded-lg bg-background py-2 px-3 text-foreground shadow-sm focus:outline-none focus:ring-2 focus:ring-accent"
/>
)}
</div>
</div>
{localTTSModel === 'gpt-4o-mini-tts' && (
<div className="space-y-2">
<label className="block text-sm font-medium text-foreground">TTS Instructions</label>
<textarea
value={localTTSInstructions}
onChange={(e) => setLocalTTSInstructions(e.target.value)}
placeholder="Enter instructions for the TTS model"
className="w-full h-24 rounded-lg bg-background py-2 px-3 text-foreground shadow-sm focus:outline-none focus:ring-2 focus:ring-accent"
/>
</div>
)}
<div className="mt-6 flex justify-end gap-2">
<Button
type="button"
@ -300,6 +387,8 @@ export function SettingsModal() {
setLocalApiKey('');
setLocalBaseUrl('');
setLocalTTSModel('tts-1');
setCustomModel('');
setLocalTTSInstructions('');
}}
>
Reset
@ -315,7 +404,9 @@ export function SettingsModal() {
apiKey: localApiKey || '',
baseUrl: localBaseUrl || '',
});
await updateConfigKey('ttsModel', localTTSModel || 'tts-1');
const finalModel = localTTSModel === 'custom' ? customModel : localTTSModel;
await updateConfigKey('ttsModel', finalModel);
await updateConfigKey('ttsInstructions', localTTSInstructions);
setIsOpen(false);
}}
>

12
src/contexts/ConfigContext.tsx
View file

@ -20,6 +20,7 @@ type ConfigValues = {
leftMargin: number;
rightMargin: number;
ttsModel: string;
ttsInstructions: string;
};
/** Interface defining the configuration context shape and functionality */
@ -36,6 +37,7 @@ interface ConfigContextType {
leftMargin: number;
rightMargin: number;
ttsModel: string;
ttsInstructions: string;
updateConfig: (newConfig: Partial<{ apiKey: string; baseUrl: string; viewType: ViewType }>) => Promise<void>;
updateConfigKey: <K extends keyof ConfigValues>(key: K, value: ConfigValues[K]) => Promise<void>;
isLoading: boolean;
@ -64,6 +66,7 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
const [leftMargin, setLeftMargin] = useState<number>(0.07);
const [rightMargin, setRightMargin] = useState<number>(0.07);
const [ttsModel, setTTSModel] = useState<string>('tts-1');
const [ttsInstructions, setTTSInstructions] = useState<string>('');
const [isLoading, setIsLoading] = useState(true);
const [isDBReady, setIsDBReady] = useState(false);
@ -88,6 +91,7 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
const cachedLeftMargin = await getItem('leftMargin');
const cachedRightMargin = await getItem('rightMargin');
const cachedTTSModel = await getItem('ttsModel');
const cachedTTSInstructions = await getItem('ttsInstructions');
// Only set API key and base URL if they were explicitly saved by the user
if (cachedApiKey) {
@ -110,6 +114,7 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
setLeftMargin(parseFloat(cachedLeftMargin || '0.07'));
setRightMargin(parseFloat(cachedRightMargin || '0.07'));
setTTSModel(cachedTTSModel || 'tts-1');
setTTSInstructions(cachedTTSInstructions || '');
// Only save non-sensitive settings by default
if (!cachedViewType) {
@ -128,6 +133,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
if (cachedTTSModel === null) {
await setItem('ttsModel', 'tts-1');
}
if (cachedTTSInstructions === null) {
await setItem('ttsInstructions', '');
}
} catch (error) {
console.error('Error initializing:', error);
@ -221,6 +229,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
case 'ttsModel':
setTTSModel(value as string);
break;
case 'ttsInstructions':
setTTSInstructions(value as string);
break;
}
} catch (error) {
console.error(`Error updating config key ${key}:`, error);
@ -244,6 +255,7 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
leftMargin,
rightMargin,
ttsModel,
ttsInstructions,
updateConfig,
updateConfigKey,
isLoading,

10
src/contexts/TTSContext.tsx
View file

@ -98,6 +98,7 @@ export function TTSProvider({ children }: { children: ReactNode }) {
voiceSpeed,
voice: configVoice,
ttsModel: configTTSModel,
ttsInstructions: configTTSInstructions,
updateConfigKey,
skipBlank,
} = useConfig();
@ -140,6 +141,7 @@ export function TTSProvider({ children }: { children: ReactNode }) {
const [speed, setSpeed] = useState(voiceSpeed);
const [voice, setVoice] = useState(configVoice);
const [ttsModel, setTTSModel] = useState(configTTSModel);
const [ttsInstructions, setTTSInstructions] = useState(configTTSInstructions);
// Track pending preload requests
const preloadRequests = useRef<Map<string, Promise<string>>>(new Map());
@ -389,8 +391,9 @@ export function TTSProvider({ children }: { children: ReactNode }) {
fetchVoices();
updateVoiceAndSpeed();
setTTSModel(configTTSModel);
setTTSInstructions(configTTSInstructions);
}
}, [configIsLoading, openApiKey, openApiBaseUrl, updateVoiceAndSpeed, fetchVoices, configTTSModel]);
}, [configIsLoading, openApiKey, openApiBaseUrl, updateVoiceAndSpeed, fetchVoices, configTTSModel, configTTSInstructions]);
/**
* Generates and plays audio for the current sentence
@ -426,7 +429,8 @@ export function TTSProvider({ children }: { children: ReactNode }) {
text: sentence,
voice: voice,
speed: speed,
model: ttsModel
model: ttsModel,
instructions: ttsModel === 'gpt-4o-mini-tts' ? ttsInstructions : undefined
}),
signal: controller.signal,
});
@ -470,7 +474,7 @@ export function TTSProvider({ children }: { children: ReactNode }) {
});
throw error;
}
}, [voice, speed, ttsModel, audioCache, openApiKey, openApiBaseUrl]);
}, [voice, speed, ttsModel, ttsInstructions, audioCache, openApiKey, openApiBaseUrl]);
/**
* Processes and plays the current sentence