diff --git a/Dockerfile b/Dockerfile
index 52a26f9..3c36a2b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,8 +1,18 @@
-# Use Node.js slim image
-FROM node:current-alpine
+# Stage 1: build whisper.cpp (no model download – the app handles that)
+FROM alpine:3.20 AS whisper-builder
-# Add ffmpeg and libreoffice using Alpine package manager
-RUN apk add --no-cache ffmpeg libreoffice-writer
+RUN apk add --no-cache git cmake build-base
+
+WORKDIR /opt
+
+RUN git clone --depth 1 https://github.com/ggml-org/whisper.cpp.git && \
+ cd whisper.cpp && \
+ cmake -B build && \
+ cmake --build build -j --config Release
+
+
+# Stage 2: build the Next.js app
+FROM node:lts-alpine AS app-builder
# Install pnpm globally
RUN npm install -g pnpm
@@ -23,8 +33,34 @@ COPY . .
RUN pnpm exec next telemetry disable
RUN pnpm build
+
+# Stage 3: minimal runtime image
+FROM node:current-alpine AS runner
+
+# Add runtime OS dependencies:
+# - ffmpeg: required for audiobook export and word-by-word alignment (/api/whisper)
+# - libreoffice-writer: required for DOCX → PDF conversion
+RUN apk add --no-cache ffmpeg libreoffice-writer
+
+# Install pnpm globally for running the app
+RUN npm install -g pnpm
+
+# App runtime directory
+WORKDIR /app
+
+# Copy built app and dependencies from the builder stage
+COPY --from=app-builder /app ./
+
+# Copy the compiled whisper.cpp build output into the runtime image
+# (includes whisper-cli and its shared libraries, e.g. libwhisper.so, libggml.so)
+COPY --from=whisper-builder /opt/whisper.cpp/build /opt/whisper.cpp/build
+
+# Point the app at the compiled whisper-cli binary and ensure its libs are discoverable
+ENV WHISPER_CPP_BIN=/opt/whisper.cpp/build/bin/whisper-cli
+ENV LD_LIBRARY_PATH=/opt/whisper.cpp/build
+
# Expose the port the app runs on
EXPOSE 3003
# Start the application
-CMD ["pnpm", "start"]
\ No newline at end of file
+CMD ["pnpm", "start"]
diff --git a/README.md b/README.md
index 691881a..02dc60d 100644
--- a/README.md
+++ b/README.md
@@ -11,8 +11,6 @@
OpenReader WebUI is an open source text to speech document reader web app built using Next.js, offering a TTS read along experience with narration for **EPUB, PDF, TXT, MD, and DOCX documents**. It supports multiple TTS providers including OpenAI, Deepinfra, and custom OpenAI-compatible endpoints like [Kokoro-FastAPI](https://github.com/remsky/Kokoro-FastAPI) and [Orpheus-FastAPI](https://github.com/Lex-au/Orpheus-FastAPI)
-- 🧠 *(New)* **Smart Sentence-Aware Narration** merges sentences across pages/chapters for smoother TTS
-- 🎧 *(New)* **Reliable Audiobook Export** in **m4b/mp3**, with resumable, chapter-based export and regeneration
- 🎯 *(New)* **Multi-Provider TTS Support**
- [**Kokoro-FastAPI**](https://github.com/remsky/Kokoro-FastAPI): Supporting multi-voice combinations (like `af_heart+af_bella`)
- [**Orpheus-FastAPI**](https://github.com/Lex-au/Orpheus-FastAPI)
@@ -20,56 +18,18 @@ OpenReader WebUI is an open source text to speech document reader web app built
- **Cloud TTS Providers (requiring API keys)**
- [**Deepinfra**](https://deepinfra.com/models/text-to-speech): Kokoro-82M + models with support for cloned voices and more
- [**OpenAI API ($$)**](https://platform.openai.com/docs/pricing#transcription-and-speech): tts-1, tts-1-hd, and gpt-4o-mini-tts w/ instructions
-- 🚀 *(New)* **Optimized Next.js TTS Proxy** with audio caching and optimized repeat playback
-- 💾 *(Updated)* **Local-First Architecture** stores documents and more in-browser with Dexie.js
- 📖 *(Updated)* **Read Along Experience** providing real-time text highlighting during playback (PDF/EPUB)
+ - *(New)* **Word-by-word** highlighting uses word-by-word timestamps generated server-side with [*whisper.cpp*](https://github.com/ggml-org/whisper.cpp) (optional)
+- 🧠 *(New)* **Smart Sentence-Aware Narration** merges sentences across pages/chapters for smoother TTS
+- 🎧 *(New)* **Reliable Audiobook Export** in **m4b/mp3**, with resumable, chapter-based export and regeneration
+- 🚀 *(New)* **Optimized Next.js TTS Proxy** with audio caching and optimized repeat playback
+- 💾 **Local-First Architecture** stores documents and more in-browser with Dexie.js
- 🛜 **Optional Server-side documents** using backend `/docstore` for all users
- 🎨 **Customizable Experience**
- 🎨 Multiple app theme options
- ⚙️ Various TTS and document handling settings
- And more ...
-
-
-
-### 🆕 What's New in v1.0.0
-
-
-
-- 🧠 **Smart sentence continuation**
- - Improved NLP handling of complex structures and quoted dialogue provides more natural sentence boundaries and a smoother audio-text flow.
- - EPUB and PDF playback now use smarter sentence splitting and continuation metadata so sentences that cross page/chapter boundaries are merged before hitting the TTS API.
- - This yields more natural narration and fewer awkward pauses when a sentence spans multiple pages or EPUB spine items.
-- 📄 **Modernized PDF text highlighting pipeline**
- - Real-time PDF text highlighting is now offloaded to a dedicated Web Worker so scrolling and playback controls remain responsive during narration.
- - A new overlay-based highlighting system draws independent highlight layers on top of the PDF, avoiding interference with the underlying text layer.
- - Upgraded fuzzy matching with Dice-based similarity improves the accuracy of mapping spoken words to on-screen text.
- - A new per-device setting lets you enable or disable real-time PDF highlighting during playback for a more tailored reading experience.
-- 🎧 **Chapter/page-based audiobook export with resume & regeneration**
- - Per-chapter/per-page generation to disk with persistent `bookId`
- - Resumable generation (can cancel and continue later)
- - Per-chapter regeneration & deletion
- - Final combined **M4B** or **MP3** download with embedded chapter metadata.
-- 💾 **Dexie-backed local storage & sync**
- - All document types (PDF, EPUB, TXT/MD-as-HTML) and config are stored via a unified Dexie layer on top of IndexedDB.
- - Document lists use live Dexie queries (no manual refresh needed), and server sync now correctly includes text/markdown documents as part of the library backup.
-- 🗣️ **Kokoro multi-voice selection & utilities**
- - Kokoro models now support multi-voice combination, with provider-aware limits and helpers (not supported on OpenAI or Deepinfra)
-- ⚡ **Faster, more efficient TTS backend proxy**
- - In-memory **LRU caching** for audio responses with configurable size/TTL
- - **ETag** support (`304` on cache hits) + `X-Cache` headers (`HIT` / `MISS` / `INFLIGHT`)
-- 📄 **More robust DOCX → PDF conversion**
- - DOCX conversion now uses isolated per-job LibreOffice profiles and temp directories, polls for a stable output file size, and aggressively cleans up temp files.
- - This reduces cross-job interference and flakiness when converting multiple DOCX files in parallel.
-- ♿ **Accessibility & layout improvements**
- - Dialogs and folder toggles expose proper roles and ARIA attributes.
- - PDF/EPUB/HTML readers use a full-height app shell with a sticky bottom TTS bar, improved scrollbars, and refined focus styles.
-- ✅ **End-to-end Playwright test suite with TTS mocks**
- - Deterministic TTS responses in tests via a reusable Playwright route mock.
- - Coverage for accessibility, upload, navigation, folder management, deletion flows, audiobook generation/export and playback across all document types.
-
-
-
## 🐳 Docker Quick Start
### Prerequisites
@@ -194,6 +154,20 @@ Optionally required for different features:
```bash
brew install libreoffice
```
+- [whisper.cpp](https://github.com/ggml-org/whisper.cpp) (optional, required for word-by-word highlighting)
+ ```bash
+ # clone and build whisper.cpp (no model download needed – OpenReader handles that)
+ git clone https://github.com/ggml-org/whisper.cpp.git
+ cd whisper.cpp
+ cmake -B build
+ cmake --build build -j --config Release
+
+ # point OpenReader to the compiled whisper-cli binary
+ echo WHISPER_CPP_BIN=\"$(pwd)/build/bin/whisper-cli\"
+ ```
+
+ > **Note:** The `WHISPER_CPP_BIN` path should be set in your `.env` file for OpenReader to use word-by-word highlighting features.
+
### Steps
1. Clone the repository:
diff --git a/package.json b/package.json
index e62e007..acc1eba 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "openreader-webui",
- "version": "v1.0.1",
+ "version": "v1.1.0",
"private": true,
"scripts": {
"dev": "next dev --turbopack -p 3003",
diff --git a/src/app/api/audio/convert/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts
similarity index 100%
rename from src/app/api/audio/convert/chapter/route.ts
rename to src/app/api/audiobook/chapter/route.ts
diff --git a/src/app/api/audio/convert/route.ts b/src/app/api/audiobook/route.ts
similarity index 89%
rename from src/app/api/audio/convert/route.ts
rename to src/app/api/audiobook/route.ts
index 4c4a3f3..17bf12a 100644
--- a/src/app/api/audio/convert/route.ts
+++ b/src/app/api/audiobook/route.ts
@@ -1,15 +1,16 @@
import { NextRequest, NextResponse } from 'next/server';
import { spawn } from 'child_process';
-import { writeFile, readFile, mkdir, unlink, readdir } from 'fs/promises';
+import { writeFile, readFile, mkdir, unlink, readdir, rm } from 'fs/promises';
import { existsSync, createReadStream } from 'fs';
import { join } from 'path';
import { randomUUID } from 'crypto';
+import type { TTSAudioBytes, TTSAudiobookFormat } from '@/types/tts';
interface ConversionRequest {
chapterTitle: string;
- buffer: number[];
+ buffer: TTSAudioBytes;
bookId?: string;
- format?: 'mp3' | 'm4b';
+ format?: TTSAudiobookFormat;
chapterIndex?: number;
}
@@ -206,9 +207,12 @@ export async function POST(request: NextRequest) {
await unlink(inputPath).catch(console.error);
return NextResponse.json({
+ index: chapterIndex,
+ title: data.chapterTitle,
+ duration,
+ status: 'completed' as const,
bookId,
- chapterIndex,
- duration
+ format
});
} catch (error) {
@@ -229,7 +233,7 @@ export async function POST(request: NextRequest) {
export async function GET(request: NextRequest) {
try {
const bookId = request.nextUrl.searchParams.get('bookId');
- const requestedFormat = request.nextUrl.searchParams.get('format') as 'mp3' | 'm4b' | null;
+ const requestedFormat = request.nextUrl.searchParams.get('format') as TTSAudiobookFormat | null;
if (!bookId) {
return NextResponse.json({ error: 'Missing bookId parameter' }, { status: 400 });
}
@@ -378,4 +382,31 @@ function streamFile(filePath: string, format: string) {
'Cache-Control': 'no-cache',
},
});
-}
\ No newline at end of file
+}
+export async function DELETE(request: NextRequest) {
+ try {
+ const bookId = request.nextUrl.searchParams.get('bookId');
+ if (!bookId) {
+ return NextResponse.json({ error: 'Missing bookId parameter' }, { status: 400 });
+ }
+
+ const docstoreDir = join(process.cwd(), 'docstore');
+ const intermediateDir = join(docstoreDir, `${bookId}-audiobook`);
+
+ // If directory doesn't exist, consider it already reset
+ if (!existsSync(intermediateDir)) {
+ return NextResponse.json({ success: true, existed: false });
+ }
+
+ // Recursively delete the entire audiobook directory
+ await rm(intermediateDir, { recursive: true, force: true });
+
+ return NextResponse.json({ success: true, existed: true });
+ } catch (error) {
+ console.error('Error resetting audiobook:', error);
+ return NextResponse.json(
+ { error: 'Failed to reset audiobook' },
+ { status: 500 }
+ );
+ }
+}
diff --git a/src/app/api/audio/convert/chapters/route.ts b/src/app/api/audiobook/status/route.ts
similarity index 66%
rename from src/app/api/audio/convert/chapters/route.ts
rename to src/app/api/audiobook/status/route.ts
index 76efd83..44e8fd6 100644
--- a/src/app/api/audio/convert/chapters/route.ts
+++ b/src/app/api/audiobook/status/route.ts
@@ -1,7 +1,8 @@
import { NextRequest, NextResponse } from 'next/server';
-import { readdir, readFile, rm } from 'fs/promises';
+import { readdir, readFile } from 'fs/promises';
import { existsSync } from 'fs';
import { join } from 'path';
+import type { TTSAudiobookFormat } from '@/types/tts';
export async function GET(request: NextRequest) {
try {
@@ -26,7 +27,7 @@ export async function GET(request: NextRequest) {
duration?: number;
status: 'completed' | 'error';
bookId: string;
- format?: 'mp3' | 'm4b';
+ format?: TTSAudiobookFormat;
}> = [];
for (const metaFile of metaFiles) {
@@ -68,31 +69,3 @@ export async function GET(request: NextRequest) {
);
}
}
-
-export async function DELETE(request: NextRequest) {
- try {
- const bookId = request.nextUrl.searchParams.get('bookId');
- if (!bookId) {
- return NextResponse.json({ error: 'Missing bookId parameter' }, { status: 400 });
- }
-
- const docstoreDir = join(process.cwd(), 'docstore');
- const intermediateDir = join(docstoreDir, `${bookId}-audiobook`);
-
- // If directory doesn't exist, consider it already reset
- if (!existsSync(intermediateDir)) {
- return NextResponse.json({ success: true, existed: false });
- }
-
- // Recursively delete the entire audiobook directory
- await rm(intermediateDir, { recursive: true, force: true });
-
- return NextResponse.json({ success: true, existed: true });
- } catch (error) {
- console.error('Error resetting audiobook:', error);
- return NextResponse.json(
- { error: 'Failed to reset audiobook' },
- { status: 500 }
- );
- }
-}
diff --git a/src/app/api/tts/route.ts b/src/app/api/tts/route.ts
index a075d41..ff3680b 100644
--- a/src/app/api/tts/route.ts
+++ b/src/app/api/tts/route.ts
@@ -4,7 +4,8 @@ import { SpeechCreateParams } from 'openai/resources/audio/speech.mjs';
import { isKokoroModel } from '@/utils/voice';
import { LRUCache } from 'lru-cache';
import { createHash } from 'crypto';
-import type { TTSRequestPayload, TTSError } from '@/types/tts';
+import type { TTSRequestPayload } from '@/types/client';
+import type { TTSError, TTSAudioBuffer } from '@/types/tts';
export const runtime = 'nodejs';
@@ -13,7 +14,7 @@ type ExtendedSpeechParams = Omit & {
voice: SpeechCreateParams['voice'] | CustomVoice;
instructions?: string;
};
-type AudioBufferValue = ArrayBuffer;
+type AudioBufferValue = TTSAudioBuffer;
const TTS_CACHE_MAX_SIZE_BYTES = Number(process.env.TTS_CACHE_MAX_SIZE_BYTES || 256 * 1024 * 1024); // 256MB
const TTS_CACHE_TTL_MS = Number(process.env.TTS_CACHE_TTL_MS || 1000 * 60 * 30); // 30 minutes
@@ -25,7 +26,7 @@ const ttsAudioCache = new LRUCache({
});
type InflightEntry = {
- promise: Promise;
+ promise: Promise;
controller: AbortController;
consumers: number;
};
@@ -40,7 +41,7 @@ async function fetchTTSBufferWithRetry(
openai: OpenAI,
createParams: ExtendedSpeechParams,
signal: AbortSignal
-): Promise {
+): Promise {
let attempt = 0;
const maxRetries = Number(process.env.TTS_MAX_RETRIES ?? 2);
let delay = Number(process.env.TTS_RETRY_INITIAL_MS ?? 250);
@@ -135,7 +136,7 @@ export async function POST(req: NextRequest) {
voice: normalizedVoice,
input: text,
speed: speed,
- response_format: format === 'aac' ? 'aac' : 'mp3',
+ response_format: format,
};
// Only add instructions if model is gpt-4o-mini-tts and instructions are provided
if ((model as string) === 'gpt-4o-mini-tts' && instructions) {
@@ -143,7 +144,7 @@ export async function POST(req: NextRequest) {
}
// Compute cache key and check LRU before making provider call
- const contentType = format === 'aac' ? 'audio/aac' : 'audio/mpeg';
+ const contentType = 'audio/mpeg';
// Preserve voice string as-is for cache key (no weight stripping)
const voiceForKey = typeof createParams.voice === 'string'
@@ -245,7 +246,7 @@ export async function POST(req: NextRequest) {
};
req.signal.addEventListener('abort', onAbort, { once: true });
- let buffer: ArrayBuffer;
+ let buffer: TTSAudioBuffer;
try {
buffer = await entry.promise;
} finally {
@@ -280,4 +281,4 @@ export async function POST(req: NextRequest) {
{ status: 500 }
);
}
-}
\ No newline at end of file
+}
diff --git a/src/app/api/whisper/route.ts b/src/app/api/whisper/route.ts
new file mode 100644
index 0000000..a0d0da7
--- /dev/null
+++ b/src/app/api/whisper/route.ts
@@ -0,0 +1,450 @@
+import { NextRequest, NextResponse } from 'next/server';
+import { createHash, randomUUID } from 'crypto';
+import { mkdtemp, writeFile, rm, access, mkdir, readFile } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { spawn } from 'child_process';
+import type { TTSSentenceAlignment, TTSAudioBytes, TTSAudioBuffer } from '@/types/tts';
+import { preprocessSentenceForAudio } from '@/lib/nlp';
+
+export const runtime = 'nodejs';
+
+interface WhisperRequestBody {
+ text: string;
+ audio: TTSAudioBytes; // raw bytes from Uint8Array
+ lang?: string;
+}
+
+interface WhisperAlignmentOptions {
+ engine?: 'whisper.cpp';
+ lang?: string;
+}
+
+interface WhisperWord {
+ start: number;
+ end: number;
+ word: string;
+}
+
+// Simple in-memory cache keyed by a hash of text+lang+audio length
+const alignmentCache = new Map();
+
+// Model management using a fixed tiny.en GGML model stored under docstore/model
+const MODEL_NAME = 'ggml-tiny.en.bin';
+const MODEL_URL =
+ 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin';
+const DOCSTORE_DIR = join(process.cwd(), 'docstore');
+const MODEL_DIR = join(DOCSTORE_DIR, 'model');
+const MODEL_PATH = join(MODEL_DIR, MODEL_NAME);
+const modelReadyPromises = new Map>();
+
+async function ensureModelAvailable(): Promise {
+ // Fast path: model already present
+ try {
+ await access(MODEL_PATH);
+ return;
+ } catch {
+ // fall through to download
+ }
+
+ const existing = modelReadyPromises.get(MODEL_PATH);
+ if (existing) return existing;
+
+ const promise = (async () => {
+ try {
+ await access(MODEL_PATH);
+ return;
+ } catch {
+ // still missing
+ }
+
+ await mkdir(MODEL_DIR, { recursive: true });
+
+ const res = await fetch(MODEL_URL);
+ if (!res.ok) {
+ throw new Error(
+ `Failed to download Whisper model from ${MODEL_URL}: ${res.status} ${res.statusText}`
+ );
+ }
+
+ const arrayBuffer = await res.arrayBuffer();
+ await writeFile(MODEL_PATH, Buffer.from(arrayBuffer));
+ })();
+
+ modelReadyPromises.set(MODEL_PATH, promise);
+ return promise;
+}
+
+async function runWhisperCpp(
+ wavPath: string,
+ opts: WhisperAlignmentOptions
+): Promise {
+ const binary = process.env.WHISPER_CPP_BIN;
+ if (!binary) {
+ throw new Error(
+ 'Whisper.cpp binary path not configured. Set WHISPER_CPP_BIN to the compiled binary.'
+ );
+ }
+
+ await ensureModelAvailable();
+
+ return new Promise((resolve, reject) => {
+ const jsonBase = `${wavPath}.json_out`;
+ const jsonPath = `${jsonBase}.json`;
+ // Request full JSON output (including token-level details when available)
+ // and ask whisper-cli not to print anything to stdout.
+ const args = [
+ '-m',
+ MODEL_PATH,
+ '-f',
+ wavPath,
+ '-of',
+ jsonBase,
+ '-ojf',
+ '-np',
+ ];
+
+ if (opts.lang) {
+ args.push('-l', opts.lang);
+ }
+
+ const child = spawn(binary, args);
+
+ let stdout = '';
+ let stderr = '';
+
+ child.stdout.on('data', (data) => {
+ stdout += data.toString();
+ });
+
+ child.stderr.on('data', (data) => {
+ stderr += data.toString();
+ });
+
+ child.on('error', (err) => {
+ reject(err);
+ });
+
+ child.on('close', (code) => {
+ if (code !== 0) {
+ console.error('whisper-cli exited with error', {
+ code,
+ wavPath,
+ stdout: stdout.slice(0, 500),
+ stderr: stderr.slice(0, 500),
+ });
+ return reject(
+ new Error(
+ `whisper.cpp exited with code ${code}: ${stderr || stdout}`
+ )
+ );
+ }
+
+ readFile(jsonPath, 'utf-8')
+ .then((content: string) => {
+ const words: WhisperWord[] = [];
+ const parsed = JSON.parse(content) as {
+ transcription?: Array<{
+ text?: string;
+ timestamps?: { from?: string; to?: string };
+ offsets?: { from?: number; to?: number };
+ tokens?: Array<{
+ text?: string;
+ timestamps?: { from?: string; to?: string };
+ offsets?: { from?: number; to?: number };
+ }>;
+ }>;
+ };
+
+ const transcription = parsed.transcription;
+
+ const parseTimecode = (value?: string): number | null => {
+ if (!value) return null;
+ const m = value.match(/(\d+):(\d+):(\d+),(\d+)/);
+ if (!m) return null;
+ const h = Number(m[1]);
+ const min = Number(m[2]);
+ const s = Number(m[3]);
+ const ms = Number(m[4]);
+ if (
+ Number.isNaN(h) ||
+ Number.isNaN(min) ||
+ Number.isNaN(s) ||
+ Number.isNaN(ms)
+ ) {
+ return null;
+ }
+ return h * 3600 + min * 60 + s + ms / 1000;
+ };
+
+ if (Array.isArray(transcription)) {
+ for (const seg of transcription) {
+ const segText = (seg.text || '').trim();
+ const segStartSecFromTs = parseTimecode(
+ seg.timestamps?.from
+ );
+ const segEndSecFromTs = parseTimecode(seg.timestamps?.to);
+ const segStartSecFromMs =
+ typeof seg.offsets?.from === 'number'
+ ? seg.offsets.from / 1000
+ : null;
+ const segEndSecFromMs =
+ typeof seg.offsets?.to === 'number'
+ ? seg.offsets.to / 1000
+ : null;
+
+ const segStartSec =
+ segStartSecFromTs ??
+ segStartSecFromMs ??
+ 0;
+ const segEndSec =
+ segEndSecFromTs ??
+ segEndSecFromMs ??
+ segStartSec;
+
+ const tokens = Array.isArray(seg.tokens)
+ ? seg.tokens
+ : [];
+
+ if (tokens.length > 0) {
+ for (const token of tokens) {
+ const rawText = token.text || '';
+ const tokenText = rawText.trim();
+ // Skip special markers like [_BEG_]
+ if (!tokenText || /^\[.*\]$/.test(tokenText)) continue;
+
+ const tokStartSecFromTs = parseTimecode(
+ token.timestamps?.from
+ );
+ const tokEndSecFromTs = parseTimecode(
+ token.timestamps?.to
+ );
+ const tokStartSecFromMs =
+ typeof token.offsets?.from === 'number'
+ ? token.offsets.from / 1000
+ : null;
+ const tokEndSecFromMs =
+ typeof token.offsets?.to === 'number'
+ ? token.offsets.to / 1000
+ : null;
+
+ const startSec =
+ tokStartSecFromTs ??
+ tokStartSecFromMs ??
+ segStartSec;
+ const endSec =
+ tokEndSecFromTs ??
+ tokEndSecFromMs ??
+ segEndSec;
+
+ words.push({
+ word: tokenText,
+ start: startSec,
+ end: endSec,
+ });
+ }
+ } else if (segText) {
+ // Fallback: no token list, approximate per-word timing within the segment
+ const segTokens = segText.split(/\s+/).filter(Boolean);
+ if (segTokens.length) {
+ const totalDur = Math.max(segEndSec - segStartSec, 0);
+ const step =
+ segTokens.length > 0
+ ? totalDur / segTokens.length
+ : 0;
+ segTokens.forEach((token, index) => {
+ const wStart =
+ step > 0
+ ? segStartSec + step * index
+ : segStartSec;
+ const wEnd =
+ step > 0
+ ? index === segTokens.length - 1
+ ? segEndSec
+ : segStartSec + step * (index + 1)
+ : segEndSec;
+ words.push({
+ word: token,
+ start: wStart,
+ end: wEnd,
+ });
+ });
+ }
+ }
+ }
+ }
+
+ resolve(words);
+ })
+ .catch((err: unknown) => {
+ reject(err);
+ });
+ });
+ });
+}
+
+function mapWordsToSentenceOffsets(
+ sentence: string,
+ words: WhisperWord[]
+): TTSSentenceAlignment {
+ const normalizedSentence = preprocessSentenceForAudio(sentence);
+ let cursor = 0;
+
+ const alignedWords = words.map((w) => {
+ const token = w.word.trim();
+ if (!token) {
+ return {
+ text: '',
+ startSec: w.start,
+ endSec: w.end,
+ charStart: cursor,
+ charEnd: cursor,
+ };
+ }
+
+ const idx = normalizedSentence
+ .toLowerCase()
+ .indexOf(token.toLowerCase(), cursor);
+
+ const start =
+ idx !== -1
+ ? idx
+ : cursor;
+ const end = start + token.length;
+
+ cursor = end;
+
+ return {
+ text: token,
+ startSec: w.start,
+ endSec: w.end,
+ charStart: start,
+ charEnd: end,
+ };
+ });
+
+ return {
+ sentence,
+ sentenceIndex: 0,
+ words: alignedWords.filter((w) => w.text.length > 0),
+ };
+}
+
+async function alignAudioWithText(
+ audioBuffer: TTSAudioBuffer,
+ text: string,
+ cacheKey?: string,
+ opts: WhisperAlignmentOptions = {}
+): Promise {
+ if (!text.trim()) {
+ return [];
+ }
+
+ if (cacheKey && alignmentCache.has(cacheKey)) {
+ return alignmentCache.get(cacheKey)!;
+ }
+
+ const tmpBase = await mkdtemp(join(tmpdir(), 'openreader-whisper-'));
+ const inputPath = join(tmpBase, `${randomUUID()}-input.bin`);
+ const wavPath = join(tmpBase, `${randomUUID()}-input.wav`);
+
+ try {
+ await writeFile(inputPath, Buffer.from(new Uint8Array(audioBuffer)));
+
+ await new Promise((resolve, reject) => {
+ const ffmpeg = spawn('ffmpeg', [
+ '-y',
+ '-i',
+ inputPath,
+ '-ar',
+ '16000',
+ '-ac',
+ '1',
+ wavPath,
+ ]);
+
+ let stderr = '';
+ ffmpeg.stderr.on('data', (data) => {
+ stderr += data.toString();
+ });
+
+ ffmpeg.on('error', (err) => {
+ reject(err);
+ });
+
+ ffmpeg.on('close', (code) => {
+ if (code === 0) {
+ resolve();
+ } else {
+ reject(
+ new Error(`ffmpeg failed with code ${code}: ${stderr}`)
+ );
+ }
+ });
+ });
+
+ const words = await runWhisperCpp(wavPath, opts);
+ const alignment = mapWordsToSentenceOffsets(text, words);
+ const result: TTSSentenceAlignment[] = [alignment];
+
+ if (cacheKey) {
+ alignmentCache.set(cacheKey, result);
+ }
+
+ return result;
+ } finally {
+ try {
+ await rm(tmpBase, { recursive: true, force: true });
+ } catch {
+ // ignore
+ }
+ }
+}
+
+function makeCacheKey(input: WhisperRequestBody) {
+ const hash = createHash('sha256')
+ .update(
+ JSON.stringify({
+ text: input.text,
+ lang: input.lang || '',
+ audioLen: input.audio?.length || 0,
+ })
+ )
+ .digest('hex');
+ return hash;
+}
+
+export async function POST(req: NextRequest) {
+ try {
+ const body = (await req.json()) as WhisperRequestBody;
+ const { text, audio, lang } = body;
+
+ if (!text || !audio || !Array.isArray(audio)) {
+ return NextResponse.json(
+ { error: 'Missing text or audio in request body' },
+ { status: 400 }
+ );
+ }
+
+ const cacheKey = makeCacheKey(body);
+ const audioBuffer = new Uint8Array(audio).buffer;
+
+ const alignments: TTSSentenceAlignment[] = await alignAudioWithText(
+ audioBuffer,
+ text,
+ cacheKey,
+ { engine: 'whisper.cpp', lang }
+ );
+
+ return NextResponse.json({ alignments }, { status: 200 });
+ } catch (error) {
+ console.error('Error in whisper route:', error);
+ return NextResponse.json(
+ {
+ error: 'WHISPER_ALIGNMENT_FAILED',
+ message: 'Failed to compute word-level alignment',
+ },
+ { status: 500 }
+ );
+ }
+}
diff --git a/src/app/epub/[id]/page.tsx b/src/app/epub/[id]/page.tsx
index 96f8b37..325b96c 100644
--- a/src/app/epub/[id]/page.tsx
+++ b/src/app/epub/[id]/page.tsx
@@ -14,6 +14,7 @@ import TTSPlayer from '@/components/player/TTSPlayer';
import { ZoomControl } from '@/components/ZoomControl';
import { AudiobookExportModal } from '@/components/AudiobookExportModal';
import { DownloadIcon } from '@/components/icons/Icons';
+import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts';
const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null;
@@ -86,8 +87,8 @@ export default function EPUBPage() {
const handleGenerateAudiobook = useCallback(async (
onProgress: (progress: number) => void,
signal: AbortSignal,
- onChapterComplete: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void,
- format: 'mp3' | 'm4b'
+ onChapterComplete: (chapter: TTSAudiobookChapter) => void,
+ format: TTSAudiobookFormat
) => {
return createEPUBAudioBook(onProgress, signal, onChapterComplete, id as string, format);
}, [createEPUBAudioBook, id]);
@@ -95,7 +96,7 @@ export default function EPUBPage() {
const handleRegenerateChapter = useCallback(async (
chapterIndex: number,
bookId: string,
- format: 'mp3' | 'm4b',
+ format: TTSAudiobookFormat,
signal: AbortSignal
) => {
return regenerateEPUBChapter(chapterIndex, bookId, format, signal);
@@ -190,4 +191,4 @@ export default function EPUBPage() {
);
-}
\ No newline at end of file
+}
diff --git a/src/app/page.tsx b/src/app/page.tsx
index 286c7e5..569a797 100644
--- a/src/app/page.tsx
+++ b/src/app/page.tsx
@@ -1,7 +1,7 @@
import { HomeContent } from '@/components/HomeContent';
import { SettingsModal } from '@/components/SettingsModal';
-// Home page redesigned for fullscreen layout: hero + document area.
+const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null;
export default function Home() {
return (
@@ -10,9 +10,9 @@ export default function Home() {
OpenReader WebUI
-
- Bring your own text-to-speech API.
- Read & listen to PDF, EPUB & HTML documents with high quality voices.
+
+ Open source document reader {isDev ? 'self-hosted server' : 'demo app'}.
+ Read & listen to PDF, EPUB, MD, and TXT documents with high quality text to speech voices.
diff --git a/src/app/pdf/[id]/page.tsx b/src/app/pdf/[id]/page.tsx
index 7cd6957..276dc52 100644
--- a/src/app/pdf/[id]/page.tsx
+++ b/src/app/pdf/[id]/page.tsx
@@ -12,6 +12,7 @@ import { SettingsIcon, DownloadIcon } from '@/components/icons/Icons';
import { Header } from '@/components/Header';
import { ZoomControl } from '@/components/ZoomControl';
import { AudiobookExportModal } from '@/components/AudiobookExportModal';
+import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts';
import TTSPlayer from '@/components/player/TTSPlayer';
const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null;
@@ -80,8 +81,8 @@ export default function PDFViewerPage() {
const handleGenerateAudiobook = useCallback(async (
onProgress: (progress: number) => void,
signal: AbortSignal,
- onChapterComplete: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void,
- format: 'mp3' | 'm4b'
+ onChapterComplete: (chapter: TTSAudiobookChapter) => void,
+ format: TTSAudiobookFormat
) => {
return createPDFAudioBook(onProgress, signal, onChapterComplete, id as string, format);
}, [createPDFAudioBook, id]);
@@ -89,7 +90,7 @@ export default function PDFViewerPage() {
const handleRegenerateChapter = useCallback(async (
chapterIndex: number,
bookId: string,
- format: 'mp3' | 'm4b',
+ format: TTSAudiobookFormat,
signal: AbortSignal
) => {
return regeneratePDFChapter(chapterIndex, bookId, format, signal);
diff --git a/src/components/AudiobookExportModal.tsx b/src/components/AudiobookExportModal.tsx
index 51fc052..d9712bd 100644
--- a/src/components/AudiobookExportModal.tsx
+++ b/src/components/AudiobookExportModal.tsx
@@ -9,7 +9,14 @@ import { DownloadIcon, CheckCircleIcon, XCircleIcon, ClockIcon, ChevronUpDownIco
import { ConfirmDialog } from '@/components/ConfirmDialog';
import { LoadingSpinner } from '@/components/Spinner';
import { useConfig } from '@/contexts/ConfigContext';
-import type { TTSAudiobookChapter } from '@/types/tts';
+import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts';
+import {
+ getAudiobookStatus,
+ deleteAudiobookChapter,
+ deleteAudiobook,
+ downloadAudiobookChapter,
+ downloadAudiobook
+} from '@/lib/client';
interface AudiobookExportModalProps {
isOpen: boolean;
setIsOpen: (isOpen: boolean) => void;
@@ -19,12 +26,12 @@ interface AudiobookExportModalProps {
onProgress: (progress: number) => void,
signal: AbortSignal,
onChapterComplete: (chapter: TTSAudiobookChapter) => void,
- format: 'mp3' | 'm4b'
+ format: TTSAudiobookFormat
) => Promise; // Returns bookId
onRegenerateChapter?: (
chapterIndex: number,
bookId: string,
- format: 'mp3' | 'm4b',
+ format: TTSAudiobookFormat,
signal: AbortSignal
) => Promise;
}
@@ -46,7 +53,7 @@ export function AudiobookExportModal({
const [isLoadingExisting, setIsLoadingExisting] = useState(false);
const [isRefreshingChapters, setIsRefreshingChapters] = useState(false);
const [currentChapter, setCurrentChapter] = useState('');
- const [format, setFormat] = useState<'mp3' | 'm4b'>('m4b');
+ const [format, setFormat] = useState('m4b');
const [regeneratingChapter, setRegeneratingChapter] = useState(null);
const abortControllerRef = useRef(null);
const [pendingDeleteChapter, setPendingDeleteChapter] = useState(null);
@@ -61,26 +68,23 @@ export function AudiobookExportModal({
setIsLoadingExisting(true);
}
try {
- const response = await fetch(`/api/audio/convert/chapters?bookId=${documentId}`);
- if (response.ok) {
- const data = await response.json();
- if (data.exists && data.chapters.length > 0) {
- setChapters(data.chapters);
- setBookId(data.bookId);
- // Set format from existing chapters - this ensures the format matches what was actually generated
- if (data.chapters[0]?.format) {
- const detectedFormat = data.chapters[0].format as 'mp3' | 'm4b';
- setFormat(detectedFormat);
- }
- // If we have a complete audiobook, we're done
- if (data.hasComplete) {
- setProgress(100);
- }
- } else {
- // If nothing exists, clear chapters/bookId to reflect current state
- setChapters([]);
- setBookId(null);
+ const data = await getAudiobookStatus(documentId);
+ if (data.exists && data.chapters.length > 0) {
+ setChapters(data.chapters);
+ setBookId(data.bookId);
+ // Set format from existing chapters - this ensures the format matches what was actually generated
+ if (data.chapters[0]?.format) {
+ const detectedFormat = data.chapters[0].format as TTSAudiobookFormat;
+ setFormat(detectedFormat);
}
+ // If we have a complete audiobook, we're done
+ if (data.hasComplete) {
+ setProgress(100);
+ }
+ } else {
+ // If nothing exists, clear chapters/bookId to reflect current state
+ setChapters([]);
+ setBookId(null);
}
} catch (error) {
console.error('Error fetching existing chapters:', error);
@@ -241,12 +245,7 @@ export function AudiobookExportModal({
const performDeleteChapter = useCallback(async () => {
if (!bookId || !pendingDeleteChapter) return;
try {
- const response = await fetch(`/api/audio/convert/chapter?bookId=${bookId}&chapterIndex=${pendingDeleteChapter.index}`, {
- method: 'DELETE'
- });
- if (!response.ok) {
- throw new Error('Delete failed');
- }
+ await deleteAudiobookChapter(bookId, pendingDeleteChapter.index);
setChapters(prev => prev.filter(c => c.index !== pendingDeleteChapter.index));
await fetchExistingChapters(true);
} catch (error) {
@@ -261,10 +260,7 @@ export function AudiobookExportModal({
const targetBookId = bookId || documentId;
if (!targetBookId) return;
try {
- const resp = await fetch(`/api/audio/convert/chapters?bookId=${targetBookId}`, { method: 'DELETE' });
- if (!resp.ok) {
- throw new Error('Reset failed');
- }
+ await deleteAudiobook(targetBookId);
setChapters([]);
setBookId(null);
setProgress(0);
@@ -281,10 +277,7 @@ export function AudiobookExportModal({
if (!chapter.bookId) return;
try {
- const response = await fetch(`/api/audio/convert/chapter?bookId=${chapter.bookId}&chapterIndex=${chapter.index}`);
- if (!response.ok) throw new Error('Download failed');
-
- const blob = await response.blob();
+ const blob = await downloadAudiobookChapter(chapter.bookId, chapter.index);
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
@@ -306,8 +299,7 @@ export function AudiobookExportModal({
setIsCombining(true);
try {
- const response = await fetch(`/api/audio/convert?bookId=${bookId}&format=${format}`);
- if (!response.ok) throw new Error('Download failed');
+ const response = await downloadAudiobook(bookId, format);
const reader = response.body?.getReader();
if (!reader) throw new Error('No response body');
diff --git a/src/components/CodeBlock.tsx b/src/components/CodeBlock.tsx
new file mode 100644
index 0000000..a887c96
--- /dev/null
+++ b/src/components/CodeBlock.tsx
@@ -0,0 +1,32 @@
+'use client';
+
+import { useState } from 'react';
+import { CopyIcon, CheckIcon } from '@/components/icons/Icons';
+
+export function CodeBlock({ children }: { children: string }) {
+ const [copied, setCopied] = useState(false);
+
+ const handleCopy = async () => {
+ await navigator.clipboard.writeText(children);
+ setCopied(true);
+ setTimeout(() => setCopied(false), 2000);
+ };
+
+ return (
+