From b89d1700e9b022386f9aa8f19828d6bbe7993956 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Sat, 1 Mar 2025 18:21:51 -0700 Subject: [PATCH] Fix for longer audiobooks --- src/app/api/audio/convert/route.ts | 225 +++++++------ src/components/DocumentSettings.tsx | 476 +++++++++++++--------------- src/components/ProgressPopup.tsx | 66 ++++ src/contexts/EPUBContext.tsx | 53 +--- src/contexts/PDFContext.tsx | 53 +--- src/utils/audio.ts | 82 ++++- 6 files changed, 505 insertions(+), 450 deletions(-) create mode 100644 src/components/ProgressPopup.tsx diff --git a/src/app/api/audio/convert/route.ts b/src/app/api/audio/convert/route.ts index 069325c..e57dcd2 100644 --- a/src/app/api/audio/convert/route.ts +++ b/src/app/api/audio/convert/route.ts @@ -1,18 +1,14 @@ import { NextRequest, NextResponse } from 'next/server'; import { spawn } from 'child_process'; -import { writeFile, mkdir, unlink, rmdir } from 'fs/promises'; -import { createReadStream } from 'fs'; -import { existsSync } from 'fs'; +import { writeFile, readFile, mkdir, unlink, rmdir, readdir } from 'fs/promises'; +import { existsSync, createReadStream } from 'fs'; import { join } from 'path'; import { randomUUID } from 'crypto'; -interface Chapter { - title: string; - buffer: number[]; -} - interface ConversionRequest { - chapters: Chapter[]; + chapterTitle: string; + buffer: number[]; + bookId?: string; } async function getAudioDuration(filePath: string): Promise { @@ -66,19 +62,9 @@ async function runFFmpeg(args: string[]): Promise { }); } -async function cleanup(files: string[], directories: string[]) { - await Promise.all([ - ...files.map(f => unlink(f).catch(console.error)), - ...directories.map(d => rmdir(d).catch(console.error)) - ]); -} - export async function POST(request: NextRequest) { - const tempFiles: string[] = []; - const tempDirs: string[] = []; - try { - // Parse the request body as a stream + // Parse the request body const data: ConversionRequest = await request.json(); // Create temp directory if it doesn't exist @@ -87,70 +73,100 @@ export async function POST(request: NextRequest) { await mkdir(tempDir); } - // Generate unique filenames - const id = randomUUID(); - const outputPath = join(tempDir, `${id}.m4b`); - const metadataPath = join(tempDir, `${id}.txt`); - const intermediateDir = join(tempDir, `${id}-intermediate`); + // Generate or use existing book ID + const bookId = data.bookId || randomUUID(); + const intermediateDir = join(tempDir, `${bookId}-intermediate`); - tempFiles.push(outputPath, metadataPath); - tempDirs.push(intermediateDir); - // Create intermediate directory if (!existsSync(intermediateDir)) { await mkdir(intermediateDir); } - // Process chapters sequentially to avoid memory issues - const chapterFiles: { path: string; title: string; duration: number }[] = []; - let currentTime = 0; + // Count existing files to determine chapter index + const files = await readdir(intermediateDir); + const wavFiles = files.filter(f => f.endsWith('.wav')); + const chapterIndex = wavFiles.length; - for (let i = 0; i < data.chapters.length; i++) { - const chapter = data.chapters[i]; - const inputPath = join(intermediateDir, `${i}-input.mp3`); - const outputPath = join(intermediateDir, `${i}.aac`); - - tempFiles.push(inputPath, outputPath); + // Write input file + const inputPath = join(intermediateDir, `${chapterIndex}-input.aac`); + const outputPath = join(intermediateDir, `${chapterIndex}.wav`); + const metadataPath = join(intermediateDir, `${chapterIndex}.meta.json`); + + // Write the chapter audio to a temp file + await writeFile(inputPath, Buffer.from(new Uint8Array(data.buffer))); + + // Convert to WAV from raw aac with consistent format + await runFFmpeg([ + '-i', inputPath, + '-f', 'wav', + '-c:a', 'copy', + '-preset', 'ultrafast', + '-threads', '0', + outputPath + ]); - // Write the chapter audio to a temp file using a Buffer chunk size of 64KB - const chunkSize = 64 * 1024; // 64KB chunks - const buffer = Buffer.from(new Uint8Array(chapter.buffer)); - const chunks: Buffer[] = []; - - for (let offset = 0; offset < buffer.length; offset += chunkSize) { - chunks.push(buffer.slice(offset, offset + chunkSize)); - } - - await writeFile(inputPath, Buffer.concat(chunks)); - chunks.length = 0; // Clear chunks array + // Get the duration and save metadata + const duration = await getAudioDuration(outputPath); + await writeFile(metadataPath, JSON.stringify({ + title: data.chapterTitle, + duration, + index: chapterIndex + })); - // Copy to AAC format for compatibility with M4B - await runFFmpeg([ - '-i', inputPath, - '-c:a', 'copy', // Use copy instead of re-encoding - outputPath - ]); - - const duration = await getAudioDuration(outputPath); - - chapterFiles.push({ - path: outputPath, - title: chapter.title, - duration - }); + // Clean up input file + await unlink(inputPath).catch(console.error); - // Clean up input file early - await unlink(inputPath).catch(console.error); - const index = tempFiles.indexOf(inputPath); - if (index > -1) { - tempFiles.splice(index, 1); - } + return NextResponse.json({ + bookId, + chapterIndex, + duration + }); + + } catch (error) { + console.error('Error processing audio chapter:', error); + return NextResponse.json( + { error: 'Failed to process audio chapter' }, + { status: 500 } + ); + } +} + +export async function GET(request: NextRequest) { + try { + const bookId = request.nextUrl.searchParams.get('bookId'); + if (!bookId) { + return NextResponse.json({ error: 'Missing bookId parameter' }, { status: 400 }); } + const tempDir = join(process.cwd(), 'temp'); + const intermediateDir = join(tempDir, `${bookId}-intermediate`); + const outputPath = join(tempDir, `${bookId}.m4b`); + const metadataPath = join(tempDir, `${bookId}-metadata.txt`); + const listPath = join(tempDir, `${bookId}-list.txt`); + + if (!existsSync(intermediateDir)) { + return NextResponse.json({ error: 'Book not found' }, { status: 404 }); + } + + // Read all chapter metadata + const files = await readdir(intermediateDir); + const metaFiles = files.filter(f => f.endsWith('.meta.json')); + const chapters: { title: string; duration: number; index: number }[] = []; + + for (const metaFile of metaFiles) { + const meta = JSON.parse(await readFile(join(intermediateDir, metaFile), 'utf-8')); + chapters.push(meta); + } + + // Sort chapters by index + chapters.sort((a, b) => a.index - b.index); + // Create chapter metadata file const metadata: string[] = []; + let currentTime = 0; - chapterFiles.forEach((chapter) => { + // Calculate chapter timings based on actual durations + chapters.forEach((chapter) => { const startMs = Math.floor(currentTime * 1000); currentTime += chapter.duration; const endMs = Math.floor(currentTime * 1000); @@ -167,72 +183,73 @@ export async function POST(request: NextRequest) { await writeFile(metadataPath, ';FFMETADATA1\n' + metadata.join('\n')); // Create list file for concat - const listPath = join(tempDir, `${id}-list.txt`); - tempFiles.push(listPath); - await writeFile( listPath, - chapterFiles.map(f => `file '${f.path}'`).join('\n') + chapters.map(c => `file '${join(intermediateDir, `${c.index}.wav`)}'`).join('\n') ); - // Combine all files into a single M4B with optimized settings + // Combine all files into a single M4B await runFFmpeg([ '-f', 'concat', '-safe', '0', '-i', listPath, '-i', metadataPath, '-map_metadata', '1', - '-c:a', 'copy', // Use macOS AudioToolbox AAC encoder + '-c:a', 'copy', // c:a is codec for audio and :a is stream specifier + //'-codec', 'wav', //'-b:a', '192k', - '-threads', '0', // Use maximum available threads - '-movflags', '+faststart', - '-preset', 'ultrafast', // Use fastest encoding preset + //'-threads', '0', // Use maximum available threads + //'-movflags', '+faststart', + //'-preset', 'ultrafast', // Use fastest encoding preset outputPath ]); - // Create a readable stream from the output file - const fileStream = createReadStream(outputPath); + // Stream the file back to the client + const stream = createReadStream(outputPath); + + // Clean up function + const cleanup = async () => { + try { + await Promise.all([ + ...chapters.map(c => unlink(join(intermediateDir, `${c.index}.wav`))), + ...chapters.map(c => unlink(join(intermediateDir, `${c.index}.meta.json`))), + unlink(metadataPath), + unlink(listPath), + unlink(outputPath), + rmdir(intermediateDir) + ]); + } catch (error) { + console.error('Cleanup error:', error); + } + }; - // Create a web-compatible ReadableStream from the Node.js stream - const webStream = new ReadableStream({ + // Clean up after streaming is complete + stream.on('end', cleanup); + + const readableWebStream = new ReadableStream({ start(controller) { - fileStream.on('data', (chunk) => { + stream.on('data', (chunk) => { controller.enqueue(chunk); }); - - fileStream.on('end', () => { + stream.on('end', () => { controller.close(); - // Clean up only after the stream has been fully sent - cleanup(tempFiles, tempDirs).catch(console.error); }); - - fileStream.on('error', (error) => { - console.error('Stream error:', error); - controller.error(error); - cleanup(tempFiles, tempDirs).catch(console.error); + stream.on('error', (err) => { + controller.error(err); }); }, - cancel() { - fileStream.destroy(); - cleanup(tempFiles, tempDirs).catch(console.error); - } }); - // Return the streaming response - return new NextResponse(webStream, { + return new NextResponse(readableWebStream, { headers: { 'Content-Type': 'audio/mp4', - 'Transfer-Encoding': 'chunked' }, }); } catch (error) { - // Clean up in case of error - await cleanup(tempFiles, tempDirs).catch(console.error); - - console.error('Error converting audio:', error); + console.error('Error creating M4B:', error); return NextResponse.json( - { error: 'Failed to convert audio format' }, + { error: 'Failed to create M4B file' }, { status: 500 } ); } diff --git a/src/components/DocumentSettings.tsx b/src/components/DocumentSettings.tsx index 1d72c87..4ffa295 100644 --- a/src/components/DocumentSettings.tsx +++ b/src/components/DocumentSettings.tsx @@ -7,7 +7,7 @@ import { ChevronUpDownIcon, CheckIcon } from '@/components/icons/Icons'; import { useEPUB } from '@/contexts/EPUBContext'; import { usePDF } from '@/contexts/PDFContext'; import { useTimeEstimation } from '@/hooks/useTimeEstimation'; -import { LoadingSpinner } from './Spinner'; +import { ProgressPopup } from '@/components/ProgressPopup'; const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null; @@ -127,48 +127,60 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro }; return ( - - setIsOpen(false)}> - -
- + <> + + + + setIsOpen(false)}> + +
+ -
-
- - - {isDev &&
- {!isGenerating ? ( +
+
+ + + {isDev &&
setAudioFormat(format as 'mp3' | 'm4b')}>
- + {audioFormat === 'mp3' ? 'MP3' : 'M4B (Audiobook)'} @@ -188,233 +200,203 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro
- ) : ( -
-
-
-
-
- - {Math.round(progress)}% complete - {estimatedTimeRemaining && ` • ${estimatedTimeRemaining} remaining`} - - -
-
- )} -
} -
- {!epub &&
-
- -
- {/* Header Margin */} -
-
- Header - {Math.round(localMargins.header * 100)}% -
- -
- - {/* Footer Margin */} -
-
- Footer - {Math.round(localMargins.footer * 100)}% -
- -
- - {/* Left Margin */} -
-
- Left - {Math.round(localMargins.left * 100)}% -
- -
- - {/* Right Margin */} -
-
- Right - {Math.round(localMargins.right * 100)}% -
- -
-
-

- Adjust margins to exclude content from edges of the page during text extraction (experimental) -

-
- updateConfigKey('viewType', newView.id as ViewType)} - > -
- - - {selectedView.name} - - - - - - - {viewTypes.map((view) => ( - - `relative cursor-pointer select-none py-2 pl-10 pr-4 ${active ? 'bg-accent/10 text-accent' : 'text-foreground' - }` - } - value={view} - > - {({ selected }) => ( - <> - - {view.name} - - {selected ? ( - - - - ) : null} - - )} - - ))} - - - {selectedView.id === 'scroll' && ( -

- Note: Continuous scroll may perform poorly for larger documents. -

- )} -
- -
} -
- -

- Automatically skip pages with no text content -

-
- {epub && ( +
+ {!epub &&
+
+ +
+ {/* Header Margin */} +
+
+ Header + {Math.round(localMargins.header * 100)}% +
+ +
+ + {/* Footer Margin */} +
+
+ Footer + {Math.round(localMargins.footer * 100)}% +
+ +
+ + {/* Left Margin */} +
+
+ Left + {Math.round(localMargins.left * 100)}% +
+ +
+ + {/* Right Margin */} +
+
+ Right + {Math.round(localMargins.right * 100)}% +
+ +
+
+

+ Adjust margins to exclude content from edges of the page during text extraction (experimental) +

+
+ updateConfigKey('viewType', newView.id as ViewType)} + > +
+ + + {selectedView.name} + + + + + + + {viewTypes.map((view) => ( + + `relative cursor-pointer select-none py-2 pl-10 pr-4 ${active ? 'bg-accent/10 text-accent' : 'text-foreground' + }` + } + value={view} + > + {({ selected }) => ( + <> + + {view.name} + + {selected ? ( + + + + ) : null} + + )} + + ))} + + + {selectedView.id === 'scroll' && ( +

+ Note: Continuous scroll may perform poorly for larger documents. +

+ )} +
+ + +
} +

- Apply the current app theme to the EPUB viewer + Automatically skip pages with no text content

- )} -
+ {epub && ( +
+ +

+ Apply the current app theme to the EPUB viewer +

+
+ )} +
-
- -
- - +
+ +
+ + +
-
-
- +
+ + ); } diff --git a/src/components/ProgressPopup.tsx b/src/components/ProgressPopup.tsx new file mode 100644 index 0000000..aedc4e2 --- /dev/null +++ b/src/components/ProgressPopup.tsx @@ -0,0 +1,66 @@ +import { Fragment } from 'react'; +import { Transition } from '@headlessui/react'; +import { LoadingSpinner } from './Spinner'; + +interface ProgressPopupProps { + isOpen: boolean; + progress: number; + estimatedTimeRemaining?: string; + onCancel: () => void; + isProcessing: boolean; +} + +export function ProgressPopup({ isOpen, progress, estimatedTimeRemaining, onCancel, isProcessing }: ProgressPopupProps) { + return ( + +
+
+
+
+
+
+
+
+
+ {Math.round(progress)}% complete + {estimatedTimeRemaining &&
+ • + {` ~${estimatedTimeRemaining}`} +
} +
+ +
+
+
+
+
+ + ); +} \ No newline at end of file diff --git a/src/contexts/EPUBContext.tsx b/src/contexts/EPUBContext.tsx index 81e7a29..9c3dbdd 100644 --- a/src/contexts/EPUBContext.tsx +++ b/src/contexts/EPUBContext.tsx @@ -19,6 +19,7 @@ import { setLastDocumentLocation } from '@/utils/indexedDB'; import { SpineItem } from 'epubjs/types/section'; import { useParams } from 'next/navigation'; import { useConfig } from './ConfigContext'; +import { combineAudioChunks } from '@/utils/audio'; interface EPUBContextType { currDocData: ArrayBuffer | undefined; @@ -230,7 +231,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { // Process each section for (let i = 0; i < sections.length; i++) { if (signal?.aborted) { - const partialBuffer = await combineAudioChunks(audioChunks, format); + const partialBuffer = await combineAudioChunks(audioChunks, format, setIsAudioCombining); return partialBuffer; } @@ -292,7 +293,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { } catch (error) { if (error instanceof Error && error.name === 'AbortError') { console.log('TTS request aborted'); - const partialBuffer = await combineAudioChunks(audioChunks, format); + const partialBuffer = await combineAudioChunks(audioChunks, format, setIsAudioCombining); return partialBuffer; } console.error('Error processing section:', error); @@ -303,59 +304,13 @@ export function EPUBProvider({ children }: { children: ReactNode }) { throw new Error('No audio was generated from the book content'); } - return combineAudioChunks(audioChunks, format); + return combineAudioChunks(audioChunks, format, setIsAudioCombining); } catch (error) { console.error('Error creating audiobook:', error); throw error; } }, [extractBookText, apiKey, baseUrl, voice, voiceSpeed]); - const combineAudioChunks = async ( - audioChunks: { buffer: ArrayBuffer; title?: string; startTime: number }[], - format: 'mp3' | 'm4b' - ): Promise => { - setIsAudioCombining(true); - try { - if (format === 'm4b') { - // Convert to M4B format using the audio conversion API - const response = await fetch('/api/audio/convert', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chapters: audioChunks - .filter(chunk => chunk.title) // Only include chunks with titles - .map(chunk => ({ - title: chunk.title, - buffer: Array.from(new Uint8Array(chunk.buffer)) - })) - }), - }); - - if (!response.ok) { - throw new Error('Failed to convert audio to M4B format'); - } - - return response.arrayBuffer(); - } - - // For MP3, just concatenate the buffers - const totalLength = audioChunks.reduce((acc, chunk) => acc + chunk.buffer.byteLength, 0); - const combinedBuffer = new Uint8Array(totalLength); - - let offset = 0; - for (const chunk of audioChunks) { - combinedBuffer.set(new Uint8Array(chunk.buffer), offset); - offset += chunk.buffer.byteLength; - } - - return combinedBuffer.buffer; - } finally { - setIsAudioCombining(false); - } - } - const setRendition = useCallback((rendition: Rendition) => { bookRef.current = rendition.book; renditionRef.current = rendition; diff --git a/src/contexts/PDFContext.tsx b/src/contexts/PDFContext.tsx index 53dc0e0..e789024 100644 --- a/src/contexts/PDFContext.tsx +++ b/src/contexts/PDFContext.tsx @@ -36,6 +36,7 @@ import { } from '@/utils/pdf'; import type { PDFDocumentProxy } from 'pdfjs-dist'; +import { combineAudioChunks } from '@/utils/audio'; /** * Interface defining all available methods and properties in the PDF context @@ -230,7 +231,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { // Second pass: process text into audio for (let i = 0; i < textPerPage.length; i++) { if (signal?.aborted) { - const partialBuffer = await combineAudioChunks(audioChunks, format); + const partialBuffer = await combineAudioChunks(audioChunks, format, setIsAudioCombining); return partialBuffer; } @@ -282,7 +283,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { } catch (error) { if (error instanceof Error && error.name === 'AbortError') { console.log('TTS request aborted'); - const partialBuffer = await combineAudioChunks(audioChunks, format); + const partialBuffer = await combineAudioChunks(audioChunks, format, setIsAudioCombining); return partialBuffer; } console.error('Error processing page:', error); @@ -293,59 +294,13 @@ export function PDFProvider({ children }: { children: ReactNode }) { throw new Error('No audio was generated from the PDF content'); } - return combineAudioChunks(audioChunks, format); + return combineAudioChunks(audioChunks, format, setIsAudioCombining); } catch (error) { console.error('Error creating audiobook:', error); throw error; } }, [pdfDocument, headerMargin, footerMargin, leftMargin, rightMargin, apiKey, baseUrl, voice, voiceSpeed]); - const combineAudioChunks = async ( - audioChunks: { buffer: ArrayBuffer; title?: string; startTime: number }[], - format: 'mp3' | 'm4b' - ): Promise => { - setIsAudioCombining(true); - try { - if (format === 'm4b') { - // Convert to M4B format using the audio conversion API - const response = await fetch('/api/audio/convert', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - chapters: audioChunks - .filter(chunk => chunk.title) // Only include chunks with titles - .map(chunk => ({ - title: chunk.title, - buffer: Array.from(new Uint8Array(chunk.buffer)) - })) - }), - }); - - if (!response.ok) { - throw new Error('Failed to convert audio to M4B format'); - } - - return response.arrayBuffer(); - } - - // For MP3, just concatenate the buffers - const totalLength = audioChunks.reduce((acc, chunk) => acc + chunk.buffer.byteLength, 0); - const combinedBuffer = new Uint8Array(totalLength); - - let offset = 0; - for (const chunk of audioChunks) { - combinedBuffer.set(new Uint8Array(chunk.buffer), offset); - offset += chunk.buffer.byteLength; - } - - return combinedBuffer.buffer; - } finally { - setIsAudioCombining(false); - } - } - // Context value memoization const contextValue = useMemo( () => ({ diff --git a/src/utils/audio.ts b/src/utils/audio.ts index 07dabeb..fd1802c 100644 --- a/src/utils/audio.ts +++ b/src/utils/audio.ts @@ -2,6 +2,12 @@ * Utility functions for audio processing */ +interface AudioChunk { + buffer: ArrayBuffer; + title?: string; + startTime: number; +} + /** * Creates a URL from an ArrayBuffer containing MP3 audio data * @param buffer The ArrayBuffer containing MP3 audio data @@ -10,4 +16,78 @@ export const audioBufferToURL = (buffer: ArrayBuffer): string => { const blob = new Blob([buffer], { type: 'audio/mp3' }); return URL.createObjectURL(blob); -}; \ No newline at end of file +}; + +/** + * Combines audio chunks into a single audio file + * @param audioChunks Array of audio chunks with metadata + * @param format Output format ('mp3' or 'm4b') + * @param setIsAudioCombining Optional callback to track combining state + * @returns Promise resolving to the combined audio buffer + */ +export const combineAudioChunks = async ( + audioChunks: AudioChunk[], + format: 'mp3' | 'm4b', + setIsAudioCombining?: (state: boolean) => void +): Promise => { + if (setIsAudioCombining) { + setIsAudioCombining(true); + } + + try { + if (format === 'm4b') { + // Filter out chunks without titles and silence buffers + const titledChunks = audioChunks.filter(chunk => chunk.title && chunk.buffer.byteLength > 48000); + + let bookId: string | undefined; + + // Upload each chunk sequentially and get book ID + for (const chunk of titledChunks) { + const response = await fetch('/api/audio/convert', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + chapterTitle: chunk.title, + buffer: Array.from(new Uint8Array(chunk.buffer)), + bookId // Will be undefined for first chunk, then set for subsequent ones + }), + }); + + if (!response.ok) { + throw new Error('Failed to upload audio chunk'); + } + + const result = await response.json(); + bookId = result.bookId; // Save book ID for subsequent chunks + } + + if (!bookId) { + throw new Error('No book ID received from server'); + } + + // Get the final combined M4B file + const m4bResponse = await fetch(`/api/audio/convert?bookId=${bookId}`); + if (!m4bResponse.ok) { + throw new Error('Failed to get combined M4B'); + } + + return await m4bResponse.arrayBuffer(); + } + + // For MP3, just concatenate the buffers + const totalLength = audioChunks.reduce((acc, chunk) => acc + chunk.buffer.byteLength, 0); + const combinedBuffer = new Uint8Array(totalLength); + + let offset = 0; + for (const chunk of audioChunks) { + combinedBuffer.set(new Uint8Array(chunk.buffer), offset); + offset += chunk.buffer.byteLength; + } + + return combinedBuffer.buffer; + } finally { + if (setIsAudioCombining) setIsAudioCombining(false); + } +} \ No newline at end of file