From 1db906f8bc65ed8b23ee1ca2367dd62d14118a65 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Mon, 24 Feb 2025 22:41:38 -0700 Subject: [PATCH 1/8] [ePub only] First working export audiobook --- src/components/DocumentSettings.tsx | 120 ++++++++++-- src/components/EPUBViewer.tsx | 92 ++------- src/contexts/EPUBContext.tsx | 294 +++++++++++++++++++++++++++- 3 files changed, 408 insertions(+), 98 deletions(-) diff --git a/src/components/DocumentSettings.tsx b/src/components/DocumentSettings.tsx index 3fad0b4..40bde50 100644 --- a/src/components/DocumentSettings.tsx +++ b/src/components/DocumentSettings.tsx @@ -1,9 +1,10 @@ 'use client'; -import { Fragment } from 'react'; +import { Fragment, useState, useRef } from 'react'; import { Dialog, DialogPanel, DialogTitle, Transition, TransitionChild, Listbox, ListboxButton, ListboxOptions, ListboxOption, Button } from '@headlessui/react'; import { useConfig, ViewType } from '@/contexts/ConfigContext'; import { ChevronUpDownIcon, CheckIcon } from '@/components/icons/Icons'; +import { useEPUB } from '@/contexts/EPUBContext'; interface DocViewSettingsProps { isOpen: boolean; @@ -19,8 +20,52 @@ const viewTypes = [ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsProps) { const { viewType, skipBlank, epubTheme, updateConfigKey } = useConfig(); + const { createFullAudioBook } = useEPUB(); + const [progress, setProgress] = useState(0); + const [isGenerating, setIsGenerating] = useState(false); + const abortControllerRef = useRef(null); const selectedView = viewTypes.find(v => v.id === viewType) || viewTypes[0]; + const handleStartGeneration = async () => { + setIsGenerating(true); + setProgress(0); + abortControllerRef.current = new AbortController(); + + try { + const audioBuffer = await createFullAudioBook( + (progress) => setProgress(progress), + abortControllerRef.current.signal + ); + + // Create and trigger download + const blob = new Blob([audioBuffer], { type: 'audio/mp3' }); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = 'audiobook.mp3'; + document.body.appendChild(a); + a.click(); + + // Clean up + setTimeout(() => { + document.body.removeChild(a); + URL.revokeObjectURL(url); + }, 100); + } catch (error) { + console.error('Error generating audiobook:', error); + } finally { + setIsGenerating(false); + setProgress(0); + abortControllerRef.current = null; + } + }; + + const handleCancel = () => { + if (abortControllerRef.current) { + abortControllerRef.current.abort(); + } + }; + return ( setIsOpen(false)}> @@ -58,8 +103,8 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro
{!epub &&
- updateConfigKey('viewType', newView.id as ViewType)} >
@@ -80,8 +125,7 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro - `relative cursor-pointer select-none py-2 pl-10 pr-4 ${ - active ? 'bg-accent/10 text-accent' : 'text-foreground' + `relative cursor-pointer select-none py-2 pl-10 pr-4 ${active ? 'bg-accent/10 text-accent' : 'text-foreground' }` } value={view} @@ -125,20 +169,58 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro

{epub && ( -
- -

- Apply the current app theme to the EPUB viewer -

-
+ <> +
+ +

+ Apply the current app theme to the EPUB viewer +

+
+
+ {!isGenerating ? ( + + ) : ( +
+
+
+
+
+ {Math.round(progress)}% complete + +
+
+ )} +
+ )}
diff --git a/src/components/EPUBViewer.tsx b/src/components/EPUBViewer.tsx index ec4a3f9..d8f6f55 100644 --- a/src/components/EPUBViewer.tsx +++ b/src/components/EPUBViewer.tsx @@ -1,15 +1,12 @@ 'use client'; import { useEffect, useRef, useCallback } from 'react'; -import { useParams } from 'next/navigation'; import dynamic from 'next/dynamic'; import { useEPUB } from '@/contexts/EPUBContext'; import { useTTS } from '@/contexts/TTSContext'; import { useConfig } from '@/contexts/ConfigContext'; import { DocumentSkeleton } from '@/components/DocumentSkeleton'; import TTSPlayer from '@/components/player/TTSPlayer'; -import { setLastDocumentLocation } from '@/utils/indexedDB'; -import type { Rendition, Book, NavItem } from 'epubjs'; import { useEPUBTheme, getThemeStyles } from '@/hooks/epub/useEPUBTheme'; import { useEPUBResize } from '@/hooks/epub/useEPUBResize'; @@ -23,88 +20,40 @@ interface EPUBViewerProps { } export function EPUBViewer({ className = '' }: EPUBViewerProps) { - const { id } = useParams(); - const { currDocData, currDocName, currDocPage, extractPageText } = useEPUB(); - const { skipToLocation, registerLocationChangeHandler, setIsEPUB, pause } = useTTS(); + const { + currDocData, + currDocName, + locationRef, + handleLocationChanged, + bookRef, + renditionRef, + tocRef, + setRendition, + extractPageText + } = useEPUB(); + const { registerLocationChangeHandler, pause } = useTTS(); const { epubTheme } = useConfig(); - const bookRef = useRef(null); - const rendition = useRef(undefined); - const toc = useRef([]); - const locationRef = useRef(currDocPage); - const { updateTheme } = useEPUBTheme(epubTheme, rendition.current); + const { updateTheme } = useEPUBTheme(epubTheme, renditionRef.current); const containerRef = useRef(null); - - const isEPUBSetOnce = useRef(false); const { isResizing, setIsResizing, dimensions } = useEPUBResize(containerRef); - const handleLocationChanged = useCallback((location: string | number) => { - // Set the EPUB flag once the location changes - if (!isEPUBSetOnce.current) { - setIsEPUB(true); - isEPUBSetOnce.current = true; - - rendition.current?.display(location.toString()); - - return; - } - - if (!bookRef.current?.isOpen || !rendition.current) return; - - // Handle special 'next' and 'prev' cases - if (location === 'next' && rendition.current) { - rendition.current.next(); - return; - } - if (location === 'prev' && rendition.current) { - rendition.current.prev(); - return; - } - - // Save the location to IndexedDB if not initial - if (id && locationRef.current !== 1) { - console.log('Saving location:', location); - setLastDocumentLocation(id as string, location.toString()); - } - - skipToLocation(location); - - locationRef.current = location; - extractPageText(bookRef.current, rendition.current); - - }, [id, skipToLocation, extractPageText, setIsEPUB]); - - const initialExtract = useCallback(() => { - if (!bookRef.current || !rendition.current?.location || isEPUBSetOnce.current) return; - extractPageText(bookRef.current, rendition.current, false); - }, [extractPageText]); - const checkResize = useCallback(() => { - if (isResizing && dimensions && bookRef.current?.isOpen && rendition.current && isEPUBSetOnce.current) { + if (isResizing && dimensions && bookRef.current?.isOpen && renditionRef.current) { pause(); // Only extract text when we have dimensions, ensuring the resize is complete - extractPageText(bookRef.current, rendition.current, true); + extractPageText(bookRef.current, renditionRef.current, true); setIsResizing(false); return true; } else { return false; } - }, [isResizing, setIsResizing, dimensions, pause, extractPageText]); + }, [isResizing, setIsResizing, dimensions, pause, bookRef, renditionRef, extractPageText]); // Check for isResizing to pause TTS and re-extract text useEffect(() => { if (checkResize()) return; - - // Load initial location when not resizing - initialExtract(); - }, [checkResize, initialExtract]); - - // Load the initial location - // useEffect(() => { - // if (!bookRef.current || !rendition.current || isEPUBSetOnce.current) return; - - // extractPageText(bookRef.current, rendition.current, false); - // }, [extractPageText]); + }, [checkResize]); // Register the location change handler useEffect(() => { @@ -127,12 +76,11 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) { locationChanged={handleLocationChanged} url={currDocData} title={currDocName} - tocChanged={(_toc) => (toc.current = _toc)} + tocChanged={(_toc) => (tocRef.current = _toc)} showToc={true} readerStyles={epubTheme && getThemeStyles() || undefined} - getRendition={(_rendition: Rendition) => { - bookRef.current = _rendition.book; - rendition.current = _rendition; + getRendition={(_rendition) => { + setRendition(_rendition); updateTheme(); }} /> diff --git a/src/contexts/EPUBContext.tsx b/src/contexts/EPUBContext.tsx index c765f41..513d889 100644 --- a/src/contexts/EPUBContext.tsx +++ b/src/contexts/EPUBContext.tsx @@ -7,11 +7,18 @@ import { ReactNode, useCallback, useMemo, + useRef, + RefObject, } from 'react'; import { indexedDBService } from '@/utils/indexedDB'; import { useTTS } from '@/contexts/TTSContext'; import { Book, Rendition } from 'epubjs'; import { createRangeCfi } from '@/utils/epub'; +import type { NavItem } from 'epubjs'; +import { setLastDocumentLocation } from '@/utils/indexedDB'; +import { SpineItem } from 'epubjs/types/section'; +import { useParams } from 'next/navigation'; +import { useConfig } from './ConfigContext'; interface EPUBContextType { currDocData: ArrayBuffer | undefined; @@ -22,6 +29,13 @@ interface EPUBContextType { setCurrentDocument: (id: string) => Promise; clearCurrDoc: () => void; extractPageText: (book: Book, rendition: Rendition, shouldPause?: boolean) => Promise; + createFullAudioBook: (onProgress: (progress: number) => void, signal?: AbortSignal) => Promise; + bookRef: RefObject; + renditionRef: RefObject; + tocRef: RefObject; + locationRef: RefObject; + handleLocationChanged: (location: string | number) => void; + setRendition: (rendition: Rendition) => void; } const EPUBContext = createContext(undefined); @@ -33,13 +47,27 @@ const EPUBContext = createContext(undefined); * @param {ReactNode} props.children - Child components to be wrapped by the provider */ export function EPUBProvider({ children }: { children: ReactNode }) { - const { setText: setTTSText, currDocPage, currDocPages, setCurrDocPages, stop } = useTTS(); - + const { setText: setTTSText, currDocPage, currDocPages, setCurrDocPages, stop, skipToLocation, setIsEPUB } = useTTS(); + const { id } = useParams(); + // Configuration context to get TTS settings + const { + apiKey, + baseUrl, + voiceSpeed, + voice, + } = useConfig(); // Current document state const [currDocData, setCurrDocData] = useState(); const [currDocName, setCurrDocName] = useState(); const [currDocText, setCurrDocText] = useState(); + // Add new refs + const bookRef = useRef(null); + const renditionRef = useRef(undefined); + const tocRef = useRef([]); + const locationRef = useRef(currDocPage); + const isEPUBSetOnce = useRef(false); + /** * Clears all current document state and stops any active TTS */ @@ -62,7 +90,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) { if (doc) { console.log('Retrieved document size:', doc.size); console.log('Retrieved ArrayBuffer size:', doc.data.byteLength); - + if (doc.data.byteLength === 0) { console.error('Retrieved ArrayBuffer is empty'); throw new Error('Empty document data'); @@ -87,18 +115,18 @@ export function EPUBProvider({ children }: { children: ReactNode }) { * @returns {Promise} The extracted text content */ const extractPageText = useCallback(async (book: Book, rendition: Rendition, shouldPause = false): Promise => { - try { + try { const { start, end } = rendition?.location; if (!start?.cfi || !end?.cfi || !book || !book.isOpen || !rendition) return ''; - + const rangeCfi = createRangeCfi(start.cfi, end.cfi); const range = await book.getRange(rangeCfi); const textContent = range.toString().trim(); - + setTTSText(textContent, shouldPause); setCurrDocText(textContent); - + return textContent; } catch (error) { console.error('Error extracting EPUB text:', error); @@ -106,6 +134,248 @@ export function EPUBProvider({ children }: { children: ReactNode }) { } }, [setTTSText]); + /** + * Extracts text content from the entire EPUB book + * @returns {Promise} Array of text content from each section + */ + const extractBookText = useCallback(async (): Promise => { + try { + if (!bookRef.current || !bookRef.current.isOpen) return ['']; + + const book = bookRef.current; + const spine = book.spine; + const promises: Promise[] = []; + + spine.each((item: SpineItem) => { + const url = item.href || ''; + if (!url) return; + + const promise = book.load(url) + .then((section) => (section as Document)) + .then((section) => { + const textContent = section.body.textContent || ''; + return textContent; + }) + .catch((err) => { + console.error(`Error loading section ${url}:`, err); + return ''; + }); + + promises.push(promise); + }); + + const textArray = await Promise.all(promises); + const filteredArray = textArray.filter(text => text.trim() !== ''); + console.log('Extracted entire EPUB text array:', filteredArray); + return filteredArray; + } catch (error) { + console.error('Error extracting EPUB text:', error); + return ['']; + } + }, []); + + /** + * Creates a complete audiobook by processing all text through NLP and TTS + * @param {string} voice - The voice to use for TTS + * @param {number} speed - The speed to use for TTS + * @returns {Promise} The complete audiobook as an ArrayBuffer + */ + const createFullAudioBook = useCallback(async ( + onProgress: (progress: number) => void, + signal?: AbortSignal + ): Promise => { + try { + // Get all text content from the book + const textArray = await extractBookText(); + if (!textArray.length) throw new Error('No text content found in book'); + + // Create an array to store all audio chunks + const audioChunks: ArrayBuffer[] = []; + let processedSentences = 0; + let totalSentences = 0; + + // First, count total sentences + for (const text of textArray) { + if (!text.trim()) continue; + + const nlpResponse = await fetch('/api/nlp', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text }), + }); + + if (!nlpResponse.ok) { + throw new Error(`NLP processing failed with status ${nlpResponse.status}`); + } + + const nlpData = await nlpResponse.json(); + const sentences = nlpData?.sentences; + if (sentences && Array.isArray(sentences)) { + totalSentences += sentences.length; + } + } + + // Process each section of text + for (const text of textArray) { + if (!text.trim()) continue; + + // Check for cancellation + if (signal?.aborted) { + // If cancelled, return what we have so far + const partialBuffer = combineAudioChunks(audioChunks); + return partialBuffer; + } + + const nlpResponse = await fetch('/api/nlp', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text }), + }); + + if (!nlpResponse.ok) { + throw new Error(`NLP processing failed with status ${nlpResponse.status}`); + } + + const nlpData = await nlpResponse.json(); + const sentences = nlpData?.sentences; + + if (!sentences || !Array.isArray(sentences) || sentences.length === 0) { + console.warn('No valid sentences returned from NLP, processing text block as single sentence'); + continue; + } + + // Process each sentence through TTS with retries + for (const sentence of sentences) { + // Check for cancellation + if (signal?.aborted) { + const partialBuffer = combineAudioChunks(audioChunks); + return partialBuffer; + } + + if (!sentence || typeof sentence !== 'string' || !sentence.trim()) { + processedSentences++; + continue; + } + + let retryCount = 0; + const maxRetries = 2; + let success = false; + + while (retryCount <= maxRetries && !success) { + try { + const ttsResponse = await fetch('/api/tts', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-openai-key': apiKey || '', + 'x-openai-base-url': baseUrl || 'https://api.openai.com/v1', + }, + body: JSON.stringify({ + text: sentence.trim(), + voice: voice, + speed: voiceSpeed, + }), + }); + + if (!ttsResponse.ok) { + throw new Error(`TTS processing failed with status ${ttsResponse.status}`); + } + + const audioBuffer = await ttsResponse.arrayBuffer(); + if (audioBuffer.byteLength === 0) { + throw new Error('Received empty audio buffer from TTS'); + } + + audioChunks.push(audioBuffer); + success = true; + } catch (error) { + retryCount++; + if (retryCount <= maxRetries) { + console.warn(`TTS generation failed, attempt ${retryCount} of ${maxRetries}:`, error); + await new Promise(resolve => setTimeout(resolve, 1000)); + } else { + console.error(`Failed to generate audio after ${maxRetries} retries, skipping sentence:`, sentence); + } + } + } + + if (success) { + // Add a small pause between sentences (500ms of silence) + const silenceBuffer = new ArrayBuffer(24000); + audioChunks.push(silenceBuffer); + } + + processedSentences++; + onProgress((processedSentences / totalSentences) * 100); + } + } + + if (audioChunks.length === 0) { + throw new Error('No audio was generated from the book content'); + } + + return combineAudioChunks(audioChunks); + } catch (error) { + console.error('Error creating audiobook:', error); + throw error; + } + }, [extractBookText, apiKey, baseUrl, voice, voiceSpeed]); + + // Helper function to combine audio chunks + const combineAudioChunks = (audioChunks: ArrayBuffer[]): ArrayBuffer => { + const totalLength = audioChunks.reduce((acc, chunk) => acc + chunk.byteLength, 0); + const combinedBuffer = new Uint8Array(totalLength); + + let offset = 0; + for (const chunk of audioChunks) { + combinedBuffer.set(new Uint8Array(chunk), offset); + offset += chunk.byteLength; + } + + return combinedBuffer.buffer; + }; + + const setRendition = useCallback((rendition: Rendition) => { + bookRef.current = rendition.book; + renditionRef.current = rendition; + }, []); + + const handleLocationChanged = useCallback((location: string | number) => { + // Set the EPUB flag once the location changes + if (!isEPUBSetOnce.current) { + setIsEPUB(true); + isEPUBSetOnce.current = true; + + renditionRef.current?.display(location.toString()); + return; + } + + if (!bookRef.current?.isOpen || !renditionRef.current) return; + + // Handle special 'next' and 'prev' cases + if (location === 'next' && renditionRef.current) { + renditionRef.current.next(); + return; + } + if (location === 'prev' && renditionRef.current) { + renditionRef.current.prev(); + return; + } + + // Save the location to IndexedDB if not initial + if (id && locationRef.current !== 1) { + console.log('Saving location:', location); + setLastDocumentLocation(id as string, location.toString()); + } + + skipToLocation(location); + + locationRef.current = location; + if (bookRef.current && renditionRef.current) { + extractPageText(bookRef.current, renditionRef.current); + } + }, [id, skipToLocation, extractPageText, setIsEPUB]); + // Context value memoization const contextValue = useMemo( () => ({ @@ -117,6 +387,13 @@ export function EPUBProvider({ children }: { children: ReactNode }) { currDocText, clearCurrDoc, extractPageText, + createFullAudioBook, + bookRef, + renditionRef, + tocRef, + locationRef, + handleLocationChanged, + setRendition, }), [ setCurrentDocument, @@ -127,6 +404,9 @@ export function EPUBProvider({ children }: { children: ReactNode }) { currDocText, clearCurrDoc, extractPageText, + createFullAudioBook, + handleLocationChanged, + setRendition, ] ); From b87b83310bd73e2e5cab4fca0098a0fa8d9cb3b8 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Tue, 25 Feb 2025 00:58:32 -0700 Subject: [PATCH 2/8] Rely on TTS API for splitting --- README.md | 2 +- src/app/api/tts/route.ts | 8 +- src/components/DocumentSettings.tsx | 230 ++++++++++++++-------------- src/contexts/EPUBContext.tsx | 142 +++++------------ .env.template => template.env | 0 5 files changed, 157 insertions(+), 225 deletions(-) rename .env.template => template.env (100%) diff --git a/README.md b/README.md index 64e2885..74a8fdb 100644 --- a/README.md +++ b/README.md @@ -103,7 +103,7 @@ services: 3. Configure the environment: ```bash - cp .env.template .env + cp template.env .env # Edit .env with your configuration settings ``` > Note: The base URL for the TTS API should be accessible and relative to the Next.js server diff --git a/src/app/api/tts/route.ts b/src/app/api/tts/route.ts index 9666b98..604a5ab 100644 --- a/src/app/api/tts/route.ts +++ b/src/app/api/tts/route.ts @@ -20,7 +20,7 @@ export async function POST(req: NextRequest) { // Initialize OpenAI client with abort signal const openai = new OpenAI({ apiKey: openApiKey, - baseURL: openApiBaseUrl || 'https://api.openai.com/v1', + baseURL: openApiBaseUrl, }); // Request audio from OpenAI and pass along the abort signal @@ -33,15 +33,15 @@ export async function POST(req: NextRequest) { // Get the audio data as array buffer // This will also be aborted if the client cancels - const arrayBuffer = await response.arrayBuffer(); + const stream = response.body; // Return audio data with appropriate headers - return new NextResponse(arrayBuffer); + return new NextResponse(stream); } catch (error) { // Check if this was an abort error if (error instanceof Error && error.name === 'AbortError') { console.log('TTS request aborted by client'); - return new Response(null, { status: 499 }); // Use 499 status for client closed request + return new NextResponse(null, { status: 499 }); // Use 499 status for client closed request } console.error('Error generating TTS:', error); diff --git a/src/components/DocumentSettings.tsx b/src/components/DocumentSettings.tsx index 40bde50..c251bbf 100644 --- a/src/components/DocumentSettings.tsx +++ b/src/components/DocumentSettings.tsx @@ -1,11 +1,13 @@ 'use client'; import { Fragment, useState, useRef } from 'react'; -import { Dialog, DialogPanel, DialogTitle, Transition, TransitionChild, Listbox, ListboxButton, ListboxOptions, ListboxOption, Button } from '@headlessui/react'; +import { Dialog, DialogPanel, Transition, TransitionChild, Listbox, ListboxButton, ListboxOptions, ListboxOption, Button } from '@headlessui/react'; import { useConfig, ViewType } from '@/contexts/ConfigContext'; import { ChevronUpDownIcon, CheckIcon } from '@/components/icons/Icons'; import { useEPUB } from '@/contexts/EPUBContext'; +const isDev = process.env.NEXT_PUBLIC_NODE_ENV !== 'production' || process.env.NODE_ENV == null; + interface DocViewSettingsProps { isOpen: boolean; setIsOpen: (isOpen: boolean) => void; @@ -93,136 +95,126 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro leaveTo="opacity-0 scale-95" > - - View Settings - -
-
- {!epub &&
- - updateConfigKey('viewType', newView.id as ViewType)} +
+ {isDev &&
+ {!isGenerating ? ( + + ) : ( +
+
+
- - {selectedView.id === 'scroll' && ( -

- Note: Continuous scroll may perform poorly for larger documents. -

- )} -
} +
+ {Math.round(progress)}% complete + +
+
+ )} +
} + {!epub &&
+ + updateConfigKey('viewType', newView.id as ViewType)} + > +
+ + {selectedView.name} + + + + + + + {viewTypes.map((view) => ( + + `relative cursor-pointer select-none py-2 pl-10 pr-4 ${active ? 'bg-accent/10 text-accent' : 'text-foreground' + }` + } + value={view} + > + {({ selected }) => ( + <> + + {view.name} + + {selected ? ( + + + + ) : null} + + )} + + ))} + + +
+ + {selectedView.id === 'scroll' && ( +

+ Note: Continuous scroll may perform poorly for larger documents. +

+ )} +
} +
+ +

+ Automatically skip pages with no text content +

+
+ {epub && (

- Automatically skip pages with no text content + Apply the current app theme to the EPUB viewer

- {epub && ( - <> -
- -

- Apply the current app theme to the EPUB viewer -

-
-
- {!isGenerating ? ( - - ) : ( -
-
-
-
-
- {Math.round(progress)}% complete - -
-
- )} -
- - )} -
+ )}
diff --git a/src/contexts/EPUBContext.tsx b/src/contexts/EPUBContext.tsx index 513d889..4e3d7ae 100644 --- a/src/contexts/EPUBContext.tsx +++ b/src/contexts/EPUBContext.tsx @@ -191,123 +191,63 @@ export function EPUBProvider({ children }: { children: ReactNode }) { // Create an array to store all audio chunks const audioChunks: ArrayBuffer[] = []; - let processedSentences = 0; - let totalSentences = 0; - - // First, count total sentences - for (const text of textArray) { - if (!text.trim()) continue; - - const nlpResponse = await fetch('/api/nlp', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ text }), - }); - - if (!nlpResponse.ok) { - throw new Error(`NLP processing failed with status ${nlpResponse.status}`); - } - - const nlpData = await nlpResponse.json(); - const sentences = nlpData?.sentences; - if (sentences && Array.isArray(sentences)) { - totalSentences += sentences.length; - } - } + let processedSections = 0; + const totalSections = textArray.length; // Process each section of text for (const text of textArray) { - if (!text.trim()) continue; - // Check for cancellation if (signal?.aborted) { - // If cancelled, return what we have so far const partialBuffer = combineAudioChunks(audioChunks); return partialBuffer; } - const nlpResponse = await fetch('/api/nlp', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ text }), - }); - - if (!nlpResponse.ok) { - throw new Error(`NLP processing failed with status ${nlpResponse.status}`); - } - - const nlpData = await nlpResponse.json(); - const sentences = nlpData?.sentences; - - if (!sentences || !Array.isArray(sentences) || sentences.length === 0) { - console.warn('No valid sentences returned from NLP, processing text block as single sentence'); + if (!text.trim()) { + processedSections++; continue; } - // Process each sentence through TTS with retries - for (const sentence of sentences) { - // Check for cancellation - if (signal?.aborted) { + try { + const ttsResponse = await fetch('/api/tts', { + method: 'POST', + headers: { + 'x-openai-key': apiKey, + 'x-openai-base-url': baseUrl, + }, + body: JSON.stringify({ + text: text.trim(), + voice: voice, + speed: voiceSpeed, + }), + signal // Pass the AbortSignal to the fetch request + }); + + if (!ttsResponse.ok) { + throw new Error(`TTS processing failed with status ${ttsResponse.status}`); + } + + const audioBuffer = await ttsResponse.arrayBuffer(); + if (audioBuffer.byteLength === 0) { + throw new Error('Received empty audio buffer from TTS'); + } + + audioChunks.push(audioBuffer); + + // Add a small pause between sections (500ms of silence) + const silenceBuffer = new ArrayBuffer(24000); + audioChunks.push(silenceBuffer); + + } catch (error) { + if (error instanceof Error && error.name === 'AbortError') { + console.log('TTS request aborted'); const partialBuffer = combineAudioChunks(audioChunks); return partialBuffer; } - - if (!sentence || typeof sentence !== 'string' || !sentence.trim()) { - processedSentences++; - continue; - } - - let retryCount = 0; - const maxRetries = 2; - let success = false; - - while (retryCount <= maxRetries && !success) { - try { - const ttsResponse = await fetch('/api/tts', { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - 'x-openai-key': apiKey || '', - 'x-openai-base-url': baseUrl || 'https://api.openai.com/v1', - }, - body: JSON.stringify({ - text: sentence.trim(), - voice: voice, - speed: voiceSpeed, - }), - }); - - if (!ttsResponse.ok) { - throw new Error(`TTS processing failed with status ${ttsResponse.status}`); - } - - const audioBuffer = await ttsResponse.arrayBuffer(); - if (audioBuffer.byteLength === 0) { - throw new Error('Received empty audio buffer from TTS'); - } - - audioChunks.push(audioBuffer); - success = true; - } catch (error) { - retryCount++; - if (retryCount <= maxRetries) { - console.warn(`TTS generation failed, attempt ${retryCount} of ${maxRetries}:`, error); - await new Promise(resolve => setTimeout(resolve, 1000)); - } else { - console.error(`Failed to generate audio after ${maxRetries} retries, skipping sentence:`, sentence); - } - } - } - - if (success) { - // Add a small pause between sentences (500ms of silence) - const silenceBuffer = new ArrayBuffer(24000); - audioChunks.push(silenceBuffer); - } - - processedSentences++; - onProgress((processedSentences / totalSentences) * 100); + console.error('Error processing section:', error); } + + processedSections++; + onProgress((processedSections / totalSections) * 100); } if (audioChunks.length === 0) { diff --git a/.env.template b/template.env similarity index 100% rename from .env.template rename to template.env From f948601e709cbb22576ff9c82184e8860ae0039a Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Tue, 25 Feb 2025 03:26:37 -0700 Subject: [PATCH 3/8] Add configurable text extraction margin for PDF processing --- src/components/DocumentSettings.tsx | 67 ++++++++++++++++++++++++----- src/contexts/ConfigContext.tsx | 12 ++++++ src/contexts/PDFContext.tsx | 6 ++- src/utils/pdf.ts | 45 +++++++++++++++++-- 4 files changed, 114 insertions(+), 16 deletions(-) diff --git a/src/components/DocumentSettings.tsx b/src/components/DocumentSettings.tsx index c251bbf..eed03c2 100644 --- a/src/components/DocumentSettings.tsx +++ b/src/components/DocumentSettings.tsx @@ -1,6 +1,6 @@ 'use client'; -import { Fragment, useState, useRef } from 'react'; +import { Fragment, useState, useRef, useCallback, useEffect } from 'react'; import { Dialog, DialogPanel, Transition, TransitionChild, Listbox, ListboxButton, ListboxOptions, ListboxOption, Button } from '@headlessui/react'; import { useConfig, ViewType } from '@/contexts/ConfigContext'; import { ChevronUpDownIcon, CheckIcon } from '@/components/icons/Icons'; @@ -21,13 +21,33 @@ const viewTypes = [ ]; export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsProps) { - const { viewType, skipBlank, epubTheme, updateConfigKey } = useConfig(); + const { viewType, skipBlank, epubTheme, textExtractionMargin, updateConfigKey } = useConfig(); const { createFullAudioBook } = useEPUB(); const [progress, setProgress] = useState(0); const [isGenerating, setIsGenerating] = useState(false); + const [localMargin, setLocalMargin] = useState(textExtractionMargin); const abortControllerRef = useRef(null); const selectedView = viewTypes.find(v => v.id === viewType) || viewTypes[0]; + //console.log(localMargin, textExtractionMargin); + + // Sync local margin with global state + useEffect(() => { + setLocalMargin(textExtractionMargin); + }, [textExtractionMargin]); + + // Handler for slider change (updates local state only) + const handleMarginChange = useCallback((event: React.ChangeEvent) => { + setLocalMargin(Number(event.target.value)); + }, []); + + // Handler for slider release + const handleMarginChangeComplete = useCallback(() => { + if (localMargin !== textExtractionMargin) { + updateConfigKey('textExtractionMargin', localMargin); + } + }, [localMargin, textExtractionMargin, updateConfigKey]); + const handleStartGeneration = async () => { setIsGenerating(true); setProgress(0); @@ -132,13 +152,38 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro
)}
} - {!epub &&
- + {!epub &&
+
+ +
+ 0% + {Math.round(localMargin * 100)}% + 20% +
+ +

+ {"Don't"} include content from outer rim of the page during text extraction (experimental) +

+
updateConfigKey('viewType', newView.id as ViewType)} > -
+
+ {selectedView.name} @@ -177,14 +222,16 @@ export function DocumentSettings({ isOpen, setIsOpen, epub }: DocViewSettingsPro ))} + {selectedView.id === 'scroll' && ( +

+ Note: Continuous scroll may perform poorly for larger documents. +

+ )}
- {selectedView.id === 'scroll' && ( -

- Note: Continuous scroll may perform poorly for larger documents. -

- )} +
} +
} {!epub &&
-