From 521eda6cff135c88b416d3c090e9235f3882dc21 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Wed, 12 Feb 2025 02:14:56 -0700 Subject: [PATCH 1/3] Move nlp processing to next server --- src/{utils/nlp.ts => app/api/nlp/route.ts} | 45 +++++--- src/contexts/TTSContext.tsx | 122 +++++++++++++-------- 2 files changed, 104 insertions(+), 63 deletions(-) rename src/{utils/nlp.ts => app/api/nlp/route.ts} (53%) diff --git a/src/utils/nlp.ts b/src/app/api/nlp/route.ts similarity index 53% rename from src/utils/nlp.ts rename to src/app/api/nlp/route.ts index a0aa8ae..276e170 100644 --- a/src/utils/nlp.ts +++ b/src/app/api/nlp/route.ts @@ -1,31 +1,23 @@ +import { NextRequest, NextResponse } from 'next/server'; import nlp from 'compromise'; -// Text preprocessing function to clean and normalize text -export const preprocessSentenceForAudio = (text: string): string => { +const MAX_BLOCK_LENGTH = 300; + +const preprocessSentenceForAudio = (text: string): string => { return text - // Replace URLs with descriptive text including domain .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') - // Remove special characters except basic punctuation - //.replace(/[^\w\s.,!?;:'"()-]/g, ' ') - // Fix hyphenated words at line breaks (word- word -> wordword) .replace(/(\w+)-\s+(\w+)/g, '$1$2') - // Replace multiple spaces with single space .replace(/\s+/g, ' ') - // Trim whitespace .trim(); }; -const MAX_BLOCK_LENGTH = 300; // Maximum characters per block - -export const splitIntoSentences = (text: string): string[] => { - // Split text into paragraphs first +const splitIntoSentences = (text: string): string[] => { const paragraphs = text.split(/\n+/); const blocks: string[] = []; for (const paragraph of paragraphs) { if (!paragraph.trim()) continue; - // Preprocess each paragraph const cleanedText = preprocessSentenceForAudio(paragraph); const doc = nlp(cleanedText); const rawSentences = doc.sentences().out('array') as string[]; @@ -35,23 +27,42 @@ export const splitIntoSentences = (text: string): string[] => { for (const sentence of rawSentences) { const trimmedSentence = sentence.trim(); - // If adding this sentence would exceed the limit, start a new block if (currentBlock && (currentBlock.length + trimmedSentence.length + 1) > MAX_BLOCK_LENGTH) { blocks.push(currentBlock.trim()); currentBlock = trimmedSentence; } else { - // Add to current block with a space if not empty currentBlock = currentBlock ? `${currentBlock} ${trimmedSentence}` : trimmedSentence; } } - // Add the last block if not empty if (currentBlock) { blocks.push(currentBlock.trim()); } } return blocks; -}; \ No newline at end of file +}; + +export async function POST(req: NextRequest) { + try { + const { text } = await req.json(); + if (!text) { + return NextResponse.json({ error: 'No text provided' }, { status: 400 }); + } + + if (text.length <= MAX_BLOCK_LENGTH) { + // Single sentence preprocessing + const cleanedText = preprocessSentenceForAudio(text); + return NextResponse.json({ sentences: [cleanedText] }); + } + + // Full text splitting into sentences + const sentences = splitIntoSentences(text); + return NextResponse.json({ sentences }); + } catch (error) { + console.error('Error processing text:', error); + return NextResponse.json({ error: 'Failed to process text' }, { status: 500 }); + } +} diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index d4969bf..51d113d 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -29,7 +29,6 @@ import toast from 'react-hot-toast'; import { useParams } from 'next/navigation'; import { useConfig } from '@/contexts/ConfigContext'; -import { splitIntoSentences, preprocessSentenceForAudio } from '@/utils/nlp'; import { audioBufferToURL } from '@/utils/audio'; import { useAudioCache } from '@/hooks/audio/useAudioCache'; import { useVoiceManagement } from '@/hooks/audio/useVoiceManagement'; @@ -152,51 +151,71 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { */ const setText = useCallback((text: string) => { console.log('Setting page text:', text); - const newSentences = splitIntoSentences(text); - - // If skipBlank is enabled and there's no text - if (isPlaying && skipBlank && newSentences.length === 0) { - if (isEPUB && locationChangeHandlerRef.current) { - // For EPUB, use the location handler to move to next section - locationChangeHandlerRef.current('next'); - - toast.success('Skipping blank section', { - id: `epub-section-skip`, - iconTheme: { - primary: 'var(--accent)', - secondary: 'var(--background)', - }, - style: { - background: 'var(--background)', - color: 'var(--accent)', - }, - duration: 1000, - position: 'top-center', - }); - return; - } else if (currDocPageNumber < currDocPages!) { - // For PDF, increment the page - incrementPage(); - - toast.success(`Skipping blank page ${currDocPageNumber}`, { - id: `page-${currDocPageNumber}`, - iconTheme: { - primary: 'var(--accent)', - secondary: 'var(--background)', - }, - style: { - background: 'var(--background)', - color: 'var(--accent)', - }, - duration: 1000, - position: 'top-center', - }); - return; - } - } - setSentences(newSentences); - setNextPageLoading(false); + fetch('/api/nlp', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text }), + }) + .then(response => { + if (!response.ok) { + throw new Error('Failed to process text'); + } + return response.json(); + }) + .then(({ sentences: newSentences }) => { + // If skipBlank is enabled and there's no text + if (isPlaying && skipBlank && newSentences.length === 0) { + if (isEPUB && locationChangeHandlerRef.current) { + locationChangeHandlerRef.current('next'); + + toast.success('Skipping blank section', { + id: `epub-section-skip`, + iconTheme: { + primary: 'var(--accent)', + secondary: 'var(--background)', + }, + style: { + background: 'var(--background)', + color: 'var(--accent)', + }, + duration: 1000, + position: 'top-center', + }); + return; + } else if (currDocPageNumber < currDocPages!) { + incrementPage(); + + toast.success(`Skipping blank page ${currDocPageNumber}`, { + id: `page-${currDocPageNumber}`, + iconTheme: { + primary: 'var(--accent)', + secondary: 'var(--background)', + }, + style: { + background: 'var(--background)', + color: 'var(--accent)', + }, + duration: 1000, + position: 'top-center', + }); + return; + } + } + + setSentences(newSentences); + setNextPageLoading(false); + }) + .catch(error => { + console.error('Error processing text:', error); + toast.error('Failed to process text', { + style: { + background: 'var(--background)', + color: 'var(--accent)', + }, + duration: 3000, + }); + }); }, [isPlaying, skipBlank, currDocPageNumber, currDocPages, incrementPage, isEPUB]); /** @@ -437,7 +456,18 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { // Only set processing state if not preloading if (!preload) setIsProcessing(true); - const cleanedSentence = preprocessSentenceForAudio(sentence); + // Preprocess the sentence using the NLP API + const response = await fetch('/api/nlp', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ text: sentence }), + }); + + if (!response.ok) { + throw new Error('Failed to process sentence'); + } + + const { sentences: [cleanedSentence] } = await response.json(); const audioBuffer = await getAudio(cleanedSentence); return audioBufferToURL(audioBuffer!); From f16116cab4c9603bdbe3f8c16bab2b49c153889b Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Wed, 12 Feb 2025 02:15:21 -0700 Subject: [PATCH 2/3] Huge PDF text extraction optimization --- src/contexts/PDFContext.tsx | 22 +++++++++++++++------- src/utils/pdf.ts | 15 ++++----------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/src/contexts/PDFContext.tsx b/src/contexts/PDFContext.tsx index d2396f4..9429cfc 100644 --- a/src/contexts/PDFContext.tsx +++ b/src/contexts/PDFContext.tsx @@ -33,6 +33,8 @@ import { handleTextClick, } from '@/utils/pdf'; +import type { PDFDocumentProxy } from 'pdfjs-dist'; + /** * Interface defining all available methods and properties in the PDF context */ @@ -43,11 +45,12 @@ interface PDFContextType { currDocPages: number | undefined; currDocPage: number; currDocText: string | undefined; + pdfDocument: PDFDocumentProxy | undefined; setCurrentDocument: (id: string) => Promise; clearCurrDoc: () => void; // PDF functionality - onDocumentLoadSuccess: ({ numPages }: { numPages: number }) => void; + onDocumentLoadSuccess: (pdf: PDFDocumentProxy) => void; highlightPattern: (text: string, pattern: string, containerRef: React.RefObject) => void; clearHighlights: () => void; handleTextClick: ( @@ -75,15 +78,17 @@ export function PDFProvider({ children }: { children: ReactNode }) { const [currDocURL, setCurrDocURL] = useState(); const [currDocName, setCurrDocName] = useState(); const [currDocText, setCurrDocText] = useState(); + const [pdfDocument, setPdfDocument] = useState(); /** * Handles successful document load * * @param {Object} param0 - Object containing numPages */ - const onDocumentLoadSuccess = useCallback(({ numPages }: { numPages: number }) => { - console.log('Document loaded:', numPages); - setCurrDocPages(numPages); + const onDocumentLoadSuccess = useCallback((pdf: PDFDocumentProxy) => { + console.log('Document loaded:', pdf.numPages); + setCurrDocPages(pdf.numPages); + setPdfDocument(pdf); }, [setCurrDocPages]); /** @@ -91,15 +96,15 @@ export function PDFProvider({ children }: { children: ReactNode }) { */ const loadCurrDocText = useCallback(async () => { try { - if (!currDocURL) return; - const text = await extractTextFromPDF(currDocURL, currDocPage); + if (!pdfDocument) return; + const text = await extractTextFromPDF(pdfDocument, currDocPage); setCurrDocText(text); setTTSText(text); } catch (error) { console.error('Error loading PDF text:', error); } - }, [currDocURL, currDocPage, setTTSText]); + }, [pdfDocument, currDocPage, setTTSText]); /** * Updates the current document text when the page changes @@ -136,6 +141,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { setCurrDocURL(undefined); setCurrDocText(undefined); setCurrDocPages(undefined); + setPdfDocument(undefined); stop(); }, [setCurrDocPages, stop]); @@ -153,6 +159,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { highlightPattern, clearHighlights, handleTextClick, + pdfDocument, }), [ onDocumentLoadSuccess, @@ -163,6 +170,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { currDocPage, currDocText, clearCurrDoc, + pdfDocument, ] ); diff --git a/src/utils/pdf.ts b/src/utils/pdf.ts index 461e610..c47447b 100644 --- a/src/utils/pdf.ts +++ b/src/utils/pdf.ts @@ -2,9 +2,11 @@ import { pdfjs } from 'react-pdf'; import nlp from 'compromise'; import stringSimilarity from 'string-similarity'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; +import type { PDFDocumentProxy } from 'pdfjs-dist'; -// Set worker from public directory +// Set worker from public directory and compatibility mode pdfjs.GlobalWorkerOptions.workerSrc = '/pdf.worker.mjs'; +pdfjs.GlobalWorkerOptions.workerPort = null; interface TextMatch { elements: HTMLElement[]; @@ -24,17 +26,8 @@ export function convertPDFDataToURL(pdfData: Blob): Promise { } // Text Processing functions -export async function extractTextFromPDF(pdfURL: string, pageNumber: number): Promise { +export async function extractTextFromPDF(pdf: PDFDocumentProxy, pageNumber: number): Promise { try { - const base64Data = pdfURL.split(',')[1]; - const binaryData = atob(base64Data); - const bytes = new Uint8Array(binaryData.length); - for (let i = 0; i < binaryData.length; i++) { - bytes[i] = binaryData.charCodeAt(i); - } - - const loadingTask = pdfjs.getDocument({ data: bytes }); - const pdf = await loadingTask.promise; const page = await pdf.getPage(pageNumber); const textContent = await page.getTextContent(); From 2a9510aa55408c1c909f9c5f94e0bce1ff2a09d0 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Wed, 12 Feb 2025 02:19:57 -0700 Subject: [PATCH 3/3] Another nlp optimization --- src/contexts/TTSContext.tsx | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index 51d113d..05d1881 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -456,20 +456,8 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { // Only set processing state if not preloading if (!preload) setIsProcessing(true); - // Preprocess the sentence using the NLP API - const response = await fetch('/api/nlp', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ text: sentence }), - }); - - if (!response.ok) { - throw new Error('Failed to process sentence'); - } - - const { sentences: [cleanedSentence] } = await response.json(); - const audioBuffer = await getAudio(cleanedSentence); - + // No need to preprocess again since setText already did it + const audioBuffer = await getAudio(sentence); return audioBufferToURL(audioBuffer!); }, [isProcessing, audioContext, getAudio]);