From 429ec3f264d6be3c9cd812c7ed2b739b5f61c962 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Sat, 25 Jan 2025 22:23:29 -0700 Subject: [PATCH 1/8] Single page viewer + Major refactor --- src/app/pdf/[id]/page.tsx | 42 +- src/components/PDFViewer.tsx | 157 ++---- src/components/Spinner.tsx | 8 + src/components/TTSPlayer.tsx | 129 ----- src/components/player/Navigator.tsx | 42 ++ src/components/player/SpeedControl.tsx | 49 ++ src/components/player/TTSPlayer.tsx | 75 +++ src/components/player/VoicesControl.tsx | 37 ++ src/contexts/PDFContext.tsx | 154 ++++-- src/contexts/TTSContext.tsx | 623 ++++++++---------------- src/services/audio.ts | 71 +++ src/services/nlp.ts | 23 + 12 files changed, 704 insertions(+), 706 deletions(-) create mode 100644 src/components/Spinner.tsx delete mode 100644 src/components/TTSPlayer.tsx create mode 100644 src/components/player/Navigator.tsx create mode 100644 src/components/player/SpeedControl.tsx create mode 100644 src/components/player/TTSPlayer.tsx create mode 100644 src/components/player/VoicesControl.tsx create mode 100644 src/services/audio.ts create mode 100644 src/services/nlp.ts diff --git a/src/app/pdf/[id]/page.tsx b/src/app/pdf/[id]/page.tsx index f9df8ca..6d6cedf 100644 --- a/src/app/pdf/[id]/page.tsx +++ b/src/app/pdf/[id]/page.tsx @@ -4,9 +4,8 @@ import dynamic from 'next/dynamic'; import { usePDF } from '@/contexts/PDFContext'; import { useParams } from 'next/navigation'; import Link from 'next/link'; -import { useEffect, useState } from 'react'; +import { use, useCallback, useEffect, useState } from 'react'; import { PDFSkeleton } from '@/components/PDFSkeleton'; -import TTSPlayer from '@/components/TTSPlayer'; import { useTTS } from '@/contexts/TTSContext'; // Dynamic import for client-side rendering only @@ -20,32 +19,32 @@ const PDFViewer = dynamic( export default function PDFViewerPage() { const { id } = useParams(); - const { getDocument } = usePDF(); + const { setCurrentDocument, currDocName } = usePDF(); const { setText, stop } = useTTS(); - const [document, setDocument] = useState<{ name: string; data: Blob } | null>(null); const [error, setError] = useState(null); const [isLoading, setIsLoading] = useState(true); const [zoomLevel, setZoomLevel] = useState(100); - useEffect(() => { - async function loadDocument() { - try { - const doc = await getDocument(id as string); - if (!doc) { - setError('Document not found'); - return; - } - setDocument(doc); - } catch (err) { - console.error('Error loading document:', err); - setError('Failed to load document'); - } finally { - setIsLoading(false); + const loadDocument = useCallback(async () => { + if (!isLoading) return; // Prevent calls when not loading new doc + console.log('Loading new document (from page.tsx)'); + try { + if (!id) { + setError('Document not found'); + return; } + setCurrentDocument(id as string); + } catch (err) { + console.error('Error loading document:', err); + setError('Failed to load document'); + } finally { + setIsLoading(false); } + }, [isLoading, id, setCurrentDocument]); + useEffect(() => { loadDocument(); - }, [id, getDocument]); + }, [loadDocument]); const handleZoomIn = () => setZoomLevel(prev => Math.min(prev + 10, 200)); const handleZoomOut = () => setZoomLevel(prev => Math.max(prev - 10, 50)); @@ -73,7 +72,6 @@ export default function PDFViewerPage() { return ( <> -
@@ -109,7 +107,7 @@ export default function PDFViewerPage() {

- {isLoading ? 'Loading...' : document?.name} + {isLoading ? 'Loading...' : currDocName}

@@ -118,7 +116,7 @@ export default function PDFViewerPage() { ) : ( - + )} ); diff --git a/src/components/PDFViewer.tsx b/src/components/PDFViewer.tsx index 642d81e..d00af3b 100644 --- a/src/components/PDFViewer.tsx +++ b/src/components/PDFViewer.tsx @@ -1,6 +1,6 @@ 'use client'; -import { RefObject } from 'react'; +import { RefObject, useCallback } from 'react'; import { Document, Page } from 'react-pdf'; import 'react-pdf/dist/Page/AnnotationLayer.css'; import 'react-pdf/dist/Page/TextLayer.css'; @@ -8,21 +8,34 @@ import { useState, useEffect, useRef } from 'react'; import { PDFSkeleton } from './PDFSkeleton'; import { useTTS } from '@/contexts/TTSContext'; import { usePDF } from '@/contexts/PDFContext'; +import TTSPlayer from '@/components/player/TTSPlayer'; interface PDFViewerProps { - pdfData: Blob | undefined; zoomLevel: number; } -export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) { - const [numPages, setNumPages] = useState(); +export function PDFViewer({ zoomLevel }: PDFViewerProps) { const [containerWidth, setContainerWidth] = useState(0); - const { setText, currentSentence, stopAndPlayFromIndex, isProcessing } = useTTS(); - const [pdfText, setPdfText] = useState(''); - const [pdfDataUrl, setPdfDataUrl] = useState(); - const [loadingError, setLoadingError] = useState(); const containerRef = useRef(null); - const { extractTextFromPDF, highlightPattern, clearHighlights, handleTextClick } = usePDF(); + + // TTS context + const { + currentSentence, + stopAndPlayFromIndex, + isProcessing + } = useTTS(); + + // PDF context + const { + highlightPattern, + clearHighlights, + handleTextClick, + onDocumentLoadSuccess, + currDocURL, + currDocPages, + currDocText, + currDocPage, + } = usePDF(); // Add static styles once during component initialization const styleElement = document.createElement('style'); @@ -44,58 +57,6 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) { }; }, [styleElement]); - useEffect(() => { - /* - * Converts PDF blob to a data URL for display. - * Cleans up by clearing the data URL when component unmounts. - * - * Dependencies: - * - pdfData: Re-run when the PDF blob changes to convert it to a new data URL - */ - if (!pdfData) return; - - const reader = new FileReader(); - reader.onload = () => { - setPdfDataUrl(reader.result as string); - }; - reader.onerror = () => { - console.error('Error reading file:', reader.error); - setLoadingError('Failed to load PDF'); - }; - reader.readAsDataURL(pdfData); - - return () => { - setPdfDataUrl(undefined); - }; - }, [pdfData]); - - useEffect(() => { - /* - * Extracts text content from the PDF once it's loaded. - * Sets the extracted text for both display and text-to-speech. - * - * Dependencies: - * - pdfDataUrl: Re-run when the data URL is ready - * - extractTextFromPDF: Function from context that could change - * - setText: Function from context that could change - * - pdfData: Source PDF blob that's being processed - */ - if (!pdfDataUrl || !pdfData) return; - - const loadPdfText = async () => { - try { - const text = await extractTextFromPDF(pdfData); - setPdfText(text); - setText(text); - } catch (error) { - console.error('Error loading PDF text:', error); - setLoadingError('Failed to extract PDF text'); - } - }; - - loadPdfText(); - }, [pdfDataUrl, extractTextFromPDF, setText, pdfData]); - useEffect(() => { /* * Sets up click event listeners for text selection in the PDF. @@ -108,10 +69,11 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) { */ const container = containerRef.current; if (!container) return; + if (!currDocText) return; const handleClick = (event: MouseEvent) => handleTextClick( event, - pdfText, + currDocText, containerRef as RefObject, stopAndPlayFromIndex, isProcessing @@ -120,7 +82,7 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) { return () => { container.removeEventListener('click', handleClick); }; - }, [pdfText, handleTextClick, stopAndPlayFromIndex, isProcessing]); + }, [currDocText, handleTextClick, stopAndPlayFromIndex, isProcessing]); useEffect(() => { /* @@ -133,25 +95,27 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) { * - highlightPattern: Function from context that could change * - clearHighlights: Function from context that could change */ + if (!currDocText) return; + const highlightTimeout = setTimeout(() => { if (containerRef.current) { - highlightPattern(pdfText, currentSentence || '', containerRef as RefObject); + highlightPattern(currDocText, currentSentence || '', containerRef as RefObject); } - }, 100); + }, 200); return () => { clearTimeout(highlightTimeout); clearHighlights(); }; - }, [pdfText, currentSentence, highlightPattern, clearHighlights]); + }, [currDocText, currentSentence, highlightPattern, clearHighlights]); // Add scale calculation function - const calculateScale = (pageWidth: number = 595) => { // 595 is default PDF width in points + const calculateScale = useCallback((pageWidth = 595): number => { const margin = 24; // 24px padding on each side const targetWidth = containerWidth - margin; const baseScale = targetWidth / pageWidth; return baseScale * (zoomLevel / 100); - }; + }, [containerWidth, zoomLevel]); // Add resize observer effect useEffect(() => { @@ -168,48 +132,35 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) { return () => observer.disconnect(); }, []); - function onDocumentLoadSuccess({ numPages }: { numPages: number }): void { - setNumPages(numPages); - } - return ( -
- {loadingError ? ( -
{loadingError}
- ) : null} +
} noData={} - file={pdfDataUrl} - onLoadSuccess={onDocumentLoadSuccess} + file={currDocURL} + onLoadSuccess={(pdf) => { + onDocumentLoadSuccess(pdf); + //handlePageChange(1); // Load first page text + }} className="flex flex-col items-center m-0" > - {Array.from( - new Array(numPages), - (el, index) => ( -
-
-

- {index + 1} / {numPages} -

-
-
- -
-
- ), - )} +
+
+ +
+
+ {}} + />
); } \ No newline at end of file diff --git a/src/components/Spinner.tsx b/src/components/Spinner.tsx new file mode 100644 index 0000000..1c4e0b2 --- /dev/null +++ b/src/components/Spinner.tsx @@ -0,0 +1,8 @@ +// Loading spinner component +export function LoadingSpinner() { + return ( +
+
+
+ ); +} \ No newline at end of file diff --git a/src/components/TTSPlayer.tsx b/src/components/TTSPlayer.tsx deleted file mode 100644 index e8b6ad5..0000000 --- a/src/components/TTSPlayer.tsx +++ /dev/null @@ -1,129 +0,0 @@ -'use client'; - -import { useTTS } from '@/contexts/TTSContext'; -import { Button, Listbox, ListboxButton, ListboxOptions, ListboxOption } from '@headlessui/react'; -import { - PlayIcon, - PauseIcon, - SkipForwardIcon, - SkipBackwardIcon, - ChevronUpDownIcon, -} from './icons/Icons'; - -// Loading spinner component -function LoadingSpinner() { - return ( -
-
-
- ); -} - -const speedOptions = [ - { value: 0.5, label: '0.5x' }, - { value: 0.75, label: '0.75x' }, - { value: 1, label: '1x' }, - { value: 1.25, label: '1.25x' }, - { value: 1.5, label: '1.5x' }, - { value: 1.75, label: '1.75x' }, - { value: 2, label: '2x' }, - { value: 2.5, label: '2.5x' }, - { value: 3, label: '3x' }, -]; - -export default function TTSPlayer() { - const { - isPlaying, - togglePlay, - skipForward, - skipBackward, - isProcessing, - speed, - setSpeedAndRestart, - voice, - setVoiceAndRestart, - availableVoices, - } = useTTS(); - - //console.log(availableVoices); - - return ( -
-
-
- - - {speed}x - - - - {speedOptions.map((option) => ( - - `relative cursor-pointer select-none py-2 px-3 ${active ? 'bg-offbase' : '' - } ${selected ? 'font-medium' : ''}` - } - > - {option.label} - - ))} - - -
- - - - - - - - -
- - - {voice} - - - - {availableVoices.map((voiceId) => ( - - `relative cursor-pointer select-none py-2 px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium' : ''}` - } - > - {voiceId} - - ))} - - -
-
-
- ); -} diff --git a/src/components/player/Navigator.tsx b/src/components/player/Navigator.tsx new file mode 100644 index 0000000..95cdba5 --- /dev/null +++ b/src/components/player/Navigator.tsx @@ -0,0 +1,42 @@ +import { Button } from '@headlessui/react'; + +export const Navigator = ({ currentPage, numPages, onPageChange }: { + currentPage: number; + numPages: number | undefined; + onPageChange: (page: number) => void; +}) => { + return ( +
+ {/* Page back */} + + + {/* Page number */} +
+

+ {currentPage} / {numPages || 1} +

+
+ + {/* Page forward */} + +
+ ); +} \ No newline at end of file diff --git a/src/components/player/SpeedControl.tsx b/src/components/player/SpeedControl.tsx new file mode 100644 index 0000000..e544e14 --- /dev/null +++ b/src/components/player/SpeedControl.tsx @@ -0,0 +1,49 @@ +import { + Listbox, + ListboxButton, + ListboxOption, + ListboxOptions, +} from '@headlessui/react'; +import { ChevronUpDownIcon } from '@/components/icons/Icons'; + +const speedOptions = [ + { value: 0.5, label: '0.5x' }, + { value: 0.75, label: '0.75x' }, + { value: 1, label: '1x' }, + { value: 1.25, label: '1.25x' }, + { value: 1.5, label: '1.5x' }, + { value: 1.75, label: '1.75x' }, + { value: 2, label: '2x' }, + { value: 2.5, label: '2.5x' }, + { value: 3, label: '3x' }, +]; + +export const SpeedControl = ({ speed, setSpeedAndRestart }: { + speed: number; + setSpeedAndRestart: (speed: number) => void; +}) => { + return ( +
+ + + {speed}x + + + + {speedOptions.map((option) => ( + + `relative cursor-pointer select-none py-2 px-3 ${active ? 'bg-offbase' : '' + } ${selected ? 'font-medium' : ''}` + } + > + {option.label} + + ))} + + +
+ ); +} \ No newline at end of file diff --git a/src/components/player/TTSPlayer.tsx b/src/components/player/TTSPlayer.tsx new file mode 100644 index 0000000..e178554 --- /dev/null +++ b/src/components/player/TTSPlayer.tsx @@ -0,0 +1,75 @@ +'use client'; + +import { useTTS } from '@/contexts/TTSContext'; +import { Button } from '@headlessui/react'; +import { + PlayIcon, + PauseIcon, + SkipForwardIcon, + SkipBackwardIcon, +} from '@/components/icons/Icons'; +import { LoadingSpinner } from '@/components/Spinner'; +import { VoicesControl } from '@/components/player/VoicesControl'; +import { SpeedControl } from '@/components/player/SpeedControl'; +import { Navigator } from '@/components/player/Navigator'; + +export default function TTSPlayer({ currentPage, numPages, onPageChange }: { + currentPage: number; + numPages: number | undefined; + onPageChange: (page: number) => void; +}) { + const { + isPlaying, + togglePlay, + skipForward, + skipBackward, + isProcessing, + speed, + setSpeedAndRestart, + voice, + setVoiceAndRestart, + availableVoices, + } = useTTS(); + + return ( +
+
+ {/* Speed control */} + + + {/* Page Navigation */} + + + {/* Playback Controls */} + + + + + + + {/* Voice control */} + +
+
+ ); +} diff --git a/src/components/player/VoicesControl.tsx b/src/components/player/VoicesControl.tsx new file mode 100644 index 0000000..a2e8366 --- /dev/null +++ b/src/components/player/VoicesControl.tsx @@ -0,0 +1,37 @@ +import { + Listbox, + ListboxButton, + ListboxOption, + ListboxOptions, +} from '@headlessui/react'; +import { ChevronUpDownIcon } from '@/components/icons/Icons'; + +export const VoicesControl = ({ voice, availableVoices, setVoiceAndRestart }: { + voice: string; + availableVoices: string[]; + setVoiceAndRestart: (voice: string) => void; +}) => { + return ( +
+ + + {voice} + + + + {availableVoices.map((voiceId) => ( + + `relative cursor-pointer select-none py-2 px-3 ${active ? 'bg-offbase' : ''} ${selected ? 'font-medium' : ''}` + } + > + {voiceId} + + ))} + + +
+ ); +} \ No newline at end of file diff --git a/src/contexts/PDFContext.tsx b/src/contexts/PDFContext.tsx index 734048e..bcf68c6 100644 --- a/src/contexts/PDFContext.tsx +++ b/src/contexts/PDFContext.tsx @@ -18,10 +18,21 @@ import nlp from 'compromise'; // Add the correct type import import type { TextItem } from 'pdfjs-dist/types/src/display/api'; import { useConfig } from '@/contexts/ConfigContext'; +import { useTTS } from '@/contexts/TTSContext'; // Set worker from public directory pdfjs.GlobalWorkerOptions.workerSrc = '/pdf.worker.mjs'; +// Convert PDF data to PDF data URL +const convertPDFDataToURL = (pdfData: Blob): Promise => { + return new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result as string); + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(pdfData); + }); +}; + interface PDFContextType { documents: PDFDocument[]; addDocument: (file: File) => Promise; @@ -29,7 +40,7 @@ interface PDFContextType { removeDocument: (id: string) => Promise; isLoading: boolean; error: string | null; - extractTextFromPDF: (pdfData: Blob) => Promise; + extractTextFromPDF: (pdfURL: string, currDocPage: number) => Promise; highlightPattern: (text: string, pattern: string, containerRef: React.RefObject) => void; clearHighlights: () => void; handleTextClick: ( @@ -39,21 +50,40 @@ interface PDFContextType { stopAndPlayFromIndex: (index: number) => void, isProcessing: boolean ) => void; + onDocumentLoadSuccess: ({ numPages }: { numPages: number }) => void; + setCurrentDocument: (id: string) => Promise; + currDocURL: string | undefined; + currDocName: string | undefined; + currDocPages: number | undefined; + currDocPage: number; + currDocText: string | undefined; } const PDFContext = createContext(undefined); export function PDFProvider({ children }: { children: ReactNode }) { - const { isDBReady } = useConfig(); const [documents, setDocuments] = useState([]); const [isLoading, setIsLoading] = useState(true); const [error, setError] = useState(null); + const { isDBReady } = useConfig(); + const { + setText: setTTSText, + currDocPage, + currDocPages, + setCurrDocPages, + } = useTTS(); + + // Current document state + const [currDocURL, setCurrDocURL] = useState(); + const [currDocName, setCurrDocName] = useState(); + const [currDocText, setCurrDocText] = useState(); + // Load documents from IndexedDB once DB is ready useEffect(() => { const loadDocuments = async () => { if (!isDBReady) return; - + try { setError(null); const docs = await indexedDBService.getAllDocuments(); @@ -117,33 +147,30 @@ export function PDFProvider({ children }: { children: ReactNode }) { } }, []); + function onDocumentLoadSuccess({ numPages }: { numPages: number }): void { + console.log('Document loaded:', numPages); + setCurrDocPages(numPages); + } + // Extract text from a PDF file - const extractTextFromPDF = useCallback(async (pdfData: Blob): Promise => { - try { - const reader = new FileReader(); - const dataUrl = await new Promise((resolve, reject) => { - reader.onload = () => resolve(reader.result as string); - reader.onerror = () => reject(reader.error); - reader.readAsDataURL(pdfData); - }); - - const base64Data = dataUrl.split(',')[1]; - const binaryData = atob(base64Data); - const bytes = new Uint8Array(binaryData.length); - for (let i = 0; i < binaryData.length; i++) { - bytes[i] = binaryData.charCodeAt(i); - } - - const loadingTask = pdfjs.getDocument({ data: bytes }); - const pdf = await loadingTask.promise; - let fullText = ''; - - for (let i = 1; i <= pdf.numPages; i++) { - const page = await pdf.getPage(i); - const textContent = await page.getTextContent(); + const extractTextFromPDF = useCallback(async (pdfURL: string, currDocPage: number): Promise => { + try { + const base64Data = pdfURL.split(',')[1]; + const binaryData = atob(base64Data); + const bytes = new Uint8Array(binaryData.length); + for (let i = 0; i < binaryData.length; i++) { + bytes[i] = binaryData.charCodeAt(i); + } + + const loadingTask = pdfjs.getDocument({ data: bytes }); + const pdf = await loadingTask.promise; + // Get only the specified page + const page = await pdf.getPage(currDocPage); + const textContent = await page.getTextContent(); + // Filter out non-text items and assert proper type - const textItems = textContent.items.filter((item): item is TextItem => + const textItems = textContent.items.filter((item): item is TextItem => 'str' in item && 'transform' in item ); @@ -173,7 +200,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { for (const line of lines) { // Sort items horizontally within the line line.sort((a, b) => a.transform[4] - b.transform[4]); - + let lineText = ''; let prevItem: TextItem | null = null; @@ -196,16 +223,50 @@ export function PDFProvider({ children }: { children: ReactNode }) { } pageText += lineText + ' '; } - - fullText += pageText + '\n'; + + return pageText.replace(/\s+/g, ' ').trim(); + } catch (error) { + console.error('Error extracting text from PDF:', error); + throw new Error('Failed to extract text from PDF'); } + }, []); - return fullText.replace(/\s+/g, ' ').trim(); + // Load curr doc text + const loadCurrDocText = useCallback(async () => { + try { + if (!currDocURL) return; + const text = await extractTextFromPDF(currDocURL, currDocPage); + setCurrDocText(text); + setTTSText(text); } catch (error) { - console.error('Error extracting text from PDF:', error); - throw new Error('Failed to extract text from PDF'); + console.error('Error loading PDF text:', error); + setError('Failed to extract PDF text'); } - }, []); + }, [currDocURL, currDocPage, extractTextFromPDF, setTTSText]); + + // Update the current document text when the page changes + useEffect(() => { + if (currDocURL) { + loadCurrDocText(); + } + }, [currDocPage, currDocURL, loadCurrDocText]); + + // Set curr document + const setCurrentDocument = useCallback(async (id: string): Promise => { + setError(null); + try { + const doc = await getDocument(id); + if (doc) { + const url = await convertPDFDataToURL(doc.data); + setCurrDocName(doc.name); + setCurrDocURL(url); + //await loadCurrDocText(); + } + } catch (error) { + console.error('Failed to get document URL:', error); + setError('Failed to retrieve the document. Please try again.'); + } + }, [getDocument, loadCurrDocText]); // Clear all highlights in the PDF viewer const clearHighlights = useCallback(() => { @@ -291,7 +352,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { // Search for the best match within the visible area first let bestMatch = findBestTextMatch(visibleNodes, cleanPattern, cleanPattern.length * 2); - + // If no good match found in visible area, search the entire document if (bestMatch.rating < 0.3) { bestMatch = findBestTextMatch(allText, cleanPattern, cleanPattern.length * 2); @@ -324,12 +385,12 @@ export function PDFProvider({ children }: { children: ReactNode }) { // Handle text click events in the PDF viewer const handleTextClick = useCallback(( event: MouseEvent, - pdfText: string, + pageText: string, // Renamed from pdfText to pageText for clarity containerRef: React.RefObject, stopAndPlayFromIndex: (index: number) => void, isProcessing: boolean ) => { - if (isProcessing) return; // Don't process clicks while TTS is processing + if (isProcessing) return; const target = event.target as HTMLElement; if (!target.matches('.react-pdf__Page__textContent span')) return; @@ -361,7 +422,8 @@ export function PDFProvider({ children }: { children: ReactNode }) { if (bestMatch.rating >= similarityThreshold) { const matchText = bestMatch.text; - const sentences = nlp(pdfText).sentences().out('array') as string[]; + // Use pageText instead of full PDF text for sentence splitting + const sentences = nlp(pageText).sentences().out('array') as string[]; let bestSentenceMatch = { sentence: '', rating: 0 }; for (const sentence of sentences) { @@ -375,7 +437,7 @@ export function PDFProvider({ children }: { children: ReactNode }) { const sentenceIndex = sentences.findIndex((sentence) => sentence === bestSentenceMatch.sentence); if (sentenceIndex !== -1) { stopAndPlayFromIndex(sentenceIndex); - highlightPattern(pdfText, bestSentenceMatch.sentence, containerRef); + highlightPattern(pageText, bestSentenceMatch.sentence, containerRef); } } } @@ -394,6 +456,13 @@ export function PDFProvider({ children }: { children: ReactNode }) { highlightPattern, clearHighlights, handleTextClick, + onDocumentLoadSuccess, + setCurrentDocument, + currDocURL, + currDocName, + currDocPages, + currDocPage, + currDocText, }), [ documents, @@ -406,6 +475,13 @@ export function PDFProvider({ children }: { children: ReactNode }) { highlightPattern, clearHighlights, handleTextClick, + onDocumentLoadSuccess, + setCurrentDocument, + currDocURL, + currDocName, + currDocPages, + currDocPage, + currDocText, ] ); diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index d090af4..f3669b4 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -8,13 +8,15 @@ import React, { useEffect, useRef, } from 'react'; -import nlp from 'compromise'; import OpenAI from 'openai'; import { LRUCache } from 'lru-cache'; // Import LRUCache directly -import { useConfig } from './ConfigContext'; import { Howl } from 'howler'; -// Add type declarations +import { useConfig } from '@/contexts/ConfigContext'; +import { splitIntoSentences, preprocessSentenceForAudio } from '@/services/nlp'; +import { audioBufferToURL } from '@/services/audio'; + +// Media globals declare global { interface Window { webkitAudioContext: typeof AudioContext; @@ -23,7 +25,7 @@ declare global { type AudioContextType = typeof window extends undefined ? never - : (AudioContext | null); + : (AudioContext); interface TTSContextType { isPlaying: boolean; @@ -34,12 +36,12 @@ interface TTSContextType { setText: (text: string) => void; currentSentence: string; audioQueue: AudioBuffer[]; - currentAudioIndex: number; stop: () => void; - setCurrentIndex: (index: number) => void; stopAndPlayFromIndex: (index: number) => void; sentences: string[]; isProcessing: boolean; + setIsProcessing: (value: boolean) => void; + setIsPlaying: (value: boolean) => void; speed: number; setSpeed: (speed: number) => void; setSpeedAndRestart: (speed: number) => void; @@ -47,6 +49,11 @@ interface TTSContextType { setVoice: (voice: string) => void; setVoiceAndRestart: (voice: string) => void; availableVoices: string[]; + currentIndex: number; + setCurrentIndex: (index: number) => void; + currDocPage: number; + currDocPages: number | undefined; + setCurrDocPages: (pages: number) => void; } const TTSContext = createContext(undefined); @@ -62,98 +69,96 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { const [currentText, setCurrentText] = useState(''); const [sentences, setSentences] = useState([]); const [currentIndex, setCurrentIndex] = useState(0); - const [audioContext, setAudioContext] = useState(null); - const currentRequestRef = useRef(null); + const [audioContext, setAudioContext] = useState(); const [activeHowl, setActiveHowl] = useState(null); const [audioQueue] = useState([]); - const [currentAudioIndex] = useState(0); const [isProcessing, setIsProcessing] = useState(false); - const skipTriggeredRef = useRef(false); - const skipTimeoutRef = useRef(null); - const isPausingRef = useRef(false); const [speed, setSpeed] = useState(1); const [voice, setVoice] = useState('alloy'); const [availableVoices, setAvailableVoices] = useState([]); + const [currDocPage, setCurrDocPage] = useState(1); + const [currDocPages, setCurrDocPages] = useState(); + const [nextPageLoading, setNextPageLoading] = useState(false); + // Audio cache using LRUCache with a maximum size of 50 entries const audioCacheRef = useRef(new LRUCache({ max: 50 })); - // Move these function declarations up before they are used + const setText = useCallback((text: string) => { + setCurrentText(text); + console.log('Setting page text:', text); + const newSentences = splitIntoSentences(text); + setSentences(newSentences); + + // Clear audio cache + //audioCacheRef.current.clear(); + + setNextPageLoading(false); + }, []); + + const abortAudio = useCallback(() => { + if (activeHowl) { + activeHowl.stop(); + setActiveHowl(null); + } + }, [activeHowl]); + const togglePlay = useCallback(() => { setIsPlaying((prev) => { if (!prev) { - isPausingRef.current = false; return true; } else { - if (activeHowl) { - isPausingRef.current = true; - activeHowl.stop(); - setActiveHowl(null); - } + abortAudio(); return false; } }); - }, [activeHowl]); + }, [abortAudio]); + + const advance = useCallback(async (backwards = false) => { + setCurrentIndex((prev) => { + const nextIndex = prev + (backwards ? -1 : 1); + if (nextIndex < sentences.length && nextIndex >= 0) { + console.log('Advancing to next sentence:', sentences[nextIndex]); + return nextIndex; + } else if (nextIndex >= sentences.length && currDocPage < currDocPages!) { + console.log('Advancing to next page:', currDocPage + 1); + + setNextPageLoading(true); + setCurrDocPage(currDocPage + 1); + + return 0; + } else if (nextIndex < 0 && currDocPage > 1) { + console.log('Advancing to previous page:', currDocPage - 1); + + setNextPageLoading(true); + setCurrDocPage(currDocPage - 1); + + return 0; + } + return prev; + }); + }, [sentences, currDocPage, currDocPages]); + const skipForward = useCallback(() => { - if (skipTimeoutRef.current) { - clearTimeout(skipTimeoutRef.current); - } - - skipTriggeredRef.current = true; setIsProcessing(true); - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - currentRequestRef.current = null; - } + abortAudio(); - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); - } + advance(); - setCurrentIndex((prev) => { - const nextIndex = Math.min(prev + 1, sentences.length - 1); - console.log('Skipping forward to:', sentences[nextIndex]); - return nextIndex; - }); - - skipTimeoutRef.current = setTimeout(() => { - skipTriggeredRef.current = false; - setIsProcessing(false); - }, 100); - }, [sentences, activeHowl]); + setIsProcessing(false); + }, [abortAudio, advance]); const skipBackward = useCallback(() => { - if (skipTimeoutRef.current) { - clearTimeout(skipTimeoutRef.current); - } - - skipTriggeredRef.current = true; setIsProcessing(true); - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - currentRequestRef.current = null; - } + abortAudio(); - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); - } + advance(true); // Pass true to go backwards - setCurrentIndex((prev) => { - const nextIndex = Math.max(prev - 1, 0); - console.log('Skipping backward to:', sentences[nextIndex]); - return nextIndex; - }); - - skipTimeoutRef.current = setTimeout(() => { - skipTriggeredRef.current = false; - setIsProcessing(false); - }, 100); - }, [sentences, activeHowl]); + setIsProcessing(false); + }, [abortAudio, advance]); // Initialize OpenAI instance when config loads useEffect(() => { @@ -206,6 +211,14 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { } } } + + return () => { + if (audioContext) { + audioContext.close().catch((error) => { + console.error('Error closing AudioContext:', error); + }); + } + } }, [audioContext]); // Now the MediaSession effect can use these functions @@ -230,110 +243,60 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { } }, [togglePlay, skipForward, skipBackward]); - // Text preprocessing function to clean and normalize text - const preprocessSentenceForAudio = (text: string): string => { - return text - // Replace URLs with descriptive text including domain - .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') - // Remove special characters except basic punctuation - //.replace(/[^\w\s.,!?;:'"()-]/g, ' ') - // Fix hyphenated words at line breaks (word- word -> wordword) - .replace(/(\w+)-\s+(\w+)/g, '$1$2') - // Replace multiple spaces with single space - .replace(/\s+/g, ' ') - // Trim whitespace - .trim(); - }; - - const splitIntoSentences = (text: string): string[] => { - // Preprocess the text before splitting into sentences - const cleanedText = preprocessSentenceForAudio(text); - const doc = nlp(cleanedText); - return doc.sentences().out('array') as string[]; - }; - - const processNextSentence = useCallback(async () => { - if (!isPlaying || currentIndex >= sentences.length - 1 || skipTriggeredRef.current) { - setIsPlaying(false); - return; + //new function to return audio buffer with caching + const getAudio = useCallback(async (sentence: string): Promise => { + // Check if the audio is already cached + const cachedAudio = audioCacheRef.current.get(sentence); + if (cachedAudio) { + console.log('Using cached audio for sentence:', sentence.substring(0, 20)); + return cachedAudio; } - setCurrentIndex((prev) => { - const nextIndex = prev + 1; - if (nextIndex < sentences.length) { - console.log('Auto-advancing to next sentence:', sentences[nextIndex]); - return nextIndex; - } - return prev; - }); - }, [isPlaying, currentIndex, sentences]); + // If not cached, fetch the audio from OpenAI API + if (openaiRef.current) { + console.log('Requesting audio for sentence:', sentence); - const processAndPlaySentence = async (sentence: string) => { - if (!audioContext || isProcessing || !openaiRef.current) return; + const response = await openaiRef.current.audio.speech.create({ + model: 'tts-1', + voice: voice as "alloy", + input: sentence, + speed: speed, + }); + const arrayBuffer = await response.arrayBuffer(); + const audioBuffer = await audioContext!.decodeAudioData(arrayBuffer); + + // Cache the audio buffer + audioCacheRef.current.set(sentence, audioBuffer); + + return audioBuffer; + } + }, [audioContext, voice, speed]); + + const processSentence = useCallback(async (sentence: string, preload = false): Promise => { + if (isProcessing && !preload) throw new Error('Audio is already being processed'); + if (!audioContext || !openaiRef.current) throw new Error('Audio context not initialized'); + + // Only set processing state if not preloading + if (!preload) setIsProcessing(true); + + const cleanedSentence = preprocessSentenceForAudio(sentence); + const audioBuffer = await getAudio(cleanedSentence); + + return audioBufferToURL(audioBuffer!); + }, [isProcessing, audioContext, getAudio]); + + const playSentenceWithHowl = useCallback(async (sentence: string) => { try { - // Only set processing if we need to fetch from API - const cleanedSentence = preprocessSentenceForAudio(sentence); - if (!audioCacheRef.current.has(cleanedSentence)) { - setIsProcessing(true); + const audioUrl = await processSentence(sentence); + if (!audioUrl) { + throw new Error('No audio URL generated'); } - - // Cancel any existing request - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - } - - // Create new abort controller for this request - currentRequestRef.current = new AbortController(); - - // Stop any currently playing audio - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); - } - - let audioBuffer = audioCacheRef.current.get(cleanedSentence); - - if (!audioBuffer) { - console.log(' Processing TTS for sentence:', cleanedSentence.substring(0, 50) + '...'); - const startTime = Date.now(); - const response = await openaiRef.current.audio.speech.create({ - model: 'tts-1', - voice: voice as "alloy", - input: cleanedSentence, - speed: speed, - }); - - const duration = Date.now() - startTime; - console.log(` TTS processing completed in ${duration}ms`); - - const arrayBuffer = await response.arrayBuffer(); - audioBuffer = await audioContext.decodeAudioData(arrayBuffer); - - // Store in cache - audioCacheRef.current.set(cleanedSentence, audioBuffer); - setIsProcessing(false); - } - - // If the request was aborted or component unmounted, do not proceed - if (!currentRequestRef.current) return; - - // Convert AudioBuffer to URL for Howler - const audioUrl = audioBufferToURL(audioBuffer!); - + const howl = new Howl({ src: [audioUrl], format: ['wav'], html5: true, - onend: () => { - setActiveHowl(null); - // Cleanup the URL when audio ends - URL.revokeObjectURL(audioUrl); - if (isPlaying && !skipTriggeredRef.current && !isPausingRef.current) { - processNextSentence(); - } - isPausingRef.current = false; - }, onplay: () => { if ('mediaSession' in navigator) { navigator.mediaSession.playbackState = 'playing'; @@ -344,281 +307,110 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { navigator.mediaSession.playbackState = 'paused'; } }, + onend: () => { + URL.revokeObjectURL(audioUrl); + setActiveHowl(null); + if (isPlaying) { + advance(); + } + }, + onloaderror: (id, error) => { + console.error('Error loading audio:', error); + setIsProcessing(false); + setActiveHowl(null); + URL.revokeObjectURL(audioUrl); + // Don't auto-advance on load error + setIsPlaying(false); + }, }); - + setActiveHowl(howl); howl.play(); - - } catch (error: unknown) { - if (error instanceof Error && error.name === 'AbortError') { - console.log('Request was cancelled'); - } else { - console.error('Error processing TTS:', error); - } + setIsProcessing(false); + + } catch (error) { + console.error('Error playing TTS:', error); setActiveHowl(null); setIsProcessing(false); - } finally { - currentRequestRef.current = null; - } - }; - - // Add utility function to convert AudioBuffer to URL - const audioBufferToURL = (audioBuffer: AudioBuffer): string => { - // Get WAV file bytes - const wavBytes = getWavBytes(audioBuffer.getChannelData(0), { - isFloat: true, // floating point or 16-bit integer - numChannels: 1, // number of channels - sampleRate: audioBuffer.sampleRate, // audio sample rate - }); - - // Create blob and URL - const blob = new Blob([wavBytes], { type: 'audio/wav' }); - return URL.createObjectURL(blob); - }; - - // Add helper function for WAV conversion - const getWavBytes = (samples: Float32Array, opts: { - isFloat?: boolean, - numChannels?: number, - sampleRate?: number, - }) => { - const { - isFloat = true, - numChannels = 1, - sampleRate = 44100, - } = opts; - - const bytesPerSample = isFloat ? 4 : 2; - const numSamples = samples.length; - - // WAV header size is 44 bytes - const buffer = new ArrayBuffer(44 + numSamples * bytesPerSample); - const dv = new DataView(buffer); - - let pos = 0; - - // Write WAV header - writeString(dv, pos, 'RIFF'); pos += 4; - dv.setUint32(pos, 36 + numSamples * bytesPerSample, true); pos += 4; - writeString(dv, pos, 'WAVE'); pos += 4; - writeString(dv, pos, 'fmt '); pos += 4; - dv.setUint32(pos, 16, true); pos += 4; - dv.setUint16(pos, isFloat ? 3 : 1, true); pos += 2; - dv.setUint16(pos, numChannels, true); pos += 2; - dv.setUint32(pos, sampleRate, true); pos += 4; - dv.setUint32(pos, sampleRate * numChannels * bytesPerSample, true); pos += 4; - dv.setUint16(pos, numChannels * bytesPerSample, true); pos += 2; - dv.setUint16(pos, bytesPerSample * 8, true); pos += 2; - writeString(dv, pos, 'data'); pos += 4; - dv.setUint32(pos, numSamples * bytesPerSample, true); pos += 4; - - if (isFloat) { - for (let i = 0; i < numSamples; i++) { - dv.setFloat32(pos, samples[i], true); - pos += bytesPerSample; - } - } else { - for (let i = 0; i < numSamples; i++) { - const s = Math.max(-1, Math.min(1, samples[i])); - dv.setInt16(pos, s < 0 ? s * 0x8000 : s * 0x7FFF, true); - pos += bytesPerSample; + //setIsPlaying(false); // Stop playback on error + + if (error instanceof Error && + (error.message.includes('Unable to decode audio data') || + error.message.includes('EncodingError'))) { + console.log('Skipping problematic sentence:', sentence); + advance(); // Skip problematic sentence } } + }, [isPlaying, processSentence, advance]); - return buffer; - }; - - const writeString = (view: DataView, offset: number, string: string): void => { - for (let i = 0; i < string.length; i++) { - view.setUint8(offset + i, string.charCodeAt(i)); - } - }; - - const setText = useCallback((text: string) => { - setCurrentText(text); - const newSentences = splitIntoSentences(text); - setSentences(newSentences); - setCurrentIndex(0); - setIsPlaying(false); - - // Clear audio cache - audioCacheRef.current.clear(); - - // Preload the first sentence immediately - if (newSentences.length > 0) { - preloadSentence(newSentences[0]).then(() => { - // Preload the second sentence after a small delay - if (newSentences[1]) { - setTimeout(() => preloadSentence(newSentences[1]), 200); - } - }); - } - }, []); - - // Preload adjacent sentences when currentIndex changes - useEffect(() => { - /* - * Preloads the next sentence in the queue to improve playback performance. - * Only preloads the next sentence to reduce API load. - * - * Dependencies: - * - currentIndex: Re-runs when the currentIndex changes - * - sentences: Re-runs when the sentences array changes - */ - const preloadAdjacentSentences = async () => { - try { - // Only preload next sentence to reduce API load - if (sentences[currentIndex + 1] && !audioCacheRef.current.has(sentences[currentIndex + 1])) { - await new Promise((resolve) => setTimeout(resolve, 200)); // Add small delay - await preloadSentence(sentences[currentIndex + 1]); - } - } catch (error) { - console.error('Error preloading adjacent sentences:', error); - } - }; - preloadAdjacentSentences(); - }, [currentIndex, sentences]); - - const isMounted = useRef(false); - - useEffect(() => { - /* - * Plays the current sentence when the component is mounted or the currentIndex changes. - * Handles audio playback and auto-advances to the next sentence when finished. - * - * Dependencies: - * - isPlaying: Re-runs when the isPlaying state changes - * - currentIndex: Re-runs when the currentIndex changes - * - sentences: Re-runs when the sentences array changes - * - isProcessing: Re-runs when the isProcessing state changes - */ - // Skip the first mount in development - if (process.env.NODE_ENV === 'development') { - if (!isMounted.current) { - isMounted.current = true; - return; - } - } - - let isEffectActive = true; - - const playAudio = async () => { - if (isPlaying && sentences[currentIndex] && !isProcessing && isEffectActive) { - await processAndPlaySentence(sentences[currentIndex]); - } - }; - - playAudio(); - - return () => { - isEffectActive = false; - // Clean up any playing audio when the effect is cleaned up - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); - } - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - currentRequestRef.current = null; - } - }; - }, [isPlaying, currentIndex, sentences, isProcessing]); - - const preloadSentence = async (sentence: string) => { - if (!audioContext || !openaiRef.current) return; - if (audioCacheRef.current.has(sentence)) return; // Already cached - + const preloadNextAudio = useCallback(() => { try { - console.log(' Preloading TTS for sentence:', sentence.substring(0, 50) + '...'); - const startTime = Date.now(); - const response = await openaiRef.current.audio.speech.create({ - model: 'tts-1', - voice: voice as "alloy", - input: sentence, - speed: speed, - }); - - const duration = Date.now() - startTime; - console.log(` Preload TTS completed in ${duration}ms`); - - const arrayBuffer = await response.arrayBuffer(); - const audioBuffer = await audioContext.decodeAudioData(arrayBuffer); - - // Store in cache - audioCacheRef.current.set(sentence, audioBuffer); - } catch (error: unknown) { - if (error instanceof Error && error.name === 'AbortError') { - console.log('Request was cancelled'); - } else { - console.error('Error preloading TTS:', error); + if (sentences[currentIndex + 1] && !audioCacheRef.current.has(sentences[currentIndex + 1])) { + processSentence(sentences[currentIndex + 1], true); // True indicates preloading } + } catch (error) { + console.error('Error preloading next sentence:', error); } - }; + }, [currentIndex, sentences, audioCacheRef, processSentence]); + + const playAudio = useCallback(async () => { + await playSentenceWithHowl(sentences[currentIndex]); + }, [sentences, currentIndex, playSentenceWithHowl]); + + // main driver useEffect + useEffect(() => { + if (!isPlaying) return; // Don't proceed if stopped + if (isProcessing) return; // Don't proceed if processing audio + if (!sentences[currentIndex]) return; // Don't proceed if no sentence to play + if (nextPageLoading) return; // Don't proceed if loading next page + if (activeHowl) return; // Don't proceed if audio is already playing + + // Play the current sentence and preload the next one if available + playAudio(); + if (sentences[currentIndex + 1]) { + preloadNextAudio(); + } + + return () => { + abortAudio(); + }; + }, [ + isPlaying, + isProcessing, + currentIndex, + sentences, + activeHowl, + nextPageLoading, + playAudio, + preloadNextAudio, + abortAudio + ]); const stop = useCallback(() => { // Cancel any ongoing request - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - currentRequestRef.current = null; - } - - // Stop current audio - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); - } + abortAudio(); setIsPlaying(false); setCurrentIndex(0); setCurrentText(''); setIsProcessing(false); - }, [activeHowl]); + }, [abortAudio]); const stopAndPlayFromIndex = useCallback((index: number) => { - // Cancel any ongoing request - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - currentRequestRef.current = null; - } - - // Stop current audio - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); - } - - // Set skip flag to prevent immediate auto-advance - skipTriggeredRef.current = true; - - // Set new index and start playing - setCurrentIndex(index); - setIsPlaying(true); - - // Reset skip flag after a short delay to allow future auto-advance - if (skipTimeoutRef.current) { - clearTimeout(skipTimeoutRef.current); - } - skipTimeoutRef.current = setTimeout(() => { - skipTriggeredRef.current = false; - }, 100); - }, [activeHowl]); + abortAudio(); + + // Set the states in the next tick to ensure clean state + setTimeout(() => { + setCurrentIndex(index); + setIsPlaying(true); + }, 50); + }, [abortAudio]); const setCurrentIndexWithoutPlay = useCallback((index: number) => { - // Cancel any ongoing request - if (currentRequestRef.current) { - currentRequestRef.current.abort(); - currentRequestRef.current = null; - } - - // Stop current audio - if (activeHowl) { - activeHowl.stop(); - setActiveHowl(null); - } + abortAudio(); setCurrentIndex(index); - skipTriggeredRef.current = false; - }, [activeHowl]); + }, [abortAudio]); const setSpeedAndRestart = useCallback((newSpeed: number) => { setSpeed(newSpeed); @@ -661,12 +453,13 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { setText, currentSentence: sentences[currentIndex] || '', audioQueue, - currentAudioIndex, stop, setCurrentIndex: setCurrentIndexWithoutPlay, stopAndPlayFromIndex, sentences, isProcessing, + setIsProcessing, + setIsPlaying, speed, setSpeed, setSpeedAndRestart, @@ -674,6 +467,10 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { setVoice, setVoiceAndRestart, availableVoices, + currentIndex, + currDocPage, + currDocPages, + setCurrDocPages, }; if (configIsLoading) { diff --git a/src/services/audio.ts b/src/services/audio.ts new file mode 100644 index 0000000..ba4d097 --- /dev/null +++ b/src/services/audio.ts @@ -0,0 +1,71 @@ +// Add utility function to convert AudioBuffer to URL +export const audioBufferToURL = (audioBuffer: AudioBuffer): string => { + // Get WAV file bytes + const wavBytes = getWavBytes(audioBuffer.getChannelData(0), { + isFloat: true, // floating point or 16-bit integer + numChannels: 1, // number of channels + sampleRate: audioBuffer.sampleRate, // audio sample rate + }); + + // Create blob and URL + const blob = new Blob([wavBytes], { type: 'audio/wav' }); + return URL.createObjectURL(blob); +}; + +// Add helper function for WAV conversion +export const getWavBytes = (samples: Float32Array, opts: { + isFloat?: boolean, + numChannels?: number, + sampleRate?: number, +}) => { + const { + isFloat = true, + numChannels = 1, + sampleRate = 44100, + } = opts; + + const bytesPerSample = isFloat ? 4 : 2; + const numSamples = samples.length; + + // WAV header size is 44 bytes + const buffer = new ArrayBuffer(44 + numSamples * bytesPerSample); + const dv = new DataView(buffer); + + let pos = 0; + + // Write WAV header + writeString(dv, pos, 'RIFF'); pos += 4; + dv.setUint32(pos, 36 + numSamples * bytesPerSample, true); pos += 4; + writeString(dv, pos, 'WAVE'); pos += 4; + writeString(dv, pos, 'fmt '); pos += 4; + dv.setUint32(pos, 16, true); pos += 4; + dv.setUint16(pos, isFloat ? 3 : 1, true); pos += 2; + dv.setUint16(pos, numChannels, true); pos += 2; + dv.setUint32(pos, sampleRate, true); pos += 4; + dv.setUint32(pos, sampleRate * numChannels * bytesPerSample, true); pos += 4; + dv.setUint16(pos, numChannels * bytesPerSample, true); pos += 2; + dv.setUint16(pos, bytesPerSample * 8, true); pos += 2; + writeString(dv, pos, 'data'); pos += 4; + dv.setUint32(pos, numSamples * bytesPerSample, true); pos += 4; + + if (isFloat) { + for (let i = 0; i < numSamples; i++) { + dv.setFloat32(pos, samples[i], true); + pos += bytesPerSample; + } + } else { + for (let i = 0; i < numSamples; i++) { + const s = Math.max(-1, Math.min(1, samples[i])); + dv.setInt16(pos, s < 0 ? s * 0x8000 : s * 0x7FFF, true); + pos += bytesPerSample; + } + } + + return buffer; +}; + +export const writeString = (view: DataView, offset: number, string: string): void => { + for (let i = 0; i < string.length; i++) { + view.setUint8(offset + i, string.charCodeAt(i)); + } +}; \ No newline at end of file diff --git a/src/services/nlp.ts b/src/services/nlp.ts new file mode 100644 index 0000000..d702be9 --- /dev/null +++ b/src/services/nlp.ts @@ -0,0 +1,23 @@ +import nlp from 'compromise'; + +// Text preprocessing function to clean and normalize text +export const preprocessSentenceForAudio = (text: string): string => { + return text + // Replace URLs with descriptive text including domain + .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') + // Remove special characters except basic punctuation + //.replace(/[^\w\s.,!?;:'"()-]/g, ' ') + // Fix hyphenated words at line breaks (word- word -> wordword) + .replace(/(\w+)-\s+(\w+)/g, '$1$2') + // Replace multiple spaces with single space + .replace(/\s+/g, ' ') + // Trim whitespace + .trim(); +}; + +export const splitIntoSentences = (text: string): string[] => { + // Preprocess the text before splitting into sentences + const cleanedText = preprocessSentenceForAudio(text); + const doc = nlp(cleanedText); + return doc.sentences().out('array') as string[]; +}; \ No newline at end of file From b6880b9910ce46e01e3b090ee362e162150daad7 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Sat, 25 Jan 2025 22:36:28 -0700 Subject: [PATCH 2/8] Fix end of document reached --- src/contexts/TTSContext.tsx | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/contexts/TTSContext.tsx b/src/contexts/TTSContext.tsx index f3669b4..6724df8 100644 --- a/src/contexts/TTSContext.tsx +++ b/src/contexts/TTSContext.tsx @@ -134,6 +134,10 @@ export function TTSProvider({ children }: { children: React.ReactNode }) { setCurrDocPage(currDocPage - 1); return 0; + } else if (nextIndex >= sentences.length && currDocPage >= currDocPages!) { + console.log('Reached end of document'); + setIsPlaying(false); + return prev; } return prev; }); From de1ad2c4da2cca696585ebc52f7df07f15bac4f8 Mon Sep 17 00:00:00 2001 From: Richard Roberson Date: Sat, 25 Jan 2025 22:57:29 -0700 Subject: [PATCH 3/8] Implement page forward and back --- src/components/PDFViewer.tsx | 1 - src/components/player/Navigator.tsx | 10 +++++----- src/components/player/TTSPlayer.tsx | 6 +++--- src/contexts/TTSContext.tsx | 25 ++++++++++++++++++++----- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/src/components/PDFViewer.tsx b/src/components/PDFViewer.tsx index d00af3b..e743410 100644 --- a/src/components/PDFViewer.tsx +++ b/src/components/PDFViewer.tsx @@ -159,7 +159,6 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) { {}} />
); diff --git a/src/components/player/Navigator.tsx b/src/components/player/Navigator.tsx index 95cdba5..3e1ca9c 100644 --- a/src/components/player/Navigator.tsx +++ b/src/components/player/Navigator.tsx @@ -1,15 +1,15 @@ import { Button } from '@headlessui/react'; -export const Navigator = ({ currentPage, numPages, onPageChange }: { +export const Navigator = ({ currentPage, numPages, skipToPage }: { currentPage: number; numPages: number | undefined; - onPageChange: (page: number) => void; + skipToPage: (page: number) => void; }) => { return (
{/* Page back */}