danvics/openreader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Huge PDF text extraction optimization

Browse source
This commit is contained in:
Richard Roberson 2025-02-12 02:15:21 -07:00
parent 521eda6cff
commit f16116cab4
2 changed files with 19 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
src/contexts/PDFContext.tsx
View file

@ -33,6 +33,8 @@ import {
handleTextClick, handleTextClick,
} from '@/utils/pdf'; } from '@/utils/pdf';
import type { PDFDocumentProxy } from 'pdfjs-dist';
/** /**
* Interface defining all available methods and properties in the PDF context * Interface defining all available methods and properties in the PDF context
*/ */
@ -43,11 +45,12 @@ interface PDFContextType {
currDocPages: number | undefined; currDocPages: number | undefined;
currDocPage: number; currDocPage: number;
currDocText: string | undefined; currDocText: string | undefined;
pdfDocument: PDFDocumentProxy | undefined;
setCurrentDocument: (id: string) => Promise<void>; setCurrentDocument: (id: string) => Promise<void>;
clearCurrDoc: () => void; clearCurrDoc: () => void;
// PDF functionality // PDF functionality
onDocumentLoadSuccess: ({ numPages }: { numPages: number }) => void; onDocumentLoadSuccess: (pdf: PDFDocumentProxy) => void;
highlightPattern: (text: string, pattern: string, containerRef: React.RefObject<HTMLDivElement>) => void; highlightPattern: (text: string, pattern: string, containerRef: React.RefObject<HTMLDivElement>) => void;
clearHighlights: () => void; clearHighlights: () => void;
handleTextClick: ( handleTextClick: (
@ -75,15 +78,17 @@ export function PDFProvider({ children }: { children: ReactNode }) {
const [currDocURL, setCurrDocURL] = useState<string>(); const [currDocURL, setCurrDocURL] = useState<string>();
const [currDocName, setCurrDocName] = useState<string>(); const [currDocName, setCurrDocName] = useState<string>();
const [currDocText, setCurrDocText] = useState<string>(); const [currDocText, setCurrDocText] = useState<string>();
const [pdfDocument, setPdfDocument] = useState<PDFDocumentProxy>();
/** /**
* Handles successful document load * Handles successful document load
* *
* @param {Object} param0 - Object containing numPages * @param {Object} param0 - Object containing numPages
*/ */
const onDocumentLoadSuccess = useCallback(({ numPages }: { numPages: number }) => { const onDocumentLoadSuccess = useCallback((pdf: PDFDocumentProxy) => {
console.log('Document loaded:', numPages); console.log('Document loaded:', pdf.numPages);
setCurrDocPages(numPages); setCurrDocPages(pdf.numPages);
setPdfDocument(pdf);
}, [setCurrDocPages]); }, [setCurrDocPages]);
/** /**
@ -91,15 +96,15 @@ export function PDFProvider({ children }: { children: ReactNode }) {
*/ */
const loadCurrDocText = useCallback(async () => { const loadCurrDocText = useCallback(async () => {
try { try {
if (!currDocURL) return; if (!pdfDocument) return;
const text = await extractTextFromPDF(currDocURL, currDocPage); const text = await extractTextFromPDF(pdfDocument, currDocPage);
setCurrDocText(text); setCurrDocText(text);
setTTSText(text); setTTSText(text);
} catch (error) { } catch (error) {
console.error('Error loading PDF text:', error); console.error('Error loading PDF text:', error);
} }
}, [currDocURL, currDocPage, setTTSText]); }, [pdfDocument, currDocPage, setTTSText]);
/** /**
* Updates the current document text when the page changes * Updates the current document text when the page changes
@ -136,6 +141,7 @@ export function PDFProvider({ children }: { children: ReactNode }) {
setCurrDocURL(undefined); setCurrDocURL(undefined);
setCurrDocText(undefined); setCurrDocText(undefined);
setCurrDocPages(undefined); setCurrDocPages(undefined);
setPdfDocument(undefined);
stop(); stop();
}, [setCurrDocPages, stop]); }, [setCurrDocPages, stop]);
@ -153,6 +159,7 @@ export function PDFProvider({ children }: { children: ReactNode }) {
highlightPattern, highlightPattern,
clearHighlights, clearHighlights,
handleTextClick, handleTextClick,
pdfDocument,
}), }),
[ [
onDocumentLoadSuccess, onDocumentLoadSuccess,
@ -163,6 +170,7 @@ export function PDFProvider({ children }: { children: ReactNode }) {
currDocPage, currDocPage,
currDocText, currDocText,
clearCurrDoc, clearCurrDoc,
pdfDocument,
] ]
); );

15
src/utils/pdf.ts
View file

@ -2,9 +2,11 @@ import { pdfjs } from 'react-pdf';
import nlp from 'compromise'; import nlp from 'compromise';
import stringSimilarity from 'string-similarity'; import stringSimilarity from 'string-similarity';
import type { TextItem } from 'pdfjs-dist/types/src/display/api'; import type { TextItem } from 'pdfjs-dist/types/src/display/api';
import type { PDFDocumentProxy } from 'pdfjs-dist';
// Set worker from public directory // Set worker from public directory and compatibility mode
pdfjs.GlobalWorkerOptions.workerSrc = '/pdf.worker.mjs'; pdfjs.GlobalWorkerOptions.workerSrc = '/pdf.worker.mjs';
pdfjs.GlobalWorkerOptions.workerPort = null;
interface TextMatch { interface TextMatch {
elements: HTMLElement[]; elements: HTMLElement[];
@ -24,17 +26,8 @@ export function convertPDFDataToURL(pdfData: Blob): Promise<string> {
} }
// Text Processing functions // Text Processing functions
export async function extractTextFromPDF(pdfURL: string, pageNumber: number): Promise<string> { export async function extractTextFromPDF(pdf: PDFDocumentProxy, pageNumber: number): Promise<string> {
try { try {
const base64Data = pdfURL.split(',')[1];
const binaryData = atob(base64Data);
const bytes = new Uint8Array(binaryData.length);
for (let i = 0; i < binaryData.length; i++) {
bytes[i] = binaryData.charCodeAt(i);
}
const loadingTask = pdfjs.getDocument({ data: bytes });
const pdf = await loadingTask.promise;
const page = await pdf.getPage(pageNumber); const page = await pdf.getPage(pageNumber);
const textContent = await page.getTextContent(); const textContent = await page.getTextContent();