From e52b7541064e12617f0745697bebf5fb7f60175d Mon Sep 17 00:00:00 2001 From: Richard R Date: Tue, 19 May 2026 19:11:51 -0600 Subject: [PATCH] refactor(pdf): remove legacy text extraction and update parse status normalization Eliminate unused PDF text extraction logic and margin-based filtering from the client library. Refactor API routes to generalize parse status normalization, removing explicit references to deprecated "unsupported" status and type variants. Update usages to accept string parse statuses for improved flexibility. Simplify document state dependency tracking in the PDF document hook. --- src/app/(app)/pdf/[id]/usePdfDocument.ts | 2 +- src/app/api/documents/[id]/parsed/route.ts | 17 +-- src/app/api/documents/route.ts | 11 +- src/lib/client/pdf.ts | 136 +-------------------- 4 files changed, 19 insertions(+), 147 deletions(-) diff --git a/src/app/(app)/pdf/[id]/usePdfDocument.ts b/src/app/(app)/pdf/[id]/usePdfDocument.ts index 6cb221e..90fe3ec 100644 --- a/src/app/(app)/pdf/[id]/usePdfDocument.ts +++ b/src/app/(app)/pdf/[id]/usePdfDocument.ts @@ -151,7 +151,7 @@ export function usePdfDocument(): PdfDocumentState { settings: documentSettings, smartSentenceSplitting, maxBlockLength: ttsSegmentMaxBlockLength, - }), [pdfDocument, parsedDocument, documentSettings, smartSentenceSplitting, ttsSegmentMaxBlockLength]); + }), [parsedDocument, documentSettings, smartSentenceSplitting, ttsSegmentMaxBlockLength]); const pageTextCacheRef = useRef>(new Map()); const [currDocPage, setCurrDocPage] = useState(currDocPageNumber); diff --git a/src/app/api/documents/[id]/parsed/route.ts b/src/app/api/documents/[id]/parsed/route.ts index d23bfa2..b2ad9c4 100644 --- a/src/app/api/documents/[id]/parsed/route.ts +++ b/src/app/api/documents/[id]/parsed/route.ts @@ -24,10 +24,12 @@ function hasAnyParsedBlocks(doc: ParsedPdfDocument | null): boolean { } function normalizeParseStatus( - status: 'pending' | 'running' | 'ready' | 'failed' | 'unsupported' | null, + status: string | null, ): 'pending' | 'running' | 'ready' | 'failed' { - if (status === 'unsupported' || status === null) return 'pending'; - return status; + if (status === 'pending' || status === 'running' || status === 'ready' || status === 'failed') { + return status; + } + return 'pending'; } export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) { @@ -59,7 +61,7 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{ id: string; userId: string; - parseStatus: 'pending' | 'running' | 'ready' | 'failed' | 'unsupported' | null; + parseStatus: string | null; }>; const row = rows.find((candidate) => candidate.userId === storageUserId) ?? rows[0]; @@ -67,7 +69,8 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string return NextResponse.json({ error: 'Not found' }, { status: 404 }); } - if (row.parseStatus === 'unsupported') { + const effectiveStatus = normalizeParseStatus(row.parseStatus); + if (row.parseStatus !== effectiveStatus) { await db .update(documents) .set({ parseStatus: 'pending' }) @@ -80,8 +83,6 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string return NextResponse.json({ parseStatus: 'pending' }, { status: 202 }); } - const effectiveStatus = normalizeParseStatus(row.parseStatus); - if (effectiveStatus === 'failed' && retryFailed) { await db .update(documents) @@ -175,7 +176,7 @@ export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string .where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{ id: string; userId: string; - parseStatus: 'pending' | 'running' | 'ready' | 'failed' | 'unsupported' | null; + parseStatus: string | null; }>; const row = rows.find((candidate) => candidate.userId === storageUserId) ?? rows[0]; diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index 966604c..0d6a7b1 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -26,10 +26,13 @@ type RegisterDocument = { }; function normalizeParseStatus( - status: 'pending' | 'running' | 'ready' | 'failed' | 'unsupported' | null, + status: string | null, ): 'pending' | 'running' | 'ready' | 'failed' | null { - if (status === 'unsupported') return 'pending'; - return status; + if (status === null) return null; + if (status === 'pending' || status === 'running' || status === 'ready' || status === 'failed') { + return status; + } + return 'pending'; } function s3NotConfiguredResponse(): NextResponse { @@ -226,7 +229,7 @@ export async function GET(req: NextRequest) { size: number; lastModified: number; filePath: string; - parseStatus: 'pending' | 'running' | 'ready' | 'failed' | 'unsupported' | null; + parseStatus: string | null; parsedJsonKey: string | null; }>; diff --git a/src/lib/client/pdf.ts b/src/lib/client/pdf.ts index e1719dc..9727d4b 100644 --- a/src/lib/client/pdf.ts +++ b/src/lib/client/pdf.ts @@ -1,10 +1,8 @@ import { pdfjs } from 'react-pdf'; -import type { TextItem } from 'pdfjs-dist/types/src/display/api'; -import { type PDFDocumentProxy, TextLayer } from 'pdfjs-dist'; +import { TextLayer } from 'pdfjs-dist'; import "core-js/proposals/promise-with-resolvers"; import type { TTSSentenceAlignment } from '@/types/tts'; -import type { ParsedPdfDocument, ParsedPdfPage, ParsedPdfBlockKind } from '@/types/parsed-pdf'; -import { buildPageTextFromBlocks } from '@/lib/client/pdf-block-text'; +import type { ParsedPdfDocument, ParsedPdfPage } from '@/types/parsed-pdf'; import { CmpStr } from 'cmpstr'; import type { TTSSegmentLocator } from '@/types/client'; @@ -188,136 +186,6 @@ const normalizeWordForMatch = (text: string): string => .replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/g, '') .toLowerCase(); -// Text Processing functions -export async function extractTextFromPDF( - pdf: PDFDocumentProxy, - pageNumber: number, - margins = { header: 0.07, footer: 0.07, left: 0.07, right: 0.07 }, - parsed?: ParsedPdfPage, - skipKinds?: ParsedPdfBlockKind[], -): Promise { - try { - if (parsed) { - return buildPageTextFromBlocks(parsed, skipKinds ?? []); - } - - const page = await pdf.getPage(pageNumber); - const textContent = await page.getTextContent(); - - const viewport = page.getViewport({ scale: 1.0 }); - const pageHeight = viewport.height; - const pageWidth = viewport.width; - - const textItems = textContent.items.filter((item): item is TextItem => { - if (!('str' in item && 'transform' in item)) return false; - - const [scaleX, skewX, skewY, scaleY, x, y] = item.transform; - - // Basic text filtering - if (Math.abs(scaleX) < 1 || Math.abs(scaleX) > 20) return false; - if (Math.abs(scaleY) < 1 || Math.abs(scaleY) > 20) return false; - if (Math.abs(skewX) > 0.5 || Math.abs(skewY) > 0.5) return false; - - // Calculate margins in PDF coordinate space (y=0 is at bottom) - const headerY = pageHeight * (1 - margins.header); // Convert from top margin to bottom-based Y - const footerY = pageHeight * margins.footer; // Footer Y stays as is since it's already bottom-based - const leftX = pageWidth * margins.left; - const rightX = pageWidth * (1 - margins.right); - - // Check margins - remember y=0 is at bottom of page in PDF coordinates - if (y > headerY || y < footerY) { // Y greater than headerY means it's in header area, less than footerY means footer area - return false; - } - - // Check horizontal margins - if (x < leftX || x > rightX) { - return false; - } - - // Sanity check for coordinates - if (x < 0 || x > pageWidth) return false; - - return item.str.trim().length > 0; - }); - - const tolerance = 2; - const lines: TextItem[][] = []; - let currentLine: TextItem[] = []; - let currentY: number | null = null; - - textItems.forEach((item) => { - const y = item.transform[5]; - if (currentY === null) { - currentY = y; - currentLine.push(item); - } else if (Math.abs(y - currentY) < tolerance) { - currentLine.push(item); - } else { - lines.push(currentLine); - currentLine = [item]; - currentY = y; - } - }); - lines.push(currentLine); - - let pageText = ''; - for (const line of lines) { - line.sort((a, b) => a.transform[4] - b.transform[4]); - let lineText = ''; - let prevItem: TextItem | null = null; - - for (const item of line) { - if (!prevItem) { - lineText = item.str; - } else { - const prevEndX = prevItem.transform[4] + (prevItem.width ?? 0); - const currentStartX = item.transform[4]; - const space = currentStartX - prevEndX; - - // Get average character width as fallback - const avgCharWidth = (item.width ?? 0) / Math.max(1, item.str.length); - - // Multiple conditions for space detection - const needsSpace = - // Primary check: significant gap between items - space > Math.max(avgCharWidth * 0.3, 2) || - // Secondary check: natural word boundary - (!/^\W/.test(item.str) && !/\W$/.test(prevItem.str)) || - // Tertiary check: items are far enough apart relative to their size - (space > ((prevItem.width ?? 0) * 0.25)); - - if (needsSpace) { - lineText += ' ' + item.str; - } else { - lineText += item.str; - } - } - prevItem = item; - } - pageText += lineText + ' '; - } - - return pageText.replace(/\s+/g, ' ').trim(); - } catch (error) { - // During Next.js fast refresh / route transitions, react-pdf can tear down the - // underlying worker and pdf.js may throw a TypeError like: - // "null is not an object (evaluating 'this.messageHandler.sendWithPromise')". - // Treat this as a cancellation so the app can ignore it. - if ( - error instanceof TypeError && - typeof error.message === 'string' && - error.message.includes('messageHandler') && - error.message.includes('sendWithPromise') - ) { - throw new DOMException('PDF worker torn down', 'AbortError'); - } - - console.error('Error extracting text from PDF:', error); - // Preserve the original error so callers can decide whether to retry/ignore. - throw error; - } -} - // Highlighting functions let highlightPatternSeq = 0;