Add PDF page virtualization + virtualized scaling + make highlight better

package-lock.json

@@ -10,6 +10,7 @@
       "dependencies": {
         "@headlessui/react": "^2.2.0",
         "@types/howler": "^2.2.12",
+        "@types/react-window": "^1.8.8",
         "@types/string-similarity": "^4.0.2",
         "@types/uuid": "^10.0.0",
         "compromise": "^14.14.4",
@@ -22,6 +23,7 @@
         "react-dom": "^19.0.0",
         "react-dropzone": "^14.3.5",
         "react-pdf": "^9.2.1",
+        "react-window": "^1.8.11",
         "string-similarity": "^4.0.4",
         "uuid": "^11.0.5"
       },
@@ -50,6 +52,18 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/@babel/runtime": {
+      "version": "7.26.0",
+      "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.26.0.tgz",
+      "integrity": "sha512-FDSOghenHTiToteC/QRlv2q3DhPZ/oOXTBoirfWNx1Cx3TMVcGWQtMMmQcSvb/JjpNeGzx8Pq/b4fKEJuWm1sw==",
+      "license": "MIT",
+      "dependencies": {
+        "regenerator-runtime": "^0.14.0"
+      },
+      "engines": {
+        "node": ">=6.9.0"
+      }
+    },
     "node_modules/@emnapi/runtime": {
       "version": "1.3.1",
       "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.3.1.tgz",
@@ -1153,7 +1167,6 @@
       "version": "19.0.7",
       "resolved": "https://registry.npmjs.org/@types/react/-/react-19.0.7.tgz",
       "integrity": "sha512-MoFsEJKkAtZCrC1r6CM8U22GzhG7u2Wir8ons/aCKH6MBdD1ibV24zOSSkdZVUKqN5i396zG5VKLYZ3yaUZdLA==",
-      "devOptional": true,
       "license": "MIT",
       "dependencies": {
         "csstype": "^3.0.2"
@@ -1169,6 +1182,15 @@
         "@types/react": "^19.0.0"
       }
     },
+    "node_modules/@types/react-window": {
+      "version": "1.8.8",
+      "resolved": "https://registry.npmjs.org/@types/react-window/-/react-window-1.8.8.tgz",
+      "integrity": "sha512-8Ls660bHR1AUA2kuRvVG9D/4XpRC6wjAaPT9dil7Ckc76eP9TKWZwwmgfq8Q1LANX3QNDnoU4Zp48A3w+zK69Q==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/react": "*"
+      }
+    },
     "node_modules/@types/string-similarity": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/@types/string-similarity/-/string-similarity-4.0.2.tgz",
@@ -2186,7 +2208,6 @@
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.3.tgz",
       "integrity": "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==",
-      "devOptional": true,
       "license": "MIT"
     },
     "node_modules/damerau-levenshtein": {
@@ -4449,6 +4470,12 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/memoize-one": {
+      "version": "5.2.1",
+      "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz",
+      "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==",
+      "license": "MIT"
+    },
     "node_modules/merge-refs": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/merge-refs/-/merge-refs-1.3.0.tgz",
@@ -5556,6 +5583,23 @@
         }
       }
     },
+    "node_modules/react-window": {
+      "version": "1.8.11",
+      "resolved": "https://registry.npmjs.org/react-window/-/react-window-1.8.11.tgz",
+      "integrity": "sha512-+SRbUVT2scadgFSWx+R1P754xHPEqvcfSfVX10QYg6POOz+WNgkN48pS+BtZNIMGiL1HYrSEiCkwsMS15QogEQ==",
+      "license": "MIT",
+      "dependencies": {
+        "@babel/runtime": "^7.0.0",
+        "memoize-one": ">=3.1.1 <6"
+      },
+      "engines": {
+        "node": ">8.0.0"
+      },
+      "peerDependencies": {
+        "react": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0",
+        "react-dom": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0 || ^19.0.0"
+      }
+    },
     "node_modules/read-cache": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz",
@@ -5617,6 +5661,12 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/regenerator-runtime": {
+      "version": "0.14.1",
+      "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz",
+      "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==",
+      "license": "MIT"
+    },
     "node_modules/regexp.prototype.flags": {
       "version": "1.5.4",
       "resolved": "https://registry.npmjs.org/regexp.prototype.flags/-/regexp.prototype.flags-1.5.4.tgz",

package.json

@@ -11,6 +11,7 @@
   "dependencies": {
     "@headlessui/react": "^2.2.0",
     "@types/howler": "^2.2.12",
+    "@types/react-window": "^1.8.8",
     "@types/string-similarity": "^4.0.2",
     "@types/uuid": "^10.0.0",
     "compromise": "^14.14.4",
@@ -23,6 +24,7 @@
     "react-dom": "^19.0.0",
     "react-dropzone": "^14.3.5",
     "react-pdf": "^9.2.1",
+    "react-window": "^1.8.11",
     "string-similarity": "^4.0.4",
     "uuid": "^11.0.5"
   },

src/app/globals.css

@@ -35,3 +35,7 @@ body {
   -webkit-font-smoothing: antialiased;
   -moz-osx-font-smoothing: grayscale;
 }
+
+.pdf-list > div {
+  padding: 0 !important;
+}

src/components/PDFViewer.tsx

@@ -1,19 +1,62 @@
 'use client';
 
-import { RefObject } from 'react';
+import { RefObject, useCallback, useState, useEffect, useRef } from 'react';
 import { Document, Page } from 'react-pdf';
+import { FixedSizeList } from 'react-window';
 import 'react-pdf/dist/Page/AnnotationLayer.css';
 import 'react-pdf/dist/Page/TextLayer.css';
-import { useState, useEffect, useRef } from 'react';
 import { PDFSkeleton } from './PDFSkeleton';
 import { useTTS } from '@/contexts/TTSContext';
 import { usePDF } from '@/contexts/PDFContext';
+import { pdfjs } from 'react-pdf';
 
 interface PDFViewerProps {
   pdfData: Blob | undefined;
   zoomLevel: number;
 }
 
+interface PageItemData {
+  numPages: number;
+  scale: number;
+  containerWidth: number;
+  pageWidth: number;
+  pageHeight: number;
+}
+
+const PageComponent = ({ index, style, data }: { index: number; style: any; data: PageItemData }) => {
+  const { numPages, scale, pageHeight } = data;
+  
+  return (
+    <div style={{
+      ...style,
+      height: pageHeight,
+      display: 'flex',
+      flexDirection: 'column',
+      alignItems: 'center'
+    }}>
+      {index==0 ? (<div className="bg-offbase m-4 px-2 py-0.5 rounded-full w-fit">
+        <p className="text-xs">
+          {index + 1} / {numPages}
+        </p>
+      </div>) :
+      (<div className="bg-offbase m-4 px-2 py-0.5 rounded-full w-fit">
+        <p className="text-xs">
+          {index + 1} / {numPages}
+        </p>
+      </div>)}
+      <div className="flex justify-center">
+        <Page
+          pageNumber={index + 1}
+          renderAnnotationLayer={true}
+          renderTextLayer={true}
+          className="shadow-lg"
+          scale={scale}
+        />
+      </div>
+    </div>
+  );
+};
+
 export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) {
   const [numPages, setNumPages] = useState<number>();
   const [containerWidth, setContainerWidth] = useState<number>(0);
@@ -23,6 +66,8 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) {
   const [loadingError, setLoadingError] = useState<string>();
   const containerRef = useRef<HTMLDivElement>(null);
   const { extractTextFromPDF, highlightPattern, clearHighlights, handleTextClick } = usePDF();
+  const [pageHeight, setPageHeight] = useState(800); // Default height
+  const [pageSize, setPageSize] = useState({ width: 595, height: 842 }); // A4 default
 
   // Add static styles once during component initialization
   const styleElement = document.createElement('style');
@@ -146,12 +191,18 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) {
   }, [pdfText, currentSentence, highlightPattern, clearHighlights]);
 
   // Add scale calculation function
-  const calculateScale = (pageWidth: number = 595) => {  // 595 is default PDF width in points
+  const calculateScale = useCallback(() => {
-    const margin = 24; // 24px padding on each side
+    //const margin = 48; // 24px padding on each side
-    const targetWidth = containerWidth - margin;
+    //const targetWidth = containerWidth - margin;
-    const baseScale = targetWidth / pageWidth;
+    const baseScale = containerWidth / pageSize.width;
     return baseScale * (zoomLevel / 100);
-  };
+  }, [containerWidth, zoomLevel, pageSize.width]);
+
+  const calculatePageHeight = useCallback((viewport: { width: number; height: number }) => {
+    const scale = calculateScale();
+    const scaledHeight = viewport.height * scale;
+    return scaledHeight + 55; // 40px padding top and bottom
+  }, [calculateScale]);
 
   // Add resize observer effect
   useEffect(() => {
@@ -168,9 +219,37 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) {
     return () => observer.disconnect();
   }, []);
 
-  function onDocumentLoadSuccess({ numPages }: { numPages: number }): void {
+  const handleLoadSuccess = useCallback(async ({ numPages }: { numPages: number }) => {
     setNumPages(numPages);
-  }
+    
+    if (pdfData) {
+      try {
+        // Convert Blob to ArrayBuffer
+        const arrayBuffer = await pdfData.arrayBuffer();
+        const pdf = await pdfjs.getDocument(arrayBuffer).promise;
+        const page = await pdf.getPage(1);
+        const viewport = page.getViewport({ scale: 1 });
+        
+        setPageSize({
+          width: viewport.width,
+          height: viewport.height
+        });
+
+        const newPageHeight = calculatePageHeight(viewport);
+        setPageHeight(newPageHeight);
+      } catch (error) {
+        console.error('Error measuring page:', error);
+      }
+    }
+  }, [pdfData, calculatePageHeight]);
+
+  // Update page height when zoom level changes
+  useEffect(() => {
+    if (pageSize.width && pageSize.height) {
+      const newPageHeight = calculatePageHeight(pageSize);
+      setPageHeight(newPageHeight);
+    }
+  }, [zoomLevel, pageSize, calculatePageHeight]);
 
   return (
     <div
@@ -185,30 +264,28 @@ export function PDFViewer({ pdfData, zoomLevel }: PDFViewerProps) {
         loading={<PDFSkeleton />}
         noData={<PDFSkeleton />}
         file={pdfDataUrl}
-        onLoadSuccess={onDocumentLoadSuccess}
+        onLoadSuccess={handleLoadSuccess}
-        className="flex flex-col items-center m-0" 
+        className="flex flex-col items-center" 
       >
-        {Array.from(
+        {numPages && containerWidth ? (
-          new Array(numPages),
+          <FixedSizeList
-          (el, index) => (
+            height={Math.min(window.innerHeight - 100, numPages * pageHeight)}
-            <div key={`page_${index + 1}`}>
+            width={containerWidth}
-              <div className="bg-offbase my-4 px-2 py-0.5 rounded-full w-fit">
+            itemCount={numPages}
-                <p className="text-xs">
+            itemSize={pageHeight}
-                  {index + 1} / {numPages}
+            itemData={{
-                </p>
+              numPages,
-              </div>
+              scale: calculateScale(),
-              <div className="flex justify-center">
+              containerWidth,
-                <Page
+              pageWidth: pageSize.width,
-                  pageNumber={index + 1}
+              pageHeight: pageHeight,
-                  renderAnnotationLayer={true}
+            }}
-                  renderTextLayer={true}
+            className="pdf-list"
-                  className="shadow-lg"
+            overscanCount={2} // Add overscanning for smoother scrolling
-                  scale={calculateScale()}
+          >
-                />
+            {PageComponent}
-              </div>
+          </FixedSizeList>
-            </div>
+        ) : null}
-          ),
-        )}
       </Document>
     </div>
   );

src/contexts/PDFContext.tsx

@@ -276,48 +276,26 @@ export function PDFProvider({ children }: { children: ReactNode }) {
       text: (node.textContent || '').trim(),
     })).filter((node) => node.text.length > 0);
 
-    // Calculate the visible area of the container
+    // Search for the best match in the entire document
-    const containerRect = container.getBoundingClientRect();
+    const bestMatch = findBestTextMatch(allText, cleanPattern, cleanPattern.length * 2);
-    const visibleTop = container.scrollTop;
-    const visibleBottom = visibleTop + containerRect.height;
-
-    // Find nodes within the visible area and a buffer zone
-    const bufferSize = containerRect.height; // One screen height buffer
-    const visibleNodes = allText.filter(({ element }) => {
-      const rect = element.getBoundingClientRect();
-      const elementTop = rect.top - containerRect.top + container.scrollTop;
-      return elementTop >= (visibleTop - bufferSize) && elementTop <= (visibleBottom + bufferSize);
-    });
-
-    // Search for the best match within the visible area first
-    let bestMatch = findBestTextMatch(visibleNodes, cleanPattern, cleanPattern.length * 2);
-    
-    // If no good match found in visible area, search the entire document
-    if (bestMatch.rating < 0.3) {
-      bestMatch = findBestTextMatch(allText, cleanPattern, cleanPattern.length * 2);
-    }
-
     const similarityThreshold = bestMatch.lengthDiff < cleanPattern.length * 0.3 ? 0.3 : 0.5;
 
-    if (bestMatch.rating >= similarityThreshold) {
+    if (bestMatch.rating >= similarityThreshold && bestMatch.elements.length > 0) {
+      // Highlight all matching elements
       bestMatch.elements.forEach((element) => {
         element.style.backgroundColor = 'grey';
         element.style.opacity = '0.4';
       });
 
-      if (bestMatch.elements.length > 0) {
+      // Get the first element to scroll to
-        const element = bestMatch.elements[0];
+      const firstElement = bestMatch.elements[0];
-        const elementRect = element.getBoundingClientRect();
+      
-        const elementTop = elementRect.top - containerRect.top + container.scrollTop;
+      // Scroll the element into view with smooth behavior
-
+      firstElement.scrollIntoView({
-        // Only scroll if the element is outside the visible area
+        behavior: 'smooth',
-        if (elementTop < visibleTop || elementTop > visibleBottom) {
+        block: 'center',
-          container.scrollTo({
+        inline: 'nearest'
-            top: elementTop - containerRect.height / 3, // Position the highlight in the top third
+      });
-            behavior: 'smooth',
-          });
-        }
-      }
     }
   }, [clearHighlights, findBestTextMatch]);