Real PDF rendering with PDF.js and word-level highlights on canvas

- PDF pages rendered by PDF.js directly from File object (no conversion shown)
- Each page gets a transparent overlay canvas for drawing highlights
- Word positions extracted from PDF.js getTextContent() in PDF coordinate space
- convertToViewportPoint() maps PDF space → canvas pixels correctly (handles y-flip)
- On each sentence: purple highlight rectangles drawn over matching words on the PDF
- Sentence→word mapping via character-position search through word stream
- PDF rendering and text extraction run in parallel (pages appear as they render)
- Nextcloud PDFs use pdf_b64 → PDF.js rendering path
- Removed iframe approach entirely

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

static/app.js

@@ -21,8 +21,12 @@ const audioCache = new Map();
 let allModels    = {};
 let currentUser  = null;
 
-// PDF blob URL for native browser viewer
+// PDF.js state
-let pdfBlobUrl = null;
+const PDF_SCALE  = 1.5;
+let pdfJsDoc     = null;
+let pdfAllWords  = [];   // [{page, x, y, w, h, str}]
+let pdfWordMap   = [];   // sentence_idx → [word_idx, ...]
+let pageOverlays = [];   // [{overlay, wrap}] per page
 
 // ── DOM ────────────────────────────────────────────────────────────
 const $  = id => document.getElementById(id);
@@ -203,52 +207,139 @@ function setView(mode) {
 }
 window.setView = setView;
 
-// ── PDF NATIVE RENDERING ───────────────────────────────────────────
+// ── PDF.js RENDERING + HIGHLIGHTING ───────────────────────────────
-function renderPDF(b64) {
+async function renderPDF(source) {
-  // Revoke previous blob to free memory
+  // source: File object OR base64 string (from Nextcloud)
-  if (pdfBlobUrl) { URL.revokeObjectURL(pdfBlobUrl); pdfBlobUrl = null; }
+  pdfjsLib.GlobalWorkerOptions.workerSrc =
+    'https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.worker.min.js';
 
-  const binary = atob(b64);
+  let data;
-  const bytes  = new Uint8Array(binary.length);
+  if (typeof source === 'string') {
-  for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
+    const bin = atob(source);
-  const blob = new Blob([bytes], { type: 'application/pdf' });
+    data = new Uint8Array(bin.length);
-  pdfBlobUrl = URL.createObjectURL(blob);
+    for (let i = 0; i < bin.length; i++) data[i] = bin.charCodeAt(i);
-
+  } else {
-  const pdfPages = $('pdf-pages');
+    data = new Uint8Array(await source.arrayBuffer());
-  pdfPages.innerHTML = '';
+  }
-
+
-  const iframe = document.createElement('iframe');
+  pdfJsDoc    = await pdfjsLib.getDocument({ data }).promise;
-  iframe.id  = 'pdf-iframe';
+  pageOverlays = [];
-  iframe.src = pdfBlobUrl;
+  pdfAllWords  = [];
-  iframe.style.cssText = 'width:100%;height:100%;border:none;display:block;';
+
-  pdfPages.appendChild(iframe);
+  const container = $('pdf-pages');
-}
+  container.innerHTML = '';
-
+
-function buildSentencePageMap(pageTexts, sentences) {
+  for (let pageNum = 1; pageNum <= pdfJsDoc.numPages; pageNum++) {
-  // For each real sentence, find which page it appears on
+    const page = await pdfJsDoc.getPage(pageNum);
-  const map = new Array(sentences.length).fill(0);
+    const vp   = page.getViewport({ scale: PDF_SCALE });
-  let pageIdx = 0;
+
-  const lowerPages = pageTexts.map(p => p.toLowerCase().replace(/\s+/g, ' '));
+    // Page wrapper (relative so overlay sits on top)
-
+    const wrap = document.createElement('div');
-  for (let i = 0; i < sentences.length; i++) {
+    wrap.className = 'pdf-page-wrap';
-    if (sentences[i].isPara) { map[i] = pageIdx; continue; }
+    wrap.style.cssText = `position:relative;width:${vp.width}px;max-width:100%;flex-shrink:0;`;
-    const snippet = sentences[i].text.slice(0, 40).toLowerCase().replace(/\s+/g, ' ');
+
-    // Search from current page forward
+    // Render canvas
-    for (let p = pageIdx; p < lowerPages.length; p++) {
+    const canvas = document.createElement('canvas');
-      if (lowerPages[p].includes(snippet)) { pageIdx = p; break; }
+    canvas.width  = vp.width;
-    }
+    canvas.height = vp.height;
-    map[i] = pageIdx;
+    canvas.style.cssText = 'display:block;width:100%;';
+    await page.render({ canvasContext: canvas.getContext('2d'), viewport: vp }).promise;
+
+    // Transparent overlay for highlights
+    const overlay = document.createElement('canvas');
+    overlay.width  = vp.width;
+    overlay.height = vp.height;
+    overlay.style.cssText = 'position:absolute;top:0;left:0;width:100%;height:100%;pointer-events:none;';
+
+    wrap.appendChild(canvas);
+    wrap.appendChild(overlay);
+    container.appendChild(wrap);
+    pageOverlays.push({ overlay, wrap });
+
+    // Extract word positions (PDF user-space → canvas pixels)
+    const tc = await page.getTextContent();
+    for (const item of tc.items) {
+      const s = item.str.trim();
+      if (!s) continue;
+      // convertToViewportPoint flips y: PDF has y-up, canvas has y-down
+      const [cx, cy] = vp.convertToViewportPoint(item.transform[4], item.transform[5]);
+      const wPx = Math.abs(item.width  * PDF_SCALE);
+      const hPx = Math.abs(item.height * PDF_SCALE);
+      pdfAllWords.push({
+        page: pageNum - 1,
+        x: cx, y: cy - hPx,   // cy is baseline; move up by height to get top
+        w: wPx, h: hPx,
+        str: s,
+      });
+    }
   }
-  return map;
 }
 
-function scrollPDFToPage(pageIdx) {
+// Map each sentence to the word indices it covers in pdfAllWords
-  if (!pdfBlobUrl) return;
+function buildPDFWordMap(sentences) {
-  const iframe = $('pdf-iframe');
+  const strs    = pdfAllWords.map(w => w.str);
-  if (!iframe) return;
+  const allText = strs.join(' ');
-  // #page= fragment navigates inside Chrome/Edge native PDF viewer
+
-  const target = pdfBlobUrl + '#page=' + (pageIdx + 1);
+  // Pre-compute character start of every word
-  if (iframe.src !== target) iframe.src = target;
+  const charStarts = [];
+  let pos = 0;
+  for (const s of strs) { charStarts.push(pos); pos += s.length + 1; }
+
+  pdfWordMap = [];
+  let searchFrom = 0;
+
+  for (const sent of sentences) {
+    if (sent.isPara) { pdfWordMap.push([]); continue; }
+
+    const needle = sent.text.replace(/\s+/g, ' ').trim();
+    let found = allText.indexOf(needle, searchFrom);
+    // Fallback: try first 50 chars (handles slight extraction differences)
+    if (found === -1) found = allText.indexOf(needle.slice(0, 50), searchFrom);
+    if (found === -1) { pdfWordMap.push([]); continue; }
+
+    const end  = found + needle.length;
+    const idxs = [];
+    for (let i = 0; i < charStarts.length; i++) {
+      if (charStarts[i] + strs[i].length > found && charStarts[i] < end) idxs.push(i);
+    }
+    pdfWordMap.push(idxs);
+    searchFrom = Math.max(0, found - 5);
+  }
+}
+
+function clearPDFHighlights() {
+  for (const { overlay } of pageOverlays) {
+    overlay.getContext('2d').clearRect(0, 0, overlay.width, overlay.height);
+  }
+}
+
+function highlightPDFSentence(sentIdx) {
+  clearPDFHighlights();
+  const idxs = pdfWordMap[sentIdx] || [];
+  if (!idxs.length) return;
+
+  // Draw highlight rects grouped by page
+  const byPage = {};
+  for (const wi of idxs) {
+    const w = pdfAllWords[wi];
+    (byPage[w.page] ??= []).push(w);
+  }
+  for (const [pg, words] of Object.entries(byPage)) {
+    const entry = pageOverlays[Number(pg)];
+    if (!entry) continue;
+    const ctx = entry.overlay.getContext('2d');
+    ctx.fillStyle = 'rgba(124,58,237,0.38)';
+    for (const w of words) {
+      ctx.fillRect(w.x - 1, w.y - 1, w.w + 2, w.h + 2);
+    }
+  }
+
+  // Scroll to page of first word
+  const firstWord = pdfAllWords[idxs[0]];
+  if (firstWord !== undefined) {
+    const entry = pageOverlays[firstWord.page];
+    if (entry) entry.wrap.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
+  }
 }
 
 // ── LOAD DOCUMENT ──────────────────────────────────────────────────
@@ -274,18 +365,16 @@ async function loadDoc(text, title = 'Document', docId = null, startIdx = 0, pdf
   renderText(state.sentences);
   textViewer.style.fontSize = `${state.fontSize}px`;
 
-  // Set up PDF view
+  // Set up PDF view (for saved Nextcloud docs that return pdf_b64)
   const viewToggle = $('view-toggle');
   if (pdfB64) {
     viewToggle.style.display = '';
-    state.sentencePage = buildSentencePageMap(pageTexts, state.sentences);
+    setView('pdf');
-    renderPDF(pdfB64);
+    renderPDF(pdfB64).then(() => buildPDFWordMap(state.sentences)).catch(console.error);
-    setView('pdf');   // default to real PDF view
   } else {
     viewToggle.style.display = 'none';
     setView('text');
-    state.sentencePage = [];
+    pdfJsDoc = null; pdfAllWords = []; pdfWordMap = []; pageOverlays = [];
-    if (pdfBlobUrl) { URL.revokeObjectURL(pdfBlobUrl); pdfBlobUrl = null; }
     $('pdf-pages').innerHTML = '';
   }
 
@@ -380,9 +469,9 @@ function highlight(idx) {
 
   scheduleSaveProgress();
 
-  // Sync PDF view to current page
+  // Highlight directly on PDF canvas
-  if (state.currentView === 'pdf' && state.sentencePage.length) {
+  if (state.currentView === 'pdf' && pdfWordMap.length) {
-    scrollPDFToPage(state.sentencePage[idx] ?? 0);
+    highlightPDFSentence(idx);
   }
 }
 
@@ -589,59 +678,57 @@ async function handlePDF(file) {
   const status = $('pdf-status');
   const title  = file.name.replace(/\.pdf$/i, '');
 
-  // ── Step 1: show PDF instantly from local file, no server needed ──
+  // Show header immediately in PDF mode
-  if (pdfBlobUrl) URL.revokeObjectURL(pdfBlobUrl);
-  pdfBlobUrl = URL.createObjectURL(file);
-
-  // Show header + PDF view right away
   $('document-header').style.display = '';
   $('document-title').textContent = title;
-  $('document-meta').textContent = 'Extracting text for playback…';
+  $('document-meta').textContent = 'Loading…';
   $('view-toggle').style.display = '';
   $('pdf-pages').innerHTML = '';
-  const iframe = document.createElement('iframe');
+  setView('pdf');
-  iframe.id = 'pdf-iframe';
+  stopPlayback();
-  iframe.src = pdfBlobUrl;
+  audioCache.clear();
-  iframe.style.cssText = 'width:100%;height:100%;border:none;display:block;';
+  state.sentences = []; state.docId = null; state.docTitle = title;
-  $('pdf-pages').appendChild(iframe);
+  pdfJsDoc = null; pdfAllWords = []; pdfWordMap = []; pageOverlays = [];
-  setView('pdf');   // ← show the real PDF immediately
+  nowPlaying.textContent = 'Loading PDF…';
+  status.textContent = 'Loading…';
 
-  nowPlaying.textContent = 'Extracting text for playback…';
+  // Run PDF rendering (shows pages as they complete) and text extraction in parallel
-  status.textContent = 'Extracting…';
+  const form = new FormData();
+  form.append('file', file);
+
+  const renderPromise  = renderPDF(file);          // reads File directly, no server
+  const extractPromise = fetch('/api/extract-pdf', { method: 'POST', body: form })
+    .then(r => r.ok ? r.json() : r.json().then(e => Promise.reject(e.detail)));
 
-  // ── Step 2: extract text in background (needed only for TTS) ──
   try {
-    const form = new FormData();
+    // Text extraction usually finishes before all pages render
-    form.append('file', file);
+    const data = await extractPromise;
-    const resp = await fetch('/api/extract-pdf', { method: 'POST', body: form });
-    if (!resp.ok) throw new Error((await resp.json()).detail || 'Failed');
-    const data = await resp.json();
 
-    // Update state with extracted sentences (keep PDF view active)
     state.sentences  = splitSentences(data.text);
     state.docTitle   = data.title || title;
     state.currentIdx = 0;
-    state.docId      = null;
-    state.pageTexts  = data.page_texts || [];
-    state.sentencePage = buildSentencePageMap(state.pageTexts, state.sentences);
-    audioCache.clear();
 
     const real = state.sentences.filter(s => !s.isPara).length;
-    $('document-meta').textContent = `${real} sentences · ready`;
+    $('document-meta').textContent = `${real} sentences`;
-    $('document-title').textContent = data.title || title;
+    $('document-title').textContent = state.docTitle;
     totSent.textContent = real;
     curSent.textContent = 0;
     progressFill.style.width = '0%';
-    nowPlaying.textContent = 'Ready — press play';
 
-    // Also render text view in background for when user switches
+    // Render text view in background for switching
     renderText(state.sentences);
     textViewer.style.fontSize = `${state.fontSize}px`;
 
+    // Wait for PDF rendering to finish, then build word→sentence map
+    await renderPromise;
+    buildPDFWordMap(state.sentences);
+
+    nowPlaying.textContent = 'Ready — press play';
     status.textContent = '✓ Ready';
   } catch (e) {
-    status.textContent = '✗ ' + e.message;
+    await renderPromise.catch(() => {});
-    nowPlaying.textContent = 'Text extraction failed';
+    status.textContent = '✗ ' + (e.message || e);
+    nowPlaying.textContent = 'Error — check PDF tab is visible';
   }
 }
 
@@ -724,7 +811,8 @@ async function ncOpenFile(path, name) {
     });
     if (!resp.ok) throw new Error((await resp.json()).detail || 'Failed');
     const data = await resp.json();
-    await loadDoc(data.text, data.title || name.replace(/\.pdf$/i, ''), null, 0, data.pdf_b64, data.page_texts || []);
+    // loadDoc handles pdf_b64 via PDF.js
+    await loadDoc(data.text, data.title || name.replace(/\.pdf$/i, ''), null, 0, data.pdf_b64 || null, data.page_texts || []);
     status.textContent = '✓ Loaded';
   } catch (e) {
     status.textContent = '✗ ' + e.message;

static/index.html

@@ -219,6 +219,7 @@
   <p id="loading-text">Loading…</p>
 </div>
 
+<script src="https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.min.js"></script>
 <script src="/static/app.js"></script>
 </body>
 </html>

static/style.css

@@ -204,8 +204,8 @@ textarea:focus, .text-field:focus { border-color: var(--accent); }
   min-height: 0;
 }
 .content-area.pdf-mode {
-  padding: 20px 20px 0;
+  padding: 16px 16px 0;
-  overflow: hidden;   /* iframe fills remaining height */
+  overflow: hidden;   /* pdf-viewer child handles its own scroll */
 }
 .document-header { margin-bottom: 28px; }
 .doc-title-row { display: flex; align-items: flex-start; gap: 12px; margin-bottom: 6px; }
@@ -286,25 +286,30 @@ textarea:focus, .text-field:focus { border-color: var(--accent); }
 /* ── PDF VIEWER ──────────────────────────── */
 .pdf-viewer {
   flex: 1;
-  min-height: 0;        /* essential: lets flex child shrink below content size */
+  min-height: 0;
+  overflow-y: auto;
+  background: #111;
+  padding: 24px;
   display: flex;
   flex-direction: column;
-  overflow: hidden;
+  align-items: center;
+  gap: 16px;
 }
 #pdf-pages {
-  flex: 1;
-  min-height: 0;
   display: flex;
   flex-direction: column;
-  overflow: hidden;
+  align-items: center;
-}
+  gap: 16px;
-#pdf-iframe {
-  flex: 1;
-  min-height: 0;
   width: 100%;
-  border: none;
-  display: block;
 }
+.pdf-page-wrap {
+  box-shadow: 0 4px 20px rgba(0,0,0,.6);
+  background: #fff;
+  border-radius: 2px;
+  position: relative;
+  max-width: 900px;
+}
+.pdf-page-wrap canvas { display: block; width: 100%; }
 
 /* ── SETTINGS SIDEBAR ────────────────────── */
 .setting-group { padding: 16px; border-bottom: 1px solid var(--border); }