Real PDF rendering with PDF.js and word-level highlights on canvas
- PDF pages rendered by PDF.js directly from File object (no conversion shown) - Each page gets a transparent overlay canvas for drawing highlights - Word positions extracted from PDF.js getTextContent() in PDF coordinate space - convertToViewportPoint() maps PDF space → canvas pixels correctly (handles y-flip) - On each sentence: purple highlight rectangles drawn over matching words on the PDF - Sentence→word mapping via character-position search through word stream - PDF rendering and text extraction run in parallel (pages appear as they render) - Nextcloud PDFs use pdf_b64 → PDF.js rendering path - Removed iframe approach entirely Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
ifedan-ed
parent
4fc76bc650
commit
e1ff5b045e
3 changed files with 192 additions and 98 deletions
258
static/app.js
258
static/app.js
|
|
@ -21,8 +21,12 @@ const audioCache = new Map();
|
|||
let allModels = {};
|
||||
let currentUser = null;
|
||||
|
||||
// PDF blob URL for native browser viewer
|
||||
let pdfBlobUrl = null;
|
||||
// PDF.js state
|
||||
const PDF_SCALE = 1.5;
|
||||
let pdfJsDoc = null;
|
||||
let pdfAllWords = []; // [{page, x, y, w, h, str}]
|
||||
let pdfWordMap = []; // sentence_idx → [word_idx, ...]
|
||||
let pageOverlays = []; // [{overlay, wrap}] per page
|
||||
|
||||
// ── DOM ────────────────────────────────────────────────────────────
|
||||
const $ = id => document.getElementById(id);
|
||||
|
|
@ -203,52 +207,139 @@ function setView(mode) {
|
|||
}
|
||||
window.setView = setView;
|
||||
|
||||
// ── PDF NATIVE RENDERING ───────────────────────────────────────────
|
||||
function renderPDF(b64) {
|
||||
// Revoke previous blob to free memory
|
||||
if (pdfBlobUrl) { URL.revokeObjectURL(pdfBlobUrl); pdfBlobUrl = null; }
|
||||
// ── PDF.js RENDERING + HIGHLIGHTING ───────────────────────────────
|
||||
async function renderPDF(source) {
|
||||
// source: File object OR base64 string (from Nextcloud)
|
||||
pdfjsLib.GlobalWorkerOptions.workerSrc =
|
||||
'https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.worker.min.js';
|
||||
|
||||
const binary = atob(b64);
|
||||
const bytes = new Uint8Array(binary.length);
|
||||
for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i);
|
||||
const blob = new Blob([bytes], { type: 'application/pdf' });
|
||||
pdfBlobUrl = URL.createObjectURL(blob);
|
||||
|
||||
const pdfPages = $('pdf-pages');
|
||||
pdfPages.innerHTML = '';
|
||||
|
||||
const iframe = document.createElement('iframe');
|
||||
iframe.id = 'pdf-iframe';
|
||||
iframe.src = pdfBlobUrl;
|
||||
iframe.style.cssText = 'width:100%;height:100%;border:none;display:block;';
|
||||
pdfPages.appendChild(iframe);
|
||||
}
|
||||
|
||||
function buildSentencePageMap(pageTexts, sentences) {
|
||||
// For each real sentence, find which page it appears on
|
||||
const map = new Array(sentences.length).fill(0);
|
||||
let pageIdx = 0;
|
||||
const lowerPages = pageTexts.map(p => p.toLowerCase().replace(/\s+/g, ' '));
|
||||
|
||||
for (let i = 0; i < sentences.length; i++) {
|
||||
if (sentences[i].isPara) { map[i] = pageIdx; continue; }
|
||||
const snippet = sentences[i].text.slice(0, 40).toLowerCase().replace(/\s+/g, ' ');
|
||||
// Search from current page forward
|
||||
for (let p = pageIdx; p < lowerPages.length; p++) {
|
||||
if (lowerPages[p].includes(snippet)) { pageIdx = p; break; }
|
||||
}
|
||||
map[i] = pageIdx;
|
||||
let data;
|
||||
if (typeof source === 'string') {
|
||||
const bin = atob(source);
|
||||
data = new Uint8Array(bin.length);
|
||||
for (let i = 0; i < bin.length; i++) data[i] = bin.charCodeAt(i);
|
||||
} else {
|
||||
data = new Uint8Array(await source.arrayBuffer());
|
||||
}
|
||||
|
||||
pdfJsDoc = await pdfjsLib.getDocument({ data }).promise;
|
||||
pageOverlays = [];
|
||||
pdfAllWords = [];
|
||||
|
||||
const container = $('pdf-pages');
|
||||
container.innerHTML = '';
|
||||
|
||||
for (let pageNum = 1; pageNum <= pdfJsDoc.numPages; pageNum++) {
|
||||
const page = await pdfJsDoc.getPage(pageNum);
|
||||
const vp = page.getViewport({ scale: PDF_SCALE });
|
||||
|
||||
// Page wrapper (relative so overlay sits on top)
|
||||
const wrap = document.createElement('div');
|
||||
wrap.className = 'pdf-page-wrap';
|
||||
wrap.style.cssText = `position:relative;width:${vp.width}px;max-width:100%;flex-shrink:0;`;
|
||||
|
||||
// Render canvas
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = vp.width;
|
||||
canvas.height = vp.height;
|
||||
canvas.style.cssText = 'display:block;width:100%;';
|
||||
await page.render({ canvasContext: canvas.getContext('2d'), viewport: vp }).promise;
|
||||
|
||||
// Transparent overlay for highlights
|
||||
const overlay = document.createElement('canvas');
|
||||
overlay.width = vp.width;
|
||||
overlay.height = vp.height;
|
||||
overlay.style.cssText = 'position:absolute;top:0;left:0;width:100%;height:100%;pointer-events:none;';
|
||||
|
||||
wrap.appendChild(canvas);
|
||||
wrap.appendChild(overlay);
|
||||
container.appendChild(wrap);
|
||||
pageOverlays.push({ overlay, wrap });
|
||||
|
||||
// Extract word positions (PDF user-space → canvas pixels)
|
||||
const tc = await page.getTextContent();
|
||||
for (const item of tc.items) {
|
||||
const s = item.str.trim();
|
||||
if (!s) continue;
|
||||
// convertToViewportPoint flips y: PDF has y-up, canvas has y-down
|
||||
const [cx, cy] = vp.convertToViewportPoint(item.transform[4], item.transform[5]);
|
||||
const wPx = Math.abs(item.width * PDF_SCALE);
|
||||
const hPx = Math.abs(item.height * PDF_SCALE);
|
||||
pdfAllWords.push({
|
||||
page: pageNum - 1,
|
||||
x: cx, y: cy - hPx, // cy is baseline; move up by height to get top
|
||||
w: wPx, h: hPx,
|
||||
str: s,
|
||||
});
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
function scrollPDFToPage(pageIdx) {
|
||||
if (!pdfBlobUrl) return;
|
||||
const iframe = $('pdf-iframe');
|
||||
if (!iframe) return;
|
||||
// #page= fragment navigates inside Chrome/Edge native PDF viewer
|
||||
const target = pdfBlobUrl + '#page=' + (pageIdx + 1);
|
||||
if (iframe.src !== target) iframe.src = target;
|
||||
// Map each sentence to the word indices it covers in pdfAllWords
|
||||
function buildPDFWordMap(sentences) {
|
||||
const strs = pdfAllWords.map(w => w.str);
|
||||
const allText = strs.join(' ');
|
||||
|
||||
// Pre-compute character start of every word
|
||||
const charStarts = [];
|
||||
let pos = 0;
|
||||
for (const s of strs) { charStarts.push(pos); pos += s.length + 1; }
|
||||
|
||||
pdfWordMap = [];
|
||||
let searchFrom = 0;
|
||||
|
||||
for (const sent of sentences) {
|
||||
if (sent.isPara) { pdfWordMap.push([]); continue; }
|
||||
|
||||
const needle = sent.text.replace(/\s+/g, ' ').trim();
|
||||
let found = allText.indexOf(needle, searchFrom);
|
||||
// Fallback: try first 50 chars (handles slight extraction differences)
|
||||
if (found === -1) found = allText.indexOf(needle.slice(0, 50), searchFrom);
|
||||
if (found === -1) { pdfWordMap.push([]); continue; }
|
||||
|
||||
const end = found + needle.length;
|
||||
const idxs = [];
|
||||
for (let i = 0; i < charStarts.length; i++) {
|
||||
if (charStarts[i] + strs[i].length > found && charStarts[i] < end) idxs.push(i);
|
||||
}
|
||||
pdfWordMap.push(idxs);
|
||||
searchFrom = Math.max(0, found - 5);
|
||||
}
|
||||
}
|
||||
|
||||
function clearPDFHighlights() {
|
||||
for (const { overlay } of pageOverlays) {
|
||||
overlay.getContext('2d').clearRect(0, 0, overlay.width, overlay.height);
|
||||
}
|
||||
}
|
||||
|
||||
function highlightPDFSentence(sentIdx) {
|
||||
clearPDFHighlights();
|
||||
const idxs = pdfWordMap[sentIdx] || [];
|
||||
if (!idxs.length) return;
|
||||
|
||||
// Draw highlight rects grouped by page
|
||||
const byPage = {};
|
||||
for (const wi of idxs) {
|
||||
const w = pdfAllWords[wi];
|
||||
(byPage[w.page] ??= []).push(w);
|
||||
}
|
||||
for (const [pg, words] of Object.entries(byPage)) {
|
||||
const entry = pageOverlays[Number(pg)];
|
||||
if (!entry) continue;
|
||||
const ctx = entry.overlay.getContext('2d');
|
||||
ctx.fillStyle = 'rgba(124,58,237,0.38)';
|
||||
for (const w of words) {
|
||||
ctx.fillRect(w.x - 1, w.y - 1, w.w + 2, w.h + 2);
|
||||
}
|
||||
}
|
||||
|
||||
// Scroll to page of first word
|
||||
const firstWord = pdfAllWords[idxs[0]];
|
||||
if (firstWord !== undefined) {
|
||||
const entry = pageOverlays[firstWord.page];
|
||||
if (entry) entry.wrap.scrollIntoView({ behavior: 'smooth', block: 'nearest' });
|
||||
}
|
||||
}
|
||||
|
||||
// ── LOAD DOCUMENT ──────────────────────────────────────────────────
|
||||
|
|
@ -274,18 +365,16 @@ async function loadDoc(text, title = 'Document', docId = null, startIdx = 0, pdf
|
|||
renderText(state.sentences);
|
||||
textViewer.style.fontSize = `${state.fontSize}px`;
|
||||
|
||||
// Set up PDF view
|
||||
// Set up PDF view (for saved Nextcloud docs that return pdf_b64)
|
||||
const viewToggle = $('view-toggle');
|
||||
if (pdfB64) {
|
||||
viewToggle.style.display = '';
|
||||
state.sentencePage = buildSentencePageMap(pageTexts, state.sentences);
|
||||
renderPDF(pdfB64);
|
||||
setView('pdf'); // default to real PDF view
|
||||
setView('pdf');
|
||||
renderPDF(pdfB64).then(() => buildPDFWordMap(state.sentences)).catch(console.error);
|
||||
} else {
|
||||
viewToggle.style.display = 'none';
|
||||
setView('text');
|
||||
state.sentencePage = [];
|
||||
if (pdfBlobUrl) { URL.revokeObjectURL(pdfBlobUrl); pdfBlobUrl = null; }
|
||||
pdfJsDoc = null; pdfAllWords = []; pdfWordMap = []; pageOverlays = [];
|
||||
$('pdf-pages').innerHTML = '';
|
||||
}
|
||||
|
||||
|
|
@ -380,9 +469,9 @@ function highlight(idx) {
|
|||
|
||||
scheduleSaveProgress();
|
||||
|
||||
// Sync PDF view to current page
|
||||
if (state.currentView === 'pdf' && state.sentencePage.length) {
|
||||
scrollPDFToPage(state.sentencePage[idx] ?? 0);
|
||||
// Highlight directly on PDF canvas
|
||||
if (state.currentView === 'pdf' && pdfWordMap.length) {
|
||||
highlightPDFSentence(idx);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -589,59 +678,57 @@ async function handlePDF(file) {
|
|||
const status = $('pdf-status');
|
||||
const title = file.name.replace(/\.pdf$/i, '');
|
||||
|
||||
// ── Step 1: show PDF instantly from local file, no server needed ──
|
||||
if (pdfBlobUrl) URL.revokeObjectURL(pdfBlobUrl);
|
||||
pdfBlobUrl = URL.createObjectURL(file);
|
||||
|
||||
// Show header + PDF view right away
|
||||
// Show header immediately in PDF mode
|
||||
$('document-header').style.display = '';
|
||||
$('document-title').textContent = title;
|
||||
$('document-meta').textContent = 'Extracting text for playback…';
|
||||
$('document-meta').textContent = 'Loading…';
|
||||
$('view-toggle').style.display = '';
|
||||
$('pdf-pages').innerHTML = '';
|
||||
const iframe = document.createElement('iframe');
|
||||
iframe.id = 'pdf-iframe';
|
||||
iframe.src = pdfBlobUrl;
|
||||
iframe.style.cssText = 'width:100%;height:100%;border:none;display:block;';
|
||||
$('pdf-pages').appendChild(iframe);
|
||||
setView('pdf'); // ← show the real PDF immediately
|
||||
setView('pdf');
|
||||
stopPlayback();
|
||||
audioCache.clear();
|
||||
state.sentences = []; state.docId = null; state.docTitle = title;
|
||||
pdfJsDoc = null; pdfAllWords = []; pdfWordMap = []; pageOverlays = [];
|
||||
nowPlaying.textContent = 'Loading PDF…';
|
||||
status.textContent = 'Loading…';
|
||||
|
||||
nowPlaying.textContent = 'Extracting text for playback…';
|
||||
status.textContent = 'Extracting…';
|
||||
// Run PDF rendering (shows pages as they complete) and text extraction in parallel
|
||||
const form = new FormData();
|
||||
form.append('file', file);
|
||||
|
||||
const renderPromise = renderPDF(file); // reads File directly, no server
|
||||
const extractPromise = fetch('/api/extract-pdf', { method: 'POST', body: form })
|
||||
.then(r => r.ok ? r.json() : r.json().then(e => Promise.reject(e.detail)));
|
||||
|
||||
// ── Step 2: extract text in background (needed only for TTS) ──
|
||||
try {
|
||||
const form = new FormData();
|
||||
form.append('file', file);
|
||||
const resp = await fetch('/api/extract-pdf', { method: 'POST', body: form });
|
||||
if (!resp.ok) throw new Error((await resp.json()).detail || 'Failed');
|
||||
const data = await resp.json();
|
||||
// Text extraction usually finishes before all pages render
|
||||
const data = await extractPromise;
|
||||
|
||||
// Update state with extracted sentences (keep PDF view active)
|
||||
state.sentences = splitSentences(data.text);
|
||||
state.docTitle = data.title || title;
|
||||
state.currentIdx = 0;
|
||||
state.docId = null;
|
||||
state.pageTexts = data.page_texts || [];
|
||||
state.sentencePage = buildSentencePageMap(state.pageTexts, state.sentences);
|
||||
audioCache.clear();
|
||||
|
||||
const real = state.sentences.filter(s => !s.isPara).length;
|
||||
$('document-meta').textContent = `${real} sentences · ready`;
|
||||
$('document-title').textContent = data.title || title;
|
||||
$('document-meta').textContent = `${real} sentences`;
|
||||
$('document-title').textContent = state.docTitle;
|
||||
totSent.textContent = real;
|
||||
curSent.textContent = 0;
|
||||
progressFill.style.width = '0%';
|
||||
nowPlaying.textContent = 'Ready — press play';
|
||||
|
||||
// Also render text view in background for when user switches
|
||||
// Render text view in background for switching
|
||||
renderText(state.sentences);
|
||||
textViewer.style.fontSize = `${state.fontSize}px`;
|
||||
|
||||
// Wait for PDF rendering to finish, then build word→sentence map
|
||||
await renderPromise;
|
||||
buildPDFWordMap(state.sentences);
|
||||
|
||||
nowPlaying.textContent = 'Ready — press play';
|
||||
status.textContent = '✓ Ready';
|
||||
} catch (e) {
|
||||
status.textContent = '✗ ' + e.message;
|
||||
nowPlaying.textContent = 'Text extraction failed';
|
||||
await renderPromise.catch(() => {});
|
||||
status.textContent = '✗ ' + (e.message || e);
|
||||
nowPlaying.textContent = 'Error — check PDF tab is visible';
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -724,7 +811,8 @@ async function ncOpenFile(path, name) {
|
|||
});
|
||||
if (!resp.ok) throw new Error((await resp.json()).detail || 'Failed');
|
||||
const data = await resp.json();
|
||||
await loadDoc(data.text, data.title || name.replace(/\.pdf$/i, ''), null, 0, data.pdf_b64, data.page_texts || []);
|
||||
// loadDoc handles pdf_b64 via PDF.js
|
||||
await loadDoc(data.text, data.title || name.replace(/\.pdf$/i, ''), null, 0, data.pdf_b64 || null, data.page_texts || []);
|
||||
status.textContent = '✓ Loaded';
|
||||
} catch (e) {
|
||||
status.textContent = '✗ ' + e.message;
|
||||
|
|
|
|||
|
|
@ -219,6 +219,7 @@
|
|||
<p id="loading-text">Loading…</p>
|
||||
</div>
|
||||
|
||||
<script src="https://cdnjs.cloudflare.com/ajax/libs/pdf.js/3.11.174/pdf.min.js"></script>
|
||||
<script src="/static/app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -204,8 +204,8 @@ textarea:focus, .text-field:focus { border-color: var(--accent); }
|
|||
min-height: 0;
|
||||
}
|
||||
.content-area.pdf-mode {
|
||||
padding: 20px 20px 0;
|
||||
overflow: hidden; /* iframe fills remaining height */
|
||||
padding: 16px 16px 0;
|
||||
overflow: hidden; /* pdf-viewer child handles its own scroll */
|
||||
}
|
||||
.document-header { margin-bottom: 28px; }
|
||||
.doc-title-row { display: flex; align-items: flex-start; gap: 12px; margin-bottom: 6px; }
|
||||
|
|
@ -286,25 +286,30 @@ textarea:focus, .text-field:focus { border-color: var(--accent); }
|
|||
/* ── PDF VIEWER ──────────────────────────── */
|
||||
.pdf-viewer {
|
||||
flex: 1;
|
||||
min-height: 0; /* essential: lets flex child shrink below content size */
|
||||
min-height: 0;
|
||||
overflow-y: auto;
|
||||
background: #111;
|
||||
padding: 24px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
}
|
||||
#pdf-pages {
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
overflow: hidden;
|
||||
}
|
||||
#pdf-iframe {
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
align-items: center;
|
||||
gap: 16px;
|
||||
width: 100%;
|
||||
border: none;
|
||||
display: block;
|
||||
}
|
||||
.pdf-page-wrap {
|
||||
box-shadow: 0 4px 20px rgba(0,0,0,.6);
|
||||
background: #fff;
|
||||
border-radius: 2px;
|
||||
position: relative;
|
||||
max-width: 900px;
|
||||
}
|
||||
.pdf-page-wrap canvas { display: block; width: 100%; }
|
||||
|
||||
/* ── SETTINGS SIDEBAR ────────────────────── */
|
||||
.setting-group { padding: 16px; border-bottom: 1px solid var(--border); }
|
||||
|
|
|
|||
Loading…
Reference in a new issue