feat(tts): add word-by-word highlighting with whisper.cpp
- Introduce `/api/whisper` endpoint which uses `whisper.cpp` (via a `WHISPER_CPP_BIN` executable) and `ffmpeg` to generate word-level audio alignments from provided audio and text. - Integrate word-level alignments into `TTSContext`, tracking the currently spoken word based on audio seek position and provided timestamps. Alignments are cached in-memory and fetched asynchronously. - Add new configuration options (`pdfWordHighlightEnabled`, `epubWordHighlightEnabled`) to `ConfigContext` and `Dexie` for enabling/disabling the feature. - Implement visual word highlighting in both `PDFViewer` and `EPUBViewer` by mapping TTS-aligned words to rendered text elements. - Enhance `EPUBContext` and `PDFContext` with new `highlightWordIndex` and `clearWordHighlights` functions, utilizing fuzzy string matching (`cmpstr`) to robustly align spoken words with displayed text for accurate highlighting. - Update `DocumentSettings` to include user-facing toggles for the new highlighting modes.
This commit is contained in:
Richard Roberson
parent
ac0b47debb
commit
733b4180eb
12 changed files with 1333 additions and 240 deletions
451
src/app/api/whisper/route.ts
Normal file
451
src/app/api/whisper/route.ts
Normal file
|
|
@ -0,0 +1,451 @@
|
|||
import { NextRequest, NextResponse } from 'next/server';
|
||||
import { createHash, randomUUID } from 'crypto';
|
||||
import { mkdtemp, writeFile, rm, access, mkdir, readFile } from 'fs/promises';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { spawn } from 'child_process';
|
||||
import type { TTSSentenceAlignment } from '@/types/tts';
|
||||
import { preprocessSentenceForAudio } from '@/lib/nlp';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
|
||||
interface WhisperRequestBody {
|
||||
text: string;
|
||||
audio: number[]; // raw bytes from Uint8Array
|
||||
lang?: string;
|
||||
}
|
||||
|
||||
interface WhisperAlignmentOptions {
|
||||
engine?: 'whisper.cpp';
|
||||
lang?: string;
|
||||
}
|
||||
|
||||
interface WhisperWord {
|
||||
start: number;
|
||||
end: number;
|
||||
word: string;
|
||||
}
|
||||
|
||||
// Simple in-memory cache keyed by a hash of text+lang+audio length
|
||||
const alignmentCache = new Map<string, TTSSentenceAlignment[]>();
|
||||
|
||||
// Model management using a fixed tiny.en GGML model stored under docstore/model
|
||||
const MODEL_NAME = 'ggml-tiny.en.bin';
|
||||
const MODEL_URL =
|
||||
'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.en.bin';
|
||||
const DOCSTORE_DIR = join(process.cwd(), 'docstore');
|
||||
const MODEL_DIR = join(DOCSTORE_DIR, 'model');
|
||||
const MODEL_PATH = join(MODEL_DIR, MODEL_NAME);
|
||||
const modelReadyPromises = new Map<string, Promise<void>>();
|
||||
|
||||
async function ensureModelAvailable(): Promise<void> {
|
||||
// Fast path: model already present
|
||||
try {
|
||||
await access(MODEL_PATH);
|
||||
return;
|
||||
} catch {
|
||||
// fall through to download
|
||||
}
|
||||
|
||||
const existing = modelReadyPromises.get(MODEL_PATH);
|
||||
if (existing) return existing;
|
||||
|
||||
const promise = (async () => {
|
||||
try {
|
||||
await access(MODEL_PATH);
|
||||
return;
|
||||
} catch {
|
||||
// still missing
|
||||
}
|
||||
|
||||
await mkdir(MODEL_DIR, { recursive: true });
|
||||
|
||||
const res = await fetch(MODEL_URL);
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`Failed to download Whisper model from ${MODEL_URL}: ${res.status} ${res.statusText}`
|
||||
);
|
||||
}
|
||||
|
||||
const arrayBuffer = await res.arrayBuffer();
|
||||
await writeFile(MODEL_PATH, Buffer.from(arrayBuffer));
|
||||
})();
|
||||
|
||||
modelReadyPromises.set(MODEL_PATH, promise);
|
||||
return promise;
|
||||
}
|
||||
|
||||
async function runWhisperCpp(
|
||||
wavPath: string,
|
||||
opts: WhisperAlignmentOptions
|
||||
): Promise<WhisperWord[]> {
|
||||
const binary = process.env.WHISPER_CPP_BIN;
|
||||
if (!binary) {
|
||||
throw new Error(
|
||||
'Whisper.cpp binary path not configured. Set WHISPER_CPP_BIN to the compiled binary.'
|
||||
);
|
||||
}
|
||||
|
||||
await ensureModelAvailable();
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const jsonBase = `${wavPath}.json_out`;
|
||||
const jsonPath = `${jsonBase}.json`;
|
||||
// Request full JSON output (including token-level details when available)
|
||||
// and ask whisper-cli not to print anything to stdout.
|
||||
const args = [
|
||||
'-m',
|
||||
MODEL_PATH,
|
||||
'-f',
|
||||
wavPath,
|
||||
'-of',
|
||||
jsonBase,
|
||||
'-ojf',
|
||||
'-np',
|
||||
];
|
||||
|
||||
if (opts.lang) {
|
||||
args.push('-l', opts.lang);
|
||||
}
|
||||
|
||||
const child = spawn(binary, args);
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
|
||||
child.stdout.on('data', (data) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
|
||||
child.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
child.on('error', (err) => {
|
||||
reject(err);
|
||||
});
|
||||
|
||||
child.on('close', (code) => {
|
||||
if (code !== 0) {
|
||||
console.error('whisper-cli exited with error', {
|
||||
code,
|
||||
wavPath,
|
||||
stdout: stdout.slice(0, 500),
|
||||
stderr: stderr.slice(0, 500),
|
||||
});
|
||||
return reject(
|
||||
new Error(
|
||||
`whisper.cpp exited with code ${code}: ${stderr || stdout}`
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
readFile(jsonPath, 'utf-8')
|
||||
.then((content: string) => {
|
||||
const words: WhisperWord[] = [];
|
||||
const parsed = JSON.parse(content) as {
|
||||
transcription?: Array<{
|
||||
text?: string;
|
||||
timestamps?: { from?: string; to?: string };
|
||||
offsets?: { from?: number; to?: number };
|
||||
tokens?: Array<{
|
||||
text?: string;
|
||||
timestamps?: { from?: string; to?: string };
|
||||
offsets?: { from?: number; to?: number };
|
||||
}>;
|
||||
}>;
|
||||
};
|
||||
|
||||
const transcription = parsed.transcription;
|
||||
|
||||
const parseTimecode = (value?: string): number | null => {
|
||||
if (!value) return null;
|
||||
const m = value.match(/(\d+):(\d+):(\d+),(\d+)/);
|
||||
if (!m) return null;
|
||||
const h = Number(m[1]);
|
||||
const min = Number(m[2]);
|
||||
const s = Number(m[3]);
|
||||
const ms = Number(m[4]);
|
||||
if (
|
||||
Number.isNaN(h) ||
|
||||
Number.isNaN(min) ||
|
||||
Number.isNaN(s) ||
|
||||
Number.isNaN(ms)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return h * 3600 + min * 60 + s + ms / 1000;
|
||||
};
|
||||
|
||||
if (Array.isArray(transcription)) {
|
||||
for (const seg of transcription) {
|
||||
const segText = (seg.text || '').trim();
|
||||
const segStartSecFromTs = parseTimecode(
|
||||
seg.timestamps?.from
|
||||
);
|
||||
const segEndSecFromTs = parseTimecode(seg.timestamps?.to);
|
||||
const segStartSecFromMs =
|
||||
typeof seg.offsets?.from === 'number'
|
||||
? seg.offsets.from / 1000
|
||||
: null;
|
||||
const segEndSecFromMs =
|
||||
typeof seg.offsets?.to === 'number'
|
||||
? seg.offsets.to / 1000
|
||||
: null;
|
||||
|
||||
const segStartSec =
|
||||
segStartSecFromTs ??
|
||||
segStartSecFromMs ??
|
||||
0;
|
||||
const segEndSec =
|
||||
segEndSecFromTs ??
|
||||
segEndSecFromMs ??
|
||||
segStartSec;
|
||||
|
||||
const tokens = Array.isArray(seg.tokens)
|
||||
? seg.tokens
|
||||
: [];
|
||||
|
||||
if (tokens.length > 0) {
|
||||
for (const token of tokens) {
|
||||
const rawText = token.text || '';
|
||||
const tokenText = rawText.trim();
|
||||
// Skip special markers like [_BEG_]
|
||||
if (!tokenText || /^\[.*\]$/.test(tokenText)) continue;
|
||||
|
||||
const tokStartSecFromTs = parseTimecode(
|
||||
token.timestamps?.from
|
||||
);
|
||||
const tokEndSecFromTs = parseTimecode(
|
||||
token.timestamps?.to
|
||||
);
|
||||
const tokStartSecFromMs =
|
||||
typeof token.offsets?.from === 'number'
|
||||
? token.offsets.from / 1000
|
||||
: null;
|
||||
const tokEndSecFromMs =
|
||||
typeof token.offsets?.to === 'number'
|
||||
? token.offsets.to / 1000
|
||||
: null;
|
||||
|
||||
const startSec =
|
||||
tokStartSecFromTs ??
|
||||
tokStartSecFromMs ??
|
||||
segStartSec;
|
||||
const endSec =
|
||||
tokEndSecFromTs ??
|
||||
tokEndSecFromMs ??
|
||||
segEndSec;
|
||||
|
||||
words.push({
|
||||
word: tokenText,
|
||||
start: startSec,
|
||||
end: endSec,
|
||||
});
|
||||
}
|
||||
} else if (segText) {
|
||||
// Fallback: no token list, approximate per-word timing within the segment
|
||||
const segTokens = segText.split(/\s+/).filter(Boolean);
|
||||
if (segTokens.length) {
|
||||
const totalDur = Math.max(segEndSec - segStartSec, 0);
|
||||
const step =
|
||||
segTokens.length > 0
|
||||
? totalDur / segTokens.length
|
||||
: 0;
|
||||
segTokens.forEach((token, index) => {
|
||||
const wStart =
|
||||
step > 0
|
||||
? segStartSec + step * index
|
||||
: segStartSec;
|
||||
const wEnd =
|
||||
step > 0
|
||||
? index === segTokens.length - 1
|
||||
? segEndSec
|
||||
: segStartSec + step * (index + 1)
|
||||
: segEndSec;
|
||||
words.push({
|
||||
word: token,
|
||||
start: wStart,
|
||||
end: wEnd,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resolve(words);
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function mapWordsToSentenceOffsets(
|
||||
sentence: string,
|
||||
words: WhisperWord[]
|
||||
): TTSSentenceAlignment {
|
||||
const normalizedSentence = preprocessSentenceForAudio(sentence);
|
||||
let cursor = 0;
|
||||
|
||||
const alignedWords = words.map((w) => {
|
||||
const token = w.word.trim();
|
||||
if (!token) {
|
||||
return {
|
||||
text: '',
|
||||
startSec: w.start,
|
||||
endSec: w.end,
|
||||
charStart: cursor,
|
||||
charEnd: cursor,
|
||||
};
|
||||
}
|
||||
|
||||
const idx = normalizedSentence
|
||||
.toLowerCase()
|
||||
.indexOf(token.toLowerCase(), cursor);
|
||||
|
||||
const start =
|
||||
idx !== -1
|
||||
? idx
|
||||
: cursor;
|
||||
const end = start + token.length;
|
||||
|
||||
cursor = end;
|
||||
|
||||
return {
|
||||
text: token,
|
||||
startSec: w.start,
|
||||
endSec: w.end,
|
||||
charStart: start,
|
||||
charEnd: end,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
sentence,
|
||||
sentenceIndex: 0,
|
||||
words: alignedWords.filter((w) => w.text.length > 0),
|
||||
};
|
||||
}
|
||||
|
||||
async function alignAudioWithText(
|
||||
audioBuffer: ArrayBuffer,
|
||||
text: string,
|
||||
cacheKey?: string,
|
||||
opts: WhisperAlignmentOptions = {}
|
||||
): Promise<TTSSentenceAlignment[]> {
|
||||
if (!text.trim()) {
|
||||
return [];
|
||||
}
|
||||
|
||||
if (cacheKey && alignmentCache.has(cacheKey)) {
|
||||
return alignmentCache.get(cacheKey)!;
|
||||
}
|
||||
|
||||
const tmpBase = await mkdtemp(join(tmpdir(), 'openreader-whisper-'));
|
||||
const inputPath = join(tmpBase, `${randomUUID()}-input.bin`);
|
||||
const wavPath = join(tmpBase, `${randomUUID()}-input.wav`);
|
||||
|
||||
try {
|
||||
await writeFile(inputPath, Buffer.from(new Uint8Array(audioBuffer)));
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const ffmpeg = spawn('ffmpeg', [
|
||||
'-y',
|
||||
'-i',
|
||||
inputPath,
|
||||
'-ar',
|
||||
'16000',
|
||||
'-ac',
|
||||
'1',
|
||||
wavPath,
|
||||
]);
|
||||
|
||||
let stderr = '';
|
||||
ffmpeg.stderr.on('data', (data) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
|
||||
ffmpeg.on('error', (err) => {
|
||||
reject(err);
|
||||
});
|
||||
|
||||
ffmpeg.on('close', (code) => {
|
||||
if (code === 0) {
|
||||
resolve();
|
||||
} else {
|
||||
reject(
|
||||
new Error(`ffmpeg failed with code ${code}: ${stderr}`)
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const words = await runWhisperCpp(wavPath, opts);
|
||||
const alignment = mapWordsToSentenceOffsets(text, words);
|
||||
const result: TTSSentenceAlignment[] = [alignment];
|
||||
|
||||
if (cacheKey) {
|
||||
alignmentCache.set(cacheKey, result);
|
||||
}
|
||||
|
||||
return result;
|
||||
} finally {
|
||||
try {
|
||||
await rm(tmpBase, { recursive: true, force: true });
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function makeCacheKey(input: WhisperRequestBody) {
|
||||
const hash = createHash('sha256')
|
||||
.update(
|
||||
JSON.stringify({
|
||||
text: input.text,
|
||||
lang: input.lang || '',
|
||||
audioLen: input.audio?.length || 0,
|
||||
})
|
||||
)
|
||||
.digest('hex');
|
||||
return hash;
|
||||
}
|
||||
|
||||
export async function POST(req: NextRequest) {
|
||||
try {
|
||||
const body = (await req.json()) as WhisperRequestBody;
|
||||
const { text, audio, lang } = body;
|
||||
|
||||
if (!text || !audio || !Array.isArray(audio)) {
|
||||
return NextResponse.json(
|
||||
{ error: 'Missing text or audio in request body' },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
const cacheKey = makeCacheKey(body);
|
||||
const audioBuffer = new Uint8Array(audio).buffer;
|
||||
|
||||
const alignments: TTSSentenceAlignment[] = await alignAudioWithText(
|
||||
audioBuffer,
|
||||
text,
|
||||
cacheKey,
|
||||
{ engine: 'whisper.cpp', lang }
|
||||
);
|
||||
|
||||
return NextResponse.json({ alignments }, { status: 200 });
|
||||
} catch (error) {
|
||||
console.error('Error in whisper route:', error);
|
||||
return NextResponse.json(
|
||||
{
|
||||
error: 'WHISPER_ALIGNMENT_FAILED',
|
||||
message: 'Failed to compute word-level alignment',
|
||||
},
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -35,6 +35,8 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
|
|||
updateConfigKey,
|
||||
pdfHighlightEnabled,
|
||||
epubHighlightEnabled,
|
||||
pdfWordHighlightEnabled,
|
||||
epubWordHighlightEnabled,
|
||||
} = useConfig();
|
||||
const { createFullAudioBook: createEPUBAudioBook, regenerateChapter: regenerateEPUBChapter } = useEPUB();
|
||||
const { createFullAudioBook: createPDFAudioBook, regenerateChapter: regeneratePDFChapter } = usePDF();
|
||||
|
|
@ -324,40 +326,82 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
|
|||
</span>
|
||||
</label>
|
||||
<p className="text-sm text-muted pl-6">
|
||||
Merge sentences across page or section breaks for smoother TTS.
|
||||
Merge sentences across page or section breaks
|
||||
</p>
|
||||
</div>
|
||||
)}
|
||||
{!epub && !html && (
|
||||
<div className="space-y-1">
|
||||
<label className="flex items-center space-x-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={pdfHighlightEnabled}
|
||||
onChange={(e) => updateConfigKey('pdfHighlightEnabled', e.target.checked)}
|
||||
className="form-checkbox h-4 w-4 text-accent rounded border-muted"
|
||||
/>
|
||||
<span className="text-sm font-medium text-foreground">Highlight text during playback</span>
|
||||
</label>
|
||||
<p className="text-sm text-muted pl-6">
|
||||
Show visual highlighting in the PDF viewer while TTS is reading.
|
||||
</p>
|
||||
<div className="space-y-2">
|
||||
<div className="space-y-1">
|
||||
<label className="flex items-center space-x-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={pdfHighlightEnabled}
|
||||
onChange={(e) => updateConfigKey('pdfHighlightEnabled', e.target.checked)}
|
||||
className="form-checkbox h-4 w-4 text-accent rounded border-muted"
|
||||
/>
|
||||
<span className="text-sm font-medium text-foreground">Highlight text during playback</span>
|
||||
</label>
|
||||
<p className="text-sm text-muted pl-6">
|
||||
Visual text playback highlighting in the PDF viewer
|
||||
</p>
|
||||
</div>
|
||||
<div className="space-y-1 pl-6">
|
||||
<label className="flex items-center space-x-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={pdfWordHighlightEnabled && pdfHighlightEnabled}
|
||||
disabled={!pdfHighlightEnabled}
|
||||
onChange={(e) =>
|
||||
updateConfigKey('pdfWordHighlightEnabled', e.target.checked)
|
||||
}
|
||||
className="form-checkbox h-4 w-4 text-accent rounded border-muted disabled:opacity-50"
|
||||
/>
|
||||
<span className="text-sm font-medium text-foreground">
|
||||
Word-by-word
|
||||
</span>
|
||||
</label>
|
||||
<p className="text-sm text-muted pl-6">
|
||||
Highlight individual words using audio timestamps generated by whisper.cpp
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{epub && (
|
||||
<div className="space-y-1">
|
||||
<label className="flex items-center space-x-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={epubHighlightEnabled}
|
||||
onChange={(e) => updateConfigKey('epubHighlightEnabled', e.target.checked)}
|
||||
className="form-checkbox h-4 w-4 text-accent rounded border-muted"
|
||||
/>
|
||||
<span className="text-sm font-medium text-foreground">Highlight text during playback</span>
|
||||
</label>
|
||||
<p className="text-sm text-muted pl-6">
|
||||
Show visual highlighting in the EPUB viewer while TTS is reading.
|
||||
</p>
|
||||
<div className="space-y-2">
|
||||
<div className="space-y-1">
|
||||
<label className="flex items-center space-x-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={epubHighlightEnabled}
|
||||
onChange={(e) => updateConfigKey('epubHighlightEnabled', e.target.checked)}
|
||||
className="form-checkbox h-4 w-4 text-accent rounded border-muted"
|
||||
/>
|
||||
<span className="text-sm font-medium text-foreground">Highlight text during playback</span>
|
||||
</label>
|
||||
<p className="text-sm text-muted pl-6">
|
||||
Visual text playback highlighting in the EPUB viewer
|
||||
</p>
|
||||
</div>
|
||||
<div className="space-y-1 pl-6">
|
||||
<label className="flex items-center space-x-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={epubWordHighlightEnabled && epubHighlightEnabled}
|
||||
disabled={!epubHighlightEnabled}
|
||||
onChange={(e) =>
|
||||
updateConfigKey('epubWordHighlightEnabled', e.target.checked)
|
||||
}
|
||||
className="form-checkbox h-4 w-4 text-accent rounded border-muted disabled:opacity-50"
|
||||
/>
|
||||
<span className="text-sm font-medium text-foreground">
|
||||
Word-by-word
|
||||
</span>
|
||||
</label>
|
||||
<p className="text-sm text-muted pl-6">
|
||||
Highlight individual words using audio timestamps generated by whisper.cpp
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{epub && (
|
||||
|
|
|
|||
|
|
@ -30,10 +30,18 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) {
|
|||
setRendition,
|
||||
extractPageText,
|
||||
highlightPattern,
|
||||
clearHighlights
|
||||
clearHighlights,
|
||||
highlightWordIndex,
|
||||
clearWordHighlights
|
||||
} = useEPUB();
|
||||
const { registerLocationChangeHandler, pause, currentSentence } = useTTS();
|
||||
const { epubTheme } = useConfig();
|
||||
const {
|
||||
registerLocationChangeHandler,
|
||||
pause,
|
||||
currentSentence,
|
||||
currentSentenceAlignment,
|
||||
currentWordIndex
|
||||
} = useTTS();
|
||||
const { epubTheme, epubHighlightEnabled, epubWordHighlightEnabled } = useConfig();
|
||||
const { updateTheme } = useEPUBTheme(epubTheme, renditionRef.current);
|
||||
const containerRef = useRef<HTMLDivElement>(null);
|
||||
const { isResizing, setIsResizing, dimensions } = useEPUBResize(containerRef);
|
||||
|
|
@ -70,6 +78,38 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) {
|
|||
}
|
||||
}, [currentSentence, highlightPattern, clearHighlights]);
|
||||
|
||||
// Word-level highlight layered on top of the block highlight
|
||||
useEffect(() => {
|
||||
if (!epubHighlightEnabled || !epubWordHighlightEnabled) {
|
||||
clearWordHighlights();
|
||||
return;
|
||||
}
|
||||
|
||||
if (currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) {
|
||||
clearWordHighlights();
|
||||
return;
|
||||
}
|
||||
|
||||
if (!currentSentenceAlignment) {
|
||||
clearWordHighlights();
|
||||
return;
|
||||
}
|
||||
|
||||
highlightWordIndex(
|
||||
currentSentenceAlignment,
|
||||
currentWordIndex,
|
||||
currentSentence || ''
|
||||
);
|
||||
}, [
|
||||
currentWordIndex,
|
||||
currentSentence,
|
||||
currentSentenceAlignment,
|
||||
epubHighlightEnabled,
|
||||
epubWordHighlightEnabled,
|
||||
clearWordHighlights,
|
||||
highlightWordIndex
|
||||
]);
|
||||
|
||||
if (!currDocData) {
|
||||
return <DocumentSkeleton />;
|
||||
}
|
||||
|
|
@ -95,4 +135,4 @@ export function EPUBViewer({ className = '' }: EPUBViewerProps) {
|
|||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,11 +26,13 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
|
|||
const { containerWidth } = usePDFResize(containerRef);
|
||||
|
||||
// Config context
|
||||
const { viewType, pdfHighlightEnabled } = useConfig();
|
||||
const { viewType, pdfHighlightEnabled, pdfWordHighlightEnabled } = useConfig();
|
||||
|
||||
// TTS context
|
||||
const {
|
||||
currentSentence,
|
||||
currentWordIndex,
|
||||
currentSentenceAlignment,
|
||||
skipToLocation,
|
||||
} = useTTS();
|
||||
|
||||
|
|
@ -38,6 +40,8 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
|
|||
const {
|
||||
highlightPattern,
|
||||
clearHighlights,
|
||||
clearWordHighlights,
|
||||
highlightWordIndex,
|
||||
onDocumentLoadSuccess,
|
||||
currDocData,
|
||||
currDocPages,
|
||||
|
|
@ -74,6 +78,46 @@ export function PDFViewer({ zoomLevel }: PDFViewerProps) {
|
|||
};
|
||||
}, [currDocText, currentSentence, highlightPattern, clearHighlights, pdfHighlightEnabled]);
|
||||
|
||||
// Word-level highlight layered on top of the block highlight
|
||||
useEffect(() => {
|
||||
if (!pdfHighlightEnabled || !pdfWordHighlightEnabled) {
|
||||
clearWordHighlights();
|
||||
return;
|
||||
}
|
||||
|
||||
if (currentWordIndex === null || currentWordIndex === undefined || currentWordIndex < 0) {
|
||||
clearWordHighlights();
|
||||
return;
|
||||
}
|
||||
|
||||
const wordEntry =
|
||||
currentSentenceAlignment &&
|
||||
currentWordIndex < currentSentenceAlignment.words.length
|
||||
? currentSentenceAlignment.words[currentWordIndex]
|
||||
: undefined;
|
||||
const wordText = wordEntry?.text || null;
|
||||
|
||||
if (!wordText) {
|
||||
clearWordHighlights();
|
||||
return;
|
||||
}
|
||||
|
||||
highlightWordIndex(
|
||||
currentSentenceAlignment,
|
||||
currentWordIndex,
|
||||
currentSentence || '',
|
||||
containerRef as RefObject<HTMLDivElement>
|
||||
);
|
||||
}, [
|
||||
currentWordIndex,
|
||||
currentSentence,
|
||||
currentSentenceAlignment,
|
||||
pdfHighlightEnabled,
|
||||
pdfWordHighlightEnabled,
|
||||
clearWordHighlights,
|
||||
highlightWordIndex
|
||||
]);
|
||||
|
||||
// Add page dimensions state
|
||||
const [pageWidth, setPageWidth] = useState<number>(595); // default A4 width
|
||||
const [pageHeight, setPageHeight] = useState<number>(842); // default A4 height
|
||||
|
|
|
|||
|
|
@ -32,7 +32,9 @@ interface ConfigContextType {
|
|||
isLoading: boolean;
|
||||
isDBReady: boolean;
|
||||
pdfHighlightEnabled: boolean;
|
||||
pdfWordHighlightEnabled: boolean;
|
||||
epubHighlightEnabled: boolean;
|
||||
epubWordHighlightEnabled: boolean;
|
||||
}
|
||||
|
||||
const ConfigContext = createContext<ConfigContextType | undefined>(undefined);
|
||||
|
|
@ -103,7 +105,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
|
|||
savedVoices,
|
||||
smartSentenceSplitting,
|
||||
pdfHighlightEnabled,
|
||||
pdfWordHighlightEnabled,
|
||||
epubHighlightEnabled,
|
||||
epubWordHighlightEnabled,
|
||||
} = config || APP_CONFIG_DEFAULTS;
|
||||
|
||||
/**
|
||||
|
|
@ -201,7 +205,9 @@ export function ConfigProvider({ children }: { children: ReactNode }) {
|
|||
isLoading,
|
||||
isDBReady,
|
||||
pdfHighlightEnabled,
|
||||
epubHighlightEnabled
|
||||
pdfWordHighlightEnabled,
|
||||
epubHighlightEnabled,
|
||||
epubWordHighlightEnabled
|
||||
}}>
|
||||
{children}
|
||||
</ConfigContext.Provider>
|
||||
|
|
|
|||
|
|
@ -22,7 +22,13 @@ import { createRangeCfi } from '@/lib/epub';
|
|||
import { useParams } from 'next/navigation';
|
||||
import { useConfig } from './ConfigContext';
|
||||
import { withRetry } from '@/utils/audio';
|
||||
import type { TTSRequestHeaders, TTSRequestPayload, TTSRetryOptions } from '@/types/tts';
|
||||
import { CmpStr } from 'cmpstr';
|
||||
import type {
|
||||
TTSRequestHeaders,
|
||||
TTSRequestPayload,
|
||||
TTSRetryOptions,
|
||||
TTSSentenceAlignment
|
||||
} from '@/types/tts';
|
||||
|
||||
interface EPUBContextType {
|
||||
currDocData: ArrayBuffer | undefined;
|
||||
|
|
@ -44,12 +50,26 @@ interface EPUBContextType {
|
|||
isAudioCombining: boolean;
|
||||
highlightPattern: (text: string) => void;
|
||||
clearHighlights: () => void;
|
||||
highlightWordIndex: (
|
||||
alignment: TTSSentenceAlignment | undefined,
|
||||
wordIndex: number | null | undefined,
|
||||
sentence: string | null | undefined
|
||||
) => void;
|
||||
clearWordHighlights: () => void;
|
||||
}
|
||||
|
||||
const EPUBContext = createContext<EPUBContextType | undefined>(undefined);
|
||||
|
||||
const EPUB_CONTINUATION_CHARS = 600;
|
||||
|
||||
const cmp = CmpStr.create().setMetric('dice').setFlags('itw');
|
||||
|
||||
const normalizeWordForMatch = (text: string): string =>
|
||||
text
|
||||
.trim()
|
||||
.replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/g, '')
|
||||
.toLowerCase();
|
||||
|
||||
const stepToNextNode = (node: Node | null, root: Node): Node | null => {
|
||||
if (!node) return null;
|
||||
if (node.firstChild) {
|
||||
|
|
@ -160,6 +180,7 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
const shouldPauseRef = useRef(true);
|
||||
// Track current highlight CFI for removal
|
||||
const currentHighlightCfi = useRef<string | null>(null);
|
||||
const currentWordHighlightCfi = useRef<string | null>(null);
|
||||
|
||||
/**
|
||||
* Clears all current document state and stops any active TTS
|
||||
|
|
@ -711,15 +732,22 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
}
|
||||
}, [id, skipToLocation, extractPageText, setIsEPUB]);
|
||||
|
||||
const clearHighlights = useCallback(() => {
|
||||
if (renditionRef.current) {
|
||||
if (currentHighlightCfi.current) {
|
||||
renditionRef.current.annotations.remove(currentHighlightCfi.current, 'highlight');
|
||||
currentHighlightCfi.current = null;
|
||||
}
|
||||
const clearWordHighlights = useCallback(() => {
|
||||
if (!renditionRef.current) return;
|
||||
if (currentWordHighlightCfi.current) {
|
||||
renditionRef.current.annotations.remove(currentWordHighlightCfi.current, 'highlight');
|
||||
currentWordHighlightCfi.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
const clearHighlights = useCallback(() => {
|
||||
if (renditionRef.current && currentHighlightCfi.current) {
|
||||
renditionRef.current.annotations.remove(currentHighlightCfi.current, 'highlight');
|
||||
currentHighlightCfi.current = null;
|
||||
}
|
||||
clearWordHighlights();
|
||||
}, [clearWordHighlights]);
|
||||
|
||||
const highlightPattern = useCallback(async (text: string) => {
|
||||
if (!renditionRef.current) return;
|
||||
|
||||
|
|
@ -772,6 +800,262 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
}
|
||||
}, [epubHighlightEnabled, clearHighlights]);
|
||||
|
||||
const highlightWordIndex = useCallback((
|
||||
alignment: TTSSentenceAlignment | undefined,
|
||||
wordIndex: number | null | undefined,
|
||||
sentence: string | null | undefined
|
||||
) => {
|
||||
clearWordHighlights();
|
||||
|
||||
if (!epubHighlightEnabled) return;
|
||||
if (!alignment) return;
|
||||
if (wordIndex === null || wordIndex === undefined || wordIndex < 0) return;
|
||||
|
||||
const words = alignment.words || [];
|
||||
if (!words.length || wordIndex >= words.length) return;
|
||||
|
||||
if (!renditionRef.current) return;
|
||||
if (!currentHighlightCfi.current) return;
|
||||
|
||||
const cleanSentence =
|
||||
sentence && sentence.trim()
|
||||
? sentence.trim().replace(/\s+/g, ' ')
|
||||
: null;
|
||||
if (!cleanSentence) return;
|
||||
|
||||
const alignmentSentenceClean = alignment.sentence
|
||||
? alignment.sentence.trim().replace(/\s+/g, ' ')
|
||||
: null;
|
||||
if (!alignmentSentenceClean || alignmentSentenceClean !== cleanSentence) {
|
||||
return;
|
||||
}
|
||||
|
||||
const contents = renditionRef.current.getContents();
|
||||
const contentsArray = Array.isArray(contents) ? contents : [contents];
|
||||
|
||||
for (const content of contentsArray) {
|
||||
let range: Range | null = null;
|
||||
try {
|
||||
range = content.range(currentHighlightCfi.current as string);
|
||||
} catch {
|
||||
range = null;
|
||||
}
|
||||
if (!range) continue;
|
||||
|
||||
const root = range.commonAncestorContainer;
|
||||
if (!root) continue;
|
||||
|
||||
const domTokens: Array<{
|
||||
node: Text;
|
||||
startOffset: number;
|
||||
endOffset: number;
|
||||
norm: string;
|
||||
}> = [];
|
||||
|
||||
const addTokensFromNode = (textNode: Text, start: number, end: number) => {
|
||||
const full = textNode.textContent || '';
|
||||
const safeStart = Math.max(0, Math.min(start, full.length));
|
||||
const safeEnd = Math.max(safeStart, Math.min(end, full.length));
|
||||
if (safeEnd <= safeStart) return;
|
||||
|
||||
const slice = full.slice(safeStart, safeEnd);
|
||||
const wordRegex = /\S+/g;
|
||||
let match: RegExpExecArray | null;
|
||||
while ((match = wordRegex.exec(slice)) !== null) {
|
||||
const raw = match[0];
|
||||
const norm = normalizeWordForMatch(raw);
|
||||
if (!norm) continue;
|
||||
const tokenStart = safeStart + match.index;
|
||||
const tokenEnd = tokenStart + raw.length;
|
||||
domTokens.push({
|
||||
node: textNode,
|
||||
startOffset: tokenStart,
|
||||
endOffset: tokenEnd,
|
||||
norm,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const nextTextNode = (node: Node | null): Text | null => {
|
||||
let next = getNextTextNode(node, root);
|
||||
while (next) {
|
||||
if (next.nodeType === Node.TEXT_NODE) {
|
||||
return next as Text;
|
||||
}
|
||||
next = getNextTextNode(next, root);
|
||||
}
|
||||
return null;
|
||||
};
|
||||
|
||||
// Collect tokens within the sentence range
|
||||
if (range.startContainer === range.endContainer && range.startContainer.nodeType === Node.TEXT_NODE) {
|
||||
addTokensFromNode(range.startContainer as Text, range.startOffset, range.endOffset);
|
||||
} else {
|
||||
let current: Text | null = null;
|
||||
|
||||
if (range.startContainer.nodeType === Node.TEXT_NODE) {
|
||||
const startText = range.startContainer as Text;
|
||||
const isEnd = range.endContainer === startText && range.endContainer.nodeType === Node.TEXT_NODE;
|
||||
const endOffset = isEnd ? range.endOffset : (startText.textContent || '').length;
|
||||
addTokensFromNode(startText, range.startOffset, endOffset);
|
||||
if (isEnd) {
|
||||
current = null;
|
||||
} else {
|
||||
current = nextTextNode(startText);
|
||||
}
|
||||
} else {
|
||||
current = nextTextNode(range.startContainer);
|
||||
}
|
||||
|
||||
while (current) {
|
||||
if (range.endContainer.nodeType === Node.TEXT_NODE && current === range.endContainer) {
|
||||
addTokensFromNode(current, 0, range.endOffset);
|
||||
break;
|
||||
} else {
|
||||
addTokensFromNode(current, 0, (current.textContent || '').length);
|
||||
}
|
||||
current = nextTextNode(current);
|
||||
}
|
||||
}
|
||||
|
||||
if (!domTokens.length) {
|
||||
return;
|
||||
}
|
||||
|
||||
const domFiltered: Array<{ tokenIndex: number; norm: string }> = [];
|
||||
for (let i = 0; i < domTokens.length; i++) {
|
||||
const norm = domTokens[i].norm;
|
||||
if (!norm) continue;
|
||||
domFiltered.push({ tokenIndex: i, norm });
|
||||
}
|
||||
|
||||
const ttsFiltered: Array<{ wordIndex: number; norm: string }> = [];
|
||||
for (let i = 0; i < words.length; i++) {
|
||||
const norm = normalizeWordForMatch(words[i].text);
|
||||
if (!norm) continue;
|
||||
ttsFiltered.push({ wordIndex: i, norm });
|
||||
}
|
||||
|
||||
const wordToToken = new Array<number>(words.length).fill(-1);
|
||||
const m = domFiltered.length;
|
||||
const n = ttsFiltered.length;
|
||||
|
||||
if (m && n) {
|
||||
const dp: number[][] = Array.from({ length: m + 1 }, () =>
|
||||
new Array<number>(n + 1).fill(Number.POSITIVE_INFINITY)
|
||||
);
|
||||
const bt: number[][] = Array.from({ length: m + 1 }, () =>
|
||||
new Array<number>(n + 1).fill(0)
|
||||
); // 0=diag,1=up,2=left
|
||||
|
||||
dp[0][0] = 0;
|
||||
const GAP_COST = 0.7;
|
||||
|
||||
for (let i = 0; i <= m; i++) {
|
||||
for (let j = 0; j <= n; j++) {
|
||||
if (i > 0 && j > 0) {
|
||||
const a = domFiltered[i - 1].norm;
|
||||
const b = ttsFiltered[j - 1].norm;
|
||||
const sim = cmp.compare(a, b);
|
||||
const subCost = 1 - sim;
|
||||
const cand = dp[i - 1][j - 1] + subCost;
|
||||
if (cand < dp[i][j]) {
|
||||
dp[i][j] = cand;
|
||||
bt[i][j] = 0;
|
||||
}
|
||||
}
|
||||
if (i > 0) {
|
||||
const cand = dp[i - 1][j] + GAP_COST;
|
||||
if (cand < dp[i][j]) {
|
||||
dp[i][j] = cand;
|
||||
bt[i][j] = 1;
|
||||
}
|
||||
}
|
||||
if (j > 0) {
|
||||
const cand = dp[i][j - 1] + GAP_COST;
|
||||
if (cand < dp[i][j]) {
|
||||
dp[i][j] = cand;
|
||||
bt[i][j] = 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let i = m;
|
||||
let j = n;
|
||||
while (i > 0 || j > 0) {
|
||||
const move = bt[i][j];
|
||||
if (i > 0 && j > 0 && move === 0) {
|
||||
const domIdx = domFiltered[i - 1].tokenIndex;
|
||||
const ttsIdx = ttsFiltered[j - 1].wordIndex;
|
||||
if (wordToToken[ttsIdx] === -1) {
|
||||
wordToToken[ttsIdx] = domIdx;
|
||||
}
|
||||
i -= 1;
|
||||
j -= 1;
|
||||
} else if (i > 0 && (move === 1 || j === 0)) {
|
||||
i -= 1;
|
||||
} else if (j > 0 && (move === 2 || i === 0)) {
|
||||
j -= 1;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Propagate nearest known mapping to fill gaps
|
||||
let lastSeen = -1;
|
||||
for (let k = 0; k < wordToToken.length; k++) {
|
||||
if (wordToToken[k] !== -1) {
|
||||
lastSeen = wordToToken[k];
|
||||
} else if (lastSeen !== -1) {
|
||||
wordToToken[k] = lastSeen;
|
||||
}
|
||||
}
|
||||
let nextSeen = -1;
|
||||
for (let k = wordToToken.length - 1; k >= 0; k--) {
|
||||
if (wordToToken[k] !== -1) {
|
||||
nextSeen = wordToToken[k];
|
||||
} else if (nextSeen !== -1) {
|
||||
wordToToken[k] = nextSeen;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const mappedIndex =
|
||||
wordIndex < wordToToken.length ? wordToToken[wordIndex] : -1;
|
||||
if (mappedIndex === -1) {
|
||||
return;
|
||||
}
|
||||
|
||||
const token = domTokens[mappedIndex];
|
||||
const doc = token.node.ownerDocument || (range.commonAncestorContainer as Document);
|
||||
const wordRange = doc.createRange();
|
||||
wordRange.setStart(token.node, token.startOffset);
|
||||
wordRange.setEnd(token.node, token.endOffset);
|
||||
|
||||
try {
|
||||
const wordCfi = content.cfiFromRange(wordRange);
|
||||
currentWordHighlightCfi.current = wordCfi;
|
||||
renditionRef.current.annotations.add(
|
||||
'highlight',
|
||||
wordCfi,
|
||||
{},
|
||||
() => {},
|
||||
'',
|
||||
{
|
||||
fill: 'var(--accent)',
|
||||
'fill-opacity': '0.4',
|
||||
'mix-blend-mode': 'multiply',
|
||||
}
|
||||
);
|
||||
} catch (error) {
|
||||
console.error('Error highlighting EPUB word:', error);
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
}, [epubHighlightEnabled, clearWordHighlights]);
|
||||
|
||||
|
||||
|
||||
// Context value memoization
|
||||
|
|
@ -796,6 +1080,8 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
isAudioCombining,
|
||||
highlightPattern,
|
||||
clearHighlights,
|
||||
highlightWordIndex,
|
||||
clearWordHighlights,
|
||||
}),
|
||||
[
|
||||
setCurrentDocument,
|
||||
|
|
@ -813,6 +1099,8 @@ export function EPUBProvider({ children }: { children: ReactNode }) {
|
|||
isAudioCombining,
|
||||
highlightPattern,
|
||||
clearHighlights,
|
||||
highlightWordIndex,
|
||||
clearWordHighlights,
|
||||
]
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -36,10 +36,16 @@ import {
|
|||
extractTextFromPDF,
|
||||
highlightPattern,
|
||||
clearHighlights,
|
||||
handleTextClick,
|
||||
clearWordHighlights,
|
||||
highlightWordIndex,
|
||||
} from '@/lib/pdf';
|
||||
|
||||
import type { TTSRequestHeaders, TTSRequestPayload, TTSRetryOptions } from '@/types/tts';
|
||||
import type {
|
||||
TTSRequestHeaders,
|
||||
TTSRequestPayload,
|
||||
TTSRetryOptions,
|
||||
TTSSentenceAlignment
|
||||
} from '@/types/tts';
|
||||
|
||||
/**
|
||||
* Interface defining all available methods and properties in the PDF context
|
||||
|
|
@ -59,13 +65,12 @@ interface PDFContextType {
|
|||
onDocumentLoadSuccess: (pdf: PDFDocumentProxy) => void;
|
||||
highlightPattern: (text: string, pattern: string, containerRef: RefObject<HTMLDivElement>) => void;
|
||||
clearHighlights: () => void;
|
||||
handleTextClick: (
|
||||
event: MouseEvent,
|
||||
pdfText: string,
|
||||
containerRef: RefObject<HTMLDivElement>,
|
||||
stopAndPlayFromIndex: (index: number) => void,
|
||||
isProcessing: boolean,
|
||||
enableHighlight?: boolean
|
||||
clearWordHighlights: () => void;
|
||||
highlightWordIndex: (
|
||||
alignment: TTSSentenceAlignment | undefined,
|
||||
wordIndex: number | null | undefined,
|
||||
sentence: string | null | undefined,
|
||||
containerRef: RefObject<HTMLDivElement>
|
||||
) => void;
|
||||
createFullAudioBook: (onProgress: (progress: number) => void, signal?: AbortSignal, onChapterComplete?: (chapter: { index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }) => void, bookId?: string, format?: 'mp3' | 'm4b') => Promise<string>;
|
||||
regenerateChapter: (chapterIndex: number, bookId: string, format: 'mp3' | 'm4b', signal: AbortSignal) => Promise<{ index: number; title: string; duration?: number; status: 'pending' | 'generating' | 'completed' | 'error'; bookId?: string; format?: 'mp3' | 'm4b' }>;
|
||||
|
|
@ -655,7 +660,8 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
|||
clearCurrDoc,
|
||||
highlightPattern,
|
||||
clearHighlights,
|
||||
handleTextClick,
|
||||
clearWordHighlights,
|
||||
highlightWordIndex,
|
||||
pdfDocument,
|
||||
createFullAudioBook,
|
||||
regenerateChapter,
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ import type {
|
|||
TTSRequestPayload,
|
||||
TTSRequestHeaders,
|
||||
TTSRetryOptions,
|
||||
TTSSentenceAlignment,
|
||||
} from '@/types/tts';
|
||||
|
||||
// Media globals
|
||||
|
|
@ -63,6 +64,10 @@ interface TTSContextType extends TTSPlaybackState {
|
|||
// Voice settings
|
||||
availableVoices: string[];
|
||||
|
||||
// Alignment metadata for the current sentence
|
||||
currentSentenceAlignment?: TTSSentenceAlignment;
|
||||
currentWordIndex?: number | null;
|
||||
|
||||
// Control functions
|
||||
togglePlay: () => void;
|
||||
skipForward: () => void;
|
||||
|
|
@ -264,6 +269,10 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
updateConfigKey,
|
||||
skipBlank,
|
||||
smartSentenceSplitting,
|
||||
pdfHighlightEnabled,
|
||||
pdfWordHighlightEnabled,
|
||||
epubHighlightEnabled,
|
||||
epubWordHighlightEnabled,
|
||||
} = useConfig();
|
||||
|
||||
// Audio and voice management hooks
|
||||
|
|
@ -331,6 +340,19 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
const epubContinuationRef = useRef<string | null>(null);
|
||||
const pageTurnEstimateRef = useRef<TTSPageTurnEstimate | null>(null);
|
||||
const pageTurnTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
const sentenceAlignmentCacheRef = useRef<Map<string, TTSSentenceAlignment>>(new Map());
|
||||
const [currentSentenceAlignment, setCurrentSentenceAlignment] = useState<TTSSentenceAlignment | undefined>();
|
||||
const [currentWordIndex, setCurrentWordIndex] = useState<number | null>(null);
|
||||
const sentencesRef = useRef<string[]>([]);
|
||||
const currentIndexRef = useRef(0);
|
||||
|
||||
useEffect(() => {
|
||||
sentencesRef.current = sentences;
|
||||
}, [sentences]);
|
||||
|
||||
useEffect(() => {
|
||||
currentIndexRef.current = currentIndex;
|
||||
}, [currentIndex]);
|
||||
|
||||
/**
|
||||
* Processes text into sentences using the shared NLP utility
|
||||
|
|
@ -370,6 +392,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
clearTimeout(pageTurnTimeoutRef.current);
|
||||
pageTurnTimeoutRef.current = null;
|
||||
}
|
||||
setCurrentWordIndex(null);
|
||||
}, [activeHowl]);
|
||||
|
||||
/**
|
||||
|
|
@ -558,6 +581,11 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
setCurrentIndex(0);
|
||||
}
|
||||
|
||||
// Reset alignment state whenever the text block changes
|
||||
sentenceAlignmentCacheRef.current.clear();
|
||||
setCurrentSentenceAlignment(undefined);
|
||||
setCurrentWordIndex(null);
|
||||
|
||||
// Compute auto page-turn estimate for PDFs when we have a continuation
|
||||
if (smartSentenceSplitting && !isEPUB && continuationCarried && normalizedOptions.nextLocation !== undefined) {
|
||||
const continuationNormalized = preprocessSentenceForAudio(continuationCarried);
|
||||
|
|
@ -706,10 +734,61 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
* @returns {Promise<ArrayBuffer | undefined>} The generated audio buffer
|
||||
*/
|
||||
const getAudio = useCallback(async (sentence: string): Promise<ArrayBuffer | undefined> => {
|
||||
const alignmentEnabledForCurrentDoc =
|
||||
(!isEPUB && pdfHighlightEnabled && pdfWordHighlightEnabled) ||
|
||||
(isEPUB && epubHighlightEnabled && epubWordHighlightEnabled);
|
||||
// Helper to ensure we have an alignment for a given
|
||||
// sentence/audio pair, even when the audio itself is
|
||||
// served from the local cache.
|
||||
const ensureAlignment = (arrayBuffer: ArrayBuffer) => {
|
||||
if (!alignmentEnabledForCurrentDoc) return;
|
||||
if (sentenceAlignmentCacheRef.current.has(sentence)) return;
|
||||
|
||||
try {
|
||||
const audioBytes = Array.from(new Uint8Array(arrayBuffer));
|
||||
const alignmentBody = {
|
||||
text: sentence,
|
||||
audio: audioBytes,
|
||||
};
|
||||
|
||||
void fetch('/api/whisper', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(alignmentBody),
|
||||
})
|
||||
.then(async (res) => {
|
||||
if (!res.ok) return;
|
||||
const data = await res.json().catch(() => null);
|
||||
if (!data || !Array.isArray(data.alignments) || !data.alignments[0]) {
|
||||
return;
|
||||
}
|
||||
const alignment = data.alignments[0] as TTSSentenceAlignment;
|
||||
sentenceAlignmentCacheRef.current.set(sentence, alignment);
|
||||
|
||||
const currentSentence = sentencesRef.current[currentIndexRef.current];
|
||||
if (currentSentence === sentence) {
|
||||
setCurrentSentenceAlignment(alignment);
|
||||
setCurrentWordIndex(null);
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
console.warn('Alignment request failed:', err);
|
||||
});
|
||||
} catch (err) {
|
||||
console.warn('Failed to start alignment request:', err);
|
||||
}
|
||||
};
|
||||
|
||||
// Check if the audio is already cached
|
||||
const cachedAudio = audioCache.get(sentence);
|
||||
if (cachedAudio) {
|
||||
console.log('Using cached audio for sentence:', sentence.substring(0, 20));
|
||||
// If we have audio but no alignment (e.g. after a
|
||||
// navigation or TTS reset), kick off a fresh alignment
|
||||
// request using the cached audio buffer.
|
||||
ensureAlignment(cachedAudio);
|
||||
return cachedAudio;
|
||||
}
|
||||
|
||||
|
|
@ -769,6 +848,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
// Cache the array buffer
|
||||
audioCache.set(sentence, arrayBuffer);
|
||||
|
||||
// Fire-and-forget alignment request; do not block audio playback
|
||||
ensureAlignment(arrayBuffer);
|
||||
|
||||
return arrayBuffer;
|
||||
} catch (error) {
|
||||
// Check if this was an abort error
|
||||
|
|
@ -788,7 +870,21 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
});
|
||||
throw error;
|
||||
}
|
||||
}, [voice, speed, ttsModel, ttsInstructions, audioCache, openApiKey, openApiBaseUrl, configTTSProvider]);
|
||||
}, [
|
||||
voice,
|
||||
speed,
|
||||
ttsModel,
|
||||
ttsInstructions,
|
||||
audioCache,
|
||||
openApiKey,
|
||||
openApiBaseUrl,
|
||||
configTTSProvider,
|
||||
isEPUB,
|
||||
pdfHighlightEnabled,
|
||||
pdfWordHighlightEnabled,
|
||||
epubHighlightEnabled,
|
||||
epubWordHighlightEnabled
|
||||
]);
|
||||
|
||||
/**
|
||||
* Processes and plays the current sentence
|
||||
|
|
@ -1004,7 +1100,17 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
}, [isPlaying, advance, activeHowl, processSentence, audioSpeed]);
|
||||
|
||||
const playAudio = useCallback(async () => {
|
||||
const howl = await playSentenceWithHowl(sentences[currentIndex], currentIndex);
|
||||
const sentence = sentences[currentIndex];
|
||||
const cachedAlignment = sentenceAlignmentCacheRef.current.get(sentence);
|
||||
if (cachedAlignment) {
|
||||
setCurrentSentenceAlignment(cachedAlignment);
|
||||
setCurrentWordIndex(null);
|
||||
} else {
|
||||
setCurrentSentenceAlignment(undefined);
|
||||
setCurrentWordIndex(null);
|
||||
}
|
||||
|
||||
const howl = await playSentenceWithHowl(sentence, currentIndex);
|
||||
if (howl) {
|
||||
howl.play();
|
||||
}
|
||||
|
|
@ -1017,6 +1123,47 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
playAudio,
|
||||
});
|
||||
|
||||
// Track the current word index during playback using Howler's seek position
|
||||
useEffect(() => {
|
||||
if (!activeHowl || !isPlaying || !currentSentenceAlignment || !currentSentenceAlignment.words.length) {
|
||||
setCurrentWordIndex(null);
|
||||
return;
|
||||
}
|
||||
|
||||
let frameId: number;
|
||||
|
||||
const tick = () => {
|
||||
try {
|
||||
const pos = activeHowl.seek() as number;
|
||||
if (typeof pos === 'number' && Number.isFinite(pos)) {
|
||||
const words = currentSentenceAlignment.words;
|
||||
let idx = -1;
|
||||
for (let i = 0; i < words.length; i++) {
|
||||
const w = words[i];
|
||||
if (pos >= w.startSec && pos < w.endSec) {
|
||||
idx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (idx !== -1) {
|
||||
setCurrentWordIndex((prev) => (prev === idx ? prev : idx));
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// ignore seek errors
|
||||
}
|
||||
frameId = requestAnimationFrame(tick);
|
||||
};
|
||||
|
||||
frameId = requestAnimationFrame(tick);
|
||||
|
||||
return () => {
|
||||
if (frameId) {
|
||||
cancelAnimationFrame(frameId);
|
||||
}
|
||||
};
|
||||
}, [activeHowl, isPlaying, currentSentenceAlignment]);
|
||||
|
||||
/**
|
||||
* Preloads the next sentence's audio
|
||||
*/
|
||||
|
|
@ -1085,6 +1232,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
setCurrDocPages(undefined);
|
||||
setIsProcessing(false);
|
||||
setIsEPUB(false);
|
||||
sentenceAlignmentCacheRef.current.clear();
|
||||
setCurrentSentenceAlignment(undefined);
|
||||
setCurrentWordIndex(null);
|
||||
}, [abortAudio]);
|
||||
|
||||
/**
|
||||
|
|
@ -1205,6 +1355,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
isProcessing,
|
||||
isBackgrounded,
|
||||
currentSentence: sentences[currentIndex] || '',
|
||||
currentSentenceAlignment,
|
||||
currentWordIndex,
|
||||
currDocPage,
|
||||
currDocPageNumber,
|
||||
currDocPages,
|
||||
|
|
@ -1248,7 +1400,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
|||
skipToLocation,
|
||||
registerLocationChangeHandler,
|
||||
registerVisualPageChangeHandler,
|
||||
setIsEPUB
|
||||
setIsEPUB,
|
||||
currentSentenceAlignment,
|
||||
currentWordIndex
|
||||
]);
|
||||
|
||||
// Use media session hook
|
||||
|
|
|
|||
|
|
@ -136,6 +136,12 @@ function buildAppConfigFromRaw(raw: RawConfigMap): AppConfigRow {
|
|||
savedVoices,
|
||||
pdfHighlightEnabled:
|
||||
raw.pdfHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.pdfHighlightEnabled,
|
||||
pdfWordHighlightEnabled:
|
||||
raw.pdfWordHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.pdfWordHighlightEnabled,
|
||||
epubHighlightEnabled:
|
||||
raw.epubHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.epubHighlightEnabled,
|
||||
epubWordHighlightEnabled:
|
||||
raw.epubWordHighlightEnabled === 'false' ? false : APP_CONFIG_DEFAULTS.epubWordHighlightEnabled,
|
||||
firstVisit: raw.firstVisit === 'true',
|
||||
documentListState,
|
||||
};
|
||||
|
|
|
|||
406
src/lib/pdf.ts
406
src/lib/pdf.ts
|
|
@ -3,9 +3,10 @@ import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
|||
import { type PDFDocumentProxy, TextLayer } from 'pdfjs-dist';
|
||||
import "core-js/proposals/promise-with-resolvers";
|
||||
import { processTextToSentences } from '@/lib/nlp';
|
||||
import type { TTSSentenceAlignment } from '@/types/tts';
|
||||
import { CmpStr } from 'cmpstr';
|
||||
|
||||
const cmp = CmpStr.create().setMetric( 'dice' ).setFlags( 'itw' );
|
||||
const cmp = CmpStr.create().setMetric('dice').setFlags('itw');
|
||||
|
||||
// Worker coordination for offloading highlight token matching
|
||||
interface HighlightTokenMatchRequest {
|
||||
|
|
@ -141,12 +142,25 @@ try {
|
|||
console.error('Error patching TextLayer.render:', e);
|
||||
}
|
||||
|
||||
interface TextMatch {
|
||||
elements: HTMLElement[];
|
||||
rating: number;
|
||||
type PDFToken = {
|
||||
spanIndex: number;
|
||||
textNode: Text;
|
||||
text: string;
|
||||
lengthDiff: number;
|
||||
}
|
||||
startOffset: number;
|
||||
endOffset: number;
|
||||
};
|
||||
|
||||
let lastSpanNodes: HTMLElement[] = [];
|
||||
let lastTokens: PDFToken[] = [];
|
||||
let lastSentenceTokenWindow: { start: number; end: number } | null = null;
|
||||
let lastSentencePattern: string | null = null;
|
||||
let lastSentenceWordToTokenMap: number[] | null = null;
|
||||
|
||||
const normalizeWordForMatch = (text: string): string =>
|
||||
text
|
||||
.trim()
|
||||
.replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/g, '')
|
||||
.toLowerCase();
|
||||
|
||||
// Text Processing functions
|
||||
export async function extractTextFromPDF(
|
||||
|
|
@ -279,50 +293,23 @@ export function clearHighlights() {
|
|||
element.parentElement.removeChild(element);
|
||||
}
|
||||
});
|
||||
const wordOverlays = document.querySelectorAll('.pdf-word-highlight-overlay');
|
||||
wordOverlays.forEach((node) => {
|
||||
const element = node as HTMLElement;
|
||||
if (element.parentElement) {
|
||||
element.parentElement.removeChild(element);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
export function findBestTextMatch(
|
||||
elements: Array<{ element: HTMLElement; text: string }>,
|
||||
targetText: string,
|
||||
maxCombinedLength: number
|
||||
): TextMatch {
|
||||
let bestMatch = {
|
||||
elements: [] as HTMLElement[],
|
||||
rating: 0,
|
||||
text: '',
|
||||
lengthDiff: Infinity,
|
||||
};
|
||||
|
||||
const SPAN_SEARCH_LIMIT = 10;
|
||||
|
||||
for (let i = 0; i < elements.length; i++) {
|
||||
let combinedText = '';
|
||||
const currentElements = [];
|
||||
for (let j = i; j < Math.min(i + SPAN_SEARCH_LIMIT, elements.length); j++) {
|
||||
const node = elements[j];
|
||||
const newText = combinedText ? `${combinedText} ${node.text}` : node.text;
|
||||
if (newText.length > maxCombinedLength) break;
|
||||
|
||||
combinedText = newText;
|
||||
currentElements.push(node.element);
|
||||
|
||||
const similarity = cmp.compare(combinedText, targetText);
|
||||
const lengthDiff = Math.abs(combinedText.length - targetText.length);
|
||||
const lengthPenalty = lengthDiff / targetText.length;
|
||||
const adjustedRating = similarity * (1 - lengthPenalty * 0.5);
|
||||
|
||||
if (adjustedRating > bestMatch.rating) {
|
||||
bestMatch = {
|
||||
elements: [...currentElements],
|
||||
rating: adjustedRating,
|
||||
text: combinedText,
|
||||
lengthDiff,
|
||||
};
|
||||
}
|
||||
export function clearWordHighlights() {
|
||||
const wordOverlays = document.querySelectorAll('.pdf-word-highlight-overlay');
|
||||
wordOverlays.forEach((node) => {
|
||||
const element = node as HTMLElement;
|
||||
if (element.parentElement) {
|
||||
element.parentElement.removeChild(element);
|
||||
}
|
||||
}
|
||||
|
||||
return bestMatch;
|
||||
});
|
||||
}
|
||||
|
||||
export function highlightPattern(
|
||||
|
|
@ -338,22 +325,18 @@ export function highlightPattern(
|
|||
|
||||
const cleanPattern = pattern.trim().replace(/\s+/g, ' ');
|
||||
if (!cleanPattern) return;
|
||||
lastSentencePattern = cleanPattern;
|
||||
lastSentenceWordToTokenMap = null;
|
||||
lastSentenceTokenWindow = null;
|
||||
|
||||
const spanNodes = Array.from(
|
||||
container.querySelectorAll('.react-pdf__Page__textContent span')
|
||||
) as HTMLElement[];
|
||||
|
||||
if (!spanNodes.length) return;
|
||||
lastSpanNodes = spanNodes;
|
||||
|
||||
type Token = {
|
||||
spanIndex: number;
|
||||
textNode: Text;
|
||||
text: string;
|
||||
startOffset: number;
|
||||
endOffset: number;
|
||||
};
|
||||
|
||||
const tokens: Token[] = [];
|
||||
const tokens: PDFToken[] = [];
|
||||
|
||||
spanNodes.forEach((span, spanIndex) => {
|
||||
const node = span.firstChild;
|
||||
|
|
@ -377,6 +360,7 @@ export function highlightPattern(
|
|||
});
|
||||
|
||||
if (!tokens.length) return;
|
||||
lastTokens = tokens;
|
||||
|
||||
const patternLen = cleanPattern.length;
|
||||
|
||||
|
|
@ -415,6 +399,11 @@ export function highlightPattern(
|
|||
bestLengthDiff < patternLen * 0.3 ? 0.3 : 0.5;
|
||||
|
||||
if (hasTokenMatch && bestRating >= similarityThreshold) {
|
||||
lastSentenceTokenWindow = {
|
||||
start: bestStart,
|
||||
end: bestEnd,
|
||||
};
|
||||
|
||||
const rangesBySpan = new Map<
|
||||
number,
|
||||
{ startOffset: number; endOffset: number }
|
||||
|
|
@ -454,65 +443,6 @@ export function highlightPattern(
|
|||
});
|
||||
}
|
||||
|
||||
// Fallback: if token-level matching failed, use span-based fuzzy matching
|
||||
if (!highlightRanges.length) {
|
||||
const spanEntries = spanNodes
|
||||
.map((node) => ({
|
||||
element: node as HTMLElement,
|
||||
text: (node.textContent || '').trim(),
|
||||
}))
|
||||
.filter((entry) => entry.text.length > 0);
|
||||
|
||||
const containerRect = container.getBoundingClientRect();
|
||||
const visibleTop = container.scrollTop;
|
||||
const visibleBottom = visibleTop + containerRect.height;
|
||||
const bufferSize = containerRect.height;
|
||||
|
||||
const visibleNodes = spanEntries.filter(({ element }) => {
|
||||
const rect = element.getBoundingClientRect();
|
||||
const elementTop =
|
||||
rect.top - containerRect.top + container.scrollTop;
|
||||
return (
|
||||
elementTop >= visibleTop - bufferSize &&
|
||||
elementTop <= visibleBottom + bufferSize
|
||||
);
|
||||
});
|
||||
|
||||
let bestMatch = findBestTextMatch(
|
||||
visibleNodes,
|
||||
cleanPattern,
|
||||
cleanPattern.length * 2
|
||||
);
|
||||
|
||||
if (bestMatch.rating < 0.3) {
|
||||
bestMatch = findBestTextMatch(
|
||||
spanEntries,
|
||||
cleanPattern,
|
||||
cleanPattern.length * 2
|
||||
);
|
||||
}
|
||||
|
||||
const spanSimilarityThreshold =
|
||||
bestMatch.lengthDiff < cleanPattern.length * 0.3 ? 0.3 : 0.5;
|
||||
|
||||
if (bestMatch.rating >= spanSimilarityThreshold) {
|
||||
bestMatch.elements.forEach((element) => {
|
||||
const node = element.firstChild;
|
||||
if (!node || node.nodeType !== Node.TEXT_NODE) return;
|
||||
const textNode = node as Text;
|
||||
const content = textNode.textContent || '';
|
||||
if (!content) return;
|
||||
|
||||
highlightRanges.push({
|
||||
textNode,
|
||||
startOffset: 0,
|
||||
endOffset: content.length,
|
||||
span: element,
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!highlightRanges.length) return;
|
||||
|
||||
// Create overlay rectangles for each range, relative to its page text layer
|
||||
|
|
@ -577,7 +507,7 @@ export function highlightPattern(
|
|||
runHighlightTokenMatch(cleanPattern, tokenTexts)
|
||||
.then((result) => {
|
||||
if (!result || result.bestStart === -1) {
|
||||
// No worker result or no good match; rely on span-level fallback
|
||||
// No worker result or no good match; nothing to highlight
|
||||
applyHighlightFromTokens(null);
|
||||
} else {
|
||||
applyHighlightFromTokens({
|
||||
|
|
@ -590,95 +520,199 @@ export function highlightPattern(
|
|||
})
|
||||
.catch((error) => {
|
||||
console.error(
|
||||
'Error in PDF highlight worker, falling back to span-based matching:',
|
||||
'Error in PDF highlight worker; no highlights applied:',
|
||||
error
|
||||
);
|
||||
applyHighlightFromTokens(null);
|
||||
});
|
||||
}
|
||||
|
||||
// Text Click Handler
|
||||
export function handleTextClick(
|
||||
event: MouseEvent,
|
||||
pdfText: string,
|
||||
containerRef: React.RefObject<HTMLDivElement>,
|
||||
stopAndPlayFromIndex: (index: number) => void,
|
||||
isProcessing: boolean,
|
||||
enableHighlight = true
|
||||
export function highlightWordIndex(
|
||||
alignment: TTSSentenceAlignment | undefined,
|
||||
wordIndex: number | null | undefined,
|
||||
sentence: string | null | undefined,
|
||||
containerRef: React.RefObject<HTMLDivElement>
|
||||
) {
|
||||
if (isProcessing) return;
|
||||
clearWordHighlights();
|
||||
|
||||
const target = event.target as HTMLElement;
|
||||
if (!target.matches('.react-pdf__Page__textContent span')) return;
|
||||
|
||||
const parentElement = target.closest('.react-pdf__Page__textContent');
|
||||
if (!parentElement) return;
|
||||
|
||||
const spans = Array.from(parentElement.querySelectorAll('span'));
|
||||
const clickedIndex = spans.indexOf(target);
|
||||
const contextWindow = 3;
|
||||
const startIndex = Math.max(0, clickedIndex - contextWindow);
|
||||
const endIndex = Math.min(spans.length - 1, clickedIndex + contextWindow);
|
||||
const contextText = spans
|
||||
.slice(startIndex, endIndex + 1)
|
||||
.map((span) => span.textContent)
|
||||
.join(' ')
|
||||
.trim();
|
||||
|
||||
if (!contextText?.trim()) return;
|
||||
|
||||
const cleanContext = contextText.trim().replace(/\s+/g, ' ');
|
||||
|
||||
// Fast path when highlight overlays are disabled:
|
||||
// avoid expensive span-level fuzzy matching and just map
|
||||
// the clicked context to a sentence using cheap string checks.
|
||||
if (!enableHighlight) {
|
||||
const sentences = processTextToSentences(pdfText);
|
||||
const idx = sentences.findIndex((sentence) => {
|
||||
const cleanSentence = sentence.trim().replace(/\s+/g, ' ');
|
||||
return (
|
||||
cleanSentence.includes(cleanContext) ||
|
||||
cleanContext.includes(cleanSentence)
|
||||
);
|
||||
});
|
||||
|
||||
if (idx !== -1) {
|
||||
stopAndPlayFromIndex(idx);
|
||||
}
|
||||
if (!alignment) return;
|
||||
if (wordIndex === null || wordIndex === undefined || wordIndex < 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const allText = Array.from(parentElement.querySelectorAll('span')).map((node) => ({
|
||||
element: node as HTMLElement,
|
||||
text: (node.textContent || '').trim(),
|
||||
})).filter((node) => node.text.length > 0);
|
||||
const words = alignment.words || [];
|
||||
if (!words.length || wordIndex >= words.length) return;
|
||||
|
||||
const bestMatch = findBestTextMatch(allText, cleanContext, cleanContext.length * 2);
|
||||
const similarityThreshold = bestMatch.lengthDiff < cleanContext.length * 0.3 ? 0.3 : 0.5;
|
||||
const container = containerRef.current;
|
||||
if (!container) return;
|
||||
if (!lastSentenceTokenWindow) return;
|
||||
if (!lastTokens.length || !lastSpanNodes.length) return;
|
||||
|
||||
if (bestMatch.rating >= similarityThreshold) {
|
||||
const matchText = bestMatch.text;
|
||||
// Use the same sentence processing logic as TTSContext for consistency
|
||||
const sentences = processTextToSentences(pdfText);
|
||||
console.log("sentences inside handleTextClick: %d", sentences.length)
|
||||
let bestSentenceMatch = { sentence: '', rating: 0 };
|
||||
const cleanSentence =
|
||||
sentence && sentence.trim()
|
||||
? sentence.trim().replace(/\s+/g, ' ')
|
||||
: null;
|
||||
if (!cleanSentence || !lastSentencePattern) return;
|
||||
if (cleanSentence !== lastSentencePattern) return;
|
||||
|
||||
for (const sentence of sentences) {
|
||||
const rating = cmp.compare(matchText, sentence);
|
||||
if (rating > bestSentenceMatch.rating) {
|
||||
bestSentenceMatch = { sentence, rating };
|
||||
}
|
||||
const start = lastSentenceTokenWindow.start;
|
||||
const end = lastSentenceTokenWindow.end;
|
||||
if (end < start) return;
|
||||
|
||||
// Lazily build or refresh the mapping from alignment word
|
||||
// indices to PDF token indices for this sentence window.
|
||||
if (
|
||||
!lastSentenceWordToTokenMap ||
|
||||
lastSentenceWordToTokenMap.length !== words.length
|
||||
) {
|
||||
const pdfFiltered: { tokenIndex: number; norm: string }[] = [];
|
||||
for (let i = start; i <= end; i++) {
|
||||
const norm = normalizeWordForMatch(lastTokens[i].text);
|
||||
if (!norm) continue;
|
||||
pdfFiltered.push({ tokenIndex: i, norm });
|
||||
}
|
||||
|
||||
if (bestSentenceMatch.rating >= 0.5) {
|
||||
const sentenceIndex = sentences.findIndex((sentence) => sentence === bestSentenceMatch.sentence);
|
||||
if (sentenceIndex !== -1) {
|
||||
stopAndPlayFromIndex(sentenceIndex);
|
||||
if (enableHighlight) {
|
||||
highlightPattern(pdfText, bestSentenceMatch.sentence, containerRef);
|
||||
const ttsFiltered: { wordIndex: number; norm: string }[] = [];
|
||||
for (let i = 0; i < words.length; i++) {
|
||||
const norm = normalizeWordForMatch(words[i].text);
|
||||
if (!norm) continue;
|
||||
ttsFiltered.push({ wordIndex: i, norm });
|
||||
}
|
||||
|
||||
const wordToToken = new Array<number>(words.length).fill(-1);
|
||||
|
||||
const m = pdfFiltered.length;
|
||||
const n = ttsFiltered.length;
|
||||
|
||||
if (m && n) {
|
||||
const dp: number[][] = Array.from({ length: m + 1 }, () =>
|
||||
new Array<number>(n + 1).fill(Number.POSITIVE_INFINITY)
|
||||
);
|
||||
const bt: number[][] = Array.from({ length: m + 1 }, () =>
|
||||
new Array<number>(n + 1).fill(0)
|
||||
); // 0=diag,1=up,2=left
|
||||
|
||||
dp[0][0] = 0;
|
||||
const GAP_COST = 0.7;
|
||||
|
||||
for (let i = 0; i <= m; i++) {
|
||||
for (let j = 0; j <= n; j++) {
|
||||
if (i > 0 && j > 0) {
|
||||
const a = pdfFiltered[i - 1].norm;
|
||||
const b = ttsFiltered[j - 1].norm;
|
||||
const sim = cmp.compare(a, b);
|
||||
const subCost = 1 - sim;
|
||||
const cand = dp[i - 1][j - 1] + subCost;
|
||||
if (cand < dp[i][j]) {
|
||||
dp[i][j] = cand;
|
||||
bt[i][j] = 0;
|
||||
}
|
||||
}
|
||||
if (i > 0) {
|
||||
const cand = dp[i - 1][j] + GAP_COST;
|
||||
if (cand < dp[i][j]) {
|
||||
dp[i][j] = cand;
|
||||
bt[i][j] = 1;
|
||||
}
|
||||
}
|
||||
if (j > 0) {
|
||||
const cand = dp[i][j - 1] + GAP_COST;
|
||||
if (cand < dp[i][j]) {
|
||||
dp[i][j] = cand;
|
||||
bt[i][j] = 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let i = m;
|
||||
let j = n;
|
||||
while (i > 0 || j > 0) {
|
||||
const move = bt[i][j];
|
||||
if (i > 0 && j > 0 && move === 0) {
|
||||
const pdfIdx = pdfFiltered[i - 1].tokenIndex;
|
||||
const ttsIdx = ttsFiltered[j - 1].wordIndex;
|
||||
if (wordToToken[ttsIdx] === -1) {
|
||||
wordToToken[ttsIdx] = pdfIdx;
|
||||
}
|
||||
i -= 1;
|
||||
j -= 1;
|
||||
} else if (i > 0 && (move === 1 || j === 0)) {
|
||||
i -= 1;
|
||||
} else if (j > 0 && (move === 2 || i === 0)) {
|
||||
j -= 1;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Propagate nearest known mapping to fill gaps
|
||||
let lastSeen = -1;
|
||||
for (let k = 0; k < wordToToken.length; k++) {
|
||||
if (wordToToken[k] !== -1) {
|
||||
lastSeen = wordToToken[k];
|
||||
} else if (lastSeen !== -1) {
|
||||
wordToToken[k] = lastSeen;
|
||||
}
|
||||
}
|
||||
let nextSeen = -1;
|
||||
for (let k = wordToToken.length - 1; k >= 0; k--) {
|
||||
if (wordToToken[k] !== -1) {
|
||||
nextSeen = wordToToken[k];
|
||||
} else if (nextSeen !== -1) {
|
||||
wordToToken[k] = nextSeen;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lastSentenceWordToTokenMap = wordToToken;
|
||||
}
|
||||
|
||||
const mappedIndex =
|
||||
lastSentenceWordToTokenMap && wordIndex < lastSentenceWordToTokenMap.length
|
||||
? lastSentenceWordToTokenMap[wordIndex]
|
||||
: -1;
|
||||
|
||||
if (mappedIndex === -1) return;
|
||||
|
||||
const chosenTokenIndex = mappedIndex;
|
||||
|
||||
const token = lastTokens[chosenTokenIndex];
|
||||
const span = lastSpanNodes[token.spanIndex];
|
||||
if (!span) return;
|
||||
|
||||
const node = token.textNode;
|
||||
if (!node || node.nodeType !== Node.TEXT_NODE) return;
|
||||
|
||||
try {
|
||||
const range = document.createRange();
|
||||
range.setStart(node, token.startOffset);
|
||||
range.setEnd(node, token.endOffset);
|
||||
|
||||
const pageLayer = span.closest(
|
||||
'.react-pdf__Page__textContent'
|
||||
) as HTMLElement | null;
|
||||
if (!pageLayer) return;
|
||||
|
||||
const pageRect = pageLayer.getBoundingClientRect();
|
||||
const rects = Array.from(range.getClientRects());
|
||||
|
||||
rects.forEach((rect) => {
|
||||
const highlight = document.createElement('div');
|
||||
highlight.className = 'pdf-word-highlight-overlay';
|
||||
highlight.style.position = 'absolute';
|
||||
highlight.style.backgroundColor = 'var(--accent)';
|
||||
highlight.style.opacity = '0.4';
|
||||
highlight.style.pointerEvents = 'none';
|
||||
highlight.style.left = `${rect.left - pageRect.left}px`;
|
||||
highlight.style.top = `${rect.top - pageRect.top}px`;
|
||||
highlight.style.width = `${rect.width}px`;
|
||||
highlight.style.height = `${rect.height}px`;
|
||||
highlight.style.zIndex = '2';
|
||||
pageLayer.appendChild(highlight);
|
||||
});
|
||||
} catch {
|
||||
// Ignore range errors
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,9 @@ export interface AppConfigValues {
|
|||
savedVoices: SavedVoices;
|
||||
smartSentenceSplitting: boolean;
|
||||
pdfHighlightEnabled: boolean;
|
||||
pdfWordHighlightEnabled: boolean;
|
||||
epubHighlightEnabled: boolean;
|
||||
epubWordHighlightEnabled: boolean;
|
||||
firstVisit: boolean;
|
||||
documentListState: DocumentListState;
|
||||
}
|
||||
|
|
@ -47,7 +49,9 @@ export const APP_CONFIG_DEFAULTS: AppConfigValues = {
|
|||
savedVoices: {},
|
||||
smartSentenceSplitting: true,
|
||||
pdfHighlightEnabled: true,
|
||||
pdfWordHighlightEnabled: true,
|
||||
epubHighlightEnabled: true,
|
||||
epubWordHighlightEnabled: true,
|
||||
firstVisit: false,
|
||||
documentListState: {
|
||||
sortBy: 'name',
|
||||
|
|
|
|||
|
|
@ -63,6 +63,22 @@ export interface TTSPageTurnEstimate {
|
|||
fraction: number;
|
||||
}
|
||||
|
||||
// Word-level alignment within a single spoken sentence/block
|
||||
export interface TTSSentenceWord {
|
||||
text: string;
|
||||
startSec: number;
|
||||
endSec: number;
|
||||
charStart: number;
|
||||
charEnd: number;
|
||||
}
|
||||
|
||||
// Alignment metadata for a single TTS sentence/block
|
||||
export interface TTSSentenceAlignment {
|
||||
sentence: string;
|
||||
sentenceIndex: number;
|
||||
words: TTSSentenceWord[];
|
||||
}
|
||||
|
||||
// Metadata for an audiobook chapter
|
||||
export interface TTSAudiobookChapter {
|
||||
index: number;
|
||||
|
|
@ -71,4 +87,4 @@ export interface TTSAudiobookChapter {
|
|||
status: 'pending' | 'generating' | 'completed' | 'error';
|
||||
bookId?: string;
|
||||
format?: 'mp3' | 'm4b';
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue