feat: Remove explicit background state and implementing text prefetching for smoother page/section transitions.
This commit is contained in:
parent
3ede092adc
commit
30ad62aa41
5 changed files with 60 additions and 53 deletions
|
|
@ -82,7 +82,7 @@ interface EPUBContextType {
|
||||||
|
|
||||||
const EPUBContext = createContext<EPUBContextType | undefined>(undefined);
|
const EPUBContext = createContext<EPUBContextType | undefined>(undefined);
|
||||||
|
|
||||||
const EPUB_CONTINUATION_CHARS = 600;
|
const EPUB_CONTINUATION_CHARS = 5000;
|
||||||
|
|
||||||
const cmp = CmpStr.create().setMetric('dice').setFlags('itw');
|
const cmp = CmpStr.create().setMetric('dice').setFlags('itw');
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -101,7 +101,6 @@ interface PDFContextType {
|
||||||
// Create the context
|
// Create the context
|
||||||
const PDFContext = createContext<PDFContextType | undefined>(undefined);
|
const PDFContext = createContext<PDFContextType | undefined>(undefined);
|
||||||
|
|
||||||
const CONTINUATION_PREVIEW_CHARS = 600;
|
|
||||||
const EMPTY_TEXT_RETRY_DELAY_MS = 120;
|
const EMPTY_TEXT_RETRY_DELAY_MS = 120;
|
||||||
const EMPTY_TEXT_MAX_RETRIES = 6;
|
const EMPTY_TEXT_MAX_RETRIES = 6;
|
||||||
|
|
||||||
|
|
@ -283,7 +282,7 @@ export function PDFProvider({ children }: { children: ReactNode }) {
|
||||||
setTTSText(text, {
|
setTTSText(text, {
|
||||||
location: currDocPageNumber,
|
location: currDocPageNumber,
|
||||||
nextLocation: nextPageNumber,
|
nextLocation: nextPageNumber,
|
||||||
nextText: nextText?.slice(0, CONTINUATION_PREVIEW_CHARS),
|
nextText: nextText,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
|
||||||
|
|
@ -36,7 +36,6 @@ import { useMediaSession } from '@/hooks/audio/useMediaSession';
|
||||||
import { useAudioContext } from '@/hooks/audio/useAudioContext';
|
import { useAudioContext } from '@/hooks/audio/useAudioContext';
|
||||||
import { getLastDocumentLocation, setLastDocumentLocation } from '@/lib/client/dexie';
|
import { getLastDocumentLocation, setLastDocumentLocation } from '@/lib/client/dexie';
|
||||||
import { getDocumentProgress, scheduleDocumentProgressSync } from '@/lib/client/api/user-state';
|
import { getDocumentProgress, scheduleDocumentProgressSync } from '@/lib/client/api/user-state';
|
||||||
import { useBackgroundState } from '@/hooks/audio/useBackgroundState';
|
|
||||||
import { withRetry, generateTTS, alignAudio } from '@/lib/client/api/audiobooks';
|
import { withRetry, generateTTS, alignAudio } from '@/lib/client/api/audiobooks';
|
||||||
import { preprocessSentenceForAudio, splitTextToTtsBlocks, splitTextToTtsBlocksEPUB } from '@/lib/shared/nlp';
|
import { preprocessSentenceForAudio, splitTextToTtsBlocks, splitTextToTtsBlocksEPUB } from '@/lib/shared/nlp';
|
||||||
import { isKokoroModel } from '@/lib/shared/kokoro';
|
import { isKokoroModel } from '@/lib/shared/kokoro';
|
||||||
|
|
@ -388,6 +387,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const [currentWordIndex, setCurrentWordIndex] = useState<number | null>(null);
|
const [currentWordIndex, setCurrentWordIndex] = useState<number | null>(null);
|
||||||
const sentencesRef = useRef<string[]>([]);
|
const sentencesRef = useRef<string[]>([]);
|
||||||
const currentIndexRef = useRef(0);
|
const currentIndexRef = useRef(0);
|
||||||
|
const setTextRef = useRef<(text: string, options?: boolean | SetTextOptions) => void>(() => { });
|
||||||
|
const prefetchedLocationTextRef = useRef<Map<string, string>>(new Map());
|
||||||
|
const pendingNextLocationRef = useRef<TTSLocation | undefined>(undefined);
|
||||||
|
|
||||||
const audioUnlockAttemptRef = useRef(0);
|
const audioUnlockAttemptRef = useRef(0);
|
||||||
|
|
||||||
|
|
@ -575,6 +577,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
*/
|
*/
|
||||||
const advance = useCallback(async (backwards = false) => {
|
const advance = useCallback(async (backwards = false) => {
|
||||||
const nextIndex = currentIndex + (backwards ? -1 : 1);
|
const nextIndex = currentIndex + (backwards ? -1 : 1);
|
||||||
|
const movingForward = !backwards && nextIndex >= sentences.length;
|
||||||
|
|
||||||
// Handle within current page bounds
|
// Handle within current page bounds
|
||||||
if (nextIndex < sentences.length && nextIndex >= 0) {
|
if (nextIndex < sentences.length && nextIndex >= 0) {
|
||||||
|
|
@ -584,6 +587,25 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
|
|
||||||
// For EPUB documents, always try to advance to next/prev section
|
// For EPUB documents, always try to advance to next/prev section
|
||||||
if (isEPUB && locationChangeHandlerRef.current) {
|
if (isEPUB && locationChangeHandlerRef.current) {
|
||||||
|
if (movingForward && typeof document !== 'undefined' && document.hidden) {
|
||||||
|
const targetLocation = pendingNextLocationRef.current;
|
||||||
|
if (targetLocation !== undefined) {
|
||||||
|
const bufferKey = normalizeLocationKey(targetLocation);
|
||||||
|
const prefetchedText = prefetchedLocationTextRef.current.get(bufferKey);
|
||||||
|
if (prefetchedText?.trim()) {
|
||||||
|
prefetchedLocationTextRef.current.delete(bufferKey);
|
||||||
|
pendingNextLocationRef.current = undefined;
|
||||||
|
setCurrDocPage(targetLocation);
|
||||||
|
setTextRef.current(prefetchedText, {
|
||||||
|
shouldPause: false,
|
||||||
|
location: targetLocation,
|
||||||
|
});
|
||||||
|
// Ask the viewer to continue turning pages/sections; this may be deferred while hidden.
|
||||||
|
locationChangeHandlerRef.current('next');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
locationChangeHandlerRef.current(nextIndex >= sentences.length ? 'next' : 'prev');
|
locationChangeHandlerRef.current(nextIndex >= sentences.length ? 'next' : 'prev');
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -593,8 +615,27 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
// Handle next/previous page transitions
|
// Handle next/previous page transitions
|
||||||
if ((nextIndex >= sentences.length && currDocPageNumber < currDocPages!) ||
|
if ((nextIndex >= sentences.length && currDocPageNumber < currDocPages!) ||
|
||||||
(nextIndex < 0 && currDocPageNumber > 1)) {
|
(nextIndex < 0 && currDocPageNumber > 1)) {
|
||||||
|
const targetLocation = currDocPageNumber + (nextIndex >= sentences.length ? 1 : -1);
|
||||||
|
|
||||||
|
// In background tabs, page text extraction can be delayed. If we already have
|
||||||
|
// prefetched text for the target page, keep speaking without waiting for viewer callbacks.
|
||||||
|
if (movingForward && typeof document !== 'undefined' && document.hidden) {
|
||||||
|
const bufferKey = normalizeLocationKey(targetLocation);
|
||||||
|
const prefetchedText = prefetchedLocationTextRef.current.get(bufferKey);
|
||||||
|
if (prefetchedText?.trim()) {
|
||||||
|
prefetchedLocationTextRef.current.delete(bufferKey);
|
||||||
|
pendingNextLocationRef.current = undefined;
|
||||||
|
setCurrDocPage(targetLocation);
|
||||||
|
setTextRef.current(prefetchedText, {
|
||||||
|
shouldPause: false,
|
||||||
|
location: targetLocation,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Pass wasPlaying to maintain playback state during page turn
|
// Pass wasPlaying to maintain playback state during page turn
|
||||||
skipToLocation(currDocPageNumber + (nextIndex >= sentences.length ? 1 : -1));
|
skipToLocation(targetLocation);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -692,6 +733,17 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
pageTurnEstimateRef.current = null;
|
pageTurnEstimateRef.current = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Keep the next-location text around so background page turns can continue when
|
||||||
|
// viewer/location callbacks are throttled by the browser.
|
||||||
|
prefetchedLocationTextRef.current.clear();
|
||||||
|
pendingNextLocationRef.current = normalizedOptions.nextLocation;
|
||||||
|
if (normalizedOptions.nextLocation !== undefined && normalizedOptions.nextText?.trim()) {
|
||||||
|
prefetchedLocationTextRef.current.set(
|
||||||
|
normalizeLocationKey(normalizedOptions.nextLocation),
|
||||||
|
normalizedOptions.nextText
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
// Check for blank section after adjustments
|
// Check for blank section after adjustments
|
||||||
if (handleBlankSection(workingText)) return;
|
if (handleBlankSection(workingText)) return;
|
||||||
|
|
||||||
|
|
@ -780,6 +832,10 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
});
|
});
|
||||||
}, [isPlaying, handleBlankSection, abortAudio, splitTextToTtsBlocksLocal, pendingRestoreIndex, isEPUB, smartSentenceSplitting]);
|
}, [isPlaying, handleBlankSection, abortAudio, splitTextToTtsBlocksLocal, pendingRestoreIndex, isEPUB, smartSentenceSplitting]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
setTextRef.current = setText;
|
||||||
|
}, [setText]);
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Toggles the playback state between playing and paused
|
* Toggles the playback state between playing and paused
|
||||||
*/
|
*/
|
||||||
|
|
@ -1408,13 +1464,6 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
}
|
}
|
||||||
}, [activeHowl, audioSpeed, applyPlaybackRateToHowl, isPlaying, startRateWatchdog]);
|
}, [activeHowl, audioSpeed, applyPlaybackRateToHowl, isPlaying, startRateWatchdog]);
|
||||||
|
|
||||||
// Place useBackgroundState after playAudio is defined
|
|
||||||
const isBackgrounded = useBackgroundState({
|
|
||||||
activeHowl,
|
|
||||||
isPlaying,
|
|
||||||
playAudio,
|
|
||||||
});
|
|
||||||
|
|
||||||
// Track the current word index during playback using Howler's seek position
|
// Track the current word index during playback using Howler's seek position
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (!activeHowl || !isPlaying || !currentSentenceAlignment || !currentSentenceAlignment.words.length) {
|
if (!activeHowl || !isPlaying || !currentSentenceAlignment || !currentSentenceAlignment.words.length) {
|
||||||
|
|
@ -1510,7 +1559,6 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
if (isProcessing) return; // Don't proceed if processing audio
|
if (isProcessing) return; // Don't proceed if processing audio
|
||||||
if (!sentences[currentIndex]) return; // Don't proceed if no sentence to play
|
if (!sentences[currentIndex]) return; // Don't proceed if no sentence to play
|
||||||
if (activeHowl) return; // Don't proceed if audio is already playing
|
if (activeHowl) return; // Don't proceed if audio is already playing
|
||||||
if (isBackgrounded) return; // Don't proceed if backgrounded
|
|
||||||
|
|
||||||
// Start playing current sentence
|
// Start playing current sentence
|
||||||
playAudio();
|
playAudio();
|
||||||
|
|
@ -1530,7 +1578,6 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
currentIndex,
|
currentIndex,
|
||||||
sentences,
|
sentences,
|
||||||
activeHowl,
|
activeHowl,
|
||||||
isBackgrounded,
|
|
||||||
playAudio,
|
playAudio,
|
||||||
preloadNextAudio,
|
preloadNextAudio,
|
||||||
abortAudio
|
abortAudio
|
||||||
|
|
@ -1674,7 +1721,6 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
const value = useMemo(() => ({
|
const value = useMemo(() => ({
|
||||||
isPlaying,
|
isPlaying,
|
||||||
isProcessing,
|
isProcessing,
|
||||||
isBackgrounded,
|
|
||||||
currentSentence: sentences[currentIndex] || '',
|
currentSentence: sentences[currentIndex] || '',
|
||||||
currentSentenceAlignment,
|
currentSentenceAlignment,
|
||||||
currentWordIndex,
|
currentWordIndex,
|
||||||
|
|
@ -1700,7 +1746,6 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
}), [
|
}), [
|
||||||
isPlaying,
|
isPlaying,
|
||||||
isProcessing,
|
isProcessing,
|
||||||
isBackgrounded,
|
|
||||||
sentences,
|
sentences,
|
||||||
currentIndex,
|
currentIndex,
|
||||||
currDocPage,
|
currDocPage,
|
||||||
|
|
|
||||||
|
|
@ -1,36 +0,0 @@
|
||||||
import { useState, useEffect } from 'react';
|
|
||||||
import { Howl } from 'howler';
|
|
||||||
|
|
||||||
interface UseBackgroundStateProps {
|
|
||||||
activeHowl: Howl | null;
|
|
||||||
isPlaying: boolean;
|
|
||||||
playAudio: () => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function useBackgroundState({ activeHowl, isPlaying, playAudio }: UseBackgroundStateProps) {
|
|
||||||
const [isBackgrounded, setIsBackgrounded] = useState(false);
|
|
||||||
|
|
||||||
useEffect(() => {
|
|
||||||
const handleVisibilityChange = () => {
|
|
||||||
setIsBackgrounded(document.hidden);
|
|
||||||
if (document.hidden) {
|
|
||||||
// When backgrounded, pause audio but maintain isPlaying state
|
|
||||||
if (activeHowl) {
|
|
||||||
activeHowl.pause();
|
|
||||||
}
|
|
||||||
} else if (isPlaying) {
|
|
||||||
// When returning to foreground, resume from current position
|
|
||||||
if (activeHowl) {
|
|
||||||
activeHowl.play();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
document.addEventListener('visibilitychange', handleVisibilityChange);
|
|
||||||
return () => {
|
|
||||||
document.removeEventListener('visibilitychange', handleVisibilityChange);
|
|
||||||
};
|
|
||||||
}, [isPlaying, activeHowl, playAudio]);
|
|
||||||
|
|
||||||
return isBackgrounded;
|
|
||||||
}
|
|
||||||
|
|
@ -23,7 +23,6 @@ export interface TTSError {
|
||||||
export interface TTSPlaybackState {
|
export interface TTSPlaybackState {
|
||||||
isPlaying: boolean;
|
isPlaying: boolean;
|
||||||
isProcessing: boolean;
|
isProcessing: boolean;
|
||||||
isBackgrounded: boolean;
|
|
||||||
currentSentence: string;
|
currentSentence: string;
|
||||||
currDocPage: TTSLocation;
|
currDocPage: TTSLocation;
|
||||||
currDocPageNumber: number;
|
currDocPageNumber: number;
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue