feat(audio): implement Media Session API and chapter position memory
Implemented comprehensive background playback and position persistence: Media Session API for Background Playback: - Enhanced useMediaSession hook with dynamic metadata support - Added action handlers for play, pause, seek, and chapter navigation - Enabled lock screen media controls on mobile devices - Fixed background tab playback in Firefox and other browsers - System media key support (keyboard, headphones, etc.) - Playback state synchronization (playing/paused) Background Playback Fix: - Removed auto-pause when tab is backgrounded (useBackgroundState) - Audio now continues playing when switching tabs - Works on mobile devices when screen is turned off - Leverages Media Session API for cross-browser compatibility Chapter Position Memory: - Extended LastLocationRow schema with optional chapterIndex - Added getLastChapterIndex() function to retrieve saved position - Updated setLastDocumentLocation() to accept chapter index - HTMLContext automatically saves chapter position on change - HTMLContext restores last chapter on document load - Position persists across app sessions via IndexedDB Technical Details: - Updated database schema (LastLocationRow interface) - Enhanced Media Session with seekbackward/seekforward actions - Added playback state management to TTSContext - Automatic position save using useEffect in HTMLContext - Chapter restoration with bounds checking - Console logging for debugging position save/restore Benefits: - Seamless audiobook listening experience - Continue from where you left off in long documents - Lock screen controls work on iOS/Android - Background playback works in Firefox - No more losing your place in multi-chapter books
This commit is contained in:
parent
b65ebd1d51
commit
3837b215bb
5 changed files with 172 additions and 42 deletions
|
|
@ -4,11 +4,12 @@ import {
|
||||||
createContext,
|
createContext,
|
||||||
useContext,
|
useContext,
|
||||||
useState,
|
useState,
|
||||||
|
useEffect,
|
||||||
ReactNode,
|
ReactNode,
|
||||||
useCallback,
|
useCallback,
|
||||||
useMemo,
|
useMemo,
|
||||||
} from 'react';
|
} from 'react';
|
||||||
import { getHtmlDocument } from '@/lib/dexie';
|
import { getHtmlDocument, getLastChapterIndex, setLastDocumentLocation } from '@/lib/dexie';
|
||||||
import { useTTS } from '@/contexts/TTSContext';
|
import { useTTS } from '@/contexts/TTSContext';
|
||||||
import { detectChapters, type Chapter } from '@/lib/chapterDetection';
|
import { detectChapters, type Chapter } from '@/lib/chapterDetection';
|
||||||
import type { TTSAudiobookChapter } from '@/types/tts';
|
import type { TTSAudiobookChapter } from '@/types/tts';
|
||||||
|
|
@ -59,6 +60,7 @@ export function HTMLProvider({ children }: { children: ReactNode }) {
|
||||||
const [currDocData, setCurrDocData] = useState<string>();
|
const [currDocData, setCurrDocData] = useState<string>();
|
||||||
const [currDocName, setCurrDocName] = useState<string>();
|
const [currDocName, setCurrDocName] = useState<string>();
|
||||||
const [currDocText, setCurrDocText] = useState<string>();
|
const [currDocText, setCurrDocText] = useState<string>();
|
||||||
|
const [currDocId, setCurrDocId] = useState<string>();
|
||||||
|
|
||||||
// Chapter state
|
// Chapter state
|
||||||
const [chapters, setChapters] = useState<Chapter[]>([]);
|
const [chapters, setChapters] = useState<Chapter[]>([]);
|
||||||
|
|
@ -71,6 +73,7 @@ export function HTMLProvider({ children }: { children: ReactNode }) {
|
||||||
setCurrDocData(undefined);
|
setCurrDocData(undefined);
|
||||||
setCurrDocName(undefined);
|
setCurrDocName(undefined);
|
||||||
setCurrDocText(undefined);
|
setCurrDocText(undefined);
|
||||||
|
setCurrDocId(undefined);
|
||||||
setChapters([]);
|
setChapters([]);
|
||||||
setCurrentChapterIndex(0);
|
setCurrentChapterIndex(0);
|
||||||
stop();
|
stop();
|
||||||
|
|
@ -86,6 +89,7 @@ export function HTMLProvider({ children }: { children: ReactNode }) {
|
||||||
try {
|
try {
|
||||||
const doc = await getHtmlDocument(id);
|
const doc = await getHtmlDocument(id);
|
||||||
if (doc) {
|
if (doc) {
|
||||||
|
setCurrDocId(id);
|
||||||
setCurrDocName(doc.name);
|
setCurrDocName(doc.name);
|
||||||
|
|
||||||
// Detect chapters in the document (splits large files)
|
// Detect chapters in the document (splits large files)
|
||||||
|
|
@ -95,16 +99,23 @@ export function HTMLProvider({ children }: { children: ReactNode }) {
|
||||||
// Document has been split into chapters
|
// Document has been split into chapters
|
||||||
console.log(`Document split into ${detectedChapters.length} chapters`);
|
console.log(`Document split into ${detectedChapters.length} chapters`);
|
||||||
setChapters(detectedChapters);
|
setChapters(detectedChapters);
|
||||||
setCurrentChapterIndex(0);
|
|
||||||
|
|
||||||
// Display only the first chapter
|
// Restore last chapter position or start from beginning
|
||||||
const firstChapter = detectedChapters[0];
|
const lastChapterIndex = await getLastChapterIndex(id);
|
||||||
setCurrDocData(firstChapter.text);
|
const startChapterIndex = lastChapterIndex !== null && lastChapterIndex < detectedChapters.length ? lastChapterIndex : 0;
|
||||||
setCurrDocText(firstChapter.text);
|
setCurrentChapterIndex(startChapterIndex);
|
||||||
setTTSText(firstChapter.text);
|
|
||||||
|
// Display the restored or first chapter
|
||||||
|
const startChapter = detectedChapters[startChapterIndex];
|
||||||
|
setCurrDocData(startChapter.text);
|
||||||
|
setCurrDocText(startChapter.text);
|
||||||
|
setTTSText(startChapter.text);
|
||||||
|
|
||||||
|
console.log(`Restored to chapter ${startChapterIndex + 1} of ${detectedChapters.length}`);
|
||||||
} else {
|
} else {
|
||||||
// Small document, no chapters needed
|
// Small document, no chapters needed
|
||||||
setChapters([]);
|
setChapters([]);
|
||||||
|
setCurrentChapterIndex(0);
|
||||||
setCurrDocData(doc.data);
|
setCurrDocData(doc.data);
|
||||||
setCurrDocText(doc.data);
|
setCurrDocText(doc.data);
|
||||||
setTTSText(doc.data);
|
setTTSText(doc.data);
|
||||||
|
|
@ -229,6 +240,15 @@ export function HTMLProvider({ children }: { children: ReactNode }) {
|
||||||
}
|
}
|
||||||
}, [chapters.length, registerLocationChangeHandler, handleLocationChange]);
|
}, [chapters.length, registerLocationChangeHandler, handleLocationChange]);
|
||||||
|
|
||||||
|
// Save chapter position whenever it changes
|
||||||
|
useEffect(() => {
|
||||||
|
if (currDocId && chapters.length > 1) {
|
||||||
|
// Save current chapter index to remember position
|
||||||
|
setLastDocumentLocation(currDocId, String(currentChapterIndex), currentChapterIndex)
|
||||||
|
.catch(error => console.error('Failed to save chapter position:', error));
|
||||||
|
}
|
||||||
|
}, [currDocId, currentChapterIndex, chapters.length]);
|
||||||
|
|
||||||
const totalChapters = chapters.length;
|
const totalChapters = chapters.length;
|
||||||
|
|
||||||
const contextValue = useMemo(() => ({
|
const contextValue = useMemo(() => ({
|
||||||
|
|
|
||||||
|
|
@ -1466,13 +1466,27 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
|
||||||
currentWordIndex
|
currentWordIndex
|
||||||
]);
|
]);
|
||||||
|
|
||||||
// Use media session hook
|
// Media Session API for background playback and lock screen controls
|
||||||
useMediaSession({
|
const { setPlaybackState } = useMediaSession({
|
||||||
togglePlay,
|
metadata: {
|
||||||
skipForward,
|
title: id ? `Document ${String(id).slice(0, 8)}...` : 'Text-to-Speech',
|
||||||
skipBackward,
|
artist: 'OpenReader WebUI',
|
||||||
|
album: 'Audiobook',
|
||||||
|
},
|
||||||
|
controls: {
|
||||||
|
togglePlay,
|
||||||
|
skipForward,
|
||||||
|
skipBackward,
|
||||||
|
// Chapter navigation will be handled by document contexts if available
|
||||||
|
},
|
||||||
|
enabled: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Update playback state when isPlaying changes
|
||||||
|
useEffect(() => {
|
||||||
|
setPlaybackState(isPlaying ? 'playing' : 'paused');
|
||||||
|
}, [isPlaying, setPlaybackState]);
|
||||||
|
|
||||||
// Load last location on mount for both EPUB and PDF
|
// Load last location on mount for both EPUB and PDF
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
if (id) {
|
if (id) {
|
||||||
|
|
|
||||||
|
|
@ -7,23 +7,23 @@ interface UseBackgroundStateProps {
|
||||||
playAudio: () => void;
|
playAudio: () => void;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Hook to track background state and maintain audio context
|
||||||
|
*
|
||||||
|
* NOTE: This hook does NOT pause audio when backgrounded.
|
||||||
|
* Audio continues playing in background tabs and when screen is off.
|
||||||
|
* This is enabled by the Media Session API for cross-browser support.
|
||||||
|
*/
|
||||||
export function useBackgroundState({ activeHowl, isPlaying, playAudio }: UseBackgroundStateProps) {
|
export function useBackgroundState({ activeHowl, isPlaying, playAudio }: UseBackgroundStateProps) {
|
||||||
const [isBackgrounded, setIsBackgrounded] = useState(false);
|
const [isBackgrounded, setIsBackgrounded] = useState(false);
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
const handleVisibilityChange = () => {
|
const handleVisibilityChange = () => {
|
||||||
setIsBackgrounded(document.hidden);
|
setIsBackgrounded(document.hidden);
|
||||||
if (document.hidden) {
|
|
||||||
// When backgrounded, pause audio but maintain isPlaying state
|
// Do NOT pause when backgrounded - allow continuous playback
|
||||||
if (activeHowl) {
|
// Media Session API handles lock screen controls and background playback
|
||||||
activeHowl.pause();
|
// Howler.js will continue playing even when page is hidden
|
||||||
}
|
|
||||||
} else if (isPlaying) {
|
|
||||||
// When returning to foreground, resume from current position
|
|
||||||
if (activeHowl) {
|
|
||||||
activeHowl.play();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
document.addEventListener('visibilitychange', handleVisibilityChange);
|
document.addEventListener('visibilitychange', handleVisibilityChange);
|
||||||
|
|
|
||||||
|
|
@ -1,30 +1,118 @@
|
||||||
'use client';
|
'use client';
|
||||||
|
|
||||||
import { useEffect } from 'react';
|
import { useEffect, useCallback } from 'react';
|
||||||
|
|
||||||
interface MediaControls {
|
export interface MediaSessionMetadata {
|
||||||
|
title: string;
|
||||||
|
artist?: string;
|
||||||
|
album?: string;
|
||||||
|
artwork?: Array<{
|
||||||
|
src: string;
|
||||||
|
sizes?: string;
|
||||||
|
type?: string;
|
||||||
|
}>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface MediaSessionControls {
|
||||||
togglePlay: () => void;
|
togglePlay: () => void;
|
||||||
skipForward: () => void;
|
skipForward: () => void;
|
||||||
skipBackward: () => void;
|
skipBackward: () => void;
|
||||||
|
nextChapter?: () => void;
|
||||||
|
previousChapter?: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface UseMediaSessionProps {
|
||||||
|
metadata?: MediaSessionMetadata;
|
||||||
|
controls: MediaSessionControls;
|
||||||
|
enabled?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Custom hook for managing media session controls
|
* Custom hook for managing Media Session API
|
||||||
* @param controls Object containing media control functions
|
* Enables background playback, lock screen controls, and system media key support
|
||||||
|
*
|
||||||
|
* Features:
|
||||||
|
* - Dynamic metadata (title, artist, artwork)
|
||||||
|
* - Lock screen media controls on mobile
|
||||||
|
* - Background tab playback in Firefox
|
||||||
|
* - System media key support (keyboard, headphones)
|
||||||
|
* - Chapter navigation for audiobooks
|
||||||
*/
|
*/
|
||||||
export function useMediaSession(controls: MediaControls) {
|
export function useMediaSession({ metadata, controls, enabled = true }: UseMediaSessionProps) {
|
||||||
useEffect(() => {
|
// Update playback state
|
||||||
|
const setPlaybackState = useCallback((state: MediaSessionPlaybackState) => {
|
||||||
if ('mediaSession' in navigator) {
|
if ('mediaSession' in navigator) {
|
||||||
navigator.mediaSession.metadata = new MediaMetadata({
|
try {
|
||||||
title: 'Text-to-Speech',
|
navigator.mediaSession.playbackState = state;
|
||||||
artist: 'OpenReader WebUI',
|
} catch (error) {
|
||||||
album: 'Current Document',
|
console.warn('Failed to set playback state:', error);
|
||||||
});
|
}
|
||||||
|
|
||||||
navigator.mediaSession.setActionHandler('play', () => controls.togglePlay());
|
|
||||||
navigator.mediaSession.setActionHandler('pause', () => controls.togglePlay());
|
|
||||||
navigator.mediaSession.setActionHandler('nexttrack', () => controls.skipForward());
|
|
||||||
navigator.mediaSession.setActionHandler('previoustrack', () => controls.skipBackward());
|
|
||||||
}
|
}
|
||||||
}, [controls]);
|
}, []);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!enabled || !('mediaSession' in navigator)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set metadata
|
||||||
|
try {
|
||||||
|
navigator.mediaSession.metadata = new MediaMetadata({
|
||||||
|
title: metadata?.title || 'Text-to-Speech',
|
||||||
|
artist: metadata?.artist || 'OpenReader WebUI',
|
||||||
|
album: metadata?.album || 'Audiobook',
|
||||||
|
artwork: metadata?.artwork || [
|
||||||
|
{
|
||||||
|
src: '/icon.png',
|
||||||
|
sizes: '512x512',
|
||||||
|
type: 'image/png',
|
||||||
|
},
|
||||||
|
],
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
console.warn('Failed to set media session metadata:', error);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Set action handlers
|
||||||
|
try {
|
||||||
|
navigator.mediaSession.setActionHandler('play', controls.togglePlay);
|
||||||
|
navigator.mediaSession.setActionHandler('pause', controls.togglePlay);
|
||||||
|
navigator.mediaSession.setActionHandler('seekbackward', controls.skipBackward);
|
||||||
|
navigator.mediaSession.setActionHandler('seekforward', controls.skipForward);
|
||||||
|
|
||||||
|
// Chapter navigation (optional)
|
||||||
|
if (controls.nextChapter) {
|
||||||
|
navigator.mediaSession.setActionHandler('nexttrack', controls.nextChapter);
|
||||||
|
} else {
|
||||||
|
navigator.mediaSession.setActionHandler('nexttrack', controls.skipForward);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (controls.previousChapter) {
|
||||||
|
navigator.mediaSession.setActionHandler('previoustrack', controls.previousChapter);
|
||||||
|
} else {
|
||||||
|
navigator.mediaSession.setActionHandler('previoustrack', controls.skipBackward);
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.warn('Failed to set media session action handlers:', error);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cleanup on unmount
|
||||||
|
return () => {
|
||||||
|
if ('mediaSession' in navigator) {
|
||||||
|
try {
|
||||||
|
navigator.mediaSession.setActionHandler('play', null);
|
||||||
|
navigator.mediaSession.setActionHandler('pause', null);
|
||||||
|
navigator.mediaSession.setActionHandler('seekbackward', null);
|
||||||
|
navigator.mediaSession.setActionHandler('seekforward', null);
|
||||||
|
navigator.mediaSession.setActionHandler('nexttrack', null);
|
||||||
|
navigator.mediaSession.setActionHandler('previoustrack', null);
|
||||||
|
navigator.mediaSession.metadata = null;
|
||||||
|
} catch (error) {
|
||||||
|
// Ignore cleanup errors
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}, [metadata, controls, enabled]);
|
||||||
|
|
||||||
|
return { setPlaybackState };
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -18,6 +18,7 @@ const JOBS_TABLE = 'jobs' as const;
|
||||||
export interface LastLocationRow {
|
export interface LastLocationRow {
|
||||||
docId: string;
|
docId: string;
|
||||||
location: string;
|
location: string;
|
||||||
|
chapterIndex?: number; // For HTML/text documents with chapters
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ConfigRow {
|
export interface ConfigRow {
|
||||||
|
|
@ -392,9 +393,16 @@ export async function getLastDocumentLocation(docId: string): Promise<string | n
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function setLastDocumentLocation(docId: string, location: string): Promise<void> {
|
export async function setLastDocumentLocation(docId: string, location: string, chapterIndex?: number): Promise<void> {
|
||||||
await withDB(async () => {
|
await withDB(async () => {
|
||||||
await db[LAST_LOCATION_TABLE].put({ docId, location });
|
await db[LAST_LOCATION_TABLE].put({ docId, location, chapterIndex });
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getLastChapterIndex(docId: string): Promise<number | null> {
|
||||||
|
return withDB(async () => {
|
||||||
|
const row = await db[LAST_LOCATION_TABLE].get(docId);
|
||||||
|
return row?.chapterIndex ?? null;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue