danvics/openreader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Retry errored TTS segments with cooldown and retry-after

Browse source
This commit is contained in:
Richard R 2026-05-14 08:31:40 -06:00
parent df841210a3
commit f300dfb01f
3 changed files with 207 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

184
src/contexts/TTSContext.tsx
View file

@ -52,6 +52,11 @@ import {
upsertWarmAudioEntry,
type WarmAudioCacheEntry,
} from '@/lib/client/tts/audio-warm-cache';
import {
isRetryableSegmentStatus,
resolveSegmentStatusRetryDelayMs,
shouldDeferSegmentRetry,
} from '@/lib/client/tts/segment-retry-policy';
import {
completedEpubBoundarySegment,
resolveEpubBoundaryHandoffStartIndex,
@ -571,6 +576,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
// Track pending preload requests
const preloadRequests = useRef<Map<string, Promise<TTSSegmentPlaybackSource | null>>>(new Map());
const segmentRetryCooldownRef = useRef<Map<string, number>>(new Map());
const warmAudioCacheRef = useRef<Map<string, WarmAudioCacheEntry>>(new Map());
// Track active abort controllers for TTS requests
const activeAbortControllers = useRef<Set<AbortController>>(new Set());
@ -759,6 +765,37 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
return true;
}, [setSegmentManifestCache]);
const setSegmentRetryCooldown = useCallback((cacheKey: string, delayMs: number) => {
const now = Date.now();
const retryAtMs = now + Math.max(0, delayMs);
const cooldowns = segmentRetryCooldownRef.current;
cooldowns.set(cacheKey, retryAtMs);
while (cooldowns.size > AUDIO_CACHE_MAX_ITEMS) {
const oldestKey = cooldowns.keys().next().value;
if (typeof oldestKey !== 'string') break;
cooldowns.delete(oldestKey);
}
}, []);
const waitForRetryDelayWithAbort = useCallback((delayMs: number, signal: AbortSignal): Promise<void> => {
if (!Number.isFinite(delayMs) || delayMs <= 0) return Promise.resolve();
if (signal.aborted) {
return Promise.reject(new DOMException('Aborted', 'AbortError'));
}
return new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
signal.removeEventListener('abort', onAbort);
resolve();
}, delayMs);
const onAbort = () => {
clearTimeout(timeout);
signal.removeEventListener('abort', onAbort);
reject(new DOMException('Aborted', 'AbortError'));
};
signal.addEventListener('abort', onAbort, { once: true });
});
}, []);
const invalidatePlaybackRun = useCallback(() => {
playbackRunIdRef.current += 1;
playbackInFlightRef.current = false;
@ -838,6 +875,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
});
activeAbortControllers.current.clear();
preloadRequests.current.clear();
segmentRetryCooldownRef.current.clear();
clearWarmAudioCache();
}
@ -1509,6 +1547,13 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
return undefined;
}
if (preload) {
const retryAtMs = segmentRetryCooldownRef.current.get(audioCacheKey);
if (shouldDeferSegmentRetry(Date.now(), retryAtMs)) {
return undefined;
}
}
const controller = new AbortController();
activeAbortControllers.current.add(controller);
@ -1529,59 +1574,91 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
try {
onTTSStart();
const persistResult = await resolveSegmentsForPersist([
{
segmentIndex: sentenceIndex,
...(segmentKey ? { segmentKey } : {}),
text: sentence,
locator,
},
]);
const persistSegments = persistResult.segments;
if (persistSegments.length === 0) {
if (!preload) setIsPlaying(false);
return undefined;
}
const ensured = await withRetry(
async () => ensureTtsSegments({
documentId,
settings: {
providerRef: configProviderRef,
providerType: configProviderType,
ttsModel,
voice,
nativeSpeed: effectiveNativeSpeed,
...(providerModelPolicy.supportsInstructions && ttsInstructions ? { ttsInstructions } : {}),
},
segments: persistSegments,
}, reqHeaders, controller.signal),
retryOptions,
);
const maxStatusRetries = preload ? 1 : 3;
let statusAttempt = 0;
const segment = ensured.segments[0];
if (!segment || segment.status !== 'completed' || !segment.audioPresignUrl || !segment.audioFallbackUrl) {
const status = segment?.status ?? 'missing';
const maybeError = segment && 'error' in segment
? (segment as { error?: unknown }).error
: undefined;
if (preload && status === 'pending') {
while (true) {
const persistResult = await resolveSegmentsForPersist([
{
segmentIndex: sentenceIndex,
...(segmentKey ? { segmentKey } : {}),
text: sentence,
locator,
},
]);
const persistSegments = persistResult.segments;
if (persistSegments.length === 0) {
if (!preload) setIsPlaying(false);
return undefined;
}
const detail = typeof maybeError === 'string' && maybeError.trim()
? ` (${maybeError})`
: '';
const ensured = await withRetry(
async () => ensureTtsSegments({
documentId,
settings: {
providerRef: configProviderRef,
providerType: configProviderType,
ttsModel,
voice,
nativeSpeed: effectiveNativeSpeed,
...(providerModelPolicy.supportsInstructions && ttsInstructions ? { ttsInstructions } : {}),
},
segments: persistSegments,
}, reqHeaders, controller.signal),
retryOptions,
);
const segment = ensured.segments[0];
if (segment?.status === 'completed' && segment.audioPresignUrl && segment.audioFallbackUrl) {
segmentRetryCooldownRef.current.delete(audioCacheKey);
setSegmentManifestCache(audioCacheKey, segment);
if (alignmentEnabledForCurrentDoc && segment.alignment) {
sentenceAlignmentCacheRef.current.set(audioCacheKey, segment.alignment);
}
return {
presignUrl: segment.audioPresignUrl,
fallbackUrl: segment.audioFallbackUrl,
manifest: segment,
};
}
const status = segment?.status ?? 'missing';
const retryAfterSeconds =
typeof segment?.error?.retryAfterSeconds === 'number'
? segment.error.retryAfterSeconds
: undefined;
const delayMs = resolveSegmentStatusRetryDelayMs({
attempt: statusAttempt,
retryAfterSeconds,
});
const canRetryStatus = isRetryableSegmentStatus(status) && statusAttempt < maxStatusRetries;
if (preload) {
if (isRetryableSegmentStatus(status)) {
setSegmentRetryCooldown(audioCacheKey, delayMs);
}
if (canRetryStatus) {
statusAttempt += 1;
await waitForRetryDelayWithAbort(delayMs, controller.signal);
continue;
}
return undefined;
}
if (canRetryStatus) {
statusAttempt += 1;
await waitForRetryDelayWithAbort(delayMs, controller.signal);
continue;
}
const detail = (() => {
if (typeof segment?.error?.detail === 'string' && segment.error.detail.trim()) {
return ` (${segment.error.detail})`;
}
return '';
})();
throw new Error(`Failed to prepare segment audio: ${status}${detail}`);
}
setSegmentManifestCache(audioCacheKey, segment);
if (alignmentEnabledForCurrentDoc && segment.alignment) {
sentenceAlignmentCacheRef.current.set(audioCacheKey, segment.alignment);
}
return {
presignUrl: segment.audioPresignUrl,
fallbackUrl: segment.audioFallbackUrl,
manifest: segment,
};
} catch (error) {
if (isAbortLikeError(error)) {
return undefined;
@ -1648,6 +1725,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
currDocPageNumber,
currentReaderType,
setSegmentManifestCache,
setSegmentRetryCooldown,
waitForRetryDelayWithAbort,
isAbortLikeError,
resolveSegmentsForPersist,
]);
@ -2241,6 +2320,11 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
preloadFromOffset(offset + 1);
return;
}
const cooldownRetryAtMs = segmentRetryCooldownRef.current.get(cacheKey);
if (shouldDeferSegmentRetry(Date.now(), cooldownRetryAtMs)) {
preloadFromOffset(offset + 1);
return;
}
const pending = preloadRequests.current.get(requestKey);
if (pending) {
void pending
@ -2366,6 +2450,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
if (seenCandidates.has(requestKey)) continue;
seenCandidates.add(requestKey);
if (segmentManifestCacheRef.current.has(cacheKey)) continue;
const cooldownRetryAtMs = segmentRetryCooldownRef.current.get(cacheKey);
if (shouldDeferSegmentRetry(Date.now(), cooldownRetryAtMs)) continue;
if (preloadRequests.current.has(requestKey)) continue;
uniqueCandidates.push({
sentence: segment.text,
@ -2539,6 +2625,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
if (seen.has(candidate.requestKey)) continue;
seen.add(candidate.requestKey);
if (segmentManifestCacheRef.current.has(candidate.cacheKey)) continue;
const cooldownRetryAtMs = segmentRetryCooldownRef.current.get(candidate.cacheKey);
if (shouldDeferSegmentRetry(Date.now(), cooldownRetryAtMs)) continue;
if (preloadRequests.current.has(candidate.requestKey)) continue;
uniqueCandidates.push(candidate);
}

32
src/lib/client/tts/segment-retry-policy.ts Normal file
View file

@ -0,0 +1,32 @@
export const SEGMENT_STATUS_RETRY_INITIAL_DELAY_MS = 400;
export const SEGMENT_STATUS_RETRY_MAX_DELAY_MS = 5000;
export function isRetryableSegmentStatus(status: string | null | undefined): boolean {
return status === 'pending' || status === 'error';
}
export function resolveSegmentStatusRetryDelayMs(input: {
attempt: number;
retryAfterSeconds?: number | null;
initialDelayMs?: number;
maxDelayMs?: number;
}): number {
const initialDelayMs = input.initialDelayMs ?? SEGMENT_STATUS_RETRY_INITIAL_DELAY_MS;
const maxDelayMs = input.maxDelayMs ?? SEGMENT_STATUS_RETRY_MAX_DELAY_MS;
const retryAfterSeconds = input.retryAfterSeconds;
if (typeof retryAfterSeconds === 'number' && Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0) {
const hintedDelayMs = Math.ceil(retryAfterSeconds * 1000);
return Math.max(initialDelayMs, Math.min(hintedDelayMs, maxDelayMs));
}
const computedDelayMs = Math.min(
initialDelayMs * Math.pow(2, Math.max(0, input.attempt)),
maxDelayMs,
);
return Math.max(initialDelayMs, computedDelayMs);
}
export function shouldDeferSegmentRetry(nowMs: number, retryAtMs: number | undefined): boolean {
return typeof retryAtMs === 'number' && Number.isFinite(retryAtMs) && retryAtMs > nowMs;
}

39
tests/unit/tts-segment-retry-policy.spec.ts Normal file
View file

@ -0,0 +1,39 @@
import { expect, test } from '@playwright/test';
import {
isRetryableSegmentStatus,
resolveSegmentStatusRetryDelayMs,
shouldDeferSegmentRetry,
} from '../../src/lib/client/tts/segment-retry-policy';
test.describe('tts segment retry policy', () => {
test('marks pending/error statuses as retryable', () => {
expect(isRetryableSegmentStatus('pending')).toBe(true);
expect(isRetryableSegmentStatus('error')).toBe(true);
expect(isRetryableSegmentStatus('completed')).toBe(false);
expect(isRetryableSegmentStatus('missing')).toBe(false);
expect(isRetryableSegmentStatus(null)).toBe(false);
});
test('uses exponential backoff when retry-after hint is missing', () => {
expect(resolveSegmentStatusRetryDelayMs({ attempt: 0 })).toBe(400);
expect(resolveSegmentStatusRetryDelayMs({ attempt: 1 })).toBe(800);
expect(resolveSegmentStatusRetryDelayMs({ attempt: 2 })).toBe(1600);
});
test('caps delay by max delay', () => {
expect(resolveSegmentStatusRetryDelayMs({ attempt: 10 })).toBe(5000);
});
test('honors retry-after hint when present', () => {
expect(resolveSegmentStatusRetryDelayMs({ attempt: 0, retryAfterSeconds: 2 })).toBe(2000);
expect(resolveSegmentStatusRetryDelayMs({ attempt: 0, retryAfterSeconds: 0.1 })).toBe(400);
expect(resolveSegmentStatusRetryDelayMs({ attempt: 0, retryAfterSeconds: 99 })).toBe(5000);
});
test('defers retries while cooldown is active', () => {
expect(shouldDeferSegmentRetry(1000, 1001)).toBe(true);
expect(shouldDeferSegmentRetry(1000, 999)).toBe(false);
expect(shouldDeferSegmentRetry(1000, undefined)).toBe(false);
});
});