From 07e27430c2e1547e7d20f7b5156a3791240691c0 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 6 Jun 2026 06:18:50 -0600 Subject: [PATCH 01/12] feat(tts): add multilingual reader foundation --- src/app/(app)/epub/[id]/page.tsx | 12 +- src/app/(app)/epub/[id]/useEpubDocument.ts | 12 +- src/app/(app)/html/[id]/page.tsx | 13 +- src/app/(app)/pdf/[id]/page.tsx | 8 ++ src/app/(app)/pdf/[id]/usePdfDocument.ts | 6 + src/app/api/audiobook/chapter/route.ts | 3 +- src/app/api/tts/segments/ensure/route.ts | 6 + src/app/api/tts/segments/manifest/route.ts | 3 +- src/components/AudiobookExportModal.tsx | 9 +- src/components/documents/DocumentSettings.tsx | 42 ++++++- src/components/views/HTMLViewer.tsx | 4 +- src/components/views/PDFViewer.tsx | 3 + src/contexts/TTSContext.tsx | 39 +++++- src/hooks/epub/useEPUBHighlighting.ts | 7 +- src/hooks/useDocumentLanguage.ts | 52 ++++++++ src/lib/client/audiobooks/pipeline.ts | 8 +- src/lib/client/epub/epub-word-highlight.ts | 43 +++---- src/lib/client/html/highlight.ts | 52 ++++---- src/lib/client/pdf.ts | 19 ++- src/lib/server/audiobooks/settings.ts | 2 + src/lib/server/tts/segments.ts | 38 +++--- src/lib/shared/document-settings.ts | 9 +- src/lib/shared/language.ts | 119 ++++++++++++++++++ src/lib/shared/nlp.ts | 48 +++---- src/lib/shared/tts-segment-plan.ts | 8 +- src/types/client.ts | 2 + src/types/document-settings.ts | 2 + tests/files/multilingual-sample.txt | 23 ++++ tests/unit/audiobook-settings.vitest.spec.ts | 2 + tests/unit/document-settings.vitest.spec.ts | 22 ++++ tests/unit/epub-word-highlight.vitest.spec.ts | 10 ++ tests/unit/language.vitest.spec.ts | 60 +++++++++ tests/unit/nlp.vitest.spec.ts | 21 ++++ tests/unit/tts-segments.vitest.spec.ts | 15 +++ 34 files changed, 590 insertions(+), 132 deletions(-) create mode 100644 src/hooks/useDocumentLanguage.ts create mode 100644 src/lib/shared/language.ts create mode 100644 tests/files/multilingual-sample.txt create mode 100644 tests/unit/document-settings.vitest.spec.ts create mode 100644 tests/unit/language.vitest.spec.ts diff --git a/src/app/(app)/epub/[id]/page.tsx b/src/app/(app)/epub/[id]/page.tsx index b08d4d4..3cc6ebd 100644 --- a/src/app/(app)/epub/[id]/page.tsx +++ b/src/app/(app)/epub/[id]/page.tsx @@ -19,6 +19,7 @@ import { RateLimitBanner } from '@/components/auth/RateLimitBanner'; import { useAuthRateLimit } from '@/contexts/AuthRateLimitContext'; import { useFeatureFlag } from '@/contexts/RuntimeConfigContext'; import { useUnmountCleanupRef } from '@/hooks/useUnmountCleanupRef'; +import { useDocumentLanguage } from '@/hooks/useDocumentLanguage'; import { ButtonLink } from '@/components/ui'; import { useEpubDocument } from './useEpubDocument'; @@ -37,7 +38,8 @@ export default function EPUBPage() { regenerateChapter: regenerateEPUBChapter, bookRef, } = epubState; - const { stop } = useTTS(); + const { stop, setDocumentLanguage } = useTTS(); + const { language, updateLanguage } = useDocumentLanguage(routeDocumentId); const { isAtLimit } = useAuthRateLimit(); const [error, setError] = useState(null); const [isLoading, setIsLoading] = useState(true); @@ -106,6 +108,10 @@ export default function EPUBPage() { useUnmountCleanupRef(clearCurrDoc); + useEffect(() => { + setDocumentLanguage(language); + }, [language, setDocumentLanguage]); + // Compute available height = viewport - (header height + tts bar height) useEffect(() => { const compute = () => { @@ -249,6 +255,10 @@ export default function EPUBPage() { epub isOpen={activeSidebar === 'settings'} setIsOpen={(isOpen) => setActiveSidebar((prev) => isOpen ? 'settings' : (prev === 'settings' ? null : prev))} + language={language} + onLanguageChange={(nextLanguage) => { + void updateLanguage(nextLanguage); + }} /> () => { diff --git a/src/app/(app)/html/[id]/page.tsx b/src/app/(app)/html/[id]/page.tsx index 09a0b45..ef0ba89 100644 --- a/src/app/(app)/html/[id]/page.tsx +++ b/src/app/(app)/html/[id]/page.tsx @@ -17,6 +17,7 @@ import { AudiobookExportModal } from '@/components/AudiobookExportModal'; import { useAuthRateLimit } from '@/contexts/AuthRateLimitContext'; import { useFeatureFlag } from '@/contexts/RuntimeConfigContext'; import { useUnmountCleanupRef } from '@/hooks/useUnmountCleanupRef'; +import { useDocumentLanguage } from '@/hooks/useDocumentLanguage'; import { ButtonLink } from '@/components/ui'; import type { TTSAudiobookChapter } from '@/types/tts'; import type { AudiobookGenerationSettings } from '@/types/client'; @@ -26,6 +27,7 @@ export default function HTMLPage() { const canExportAudiobook = useFeatureFlag('enableAudiobookExport'); const { id } = useParams(); const router = useRouter(); + const routeDocumentId = typeof id === 'string' ? id : undefined; const htmlState = useHtmlDocument(); const { setCurrentDocument, @@ -38,7 +40,8 @@ export default function HTMLPage() { createFullAudioBook, regenerateChapter, } = htmlState; - const { stop } = useTTS(); + const { stop, setDocumentLanguage } = useTTS(); + const { language, updateLanguage } = useDocumentLanguage(routeDocumentId); const { isAtLimit } = useAuthRateLimit(); const [error, setError] = useState(null); const [isLoading, setIsLoading] = useState(true); @@ -106,6 +109,10 @@ export default function HTMLPage() { useUnmountCleanupRef(clearCurrDoc); + useEffect(() => { + setDocumentLanguage(language); + }, [language, setDocumentLanguage]); + // Compute available height = viewport - (header height + tts bar height) useEffect(() => { const compute = () => { @@ -240,6 +247,10 @@ export default function HTMLPage() { html isOpen={activeSidebar === 'settings'} setIsOpen={(isOpen) => setActiveSidebar((prev) => isOpen ? 'settings' : (prev === 'settings' ? null : prev))} + language={language} + onLanguageChange={(nextLanguage) => { + void updateLanguage(nextLanguage); + }} /> setActiveSidebar((prev) => isOpen ? 'settings' : (prev === 'settings' ? null : prev))} + language={documentSettings.language ?? 'auto'} + onLanguageChange={(language) => { + void updateDocumentSettings({ + ...documentSettings, + schemaVersion: 1, + language, + }); + }} pdf={{ parseStatus, parsedOverlayEnabled, diff --git a/src/app/(app)/pdf/[id]/usePdfDocument.ts b/src/app/(app)/pdf/[id]/usePdfDocument.ts index 8ec76c4..86cebf9 100644 --- a/src/app/(app)/pdf/[id]/usePdfDocument.ts +++ b/src/app/(app)/pdf/[id]/usePdfDocument.ts @@ -91,6 +91,7 @@ export interface PdfDocumentState { parsedDocument?: ParsedPdfDocument | null; locator?: TTSSegmentLocator | null; useBlockGeometryOnly?: boolean; + language?: string; }, ) => void; clearHighlights: () => void; @@ -134,6 +135,7 @@ export function usePdfDocument(): PdfDocumentState { currDocPages, setCurrDocPages, setIsEPUB, + setDocumentLanguage, registerVisualPageChangeHandler, } = useTTS(); const { @@ -156,6 +158,10 @@ export function usePdfDocument(): PdfDocumentState { const [parseProgress, setParseProgress] = useState(null); const [, setActiveParseOpId] = useState(null); const [documentSettings, setDocumentSettings] = useState(DEFAULT_DOCUMENT_SETTINGS); + useEffect(() => { + setDocumentLanguage(documentSettings.language ?? 'auto'); + lastPreparedPlaybackPageRef.current = null; + }, [documentSettings.language, setDocumentLanguage]); const [parsedOverlayEnabled, setParsedOverlayEnabled] = useState(false); const [isAudioCombining] = useState(false); const audiobookAdapter = useMemo(() => createPdfAudiobookSourceAdapter({ diff --git a/src/app/api/audiobook/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts index 70e78f7..31f40e6 100644 --- a/src/app/api/audiobook/chapter/route.ts +++ b/src/app/api/audiobook/chapter/route.ts @@ -432,7 +432,8 @@ export async function POST(request: NextRequest) { normalizedExistingSettings.nativeSpeed !== normalizedIncomingSettings.nativeSpeed || normalizedExistingSettings.postSpeed !== normalizedIncomingSettings.postSpeed || normalizedExistingSettings.format !== normalizedIncomingSettings.format || - (normalizedExistingSettings.ttsInstructions || '') !== (normalizedIncomingSettings.ttsInstructions || ''); + (normalizedExistingSettings.ttsInstructions || '') !== (normalizedIncomingSettings.ttsInstructions || '') || + (normalizedExistingSettings.language || '') !== (normalizedIncomingSettings.language || ''); if (mismatch) { return NextResponse.json({ error: 'Audiobook settings mismatch', settings: normalizedExistingSettings }, { status: 409 }); } diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts index fdd6b2d..e892784 100644 --- a/src/app/api/tts/segments/ensure/route.ts +++ b/src/app/api/tts/segments/ensure/route.ts @@ -33,6 +33,7 @@ import { getUpstreamRetryAfterSeconds, getUpstreamStatus } from '@/lib/server/tt import { userWhisperAlignJob } from '@/lib/server/jobs/user-whisper-align-job'; import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { resolveTtsModelForProvider } from '@/lib/shared/tts-provider-policy'; +import { normalizeLanguageTag } from '@/lib/shared/language'; import { resolveSegmentAudioUrls } from '@/lib/server/tts/segment-audio-urls'; import { createRequestLogger, errorToLog } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; @@ -62,6 +63,8 @@ function parseSettings(value: unknown): TTSSegmentSettings | null { if (typeof rec.voice !== 'string') return null; if (!Number.isFinite(Number(rec.nativeSpeed))) return null; if (rec.ttsInstructions !== undefined && typeof rec.ttsInstructions !== 'string') return null; + if (rec.language !== undefined && typeof rec.language !== 'string') return null; + if (typeof rec.language === 'string' && rec.language.length > 64) return null; return { providerRef: rec.providerRef, @@ -70,6 +73,7 @@ function parseSettings(value: unknown): TTSSegmentSettings | null { voice: rec.voice, nativeSpeed: Number(rec.nativeSpeed), ...(typeof rec.ttsInstructions === 'string' ? { ttsInstructions: rec.ttsInstructions } : {}), + ...(typeof rec.language === 'string' ? { language: normalizeLanguageTag(rec.language) } : {}), }; } @@ -406,6 +410,7 @@ export async function POST(request: NextRequest) { alignment = await userWhisperAlignJob({ audioObjectKey: existing.audioKey, text: segment.text, + lang: effectiveSettings.language, sentenceIndex: segment.original.segmentIndex, }); stageTimings.selfHealAlignMs = Date.now() - alignStartedAt; @@ -674,6 +679,7 @@ export async function POST(request: NextRequest) { alignment = await userWhisperAlignJob({ audioObjectKey: audioKey, text: segment.text, + lang: effectiveSettings.language, sentenceIndex: segment.original.segmentIndex, }); stageTimings.whisperAlignMs = Date.now() - alignStartedAt; diff --git a/src/app/api/tts/segments/manifest/route.ts b/src/app/api/tts/segments/manifest/route.ts index c78e602..686555c 100644 --- a/src/app/api/tts/segments/manifest/route.ts +++ b/src/app/api/tts/segments/manifest/route.ts @@ -71,9 +71,10 @@ function parseSettingsValue(value: unknown): TTSSegmentSettings | null { const nativeSpeed = Number.isFinite(Number(speedSource)) ? Number(speedSource) : 1; const instructionsSource = rec.ttsInstructions ?? rec.instructions; const ttsInstructions = typeof instructionsSource === 'string' ? instructionsSource : ''; + const language = typeof rec.language === 'string' ? rec.language : 'en'; if (!providerRef || !providerType || !ttsModel || !voice) return null; - return { providerRef, providerType, ttsModel, voice, nativeSpeed, ttsInstructions }; + return { providerRef, providerType, ttsModel, voice, nativeSpeed, ttsInstructions, language }; } function locatorFromProjection(row: ManifestGroupRow): TTSSegmentLocator | null { diff --git a/src/components/AudiobookExportModal.tsx b/src/components/AudiobookExportModal.tsx index 0bd98b5..9595c46 100644 --- a/src/components/AudiobookExportModal.tsx +++ b/src/components/AudiobookExportModal.tsx @@ -12,6 +12,7 @@ import { useTTS } from '@/contexts/TTSContext'; import { VoicesControlBase } from '@/components/player/VoicesControlBase'; import { ReaderSidebarShell } from '@/components/reader/ReaderSidebarShell'; import { resolveTtsProviderModelPolicy } from '@/lib/shared/tts-provider-policy'; +import { resolveTtsLanguage } from '@/lib/shared/language'; import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts'; import { Button, Card, IconButton, MenuActionItem, MenuItemsSurface, RangeInput, SharedListboxButton, SharedListboxOption, SharedListboxOptions } from '@/components/ui'; import { @@ -50,7 +51,7 @@ export function AudiobookExportModal({ onRegenerateChapter }: AudiobookExportModalProps) { const { isLoading, isDBReady, providerRef, providerType, ttsModel, ttsInstructions, voice: configVoice, voiceSpeed, audioPlayerSpeed } = useConfig(); - const { availableVoices } = useTTS(); + const { availableVoices, documentLanguage } = useTTS(); const { progress, setProgress, estimatedTimeRemaining } = useTimeEstimation(); const [isGenerating, setIsGenerating] = useState(false); const [chapters, setChapters] = useState([]); @@ -125,8 +126,12 @@ export function AudiobookExportModal({ postSpeed, format, ttsInstructions: providerModelPolicy.supportsInstructions ? ttsInstructions : undefined, + language: resolveTtsLanguage({ + configuredLanguage: documentLanguage, + voice: nextVoice, + }), }; - }, [savedSettings, audiobookVoice, configVoice, availableVoices, providerRef, providerType, ttsModel, ttsInstructions, effectiveNativeSpeed, postSpeed, format, providerModelPolicy.supportsInstructions]); + }, [savedSettings, audiobookVoice, configVoice, availableVoices, providerRef, providerType, ttsModel, ttsInstructions, effectiveNativeSpeed, postSpeed, format, providerModelPolicy.supportsInstructions, documentLanguage]); const fetchExistingChapters = useCallback(async (soft: boolean = false) => { if (soft) { diff --git a/src/components/documents/DocumentSettings.tsx b/src/components/documents/DocumentSettings.tsx index 07442c2..0e1b0ca 100644 --- a/src/components/documents/DocumentSettings.tsx +++ b/src/components/documents/DocumentSettings.tsx @@ -50,6 +50,20 @@ const viewTypeTextMapping = [ { id: 'scroll', name: 'Continuous Scroll' }, ]; +const DOCUMENT_LANGUAGE_OPTIONS = [ + { value: 'auto', label: 'Automatic (voice or metadata)' }, + { value: 'en', label: 'English' }, + { value: 'es', label: 'Spanish' }, + { value: 'fr', label: 'French' }, + { value: 'hi', label: 'Hindi' }, + { value: 'it', label: 'Italian' }, + { value: 'ja', label: 'Japanese' }, + { value: 'pt-BR', label: 'Portuguese (Brazil)' }, + { value: 'zh-CN', label: 'Chinese (Simplified)' }, + { value: 'ar', label: 'Arabic' }, + { value: 'th', label: 'Thai' }, +]; + type RangeSettingProps = { label: string; value: number; @@ -92,11 +106,13 @@ function RangeSetting({ ); } -export function DocumentSettings({ isOpen, setIsOpen, epub, html, pdf }: { +export function DocumentSettings({ isOpen, setIsOpen, epub, html, language, onLanguageChange, pdf }: { isOpen: boolean, setIsOpen: (isOpen: boolean) => void, epub?: boolean, html?: boolean, + language?: string, + onLanguageChange?: (language: string) => void, pdf?: { parseStatus: PdfParseStatus | null; parsedOverlayEnabled: boolean; @@ -151,6 +167,30 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html, pdf }: { panelClassName="w-full sm:w-[30rem]" >
+ {language && onLanguageChange ? ( +
+ +
+ ) : null} {isPdfMode && pdf && (
void; setAudioPlayerSpeedAndRestart: (speed: number) => void; setVoiceAndRestart: (voice: string) => void; + documentLanguage: string; + resolvedLanguage: string; + setDocumentLanguage: (language: string) => void; clearSegmentCaches: () => void; skipToLocation: (location: TTSLocation, shouldPause?: boolean) => void; registerLocationChangeHandler: (handler: ((location: TTSLocation) => void) | null) => void; // EPUB-only: Handles chapter navigation @@ -234,8 +238,9 @@ const normalizeLocationKey = normalizeTtsLocationKey; const normalizeBlockFingerprint = (text: string): string => { const normalized = preprocessSentenceForAudio(text) + .normalize('NFKC') .toLowerCase() - .replace(/[^a-z0-9\s]/g, ' ') + .replace(/[^\p{L}\p{N}\p{M}\s]/gu, ' ') .replace(/\s+/g, ' ') .trim(); return normalized.slice(0, 200); @@ -249,6 +254,7 @@ const buildCacheKey = ( model: string, providerType: string, instructions: string, + language: string, ) => { return [ `provider=${provider || ''}`, @@ -257,6 +263,7 @@ const buildCacheKey = ( `voice=${voice || ''}`, `speed=${Number.isFinite(speed) ? speed : ''}`, `instructions=${instructions || ''}`, + `language=${language || ''}`, `text=${sentence}`, ].join('|'); }; @@ -271,10 +278,11 @@ const buildScopedSegmentCacheKey = ( model: string, providerType: string, instructions: string, + language: string, segmentKey?: string | null, ) => { return [ - buildCacheKey(sentence, voice, speed, provider, model, providerType, instructions), + buildCacheKey(sentence, voice, speed, provider, model, providerType, instructions, language), `segmentKey=${segmentKey || ''}`, `locator=${segmentKey ? '' : buildLocatorRequestKey(locator)}`, `segmentIndex=${segmentKey ? '' : segmentIndex}`, @@ -567,6 +575,11 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement const [voice, setVoice] = useState(configVoice); const [ttsModel, setTTSModel] = useState(configTTSModel); const [ttsInstructions, setTTSInstructions] = useState(configTTSInstructions); + const [documentLanguage, setDocumentLanguage] = useState('auto'); + const resolvedLanguage = useMemo( + () => resolveTtsLanguage({ configuredLanguage: documentLanguage, voice }), + [documentLanguage, voice], + ); const providerModelPolicy = useMemo( () => resolveTtsProviderModelPolicy({ providerRef: configProviderRef, @@ -1231,6 +1244,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement maxBlockLength: ttsSegmentMaxBlockLength, keyPrefix: buildSegmentKeyPrefix(documentId, activeReaderType), enforceSourceBoundaries: activeReaderType === 'pdf' && currentUnits !== null && currentUnits.length > 0, + language: resolvedLanguage, }); const currentSegments = plan.segments.filter((segment) => currentSourceKeySet.has(segment.ownerSourceKey)); const newSentences = currentSegments.map((segment) => segment.text); @@ -1383,6 +1397,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement currDocPage, documentId, ttsSegmentMaxBlockLength, + resolvedLanguage, clearPendingEpubJump, ]); @@ -1496,6 +1511,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement voice, effectiveNativeSpeed, providerModelPolicy.supportsInstructions ? ttsInstructions : '', + resolvedLanguage, ttsSegmentMaxBlockLength, ].join('|'); @@ -1516,6 +1532,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement effectiveNativeSpeed, providerModelPolicy.supportsInstructions, ttsInstructions, + resolvedLanguage, ttsSegmentMaxBlockLength, clearPendingEpubJump, bumpEpubPreloadGeneration, @@ -1589,6 +1606,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement ttsModel, configProviderType, providerModelPolicy.supportsInstructions ? ttsInstructions : '', + resolvedLanguage, segmentKey, ); @@ -1669,6 +1687,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement voice, nativeSpeed: effectiveNativeSpeed, ...(providerModelPolicy.supportsInstructions && ttsInstructions ? { ttsInstructions } : {}), + language: resolvedLanguage, }, segments: persistSegments, }, reqHeaders, controller.signal), @@ -1772,6 +1791,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement effectiveNativeSpeed, ttsModel, ttsInstructions, + resolvedLanguage, openApiKey, openApiBaseUrl, configProviderRef, @@ -2253,6 +2273,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement ttsModel, configProviderType, providerModelPolicy.supportsInstructions ? ttsInstructions : '', + resolvedLanguage, playbackSegment?.key, ); const cachedAlignment = sentenceAlignmentCacheRef.current.get(alignmentKey); @@ -2290,6 +2311,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement configProviderType, providerModelPolicy.supportsInstructions, ttsInstructions, + resolvedLanguage, isEPUB, currDocPage, currDocPageNumber, @@ -2388,6 +2410,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement ttsModel, configProviderType, providerModelPolicy.supportsInstructions ? ttsInstructions : '', + resolvedLanguage, nextSegment?.key, ); if (segmentManifestCacheRef.current.has(cacheKey)) { @@ -2498,6 +2521,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement readerType: 'epub', maxBlockLength: ttsSegmentMaxBlockLength, keyPrefix: buildSegmentKeyPrefix(documentId, 'epub'), + language: resolvedLanguage, }); const uniqueCandidates: Array = []; const seenCandidates = new Set(); @@ -2520,6 +2544,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement ttsModel, configProviderType, providerModelPolicy.supportsInstructions ? ttsInstructions : '', + resolvedLanguage, segment.key, ); if (seenCandidates.has(requestKey)) continue; @@ -2564,6 +2589,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement voice, nativeSpeed: effectiveNativeSpeed, ...(providerModelPolicy.supportsInstructions && ttsInstructions ? { ttsInstructions } : {}), + language: resolvedLanguage, }, segments: persistPayload, }, reqHeaders, controller.signal), @@ -2660,6 +2686,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement ttsModel, configProviderType, providerModelPolicy.supportsInstructions ? ttsInstructions : '', + resolvedLanguage, plannedSegment?.key, ); const requestKey = buildSegmentRequestKey(locator, sentenceIndex, sentence, plannedSegment?.key); @@ -2691,6 +2718,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement ttsModel, configProviderType, providerModelPolicy.supportsInstructions ? ttsInstructions : '', + resolvedLanguage, segment.key, ); const requestKey = buildSegmentRequestKey(segmentLocator, index, sentence, segment.key); @@ -2755,6 +2783,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement voice, nativeSpeed: effectiveNativeSpeed, ...(providerModelPolicy.supportsInstructions && ttsInstructions ? { ttsInstructions } : {}), + language: resolvedLanguage, }, segments: persistPayload, }, reqHeaders, controller.signal), @@ -2841,6 +2870,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement openApiBaseUrl, providerModelPolicy.supportsInstructions, ttsInstructions, + resolvedLanguage, onTTSStart, onTTSComplete, processSentence, @@ -3145,6 +3175,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement setSpeedAndRestart, setAudioPlayerSpeedAndRestart, setVoiceAndRestart, + documentLanguage, + resolvedLanguage, + setDocumentLanguage, clearSegmentCaches, skipToLocation, registerLocationChangeHandler, @@ -3176,6 +3209,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement setSpeedAndRestart, setAudioPlayerSpeedAndRestart, setVoiceAndRestart, + documentLanguage, + resolvedLanguage, clearSegmentCaches, skipToLocation, registerLocationChangeHandler, diff --git a/src/hooks/epub/useEPUBHighlighting.ts b/src/hooks/epub/useEPUBHighlighting.ts index 8255074..8c31d34 100644 --- a/src/hooks/epub/useEPUBHighlighting.ts +++ b/src/hooks/epub/useEPUBHighlighting.ts @@ -30,6 +30,7 @@ type UseEpubHighlightingParams = { currentWordHighlightCfiRef: MutableRefObject; renderedTextMapsRef: MutableRefObject; wordHighlightMapCacheRef: MutableRefObject; + language?: string; }; type UseEpubHighlightingResult = { @@ -52,6 +53,7 @@ export function useEPUBHighlighting({ currentWordHighlightCfiRef, renderedTextMapsRef, wordHighlightMapCacheRef, + language, }: UseEpubHighlightingParams): UseEpubHighlightingResult { const clearWordHighlights = useCallback(() => { if (!renditionRef.current) return; @@ -116,9 +118,9 @@ export function useEPUBHighlighting({ const resolved = resolveVisibleSegmentRange(renderedTextMapsRef.current, segment); if (!resolved || segment.startAnchor.sourceKey !== resolved.map.sourceKey) return; - const cacheKey = buildWordHighlightCacheKey(segment, alignment); + const cacheKey = buildWordHighlightCacheKey(segment, alignment, language); if (wordHighlightMapCacheRef.current?.key !== cacheKey) { - const tokens = tokenizeCanonicalSegment(segment); + const tokens = tokenizeCanonicalSegment(segment, language); wordHighlightMapCacheRef.current = { key: cacheKey, tokens, @@ -162,6 +164,7 @@ export function useEPUBHighlighting({ renderedTextMapsRef, renditionRef, wordHighlightMapCacheRef, + language, ]); const setRenderedTextMaps = useCallback((maps: EpubRenderedTextMap[]) => { diff --git a/src/hooks/useDocumentLanguage.ts b/src/hooks/useDocumentLanguage.ts new file mode 100644 index 0000000..f8474ab --- /dev/null +++ b/src/hooks/useDocumentLanguage.ts @@ -0,0 +1,52 @@ +'use client'; + +import { useCallback, useEffect, useState } from 'react'; + +import { getDocumentSettings, putDocumentSettings } from '@/lib/client/api/documents'; +import { mergeDocumentSettings } from '@/lib/shared/document-settings'; +import { DEFAULT_DOCUMENT_SETTINGS, type DocumentSettings } from '@/types/document-settings'; + +export function useDocumentLanguage(documentId: string | undefined): { + language: string; + updateLanguage: (language: string) => Promise; +} { + const [settings, setSettings] = useState(DEFAULT_DOCUMENT_SETTINGS); + + useEffect(() => { + setSettings(DEFAULT_DOCUMENT_SETTINGS); + if (!documentId) return; + + const controller = new AbortController(); + void getDocumentSettings(documentId, { signal: controller.signal }) + .then((response) => { + setSettings(mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, response.settings)); + }) + .catch((error) => { + if (error instanceof DOMException && error.name === 'AbortError') return; + console.warn('Failed to load document language, using automatic detection:', error); + }); + + return () => controller.abort(); + }, [documentId]); + + const updateLanguage = useCallback(async (language: string): Promise => { + if (!documentId) return; + const next = mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, { + ...settings, + schemaVersion: 1, + language, + }); + setSettings(next); + try { + const response = await putDocumentSettings(documentId, next); + setSettings(mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, response.settings)); + } catch (error) { + console.warn('Failed to persist document language:', error); + } + }, [documentId, settings]); + + return { + language: settings.language ?? 'auto', + updateLanguage, + }; +} diff --git a/src/lib/client/audiobooks/pipeline.ts b/src/lib/client/audiobooks/pipeline.ts index e9e8675..d2e7c4e 100644 --- a/src/lib/client/audiobooks/pipeline.ts +++ b/src/lib/client/audiobooks/pipeline.ts @@ -12,6 +12,7 @@ import type { TTSAudiobookChapter, TTSAudiobookFormat, } from '@/types/tts'; +import { normalizeTextForTts } from '@/lib/shared/nlp'; export interface PreparedAudiobookChapter { index: number; @@ -109,7 +110,10 @@ export async function runAudiobookGeneration({ maxDelay: 300, }, }: RunAudiobookGenerationOptions): Promise { - const chapters = await adapter.prepareChapters(); + const chapters = (await adapter.prepareChapters()).map((chapter) => ({ + ...chapter, + text: normalizeTextForTts(chapter.text, { language: settings?.language }), + })); const totalLength = chapters.reduce((sum, chapter) => sum + chapter.text.trim().length, 0); if (totalLength === 0) { throw new Error(adapter.noContentMessage); @@ -228,7 +232,7 @@ export async function regenerateAudiobookChapter({ }, }: RegenerateAudiobookChapterOptions): Promise { const chapter = await adapter.prepareChapter(chapterIndex); - const trimmedText = chapter.text.trim(); + const trimmedText = normalizeTextForTts(chapter.text, { language: settings?.language }).trim(); if (!trimmedText) { throw new Error(adapter.noContentMessage); } diff --git a/src/lib/client/epub/epub-word-highlight.ts b/src/lib/client/epub/epub-word-highlight.ts index a249b7e..5e90cda 100644 --- a/src/lib/client/epub/epub-word-highlight.ts +++ b/src/lib/client/epub/epub-word-highlight.ts @@ -1,5 +1,6 @@ import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; import type { TTSSentenceAlignment } from '@/types/tts'; +import { normalizeUnicodeToken, segmentWords } from '@/lib/shared/language'; export type EpubCanonicalWordToken = { norm: string; @@ -8,35 +9,19 @@ export type EpubCanonicalWordToken = { }; export const normalizeWordForHighlight = (text: string): string => - text - .toLowerCase() - .replace(/[^a-z0-9]+/g, ''); + normalizeUnicodeToken(text); -export const tokenizeCanonicalSegment = (segment: CanonicalTtsSegment): EpubCanonicalWordToken[] => { - const tokens: EpubCanonicalWordToken[] = []; - const wordRegex = /\S+/g; - let match: RegExpExecArray | null; - - while ((match = wordRegex.exec(segment.text)) !== null) { - const raw = match[0]; - const leading = raw.match(/^[^A-Za-z0-9]*/)?.[0].length ?? 0; - const trailing = raw.match(/[^A-Za-z0-9]*$/)?.[0].length ?? 0; - const start = match.index + leading; - const end = match.index + raw.length - trailing; - if (end <= start) continue; - - const norm = normalizeWordForHighlight(raw.slice(leading, raw.length - trailing)); - if (!norm) continue; - - tokens.push({ - norm, - sourceStart: segment.startAnchor.offset + start, - sourceEnd: segment.startAnchor.offset + end, - }); - } - - return tokens; -}; +export const tokenizeCanonicalSegment = ( + segment: CanonicalTtsSegment, + language?: string, +): EpubCanonicalWordToken[] => + segmentWords(segment.text, language) + .map((token) => ({ + norm: normalizeWordForHighlight(token.text), + sourceStart: segment.startAnchor.offset + token.start, + sourceEnd: segment.startAnchor.offset + token.end, + })) + .filter((token) => Boolean(token.norm)); export const buildMonotonicWordToTokenMap = ( alignmentWords: TTSSentenceAlignment['words'], @@ -105,10 +90,12 @@ export const buildMonotonicWordToTokenMap = ( export const buildWordHighlightCacheKey = ( segment: CanonicalTtsSegment, alignment: TTSSentenceAlignment, + language?: string, ): string => [ segment.key, segment.text.length, + language || '', alignment.words.length, alignment.words.map((word) => normalizeWordForHighlight(word.text)).join('|'), ].join('::'); diff --git a/src/lib/client/html/highlight.ts b/src/lib/client/html/highlight.ts index 104765d..db26cc2 100644 --- a/src/lib/client/html/highlight.ts +++ b/src/lib/client/html/highlight.ts @@ -18,6 +18,7 @@ */ import { CmpStr } from 'cmpstr'; import type { TTSSentenceAlignment } from '@/types/tts'; +import { normalizeUnicodeToken, segmentWords } from '@/lib/shared/language'; export const HTML_SENTENCE_CLASS = 'openreader-html-highlight-sentence'; export const HTML_WORD_CLASS = 'openreader-html-highlight-word'; @@ -53,21 +54,11 @@ interface SentenceState { let sentenceState: SentenceState | null = null; function normalizeWord(word: string): string { - return word - .toLowerCase() - .replace(/[\p{P}\p{S}]+/gu, '') - .trim(); + return normalizeUnicodeToken(word); } -function tokenizePattern(pattern: string): string[] { - const out: string[] = []; - const wordRe = /\S+/g; - let m: RegExpExecArray | null; - while ((m = wordRe.exec(pattern)) !== null) { - const norm = normalizeWord(m[0]); - if (norm) out.push(norm); - } - return out; +function tokenizePattern(pattern: string, language?: string): string[] { + return segmentWords(pattern, language).map((token) => normalizeWord(token.text)).filter(Boolean); } function unwrap(span: HTMLSpanElement): void { @@ -108,7 +99,11 @@ function isHighlightWrapper(node: Node | null): node is HTMLSpanElement { return el.classList.contains(HTML_SENTENCE_CLASS) || el.classList.contains(HTML_WORD_CLASS); } -function collectDomTokens(root: HTMLElement, opts: { skipHighlightWraps: boolean } = { skipHighlightWraps: true }): DomToken[] { +function collectDomTokens( + root: HTMLElement, + language?: string, + opts: { skipHighlightWraps: boolean } = { skipHighlightWraps: true }, +): DomToken[] { const walker = document.createTreeWalker(root, NodeFilter.SHOW_TEXT, { acceptNode(node) { const parent = node.parentElement; @@ -131,15 +126,13 @@ function collectDomTokens(root: HTMLElement, opts: { skipHighlightWraps: boolean while (current) { const textNode = current as Text; const text = textNode.nodeValue || ''; - const wordRe = /\S+/g; - let m: RegExpExecArray | null; - while ((m = wordRe.exec(text)) !== null) { - const norm = normalizeWord(m[0]); + for (const token of segmentWords(text, language)) { + const norm = normalizeWord(token.text); if (!norm) continue; tokens.push({ textNode, - startOffset: m.index, - endOffset: m.index + m[0].length, + startOffset: token.start, + endOffset: token.end, norm, }); } @@ -153,7 +146,7 @@ function collectDomTokens(root: HTMLElement, opts: { skipHighlightWraps: boolean * wrap is applied; lets us index just the words *within* the highlighted * sentence rather than the whole document). */ -function collectTokensInsideWraps(wraps: HTMLSpanElement[]): DomToken[] { +function collectTokensInsideWraps(wraps: HTMLSpanElement[], language?: string): DomToken[] { const tokens: DomToken[] = []; for (const wrap of wraps) { const walker = document.createTreeWalker(wrap, NodeFilter.SHOW_TEXT); @@ -161,15 +154,13 @@ function collectTokensInsideWraps(wraps: HTMLSpanElement[]): DomToken[] { while (current) { const t = current as Text; const text = t.nodeValue || ''; - const wordRe = /\S+/g; - let m: RegExpExecArray | null; - while ((m = wordRe.exec(text)) !== null) { - const norm = normalizeWord(m[0]); + for (const token of segmentWords(text, language)) { + const norm = normalizeWord(token.text); if (!norm) continue; tokens.push({ textNode: t, - startOffset: m.index, - endOffset: m.index + m[0].length, + startOffset: token.start, + endOffset: token.end, norm, }); } @@ -260,14 +251,15 @@ function wrapTokenRange(tokens: DomToken[], start: number, end: number, classNam export function highlightHtmlSentence( container: HTMLElement | null | undefined, sentence: string | null | undefined, + language?: string, ): boolean { clearHtmlSentenceHighlight(); if (!container || !sentence?.trim()) return false; - const patternTokens = tokenizePattern(sentence); + const patternTokens = tokenizePattern(sentence, language); if (!patternTokens.length) return false; - const domTokens = collectDomTokens(container); + const domTokens = collectDomTokens(container, language); if (!domTokens.length) return false; const win = findBestWindow(domTokens, patternTokens); @@ -280,7 +272,7 @@ export function highlightHtmlSentence( // can look up individual word tokens without re-walking the doc. sentenceState = { sentence, - wordTokens: collectTokensInsideWraps(sentenceWraps), + wordTokens: collectTokensInsideWraps(sentenceWraps, language), alignment: null, wordToToken: null, }; diff --git a/src/lib/client/pdf.ts b/src/lib/client/pdf.ts index 9727d4b..32a1a7e 100644 --- a/src/lib/client/pdf.ts +++ b/src/lib/client/pdf.ts @@ -5,6 +5,7 @@ import type { TTSSentenceAlignment } from '@/types/tts'; import type { ParsedPdfDocument, ParsedPdfPage } from '@/types/parsed-pdf'; import { CmpStr } from 'cmpstr'; import type { TTSSegmentLocator } from '@/types/client'; +import { normalizeUnicodeToken, segmentWords } from '@/lib/shared/language'; const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); @@ -181,10 +182,7 @@ function getOrCreateHighlightLayer(span: HTMLElement): { } const normalizeWordForMatch = (text: string): string => - text - .trim() - .replace(/^[^A-Za-z0-9]+|[^A-Za-z0-9]+$/g, '') - .toLowerCase(); + normalizeUnicodeToken(text); // Highlighting functions let highlightPatternSeq = 0; @@ -193,6 +191,7 @@ type HighlightPatternOptions = { parsedDocument?: ParsedPdfDocument | null; locator?: TTSSegmentLocator | null; useBlockGeometryOnly?: boolean; + language?: string; }; function getHighlightLayerForPage(pageElement: HTMLElement): { @@ -451,17 +450,13 @@ export function highlightPattern( const textNode = node as Text; const textContent = textNode.textContent || ''; - const wordRegex = /\S+/g; - let match: RegExpExecArray | null; - - while ((match = wordRegex.exec(textContent)) !== null) { - const word = match[0]; + for (const token of segmentWords(textContent, options?.language)) { tokens.push({ spanIndex, textNode, - text: word, - startOffset: match.index, - endOffset: match.index + word.length, + text: token.text, + startOffset: token.start, + endOffset: token.end, }); } }); diff --git a/src/lib/server/audiobooks/settings.ts b/src/lib/server/audiobooks/settings.ts index cf74673..cdef976 100644 --- a/src/lib/server/audiobooks/settings.ts +++ b/src/lib/server/audiobooks/settings.ts @@ -9,6 +9,7 @@ import type { AudiobookGenerationSettings } from '@/types/client'; import type { TTSAudiobookFormat } from '@/types/tts'; import { resolveEffectiveTtsInstructions } from '@/lib/server/admin/tts-instructions'; import { resolvePreferredSharedProviderSlug } from '@/lib/shared/shared-provider-selection'; +import { normalizeLanguageTag } from '@/lib/shared/language'; function isAudiobookFormat(value: unknown): value is TTSAudiobookFormat { return value === 'mp3' || value === 'm4b'; @@ -68,6 +69,7 @@ export function coerceAudiobookGenerationSettings( postSpeed, format, ...(typeof record.ttsInstructions === 'string' ? { ttsInstructions: record.ttsInstructions } : {}), + ...(typeof record.language === 'string' ? { language: normalizeLanguageTag(record.language) } : {}), }; const migrated = diff --git a/src/lib/server/tts/segments.ts b/src/lib/server/tts/segments.ts index d1bb3c7..e3618ae 100644 --- a/src/lib/server/tts/segments.ts +++ b/src/lib/server/tts/segments.ts @@ -3,6 +3,7 @@ import { mkdtemp, rm, writeFile } from 'fs/promises'; import { join } from 'path'; import { tmpdir } from 'os'; import { preprocessSentenceForAudio } from '@/lib/shared/nlp'; +import { normalizeUnicodeToken, segmentWords } from '@/lib/shared/language'; import { locatorIdentityKey } from '@/lib/shared/tts-locator'; import { ffprobeAudio } from '@/lib/server/audiobooks/chapters'; import type { @@ -32,6 +33,7 @@ function settingsCanonical(settings: TTSSegmentSettings): string { voice: settings.voice, speed: Number.isFinite(Number(settings.nativeSpeed)) ? Number(settings.nativeSpeed) : 1, instructions: settings.ttsInstructions || '', + language: settings.language || 'en', format: 'mp3', }); } @@ -48,6 +50,7 @@ export function buildTtsSegmentSettingsJson(settings: TTSSegmentSettings): TTSSe voice: settings.voice, nativeSpeed: Number.isFinite(Number(settings.nativeSpeed)) ? Number(settings.nativeSpeed) : 1, ttsInstructions: settings.ttsInstructions || '', + language: settings.language || 'en', }; // Postgres jsonb accepts the object directly; SQLite text needs a canonical JSON string. return process.env.POSTGRES_URL ? canonical : settingsCanonical(settings); @@ -242,35 +245,24 @@ export async function probeAudioDurationMsFromBuffer(buffer: Buffer, signal?: Ab } } -function alignWordsToText(sentence: string): Array<{ text: string; charStart: number; charEnd: number }> { - const words = sentence.match(/\S+/g) || []; - const aligned: Array<{ text: string; charStart: number; charEnd: number }> = []; - let cursor = 0; - const lowerSentence = sentence.toLowerCase(); - - for (const token of words) { - const clean = token.trim(); - if (!clean) continue; - const idx = lowerSentence.indexOf(clean.toLowerCase(), cursor); - const start = idx >= 0 ? idx : cursor; - const end = Math.min(sentence.length, start + clean.length); - cursor = Math.max(cursor, end); - aligned.push({ - text: clean, - charStart: start, - charEnd: end, - }); - } - - return aligned; +function alignWordsToText( + sentence: string, + language?: string, +): Array<{ text: string; charStart: number; charEnd: number }> { + return segmentWords(sentence, language).map((token) => ({ + text: token.text, + charStart: token.start, + charEnd: token.end, + })); } export function buildProportionalAlignment(input: { sentence: string; sentenceIndex: number; durationMs: number; + language?: string; }): TTSSentenceAlignment { - const wordsWithOffsets = alignWordsToText(input.sentence); + const wordsWithOffsets = alignWordsToText(input.sentence, input.language); if (wordsWithOffsets.length === 0 || input.durationMs <= 0) { return { sentence: input.sentence, @@ -281,7 +273,7 @@ export function buildProportionalAlignment(input: { const weighted = wordsWithOffsets.map((word) => ({ ...word, - weight: Math.max(1, word.text.replace(/[^a-zA-Z0-9]/g, '').length), + weight: Math.max(1, normalizeUnicodeToken(word.text).length), })); const totalWeight = weighted.reduce((sum, word) => sum + word.weight, 0); diff --git a/src/lib/shared/document-settings.ts b/src/lib/shared/document-settings.ts index 26b45e0..208f76c 100644 --- a/src/lib/shared/document-settings.ts +++ b/src/lib/shared/document-settings.ts @@ -3,6 +3,7 @@ import { type DocumentSettings, } from '@/types/document-settings'; import { PARSED_PDF_BLOCK_KINDS, type ParsedPdfBlockKind } from '@/types/parsed-pdf'; +import { normalizeLanguageTag } from '@/lib/shared/language'; function normalizeSkipKinds(value: unknown): ParsedPdfBlockKind[] { if (!Array.isArray(value)) return [...(DEFAULT_DOCUMENT_SETTINGS.pdf?.skipBlockKinds ?? [])]; @@ -22,6 +23,7 @@ export function mergeDocumentSettings( ): DocumentSettings { const base: DocumentSettings = { schemaVersion: 1, + language: defaults.language || 'auto', pdf: { skipBlockKinds: [...(defaults.pdf?.skipBlockKinds ?? [])], }, @@ -29,12 +31,17 @@ export function mergeDocumentSettings( if (!stored || typeof stored !== 'object') return base; const rec = stored as Record; + const rawLanguage = typeof rec.language === 'string' ? rec.language.trim() : ''; + const language = !rawLanguage || rawLanguage.toLowerCase() === 'auto' + ? 'auto' + : normalizeLanguageTag(rawLanguage, defaults.language || 'en'); const pdf = rec.pdf; - if (!pdf || typeof pdf !== 'object') return base; + if (!pdf || typeof pdf !== 'object') return { ...base, language }; const pdfRec = pdf as Record; return { schemaVersion: 1, + language, pdf: { skipBlockKinds: normalizeSkipKinds(pdfRec.skipBlockKinds), }, diff --git a/src/lib/shared/language.ts b/src/lib/shared/language.ts new file mode 100644 index 0000000..b2e3437 --- /dev/null +++ b/src/lib/shared/language.ts @@ -0,0 +1,119 @@ +export const DEFAULT_TTS_LANGUAGE = 'en'; + +export interface TextToken { + text: string; + start: number; + end: number; +} + +const KOKORO_LANGUAGE_BY_PREFIX: Readonly> = { + af: 'en-US', + am: 'en-US', + bf: 'en-GB', + bm: 'en-GB', + ef: 'es', + em: 'es', + ff: 'fr', + hf: 'hi', + hm: 'hi', + if: 'it', + im: 'it', + jf: 'ja', + jm: 'ja', + pf: 'pt-BR', + pm: 'pt-BR', + zf: 'zh-CN', + zm: 'zh-CN', +}; + +export function normalizeLanguageTag( + language: string | null | undefined, + fallback = DEFAULT_TTS_LANGUAGE, +): string { + const candidate = language?.trim(); + if (!candidate || candidate.toLowerCase() === 'auto') return fallback; + + try { + return Intl.getCanonicalLocales(candidate)[0] ?? fallback; + } catch { + return fallback; + } +} + +export function toBaseLanguageCode(language: string | null | undefined): string { + const normalized = normalizeLanguageTag(language); + try { + return new Intl.Locale(normalized).language; + } catch { + return normalized.split('-')[0]?.toLowerCase() || DEFAULT_TTS_LANGUAGE; + } +} + +export function inferKokoroLanguageFromVoice(voice: string | null | undefined): string | null { + if (!voice?.trim()) return null; + + const languages = new Set( + voice + .split('+') + .map((part) => part.trim().replace(/\([^)]*\)/g, '')) + .map((name) => KOKORO_LANGUAGE_BY_PREFIX[name.slice(0, 2).toLowerCase()]) + .filter((language): language is string => Boolean(language)), + ); + + return languages.size === 1 ? [...languages][0] : null; +} + +export function resolveTtsLanguage(input: { + configuredLanguage?: string | null; + voice?: string | null; +}): string { + const configured = input.configuredLanguage?.trim(); + if (configured && configured.toLowerCase() !== 'auto') { + return normalizeLanguageTag(configured); + } + + return inferKokoroLanguageFromVoice(input.voice) ?? DEFAULT_TTS_LANGUAGE; +} + +export function normalizeUnicodeToken(text: string): string { + return text + .normalize('NFKC') + .toLocaleLowerCase() + .replace(/[^\p{L}\p{N}\p{M}]+/gu, ''); +} + +export function segmentSentences(text: string, language?: string | null): string[] { + const normalizedLanguage = normalizeLanguageTag(language); + try { + return [...new Intl.Segmenter(normalizedLanguage, { granularity: 'sentence' }).segment(text)] + .map(({ segment }) => segment.trim()) + .filter(Boolean); + } catch { + return text.split(/(?<=[.!?。！？؟।])\s*/u).map((segment) => segment.trim()).filter(Boolean); + } +} + +export function segmentWords(text: string, language?: string | null): TextToken[] { + const normalizedLanguage = normalizeLanguageTag(language); + try { + return [...new Intl.Segmenter(normalizedLanguage, { granularity: 'word' }).segment(text)] + .filter((segment) => segment.isWordLike) + .map((segment) => ({ + text: segment.segment, + start: segment.index, + end: segment.index + segment.segment.length, + })); + } catch { + const tokens: TextToken[] = []; + const wordRegex = /\S+/gu; + let match: RegExpExecArray | null; + while ((match = wordRegex.exec(text)) !== null) { + tokens.push({ + text: match[0], + start: match.index, + end: match.index + match[0].length, + }); + } + return tokens; + } +} diff --git a/src/lib/shared/nlp.ts b/src/lib/shared/nlp.ts index b254102..6712116 100644 --- a/src/lib/shared/nlp.ts +++ b/src/lib/shared/nlp.ts @@ -5,13 +5,14 @@ * It handles text preprocessing, sentence splitting, and block creation for optimal TTS processing. */ -import nlp from 'compromise'; +import { normalizeLanguageTag, segmentSentences, toBaseLanguageCode } from '@/lib/shared/language'; export const MAX_BLOCK_LENGTH = 450; const MIN_BLOCK_LENGTH = 50; export interface TtsSplitOptions { maxBlockLength?: number; + language?: string; } function resolveMaxBlockLength(options?: TtsSplitOptions): number { @@ -27,9 +28,9 @@ const splitOversizedText = (text: string, maxLen: number): string[] => { const parts: string[] = []; const MAX_OVERFLOW = maxLen; // allow finishing the sentence up to +maxLen chars - const CLOSERS = new Set(['"', "'", '”', '’', ')', ']', '}']); - const BREAK_CHARS = new Set(['.', '!', '?']); - const SOFT_BREAK_CHARS = new Set([';', ':']); + const CLOSERS = new Set(['"', "'", '”', '’', ')', ']', '}', '」', '』', '】']); + const BREAK_CHARS = new Set(['.', '!', '?', '。', '！', '？', '؟', '।']); + const SOFT_BREAK_CHARS = new Set([';', ':', '；', '：']); const findPunctuationCut = (s: string, limit: number): number | null => { for (let i = limit; i >= 0; i--) { @@ -48,7 +49,7 @@ const splitOversizedText = (text: string, maxLen: number): string[] => { // Allow a boundary at end/whitespace, or common PDF artifact where // the next sentence starts immediately with an uppercase letter. - if (!after || /\s/.test(after) || /[A-Z]/.test(after)) return end; + if (!after || /\s/u.test(after) || /\p{Lu}/u.test(after)) return end; } return null; }; @@ -74,7 +75,7 @@ const splitOversizedText = (text: string, maxLen: number): string[] => { while (cut < s.length && CLOSERS.has(s[cut])) cut++; const after = cut < s.length ? s[cut] : ''; - if (!after || /\s/.test(after) || /[A-Z]/.test(after)) return cut; + if (!after || /\s/u.test(after) || /\p{Lu}/u.test(after)) return cut; } return null; }; @@ -87,7 +88,7 @@ const splitOversizedText = (text: string, maxLen: number): string[] => { let end = i + 1; while (end < s.length && CLOSERS.has(s[end])) end++; const after = end < s.length ? s[end] : ''; - if (!after || /\s/.test(after) || /[A-Z]/.test(after)) return end; + if (!after || /\s/u.test(after) || /\p{Lu}/u.test(after)) return end; } return null; }; @@ -123,8 +124,8 @@ const normalizeSentenceBoundariesForNlp = (text: string): string => { // Insert a space only when it looks like a sentence boundary (lower/digit before, // uppercase after) to avoid breaking abbreviations like "U.S.A". return text - .replace(/([a-z0-9])([.!?])(?=[A-Z])/g, '$1$2 ') - .replace(/([a-z0-9][.!?][\"”’)\]])(?=[A-Z])/g, '$1 '); + .replace(/([\p{Ll}\p{N}])([.!?])(?=\p{Lu})/gu, '$1$2 ') + .replace(/([\p{Ll}\p{N}][.!?]["”’)\]])(?=\p{Lu})/gu, '$1 '); }; /** @@ -136,7 +137,7 @@ const normalizeSentenceBoundariesForNlp = (text: string): string => { export const preprocessSentenceForAudio = (text: string): string => { return text .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') - .replace(/(\w+)-\s+(\w+)/g, '$1$2') // Remove hyphenation + .replace(/([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu, '$1$2') // Remove hyphenation // Remove special character * .replace(/\*/g, '') .replace(/\s+/g, ' ') @@ -151,6 +152,8 @@ export const preprocessSentenceForAudio = (text: string): string => { */ export const splitTextToTtsBlocks = (text: string, options?: TtsSplitOptions): string[] => { const maxBlockLength = resolveMaxBlockLength(options); + const language = normalizeLanguageTag(options?.language); + const blockSeparator = ['ja', 'zh'].includes(toBaseLanguageCode(language)) ? '' : ' '; // Treat double-newlines as paragraph boundaries; single newlines are usually // just PDF line wrapping and should not force sentence/block boundaries. const paragraphs = text.split(/\n{2,}/); @@ -162,8 +165,7 @@ export const splitTextToTtsBlocks = (text: string, options?: TtsSplitOptions): s const cleanedText = normalizeSentenceBoundariesForNlp( preprocessSentenceForAudio(paragraph) ); - const doc = nlp(cleanedText); - const rawSentences = doc.sentences().out('array') as string[]; + const rawSentences = segmentSentences(cleanedText, language); // Merge multi-sentence dialogue enclosed in quotes into single items const mergedSentences = mergeQuotedDialogue(rawSentences); @@ -179,8 +181,8 @@ export const splitTextToTtsBlocks = (text: string, options?: TtsSplitOptions): s blocks.push(currentBlock.trim()); currentBlock = sentencePart; } else { - currentBlock = currentBlock - ? `${currentBlock} ${sentencePart}` + currentBlock = currentBlock + ? `${currentBlock}${blockSeparator}${sentencePart}` : sentencePart; } } @@ -200,6 +202,8 @@ export const splitTextToTtsBlocks = (text: string, options?: TtsSplitOptions): s */ export const splitTextToTtsBlocksEPUB = (text: string, options?: TtsSplitOptions): string[] => { const maxBlockLength = resolveMaxBlockLength(options); + const language = normalizeLanguageTag(options?.language); + const blockSeparator = ['ja', 'zh'].includes(toBaseLanguageCode(language)) ? '' : ' '; const paragraphs = text.split(/\n+/); const blocks: string[] = []; @@ -207,8 +211,7 @@ export const splitTextToTtsBlocksEPUB = (text: string, options?: TtsSplitOptions if (!paragraph.trim()) continue; const cleanedText = preprocessSentenceForAudio(paragraph); - const doc = nlp(cleanedText); - const rawSentences = doc.sentences().out('array') as string[]; + const rawSentences = segmentSentences(cleanedText, language); const mergedSentences = mergeQuotedDialogue(rawSentences); @@ -227,7 +230,7 @@ export const splitTextToTtsBlocksEPUB = (text: string, options?: TtsSplitOptions currentBlock = sentencePart; } else { currentBlock = currentBlock - ? `${currentBlock} ${sentencePart}` + ? `${currentBlock}${blockSeparator}${sentencePart}` : sentencePart; } } @@ -248,8 +251,11 @@ export const splitTextToTtsBlocksEPUB = (text: string, options?: TtsSplitOptions * @param {string} text - The text to process * @returns {string} Normalized text */ -export const normalizeTextForTts = (text: string, options?: TtsSplitOptions): string => - splitTextToTtsBlocks(text, options).join(' '); +export const normalizeTextForTts = (text: string, options?: TtsSplitOptions): string => { + const language = normalizeLanguageTag(options?.language); + const separator = ['ja', 'zh'].includes(toBaseLanguageCode(language)) ? '' : ' '; + return splitTextToTtsBlocks(text, options).join(separator); +}; // Helper functions to merge quoted dialogue across sentences const countDoubleQuotes = (s: string): number => { @@ -265,8 +271,8 @@ const countNonApostropheSingleQuotes = (s: string): number => { if (ch === "'" || ch === '‘' || ch === '’') { const prev = i > 0 ? s[i - 1] : ''; const next = i + 1 < s.length ? s[i + 1] : ''; - const isPrevAlphaNum = /[A-Za-z0-9]/.test(prev); - const isNextAlphaNum = /[A-Za-z0-9]/.test(next); + const isPrevAlphaNum = /[\p{L}\p{N}\p{M}]/u.test(prev); + const isNextAlphaNum = /[\p{L}\p{N}\p{M}]/u.test(next); // Treat as a real quote mark only when it's not clearly an apostrophe // between two alphanumeric characters (e.g., don't, WizardLM’s). if (!(isPrevAlphaNum && isNextAlphaNum)) { diff --git a/src/lib/shared/tts-segment-plan.ts b/src/lib/shared/tts-segment-plan.ts index b33ca15..6f76e31 100644 --- a/src/lib/shared/tts-segment-plan.ts +++ b/src/lib/shared/tts-segment-plan.ts @@ -2,7 +2,7 @@ import { preprocessSentenceForAudio, splitTextToTtsBlocks, splitTextToTtsBlocksE import type { TTSSegmentLocator } from '@/types/client'; import type { ReaderType } from '@/types/user-state'; -export const TTS_SEGMENT_PLAN_VERSION = 'tts-segment-plan-v1'; +export const TTS_SEGMENT_PLAN_VERSION = 'tts-segment-plan-v2'; export interface CanonicalTtsSourceUnit { sourceKey: string; @@ -38,6 +38,7 @@ export interface CanonicalTtsSegmentPlanOptions { maxBlockLength?: number; keyPrefix?: string; enforceSourceBoundaries?: boolean; + language?: string; } interface PreparedSourceUnit { @@ -215,7 +216,10 @@ export function planCanonicalTtsSegments( } const canonicalText = textParts.join(''); - const splitOptions = { maxBlockLength: options.maxBlockLength }; + const splitOptions = { + maxBlockLength: options.maxBlockLength, + language: options.language, + }; const splitIntoBlocks = (text: string): string[] => readerType === 'epub' ? splitTextToTtsBlocksEPUB(text, splitOptions) diff --git a/src/types/client.ts b/src/types/client.ts index 0b94363..f9c7440 100644 --- a/src/types/client.ts +++ b/src/types/client.ts @@ -45,6 +45,7 @@ export interface AudiobookGenerationSettings { postSpeed: number; format: TTSAudiobookFormat; ttsInstructions?: string; + language?: string; } export interface CreateChapterPayload { @@ -70,6 +71,7 @@ export interface TTSSegmentSettings { voice: string; nativeSpeed: number; ttsInstructions?: string; + language?: string; } type TTSReaderType = 'pdf' | 'epub' | 'html'; diff --git a/src/types/document-settings.ts b/src/types/document-settings.ts index 6908f33..1d6b358 100644 --- a/src/types/document-settings.ts +++ b/src/types/document-settings.ts @@ -2,6 +2,7 @@ import type { ParsedPdfBlockKind } from '@/types/parsed-pdf'; export interface DocumentSettings { schemaVersion: 1; + language?: string; pdf?: { skipBlockKinds: ParsedPdfBlockKind[]; }; @@ -9,6 +10,7 @@ export interface DocumentSettings { export const DEFAULT_DOCUMENT_SETTINGS: DocumentSettings = { schemaVersion: 1, + language: 'auto', pdf: { skipBlockKinds: ['header', 'footer', 'footnote', 'vision_footnote'], }, diff --git a/tests/files/multilingual-sample.txt b/tests/files/multilingual-sample.txt new file mode 100644 index 0000000..fb517aa --- /dev/null +++ b/tests/files/multilingual-sample.txt @@ -0,0 +1,23 @@ +English +OpenReader should split this sentence correctly. This is the second sentence. + +French +OpenReader doit correctement segmenter cette phrase. Voici la deuxieme phrase. + +Spanish +OpenReader debe segmentar correctamente esta frase. Esta es la segunda frase. + +Arabic +يجب أن يقسم أوبن ريدر هذه الجملة بشكل صحيح. هذه هي الجملة الثانية. + +Hindi +ओपनरीडर को इस वाक्य को सही ढंग से विभाजित करना चाहिए। यह दूसरा वाक्य है। + +Japanese +OpenReaderはこの文を正しく分割する必要があります。これは二番目の文です。 + +Chinese +OpenReader应该正确地分割这个句子。这是第二句话。 + +Thai +OpenReader ควรแบ่งประโยคนี้อย่างถูกต้อง นี่คือประโยคที่สอง diff --git a/tests/unit/audiobook-settings.vitest.spec.ts b/tests/unit/audiobook-settings.vitest.spec.ts index 1cc371c..9cd68fe 100644 --- a/tests/unit/audiobook-settings.vitest.spec.ts +++ b/tests/unit/audiobook-settings.vitest.spec.ts @@ -15,6 +15,7 @@ describe('coerceAudiobookGenerationSettings', () => { postSpeed: 1, format: 'mp3', ttsInstructions: 'keep calm', + language: 'ja-jp', }); expect(result.migrated).toBe(false); @@ -27,6 +28,7 @@ describe('coerceAudiobookGenerationSettings', () => { postSpeed: 1, format: 'mp3', ttsInstructions: 'keep calm', + language: 'ja-JP', }); }); diff --git a/tests/unit/document-settings.vitest.spec.ts b/tests/unit/document-settings.vitest.spec.ts new file mode 100644 index 0000000..9a2256c --- /dev/null +++ b/tests/unit/document-settings.vitest.spec.ts @@ -0,0 +1,22 @@ +import { describe, expect, test } from 'vitest'; + +import { mergeDocumentSettings } from '../../src/lib/shared/document-settings'; +import { DEFAULT_DOCUMENT_SETTINGS } from '../../src/types/document-settings'; + +describe('document settings language', () => { + test('defaults to automatic language resolution', () => { + expect(mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, null).language).toBe('auto'); + }); + + test('normalizes explicit BCP 47 language tags', () => { + expect(mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, { language: 'zh-cn' }).language).toBe('zh-CN'); + expect(mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, { language: 'JA' }).language).toBe('ja'); + }); + + test('preserves language when PDF settings are absent', () => { + expect(mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, { language: 'fr' })).toMatchObject({ + language: 'fr', + pdf: DEFAULT_DOCUMENT_SETTINGS.pdf, + }); + }); +}); diff --git a/tests/unit/epub-word-highlight.vitest.spec.ts b/tests/unit/epub-word-highlight.vitest.spec.ts index 8899dc8..ad1f53c 100644 --- a/tests/unit/epub-word-highlight.vitest.spec.ts +++ b/tests/unit/epub-word-highlight.vitest.spec.ts @@ -28,6 +28,16 @@ const alignmentWords = (words: string[]): TTSSentenceAlignment['words'] => })); describe('EPUB word highlight mapping', () => { + test('tokenizes Japanese and Chinese using locale-aware word boundaries', () => { + const japanese = tokenizeCanonicalSegment(segment('これは日本語です。', 5), 'ja'); + expect(japanese.length).toBeGreaterThan(1); + expect(japanese.every((token) => token.norm.length > 0)).toBe(true); + + const chinese = tokenizeCanonicalSegment(segment('这是中文。', 10), 'zh'); + expect(chinese.length).toBeGreaterThan(1); + expect(chinese.map((token) => token.norm).join('')).toBe('这是中文'); + }); + test('tokenizes canonical segment words with source offsets', () => { const tokens = tokenizeCanonicalSegment(segment('"Hello," she said.', 12)); diff --git a/tests/unit/language.vitest.spec.ts b/tests/unit/language.vitest.spec.ts new file mode 100644 index 0000000..b224a2f --- /dev/null +++ b/tests/unit/language.vitest.spec.ts @@ -0,0 +1,60 @@ +import { describe, expect, test } from 'vitest'; +import { readFileSync } from 'fs'; +import { resolve } from 'path'; + +import { + inferKokoroLanguageFromVoice, + normalizeUnicodeToken, + resolveTtsLanguage, + segmentSentences, + segmentWords, + toBaseLanguageCode, +} from '../../src/lib/shared/language'; + +describe('multilingual language utilities', () => { + test('keeps the multilingual document fixture readable as UTF-8', () => { + const fixture = readFileSync(resolve(process.cwd(), 'tests/files/multilingual-sample.txt'), 'utf8'); + expect(fixture).toContain('これは二番目の文です。'); + expect(fixture).toContain('这是第二句话。'); + expect(fixture).toContain('هذه هي الجملة الثانية.'); + }); + + test('infers Kokoro language from a single-language voice selection', () => { + expect(inferKokoroLanguageFromVoice('jf_alpha')).toBe('ja'); + expect(inferKokoroLanguageFromVoice('zf_xiaobei(0.5)+zm_yunxi(0.5)')).toBe('zh-CN'); + expect(inferKokoroLanguageFromVoice('ff_siwis+jf_alpha')).toBeNull(); + }); + + test('prefers an explicit language and resolves regional tags for Whisper', () => { + expect(resolveTtsLanguage({ configuredLanguage: 'pt-BR', voice: 'jf_alpha' })).toBe('pt-BR'); + expect(resolveTtsLanguage({ configuredLanguage: 'auto', voice: 'jf_alpha' })).toBe('ja'); + expect(toBaseLanguageCode('zh-CN')).toBe('zh'); + }); + + test('normalizes Unicode tokens without dropping non-Latin scripts', () => { + expect(normalizeUnicodeToken('École!')).toBe('école'); + expect(normalizeUnicodeToken('日本語。')).toBe('日本語'); + expect(normalizeUnicodeToken('العربية؟')).toBe('العربية'); + expect(normalizeUnicodeToken('हिन्दी।')).toBe('हिन्दी'); + }); + + test('segments CJK sentences and no-space language words', () => { + expect(segmentSentences('これは最初の文です。これは二番目の文です。', 'ja')).toEqual([ + 'これは最初の文です。', + 'これは二番目の文です。', + ]); + + const chineseWords = segmentWords('这是第一句话。', 'zh').map((token) => token.text); + expect(chineseWords.length).toBeGreaterThan(1); + expect(chineseWords.join('')).toBe('这是第一句话'); + }); + + test('preserves source offsets for Thai word segmentation', () => { + const text = 'นี่คือประโยคแรก'; + const words = segmentWords(text, 'th'); + expect(words.length).toBeGreaterThan(1); + for (const word of words) { + expect(text.slice(word.start, word.end)).toBe(word.text); + } + }); +}); diff --git a/tests/unit/nlp.vitest.spec.ts b/tests/unit/nlp.vitest.spec.ts index 4753b20..b259690 100644 --- a/tests/unit/nlp.vitest.spec.ts +++ b/tests/unit/nlp.vitest.spec.ts @@ -46,6 +46,20 @@ describe('preprocessSentenceForAudio', () => { }); describe('splitTextToTtsBlocks (PDF-oriented)', () => { + test('uses locale-aware sentence boundaries for Japanese and Chinese', () => { + expect(splitTextToTtsBlocks( + 'これは最初の文です。これは二番目の文です。', + { language: 'ja', maxBlockLength: 50 }, + )).toEqual(['これは最初の文です。これは二番目の文です。']); + + const chinese = splitTextToTtsBlocks( + Array(12).fill('这是一个用于测试分句的中文句子。').join(''), + { language: 'zh', maxBlockLength: 50 }, + ); + expect(chinese.length).toBeGreaterThan(1); + expect(chinese.join('')).toContain('用于测试分句'); + }); + test('returns [] for empty input', () => { expect(splitTextToTtsBlocks('')).toEqual([]); expect(splitTextToTtsBlocks(' ')).toEqual([]); @@ -201,4 +215,11 @@ describe('normalizeTextForTts', () => { expect(normalized).not.toMatch(/\s{2,}/); expect(normalized.length).toBeGreaterThan(0); }); + + test('does not insert spaces between normalized Japanese blocks', () => { + expect(normalizeTextForTts('最初の文です。次の文です。', { + language: 'ja', + maxBlockLength: 7, + })).toBe('最初の文です。次の文です。'); + }); }); diff --git a/tests/unit/tts-segments.vitest.spec.ts b/tests/unit/tts-segments.vitest.spec.ts index def8e9f..a448afe 100644 --- a/tests/unit/tts-segments.vitest.spec.ts +++ b/tests/unit/tts-segments.vitest.spec.ts @@ -195,4 +195,19 @@ describe('tts segment helpers', () => { expect(alignment.words[2].endSec).toBeGreaterThan(1.4); expect(alignment.words[1].charStart).toBeGreaterThan(alignment.words[0].charStart); }); + + test('builds proportional alignment for no-space languages', () => { + const sentence = 'これは日本語です'; + const alignment = buildProportionalAlignment({ + sentence, + sentenceIndex: 2, + durationMs: 1200, + language: 'ja', + }); + + expect(alignment.words.length).toBeGreaterThan(1); + for (const word of alignment.words) { + expect(sentence.slice(word.charStart, word.charEnd)).toBe(word.text); + } + }); }); From 696d3e488f60c1f48b0673f2a7670806806e359f Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 6 Jun 2026 09:29:39 -0600 Subject: [PATCH 02/12] feat(tts): detect EPUB language and warn on voice mismatch --- src/app/(app)/epub/[id]/page.tsx | 6 +- src/app/(app)/epub/[id]/useEpubDocument.ts | 22 ++++- src/components/AudiobookExportModal.tsx | 12 ++- src/components/documents/DocumentSettings.tsx | 22 ++++- src/components/player/VoicesControl.tsx | 34 ++++++-- src/components/reader/SegmentsSidebar.tsx | 5 +- src/lib/shared/language.ts | 82 +++++++++++++++++-- tests/unit/language.vitest.spec.ts | 48 +++++++++++ 8 files changed, 212 insertions(+), 19 deletions(-) diff --git a/src/app/(app)/epub/[id]/page.tsx b/src/app/(app)/epub/[id]/page.tsx index 3cc6ebd..38472f8 100644 --- a/src/app/(app)/epub/[id]/page.tsx +++ b/src/app/(app)/epub/[id]/page.tsx @@ -37,6 +37,7 @@ export default function EPUBPage() { createFullAudioBook: createEPUBAudioBook, regenerateChapter: regenerateEPUBChapter, bookRef, + metadataLanguage, } = epubState; const { stop, setDocumentLanguage } = useTTS(); const { language, updateLanguage } = useDocumentLanguage(routeDocumentId); @@ -109,8 +110,8 @@ export default function EPUBPage() { useUnmountCleanupRef(clearCurrDoc); useEffect(() => { - setDocumentLanguage(language); - }, [language, setDocumentLanguage]); + setDocumentLanguage(language === 'auto' ? metadataLanguage ?? 'auto' : language); + }, [language, metadataLanguage, setDocumentLanguage]); // Compute available height = viewport - (header height + tts bar height) useEffect(() => { @@ -256,6 +257,7 @@ export default function EPUBPage() { isOpen={activeSidebar === 'settings'} setIsOpen={(isOpen) => setActiveSidebar((prev) => isOpen ? 'settings' : (prev === 'settings' ? null : prev))} language={language} + detectedLanguage={metadataLanguage} onLanguageChange={(nextLanguage) => { void updateLanguage(nextLanguage); }} diff --git a/src/app/(app)/epub/[id]/useEpubDocument.ts b/src/app/(app)/epub/[id]/useEpubDocument.ts index 21e1f0a..14c38fa 100644 --- a/src/app/(app)/epub/[id]/useEpubDocument.ts +++ b/src/app/(app)/epub/[id]/useEpubDocument.ts @@ -44,6 +44,7 @@ import type { import type { AudiobookGenerationSettings, TTSSegmentLocator } from '@/types/client'; import { isStableEpubLocator } from '@/types/client'; import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; +import { normalizeOptionalLanguageTag } from '@/lib/shared/language'; export interface EpubDocumentState { currDocData: ArrayBuffer | undefined; @@ -51,6 +52,7 @@ export interface EpubDocumentState { currDocPages: number | undefined; currDocPage: number | string; currDocText: string | undefined; + metadataLanguage: string | null; isPlaybackReady: boolean; setCurrentDocument: (id: string) => Promise; clearCurrDoc: () => void; @@ -116,6 +118,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { const [currDocData, setCurrDocData] = useState(); const [currDocName, setCurrDocName] = useState(); const [currDocText, setCurrDocText] = useState(); + const [metadataLanguage, setMetadataLanguage] = useState(null); const [isPlaybackReady, setIsPlaybackReady] = useState(false); // Mirror state into a ref so resolveEpubLocator (registered once with // TTSContext via a stable callback) can always read the latest page text @@ -172,6 +175,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { setCurrDocData(undefined); setCurrDocName(undefined); setCurrDocText(undefined); + setMetadataLanguage(null); setIsPlaybackReady(false); setCurrDocPages(undefined); isEPUBSetOnce.current = false; @@ -193,6 +197,9 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { const setCurrentDocument = useCallback(async (id: string): Promise => { try { setIsPlaybackReady(false); + setMetadataLanguage(null); + bookRef.current = null; + renditionRef.current = undefined; const meta = await getDocumentMetadata(id); if (!meta) { clearCurrDoc(); @@ -380,8 +387,19 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { }); const setRendition = useCallback((rendition: Rendition) => { - bookRef.current = rendition.book; + const book = rendition.book; + bookRef.current = book; renditionRef.current = rendition; + void book.loaded.metadata + .then((metadata) => { + if (bookRef.current !== book) return; + setMetadataLanguage(normalizeOptionalLanguageTag(metadata.language)); + }) + .catch((error) => { + if (bookRef.current !== book) return; + setMetadataLanguage(null); + console.warn('Failed to read EPUB language metadata:', error); + }); }, []); const handleLocationChanged = useEPUBLocationController({ @@ -406,6 +424,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { currDocPages, currDocPage, currDocText, + metadataLanguage, isPlaybackReady, clearCurrDoc, extractPageText, @@ -432,6 +451,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { currDocPages, currDocPage, currDocText, + metadataLanguage, isPlaybackReady, clearCurrDoc, extractPageText, diff --git a/src/components/AudiobookExportModal.tsx b/src/components/AudiobookExportModal.tsx index 9595c46..b88efda 100644 --- a/src/components/AudiobookExportModal.tsx +++ b/src/components/AudiobookExportModal.tsx @@ -12,7 +12,7 @@ import { useTTS } from '@/contexts/TTSContext'; import { VoicesControlBase } from '@/components/player/VoicesControlBase'; import { ReaderSidebarShell } from '@/components/reader/ReaderSidebarShell'; import { resolveTtsProviderModelPolicy } from '@/lib/shared/tts-provider-policy'; -import { resolveTtsLanguage } from '@/lib/shared/language'; +import { getTtsLanguageCompatibilityWarnings, resolveTtsLanguage } from '@/lib/shared/language'; import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts'; import { Button, Card, IconButton, MenuActionItem, MenuItemsSurface, RangeInput, SharedListboxButton, SharedListboxOption, SharedListboxOptions } from '@/components/ui'; import { @@ -132,6 +132,11 @@ export function AudiobookExportModal({ }), }; }, [savedSettings, audiobookVoice, configVoice, availableVoices, providerRef, providerType, ttsModel, ttsInstructions, effectiveNativeSpeed, postSpeed, format, providerModelPolicy.supportsInstructions, documentLanguage]); + const languageWarnings = useMemo(() => getTtsLanguageCompatibilityWarnings({ + model: effectiveSettings?.ttsModel, + voice: effectiveSettings?.voice, + documentLanguage: effectiveSettings?.language, + }), [effectiveSettings]); const fetchExistingChapters = useCallback(async (soft: boolean = false) => { if (soft) { @@ -610,6 +615,11 @@ export function AudiobookExportModal({ )}
+ {languageWarnings.map((warning) => ( +

+ {warning} +

+ ))} {/* Speed controls */} diff --git a/src/components/documents/DocumentSettings.tsx b/src/components/documents/DocumentSettings.tsx index 0e1b0ca..7d4e6f9 100644 --- a/src/components/documents/DocumentSettings.tsx +++ b/src/components/documents/DocumentSettings.tsx @@ -2,6 +2,7 @@ import { useState, useEffect } from 'react'; import { useConfig, ViewType } from '@/contexts/ConfigContext'; +import { useTTS } from '@/contexts/TTSContext'; import { ReaderSidebarShell } from '@/components/reader/ReaderSidebarShell'; import { SEGMENT_PRELOAD_DEPTH_MIN, @@ -19,6 +20,7 @@ import { IconButton, RangeInput, Section, ToggleRow, CheckItem, SegmentedControl import { RefreshIcon, SparkleIcon } from '@/components/icons/Icons'; import type { ParsedPdfBlockKind, PdfParseStatus } from '@/types/parsed-pdf'; import { isForceReparseDisabled } from '@/lib/client/pdf/force-reparse'; +import { getLanguageDisplayName, getTtsLanguageCompatibilityWarnings } from '@/lib/shared/language'; const PDF_SKIP_KIND_OPTIONS: Array<{ kind: ParsedPdfBlockKind; label: string }> = [ { kind: 'header', label: 'Header' }, @@ -106,12 +108,13 @@ function RangeSetting({ ); } -export function DocumentSettings({ isOpen, setIsOpen, epub, html, language, onLanguageChange, pdf }: { +export function DocumentSettings({ isOpen, setIsOpen, epub, html, language, detectedLanguage, onLanguageChange, pdf }: { isOpen: boolean, setIsOpen: (isOpen: boolean) => void, epub?: boolean, html?: boolean, language?: string, + detectedLanguage?: string | null, onLanguageChange?: (language: string) => void, pdf?: { parseStatus: PdfParseStatus | null; @@ -137,7 +140,14 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html, language, onLa epubWordHighlightEnabled, htmlHighlightEnabled, htmlWordHighlightEnabled, + ttsModel, } = useConfig(); + const { voice, resolvedLanguage } = useTTS(); + const languageWarnings = getTtsLanguageCompatibilityWarnings({ + model: ttsModel, + voice, + documentLanguage: resolvedLanguage, + }); const selectedView = viewTypeTextMapping.find(v => v.id === viewType) || viewTypeTextMapping[0]; const isPdfMode = !epub && !html && !!pdf; const [localPreloadDepth, setLocalPreloadDepth] = useState(segmentPreloadDepthPages); @@ -189,6 +199,16 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html, language, onLa ))} + {language === 'auto' && detectedLanguage ? ( +

+ Detected from document metadata: {getLanguageDisplayName(detectedLanguage)} +

+ ) : null} + {languageWarnings.map((warning) => ( +

+ {warning} +

+ ))} ) : null} {isPdfMode && pdf && ( diff --git a/src/components/player/VoicesControl.tsx b/src/components/player/VoicesControl.tsx index 2e67b86..b2b8661 100644 --- a/src/components/player/VoicesControl.tsx +++ b/src/components/player/VoicesControl.tsx @@ -4,22 +4,40 @@ import { useConfig } from '@/contexts/ConfigContext'; import { useTTS } from '@/contexts/TTSContext'; import { useCallback } from 'react'; import { VoicesControlBase } from '@/components/player/VoicesControlBase'; +import { InfoIcon } from '@/components/icons/Icons'; +import { getTtsLanguageCompatibilityWarnings } from '@/lib/shared/language'; export const VoicesControl = ({ availableVoices, setVoiceAndRestart }: { availableVoices: string[]; setVoiceAndRestart: (voice: string) => void; }) => { const { ttsModel, providerType } = useConfig(); - const { voice } = useTTS(); + const { voice, resolvedLanguage } = useTTS(); const onChangeVoice = useCallback((nextVoice: string) => setVoiceAndRestart(nextVoice), [setVoiceAndRestart]); + const languageWarnings = getTtsLanguageCompatibilityWarnings({ + model: ttsModel, + voice, + documentLanguage: resolvedLanguage, + }); return ( - +
+ + {languageWarnings.length > 0 ? ( + + + + ) : null} +
); } diff --git a/src/components/reader/SegmentsSidebar.tsx b/src/components/reader/SegmentsSidebar.tsx index d9ff378..34b6f3b 100644 --- a/src/components/reader/SegmentsSidebar.tsx +++ b/src/components/reader/SegmentsSidebar.tsx @@ -94,6 +94,7 @@ function settingsAreEqual(a: TTSSegmentSettings | null, b: TTSSegmentSettings | && a.voice === b.voice && Number(a.nativeSpeed) === Number(b.nativeSpeed) && (a.ttsInstructions || '') === (b.ttsInstructions || '') + && (a.language || 'en') === (b.language || 'en') ); } @@ -186,6 +187,7 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId, epubBookRef }: playFromSegment, activeReaderType, clearSegmentCaches, + resolvedLanguage, } = useTTS(); const { providerRef, @@ -348,7 +350,8 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId, epubBookRef }: voice, nativeSpeed: Number.isFinite(Number(voiceSpeed)) ? Number(voiceSpeed) : 1, ttsInstructions: ttsInstructions || '', - }), [providerRef, providerType, ttsModel, voice, voiceSpeed, ttsInstructions]); + language: resolvedLanguage, + }), [providerRef, providerType, ttsModel, voice, voiceSpeed, ttsInstructions, resolvedLanguage]); const handleClearCache = useCallback(async () => { if (!documentId || isClearingSegments) return; diff --git a/src/lib/shared/language.ts b/src/lib/shared/language.ts index b2e3437..9432235 100644 --- a/src/lib/shared/language.ts +++ b/src/lib/shared/language.ts @@ -1,3 +1,5 @@ +import { isKokoroModel } from '@/lib/shared/kokoro'; + export const DEFAULT_TTS_LANGUAGE = 'en'; export interface TextToken { @@ -26,6 +28,17 @@ const KOKORO_LANGUAGE_BY_PREFIX: Readonly> = { zm: 'zh-CN', }; +export const KOKORO_SUPPORTED_LANGUAGES = [ + 'en', + 'es', + 'fr', + 'hi', + 'it', + 'ja', + 'pt-BR', + 'zh-CN', +] as const; + export function normalizeLanguageTag( language: string | null | undefined, fallback = DEFAULT_TTS_LANGUAGE, @@ -40,6 +53,17 @@ export function normalizeLanguageTag( } } +export function normalizeOptionalLanguageTag(language: unknown): string | null { + if (typeof language !== 'string') return null; + const candidate = language.trim().split(/[,\s]+/u)[0]; + if (!candidate) return null; + try { + return Intl.getCanonicalLocales(candidate)[0] ?? null; + } catch { + return null; + } +} + export function toBaseLanguageCode(language: string | null | undefined): string { const normalized = normalizeLanguageTag(language); try { @@ -50,17 +74,20 @@ export function toBaseLanguageCode(language: string | null | undefined): string } export function inferKokoroLanguageFromVoice(voice: string | null | undefined): string | null { - if (!voice?.trim()) return null; + const languages = new Set(getKokoroVoiceLanguages(voice)); - const languages = new Set( + return languages.size === 1 ? [...languages][0] : null; +} + +export function getKokoroVoiceLanguages(voice: string | null | undefined): string[] { + if (!voice?.trim()) return []; + return Array.from(new Set( voice .split('+') .map((part) => part.trim().replace(/\([^)]*\)/g, '')) .map((name) => KOKORO_LANGUAGE_BY_PREFIX[name.slice(0, 2).toLowerCase()]) .filter((language): language is string => Boolean(language)), - ); - - return languages.size === 1 ? [...languages][0] : null; + )); } export function resolveTtsLanguage(input: { @@ -75,6 +102,51 @@ export function resolveTtsLanguage(input: { return inferKokoroLanguageFromVoice(input.voice) ?? DEFAULT_TTS_LANGUAGE; } +export function getLanguageDisplayName(language: string): string { + try { + return new Intl.DisplayNames(['en'], { type: 'language' }).of(language) || language; + } catch { + return language; + } +} + +export function getTtsLanguageCompatibilityWarnings(input: { + model?: string | null; + voice?: string | null; + documentLanguage?: string | null; +}): string[] { + if (!isKokoroModel(input.model || '')) return []; + + const documentLanguage = normalizeLanguageTag(input.documentLanguage); + const documentBaseLanguage = toBaseLanguageCode(documentLanguage); + const supportedBaseLanguages = new Set(KOKORO_SUPPORTED_LANGUAGES.map((language) => toBaseLanguageCode(language))); + const voiceLanguages = getKokoroVoiceLanguages(input.voice); + const voiceBaseLanguages = new Set(voiceLanguages.map((language) => toBaseLanguageCode(language))); + const warnings: string[] = []; + + if (!supportedBaseLanguages.has(documentBaseLanguage)) { + warnings.push( + `Kokoro's built-in voice catalog does not include ${getLanguageDisplayName(documentLanguage)}.`, + ); + } + + if (voiceBaseLanguages.size > 1) { + warnings.push( + `Selected Kokoro voices use multiple languages (${voiceLanguages.map(getLanguageDisplayName).join(', ')}).`, + ); + return warnings; + } + + const voiceLanguage = voiceLanguages[0]; + if (voiceLanguage && toBaseLanguageCode(voiceLanguage) !== documentBaseLanguage) { + warnings.push( + `Selected Kokoro voice is ${getLanguageDisplayName(voiceLanguage)}, but the document is ${getLanguageDisplayName(documentLanguage)}.`, + ); + } + + return warnings; +} + export function normalizeUnicodeToken(text: string): string { return text .normalize('NFKC') diff --git a/tests/unit/language.vitest.spec.ts b/tests/unit/language.vitest.spec.ts index b224a2f..2842d80 100644 --- a/tests/unit/language.vitest.spec.ts +++ b/tests/unit/language.vitest.spec.ts @@ -3,7 +3,9 @@ import { readFileSync } from 'fs'; import { resolve } from 'path'; import { + getTtsLanguageCompatibilityWarnings, inferKokoroLanguageFromVoice, + normalizeOptionalLanguageTag, normalizeUnicodeToken, resolveTtsLanguage, segmentSentences, @@ -25,6 +27,52 @@ describe('multilingual language utilities', () => { expect(inferKokoroLanguageFromVoice('ff_siwis+jf_alpha')).toBeNull(); }); + test('normalizes valid EPUB metadata language tags and rejects invalid metadata', () => { + expect(normalizeOptionalLanguageTag(' ja-jp ')).toBe('ja-JP'); + expect(normalizeOptionalLanguageTag('fr, en')).toBe('fr'); + expect(normalizeOptionalLanguageTag('not_a_language')).toBeNull(); + expect(normalizeOptionalLanguageTag(undefined)).toBeNull(); + }); + + test('warns only for provable Kokoro language compatibility problems', () => { + expect(getTtsLanguageCompatibilityWarnings({ + model: 'kokoro', + voice: 'jf_alpha', + documentLanguage: 'ja', + })).toEqual([]); + + expect(getTtsLanguageCompatibilityWarnings({ + model: 'kokoro', + voice: 'af_sarah', + documentLanguage: 'ja', + })).toEqual([ + 'Selected Kokoro voice is American English, but the document is Japanese.', + ]); + + expect(getTtsLanguageCompatibilityWarnings({ + model: 'kokoro', + voice: 'ff_siwis+jf_alpha', + documentLanguage: 'fr', + })).toEqual([ + 'Selected Kokoro voices use multiple languages (French, Japanese).', + ]); + + expect(getTtsLanguageCompatibilityWarnings({ + model: 'kokoro', + voice: 'af_sarah', + documentLanguage: 'ar', + })).toEqual([ + "Kokoro's built-in voice catalog does not include Arabic.", + 'Selected Kokoro voice is American English, but the document is Arabic.', + ]); + + expect(getTtsLanguageCompatibilityWarnings({ + model: 'custom-unknown-model', + voice: 'af_sarah', + documentLanguage: 'ja', + })).toEqual([]); + }); + test('prefers an explicit language and resolves regional tags for Whisper', () => { expect(resolveTtsLanguage({ configuredLanguage: 'pt-BR', voice: 'jf_alpha' })).toBe('pt-BR'); expect(resolveTtsLanguage({ configuredLanguage: 'auto', voice: 'jf_alpha' })).toBe('ja'); From facbda247749c0d0de1bfc0032024f7dfc549296 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 6 Jun 2026 09:38:14 -0600 Subject: [PATCH 03/12] feat(tts): forward language to capable providers --- src/app/api/audiobook/chapter/route.ts | 1 + src/app/api/tts/segments/ensure/route.ts | 1 + src/lib/server/tts/generate.ts | 85 +++++++++++++++++-- src/lib/server/tts/voice-resolution.ts | 64 ++++++++++++++ tests/unit/tts-generate.vitest.spec.ts | 32 ++++++- .../unit/tts-provider-catalog.vitest.spec.ts | 50 ++++++++++- 6 files changed, 226 insertions(+), 7 deletions(-) diff --git a/src/app/api/audiobook/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts index 31f40e6..e63d045 100644 --- a/src/app/api/audiobook/chapter/route.ts +++ b/src/app/api/audiobook/chapter/route.ts @@ -597,6 +597,7 @@ export async function POST(request: NextRequest) { format: 'mp3', model, instructions, + language: mergedSettings?.language, provider, apiKey: openApiKey, baseUrl: openApiBaseUrl, diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts index e892784..f089231 100644 --- a/src/app/api/tts/segments/ensure/route.ts +++ b/src/app/api/tts/segments/ensure/route.ts @@ -642,6 +642,7 @@ export async function POST(request: NextRequest) { format: 'mp3', model: effectiveSettings.ttsModel, instructions: effectiveSettings.ttsInstructions, + language: effectiveSettings.language, provider: requestCreds.provider, apiKey: requestCreds.apiKey || 'none', baseUrl: requestCreds.baseUrl, diff --git a/src/lib/server/tts/generate.ts b/src/lib/server/tts/generate.ts index 01a7978..c235e88 100644 --- a/src/lib/server/tts/generate.ts +++ b/src/lib/server/tts/generate.ts @@ -6,12 +6,16 @@ import { REPLICATE_KOKORO_82M_VERSIONED_MODEL, } from '@/lib/shared/tts-provider-catalog'; import { resolveTtsProviderModelPolicy } from '@/lib/shared/tts-provider-policy'; -import { resolveReplicateVoiceInputKey } from '@/lib/server/tts/voice-resolution'; +import { + resolveReplicateLanguageInput, + resolveReplicateVoiceInputKey, +} from '@/lib/server/tts/voice-resolution'; import { getUpstreamRetryAfterSeconds, getUpstreamStatus } from '@/lib/server/tts/upstream-response'; import { LRUCache } from 'lru-cache'; import { createHash } from 'crypto'; import { access, readFile } from 'fs/promises'; import { resolve } from 'path'; +import { getLanguageDisplayName, toBaseLanguageCode } from '@/lib/shared/language'; export interface ServerTTSRequest { text: string; @@ -20,6 +24,7 @@ export interface ServerTTSRequest { format?: string; model?: string | null; instructions?: string; + language?: string; provider: string; apiKey: string; baseUrl?: string; @@ -30,6 +35,7 @@ type CustomVoice = string; type ExtendedSpeechParams = Omit & { voice: SpeechCreateParams['voice'] | CustomVoice; instructions?: string; + language?: string; }; type ResolvedServerTTSRequest = { @@ -39,6 +45,7 @@ type ResolvedServerTTSRequest = { format: string; model: SpeechCreateParams['model']; instructions?: string; + language?: string; provider: string; apiKey: string; baseUrl?: string; @@ -55,6 +62,7 @@ const REPLICATE_COOLDOWN_SCOPE_CACHE_MAX_ENTRIES = 512; const replicateBlockedUntilByScope = new LRUCache({ max: REPLICATE_COOLDOWN_SCOPE_CACHE_MAX_ENTRIES, }); +const openAiCompatibleLanguageUnsupported = new LRUCache({ max: 256 }); const DEFAULT_TTS_CACHE_MAX_SIZE_BYTES = 256 * 1024 * 1024; const DEFAULT_TTS_CACHE_TTL_MS = 1000 * 60 * 30; @@ -323,6 +331,7 @@ function resolveTTSRequest(input: ServerTTSRequest): ResolvedServerTTSRequest { format, model, instructions, + language: input.language, provider, apiKey: input.apiKey, baseUrl: input.baseUrl, @@ -338,6 +347,7 @@ function makeCacheKey(input: { format: string; text: string; instructions?: string; + language?: string; testNamespace?: string | null; }) { const canonical = { @@ -348,6 +358,7 @@ function makeCacheKey(input: { format: input.format, text: input.text, instructions: input.instructions || undefined, + language: input.language || undefined, testNamespace: input.testNamespace || undefined, }; return createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); @@ -363,6 +374,7 @@ export function buildTTSCacheKey(request: ServerTTSRequest): string { format: resolved.format, text: resolved.text, instructions: resolved.instructions, + language: resolved.language, testNamespace: resolved.testNamespace, }); } @@ -433,7 +445,7 @@ async function buildReplicateInput(request: ResolvedServerTTSRequest): Promise = { text: request.text }; @@ -498,9 +510,46 @@ async function buildReplicateInput(request: ResolvedServerTTSRequest): Promise, + request: ResolvedServerTTSRequest, +): Promise> { + if (!request.language) return input; + const languageInput = await resolveReplicateLanguageInput({ + provider: 'replicate', + model: request.model as string, + apiKey: request.apiKey, + }); + if (languageInput) { + const resolvedValue = resolveReplicateLanguageValue(request.language, languageInput.allowedValues); + if (resolvedValue) { + input[languageInput.key] = resolvedValue; + } + } return input; } +export function resolveReplicateLanguageValue(language: string, allowedValues: string[]): string | null { + if (allowedValues.length === 0) return language; + + const baseLanguage = toBaseLanguageCode(language); + const candidates = [ + language, + baseLanguage, + getLanguageDisplayName(language), + getLanguageDisplayName(baseLanguage), + ]; + const allowedByLowercase = new Map( + allowedValues.map((value) => [value.toLocaleLowerCase(), value]), + ); + return candidates + .map((candidate) => allowedByLowercase.get(candidate.toLocaleLowerCase())) + .find((candidate): candidate is string => Boolean(candidate)) ?? null; +} + async function runReplicateRequest( request: ResolvedServerTTSRequest, signal: AbortSignal, @@ -594,6 +643,31 @@ async function runProviderRequest( createParams.instructions = request.instructions; } + if (request.provider !== 'openai' && request.language) { + const supportKey = `${request.baseUrl || ''}|${request.model as string}`; + if (!openAiCompatibleLanguageUnsupported.has(supportKey)) { + try { + return await fetchTTSBufferWithRetry( + openai, + { ...createParams, language: request.language }, + signal, + upstreamSettings.ttsUpstreamMaxRetries, + ); + } catch (error) { + const status = getUpstreamStatus(error); + if (status !== 400 && status !== 422) throw error; + const fallback = await fetchTTSBufferWithRetry( + openai, + createParams, + signal, + upstreamSettings.ttsUpstreamMaxRetries, + ); + openAiCompatibleLanguageUnsupported.set(supportKey, true); + return fallback; + } + } + } + return fetchTTSBufferWithRetry(openai, createParams, signal, upstreamSettings.ttsUpstreamMaxRetries); } @@ -614,6 +688,7 @@ export async function generateTTSBuffer( format: resolved.format, text: resolved.text, instructions: resolved.instructions, + language: resolved.language, testNamespace: resolved.testNamespace, }); diff --git a/src/lib/server/tts/voice-resolution.ts b/src/lib/server/tts/voice-resolution.ts index d42ab3e..1fd9420 100644 --- a/src/lib/server/tts/voice-resolution.ts +++ b/src/lib/server/tts/voice-resolution.ts @@ -102,6 +102,19 @@ function walkRecordGraph(root: unknown, visit: (node: Record) = } const REPLICATE_VOICE_KEYS = ['voice', 'voice_id', 'speaker'] as const satisfies readonly ReplicateVoiceInputKey[]; +const REPLICATE_LANGUAGE_KEYS = [ + 'language', + 'lang', + 'language_code', + 'locale', + 'language_id', + 'language_boost', +] as const; +export type ReplicateLanguageInputKey = typeof REPLICATE_LANGUAGE_KEYS[number]; +export type ReplicateLanguageInput = { + key: ReplicateLanguageInputKey; + allowedValues: string[]; +}; const REPLICATE_BUILT_IN_MODELS = new Set( resolveProviderModels('replicate') .map((model) => model.id) @@ -150,6 +163,26 @@ function extractReplicateVoiceInputKeyFromOpenApiSchema(openApiSchema: unknown): return found; } +function extractReplicateLanguageInputFromOpenApiSchema(openApiSchema: unknown): ReplicateLanguageInput | null { + let found: ReplicateLanguageInput | null = null; + + walkRecordGraph(openApiSchema, (node) => { + const properties = node.properties; + if (!isRecord(properties)) return; + for (const key of REPLICATE_LANGUAGE_KEYS) { + if (key in properties) { + found = { + key, + allowedValues: extractSchemaStringEnums(properties[key]), + }; + return true; + } + } + }); + + return found; +} + async function fetchReplicateOpenApiSchema(apiKey: string, model: string): Promise { const parsedModel = parseReplicateModelIdentifier(model); if (!parsedModel) { @@ -207,11 +240,15 @@ async function fetchReplicateOpenApiSchema(apiKey: string, model: string): Promi } const REPLICATE_VOICE_INPUT_KEY_CACHE_MAX_ENTRIES = 128; +const REPLICATE_LANGUAGE_INPUT_KEY_CACHE_MAX_ENTRIES = 128; const REPLICATE_OPENAPI_SCHEMA_PROMISE_CACHE_MAX_ENTRIES = 128; const replicateVoiceInputKeyCache = new LRUCache({ max: REPLICATE_VOICE_INPUT_KEY_CACHE_MAX_ENTRIES, }); +const replicateLanguageInputCache = new LRUCache({ + max: REPLICATE_LANGUAGE_INPUT_KEY_CACHE_MAX_ENTRIES, +}); const replicateOpenApiSchemaPromiseCache = new LRUCache>({ max: REPLICATE_OPENAPI_SCHEMA_PROMISE_CACHE_MAX_ENTRIES, }); @@ -260,6 +297,33 @@ export async function resolveReplicateVoiceInputKey({ return inputKey; } +export async function resolveReplicateLanguageInputKey({ + provider, + model, + apiKey = '', +}: ResolveVoicesOptions): Promise { + const input = await resolveReplicateLanguageInput({ provider, model, apiKey }); + return input?.key ?? null; +} + +export async function resolveReplicateLanguageInput({ + provider, + model, + apiKey = '', +}: ResolveVoicesOptions): Promise { + if (provider !== 'replicate' || !apiKey) return null; + + const cached = replicateLanguageInputCache.get(model); + if (cached) return cached; + + const openApiSchema = await getReplicateOpenApiSchemaCached(apiKey, model); + const input = extractReplicateLanguageInputFromOpenApiSchema(openApiSchema); + if (input) { + replicateLanguageInputCache.set(model, input); + } + return input; +} + async function fetchDeepinfraVoices(apiKey: string): Promise { const controller = new AbortController(); const timeoutId = setTimeout(() => { diff --git a/tests/unit/tts-generate.vitest.spec.ts b/tests/unit/tts-generate.vitest.spec.ts index a601952..4b35d02 100644 --- a/tests/unit/tts-generate.vitest.spec.ts +++ b/tests/unit/tts-generate.vitest.spec.ts @@ -1,6 +1,10 @@ import { describe, expect, test } from 'vitest'; -import { extractReplicateAudioUrl } from '../../src/lib/server/tts/generate'; +import { + buildTTSCacheKey, + extractReplicateAudioUrl, + resolveReplicateLanguageValue, +} from '../../src/lib/server/tts/generate'; describe('replicate output URL extraction', () => { test('returns direct URL string output', () => { @@ -41,3 +45,29 @@ describe('replicate output URL extraction', () => { expect(extractReplicateAudioUrl(output)).toBeNull(); }); }); + +describe('TTS upstream cache identity', () => { + test('includes language in the upstream audio cache key', () => { + const request = { + text: '同じ文章です。', + voice: 'jf_alpha', + speed: 1, + format: 'mp3', + model: 'kokoro', + provider: 'custom-openai', + apiKey: 'test', + }; + + expect(buildTTSCacheKey({ ...request, language: 'ja' })) + .not.toBe(buildTTSCacheKey({ ...request, language: 'en' })); + }); +}); + +describe('Replicate language schema values', () => { + test('uses language codes or advertised display names without a model table', () => { + expect(resolveReplicateLanguageValue('ja-JP', [])).toBe('ja-JP'); + expect(resolveReplicateLanguageValue('ja-JP', ['en', 'ja', 'zh'])).toBe('ja'); + expect(resolveReplicateLanguageValue('ja-JP', ['English', 'Japanese'])).toBe('Japanese'); + expect(resolveReplicateLanguageValue('ja-JP', ['English', 'French'])).toBeNull(); + }); +}); diff --git a/tests/unit/tts-provider-catalog.vitest.spec.ts b/tests/unit/tts-provider-catalog.vitest.spec.ts index 334ea16..8996617 100644 --- a/tests/unit/tts-provider-catalog.vitest.spec.ts +++ b/tests/unit/tts-provider-catalog.vitest.spec.ts @@ -9,7 +9,11 @@ import { supportsTtsInstructions, } from '../../src/lib/shared/tts-provider-catalog'; import { normalizeLegacyProviderRef, resolveProviderDefaults, resolveTtsModelForProvider } from '../../src/lib/shared/tts-provider-policy'; -import { resolveReplicateVoiceInputKey, resolveVoices } from '../../src/lib/server/tts/voice-resolution'; +import { + resolveReplicateLanguageInputKey, + resolveReplicateVoiceInputKey, + resolveVoices, +} from '../../src/lib/server/tts/voice-resolution'; import { applyConfigUpdate, getVoicePreferenceKey } from '../../src/lib/client/config/updates'; import { buildSyncedPreferencePatch } from '../../src/lib/client/config/preferences'; @@ -387,6 +391,50 @@ describe('tts provider catalog', () => { globalThis.fetch = originalFetch; } }); + + test('discovers Replicate language input key from model schema', async () => { + const originalFetch = globalThis.fetch; + let calls = 0; + globalThis.fetch = async () => { + calls += 1; + return { + ok: true, + json: async () => ({ + latest_version: { + openapi_schema: { + components: { + schemas: { + Input: { + type: 'object', + properties: { + text: { type: 'string' }, + language_code: { type: 'string' }, + }, + }, + }, + }, + }, + }, + }), + } as Response; + }; + + try { + await expect(resolveReplicateLanguageInputKey({ + provider: 'replicate', + model: 'acme/schema-language-model', + apiKey: 'r8_token', + })).resolves.toBe('language_code'); + await expect(resolveReplicateLanguageInputKey({ + provider: 'replicate', + model: 'acme/schema-language-model', + apiKey: 'r8_token', + })).resolves.toBe('language_code'); + expect(calls).toBe(1); + } finally { + globalThis.fetch = originalFetch; + } + }); }); describe('config helpers', () => { From 3f15636e0e948325dc06e089c316688cecc902e3 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 6 Jun 2026 10:37:55 -0600 Subject: [PATCH 04/12] fix(highlight): support Japanese token matching --- src/hooks/epub/useEPUBHighlighting.ts | 35 ++++++ src/lib/client/epub/epub-word-highlight.ts | 16 ++- src/lib/client/pdf-highlight-match.ts | 101 +++++++++++++++ src/lib/client/pdf-highlight-worker.ts | 116 ++---------------- src/lib/client/pdf.ts | 9 +- tests/unit/epub-word-highlight.vitest.spec.ts | 30 +++++ tests/unit/pdf-highlight-match.vitest.spec.ts | 29 +++++ 7 files changed, 223 insertions(+), 113 deletions(-) create mode 100644 src/lib/client/pdf-highlight-match.ts create mode 100644 tests/unit/pdf-highlight-match.vitest.spec.ts diff --git a/src/hooks/epub/useEPUBHighlighting.ts b/src/hooks/epub/useEPUBHighlighting.ts index 8c31d34..83beafa 100644 --- a/src/hooks/epub/useEPUBHighlighting.ts +++ b/src/hooks/epub/useEPUBHighlighting.ts @@ -6,6 +6,7 @@ import type { Rendition } from 'epubjs'; import { buildMonotonicWordToTokenMap, buildWordHighlightCacheKey, + resolveAlignmentWordSourceRange, tokenizeCanonicalSegment, type EpubCanonicalWordToken, } from '@/lib/client/epub/epub-word-highlight'; @@ -118,6 +119,40 @@ export function useEPUBHighlighting({ const resolved = resolveVisibleSegmentRange(renderedTextMapsRef.current, segment); if (!resolved || segment.startAnchor.sourceKey !== resolved.map.sourceKey) return; + const alignmentRange = resolveAlignmentWordSourceRange(segment, words[wordIndex]); + if ( + alignmentRange + && alignmentRange.sourceStart >= resolved.startOffset + && alignmentRange.sourceEnd <= resolved.endOffset + ) { + const wordRange = createRangeFromMappedOffsets( + resolved.map, + alignmentRange.sourceStart, + alignmentRange.sourceEnd, + ); + if (wordRange) { + try { + const wordCfi = resolved.map.content.cfiFromRange(wordRange); + currentWordHighlightCfiRef.current = wordCfi; + renditionRef.current.annotations.add( + 'highlight', + wordCfi, + {}, + () => { }, + '', + { + fill: 'var(--accent)', + 'fill-opacity': '0.4', + 'mix-blend-mode': 'multiply', + } + ); + return; + } catch (error) { + console.error('Error highlighting EPUB word from alignment offsets:', error); + } + } + } + const cacheKey = buildWordHighlightCacheKey(segment, alignment, language); if (wordHighlightMapCacheRef.current?.key !== cacheKey) { const tokens = tokenizeCanonicalSegment(segment, language); diff --git a/src/lib/client/epub/epub-word-highlight.ts b/src/lib/client/epub/epub-word-highlight.ts index 5e90cda..7007ce5 100644 --- a/src/lib/client/epub/epub-word-highlight.ts +++ b/src/lib/client/epub/epub-word-highlight.ts @@ -1,5 +1,5 @@ import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; -import type { TTSSentenceAlignment } from '@/types/tts'; +import type { TTSSentenceAlignment, TTSSentenceWord } from '@/types/tts'; import { normalizeUnicodeToken, segmentWords } from '@/lib/shared/language'; export type EpubCanonicalWordToken = { @@ -11,6 +11,20 @@ export type EpubCanonicalWordToken = { export const normalizeWordForHighlight = (text: string): string => normalizeUnicodeToken(text); +export const resolveAlignmentWordSourceRange = ( + segment: CanonicalTtsSegment, + word: TTSSentenceWord, +): { sourceStart: number; sourceEnd: number } | null => { + const { charStart, charEnd } = word; + if (!Number.isInteger(charStart) || !Number.isInteger(charEnd)) return null; + if (charStart < 0 || charEnd <= charStart || charEnd > segment.text.length) return null; + + return { + sourceStart: segment.startAnchor.offset + charStart, + sourceEnd: segment.startAnchor.offset + charEnd, + }; +}; + export const tokenizeCanonicalSegment = ( segment: CanonicalTtsSegment, language?: string, diff --git a/src/lib/client/pdf-highlight-match.ts b/src/lib/client/pdf-highlight-match.ts new file mode 100644 index 0000000..b16ef5d --- /dev/null +++ b/src/lib/client/pdf-highlight-match.ts @@ -0,0 +1,101 @@ +import { CmpStr } from 'cmpstr'; + +const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); + +export interface HighlightTokenMatchResult { + bestStart: number; + bestEnd: number; + rating: number; + lengthDiff: number; +} + +export function findBestHighlightTokenMatch( + patternTokens: string[], + tokenTexts: string[], +): HighlightTokenMatchResult { + const cleanPatternTokens = patternTokens.map((token) => token.trim()).filter(Boolean); + const cleanPattern = cleanPatternTokens.join(''); + const patternLen = cleanPattern.length; + const responseBase: HighlightTokenMatchResult = { + bestStart: -1, + bestEnd: -1, + rating: 0, + lengthDiff: Number.POSITIVE_INFINITY, + }; + + if (!patternLen || !tokenTexts.length) return responseBase; + + const patternTokenCount = cleanPatternTokens.length; + const minWindowTokens = Math.max(1, Math.floor(patternTokenCount * 0.6)); + const maxWindowTokens = Math.max( + minWindowTokens, + Math.ceil(patternTokenCount * 1.4), + ); + + let bestStart = -1; + let bestEnd = -1; + let bestRating = 0; + let bestLengthDiff = Number.POSITIVE_INFINITY; + + for (let start = 0; start < tokenTexts.length; start += 1) { + let combined = ''; + + for ( + let offset = 0; + offset < maxWindowTokens && start + offset < tokenTexts.length; + offset += 1 + ) { + const token = tokenTexts[start + offset]; + combined += token; + + const windowSize = offset + 1; + if (windowSize < minWindowTokens) continue; + if (combined.length > patternLen * 2) break; + + const similarity = cmp.compare(combined, cleanPattern); + const lengthDiff = Math.abs(combined.length - patternLen); + const lengthPenalty = lengthDiff / patternLen; + const adjustedRating = similarity * (1 - lengthPenalty * 0.3); + + let boostedRating = adjustedRating; + const windowTokens = tokenTexts.slice(start, start + windowSize); + const maxPrefixCheck = Math.min( + windowTokens.length, + cleanPatternTokens.length, + 5, + ); + + let prefixMatches = 0; + for (let i = 0; i < maxPrefixCheck; i += 1) { + const tokenSim = cmp.compare(windowTokens[i], cleanPatternTokens[i]); + if (tokenSim < 0.8) break; + prefixMatches += 1; + } + + if (prefixMatches > 0) { + const prefixRatio = prefixMatches / maxPrefixCheck; + boostedRating = adjustedRating * (1 + prefixRatio * 0.25); + } + + if ( + boostedRating > bestRating + || ( + Math.abs(boostedRating - bestRating) < 1e-3 + && lengthDiff < bestLengthDiff + ) + ) { + bestRating = boostedRating; + bestLengthDiff = lengthDiff; + bestStart = start; + bestEnd = start + offset; + } + } + } + + return { + bestStart, + bestEnd, + rating: bestRating, + lengthDiff: bestLengthDiff, + }; +} diff --git a/src/lib/client/pdf-highlight-worker.ts b/src/lib/client/pdf-highlight-worker.ts index 5b41d45..6b396af 100644 --- a/src/lib/client/pdf-highlight-worker.ts +++ b/src/lib/client/pdf-highlight-worker.ts @@ -1,13 +1,11 @@ /// -import { CmpStr } from 'cmpstr'; - -const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); +import { findBestHighlightTokenMatch } from './pdf-highlight-match'; interface TokenMatchRequest { id: string; type: 'tokenMatch'; - pattern: string; + patternTokens: string[]; tokenTexts: string[]; } @@ -39,112 +37,14 @@ self.onmessage = (event: MessageEvent) => { const data = event.data; if (!data || data.type !== 'tokenMatch') return; - const { id, pattern, tokenTexts } = data; - - const cleanPattern = pattern.trim().replace(/\s+/g, ' '); - const patternLen = cleanPattern.length; - - const responseBase: TokenMatchResponse = { - id, - type: 'tokenMatchResult', - bestStart: -1, - bestEnd: -1, - rating: 0, - lengthDiff: Number.POSITIVE_INFINITY, - }; - - if (!patternLen || !tokenTexts.length) { - (self as unknown as DedicatedWorkerGlobalScope).postMessage(responseBase); - return; - } - - const patternTokens = cleanPattern.split(' ').filter(Boolean); - const patternTokenCount = patternTokens.length || 1; - - const minWindowTokens = Math.max(1, Math.floor(patternTokenCount * 0.6)); - const maxWindowTokens = Math.max( - minWindowTokens, - Math.ceil(patternTokenCount * 1.4) - ); - - let bestStart = -1; - let bestEnd = -1; - let bestRating = 0; - let bestLengthDiff = Number.POSITIVE_INFINITY; - - for (let start = 0; start < tokenTexts.length; start++) { - let combined = ''; - - for ( - let offset = 0; - offset < maxWindowTokens && start + offset < tokenTexts.length; - offset++ - ) { - const token = tokenTexts[start + offset]; - combined = combined ? `${combined} ${token}` : token; - - const windowSize = offset + 1; - if (windowSize < minWindowTokens) continue; - if (combined.length > patternLen * 2) break; - - const similarity = cmp.compare(combined, cleanPattern); - const lengthDiff = Math.abs(combined.length - patternLen); - const lengthPenalty = lengthDiff / patternLen; - const adjustedRating = similarity * (1 - lengthPenalty * 0.3); - - // Prefix-alignment boost: - // Favour windows whose first few tokens closely match the beginning - // of the pattern, so we are less likely to cut off the first 1–2 words. - let boostedRating = adjustedRating; - if (patternTokens.length > 0) { - const windowTokens = tokenTexts.slice(start, start + windowSize); - const maxPrefixCheck = Math.min( - windowTokens.length, - patternTokens.length, - 5 - ); - - let prefixMatches = 0; - for (let i = 0; i < maxPrefixCheck; i++) { - const tokenText = windowTokens[i]; - const patternToken = patternTokens[i]; - // Require reasonably strong similarity for a prefix match - const tokenSim = cmp.compare(tokenText, patternToken); - if (tokenSim >= 0.8) { - prefixMatches++; - } else { - // Stop at the first non-matching leading token - break; - } - } - - if (prefixMatches > 0) { - const prefixRatio = prefixMatches / maxPrefixCheck; - const PREFIX_BOOST_FACTOR = 0.25; // up to +25% boost - boostedRating = adjustedRating * (1 + prefixRatio * PREFIX_BOOST_FACTOR); - } - } - - if ( - boostedRating > bestRating || - (Math.abs(boostedRating - bestRating) < 1e-3 && - lengthDiff < bestLengthDiff) - ) { - bestRating = boostedRating; - bestLengthDiff = lengthDiff; - bestStart = start; - bestEnd = start + offset; - } - } - } + const { id, patternTokens, tokenTexts } = data; + const result = findBestHighlightTokenMatch(patternTokens, tokenTexts); const response: TokenMatchResponse = { - ...responseBase, - bestStart, - bestEnd, - rating: bestRating, - lengthDiff: bestLengthDiff, + id, + type: 'tokenMatchResult', + ...result, }; (self as unknown as DedicatedWorkerGlobalScope).postMessage(response); -}; \ No newline at end of file +}; diff --git a/src/lib/client/pdf.ts b/src/lib/client/pdf.ts index 32a1a7e..b81234d 100644 --- a/src/lib/client/pdf.ts +++ b/src/lib/client/pdf.ts @@ -13,7 +13,7 @@ const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); interface HighlightTokenMatchRequest { id: string; type: 'tokenMatch'; - pattern: string; + patternTokens: string[]; tokenTexts: string[]; } @@ -46,7 +46,7 @@ function getHighlightWorker(): Worker | null { } function runHighlightTokenMatch( - pattern: string, + patternTokens: string[], tokenTexts: string[] ): Promise { const worker = getHighlightWorker(); @@ -71,7 +71,7 @@ function runHighlightTokenMatch( const message: HighlightTokenMatchRequest = { id, type: 'tokenMatch', - pattern, + patternTokens, tokenTexts, }; worker.postMessage(message); @@ -612,9 +612,10 @@ export function highlightPattern( }; const tokenTexts = tokens.map((t) => t.text); + const patternTokens = segmentWords(cleanPattern, options?.language).map((token) => token.text); // Fire-and-forget async worker call; UI thread returns immediately - runHighlightTokenMatch(cleanPattern, tokenTexts) + runHighlightTokenMatch(patternTokens, tokenTexts) .then((result) => { if (seq !== highlightPatternSeq) return; if (!result || result.bestStart === -1) { diff --git a/tests/unit/epub-word-highlight.vitest.spec.ts b/tests/unit/epub-word-highlight.vitest.spec.ts index ad1f53c..20f0266 100644 --- a/tests/unit/epub-word-highlight.vitest.spec.ts +++ b/tests/unit/epub-word-highlight.vitest.spec.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'vitest'; import { buildMonotonicWordToTokenMap, + resolveAlignmentWordSourceRange, tokenizeCanonicalSegment, } from '../../src/lib/client/epub/epub-word-highlight'; import type { CanonicalTtsSegment } from '../../src/lib/shared/tts-segment-plan'; @@ -38,6 +39,35 @@ describe('EPUB word highlight mapping', () => { expect(chinese.map((token) => token.norm).join('')).toBe('这是中文'); }); + test('resolves Japanese alignment chunks directly from character offsets', () => { + const japanese = segment('これは日本語です。', 25); + const word: TTSSentenceAlignment['words'][number] = { + text: 'これは', + startSec: 0, + endSec: 0.5, + charStart: 0, + charEnd: 3, + }; + + expect(resolveAlignmentWordSourceRange(japanese, word)).toEqual({ + sourceStart: 25, + sourceEnd: 28, + }); + }); + + test('rejects invalid alignment character offsets so token mapping can be used', () => { + const japanese = segment('これは日本語です。', 25); + const word: TTSSentenceAlignment['words'][number] = { + text: '範囲外', + startSec: 0, + endSec: 0.5, + charStart: 20, + charEnd: 23, + }; + + expect(resolveAlignmentWordSourceRange(japanese, word)).toBeNull(); + }); + test('tokenizes canonical segment words with source offsets', () => { const tokens = tokenizeCanonicalSegment(segment('"Hello," she said.', 12)); diff --git a/tests/unit/pdf-highlight-match.vitest.spec.ts b/tests/unit/pdf-highlight-match.vitest.spec.ts new file mode 100644 index 0000000..716fc26 --- /dev/null +++ b/tests/unit/pdf-highlight-match.vitest.spec.ts @@ -0,0 +1,29 @@ +import { describe, expect, test } from 'vitest'; + +import { findBestHighlightTokenMatch } from '../../src/lib/client/pdf-highlight-match'; +import { segmentWords } from '../../src/lib/shared/language'; + +describe('PDF highlight token matching', () => { + test('matches a complete Japanese sentence using locale-aware token count', () => { + const sentence = 'これは日本語です。'; + const patternTokens = segmentWords(sentence, 'ja').map((token) => token.text); + const tokenTexts = ['前文', ...patternTokens, '次文']; + + expect(findBestHighlightTokenMatch(patternTokens, tokenTexts)).toMatchObject({ + bestStart: 1, + bestEnd: patternTokens.length, + lengthDiff: 0, + }); + }); + + test('matches spaced Latin text without relying on whitespace tokens', () => { + expect(findBestHighlightTokenMatch( + ['hello', 'world'], + ['before', 'hello', 'world', 'after'], + )).toMatchObject({ + bestStart: 1, + bestEnd: 2, + lengthDiff: 0, + }); + }); +}); From 81bd355bd1d225f485e5a84f569a996db0f2d9ec Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 6 Jun 2026 10:45:19 -0600 Subject: [PATCH 05/12] refactor(highlight): share multilingual token alignment --- src/hooks/epub/useEPUBHighlighting.ts | 30 ++- src/lib/client/epub/epub-word-highlight.ts | 69 +----- src/lib/client/highlight-token-alignment.ts | 232 ++++++++++++++++++ src/lib/client/html/highlight.ts | 187 ++------------ src/lib/client/pdf-highlight-match.ts | 101 -------- src/lib/client/pdf-highlight-worker.ts | 7 +- src/lib/client/pdf.ts | 203 ++++----------- tests/unit/epub-word-highlight.vitest.spec.ts | 29 --- .../highlight-token-alignment.vitest.spec.ts | 92 +++++++ tests/unit/pdf-highlight-match.vitest.spec.ts | 29 --- 10 files changed, 432 insertions(+), 547 deletions(-) create mode 100644 src/lib/client/highlight-token-alignment.ts delete mode 100644 src/lib/client/pdf-highlight-match.ts create mode 100644 tests/unit/highlight-token-alignment.vitest.spec.ts delete mode 100644 tests/unit/pdf-highlight-match.vitest.spec.ts diff --git a/src/hooks/epub/useEPUBHighlighting.ts b/src/hooks/epub/useEPUBHighlighting.ts index 83beafa..7362d3e 100644 --- a/src/hooks/epub/useEPUBHighlighting.ts +++ b/src/hooks/epub/useEPUBHighlighting.ts @@ -4,12 +4,15 @@ import { useCallback, useEffect, type MutableRefObject, type RefObject } from 'r import type { Rendition } from 'epubjs'; import { - buildMonotonicWordToTokenMap, buildWordHighlightCacheKey, resolveAlignmentWordSourceRange, tokenizeCanonicalSegment, type EpubCanonicalWordToken, } from '@/lib/client/epub/epub-word-highlight'; +import { + buildAlignmentTokenRanges, + type HighlightTokenRange, +} from '@/lib/client/highlight-token-alignment'; import { createRangeFromMappedOffsets, resolveVisibleSegmentRange, @@ -20,7 +23,7 @@ import type { TTSSentenceAlignment } from '@/types/tts'; export type EpubWordHighlightMapCache = { key: string; - wordToToken: number[]; + wordToTokenRange: Array; tokens: EpubCanonicalWordToken[]; }; @@ -159,19 +162,28 @@ export function useEPUBHighlighting({ wordHighlightMapCacheRef.current = { key: cacheKey, tokens, - wordToToken: buildMonotonicWordToTokenMap(words, tokens), + wordToTokenRange: buildAlignmentTokenRanges( + words, + tokens.map((token) => token.norm), + { minimumSimilarity: 0.8 }, + ), }; } const cached = wordHighlightMapCacheRef.current; - const tokenIndex = cached.wordToToken[wordIndex] ?? -1; - if (tokenIndex < 0) return; + const tokenRange = cached.wordToTokenRange[wordIndex]; + if (!tokenRange) return; - const token = cached.tokens[tokenIndex]; - if (!token) return; - if (token.sourceStart < resolved.startOffset || token.sourceEnd > resolved.endOffset) return; + const firstToken = cached.tokens[tokenRange.start]; + const lastToken = cached.tokens[tokenRange.end]; + if (!firstToken || !lastToken) return; + if (firstToken.sourceStart < resolved.startOffset || lastToken.sourceEnd > resolved.endOffset) return; - const wordRange = createRangeFromMappedOffsets(resolved.map, token.sourceStart, token.sourceEnd); + const wordRange = createRangeFromMappedOffsets( + resolved.map, + firstToken.sourceStart, + lastToken.sourceEnd, + ); if (!wordRange) return; try { diff --git a/src/lib/client/epub/epub-word-highlight.ts b/src/lib/client/epub/epub-word-highlight.ts index 7007ce5..de44021 100644 --- a/src/lib/client/epub/epub-word-highlight.ts +++ b/src/lib/client/epub/epub-word-highlight.ts @@ -1,6 +1,7 @@ import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; import type { TTSSentenceAlignment, TTSSentenceWord } from '@/types/tts'; -import { normalizeUnicodeToken, segmentWords } from '@/lib/shared/language'; +import { segmentWords } from '@/lib/shared/language'; +import { normalizeHighlightToken } from '@/lib/client/highlight-token-alignment'; export type EpubCanonicalWordToken = { norm: string; @@ -9,7 +10,7 @@ export type EpubCanonicalWordToken = { }; export const normalizeWordForHighlight = (text: string): string => - normalizeUnicodeToken(text); + normalizeHighlightToken(text); export const resolveAlignmentWordSourceRange = ( segment: CanonicalTtsSegment, @@ -37,70 +38,6 @@ export const tokenizeCanonicalSegment = ( })) .filter((token) => Boolean(token.norm)); -export const buildMonotonicWordToTokenMap = ( - alignmentWords: TTSSentenceAlignment['words'], - segmentTokens: EpubCanonicalWordToken[], -): number[] => { - const alignmentTokens = alignmentWords.map((word) => normalizeWordForHighlight(word.text)); - const wordToToken = new Array(alignmentWords.length).fill(-1); - const m = alignmentTokens.length; - const n = segmentTokens.length; - if (!m || !n) return wordToToken; - - const dp: number[][] = Array.from({ length: m + 1 }, () => new Array(n + 1).fill(0)); - const bt: number[][] = Array.from({ length: m + 1 }, () => new Array(n + 1).fill(0)); - - for (let i = 1; i <= m; i += 1) { - for (let j = 1; j <= n; j += 1) { - let best = dp[i - 1][j - 1]; - let move = 0; - - const alignmentNorm = alignmentTokens[i - 1]; - const segmentNorm = segmentTokens[j - 1].norm; - if (alignmentNorm && alignmentNorm === segmentNorm) { - const positionPenalty = - m <= 1 || n <= 1 - ? 0 - : Math.abs((i - 1) / (m - 1) - (j - 1) / (n - 1)); - best = dp[i - 1][j - 1] + 10 - positionPenalty; - move = 1; - } - - if (dp[i - 1][j] > best) { - best = dp[i - 1][j]; - move = 2; - } - if (dp[i][j - 1] > best) { - best = dp[i][j - 1]; - move = 3; - } - - dp[i][j] = best; - bt[i][j] = move; - } - } - - let i = m; - let j = n; - while (i > 0 && j > 0) { - const move = bt[i][j]; - if (move === 1) { - wordToToken[i - 1] = j - 1; - i -= 1; - j -= 1; - } else if (move === 2) { - i -= 1; - } else if (move === 3) { - j -= 1; - } else { - i -= 1; - j -= 1; - } - } - - return wordToToken; -}; - export const buildWordHighlightCacheKey = ( segment: CanonicalTtsSegment, alignment: TTSSentenceAlignment, diff --git a/src/lib/client/highlight-token-alignment.ts b/src/lib/client/highlight-token-alignment.ts new file mode 100644 index 0000000..26210ad --- /dev/null +++ b/src/lib/client/highlight-token-alignment.ts @@ -0,0 +1,232 @@ +import { CmpStr } from 'cmpstr'; + +import { normalizeUnicodeToken } from '@/lib/shared/language'; +import type { TTSSentenceAlignment } from '@/types/tts'; + +const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); + +export interface HighlightTokenRange { + start: number; + end: number; +} + +export interface HighlightTokenMatchResult extends HighlightTokenRange { + rating: number; + lengthDiff: number; +} + +export function normalizeHighlightToken(text: string): string { + return normalizeUnicodeToken(text); +} + +export function findBestHighlightTokenMatch( + patternTokens: string[], + tokenTexts: string[], +): HighlightTokenMatchResult { + const normalizedPattern = patternTokens.map(normalizeHighlightToken).filter(Boolean); + const normalizedTargets = tokenTexts.map(normalizeHighlightToken); + const cleanPattern = normalizedPattern.join(''); + const patternLen = cleanPattern.length; + const responseBase: HighlightTokenMatchResult = { + start: -1, + end: -1, + rating: 0, + lengthDiff: Number.POSITIVE_INFINITY, + }; + + if (!patternLen || !normalizedTargets.length) return responseBase; + + const patternTokenCount = normalizedPattern.length; + const minWindowTokens = Math.max(1, Math.floor(patternTokenCount * 0.6)); + const maxWindowTokens = Math.max( + minWindowTokens, + Math.ceil(patternTokenCount * 1.4), + ); + + let bestStart = -1; + let bestEnd = -1; + let bestRating = 0; + let bestLengthDiff = Number.POSITIVE_INFINITY; + + for (let start = 0; start < normalizedTargets.length; start += 1) { + let combined = ''; + + for ( + let offset = 0; + offset < maxWindowTokens && start + offset < normalizedTargets.length; + offset += 1 + ) { + combined += normalizedTargets[start + offset]; + + const windowSize = offset + 1; + if (windowSize < minWindowTokens) continue; + if (combined.length > patternLen * 2) break; + + const similarity = cmp.compare(combined, cleanPattern); + const lengthDiff = Math.abs(combined.length - patternLen); + const lengthPenalty = lengthDiff / patternLen; + const adjustedRating = similarity * (1 - lengthPenalty * 0.3); + + let boostedRating = adjustedRating; + const maxPrefixCheck = Math.min(windowSize, normalizedPattern.length, 5); + let prefixMatches = 0; + for (let i = 0; i < maxPrefixCheck; i += 1) { + const tokenSim = cmp.compare(normalizedTargets[start + i], normalizedPattern[i]); + if (tokenSim < 0.8) break; + prefixMatches += 1; + } + + if (prefixMatches > 0) { + boostedRating = adjustedRating * (1 + (prefixMatches / maxPrefixCheck) * 0.25); + } + + if ( + boostedRating > bestRating + || ( + Math.abs(boostedRating - bestRating) < 1e-3 + && lengthDiff < bestLengthDiff + ) + ) { + bestRating = boostedRating; + bestLengthDiff = lengthDiff; + bestStart = start; + bestEnd = start + offset; + } + } + } + + return { + start: bestStart, + end: bestEnd, + rating: bestRating, + lengthDiff: bestLengthDiff, + }; +} + +function buildExactConcatenatedRanges( + alignmentNorms: string[], + targetNorms: string[], +): Array | null { + if (alignmentNorms.join('') !== targetNorms.join('')) return null; + + const targetOffsets: HighlightTokenRange[] = []; + let targetCursor = 0; + for (const norm of targetNorms) { + targetOffsets.push({ start: targetCursor, end: targetCursor + norm.length }); + targetCursor += norm.length; + } + + const ranges: Array = []; + let alignmentCursor = 0; + for (const norm of alignmentNorms) { + const alignmentStart = alignmentCursor; + const alignmentEnd = alignmentStart + norm.length; + alignmentCursor = alignmentEnd; + + let first = -1; + let last = -1; + for (let i = 0; i < targetOffsets.length; i += 1) { + const target = targetOffsets[i]; + if (target.end <= alignmentStart) continue; + if (target.start >= alignmentEnd) break; + if (first === -1) first = i; + last = i; + } + ranges.push(first === -1 ? null : { start: first, end: last }); + } + + return ranges; +} + +export function buildAlignmentTokenRanges( + alignmentWords: TTSSentenceAlignment['words'], + targetTexts: string[], + options: { fillGaps?: boolean; minimumSimilarity?: number } = {}, +): Array { + const alignmentNorms = alignmentWords.map((word) => normalizeHighlightToken(word.text)); + const targetNorms = targetTexts.map(normalizeHighlightToken); + const ranges = new Array(alignmentWords.length).fill(null); + + const exactRanges = buildExactConcatenatedRanges(alignmentNorms, targetNorms); + if (exactRanges) return exactRanges; + + const targets = targetNorms + .map((norm, tokenIndex) => ({ norm, tokenIndex })) + .filter((token) => Boolean(token.norm)); + const alignments = alignmentNorms + .map((norm, wordIndex) => ({ norm, wordIndex })) + .filter((word) => Boolean(word.norm)); + const m = targets.length; + const n = alignments.length; + if (!m || !n) return ranges; + + const dp: number[][] = Array.from({ length: m + 1 }, () => + new Array(n + 1).fill(Number.POSITIVE_INFINITY), + ); + const bt: number[][] = Array.from({ length: m + 1 }, () => + new Array(n + 1).fill(0), + ); + dp[0][0] = 0; + const gapCost = 0.7; + + for (let i = 0; i <= m; i += 1) { + for (let j = 0; j <= n; j += 1) { + if (i > 0 && j > 0) { + const a = targets[i - 1].norm; + const b = alignments[j - 1].norm; + const similarity = a === b ? 1 : cmp.compare(a, b); + const candidate = dp[i - 1][j - 1] + (1 - similarity); + if (candidate < dp[i][j]) { + dp[i][j] = candidate; + bt[i][j] = 0; + } + } + if (i > 0 && dp[i - 1][j] + gapCost < dp[i][j]) { + dp[i][j] = dp[i - 1][j] + gapCost; + bt[i][j] = 1; + } + if (j > 0 && dp[i][j - 1] + gapCost < dp[i][j]) { + dp[i][j] = dp[i][j - 1] + gapCost; + bt[i][j] = 2; + } + } + } + + let i = m; + let j = n; + while (i > 0 || j > 0) { + const move = bt[i][j]; + if (i > 0 && j > 0 && move === 0) { + const tokenIndex = targets[i - 1].tokenIndex; + const a = targets[i - 1].norm; + const b = alignments[j - 1].norm; + const similarity = a === b ? 1 : cmp.compare(a, b); + if (similarity >= (options.minimumSimilarity ?? 0)) { + ranges[alignments[j - 1].wordIndex] = { start: tokenIndex, end: tokenIndex }; + } + i -= 1; + j -= 1; + } else if (i > 0 && (move === 1 || j === 0)) { + i -= 1; + } else if (j > 0 && (move === 2 || i === 0)) { + j -= 1; + } else { + break; + } + } + + if (!options.fillGaps) return ranges; + + let lastSeen: HighlightTokenRange | null = null; + for (let k = 0; k < ranges.length; k += 1) { + if (ranges[k]) lastSeen = ranges[k]; + else if (lastSeen) ranges[k] = lastSeen; + } + let nextSeen: HighlightTokenRange | null = null; + for (let k = ranges.length - 1; k >= 0; k -= 1) { + if (ranges[k]) nextSeen = ranges[k]; + else if (nextSeen) ranges[k] = nextSeen; + } + + return ranges; +} diff --git a/src/lib/client/html/highlight.ts b/src/lib/client/html/highlight.ts index db26cc2..a78a70a 100644 --- a/src/lib/client/html/highlight.ts +++ b/src/lib/client/html/highlight.ts @@ -11,14 +11,18 @@ * sentence * - `WORD` — saturated background on the currently-spoken word * - * Word-to-DOM alignment is done via Needleman-Wunsch (same approach the PDF - * reader uses) so DOM token counts that diverge from whisper's word count - * still produce a smooth, monotonic word highlight rather than a proportional - * approximation that snaps around when the counts disagree. + * Word-to-DOM alignment uses the shared viewer token-range mapper so DOM token + * counts that diverge from the timed alignment still produce a smooth, + * monotonic highlight across languages. */ -import { CmpStr } from 'cmpstr'; import type { TTSSentenceAlignment } from '@/types/tts'; -import { normalizeUnicodeToken, segmentWords } from '@/lib/shared/language'; +import { segmentWords } from '@/lib/shared/language'; +import { + buildAlignmentTokenRanges, + findBestHighlightTokenMatch, + normalizeHighlightToken, + type HighlightTokenRange, +} from '@/lib/client/highlight-token-alignment'; export const HTML_SENTENCE_CLASS = 'openreader-html-highlight-sentence'; export const HTML_WORD_CLASS = 'openreader-html-highlight-word'; @@ -30,8 +34,6 @@ interface DomToken { norm: string; } -const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); - let sentenceWraps: HTMLSpanElement[] = []; let wordWraps: HTMLSpanElement[] = []; @@ -46,15 +48,15 @@ interface SentenceState { // is in place. Stable across word wrap/unwrap cycles because clear() calls // `parent.normalize()` which restores the original text-node structure. wordTokens: DomToken[]; - // For an alignment we've already seen, the cached wordIndex → tokenIndex map. + // For an alignment we've already seen, the cached wordIndex → token range map. alignment: TTSSentenceAlignment | null; - wordToToken: number[] | null; + wordToTokenRange: Array | null; } let sentenceState: SentenceState | null = null; function normalizeWord(word: string): string { - return normalizeUnicodeToken(word); + return normalizeHighlightToken(word); } function tokenizePattern(pattern: string, language?: string): string[] { @@ -171,43 +173,9 @@ function collectTokensInsideWraps(wraps: HTMLSpanElement[], language?: string): } function findBestWindow(tokens: DomToken[], patternTokens: string[]): { start: number; end: number } | null { - if (!tokens.length || !patternTokens.length) return null; - const pLen = patternTokens.length; - - let bestStart = -1; - let bestEnd = -1; - let bestScore = 0; - - for (let i = 0; i + Math.max(1, Math.ceil(pLen * 0.5)) - 1 < tokens.length; i += 1) { - if (tokens[i].norm !== patternTokens[0]) continue; - let matches = 1; - let domCursor = i + 1; - for (let p = 1; p < pLen && domCursor < tokens.length; p += 1) { - let stepped = false; - for (let k = 0; k < 3 && domCursor + k < tokens.length; k += 1) { - if (tokens[domCursor + k].norm === patternTokens[p]) { - matches += 1; - domCursor += k + 1; - stepped = true; - break; - } - } - if (!stepped) domCursor += 1; - } - const end = Math.min(tokens.length - 1, domCursor - 1); - const score = matches / pLen; - if (score > bestScore) { - bestScore = score; - bestStart = i; - bestEnd = end; - if (score >= 0.95) break; - } - } - - if (bestScore >= 0.5 && bestStart !== -1) { - return { start: bestStart, end: bestEnd }; - } - return null; + const match = findBestHighlightTokenMatch(patternTokens, tokens.map((token) => token.norm)); + if (match.start === -1 || match.rating < 0.5) return null; + return { start: match.start, end: match.end }; } function wrapTokenRange(tokens: DomToken[], start: number, end: number, className: string): HTMLSpanElement[] { @@ -274,116 +242,11 @@ export function highlightHtmlSentence( sentence, wordTokens: collectTokensInsideWraps(sentenceWraps, language), alignment: null, - wordToToken: null, + wordToTokenRange: null, }; return true; } -/** - * Build a wordIndex → tokenIndex map via Needleman-Wunsch alignment between - * whisper's word list and the DOM tokens inside the sentence. Mirrors the - * approach in `src/lib/client/pdf.ts#highlightWordIndex` so PDF and HTML - * highlights behave the same way under count mismatches (contractions, - * stripped punctuation, missing whitespace, etc.). - */ -function buildAlignmentMap( - alignment: TTSSentenceAlignment, - domTokens: DomToken[], -): number[] { - const words = alignment.words || []; - const wordToToken = new Array(words.length).fill(-1); - - const domFiltered: { tokenIndex: number; norm: string }[] = []; - for (let i = 0; i < domTokens.length; i += 1) { - const norm = domTokens[i].norm; - if (norm) domFiltered.push({ tokenIndex: i, norm }); - } - - const ttsFiltered: { wordIndex: number; norm: string }[] = []; - for (let i = 0; i < words.length; i += 1) { - const norm = normalizeWord(words[i].text); - if (norm) ttsFiltered.push({ wordIndex: i, norm }); - } - - const m = domFiltered.length; - const n = ttsFiltered.length; - if (!m || !n) return wordToToken; - - const dp: number[][] = Array.from({ length: m + 1 }, () => - new Array(n + 1).fill(Number.POSITIVE_INFINITY), - ); - const bt: number[][] = Array.from({ length: m + 1 }, () => - new Array(n + 1).fill(0), - ); // 0=diag (substitute), 1=up (skip dom), 2=left (skip tts) - - dp[0][0] = 0; - const GAP_COST = 0.7; - - for (let i = 0; i <= m; i += 1) { - for (let j = 0; j <= n; j += 1) { - if (i > 0 && j > 0) { - const a = domFiltered[i - 1].norm; - const b = ttsFiltered[j - 1].norm; - const sim = a === b ? 1 : cmp.compare(a, b); - const cand = dp[i - 1][j - 1] + (1 - sim); - if (cand < dp[i][j]) { - dp[i][j] = cand; - bt[i][j] = 0; - } - } - if (i > 0) { - const cand = dp[i - 1][j] + GAP_COST; - if (cand < dp[i][j]) { - dp[i][j] = cand; - bt[i][j] = 1; - } - } - if (j > 0) { - const cand = dp[i][j - 1] + GAP_COST; - if (cand < dp[i][j]) { - dp[i][j] = cand; - bt[i][j] = 2; - } - } - } - } - - let i = m; - let j = n; - while (i > 0 || j > 0) { - const move = bt[i][j]; - if (i > 0 && j > 0 && move === 0) { - const domIdx = domFiltered[i - 1].tokenIndex; - const ttsIdx = ttsFiltered[j - 1].wordIndex; - if (wordToToken[ttsIdx] === -1) wordToToken[ttsIdx] = domIdx; - i -= 1; - j -= 1; - } else if (i > 0 && (move === 1 || j === 0)) { - i -= 1; - } else if (j > 0 && (move === 2 || i === 0)) { - j -= 1; - } else { - break; - } - } - - // Forward-fill, then backward-fill, so every wordIndex has a nearest known - // DOM token. This keeps the word highlight stable when whisper emits a - // word that didn't survive normalization (e.g. an apostrophe-only token). - let lastSeen = -1; - for (let k = 0; k < wordToToken.length; k += 1) { - if (wordToToken[k] !== -1) lastSeen = wordToToken[k]; - else if (lastSeen !== -1) wordToToken[k] = lastSeen; - } - let nextSeen = -1; - for (let k = wordToToken.length - 1; k >= 0; k -= 1) { - if (wordToToken[k] !== -1) nextSeen = wordToToken[k]; - else if (nextSeen !== -1) wordToToken[k] = nextSeen; - } - - return wordToToken; -} - export function highlightHtmlWord( container: HTMLElement | null | undefined, alignment: TTSSentenceAlignment | undefined, @@ -403,16 +266,20 @@ export function highlightHtmlWord( if (!words.length || wordIndex >= words.length) return false; // (Re)build the alignment map when this is a new alignment object. - if (sentenceState.alignment !== alignment || !sentenceState.wordToToken) { + if (sentenceState.alignment !== alignment || !sentenceState.wordToTokenRange) { sentenceState.alignment = alignment; - sentenceState.wordToToken = buildAlignmentMap(alignment, sentenceState.wordTokens); + sentenceState.wordToTokenRange = buildAlignmentTokenRanges( + alignment.words, + sentenceState.wordTokens.map((token) => token.norm), + { fillGaps: true }, + ); } - const tokenIndex = sentenceState.wordToToken[wordIndex]; - if (tokenIndex === undefined || tokenIndex < 0) return false; - if (tokenIndex >= sentenceState.wordTokens.length) return false; + const tokenRange = sentenceState.wordToTokenRange[wordIndex]; + if (!tokenRange) return false; + if (tokenRange.start < 0 || tokenRange.end >= sentenceState.wordTokens.length) return false; - wordWraps = wrapTokenRange(sentenceState.wordTokens, tokenIndex, tokenIndex, HTML_WORD_CLASS); + wordWraps = wrapTokenRange(sentenceState.wordTokens, tokenRange.start, tokenRange.end, HTML_WORD_CLASS); return wordWraps.length > 0; } diff --git a/src/lib/client/pdf-highlight-match.ts b/src/lib/client/pdf-highlight-match.ts deleted file mode 100644 index b16ef5d..0000000 --- a/src/lib/client/pdf-highlight-match.ts +++ /dev/null @@ -1,101 +0,0 @@ -import { CmpStr } from 'cmpstr'; - -const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); - -export interface HighlightTokenMatchResult { - bestStart: number; - bestEnd: number; - rating: number; - lengthDiff: number; -} - -export function findBestHighlightTokenMatch( - patternTokens: string[], - tokenTexts: string[], -): HighlightTokenMatchResult { - const cleanPatternTokens = patternTokens.map((token) => token.trim()).filter(Boolean); - const cleanPattern = cleanPatternTokens.join(''); - const patternLen = cleanPattern.length; - const responseBase: HighlightTokenMatchResult = { - bestStart: -1, - bestEnd: -1, - rating: 0, - lengthDiff: Number.POSITIVE_INFINITY, - }; - - if (!patternLen || !tokenTexts.length) return responseBase; - - const patternTokenCount = cleanPatternTokens.length; - const minWindowTokens = Math.max(1, Math.floor(patternTokenCount * 0.6)); - const maxWindowTokens = Math.max( - minWindowTokens, - Math.ceil(patternTokenCount * 1.4), - ); - - let bestStart = -1; - let bestEnd = -1; - let bestRating = 0; - let bestLengthDiff = Number.POSITIVE_INFINITY; - - for (let start = 0; start < tokenTexts.length; start += 1) { - let combined = ''; - - for ( - let offset = 0; - offset < maxWindowTokens && start + offset < tokenTexts.length; - offset += 1 - ) { - const token = tokenTexts[start + offset]; - combined += token; - - const windowSize = offset + 1; - if (windowSize < minWindowTokens) continue; - if (combined.length > patternLen * 2) break; - - const similarity = cmp.compare(combined, cleanPattern); - const lengthDiff = Math.abs(combined.length - patternLen); - const lengthPenalty = lengthDiff / patternLen; - const adjustedRating = similarity * (1 - lengthPenalty * 0.3); - - let boostedRating = adjustedRating; - const windowTokens = tokenTexts.slice(start, start + windowSize); - const maxPrefixCheck = Math.min( - windowTokens.length, - cleanPatternTokens.length, - 5, - ); - - let prefixMatches = 0; - for (let i = 0; i < maxPrefixCheck; i += 1) { - const tokenSim = cmp.compare(windowTokens[i], cleanPatternTokens[i]); - if (tokenSim < 0.8) break; - prefixMatches += 1; - } - - if (prefixMatches > 0) { - const prefixRatio = prefixMatches / maxPrefixCheck; - boostedRating = adjustedRating * (1 + prefixRatio * 0.25); - } - - if ( - boostedRating > bestRating - || ( - Math.abs(boostedRating - bestRating) < 1e-3 - && lengthDiff < bestLengthDiff - ) - ) { - bestRating = boostedRating; - bestLengthDiff = lengthDiff; - bestStart = start; - bestEnd = start + offset; - } - } - } - - return { - bestStart, - bestEnd, - rating: bestRating, - lengthDiff: bestLengthDiff, - }; -} diff --git a/src/lib/client/pdf-highlight-worker.ts b/src/lib/client/pdf-highlight-worker.ts index 6b396af..fa3fbd6 100644 --- a/src/lib/client/pdf-highlight-worker.ts +++ b/src/lib/client/pdf-highlight-worker.ts @@ -1,6 +1,6 @@ /// -import { findBestHighlightTokenMatch } from './pdf-highlight-match'; +import { findBestHighlightTokenMatch } from './highlight-token-alignment'; interface TokenMatchRequest { id: string; @@ -43,7 +43,10 @@ self.onmessage = (event: MessageEvent) => { const response: TokenMatchResponse = { id, type: 'tokenMatchResult', - ...result, + bestStart: result.start, + bestEnd: result.end, + rating: result.rating, + lengthDiff: result.lengthDiff, }; (self as unknown as DedicatedWorkerGlobalScope).postMessage(response); diff --git a/src/lib/client/pdf.ts b/src/lib/client/pdf.ts index b81234d..1293957 100644 --- a/src/lib/client/pdf.ts +++ b/src/lib/client/pdf.ts @@ -3,11 +3,12 @@ import { TextLayer } from 'pdfjs-dist'; import "core-js/proposals/promise-with-resolvers"; import type { TTSSentenceAlignment } from '@/types/tts'; import type { ParsedPdfDocument, ParsedPdfPage } from '@/types/parsed-pdf'; -import { CmpStr } from 'cmpstr'; import type { TTSSegmentLocator } from '@/types/client'; -import { normalizeUnicodeToken, segmentWords } from '@/lib/shared/language'; - -const cmp = CmpStr.create().setMetric('dice').setFlags('itw'); +import { segmentWords } from '@/lib/shared/language'; +import { + buildAlignmentTokenRanges, + type HighlightTokenRange, +} from '@/lib/client/highlight-token-alignment'; // Worker coordination for offloading highlight token matching interface HighlightTokenMatchRequest { @@ -152,7 +153,7 @@ let lastSpanNodes: HTMLElement[] = []; let lastTokens: PDFToken[] = []; let lastSentenceTokenWindow: { start: number; end: number } | null = null; let lastSentencePattern: string | null = null; -let lastSentenceWordToTokenMap: number[] | null = null; +let lastSentenceWordToTokenRangeMap: Array | null = null; function getOrCreateHighlightLayer(span: HTMLElement): { layer: HTMLElement; @@ -181,9 +182,6 @@ function getOrCreateHighlightLayer(span: HTMLElement): { return { layer, pageElement, pageRect: pageElement.getBoundingClientRect() }; } -const normalizeWordForMatch = (text: string): string => - normalizeUnicodeToken(text); - // Highlighting functions let highlightPatternSeq = 0; @@ -421,7 +419,7 @@ export function highlightPattern( const cleanPattern = pattern.trim().replace(/\s+/g, ' '); if (!cleanPattern) return; lastSentencePattern = cleanPattern; - lastSentenceWordToTokenMap = null; + lastSentenceWordToTokenRangeMap = null; lastSentenceTokenWindow = null; const parsedDocument = options?.parsedDocument ?? null; const locator = options?.locator ?? null; @@ -672,158 +670,61 @@ export function highlightWordIndex( const end = lastSentenceTokenWindow.end; if (end < start) return; - // Lazily build or refresh the mapping from alignment word - // indices to PDF token indices for this sentence window. + // Lazily build or refresh the shared mapping from alignment words to PDF + // token ranges for this sentence window. if ( - !lastSentenceWordToTokenMap || - lastSentenceWordToTokenMap.length !== words.length + !lastSentenceWordToTokenRangeMap || + lastSentenceWordToTokenRangeMap.length !== words.length ) { - const pdfFiltered: { tokenIndex: number; norm: string }[] = []; - for (let i = start; i <= end; i++) { - const norm = normalizeWordForMatch(lastTokens[i].text); - if (!norm) continue; - pdfFiltered.push({ tokenIndex: i, norm }); - } - - const ttsFiltered: { wordIndex: number; norm: string }[] = []; - for (let i = 0; i < words.length; i++) { - const norm = normalizeWordForMatch(words[i].text); - if (!norm) continue; - ttsFiltered.push({ wordIndex: i, norm }); - } - - const wordToToken = new Array(words.length).fill(-1); - - const m = pdfFiltered.length; - const n = ttsFiltered.length; - - if (m && n) { - const dp: number[][] = Array.from({ length: m + 1 }, () => - new Array(n + 1).fill(Number.POSITIVE_INFINITY) - ); - const bt: number[][] = Array.from({ length: m + 1 }, () => - new Array(n + 1).fill(0) - ); // 0=diag,1=up,2=left - - dp[0][0] = 0; - const GAP_COST = 0.7; - - for (let i = 0; i <= m; i++) { - for (let j = 0; j <= n; j++) { - if (i > 0 && j > 0) { - const a = pdfFiltered[i - 1].norm; - const b = ttsFiltered[j - 1].norm; - const sim = cmp.compare(a, b); - const subCost = 1 - sim; - const cand = dp[i - 1][j - 1] + subCost; - if (cand < dp[i][j]) { - dp[i][j] = cand; - bt[i][j] = 0; - } - } - if (i > 0) { - const cand = dp[i - 1][j] + GAP_COST; - if (cand < dp[i][j]) { - dp[i][j] = cand; - bt[i][j] = 1; - } - } - if (j > 0) { - const cand = dp[i][j - 1] + GAP_COST; - if (cand < dp[i][j]) { - dp[i][j] = cand; - bt[i][j] = 2; - } - } - } - } - - let i = m; - let j = n; - while (i > 0 || j > 0) { - const move = bt[i][j]; - if (i > 0 && j > 0 && move === 0) { - const pdfIdx = pdfFiltered[i - 1].tokenIndex; - const ttsIdx = ttsFiltered[j - 1].wordIndex; - if (wordToToken[ttsIdx] === -1) { - wordToToken[ttsIdx] = pdfIdx; - } - i -= 1; - j -= 1; - } else if (i > 0 && (move === 1 || j === 0)) { - i -= 1; - } else if (j > 0 && (move === 2 || i === 0)) { - j -= 1; - } else { - break; - } - } - - // Propagate nearest known mapping to fill gaps - let lastSeen = -1; - for (let k = 0; k < wordToToken.length; k++) { - if (wordToToken[k] !== -1) { - lastSeen = wordToToken[k]; - } else if (lastSeen !== -1) { - wordToToken[k] = lastSeen; - } - } - let nextSeen = -1; - for (let k = wordToToken.length - 1; k >= 0; k--) { - if (wordToToken[k] !== -1) { - nextSeen = wordToToken[k]; - } else if (nextSeen !== -1) { - wordToToken[k] = nextSeen; - } - } - } - - lastSentenceWordToTokenMap = wordToToken; + const relativeRanges = buildAlignmentTokenRanges( + words, + lastTokens.slice(start, end + 1).map((token) => token.text), + { fillGaps: true }, + ); + lastSentenceWordToTokenRangeMap = relativeRanges.map((range) => ( + range ? { start: range.start + start, end: range.end + start } : null + )); } - const mappedIndex = - lastSentenceWordToTokenMap && wordIndex < lastSentenceWordToTokenMap.length - ? lastSentenceWordToTokenMap[wordIndex] - : -1; + const tokenRange = lastSentenceWordToTokenRangeMap[wordIndex]; + if (!tokenRange) return; - if (mappedIndex === -1) return; + for (let tokenIndex = tokenRange.start; tokenIndex <= tokenRange.end; tokenIndex += 1) { + const token = lastTokens[tokenIndex]; + const span = lastSpanNodes[token.spanIndex]; + if (!span) continue; - const chosenTokenIndex = mappedIndex; + const node = token.textNode; + if (!node || node.nodeType !== Node.TEXT_NODE) continue; - const token = lastTokens[chosenTokenIndex]; - const span = lastSpanNodes[token.spanIndex]; - if (!span) return; + try { + const range = document.createRange(); + range.setStart(node, token.startOffset); + range.setEnd(node, token.endOffset); - const node = token.textNode; - if (!node || node.nodeType !== Node.TEXT_NODE) return; + const highlightTarget = getOrCreateHighlightLayer(span); + if (!highlightTarget) continue; - try { - const range = document.createRange(); - range.setStart(node, token.startOffset); - range.setEnd(node, token.endOffset); + const { layer: highlightLayer, pageRect } = highlightTarget; + const rects = Array.from(range.getClientRects()); - const highlightTarget = getOrCreateHighlightLayer(span); - if (!highlightTarget) return; - - const { layer: highlightLayer, pageRect } = highlightTarget; - const rects = Array.from(range.getClientRects()); - - rects.forEach((rect) => { - const highlight = document.createElement('div'); - highlight.className = 'pdf-word-highlight-overlay'; - highlight.style.position = 'absolute'; - highlight.style.backgroundColor = 'var(--accent)'; - highlight.style.opacity = '0.4'; - highlight.style.pointerEvents = 'none'; - highlight.style.left = `${rect.left - pageRect.left}px`; - highlight.style.top = `${rect.top - pageRect.top}px`; - highlight.style.width = `${rect.width}px`; - highlight.style.height = `${rect.height}px`; - highlight.style.zIndex = '2'; - highlightLayer.appendChild(highlight); - }); - } catch { - // Ignore range errors + rects.forEach((rect) => { + const highlight = document.createElement('div'); + highlight.className = 'pdf-word-highlight-overlay'; + highlight.style.position = 'absolute'; + highlight.style.backgroundColor = 'var(--accent)'; + highlight.style.opacity = '0.4'; + highlight.style.pointerEvents = 'none'; + highlight.style.left = `${rect.left - pageRect.left}px`; + highlight.style.top = `${rect.top - pageRect.top}px`; + highlight.style.width = `${rect.width}px`; + highlight.style.height = `${rect.height}px`; + highlight.style.zIndex = '2'; + highlightLayer.appendChild(highlight); + }); + } catch { + // Ignore range errors + } } } diff --git a/tests/unit/epub-word-highlight.vitest.spec.ts b/tests/unit/epub-word-highlight.vitest.spec.ts index 20f0266..6d0e3cc 100644 --- a/tests/unit/epub-word-highlight.vitest.spec.ts +++ b/tests/unit/epub-word-highlight.vitest.spec.ts @@ -1,7 +1,6 @@ import { describe, expect, test } from 'vitest'; import { - buildMonotonicWordToTokenMap, resolveAlignmentWordSourceRange, tokenizeCanonicalSegment, } from '../../src/lib/client/epub/epub-word-highlight'; @@ -19,15 +18,6 @@ const segment = (text: string, offset = 0): CanonicalTtsSegment => ({ spansSourceBoundary: false, }); -const alignmentWords = (words: string[]): TTSSentenceAlignment['words'] => - words.map((word, index) => ({ - text: word, - startSec: index, - endSec: index + 0.5, - charStart: 0, - charEnd: word.length, - })); - describe('EPUB word highlight mapping', () => { test('tokenizes Japanese and Chinese using locale-aware word boundaries', () => { const japanese = tokenizeCanonicalSegment(segment('これは日本語です。', 5), 'ja'); @@ -78,23 +68,4 @@ describe('EPUB word highlight mapping', () => { ]); }); - test('maps repeated words monotonically instead of jumping to later duplicates', () => { - const tokens = tokenizeCanonicalSegment(segment('the light and the shadow and the light returned')); - const map = buildMonotonicWordToTokenMap( - alignmentWords(['the', 'light', 'and', 'the', 'shadow', 'and', 'the', 'light', 'returned']), - tokens, - ); - - expect(map).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8]); - }); - - test('leaves unmatched alignment words unhighlighted instead of borrowing a neighbor', () => { - const tokens = tokenizeCanonicalSegment(segment('alpha beta gamma')); - const map = buildMonotonicWordToTokenMap( - alignmentWords(['alpha', 'missing', 'gamma']), - tokens, - ); - - expect(map).toEqual([0, -1, 2]); - }); }); diff --git a/tests/unit/highlight-token-alignment.vitest.spec.ts b/tests/unit/highlight-token-alignment.vitest.spec.ts new file mode 100644 index 0000000..7ee3212 --- /dev/null +++ b/tests/unit/highlight-token-alignment.vitest.spec.ts @@ -0,0 +1,92 @@ +import { describe, expect, test } from 'vitest'; + +import { + buildAlignmentTokenRanges, + findBestHighlightTokenMatch, +} from '../../src/lib/client/highlight-token-alignment'; +import { segmentWords } from '../../src/lib/shared/language'; +import type { TTSSentenceAlignment } from '../../src/types/tts'; + +const alignmentWords = ( + words: string[], +): TTSSentenceAlignment['words'] => + words.map((word, index) => ({ + text: word, + startSec: index, + endSec: index + 0.5, + charStart: 0, + charEnd: word.length, + })); + +describe('shared viewer highlight token alignment', () => { + test('matches a complete Japanese sentence using locale-aware token count', () => { + const sentence = 'これは日本語です。'; + const patternTokens = segmentWords(sentence, 'ja').map((token) => token.text); + const tokenTexts = ['前文', ...patternTokens, '次文']; + + expect(findBestHighlightTokenMatch(patternTokens, tokenTexts)).toMatchObject({ + start: 1, + end: patternTokens.length, + lengthDiff: 0, + }); + }); + + test('matches spaced Latin text without relying on whitespace tokens', () => { + expect(findBestHighlightTokenMatch( + ['hello', 'world'], + ['before', 'hello', 'world', 'after'], + )).toMatchObject({ + start: 1, + end: 2, + lengthDiff: 0, + }); + }); + + test('maps a Japanese timed chunk across every visible token it spans', () => { + expect(buildAlignmentTokenRanges( + alignmentWords(['これは', '日本語', 'です']), + ['これ', 'は', '日本語', 'です'], + )).toEqual([ + { start: 0, end: 1 }, + { start: 2, end: 2 }, + { start: 3, end: 3 }, + ]); + }); + + test('maps visible chunks back to a larger timed Japanese token', () => { + expect(buildAlignmentTokenRanges( + alignmentWords(['これ', 'は', '日本語', 'です']), + ['これは', '日本語', 'です'], + )).toEqual([ + { start: 0, end: 0 }, + { start: 0, end: 0 }, + { start: 1, end: 1 }, + { start: 2, end: 2 }, + ]); + }); + + test('maps repeated words monotonically', () => { + expect(buildAlignmentTokenRanges( + alignmentWords(['the', 'light', 'and', 'the', 'light']), + ['the', 'light', 'and', 'the', 'light'], + )).toEqual([ + { start: 0, end: 0 }, + { start: 1, end: 1 }, + { start: 2, end: 2 }, + { start: 3, end: 3 }, + { start: 4, end: 4 }, + ]); + }); + + test('can leave unrelated fallback tokens unmapped for strict viewers', () => { + expect(buildAlignmentTokenRanges( + alignmentWords(['alpha', 'missing', 'gamma']), + ['alpha', 'beta', 'gamma'], + { minimumSimilarity: 0.8 }, + )).toEqual([ + { start: 0, end: 0 }, + null, + { start: 2, end: 2 }, + ]); + }); +}); diff --git a/tests/unit/pdf-highlight-match.vitest.spec.ts b/tests/unit/pdf-highlight-match.vitest.spec.ts deleted file mode 100644 index 716fc26..0000000 --- a/tests/unit/pdf-highlight-match.vitest.spec.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { describe, expect, test } from 'vitest'; - -import { findBestHighlightTokenMatch } from '../../src/lib/client/pdf-highlight-match'; -import { segmentWords } from '../../src/lib/shared/language'; - -describe('PDF highlight token matching', () => { - test('matches a complete Japanese sentence using locale-aware token count', () => { - const sentence = 'これは日本語です。'; - const patternTokens = segmentWords(sentence, 'ja').map((token) => token.text); - const tokenTexts = ['前文', ...patternTokens, '次文']; - - expect(findBestHighlightTokenMatch(patternTokens, tokenTexts)).toMatchObject({ - bestStart: 1, - bestEnd: patternTokens.length, - lengthDiff: 0, - }); - }); - - test('matches spaced Latin text without relying on whitespace tokens', () => { - expect(findBestHighlightTokenMatch( - ['hello', 'world'], - ['before', 'hello', 'world', 'after'], - )).toMatchObject({ - bestStart: 1, - bestEnd: 2, - lengthDiff: 0, - }); - }); -}); From 112dbc0da9fde5f2ee622024fa5fb105ce84fa19 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 6 Jun 2026 10:50:05 -0600 Subject: [PATCH 06/12] fix(voices): prevent mixed-language Kokoro selections --- src/components/player/VoicesControlBase.tsx | 26 ++++++++++++++------- src/lib/shared/language.ts | 24 ++++++++++++------- tests/unit/language.vitest.spec.ts | 22 ++++++++++++++--- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/src/components/player/VoicesControlBase.tsx b/src/components/player/VoicesControlBase.tsx index fd2cad9..b314902 100644 --- a/src/components/player/VoicesControlBase.tsx +++ b/src/components/player/VoicesControlBase.tsx @@ -6,6 +6,7 @@ import { import { ChevronUpDownIcon, AudioWaveIcon, CheckIcon } from '@/components/icons/Icons'; import { useEffect, useMemo, useState } from 'react'; import { buildKokoroVoiceString, parseKokoroVoiceNames } from '@/lib/shared/kokoro'; +import { keepKokoroVoicesInOneLanguage } from '@/lib/shared/language'; import { type TtsProviderType } from '@/lib/shared/tts-provider-catalog'; import { resolveTtsProviderModelPolicy } from '@/lib/shared/tts-provider-policy'; import { SharedListboxButton, SharedListboxOption, SharedListboxOptions, cn } from '@/components/ui'; @@ -70,8 +71,13 @@ export function VoicesControlBase({ if (initial.length > maxVoices) { initial = initial.slice(0, maxVoices); } + initial = keepKokoroVoicesInOneLanguage(initial); setSelectedVoices(initial); - }, [isKokoro, maxVoices, voice, availableVoices]); + const combined = buildKokoroVoiceString(initial); + if (combined && combined !== voice) { + onChangeVoice(combined); + } + }, [isKokoro, maxVoices, voice, availableVoices, onChangeVoice]); const currentVoice = useMemo(() => { if (isKokoro && maxVoices > 1) { @@ -101,15 +107,19 @@ export function VoicesControlBase({ onChange={(vals: string[]) => { if (!vals || vals.length === 0) return; - let next = vals; - if (vals.length > maxVoices) { - const newlyAdded = vals.find((v) => !selectedVoices.includes(v)); + const newlyAdded = vals.find((v) => !selectedVoices.includes(v)); + let next = keepKokoroVoicesInOneLanguage(vals, newlyAdded); + if (next.length > maxVoices) { if (newlyAdded) { - const lastPrev = selectedVoices[selectedVoices.length - 1] ?? selectedVoices[0] ?? ''; - const pair = Array.from(new Set([lastPrev, newlyAdded])).filter(Boolean); - next = pair.slice(0, maxVoices); + const compatiblePrevious = selectedVoices.filter( + (voiceId) => voiceId !== newlyAdded && next.includes(voiceId), + ); + next = [ + ...compatiblePrevious.slice(-(maxVoices - 1)), + newlyAdded, + ]; } else { - next = vals.slice(-maxVoices); + next = next.slice(-maxVoices); } } diff --git a/src/lib/shared/language.ts b/src/lib/shared/language.ts index 9432235..b278bd0 100644 --- a/src/lib/shared/language.ts +++ b/src/lib/shared/language.ts @@ -90,6 +90,21 @@ export function getKokoroVoiceLanguages(voice: string | null | undefined): strin )); } +export function keepKokoroVoicesInOneLanguage( + voices: string[], + preferredVoice?: string | null, +): string[] { + const preferredLanguage = getKokoroVoiceLanguages(preferredVoice)[0] + ?? voices.flatMap((voice) => getKokoroVoiceLanguages(voice))[0]; + if (!preferredLanguage) return voices; + + const preferredBaseLanguage = toBaseLanguageCode(preferredLanguage); + return voices.filter((voice) => { + const voiceLanguage = getKokoroVoiceLanguages(voice)[0]; + return !voiceLanguage || toBaseLanguageCode(voiceLanguage) === preferredBaseLanguage; + }); +} + export function resolveTtsLanguage(input: { configuredLanguage?: string | null; voice?: string | null; @@ -130,14 +145,7 @@ export function getTtsLanguageCompatibilityWarnings(input: { ); } - if (voiceBaseLanguages.size > 1) { - warnings.push( - `Selected Kokoro voices use multiple languages (${voiceLanguages.map(getLanguageDisplayName).join(', ')}).`, - ); - return warnings; - } - - const voiceLanguage = voiceLanguages[0]; + const voiceLanguage = voiceBaseLanguages.size === 1 ? voiceLanguages[0] : undefined; if (voiceLanguage && toBaseLanguageCode(voiceLanguage) !== documentBaseLanguage) { warnings.push( `Selected Kokoro voice is ${getLanguageDisplayName(voiceLanguage)}, but the document is ${getLanguageDisplayName(documentLanguage)}.`, diff --git a/tests/unit/language.vitest.spec.ts b/tests/unit/language.vitest.spec.ts index 2842d80..b50552c 100644 --- a/tests/unit/language.vitest.spec.ts +++ b/tests/unit/language.vitest.spec.ts @@ -5,6 +5,7 @@ import { resolve } from 'path'; import { getTtsLanguageCompatibilityWarnings, inferKokoroLanguageFromVoice, + keepKokoroVoicesInOneLanguage, normalizeOptionalLanguageTag, normalizeUnicodeToken, resolveTtsLanguage, @@ -27,6 +28,23 @@ describe('multilingual language utilities', () => { expect(inferKokoroLanguageFromVoice('ff_siwis+jf_alpha')).toBeNull(); }); + test('keeps Kokoro multi-voice selections within the newly selected language', () => { + expect(keepKokoroVoicesInOneLanguage( + ['af_sarah', 'bf_emma', 'jf_alpha'], + 'jf_alpha', + )).toEqual(['jf_alpha']); + + expect(keepKokoroVoicesInOneLanguage( + ['zf_xiaobei', 'zm_yunxi', 'af_sarah'], + 'zm_yunxi', + )).toEqual(['zf_xiaobei', 'zm_yunxi']); + + expect(keepKokoroVoicesInOneLanguage( + ['af_sarah', 'bf_emma'], + 'bf_emma', + )).toEqual(['af_sarah', 'bf_emma']); + }); + test('normalizes valid EPUB metadata language tags and rejects invalid metadata', () => { expect(normalizeOptionalLanguageTag(' ja-jp ')).toBe('ja-JP'); expect(normalizeOptionalLanguageTag('fr, en')).toBe('fr'); @@ -53,9 +71,7 @@ describe('multilingual language utilities', () => { model: 'kokoro', voice: 'ff_siwis+jf_alpha', documentLanguage: 'fr', - })).toEqual([ - 'Selected Kokoro voices use multiple languages (French, Japanese).', - ]); + })).toEqual([]); expect(getTtsLanguageCompatibilityWarnings({ model: 'kokoro', From 8071ae1a61ca3ec4919a202f9d11a5daa21c5f66 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 6 Jun 2026 11:05:36 -0600 Subject: [PATCH 07/12] refactor(ui): centralize standard select controls --- src/components/AudiobookExportModal.tsx | 64 +++----- src/components/SettingsModal.tsx | 144 +++++------------- src/components/admin/AdminFeaturesPanel.tsx | 55 ++----- src/components/admin/AdminProvidersPanel.tsx | 127 +++++---------- src/components/documents/DocumentSettings.tsx | 32 ++-- src/components/ui/select.tsx | 107 +++++++++---- .../shared-select-consumers.vitest.spec.ts | 31 ++++ 7 files changed, 235 insertions(+), 325 deletions(-) create mode 100644 tests/unit/shared-select-consumers.vitest.spec.ts diff --git a/src/components/AudiobookExportModal.tsx b/src/components/AudiobookExportModal.tsx index b88efda..254ac1f 100644 --- a/src/components/AudiobookExportModal.tsx +++ b/src/components/AudiobookExportModal.tsx @@ -1,11 +1,11 @@ 'use client'; import { Fragment, useState, useRef, useCallback, useEffect, useMemo } from 'react'; -import { Transition, Listbox, Menu, MenuButton } from '@headlessui/react'; +import { Transition, Menu, MenuButton } from '@headlessui/react'; import { useTimeEstimation } from '@/hooks/useTimeEstimation'; import { ProgressPopup } from '@/components/ProgressPopup'; import { ProgressCard } from '@/components/ProgressCard'; -import { DownloadIcon, CheckCircleIcon, XCircleIcon, ClockIcon, ChevronUpDownIcon, RefreshIcon, DotsVerticalIcon } from '@/components/icons/Icons'; +import { DownloadIcon, CheckCircleIcon, XCircleIcon, ClockIcon, RefreshIcon, DotsVerticalIcon } from '@/components/icons/Icons'; import { ConfirmDialog } from '@/components/ConfirmDialog'; import { useConfig } from '@/contexts/ConfigContext'; import { useTTS } from '@/contexts/TTSContext'; @@ -14,7 +14,7 @@ import { ReaderSidebarShell } from '@/components/reader/ReaderSidebarShell'; import { resolveTtsProviderModelPolicy } from '@/lib/shared/tts-provider-policy'; import { getTtsLanguageCompatibilityWarnings, resolveTtsLanguage } from '@/lib/shared/language'; import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts'; -import { Button, Card, IconButton, MenuActionItem, MenuItemsSurface, RangeInput, SharedListboxButton, SharedListboxOption, SharedListboxOptions } from '@/components/ui'; +import { Button, Card, IconButton, MenuActionItem, MenuItemsSurface, RangeInput, Select } from '@/components/ui'; import { getAudiobookStatus, deleteAudiobookChapter, @@ -565,51 +565,25 @@ export function AudiobookExportModal({
{chapters.length === 0 ? ( - setFormat(newFormat)} + options={['m4b', 'mp3'] as const} disabled={chapters.length > 0 || settingsLocked} - > -
- - {format.toUpperCase()} - - - - - - - - {({ selected }) => ( - - M4B - - )} - - - {({ selected }) => ( - - MP3 - - )} - - - -
- + renderValue={(option) => ( + {option.toUpperCase()} + )} + renderOption={(option, { selected }) => ( + + {option.toUpperCase()} + + )} + buttonClassName="bg-surface" + chevronClassName="h-4 w-4 text-soft" + optionInset="none" + optionItemClassName="py-2" + showCheckmark={false} + /> ) : (
{format.toUpperCase()}
)} diff --git a/src/components/SettingsModal.tsx b/src/components/SettingsModal.tsx index a5ad457..c15bec2 100644 --- a/src/components/SettingsModal.tsx +++ b/src/components/SettingsModal.tsx @@ -1,14 +1,10 @@ 'use client'; import { Fragment, useState, useEffect, useCallback, useMemo } from 'react'; -import { - Transition, - Listbox, -} from '@headlessui/react'; import Link from 'next/link'; import { useTheme } from '@/contexts/ThemeContext'; import { useConfig } from '@/contexts/ConfigContext'; -import { ChevronUpDownIcon, CheckIcon, SettingsIcon, KeyIcon, PaletteIcon, DocumentIcon, UserIcon, DownloadIcon, ChevronRightIcon } from '@/components/icons/Icons'; +import { CheckIcon, SettingsIcon, KeyIcon, PaletteIcon, DocumentIcon, UserIcon, DownloadIcon, ChevronRightIcon } from '@/components/icons/Icons'; import { useDocuments } from '@/contexts/DocumentContext'; import { ConfirmDialog } from '@/components/ConfirmDialog'; import { ProgressPopup } from '@/components/ProgressPopup'; @@ -52,10 +48,8 @@ import { Input, ModalFrame, ModalTitle, + Select, inputClass, - SharedListboxButton, - SharedListboxOption, - SharedListboxOptions, } from '@/components/ui'; import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; @@ -654,8 +648,16 @@ export function SettingsModal({ User API keys are restricted and no shared provider is configured. Ask an admin to add one.

) : ( - provider.id} + renderValue={(provider) => provider.name} + renderOption={(provider, { selected }) => ( + + {provider.name} + + )} onChange={(provider) => { const defaults = resolveProviderDefaults({ providerRef: provider.id, @@ -675,46 +677,7 @@ export function SettingsModal({ } setCustomModelInput(''); }} - > - - - {selectedProviderOption?.name || 'Select Provider'} - - - - - - - - {ttsProviders.map((provider) => ( - - {({ selected }) => ( - <> - - {provider.name} - - {selected && ( - - - - )} - - )} - - ))} - - - + /> )}
{restrictUserApiKeys && ( @@ -768,8 +731,30 @@ export function SettingsModal({

)}
- m.id === selectedModelId) || ttsModels[0]} + {renderSource('defaultTtsProvider')}
{providerOptions.length > 0 ? ( - - - {selectedProviderOption?.name ?? 'Select provider'} - - + model.id} + renderValue={(model) => model.name} + renderOption={(model, { selected }) => ( + + {model.name} + {model.id.includes(':') ? ( + + {model.id.slice(model.id.indexOf(':'))} + + ) : null} + + )} + placeholder="Select model" + chevronClassName="h-4 w-4 text-muted" + onChange={(model) => { + if (model.id === 'custom') { const nextModel = customModelInput.trim(); setForm({ ...form, @@ -476,54 +463,12 @@ export function AdminProvidersPanel() { } setForm({ ...form, - defaultModel: modelId, - defaultInstructions: modelSupportsInstructions(modelId) ? form.defaultInstructions : '', + defaultModel: model.id, + defaultInstructions: modelSupportsInstructions(model.id) ? form.defaultInstructions : '', }); setCustomModelInput(''); }} - > - - - {selectedModelDefinition?.name ?? 'Select model'} - - - - - - - - {modelDefinitions.map((model) => ( - - {({ selected }) => ( - <> - - {model.name} - {model.id.includes(':') && ( - - {model.id.slice(model.id.indexOf(':'))} - - )} - - {selected && ( - - - - )} - - )} - - ))} - - - + /> {supportsCustomModel && selectedModelId === 'custom' && ( option.value === language) + ?? DOCUMENT_LANGUAGE_OPTIONS[0]; const selectedView = viewTypeTextMapping.find(v => v.id === viewType) || viewTypeTextMapping[0]; const isPdfMode = !epub && !html && !!pdf; const [localPreloadDepth, setLocalPreloadDepth] = useState(segmentPreloadDepthPages); @@ -183,22 +193,16 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html, language, dete subtitle="Controls sentence splitting and synchronized word alignment." variant="flat" > -