From 696d3e488f60c1f48b0673f2a7670806806e359f Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 6 Jun 2026 09:29:39 -0600 Subject: [PATCH] feat(tts): detect EPUB language and warn on voice mismatch --- src/app/(app)/epub/[id]/page.tsx | 6 +- src/app/(app)/epub/[id]/useEpubDocument.ts | 22 ++++- src/components/AudiobookExportModal.tsx | 12 ++- src/components/documents/DocumentSettings.tsx | 22 ++++- src/components/player/VoicesControl.tsx | 34 ++++++-- src/components/reader/SegmentsSidebar.tsx | 5 +- src/lib/shared/language.ts | 82 +++++++++++++++++-- tests/unit/language.vitest.spec.ts | 48 +++++++++++ 8 files changed, 212 insertions(+), 19 deletions(-) diff --git a/src/app/(app)/epub/[id]/page.tsx b/src/app/(app)/epub/[id]/page.tsx index 3cc6ebd..38472f8 100644 --- a/src/app/(app)/epub/[id]/page.tsx +++ b/src/app/(app)/epub/[id]/page.tsx @@ -37,6 +37,7 @@ export default function EPUBPage() { createFullAudioBook: createEPUBAudioBook, regenerateChapter: regenerateEPUBChapter, bookRef, + metadataLanguage, } = epubState; const { stop, setDocumentLanguage } = useTTS(); const { language, updateLanguage } = useDocumentLanguage(routeDocumentId); @@ -109,8 +110,8 @@ export default function EPUBPage() { useUnmountCleanupRef(clearCurrDoc); useEffect(() => { - setDocumentLanguage(language); - }, [language, setDocumentLanguage]); + setDocumentLanguage(language === 'auto' ? metadataLanguage ?? 'auto' : language); + }, [language, metadataLanguage, setDocumentLanguage]); // Compute available height = viewport - (header height + tts bar height) useEffect(() => { @@ -256,6 +257,7 @@ export default function EPUBPage() { isOpen={activeSidebar === 'settings'} setIsOpen={(isOpen) => setActiveSidebar((prev) => isOpen ? 'settings' : (prev === 'settings' ? null : prev))} language={language} + detectedLanguage={metadataLanguage} onLanguageChange={(nextLanguage) => { void updateLanguage(nextLanguage); }} diff --git a/src/app/(app)/epub/[id]/useEpubDocument.ts b/src/app/(app)/epub/[id]/useEpubDocument.ts index 21e1f0a..14c38fa 100644 --- a/src/app/(app)/epub/[id]/useEpubDocument.ts +++ b/src/app/(app)/epub/[id]/useEpubDocument.ts @@ -44,6 +44,7 @@ import type { import type { AudiobookGenerationSettings, TTSSegmentLocator } from '@/types/client'; import { isStableEpubLocator } from '@/types/client'; import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; +import { normalizeOptionalLanguageTag } from '@/lib/shared/language'; export interface EpubDocumentState { currDocData: ArrayBuffer | undefined; @@ -51,6 +52,7 @@ export interface EpubDocumentState { currDocPages: number | undefined; currDocPage: number | string; currDocText: string | undefined; + metadataLanguage: string | null; isPlaybackReady: boolean; setCurrentDocument: (id: string) => Promise; clearCurrDoc: () => void; @@ -116,6 +118,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { const [currDocData, setCurrDocData] = useState(); const [currDocName, setCurrDocName] = useState(); const [currDocText, setCurrDocText] = useState(); + const [metadataLanguage, setMetadataLanguage] = useState(null); const [isPlaybackReady, setIsPlaybackReady] = useState(false); // Mirror state into a ref so resolveEpubLocator (registered once with // TTSContext via a stable callback) can always read the latest page text @@ -172,6 +175,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { setCurrDocData(undefined); setCurrDocName(undefined); setCurrDocText(undefined); + setMetadataLanguage(null); setIsPlaybackReady(false); setCurrDocPages(undefined); isEPUBSetOnce.current = false; @@ -193,6 +197,9 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { const setCurrentDocument = useCallback(async (id: string): Promise => { try { setIsPlaybackReady(false); + setMetadataLanguage(null); + bookRef.current = null; + renditionRef.current = undefined; const meta = await getDocumentMetadata(id); if (!meta) { clearCurrDoc(); @@ -380,8 +387,19 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { }); const setRendition = useCallback((rendition: Rendition) => { - bookRef.current = rendition.book; + const book = rendition.book; + bookRef.current = book; renditionRef.current = rendition; + void book.loaded.metadata + .then((metadata) => { + if (bookRef.current !== book) return; + setMetadataLanguage(normalizeOptionalLanguageTag(metadata.language)); + }) + .catch((error) => { + if (bookRef.current !== book) return; + setMetadataLanguage(null); + console.warn('Failed to read EPUB language metadata:', error); + }); }, []); const handleLocationChanged = useEPUBLocationController({ @@ -406,6 +424,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { currDocPages, currDocPage, currDocText, + metadataLanguage, isPlaybackReady, clearCurrDoc, extractPageText, @@ -432,6 +451,7 @@ export function useEpubDocument(documentId?: string): EpubDocumentState { currDocPages, currDocPage, currDocText, + metadataLanguage, isPlaybackReady, clearCurrDoc, extractPageText, diff --git a/src/components/AudiobookExportModal.tsx b/src/components/AudiobookExportModal.tsx index 9595c46..b88efda 100644 --- a/src/components/AudiobookExportModal.tsx +++ b/src/components/AudiobookExportModal.tsx @@ -12,7 +12,7 @@ import { useTTS } from '@/contexts/TTSContext'; import { VoicesControlBase } from '@/components/player/VoicesControlBase'; import { ReaderSidebarShell } from '@/components/reader/ReaderSidebarShell'; import { resolveTtsProviderModelPolicy } from '@/lib/shared/tts-provider-policy'; -import { resolveTtsLanguage } from '@/lib/shared/language'; +import { getTtsLanguageCompatibilityWarnings, resolveTtsLanguage } from '@/lib/shared/language'; import type { TTSAudiobookChapter, TTSAudiobookFormat } from '@/types/tts'; import { Button, Card, IconButton, MenuActionItem, MenuItemsSurface, RangeInput, SharedListboxButton, SharedListboxOption, SharedListboxOptions } from '@/components/ui'; import { @@ -132,6 +132,11 @@ export function AudiobookExportModal({ }), }; }, [savedSettings, audiobookVoice, configVoice, availableVoices, providerRef, providerType, ttsModel, ttsInstructions, effectiveNativeSpeed, postSpeed, format, providerModelPolicy.supportsInstructions, documentLanguage]); + const languageWarnings = useMemo(() => getTtsLanguageCompatibilityWarnings({ + model: effectiveSettings?.ttsModel, + voice: effectiveSettings?.voice, + documentLanguage: effectiveSettings?.language, + }), [effectiveSettings]); const fetchExistingChapters = useCallback(async (soft: boolean = false) => { if (soft) { @@ -610,6 +615,11 @@ export function AudiobookExportModal({ )} + {languageWarnings.map((warning) => ( +

+ {warning} +

+ ))} {/* Speed controls */} diff --git a/src/components/documents/DocumentSettings.tsx b/src/components/documents/DocumentSettings.tsx index 0e1b0ca..7d4e6f9 100644 --- a/src/components/documents/DocumentSettings.tsx +++ b/src/components/documents/DocumentSettings.tsx @@ -2,6 +2,7 @@ import { useState, useEffect } from 'react'; import { useConfig, ViewType } from '@/contexts/ConfigContext'; +import { useTTS } from '@/contexts/TTSContext'; import { ReaderSidebarShell } from '@/components/reader/ReaderSidebarShell'; import { SEGMENT_PRELOAD_DEPTH_MIN, @@ -19,6 +20,7 @@ import { IconButton, RangeInput, Section, ToggleRow, CheckItem, SegmentedControl import { RefreshIcon, SparkleIcon } from '@/components/icons/Icons'; import type { ParsedPdfBlockKind, PdfParseStatus } from '@/types/parsed-pdf'; import { isForceReparseDisabled } from '@/lib/client/pdf/force-reparse'; +import { getLanguageDisplayName, getTtsLanguageCompatibilityWarnings } from '@/lib/shared/language'; const PDF_SKIP_KIND_OPTIONS: Array<{ kind: ParsedPdfBlockKind; label: string }> = [ { kind: 'header', label: 'Header' }, @@ -106,12 +108,13 @@ function RangeSetting({ ); } -export function DocumentSettings({ isOpen, setIsOpen, epub, html, language, onLanguageChange, pdf }: { +export function DocumentSettings({ isOpen, setIsOpen, epub, html, language, detectedLanguage, onLanguageChange, pdf }: { isOpen: boolean, setIsOpen: (isOpen: boolean) => void, epub?: boolean, html?: boolean, language?: string, + detectedLanguage?: string | null, onLanguageChange?: (language: string) => void, pdf?: { parseStatus: PdfParseStatus | null; @@ -137,7 +140,14 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html, language, onLa epubWordHighlightEnabled, htmlHighlightEnabled, htmlWordHighlightEnabled, + ttsModel, } = useConfig(); + const { voice, resolvedLanguage } = useTTS(); + const languageWarnings = getTtsLanguageCompatibilityWarnings({ + model: ttsModel, + voice, + documentLanguage: resolvedLanguage, + }); const selectedView = viewTypeTextMapping.find(v => v.id === viewType) || viewTypeTextMapping[0]; const isPdfMode = !epub && !html && !!pdf; const [localPreloadDepth, setLocalPreloadDepth] = useState(segmentPreloadDepthPages); @@ -189,6 +199,16 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html, language, onLa ))} + {language === 'auto' && detectedLanguage ? ( +

+ Detected from document metadata: {getLanguageDisplayName(detectedLanguage)} +

+ ) : null} + {languageWarnings.map((warning) => ( +

+ {warning} +

+ ))} ) : null} {isPdfMode && pdf && ( diff --git a/src/components/player/VoicesControl.tsx b/src/components/player/VoicesControl.tsx index 2e67b86..b2b8661 100644 --- a/src/components/player/VoicesControl.tsx +++ b/src/components/player/VoicesControl.tsx @@ -4,22 +4,40 @@ import { useConfig } from '@/contexts/ConfigContext'; import { useTTS } from '@/contexts/TTSContext'; import { useCallback } from 'react'; import { VoicesControlBase } from '@/components/player/VoicesControlBase'; +import { InfoIcon } from '@/components/icons/Icons'; +import { getTtsLanguageCompatibilityWarnings } from '@/lib/shared/language'; export const VoicesControl = ({ availableVoices, setVoiceAndRestart }: { availableVoices: string[]; setVoiceAndRestart: (voice: string) => void; }) => { const { ttsModel, providerType } = useConfig(); - const { voice } = useTTS(); + const { voice, resolvedLanguage } = useTTS(); const onChangeVoice = useCallback((nextVoice: string) => setVoiceAndRestart(nextVoice), [setVoiceAndRestart]); + const languageWarnings = getTtsLanguageCompatibilityWarnings({ + model: ttsModel, + voice, + documentLanguage: resolvedLanguage, + }); return ( - +
+ + {languageWarnings.length > 0 ? ( + + + + ) : null} +
); } diff --git a/src/components/reader/SegmentsSidebar.tsx b/src/components/reader/SegmentsSidebar.tsx index d9ff378..34b6f3b 100644 --- a/src/components/reader/SegmentsSidebar.tsx +++ b/src/components/reader/SegmentsSidebar.tsx @@ -94,6 +94,7 @@ function settingsAreEqual(a: TTSSegmentSettings | null, b: TTSSegmentSettings | && a.voice === b.voice && Number(a.nativeSpeed) === Number(b.nativeSpeed) && (a.ttsInstructions || '') === (b.ttsInstructions || '') + && (a.language || 'en') === (b.language || 'en') ); } @@ -186,6 +187,7 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId, epubBookRef }: playFromSegment, activeReaderType, clearSegmentCaches, + resolvedLanguage, } = useTTS(); const { providerRef, @@ -348,7 +350,8 @@ export function SegmentsSidebar({ isOpen, setIsOpen, documentId, epubBookRef }: voice, nativeSpeed: Number.isFinite(Number(voiceSpeed)) ? Number(voiceSpeed) : 1, ttsInstructions: ttsInstructions || '', - }), [providerRef, providerType, ttsModel, voice, voiceSpeed, ttsInstructions]); + language: resolvedLanguage, + }), [providerRef, providerType, ttsModel, voice, voiceSpeed, ttsInstructions, resolvedLanguage]); const handleClearCache = useCallback(async () => { if (!documentId || isClearingSegments) return; diff --git a/src/lib/shared/language.ts b/src/lib/shared/language.ts index b2e3437..9432235 100644 --- a/src/lib/shared/language.ts +++ b/src/lib/shared/language.ts @@ -1,3 +1,5 @@ +import { isKokoroModel } from '@/lib/shared/kokoro'; + export const DEFAULT_TTS_LANGUAGE = 'en'; export interface TextToken { @@ -26,6 +28,17 @@ const KOKORO_LANGUAGE_BY_PREFIX: Readonly> = { zm: 'zh-CN', }; +export const KOKORO_SUPPORTED_LANGUAGES = [ + 'en', + 'es', + 'fr', + 'hi', + 'it', + 'ja', + 'pt-BR', + 'zh-CN', +] as const; + export function normalizeLanguageTag( language: string | null | undefined, fallback = DEFAULT_TTS_LANGUAGE, @@ -40,6 +53,17 @@ export function normalizeLanguageTag( } } +export function normalizeOptionalLanguageTag(language: unknown): string | null { + if (typeof language !== 'string') return null; + const candidate = language.trim().split(/[,\s]+/u)[0]; + if (!candidate) return null; + try { + return Intl.getCanonicalLocales(candidate)[0] ?? null; + } catch { + return null; + } +} + export function toBaseLanguageCode(language: string | null | undefined): string { const normalized = normalizeLanguageTag(language); try { @@ -50,17 +74,20 @@ export function toBaseLanguageCode(language: string | null | undefined): string } export function inferKokoroLanguageFromVoice(voice: string | null | undefined): string | null { - if (!voice?.trim()) return null; + const languages = new Set(getKokoroVoiceLanguages(voice)); - const languages = new Set( + return languages.size === 1 ? [...languages][0] : null; +} + +export function getKokoroVoiceLanguages(voice: string | null | undefined): string[] { + if (!voice?.trim()) return []; + return Array.from(new Set( voice .split('+') .map((part) => part.trim().replace(/\([^)]*\)/g, '')) .map((name) => KOKORO_LANGUAGE_BY_PREFIX[name.slice(0, 2).toLowerCase()]) .filter((language): language is string => Boolean(language)), - ); - - return languages.size === 1 ? [...languages][0] : null; + )); } export function resolveTtsLanguage(input: { @@ -75,6 +102,51 @@ export function resolveTtsLanguage(input: { return inferKokoroLanguageFromVoice(input.voice) ?? DEFAULT_TTS_LANGUAGE; } +export function getLanguageDisplayName(language: string): string { + try { + return new Intl.DisplayNames(['en'], { type: 'language' }).of(language) || language; + } catch { + return language; + } +} + +export function getTtsLanguageCompatibilityWarnings(input: { + model?: string | null; + voice?: string | null; + documentLanguage?: string | null; +}): string[] { + if (!isKokoroModel(input.model || '')) return []; + + const documentLanguage = normalizeLanguageTag(input.documentLanguage); + const documentBaseLanguage = toBaseLanguageCode(documentLanguage); + const supportedBaseLanguages = new Set(KOKORO_SUPPORTED_LANGUAGES.map((language) => toBaseLanguageCode(language))); + const voiceLanguages = getKokoroVoiceLanguages(input.voice); + const voiceBaseLanguages = new Set(voiceLanguages.map((language) => toBaseLanguageCode(language))); + const warnings: string[] = []; + + if (!supportedBaseLanguages.has(documentBaseLanguage)) { + warnings.push( + `Kokoro's built-in voice catalog does not include ${getLanguageDisplayName(documentLanguage)}.`, + ); + } + + if (voiceBaseLanguages.size > 1) { + warnings.push( + `Selected Kokoro voices use multiple languages (${voiceLanguages.map(getLanguageDisplayName).join(', ')}).`, + ); + return warnings; + } + + const voiceLanguage = voiceLanguages[0]; + if (voiceLanguage && toBaseLanguageCode(voiceLanguage) !== documentBaseLanguage) { + warnings.push( + `Selected Kokoro voice is ${getLanguageDisplayName(voiceLanguage)}, but the document is ${getLanguageDisplayName(documentLanguage)}.`, + ); + } + + return warnings; +} + export function normalizeUnicodeToken(text: string): string { return text .normalize('NFKC') diff --git a/tests/unit/language.vitest.spec.ts b/tests/unit/language.vitest.spec.ts index b224a2f..2842d80 100644 --- a/tests/unit/language.vitest.spec.ts +++ b/tests/unit/language.vitest.spec.ts @@ -3,7 +3,9 @@ import { readFileSync } from 'fs'; import { resolve } from 'path'; import { + getTtsLanguageCompatibilityWarnings, inferKokoroLanguageFromVoice, + normalizeOptionalLanguageTag, normalizeUnicodeToken, resolveTtsLanguage, segmentSentences, @@ -25,6 +27,52 @@ describe('multilingual language utilities', () => { expect(inferKokoroLanguageFromVoice('ff_siwis+jf_alpha')).toBeNull(); }); + test('normalizes valid EPUB metadata language tags and rejects invalid metadata', () => { + expect(normalizeOptionalLanguageTag(' ja-jp ')).toBe('ja-JP'); + expect(normalizeOptionalLanguageTag('fr, en')).toBe('fr'); + expect(normalizeOptionalLanguageTag('not_a_language')).toBeNull(); + expect(normalizeOptionalLanguageTag(undefined)).toBeNull(); + }); + + test('warns only for provable Kokoro language compatibility problems', () => { + expect(getTtsLanguageCompatibilityWarnings({ + model: 'kokoro', + voice: 'jf_alpha', + documentLanguage: 'ja', + })).toEqual([]); + + expect(getTtsLanguageCompatibilityWarnings({ + model: 'kokoro', + voice: 'af_sarah', + documentLanguage: 'ja', + })).toEqual([ + 'Selected Kokoro voice is American English, but the document is Japanese.', + ]); + + expect(getTtsLanguageCompatibilityWarnings({ + model: 'kokoro', + voice: 'ff_siwis+jf_alpha', + documentLanguage: 'fr', + })).toEqual([ + 'Selected Kokoro voices use multiple languages (French, Japanese).', + ]); + + expect(getTtsLanguageCompatibilityWarnings({ + model: 'kokoro', + voice: 'af_sarah', + documentLanguage: 'ar', + })).toEqual([ + "Kokoro's built-in voice catalog does not include Arabic.", + 'Selected Kokoro voice is American English, but the document is Arabic.', + ]); + + expect(getTtsLanguageCompatibilityWarnings({ + model: 'custom-unknown-model', + voice: 'af_sarah', + documentLanguage: 'ja', + })).toEqual([]); + }); + test('prefers an explicit language and resolves regional tags for Whisper', () => { expect(resolveTtsLanguage({ configuredLanguage: 'pt-BR', voice: 'jf_alpha' })).toBe('pt-BR'); expect(resolveTtsLanguage({ configuredLanguage: 'auto', voice: 'jf_alpha' })).toBe('ja');