openreader/tests/unit/language.vitest.spec.ts

import { describe, expect, test } from 'vitest';
import { readFileSync } from 'fs';
import { resolve } from 'path';

import {
  getTtsLanguageCompatibilityWarnings,
  inferKokoroLanguageFromVoice,
  keepKokoroVoicesInOneLanguage,
  normalizeOptionalLanguageTag,
  normalizeUnicodeToken,
  resolveReplicateKokoroLanguageCode,
  resolveTtsLanguage,
  segmentSentences,
  segmentWords,
  toBaseLanguageCode,
} from '../../src/lib/shared/language';

describe('multilingual language utilities', () => {
  test('keeps the multilingual document fixture readable as UTF-8', () => {
    const fixture = readFileSync(resolve(process.cwd(), 'tests/files/multilingual-sample.txt'), 'utf8');
    expect(fixture).toContain('これは二番目の文です。');
    expect(fixture).toContain('这是第二句话。');
    expect(fixture).toContain('هذه هي الجملة الثانية.');
  });

  test('infers Kokoro language from a single-language voice selection', () => {
    expect(inferKokoroLanguageFromVoice('jf_alpha')).toBe('ja');
    expect(inferKokoroLanguageFromVoice('zf_xiaobei(0.5)+zm_yunxi(0.5)')).toBe('zh-CN');
    expect(inferKokoroLanguageFromVoice('ff_siwis+jf_alpha')).toBeNull();
  });

  test('keeps Kokoro multi-voice selections within the newly selected language', () => {
    expect(keepKokoroVoicesInOneLanguage(
      ['af_sarah', 'bf_emma', 'jf_alpha'],
      'jf_alpha',
    )).toEqual(['jf_alpha']);

    expect(keepKokoroVoicesInOneLanguage(
      ['zf_xiaobei', 'zm_yunxi', 'af_sarah'],
      'zm_yunxi',
    )).toEqual(['zf_xiaobei', 'zm_yunxi']);

    expect(keepKokoroVoicesInOneLanguage(
      ['af_sarah', 'bf_emma'],
      'bf_emma',
    )).toEqual(['af_sarah', 'bf_emma']);
  });

  test('normalizes valid EPUB metadata language tags and rejects invalid metadata', () => {
    expect(normalizeOptionalLanguageTag(' ja-jp ')).toBe('ja-JP');
    expect(normalizeOptionalLanguageTag('fr, en')).toBe('fr');
    expect(normalizeOptionalLanguageTag('not_a_language')).toBeNull();
    expect(normalizeOptionalLanguageTag(undefined)).toBeNull();
  });

  test('warns only for provable Kokoro language compatibility problems', () => {
    expect(getTtsLanguageCompatibilityWarnings({
      model: 'kokoro',
      voice: 'jf_alpha',
      documentLanguage: 'ja',
    })).toEqual([]);

    expect(getTtsLanguageCompatibilityWarnings({
      model: 'kokoro',
      voice: 'af_sarah',
      documentLanguage: 'ja',
    })).toEqual([
      'Selected Kokoro voice is American English, but the document is Japanese.',
    ]);

    expect(getTtsLanguageCompatibilityWarnings({
      model: 'kokoro',
      voice: 'ff_siwis+jf_alpha',
      documentLanguage: 'fr',
    })).toEqual([]);

    expect(getTtsLanguageCompatibilityWarnings({
      model: 'kokoro',
      voice: 'af_sarah',
      documentLanguage: 'ar',
    })).toEqual([
      "Kokoro's built-in voice catalog does not include Arabic.",
      'Selected Kokoro voice is American English, but the document is Arabic.',
    ]);

    expect(getTtsLanguageCompatibilityWarnings({
      model: 'custom-unknown-model',
      voice: 'af_sarah',
      documentLanguage: 'ja',
    })).toEqual([]);
  });

  test('prefers an explicit language and resolves regional tags for Whisper', () => {
    expect(resolveTtsLanguage({ configuredLanguage: 'pt-BR', voice: 'jf_alpha' })).toBe('pt-BR');
    expect(resolveTtsLanguage({ configuredLanguage: 'auto', voice: 'jf_alpha' })).toBe('ja');
    expect(toBaseLanguageCode('zh-CN')).toBe('zh');
  });

  test('maps Kokoro voices and normalized language tags to Replicate language codes', () => {
    expect(resolveReplicateKokoroLanguageCode({ language: 'en', voice: 'af_sarah' })).toBe('a');
    expect(resolveReplicateKokoroLanguageCode({ language: 'en', voice: 'bf_emma' })).toBe('b');
    expect(resolveReplicateKokoroLanguageCode({ language: 'ja-JP', voice: 'jf_alpha' })).toBe('j');
    expect(resolveReplicateKokoroLanguageCode({ language: 'zh-TW', voice: 'zf_xiaobei' })).toBe('z');
  });

  test('normalizes Unicode tokens without dropping non-Latin scripts', () => {
    expect(normalizeUnicodeToken('École!')).toBe('école');
    expect(normalizeUnicodeToken('日本語。')).toBe('日本語');
    expect(normalizeUnicodeToken('العربية؟')).toBe('العربية');
    expect(normalizeUnicodeToken('हिन्दी।')).toBe('हिन्दी');
  });

  test('segments CJK sentences and no-space language words', () => {
    expect(segmentSentences('これは最初の文です。これは二番目の文です。', 'ja')).toEqual([
      'これは最初の文です。',
      'これは二番目の文です。',
    ]);

    const chineseWords = segmentWords('这是第一句话。', 'zh').map((token) => token.text);
    expect(chineseWords.length).toBeGreaterThan(1);
    expect(chineseWords.join('')).toBe('这是第一句话');
  });

  test('preserves source offsets for Thai word segmentation', () => {
    const text = 'นี่คือประโยคแรก';
    const words = segmentWords(text, 'th');
    expect(words.length).toBeGreaterThan(1);
    for (const word of words) {
      expect(text.slice(word.start, word.end)).toBe(word.text);
    }
  });
});