Merge pull request #114 from richardr1126/fix/text-cleanup

fix(tts): strip bullet/decorative glyphs and centralize audio-text cleaning

compute/core/src/whisper/alignment-map.ts

@@ -1,15 +1,18 @@
 import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts';
 
-// Keep byte-for-byte in lock-step with `preprocessSentenceForAudio` in
+// Worker-side mirror of the app's canonical audio-text cleaning rules in
-// `src/lib/shared/nlp.ts` (and the position-preserving copy in
+// `src/lib/shared/audio-text.ts`. This is a separate build target and cannot
-// `src/lib/client/highlight-char-map.ts`). The word `charStart`/`charEnd`
+// import from `@/lib`, so the rules are duplicated here on purpose. The word
-// offsets this module emits are consumed against text normalized by those
+// `charStart`/`charEnd` offsets this module emits are consumed against text
-// client functions, so any divergence shifts viewer highlights off-word.
+// normalized by that shared module, so any divergence shifts viewer highlights
+// off-word — keep this byte-for-byte in sync with `audio-text.ts`.
+const STRIPPED_GLYPHS = /[*•◦‣⁃∙▪▫■□●○◆◇★☆▶▸►▹➤➢❖]/g;
+
 function preprocessSentenceForAudio(text: string): string {
   return text
     .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -')
     .replace(/([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu, '$1$2')
-    .replace(/\*/g, '')
+    .replace(STRIPPED_GLYPHS, '')
     .replace(/\s+/g, ' ')
     .trim();
 }

src/lib/client/highlight-char-map.ts

@@ -3,27 +3,33 @@
  *
  * TTS word offsets (`charStart`/`charEnd`) are computed against the canonical
  * "audio" form of a sentence — URLs rewritten, line-break hyphenation joined,
- * `*` stripped, whitespace collapsed. To map those offsets back onto the
+ * decorative glyphs stripped, whitespace collapsed. To map those offsets back
- * rendered DOM we have to normalize the DOM text the SAME way while remembering
+ * onto the rendered DOM we have to normalize the DOM text the SAME way while
- * which DOM position each surviving character came from.
+ * remembering which DOM position each surviving character came from.
+ *
+ * The cleaning rules (patterns + glyph set) are imported from
+ * `@/lib/shared/audio-text` so this position-preserving variant and the plain
+ * string `preprocessSentenceForAudio` can never drift apart. This module only
+ * owns the position-tracking mechanics.
  *
  * This module operates on an opaque position type so both the EPUB renderer
  * (position = `{ node, offset }` in an iframe) and the HTML/TXT renderer
  * (position = a `Text` node + offset in the main document) share one identical
- * normalization. Keep the transforms here in lock-step with
+ * normalization.
- * `preprocessSentenceForAudio` in `src/lib/shared/nlp.ts` and the copy in
- * `compute/core/src/whisper/alignment-map.ts`.
  */
 
+import {
+  HYPHENATION_PATTERN,
+  URL_PATTERN,
+  isStrippedGlyph,
+  linkReplacement,
+} from '@/lib/shared/audio-text';
+
 export interface MappedChar<TPos> {
   char: string;
   pos: TPos;
 }
 
-const URL_PATTERN = /\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi;
-// Unicode-aware to match preprocessSentenceForAudio (nlp.ts / alignment-map.ts).
-const HYPHENATION_PATTERN = /([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu;
-
 const cloneMappedChar = <TPos>(char: string, source: MappedChar<TPos>): MappedChar<TPos> => ({
   char,
   pos: source.pos,
@@ -43,7 +49,7 @@ const replaceMappedUrls = <TPos>(tokens: MappedChar<TPos>[]): MappedChar<TPos>[]
 
     const anchor = tokens[start] ?? tokens[Math.max(0, end - 1)];
     if (anchor) {
-      const replacement = `- (link to ${match[1]}) -`;
+      const replacement = linkReplacement(match[1]);
       // Spread the replacement characters across the original URL span so the
       // mapped positions keep their positional spread instead of collapsing the
       // whole URL onto a single DOM anchor.
@@ -106,7 +112,7 @@ export const normalizeMappedChars = <TPos>(tokens: MappedChar<TPos>[]): MappedCh
   };
 
   for (const token of withoutHyphenation) {
-    if (token.char === '*') continue;
+    if (isStrippedGlyph(token.char)) continue;
     if (/\s/.test(token.char)) {
       pendingWhitespace ??= token;
       continue;

src/lib/shared/audio-text.ts

@@ -0,0 +1,49 @@
+/**
+ * Canonical text-cleaning rules for the "audio" form of a sentence.
+ *
+ * This file is the single source of truth in the app. Two consumers apply these
+ * rules:
+ *  - `preprocessSentenceForAudio` (below) — the plain-string form used wherever
+ *    TTS input text is produced.
+ *  - `normalizeMappedChars` in `src/lib/client/highlight-char-map.ts` — a
+ *    position-preserving variant that reuses the same patterns/glyph set so
+ *    viewer highlight offsets line up with the audio text.
+ *
+ * The compute/worker package keeps its own mirror in
+ * `compute/core/src/whisper/alignment-map.ts` (a separate build target that
+ * cannot import from `@/lib`). Keep that copy in lock-step with these rules.
+ */
+
+/** Matches a bare or explicit URL token (http(s):// or www.). Capture 1 = domain. */
+export const URL_PATTERN = /\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi;
+
+/** Spoken-friendly stand-in for a stripped URL. */
+export const linkReplacement = (domain: string): string => `- (link to ${domain}) -`;
+
+/** Line-break hyphenation: "exam- ple" -> "example". Captures 1 + 2 = joined word. */
+export const HYPHENATION_PATTERN = /([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu;
+
+// Asterisk plus bullet / list-marker / decorative glyphs that carry no spoken
+// value and cause strict TTS engines (e.g. supertonic) to reject the input with
+// "unsupported character(s)". Both flavors below derive from this one list.
+const STRIPPED_GLYPH_CHARS = '*•◦‣⁃∙▪▫■□●○◆◇★☆▶▸►▹➤➢❖';
+const STRIPPED_GLYPH_SET = new Set(STRIPPED_GLYPH_CHARS);
+
+/** Global regex for string `.replace`. */
+export const STRIPPED_GLYPHS = new RegExp(`[${STRIPPED_GLYPH_CHARS}]`, 'g');
+
+/** Stateless single-character membership test for the position-preserving path. */
+export const isStrippedGlyph = (char: string): boolean => STRIPPED_GLYPH_SET.has(char);
+
+/**
+ * Preprocesses text for audio generation by cleaning up various artifacts:
+ * rewrites URLs to a spoken form, joins line-break hyphenation, strips
+ * decorative glyphs, and collapses whitespace.
+ */
+export const preprocessSentenceForAudio = (text: string): string =>
+  text
+    .replace(URL_PATTERN, (_match, domain: string) => linkReplacement(domain))
+    .replace(HYPHENATION_PATTERN, '$1$2')
+    .replace(STRIPPED_GLYPHS, '')
+    .replace(/\s+/g, ' ')
+    .trim();

src/lib/shared/nlp.ts

@@ -6,6 +6,11 @@
  */
 
 import { normalizeLanguageTag, segmentSentences, toBaseLanguageCode } from '@/lib/shared/language';
+import { preprocessSentenceForAudio } from '@/lib/shared/audio-text';
+
+// Re-exported so existing `@/lib/shared/nlp` importers keep working; the rules
+// themselves live in `@/lib/shared/audio-text`.
+export { preprocessSentenceForAudio };
 
 export const MAX_BLOCK_LENGTH = 450;
 const MIN_BLOCK_LENGTH = 50;
@@ -128,27 +133,6 @@ const normalizeSentenceBoundariesForNlp = (text: string): string => {
     .replace(/([\p{Ll}\p{N}][.!?]["”’)\]])(?=\p{Lu})/gu, '$1 ');
 };
 
-/**
- * Preprocesses text for audio generation by cleaning up various text artifacts.
- *
- * Source of truth for the canonical "audio" text form. Keep byte-for-byte in
- * lock-step with the copy in `compute/core/src/whisper/alignment-map.ts` (which
- * computes word char offsets) and the position-preserving variant in
- * `src/lib/client/highlight-char-map.ts` (which maps those offsets to the DOM).
- *
- * @param {string} text - The text to preprocess
- * @returns {string} The cleaned text
- */
-export const preprocessSentenceForAudio = (text: string): string => {
-  return text
-    .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -')
-    .replace(/([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu, '$1$2') // Remove hyphenation
-    // Remove special character *
-    .replace(/\*/g, '')
-    .replace(/\s+/g, ' ')
-    .trim();
-};
-
 /**
  * Splits text into sentences and groups them into blocks suitable for TTS processing
  * 

tests/unit/nlp.vitest.spec.ts

@@ -33,6 +33,20 @@ describe('preprocessSentenceForAudio', () => {
         input: 'This is *bold* text',
         expected: 'This is bold text',
       },
+      {
+        input: '▪ First item\n▪ Second item',
+        expected: 'First item Second item',
+      },
+      {
+        input: '• Bullet ‣ marker ◦ list',
+        expected: 'Bullet marker list',
+      },
+      {
+        // U+00B7 middle dot is a real letter in Catalan ("col·legi") and must
+        // NOT be stripped like decorative bullets, since OpenReader is multilingual.
+        input: 'Col·legi de l·l',
+        expected: 'Col·legi de l·l',
+      },
       {
         input: 'Multiple    spaces',
         expected: 'Multiple spaces',