diff --git a/docs/v1/ADR-0001-playback-architecture.md b/docs/v1/ADR-0001-playback-architecture.md new file mode 100644 index 0000000..49dea6a --- /dev/null +++ b/docs/v1/ADR-0001-playback-architecture.md @@ -0,0 +1,250 @@ +# ADR-0001 Playback architecture for v1 + +Status: Accepted +Date: 2025-11-10 +Owner: @richardr1126 +Related: +- Plan checklist [docs/v1/todo.md](docs/v1/todo.md) +- Issue triage and mapping [docs/v1/issues-to-components.md](docs/v1/issues-to-components.md) + +## Decision +Adopt a new, single playback engine built around HTMLAudioElement with Media Source Extensions where available, replacing Howler entirely. Introduce a provider-agnostic TTS interface and document adapters that yield stable location tokens and sentence blocks. Replace the existing IndexedDB utility with Dexie.js for persistence and caching. Ship the new engine as a clean cutover without running dual engines. + +## Context +The current 0.x implementation couples TTS, viewers, and playback control in ways that create fragile flows and race conditions. Playback requires multiple edits across contexts to add features and is sensitive to timing between NLP, preloading, and Howler lifecycle. Issues highlight problems with large export downloads, dialog chunking, PDF margin extraction, and new feature support such as voice combination and chapter based exports. + +Guiding constraints from v1 scope: +- Streaming first playback +- Replace Howler +- Dexie.js as client storage layer +- Preserve audiobook m4b and add chapter based MP3 export +- Keep server side document sync +- Browsers: Chrome, Firefox, Edge, Safari 16+ + +## Goals +- Simplify the playback pipeline with a clear state machine and strict cancellation +- Decouple document parsing from playback via adapters +- Standardize provider integration behind a unified TTS interface +- Improve resilience for long running operations and large audio artifacts +- Make preloading, skipping, voice switching predictable and race free +- Persist user state and caches using Dexie repositories + +## Non goals +- Running the legacy engine in parallel with v1 +- Rewriting existing viewers wholesale beyond adapter wiring and highlighting seams +- Guaranteeing true streaming for providers that only return whole file responses + +## Architecture overview + +```mermaid +flowchart TD + Views[PDF viewer EPUB viewer HTML viewer] --> Adapters[Document adapters] + Adapters --> Splitter[Sentence splitter and mapping] + Splitter --> Queue[Sentence queue and preloader] + Queue --> Engine[Playback engine state machine] + Engine --> Media[Media controller HTMLAudioElement MSE] + Media --> Output[Audio output media session background handling] + Engine --> Cache[Audio cache Dexie] + Engine --> TTS[TTS providers OpenAI Deepinfra Custom] + Engine --> Position[Resume position store] +``` + +## Component responsibilities + +- Adapters + - Yield text blocks plus stable locationToken + - Handle next prev navigation semantics per format + - Provide highlight mapping strategies + - Files: + - [src/v1/adapters/DocumentAdapter.ts](src/v1/adapters/DocumentAdapter.ts) + - [src/v1/adapters/PdfAdapter.ts](src/v1/adapters/PdfAdapter.ts) + - [src/v1/adapters/EpubAdapter.ts](src/v1/adapters/EpubAdapter.ts) + - [src/v1/adapters/HtmlAdapter.ts](src/v1/adapters/HtmlAdapter.ts) + +- NLP splitter + - Builds sentence blocks with quote aware grouping + - Exposes mapping to raw sentences for highlighting + - Files: + - [src/v1/nlp/sentences.ts](src/v1/nlp/sentences.ts) + - Uses [src/utils/nlp.ts](src/utils/nlp.ts:1) + +- Playback engine + - Drives state transitions, cancellation, preloading, and error handling + - Integrates with MediaController and TTS providers + - Files: + - [src/v1/playback/state.ts](src/v1/playback/state.ts) + - [src/v1/playback/queue.ts](src/v1/playback/queue.ts) + - [src/v1/playback/engine.ts](src/v1/playback/engine.ts) + - [src/v1/playback/hooks/usePlayback.ts](src/v1/playback/hooks/usePlayback.ts) + +- Media controller + - Owns HTMLAudioElement lifecycle and Media Source Extensions when supported + - Provides blob fallback and gapless segment chaining for Safari 16 plus + - Integrates media session and background visibility behaviors + - Files: + - [src/v1/playback/media/MediaController.ts](src/v1/playback/media/MediaController.ts) + - [src/v1/playback/media/mediaSession.ts](src/v1/playback/media/mediaSession.ts) + - [src/v1/playback/media/background.ts](src/v1/playback/media/background.ts) + +- TTS providers + - Unified interface for synth requests and voice listing + - Pass through custom voice strings including plus syntax when supported + - Files: + - [src/v1/tts/types.ts](src/v1/tts/types.ts) + - [src/v1/tts/Provider.ts](src/v1/tts/Provider.ts) + - [src/v1/tts/providers/OpenAIProvider.ts](src/v1/tts/providers/OpenAIProvider.ts) + - [src/v1/tts/providers/DeepinfraProvider.ts](src/v1/tts/providers/DeepinfraProvider.ts) + - [src/v1/tts/providers/CustomOpenAIProvider.ts](src/v1/tts/providers/CustomOpenAIProvider.ts) + - [src/v1/tts/voices.ts](src/v1/tts/voices.ts) + +- Persistence and caching + - Dexie schema for documents, config, audio cache, positions, voices + - Repositories expose typed APIs and transactions + - Files: + - [src/v1/db/schema.ts](src/v1/db/schema.ts) + - [src/v1/db/client.ts](src/v1/db/client.ts) + - [src/v1/db/repositories/DocumentsRepo.ts](src/v1/db/repositories/DocumentsRepo.ts) + - [src/v1/db/repositories/ConfigRepo.ts](src/v1/db/repositories/ConfigRepo.ts) + - [src/v1/db/repositories/AudioCacheRepo.ts](src/v1/db/repositories/AudioCacheRepo.ts) + - [src/v1/db/repositories/VoicesRepo.ts](src/v1/db/repositories/VoicesRepo.ts) + - [src/v1/playback/positionStore.ts](src/v1/playback/positionStore.ts) + +- API surface + - Streaming route for providers that support chunked responses + - Range enabled audio download for large m4b artifacts + - Files: + - [src/app/api/tts/stream/route.ts](src/app/api/tts/stream/route.ts) + - [src/app/api/tts/route.ts](src/app/api/tts/route.ts:1) + - [src/app/api/tts/voices/route.ts](src/app/api/tts/voices/route.ts:1) + - [src/app/api/audio/convert/route.ts](src/app/api/audio/convert/route.ts:1) + +## Playback state machine + +States +- idle +- preparing +- buffering +- playing +- paused +- stopping +- error + +Transitions +- idle -> preparing on play with valid queue head +- preparing -> buffering after first audio segment request +- buffering -> playing on enough data available +- playing -> buffering when underflow or on skip voice change +- playing -> paused on user pause +- any -> stopping on stop clear queue cancel requests +- any -> error on unrecoverable error with context + +Guards and effects +- All requests carry AbortController scoped to the current token +- Config changes produce a new token and cancel in flight +- Preloading is capped and respects cache budgets + +## Media pipeline + +- Try MSE with a SourceBuffer of audio mpeg or aac when available +- Else use short blob segments and chain playback with minimal gaps +- Apply rate changes via playbackRate for audio player speed separate from voice speed at synth time +- Integrate Media Session actions play pause next previous +- Pause on background visibility and auto resume on foreground if user was playing + +## Text and highlighting + +- Adapters provide raw to processed sentence mapping for highlight +- PDF adapter normalizes x positions to page width and respects left right margins +- EPUB adapter yields location tokens and section navigation +- HTML adapter passes text and uses markdown rendering only for view + +## Dexie schema outline + +Tables and indicative indexes +- documents id type name lastModified size dataRef +- config key value +- audioCache key createdAt expiresAt size bytesRef or chunkRefs +- positions docId locationToken sentenceIndex updatedAt +- voices provider model voices updatedAt + +Exact table definitions will be codified in [src/v1/db/schema.ts](src/v1/db/schema.ts) + +## API notes + +- TTS stream route + - POST returns chunked audio where provider supports it + - Fallback to full array buffer with progressive delivery +- Audio convert route + - Supports mp3 per chapter mode and m4b + - Adds GET download with Accept Ranges for large files + +References: +- Current TTS route [src/app/api/tts/route.ts](src/app/api/tts/route.ts:1) +- Current voices route [src/app/api/tts/voices/route.ts](src/app/api/tts/voices/route.ts:1) +- Current audio convert [src/app/api/audio/convert/route.ts](src/app/api/audio/convert/route.ts:1) + +## Migration plan + +- One time importer reads from legacy store helpers in [src/utils/indexedDB.ts](src/utils/indexedDB.ts:1) and writes to Dexie +- Progress UI and retryable steps +- After cutover remove legacy modules and dependencies including Howler + +## Issue alignment + +- #59 chapter mp3 export via chapterized pipeline and streamed zip +- #48 large m4b download via range enabled download endpoint and persistent temp artifacts +- #47 voice combination via free form voice string pass through on Deepinfra and custom +- #44 dialog chunking via quote aware grouping in splitter +- #40 pdf margins via normalized x width and better width fallback + +See details in [docs/v1/issues-to-components.md](docs/v1/issues-to-components.md) + +## Alternatives considered + +- Keep Howler and harden with retries + - Rejected due to continued complexity and limited streaming control +- Keep raw IndexedDB helper + - Rejected due to ergonomics, schema evolution, and repo patterns desired +- Dual engine migration + - Rejected to avoid complexity and surface area during refactor + +## Risks and mitigations + +- MSE availability and Safari variance + - Provide blob segment fallback and small segment chaining +- Provider streaming differences + - Design stream route with capability detection and fallbacks +- Large artifact memory pressure + - Range enabled downloads and file backed buffers where possible +- Cache growth + - Dexie TTL LRU and size budget enforcement with telemetry + +## Rollout + +- Alpha + - HTML adapter wired end to end with engine and streaming + - Basic Dexie schema and caches +- Beta + - PDF and EPUB adapters with highlighting and resume + - Chapter mp3 export + - Migration UI +- GA + - m4b and sync hardened + - E2E and performance checks + - Legacy removal + +## Acceptance criteria + +- Streaming start to speech under reasonable latency on cached sentences +- Voice change mid playback cancels and resumes with a single buffer rebuild +- 1 to 2 GB m4b export downloads stably in Docker with Range support +- Chapter zip exports are correct and stream without UI stalls +- Dialog is chunked appropriately without regressing non dialog cases +- PDF margins trimming is reliable across test samples + +## Next actions + +- Finalize checklist and sequencing in [docs/v1/todo.md](docs/v1/todo.md) +- Create v1 code skeleton and Dexie schema +- Implement engine state machine and MediaController baseline +- Wire HTML adapter and stream route for first alpha milestone \ No newline at end of file diff --git a/docs/v1/issues-to-components.md b/docs/v1/issues-to-components.md new file mode 100644 index 0000000..ecaec04 --- /dev/null +++ b/docs/v1/issues-to-components.md @@ -0,0 +1,160 @@ +# OpenReader v1 Issue Triage and Mapping + +Repository: https://github.com/richardr1126/OpenReader-WebUI/issues +Reviewed via gh at 2025-11-10. + +Summary of open items included: +- #59 Feature: Chapter-Based MP3 Export +- #48 Bug: Failed to export after complete render with Kokoro +- #47 Feature: Combine voices (Kokoro “plus” syntax) +- #44 Bug: Dialog not chunked together +- #40 Bug: PDF left/right extraction margins not working + +Global 1.0 guardrails +- Streaming-first playback (replace Howler with HTMLAudioElement/MSE) +- Dexie DB replacing vanilla IndexedDB +- Single engine cut-over (no dual engines) +- Keep audiobook (m4b) and server-side sync + +Issue #59 — Chapter-Based MP3 Export +Type: Feature +Hypothesis / intent: +- Users want one-mp3-per-chapter output (besides full-book m4b). +v1 mapping: +- Add chapterized MP3 pipeline that chunks by adapter “chapter” units. +- Provide ZIP export for many MP3 files (streamed). +Proposed modules: +- [src/v1/playback/audiobook.ts](src/v1/playback/audiobook.ts:1) +- [src/v1/adapters/EpubAdapter.ts](src/v1/adapters/EpubAdapter.ts:1) +- [src/app/api/audio/convert/route.ts](src/app/api/audio/convert/route.ts:1) (extend to “mp3-per-chapter” mode) +- [src/v1/db/repositories/DocumentsRepo.ts](src/v1/db/repositories/DocumentsRepo.ts:1) +API design: +- POST /api/audio/convert?mode=chapters&format=mp3 -> returns stream of a ZIP. +Logging to add: +- Chapter boundaries, byte sizes, cumulative progress in [src/app/api/audio/convert/route.ts](src/app/api/audio/convert/route.ts:1). +Acceptance: +- A multi-chapter EPUB produces N mp3 parts named “NN - Chapter Title.mp3”; total duration ~ sum of parts; ZIP streamed without timeouts in Docker. + +Issue #48 — Failed export after complete render with Kokoro +Type: Bug (large-book export) +Observations: +- UI reaches 100% then resets, no download. +- Docker shows “fin … undefined”, likely final transfer problem (not TTS). +Likely root causes: +- Final m4b delivery uses single huge arrayBuffer; browser memory/timeout. +- Missing Content-Disposition/Range; no resumable download. +- Temp-file lifecycle cleanup racing with response. +v1 mitigation: +- Serve final artifact as file on disk with streaming and Range: + - New endpoint: GET /api/audio/convert/download?bookId=… that streams file with Accept-Ranges. + - UI performs streamed download; no arrayBuffer buffering. +- Optionally support S3-compatible offload in future. +Touched modules: +- [src/app/api/audio/convert/route.ts](src/app/api/audio/convert/route.ts:1) +- [src/v1/playback/audiobook.ts](src/v1/playback/audiobook.ts:1) +- [src/v1/components/Progress/MigrationProgress.tsx](src/v1/components/Progress/MigrationProgress.tsx:1) +Instrumentation to add: +- Book ID, file size on disk, stream chunk counts, and client-abort detection. +- Add explicit log/error surfaces at: + - [src/app/api/audio/convert/route.ts](src/app/api/audio/convert/route.ts:1) +Acceptance: +- 1–2 GB m4b exports download in Docker without UI reset; bookId persists until deletion; status 206 Range works. + +Issue #47 — Kokoro combined voices (“+” syntax) +Type: Feature +Intent: +- Support voice strings like “bf_emma+af_heart” when provider is Kokoro/FastAPI or DeepInfra Kokoro. +v1 plan: +- Allow free-form voice string entry and pass-through if not in known set. +- Validate known providers; if “+” present, skip voice validation list. +Components: +- [src/v1/tts/types.ts](src/v1/tts/types.ts:1) +- [src/v1/tts/providers/DeepinfraProvider.ts](src/v1/tts/providers/DeepinfraProvider.ts:1) +- [src/v1/tts/providers/CustomOpenAIProvider.ts](src/v1/tts/providers/CustomOpenAIProvider.ts:1) +- [src/v1/components/player/VoicesControl.tsx](src/v1/components/player/VoicesControl.tsx:1) (v1) +- [src/app/api/tts/route.ts](src/app/api/tts/route.ts:1) ensure it does not coerce voice when provider is deepinfra/custom. +Logging: +- Emit provider, model, raw voice string in request (no PII). +Tests: +- “a+b” voices produce audible combined output; works via DeepInfra pass-through and Kokoro-FastAPI; voice dropdown offers “Custom voice…” input. + +Issue #44 — Dialog not being chunked together +Type: Bug (NLP splitting) +Observations: +- Dialog lines split mid-quote; needs grouping. +Proposed improvements: +- Extend [splitIntoSentences()](src/utils/nlp.ts:34) to apply quote-aware grouping. +- When a sentence begins with an opening quote and the next ends with a closing quote, join them before MAX_BLOCK_LENGTH checks. +v1 mapping: +- Provide composable splitter strategy in [src/v1/nlp/sentences.ts](src/v1/nlp/sentences.ts:1) wrapping existing utility. +- Add “dialog-preserve” flag toggled in settings. +Logging to add: +- Count of quote-joined sentences; per-page example of before/after lengths. +Acceptance: +- Quoted dialog flows as single units unless exceeding MAX_BLOCK_LENGTH by > X%; regression-safe for non-dialog text. + +Issue #40 — PDF left/right extraction margins not working +Type: Bug (PDF extraction) +Observations: +- Current filter uses transform[4]/[5] with width heuristics. Edge cases: missing width, skew, page scale differences. +Likely causes: +- Some pdf.js TextItem miss width; horizontal margins computed but not applied due to scale mismatch. +- Defaults in [src/contexts/ConfigContext.tsx](src/contexts/ConfigContext.tsx:216) set left/right to '0.0' on first run; UX confusion. +Remediation: +- Compute glyph bbox width when width absent using transform matrix. +- Normalize x to [0..1] by dividing by pageWidth; compare to margins reliably. +- Add visual debug overlay (dev mode) to draw margin boxes while extracting. +Code touchpoints: +- [src/utils/pdf.ts](src/utils/pdf.ts:60) extractTextFromPDF(): margin math and width fallback. +- v1 adapter: [src/v1/adapters/PdfAdapter.ts](src/v1/adapters/PdfAdapter.ts:1) +Diagnostics to add: +- Per-page: kept/filtered counts, extremes of x positions, computed margins. +Acceptance: +- Test PDFs show left/right trimming correctly; e2e highlight still robust. + +Cross-cutting improvements from issues +- Abort discipline: Any config change cancels in-flight TTS and preloads with unique request keys. Implemented in [src/v1/playback/engine.ts](src/v1/playback/engine.ts:1). +- Streaming-first: Introduce [src/app/api/tts/stream/route.ts](src/app/api/tts/stream/route.ts:1) for chunked speech where supported; fallback to progressive MP3/AAC. +- Dexie-backed caches: Audio buffers cached with TTL and LRU in [src/v1/db/repositories/AudioCacheRepo.ts](src/v1/db/repositories/AudioCacheRepo.ts:1). +- Resume positions: store stable location tokens per adapter in [src/v1/playback/positionStore.ts](src/v1/playback/positionStore.ts:1). + +Minimal logging plan (to validate assumptions) +- Export path (#48): + - Log: bookId, tmp file sizes, final m4b size; response headers; client disconnects. +- NLP chunking (#44): + - Log: dialog-join counts; sample joined strings (first 80 chars). +- PDF margins (#40): + - Log: normalized x min/max; filtered vs kept counts per page. +- Voice combo (#47): + - Log: provider/model/voice string; server echo ensures pass-through. +- Chapter MP3 export (#59): + - Log: chapter count, per-chapter byte sizes, total ZIP size. + +Acceptance test inventory (added to 1.0) +- Streaming playback start-to-speech under 500ms on cached sentences. +- Voice switch mid-playback produces single cancellation and single rebuffer. +- 1–2 GB m4b export succeeds and downloads via streaming. +- Chapterized MP3 ZIP streams and extracts correctly. +- Dialog detection joins quotes for typical novels. +- PDF margin sliders alter extracted text deterministically. + +Action items added to v1 backlog +- Implement chapterized MP3 export (ZIP) path. +- Add range-enabled download endpoint for m4b artifacts. +- Add custom voice string input and provider pass-through. +- Implement quote-aware sentence grouping in splitter. +- Harden PDF x/width margin filtering and add debug overlay. + +References (current code) +- Export pipeline: [src/app/api/audio/convert/route.ts](src/app/api/audio/convert/route.ts:1) +- TTS routing: [src/app/api/tts/route.ts](src/app/api/tts/route.ts:1) +- PDF extraction: [src/utils/pdf.ts](src/utils/pdf.ts:60) +- NLP: [src/utils/nlp.ts](src/utils/nlp.ts:34) + +References (v1 new modules) +- [src/v1/playback/engine.ts](src/v1/playback/engine.ts:1) +- [src/v1/playback/media/MediaController.ts](src/v1/playback/media/MediaController.ts:1) +- [src/v1/tts/providers/DeepinfraProvider.ts](src/v1/tts/providers/DeepinfraProvider.ts:1) +- [src/v1/adapters/PdfAdapter.ts](src/v1/adapters/PdfAdapter.ts:1) +- [src/v1/nlp/sentences.ts](src/v1/nlp/sentences.ts:1) +- [src/v1/db/repositories/AudioCacheRepo.ts](src/v1/db/repositories/AudioCacheRepo.ts:1) \ No newline at end of file diff --git a/docs/v1/todo.md b/docs/v1/todo.md new file mode 100644 index 0000000..960db24 --- /dev/null +++ b/docs/v1/todo.md @@ -0,0 +1,651 @@ +# OpenReader v1 Todo and Plan + +This document captures the 1.0 rewrite plan, prioritized for a clean engine replacement and Dexie adoption. No dual engines during cutover. + +See triage mapping: [docs/v1/issues-to-components.md](docs/v1/issues-to-components.md) + +## Scope and guardrails +- Streaming first playback with HTMLAudioElement plus MediaSource, blob fallback +- Replace Howler across codebase +- Dexie.js for client storage, with one time migration +- Keep audiobook m4b and add chapter based MP3 export +- Keep server side document sync +- Browsers: Chrome, Firefox, Edge, Safari 16+ + +## Architecture overview + +```mermaid +flowchart TD + Viewers[PDF viewer EPUB viewer HTML viewer] --> Adapters[Document adapters] + Adapters --> Splitter[Sentence splitter and mapping] + Splitter --> Queue[Sentence queue and preloader] + Queue --> Engine[Playback engine state machine] + Engine --> Media[Media controller HTMLAudioElement MSE] + Media --> Output[Audio output media session background handling] + Engine --> Cache[Audio cache Dexie] + Engine --> TTS[TTS providers OpenAI Deepinfra Custom] +``` + +## Phased milestones +- 1.0.0 alpha: skeleton, Dexie, engine minimal wired to HTML viewer, streaming route +- 1.0.0 beta: PDF and EPUB adapters, highlighting, chapter MP3, migration UI +- 1.0.0: m4b stability, server sync, remove legacy, tests and a11y + +## Master checklist + +- [ ] Define 1.0 scope and architecture decision record in [docs/v1/ADR-0001-playback-architecture.md](docs/v1/ADR-0001-playback-architecture.md) +- [ ] Set up v1 codebase skeleton (new side by side paths): [src/v1/](src/v1/) +- [ ] Add Dexie dependency and pin version in [package.json](package.json:1) +- [ ] Implement Dexie schema and client: [src/v1/db/schema.ts](src/v1/db/schema.ts), [src/v1/db/client.ts](src/v1/db/client.ts) +- [ ] Implement repositories over Dexie: [src/v1/db/repositories/DocumentsRepo.ts](src/v1/db/repositories/DocumentsRepo.ts), [src/v1/db/repositories/ConfigRepo.ts](src/v1/db/repositories/ConfigRepo.ts), [src/v1/db/repositories/AudioCacheRepo.ts](src/v1/db/repositories/AudioCacheRepo.ts), [src/v1/db/repositories/VoicesRepo.ts](src/v1/db/repositories/VoicesRepo.ts) +- [ ] Implement 1 time migration from v0 IndexedDB to Dexie: [src/v1/migrations/migrateFromV0.ts](src/v1/migrations/migrateFromV0.ts) +- [ ] Create TTS provider interface and types: [src/v1/tts/types.ts](src/v1/tts/types.ts), [src/v1/tts/Provider.ts](src/v1/tts/Provider.ts) +- [ ] Implement providers: [src/v1/tts/providers/OpenAIProvider.ts](src/v1/tts/providers/OpenAIProvider.ts), [src/v1/tts/providers/DeepinfraProvider.ts](src/v1/tts/providers/DeepinfraProvider.ts), [src/v1/tts/providers/CustomOpenAIProvider.ts](src/v1/tts/providers/CustomOpenAIProvider.ts) +- [ ] Implement voices service unified across providers: [src/v1/tts/voices.ts](src/v1/tts/voices.ts) +- [ ] Design playback state machine no Howler: [src/v1/playback/state.ts](src/v1/playback/state.ts) +- [ ] Implement sentence queue and preloading: [src/v1/playback/queue.ts](src/v1/playback/queue.ts) +- [ ] Implement MediaController with HTMLAudioElement and MSE: [src/v1/playback/media/MediaController.ts](src/v1/playback/media/MediaController.ts) +- [ ] Implement playback engine orchestrator: [src/v1/playback/engine.ts](src/v1/playback/engine.ts) +- [ ] Implement React hook facade: [src/v1/playback/hooks/usePlayback.ts](src/v1/playback/hooks/usePlayback.ts) +- [ ] Implement unified NLP splitting wrapper: [src/v1/nlp/sentences.ts](src/v1/nlp/sentences.ts) +- [ ] Define document adapter interface and types: [src/v1/adapters/DocumentAdapter.ts](src/v1/adapters/DocumentAdapter.ts) +- [ ] Implement PDF adapter: [src/v1/adapters/PdfAdapter.ts](src/v1/adapters/PdfAdapter.ts) +- [ ] Implement EPUB adapter: [src/v1/adapters/EpubAdapter.ts](src/v1/adapters/EpubAdapter.ts) +- [ ] Implement HTML Markdown adapter: [src/v1/adapters/HtmlAdapter.ts](src/v1/adapters/HtmlAdapter.ts) +- [ ] Implement highlight strategies decoupled from viewers: [src/v1/highlight/pdfHighlighter.ts](src/v1/highlight/pdfHighlighter.ts), [src/v1/highlight/epubHighlighter.ts](src/v1/highlight/epubHighlighter.ts) +- [ ] Implement v1 contexts: [src/v1/contexts/PlaybackProvider.tsx](src/v1/contexts/PlaybackProvider.tsx), [src/v1/contexts/ConfigProviderV1.tsx](src/v1/contexts/ConfigProviderV1.tsx), [src/v1/contexts/DocumentProviderV1.tsx](src/v1/contexts/DocumentProviderV1.tsx) +- [ ] Implement v1 player components: [src/v1/components/player/Player.tsx](src/v1/components/player/Player.tsx), [src/v1/components/player/SpeedControl.tsx](src/v1/components/player/SpeedControl.tsx), [src/v1/components/player/VoicesControl.tsx](src/v1/components/player/VoicesControl.tsx), [src/v1/components/player/Navigator.tsx](src/v1/components/player/Navigator.tsx) +- [ ] Implement v1 viewers wired to adapters and playback: [src/v1/components/PDFViewerV1.tsx](src/v1/components/PDFViewerV1.tsx), [src/v1/components/EPUBViewerV1.tsx](src/v1/components/EPUBViewerV1.tsx), [src/v1/components/HTMLViewerV1.tsx](src/v1/components/HTMLViewerV1.tsx) +- [ ] Add streaming TTS API route: [src/app/api/tts/stream/route.ts](src/app/api/tts/stream/route.ts) +- [ ] Update existing TTS endpoints parity: [src/app/api/tts/route.ts](src/app/api/tts/route.ts:1), [src/app/api/tts/voices/route.ts](src/app/api/tts/voices/route.ts:1) +- [ ] Keep audiobook creation m4b: integrate providers with [src/app/api/audio/convert/route.ts](src/app/api/audio/convert/route.ts:1) and add [src/v1/playback/audiobook.ts](src/v1/playback/audiobook.ts) +- [ ] Implement chapter based MP3 export ZIP +- [ ] Implement server side document sync using Dexie repos: [src/v1/db/sync.ts](src/v1/db/sync.ts) +- [ ] Implement resume position with stable location tokens: [src/v1/playback/positionStore.ts](src/v1/playback/positionStore.ts) +- [ ] Implement robust cancellation on config changes in engine +- [ ] Implement skip blank heuristics in engine +- [ ] Implement media session integration: [src/v1/playback/media/mediaSession.ts](src/v1/playback/media/mediaSession.ts) +- [ ] Implement background visibility handling: [src/v1/playback/media/background.ts](src/v1/playback/media/background.ts) +- [ ] Safari 16 plus fallback path in MediaController +- [ ] Integrate v1 providers into app [src/app/providers.tsx](src/app/providers.tsx:1) +- [ ] Swap viewers on routes to v1: [src/app/pdf/[id]/page.tsx](src/app/pdf/[id]/page.tsx:1), [src/app/epub/[id]/page.tsx](src/app/epub/[id]/page.tsx:1), [src/app/html/[id]/page.tsx](src/app/html/[id]/page.tsx:1) +- [ ] Remove Howler usage and dependency across codebase and [package.json](package.json:12) +- [ ] Triaging GitHub issues and map to v1 components milestone 1.0 +- [ ] Unit tests for playback engine transitions: [src/v1/playback/__tests__/engine.spec.ts](src/v1/playback/__tests__/engine.spec.ts) +- [ ] Unit tests for adapters sentence boundaries and tokens: [src/v1/adapters/__tests__/adapters.spec.ts](src/v1/adapters/__tests__/adapters.spec.ts) +- [ ] Integration tests streaming playback and voice switch: [tests/v1/playback.spec.ts](tests/v1/playback.spec.ts) +- [ ] Update Playwright e2e flows to v1: [tests/play.spec.ts](tests/play.spec.ts:1), [tests/upload.spec.ts](tests/upload.spec.ts:1) +- [ ] Data migration first run progress UI: [src/v1/components/Progress/MigrationProgress.tsx](src/v1/components/Progress/MigrationProgress.tsx) +- [ ] Performance checks preloading policy and cache thresholds +- [ ] Accessibility passes for v1 player and viewers +- [ ] Developer docs v1 overview [docs/v1/README.md](docs/v1/README.md) and update [README.md](README.md:10) +- [ ] Bump version to 1.0.0 in [package.json](package.json:3) and [CHANGELOG.md](CHANGELOG.md) +- [ ] Remove deprecated v0 modules after cutover: delete [src/contexts/TTSContext.tsx](src/contexts/TTSContext.tsx:1), [src/utils/indexedDB.ts](src/utils/indexedDB.ts:1), legacy audio hooks +- [ ] Release plan 1.0.0 alpha beta stable with regression gates + +## Dependency ordering +- Dexie schema and repos precede engine cache features +- Engine and MediaController precede viewers +- Adapters precede highlighting and resume position +- Streaming route precedes engine integration tests + +## Issue mapping alignment +- #59 Chapter MP3 export -> audiobook.ts stream ZIP +- #48 Large m4b download -> range enabled download streaming response bookId lifecycle +- #47 Voice combo plus -> allow free form voice strings and pass through for Kokoro +- #44 Dialog chunking -> quote aware grouping in splitter strategy +- #40 PDF margins -> normalize x width and add debug overlay + +## Decision record outline +- Engine is single source of truth for playback +- Media controller abstracts HTMLAudioElement and MSE +- Providers are stateless except for voice lists +- Dexie stores documents configs audio cache positions + +## Checkpoint gates +- Alpha: streaming route returns audio engine plays cached sentences HTML viewer wired +- Beta: EPUB and PDF adapters stable highlighting chapter MP3 migration UI +- GA: m4b and server sync robust e2e tests green legacy removed + +## Open questions +- Do we support true streaming from all providers or progressive only for some +- Do we want optional S3 offload for very large m4b files in Docker + +## Notes +- Use strict abort controller scoping on every request to avoid double requests +- Use unique cache keys text hash plus voice plus speed plus model +- Ensure token budgets by chunk length not chars only for providers with limits + +--- + +## Expanded planning details + +To make the v1 implementation predictable and fast to execute, this section adds concrete specifications (without code) for schemas, contracts, state machine events, milestones, risks, and acceptance criteria. + +### Milestone breakdown and exit criteria + +Alpha (focus: Dexie + Engine + HTML adapter + streaming route) +- Deliverables: + - Dexie schema + repositories + - Playback engine (core states and cancellation) + - MediaController with MSE and blob fallback baseline + - HTML viewer wired to adapters and engine + - TTS streaming route (chunked where possible, progressive fallback) +- Acceptance: + - Start-to-speech under 700ms on cached blocks + - Play/Pause/Skip stable with single in-flight cancellation + - Voice change cancels and resumes with one request wave + - Basic resume position works for HTML + +Beta (focus: PDF/EPUB adapters + highlighting + chapter MP3 + migration UI) +- Deliverables: + - PDF adapter + highlighter; EPUB adapter + highlighter + - Quote-aware sentence grouping toggle in splitter + - Chapter-based MP3 export (ZIP streamed) + - First-run migration UI (legacy to Dexie) +- Acceptance: + - Highlight tracks current sentence reliably in PDF and EPUB + - Chapter ZIP exports on sample EPUBs with correct file naming + - Migration completes with progress and resumability + - PDF margins trimming reproducible across sample PDFs + +GA (focus: m4b stability + server sync + tests + removal of legacy) +- Deliverables: + - Stable m4b creation with range-enabled download endpoint + - Dexie-backed client sync using existing server route + - Full test suite and a11y pass; remove legacy modules and Howler +- Acceptance: + - Large (1–2 GB) m4b downloads complete in Docker with Range support + - Sync round-trips documents reliably (basic CRUD and bulk) + - E2E tests for PDF/EPUB/HTML, streaming, exports all green + - Legacy removed and version bumped to 1.0.0 + +### Dexie schema (draft, no code) + +Tables and primary keys: +- documents + - id (string, pk), type (enum pdf|epub|html), name, size, lastModified, dataRef (blob pointer or chunk refs), createdAt, updatedAt +- config + - key (string, pk), value (string), updatedAt +- audioCache + - key (string, pk: hash of text+voice+speed+model+format), createdAt, expiresAt, size, bytesRef (blob pointer or chunk refs), provider, model, voice, speed +- positions + - id (auto), docId (string, idx), locationToken (string), sentenceIndex (number), updatedAt +- voices + - id (auto), provider (string, idx), model (string, idx), voices (json), updatedAt + +Repository responsibilities: +- DocumentsRepo: CRUD, list, size budgeting, typed loaders for different doc types. +- ConfigRepo: get/set/clear keys, transactional updates for related keys (e.g., provider+model+savedVoices). +- AudioCacheRepo: TTL+LRU enforcement, size budgets (e.g., 100–250MB), purge policies. +- VoicesRepo: cache list per provider-model; freshness TTL. + +Migration notes: +- One-time importer reads via legacy helpers in [src/utils/indexedDB.ts](src/utils/indexedDB.ts:1) and writes to Dexie with conversion of binary arrays to blobs/references. +- Idempotent: re-runs skip if target record exists with same hash. +- Progress UI surfaces counts and bytes processed. + +### Playback engine state machine (events and guards) + +States: +- idle, preparing, buffering, playing, paused, stopping, error + +Key events: +- playRequested (with queue head present) +- pauseRequested +- stopRequested +- skipForwardRequested, skipBackwardRequested +- configChanged (voice/model/speed) -> must produce a new request token and cancel in-flight +- dataAvailable (buffer threshold met) +- underflowDetected (drop to buffering) +- errorOccurred (carry error type; decide recoverable vs terminal) + +Guards: +- When configChanged, any previously created network or decode work is cancelled using an AbortController keyed by the current session token. +- Single active playback pipeline: next preloads are concurrent but bounded (e.g., N=1–2 lookahead). + +Effects: +- Queue manages sentence blocks; emits prefetch intents; respects AudioCacheRepo. +- MediaController mediates HTMLAudioElement load/append and playbackRate changes (decoupled from TTS voice speed). +- PositionStore persists docId+locationToken+sentenceIndex at debounced intervals (e.g., 800–1200ms) to positions table. + +### MediaController behavior (MSE-first, blob fallback) + +Capabilities: +- Prefer MSE with audio/mpeg or audio/aac SourceBuffer where supported (Chrome, Edge, Firefox). +- Safari 16+ fallback path: small blob segment chaining to minimize gaps. +- PlaybackRate applies to the element (audio player speed). Voice speed remains a synth-time parameter. +- Integrate Media Session actions (play, pause, next, previous). +- Visibility changes: pause when hidden (configurable), resume on foreground if engine state is playing. + +Error handling: +- MSE append errors cause a controlled transition to error or fallback to blob-segment mode, depending on browser capability checks. +- Network stalls (no chunks for T ms) trigger buffering state. + +### TTS provider interface expectations (descriptive) + +Responsibilities: +- Accept text block, provider, model, voice (free-form for Kokoro plus syntax), speed, and format (mp3|aac). +- Return either: + - A stream of audio bytes (preferred), or + - A complete audio buffer (fallback) +- Voice listing may return default sets or pass-through if API supports dynamic retrieval. + +Provider notes: +- Deepinfra/Kokoro: permit plus syntax in voice; skip strict validation if “+” in string. +- Custom OpenAI-compatible: try /audio/voices endpoint for list; fallback to default. +- OpenAI: standard tts-1 or gpt-4o-mini-tts with optional instructions (for supported models). + +### Streaming route contract (descriptive) + +Route: POST /api/tts/stream +- Request: json body includes text, provider, model, voice, speed, response_format (mp3|aac). +- Behavior: + - If upstream supports chunked streaming, proxy as chunked response to client. + - Else, progressively stream the full buffer to client without buffering entire file in memory when possible. +- Headers: + - Content-Type audio/mpeg|audio/aac + - Prefer Transfer-Encoding chunked if supported. +- Cancellation: + - Must respect AbortSignal to cancel upstream and downstream. + +### Document adapters and location tokens + +Requirements: +- Adapters yield tuples { textBlock, locationToken, rawMap } where: + - textBlock is the processed snippet the engine will synthesize/play. + - locationToken is a stable opaque token for resume and navigation (for PDFs: page:sentence; for EPUB: cfi-based; for HTML: line-offset or logical chunk index). + - rawMap enables highlight alignment (processed->raw mapping). +- Navigation: + - next/prev logic is adapter-specific (paginate for PDF, spine/section for EPUB, block index for HTML). +- PDF margins: + - Normalize x in [0..1] by dividing by pageWidth and compare to left/right margins. + - Compute fallback width when TextItem.width is missing; exclude header/footer using normalized y. + +### Splitter strategy (dialog-aware grouping) + +Behavior: +- MAX_BLOCK_LENGTH remains bounded (e.g., 300 characters target). +- Quoted dialog grouping heuristic: + - When a sentence begins with an opening quote and the next ends with a closing quote, join unless combined length exceeds threshold; allow small overruns (e.g., +10%) to preserve dialog continuity. +- Toggle in settings to enable/disable dialog grouping; default enabled for EPUB and HTML, opt-in for PDFs. + +### Exports: chapter MP3 and m4b improvements + +Chapter MP3: +- Emitter builds per-chapter tasks using EPUB table of contents mapping; fallback to “Unknown Section - N” when absent. +- ZIP stream response for many files; return progressively to client; ensure Content-Disposition triggers download reliably. + +m4b improvements: +- Post-processing retains bookId and serves artifact via a dedicated GET endpoint that supports Accept-Ranges. +- UI downloads via streamed response, not loading entire buffer in memory. +- Clean-up policy configurable (e.g., keep artifacts for X hours/days). + +### Server-side sync plan (client-driven) + +Client sync service uses repositories to: +- Read all documents and upload in batches to [src/app/api/documents/route.ts](src/app/api/documents/route.ts:1). +- Download and write back to Dexie (convert numeric arrays to ArrayBuffer/Blob). +- Provide progress events to UI with percentage and status text. + +### Testing matrix and acceptance + +Unit (engine and adapters): +- Engine: transitions, cancellation on config change, underflow to buffering, single resume path. +- Adapters: PDF margin trimming, EPUB section mapping, HTML block indexing, dialog grouping on/off. + +Integration: +- Streaming playback path: start-to-speech latency, skip forward/back, voice switch. +- Chapter MP3 export on sample EPUB: counts and filenames match TOC. +- m4b large export streaming with Range download in Docker; verify file size and integrity. + +E2E: +- Upload and open PDF/EPUB/HTML; play with highlight; persist/resume position. +- Settings changes (provider/model/voice/speeds) do not cause double requests or deadlocks. + +### Risks and mitigations + +- Provider streaming variability: + - Route capability detection and fallback; progressive responses when chunked not possible. +- Safari inconsistencies: + - Proactive blob-fallback and small segment chaining; limit segment durations to reduce gaps. +- Cache growth and quota: + - Dexie TTL/size budgets; purge on startup and opportunistically; surface approximate disk usage to users in settings. +- Large artifacts: + - Range-enabled endpoints; avoid buffering full files in memory. + +### Work sequencing (implementation order) + +1) Dexie foundation +- Schema + client + repositories +- One-time migration script and Progress UI + +2) Engine baseline +- State, queue, cancellation, MediaController baseline +- HTML adapter and streaming route +- Hook facade for React + +3) PDF/EPUB adapters + highlighting +- PDF first, then EPUB (navigation and CFI) +- Splitter dialog-aware grouping + toggle + +4) Exports and downloads +- Chapter MP3 (ZIP), m4b GET download with Range + +5) Sync + polish +- Client sync service +- A11y, performance, tests, legacy removal, version bump + +### Acceptance gates (per milestone) + +- Alpha gate: + - Play cached text in HTML with streaming route; skip/voice switch stable +- Beta gate: + - PDF and EPUB highlight track sentences; chapter export functional; migration works reliably +- GA gate: + - m4b Range download stable for large files; server sync; E2E suite green; legacy removed; version bumped + + +--- + +## Scaffold and naming (authoritative file list) + +The following v1 files will be created verbatim to avoid dual-engine complexity. Each path is the single source of truth for its concern. + +Playback and media +- [src/v1/playback/state.ts](src/v1/playback/state.ts) +- [src/v1/playback/queue.ts](src/v1/playback/queue.ts) +- [src/v1/playback/engine.ts](src/v1/playback/engine.ts) +- [src/v1/playback/hooks/usePlayback.ts](src/v1/playback/hooks/usePlayback.ts) +- [src/v1/playback/media/MediaController.ts](src/v1/playback/media/MediaController.ts) +- [src/v1/playback/media/mediaSession.ts](src/v1/playback/media/mediaSession.ts) +- [src/v1/playback/media/background.ts](src/v1/playback/media/background.ts) +- [src/v1/playback/positionStore.ts](src/v1/playback/positionStore.ts) +- [src/v1/playback/audiobook.ts](src/v1/playback/audiobook.ts) + +TTS providers +- [src/v1/tts/types.ts](src/v1/tts/types.ts) +- [src/v1/tts/Provider.ts](src/v1/tts/Provider.ts) +- [src/v1/tts/providers/OpenAIProvider.ts](src/v1/tts/providers/OpenAIProvider.ts) +- [src/v1/tts/providers/DeepinfraProvider.ts](src/v1/tts/providers/DeepinfraProvider.ts) +- [src/v1/tts/providers/CustomOpenAIProvider.ts](src/v1/tts/providers/CustomOpenAIProvider.ts) +- [src/v1/tts/voices.ts](src/v1/tts/voices.ts) + +Adapters and highlighting +- [src/v1/adapters/DocumentAdapter.ts](src/v1/adapters/DocumentAdapter.ts) +- [src/v1/adapters/PdfAdapter.ts](src/v1/adapters/PdfAdapter.ts) +- [src/v1/adapters/EpubAdapter.ts](src/v1/adapters/EpubAdapter.ts) +- [src/v1/adapters/HtmlAdapter.ts](src/v1/adapters/HtmlAdapter.ts) +- [src/v1/highlight/pdfHighlighter.ts](src/v1/highlight/pdfHighlighter.ts) +- [src/v1/highlight/epubHighlighter.ts](src/v1/highlight/epubHighlighter.ts) + +NLP +- [src/v1/nlp/sentences.ts](src/v1/nlp/sentences.ts) + +Dexie database +- [src/v1/db/schema.ts](src/v1/db/schema.ts) +- [src/v1/db/client.ts](src/v1/db/client.ts) +- [src/v1/db/repositories/DocumentsRepo.ts](src/v1/db/repositories/DocumentsRepo.ts) +- [src/v1/db/repositories/ConfigRepo.ts](src/v1/db/repositories/ConfigRepo.ts) +- [src/v1/db/repositories/AudioCacheRepo.ts](src/v1/db/repositories/AudioCacheRepo.ts) +- [src/v1/db/repositories/VoicesRepo.ts](src/v1/db/repositories/VoicesRepo.ts) +- [src/v1/db/sync.ts](src/v1/db/sync.ts) +- [src/v1/migrations/migrateFromV0.ts](src/v1/migrations/migrateFromV0.ts) + +React contexts and components +- [src/v1/contexts/PlaybackProvider.tsx](src/v1/contexts/PlaybackProvider.tsx) +- [src/v1/contexts/ConfigProviderV1.tsx](src/v1/contexts/ConfigProviderV1.tsx) +- [src/v1/contexts/DocumentProviderV1.tsx](src/v1/contexts/DocumentProviderV1.tsx) +- [src/v1/components/player/Player.tsx](src/v1/components/player/Player.tsx) +- [src/v1/components/player/SpeedControl.tsx](src/v1/components/player/SpeedControl.tsx) +- [src/v1/components/player/VoicesControl.tsx](src/v1/components/player/VoicesControl.tsx) +- [src/v1/components/player/Navigator.tsx](src/v1/components/player/Navigator.tsx) +- [src/v1/components/PDFViewerV1.tsx](src/v1/components/PDFViewerV1.tsx) +- [src/v1/components/EPUBViewerV1.tsx](src/v1/components/EPUBViewerV1.tsx) +- [src/v1/components/HTMLViewerV1.tsx](src/v1/components/HTMLViewerV1.tsx) +- [src/v1/components/Progress/MigrationProgress.tsx](src/v1/components/Progress/MigrationProgress.tsx) + +API routes +- [src/app/api/tts/stream/route.ts](src/app/api/tts/stream/route.ts) +- [src/app/api/tts/route.ts](src/app/api/tts/route.ts:1) +- [src/app/api/tts/voices/route.ts](src/app/api/tts/voices/route.ts:1) +- [src/app/api/audio/convert/route.ts](src/app/api/audio/convert/route.ts:1) + +Tests +- [src/v1/playback/__tests__/engine.spec.ts](src/v1/playback/__tests__/engine.spec.ts) +- [src/v1/adapters/__tests__/adapters.spec.ts](src/v1/adapters/__tests__/adapters.spec.ts) +- [tests/v1/playback.spec.ts](tests/v1/playback.spec.ts) + + +## Sequenced next actions (implementation-ready) + +1) Documentation and dependencies +- [ ] Pin Dexie dependency in [package.json](package.json:1); plan minimal types footprint +- [ ] Lock ADR baseline: [docs/v1/ADR-0001-playback-architecture.md](docs/v1/ADR-0001-playback-architecture.md) + +2) Dexie foundation +- [ ] Author schema tables in [src/v1/db/schema.ts](src/v1/db/schema.ts) +- [ ] Implement [src/v1/db/client.ts](src/v1/db/client.ts) with versioning and migrations +- [ ] Build repositories (Documents, Config, AudioCache, Voices) + +3) Migration and UI +- [ ] Implement [src/v1/migrations/migrateFromV0.ts](src/v1/migrations/migrateFromV0.ts) reading [src/utils/indexedDB.ts](src/utils/indexedDB.ts:1) +- [ ] Wire [src/v1/components/Progress/MigrationProgress.tsx](src/v1/components/Progress/MigrationProgress.tsx) into a first-run flow + +4) Playback engine baseline +- [ ] Implement [src/v1/playback/state.ts](src/v1/playback/state.ts) states and events +- [ ] Implement [src/v1/playback/queue.ts](src/v1/playback/queue.ts) with lookahead and cache keys +- [ ] Implement [src/v1/playback/media/MediaController.ts](src/v1/playback/media/MediaController.ts) with MSE first, blob fallback +- [ ] Implement [src/v1/playback/engine.ts](src/v1/playback/engine.ts) with strict AbortController scoping +- [ ] React facade in [src/v1/playback/hooks/usePlayback.ts](src/v1/playback/hooks/usePlayback.ts) + +5) HTML path to first audio +- [ ] Implement [src/v1/adapters/HtmlAdapter.ts](src/v1/adapters/HtmlAdapter.ts) +- [ ] Implement [src/app/api/tts/stream/route.ts](src/app/api/tts/stream/route.ts) (chunked and progressive fallback) +- [ ] Minimal [src/v1/components/HTMLViewerV1.tsx](src/v1/components/HTMLViewerV1.tsx) + [src/v1/components/player/Player.tsx](src/v1/components/player/Player.tsx) to validate streaming + +6) PDF and EPUB adapters +- [ ] Implement [src/v1/adapters/PdfAdapter.ts](src/v1/adapters/PdfAdapter.ts) with normalized margins and debug toggles, and [src/v1/highlight/pdfHighlighter.ts](src/v1/highlight/pdfHighlighter.ts) +- [ ] Implement [src/v1/adapters/EpubAdapter.ts](src/v1/adapters/EpubAdapter.ts) and [src/v1/highlight/epubHighlighter.ts](src/v1/highlight/epubHighlighter.ts) + +7) NLP dialog grouping +- [ ] Wrap splitter in [src/v1/nlp/sentences.ts](src/v1/nlp/sentences.ts) and honor a quote-aware grouping flag + +8) Exports +- [ ] Add chapter MP3 ZIP path via [src/v1/playback/audiobook.ts](src/v1/playback/audiobook.ts) +- [ ] Harden m4b downloads using Range in [src/app/api/audio/convert/route.ts](src/app/api/audio/convert/route.ts:1) + +9) Sync +- [ ] Implement [src/v1/db/sync.ts](src/v1/db/sync.ts) client-driven sync flow + +10) Switch-over and removal +- [ ] Replace providers in [src/app/providers.tsx](src/app/providers.tsx:1) to v1 +- [ ] Swap viewers on routes to v1 +- [ ] Remove Howler dependency in [package.json](package.json:12) after cutover +- [ ] Delete legacy modules [src/contexts/TTSContext.tsx](src/contexts/TTSContext.tsx:1), [src/utils/indexedDB.ts](src/utils/indexedDB.ts:1) + + +## Acceptance tests to author (explicit) + +Playback engine +- [ ] Single inflight: issuing toggle or voice change cancels prior token and yields exactly one fresh request wave +- [ ] Underflow causes buffering and recovers on next data +- [ ] Background/foreground preserves intended state + +Streaming route +- [ ] Chunked path sends Content-Type audio/mpeg|aac; progressive fallback functions when chunking is not supported +- [ ] Client AbortSignal cancels upstream + +Adapters and highlight +- [ ] PDF: left/right margins trim deterministically across sample PDFs; highlight aligns with playing sentence +- [ ] EPUB: section navigation next/prev works; highlight aligns; resume from location token +- [ ] HTML: block indexing stable and resume works + +NLP grouping +- [ ] Dialog passages grouped unless size exceeds threshold; toggle off reverts to baseline splitting + +Exports +- [ ] Chapter MP3 ZIP produces files named NN - Chapter Title.mp3; ZIP streams without buffering entire archive +- [ ] m4b Range download stable for 1–2 GB files under Docker + +Sync and migration +- [ ] First-run migration completes with retriable steps and progress +- [ ] Sync uploads/downloads all docs with correct counts and bytes + + +## Risks checklist and mitigations + +- [ ] Provider streaming variability -> route capability checks and progressive response fallback +- [ ] Safari MSE variance -> short blob segment chaining; segment duration caps +- [ ] Cache quota pressure -> Dexie TTL/LRU budgets and purge on startup +- [ ] Large artifact memory -> Range endpoints; avoid in-memory buffers; disk-backed streams +- [ ] Race conditions -> session tokens and abort scoping for network and decode paths + + +## Notes on versioning and flags + +- [ ] Feature flags: dialog grouping default on for EPUB/HTML, off for PDF +- [ ] Schema version pin in [src/v1/db/client.ts](src/v1/db/client.ts) with explicit upgrade function +- [ ] Saved voices map by provider:model in ConfigRepo + + +## Done criteria for 1.0.0 + +- [ ] Legacy engine removed; all routes/viewers wired to v1 +- [ ] Exports (m4b and chapter MP3) robust under Docker and local +- [ ] E2E tests cover PDF/EPUB/HTML reading, streaming playback, skip, voice switch, resume, exports +- [ ] Docs updated: [docs/v1/README.md](docs/v1/README.md), [README.md](README.md:10), changelog updated and version bumped in [package.json](package.json:3) + + +--- + +## Capability matrix and detection + +Provider streaming capability (planned handling) +- OpenAI (audio.speech): treat as progressively readable stream from the Response body; prefer chunked transfer in [src/app/api/tts/stream/route.ts](src/app/api/tts/stream/route.ts) +- Deepinfra: prefer progressive streaming when supported; fallback to full buffer proxy +- Custom OpenAI-compatible: probe endpoint with a tiny synth call in a capability check, then cache provider features in [src/v1/tts/voices.ts](src/v1/tts/voices.ts) or [src/v1/tts/Provider.ts](src/v1/tts/Provider.ts) + +Browser feature checks (performed once per session) +- MSE availability and supported MIME types for audio/mpeg and audio/aac; cache in memory in [src/v1/playback/media/MediaController.ts](src/v1/playback/media/MediaController.ts) +- Visibility API handling (document.hidden) and Media Session handlers: integrated in [src/v1/playback/media/background.ts](src/v1/playback/media/background.ts) and [src/v1/playback/media/mediaSession.ts](src/v1/playback/media/mediaSession.ts) + +Decision rules +- If MSE and provider stream: MSE pipeline +- If MSE but provider returns full buffer only: still MSE if chunking the buffer is practical; else blob fallback +- If no MSE: blob fallback with small segments to minimize gaps + +## Migration plan (step-by-step) + +Pre-migration checks +- Detect legacy stores via [src/utils/indexedDB.ts](src/utils/indexedDB.ts:1) +- Estimate total bytes and number of documents; show plan to user + +Execution steps (idempotent) +1) Create Dexie stores in [src/v1/db/client.ts](src/v1/db/client.ts) +2) Copy documents from legacy to Dexie in batches; convert numeric arrays to ArrayBuffer/Blob +3) Copy config items; normalize provider, model, baseUrl; write to [src/v1/db/repositories/ConfigRepo.ts](src/v1/db/repositories/ConfigRepo.ts) +4) Copy last-location metadata into [src/v1/playback/positionStore.ts](src/v1/playback/positionStore.ts) +5) Report progress and allow retry on failure (resume from last completed batch) + +Post-migration +- Mark legacy as migrated (config flag) to avoid future runs +- Option to purge legacy data (explicit user action) + +UI +- Show a first-run guided modal in [src/v1/components/Progress/MigrationProgress.tsx](src/v1/components/Progress/MigrationProgress.tsx) with percentage and current step + +## Chapter MP3 export plan + +Source of chapter boundaries +- EPUB: TOC mapping in adapter; chapter per TOC item, merge small sections to avoid micro-chapters +- PDF: future enhancement (out of initial scope unless PDF outlines/TOC present) + +Export path +- Build per-chapter audio buffers using providers via the same synth path as playback (no separate engine) +- Stream a ZIP response with ZIP64 support for large aggregate sizes through [src/app/api/audio/convert/route.ts](src/app/api/audio/convert/route.ts:1) with a mode switch +- Names: NN - Chapter Title.mp3 with zero-padded NN +- Progress events (client-side): show current chapter index, bytes accumulated + +Acceptance criteria +- A 50+ chapter EPUB exports as a streamed ZIP without UI stalls or memory spikes +- ZIP integrity verified by unzip on client + +## Large m4b export stability (range-enabled) + +Server +- Add GET download path on [src/app/api/audio/convert/route.ts](src/app/api/audio/convert/route.ts:1) to stream artifacts from disk via Accept-Ranges with 206 responses +- Keep temp artifacts by bookId; implement cleanup policy (e.g., TTL-based) and a management UI later + +Client +- Perform HTTP range-enabled download; avoid buffering into memory; use streams to file via browser download APIs + +Acceptance criteria +- 1–2 GB m4b downloads complete in Docker with stable throughput; resume on flaky connections is acceptable via range retries + +## Highlight alignment and sentence mapping + +PDF +- Normalize x to [0..1]; apply left/right margins strictly +- When width is missing, estimate from glyph metrics; exclude header/footer using normalized y +- Highlighter replays processed-to-raw mapping from adapter map in [src/v1/highlight/pdfHighlighter.ts](src/v1/highlight/pdfHighlighter.ts) + +EPUB +- Use adapter-provided raw mapping and section context to highlight current text span in [src/v1/highlight/epubHighlighter.ts](src/v1/highlight/epubHighlighter.ts) + +HTML +- Use block index mapping; highlighting simpler (no layout transforms) + +## Performance and budgets + +Audio cache (Dexie) +- Budget target: 100–250 MB, configurable +- Eviction: TTL and LRU; purge on startup and opportunistically when budgets exceeded +- Keys: hash(text + provider + model + voice + speed + format) + +Queue and prefetch +- Lookahead of 1–2 blocks, adaptive to latency +- Abort prefetch on config change or navigation + +Target latencies +- Cached block start-to-speech < 500–700 ms +- Non-cached with stream: first audible audio chunk < 1.5–2.5 s (provider-dependent) + +## Accessibility checklist + +Player +- Buttons keyboard accessible, ARIA labels, focus ring +- Space/Enter to toggle play/pause; left/right to skip; up/down for speed adjustments + +Viewers +- Highlight announced via ARIA-live optional verbosity setting +- Navigator provides input with label and validation errors announced + +## Work package owners (placeholders) + +- Dexie schema and repos: assign to Core +- Engine and MediaController: assign to Core +- HTML adapter path to first audio: assign to Core +- PDF adapter + highlighting: assign to PDF +- EPUB adapter + highlighting: assign to EPUB +- Chapter MP3 export and m4b Range path: assign to Audio +- Migration UI: assign to UI +- Tests (unit, integration, e2e): assign to QA + +## Timeline (tentative, after approval) + +- Week 1–2: Dexie + Engine baseline + HTML path + stream route (Alpha) +- Week 3–4: PDF + EPUB adapters + highlighting + dialog grouping + migration UI (Beta) +- Week 5–6: Chapter MP3 + m4b Range + sync + tests + a11y + legacy removal (GA) + +## Open questions (to resolve before coding) + +- Exact provider streaming capabilities to lock in API adapter behavior in [src/app/api/tts/stream/route.ts](src/app/api/tts/stream/route.ts) +- Target audio formats default (mp3 vs aac) per provider for best MSE compatibility +- Chapter detection for PDFs: scope to outlines only in v1.0 or defer to 1.1 + +## Planning links + +- Issue mapping: [docs/v1/issues-to-components.md](docs/v1/issues-to-components.md) +- Architecture decision: [docs/v1/ADR-0001-playback-architecture.md](docs/v1/ADR-0001-playback-architecture.md) +- Implementation scaffold list above mirrors the master checklist paths diff --git a/package.json b/package.json index 8b2e7d6..026ce35 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "openreader-webui", - "version": "0.3.1", + "version": "0.4.0", "private": true, "scripts": { "dev": "next dev --turbopack -p 3003", @@ -10,43 +10,42 @@ "test": "playwright test" }, "dependencies": { - "@headlessui/react": "^2.2.0", + "@headlessui/react": "^2.2.9", "@types/howler": "^2.2.12", - "@types/string-similarity": "^4.0.2", "@types/uuid": "^10.0.0", "@vercel/analytics": "^1.5.0", + "cmpstr": "^3.0.4", "compromise": "^14.14.4", - "core-js": "^3.41.0", + "core-js": "^3.46.0", "epubjs": "^0.3.93", "howler": "^2.2.4", - "lru-cache": "^11.0.2", - "next": "^15.2.1", - "openai": "^4.86.1", + "lru-cache": "^11.2.2", + "next": "^15.5.6", + "openai": "^4.104.0", "pdfjs-dist": "4.8.69", - "react": "^19.0.0", + "react": "^19.2.0", "react-dnd": "^16.0.1", "react-dnd-html5-backend": "^16.0.1", - "react-dom": "^19.0.0", + "react-dom": "^19.2.0", "react-dropzone": "^14.3.8", - "react-hot-toast": "^2.5.2", + "react-hot-toast": "^2.6.0", "react-markdown": "^10.1.0", "react-pdf": "^9.2.1", - "react-reader": "^2.0.12", + "react-reader": "^2.0.15", "remark-gfm": "^4.0.1", - "string-similarity": "^4.0.4", "uuid": "^11.1.0" }, "devDependencies": { "@eslint/eslintrc": "^3", - "@playwright/test": "^1.50.1", - "@tailwindcss/typography": "^0.5.16", - "@types/node": "^20", - "@types/react": "^19", - "@types/react-dom": "^19", - "eslint": "^9", - "eslint-config-next": "15.1.5", - "postcss": "^8", - "tailwindcss": "^3.4.1", - "typescript": "^5" + "@playwright/test": "^1.56.1", + "@tailwindcss/typography": "^0.5.19", + "@types/node": "^20.19.24", + "@types/react": "^19.2.2", + "@types/react-dom": "^19.2.2", + "eslint": "^9.39.1", + "eslint-config-next": "^15.5.6", + "postcss": "^8.5.6", + "tailwindcss": "^3.4.18", + "typescript": "^5.9.3" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index cb31e0b..1bbecc8 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,25 +9,25 @@ importers: .: dependencies: '@headlessui/react': - specifier: ^2.2.0 + specifier: ^2.2.9 version: 2.2.9(react-dom@19.2.0(react@19.2.0))(react@19.2.0) '@types/howler': specifier: ^2.2.12 version: 2.2.12 - '@types/string-similarity': - specifier: ^4.0.2 - version: 4.0.2 '@types/uuid': specifier: ^10.0.0 version: 10.0.0 '@vercel/analytics': specifier: ^1.5.0 version: 1.5.0(next@15.5.6(@playwright/test@1.56.1)(react-dom@19.2.0(react@19.2.0))(react@19.2.0))(react@19.2.0) + cmpstr: + specifier: ^3.0.4 + version: 3.0.4 compromise: specifier: ^14.14.4 version: 14.14.4 core-js: - specifier: ^3.41.0 + specifier: ^3.46.0 version: 3.46.0 epubjs: specifier: ^0.3.93 @@ -36,19 +36,19 @@ importers: specifier: ^2.2.4 version: 2.2.4 lru-cache: - specifier: ^11.0.2 + specifier: ^11.2.2 version: 11.2.2 next: - specifier: ^15.2.1 + specifier: ^15.5.6 version: 15.5.6(@playwright/test@1.56.1)(react-dom@19.2.0(react@19.2.0))(react@19.2.0) openai: - specifier: ^4.86.1 + specifier: ^4.104.0 version: 4.104.0 pdfjs-dist: specifier: 4.8.69 version: 4.8.69 react: - specifier: ^19.0.0 + specifier: ^19.2.0 version: 19.2.0 react-dnd: specifier: ^16.0.1 @@ -57,13 +57,13 @@ importers: specifier: ^16.0.1 version: 16.0.1 react-dom: - specifier: ^19.0.0 + specifier: ^19.2.0 version: 19.2.0(react@19.2.0) react-dropzone: specifier: ^14.3.8 version: 14.3.8(react@19.2.0) react-hot-toast: - specifier: ^2.5.2 + specifier: ^2.6.0 version: 2.6.0(react-dom@19.2.0(react@19.2.0))(react@19.2.0) react-markdown: specifier: ^10.1.0 @@ -72,14 +72,11 @@ importers: specifier: ^9.2.1 version: 9.2.1(@types/react@19.2.2)(react-dom@19.2.0(react@19.2.0))(react@19.2.0) react-reader: - specifier: ^2.0.12 + specifier: ^2.0.15 version: 2.0.15(react@19.2.0) remark-gfm: specifier: ^4.0.1 version: 4.0.1 - string-similarity: - specifier: ^4.0.4 - version: 4.0.4 uuid: specifier: ^11.1.0 version: 11.1.0 @@ -88,34 +85,34 @@ importers: specifier: ^3 version: 3.3.1 '@playwright/test': - specifier: ^1.50.1 + specifier: ^1.56.1 version: 1.56.1 '@tailwindcss/typography': - specifier: ^0.5.16 + specifier: ^0.5.19 version: 0.5.19(tailwindcss@3.4.18) '@types/node': - specifier: ^20 + specifier: ^20.19.24 version: 20.19.24 '@types/react': - specifier: ^19 + specifier: ^19.2.2 version: 19.2.2 '@types/react-dom': - specifier: ^19 + specifier: ^19.2.2 version: 19.2.2(@types/react@19.2.2) eslint: - specifier: ^9 + specifier: ^9.39.1 version: 9.39.1(jiti@1.21.7) eslint-config-next: - specifier: 15.1.5 - version: 15.1.5(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) + specifier: ^15.5.6 + version: 15.5.6(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) postcss: - specifier: ^8 + specifier: ^8.5.6 version: 8.5.6 tailwindcss: - specifier: ^3.4.1 + specifier: ^3.4.18 version: 3.4.18 typescript: - specifier: ^5 + specifier: ^5.9.3 version: 5.9.3 packages: @@ -379,8 +376,8 @@ packages: '@next/env@15.5.6': resolution: {integrity: sha512-3qBGRW+sCGzgbpc5TS1a0p7eNxnOarGVQhZxfvTdnV0gFI61lX7QNtQ4V1TSREctXzYn5NetbUsLvyqwLFJM6Q==} - '@next/eslint-plugin-next@15.1.5': - resolution: {integrity: sha512-3cCrXBybsqe94UxD6DBQCYCCiP9YohBMgZ5IzzPYHmPzj8oqNlhBii5b6o1HDDaRHdz2pVnSsAROCtrczy8O0g==} + '@next/eslint-plugin-next@15.5.6': + resolution: {integrity: sha512-YxDvsT2fwy1j5gMqk3ppXlsgDopHnkM4BoxSVASbvvgh5zgsK8lvWerDzPip8k3WVzsTZ1O7A7si1KNfN4OZfQ==} '@next/swc-darwin-arm64@15.5.6': resolution: {integrity: sha512-ES3nRz7N+L5Umz4KoGfZ4XX6gwHplwPhioVRc25+QNsDa7RtUF/z8wJcbuQ2Tffm5RZwuN2A063eapoJ1u4nPg==} @@ -578,9 +575,6 @@ packages: '@types/react@19.2.2': resolution: {integrity: sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA==} - '@types/string-similarity@4.0.2': - resolution: {integrity: sha512-LkJQ/jsXtCVMK+sKYAmX/8zEq+/46f1PTQw7YtmQwb74jemS1SlNLmARM2Zml9DgdDTWKAtc5L13WorpHPDjDA==} - '@types/unist@2.0.11': resolution: {integrity: sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==} @@ -979,6 +973,10 @@ packages: resolution: {integrity: sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==} engines: {node: '>=6'} + cmpstr@3.0.4: + resolution: {integrity: sha512-GPkstf+gjng88j2UNUMGipRLr+HpnZq8Qct6vlAeRdFYKiFJYa2rFX1y4ikxwKPsttIBOZQ2lhpMBVZWRiSuVw==} + engines: {node: '>=18.0.0'} + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -1182,8 +1180,8 @@ packages: resolution: {integrity: sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==} engines: {node: '>=12'} - eslint-config-next@15.1.5: - resolution: {integrity: sha512-Awm7iUJY8toOR+fU8yTxZnA7/LyOGUGOd6cENCuDfJ3gucHOSmLdOSGJ4u+nlrs8p5qXemua42bZmq+uOzxl6Q==} + eslint-config-next@15.5.6: + resolution: {integrity: sha512-cGr3VQlPsZBEv8rtYp4BpG1KNXDqGvPo9VC1iaCgIA11OfziC/vczng+TnAS3WpRIR3Q5ye/6yl+CRUuZ1fPGg==} peerDependencies: eslint: ^7.23.0 || ^8.0.0 || ^9.0.0 typescript: '>=3.3.1' @@ -2493,10 +2491,6 @@ packages: resolution: {integrity: sha512-eLoXW/DHyl62zxY4SCaIgnRhuMr6ri4juEYARS8E6sCEqzKpOiE521Ucofdx+KnDZl5xmvGYaaKCk5FEOxJCoQ==} engines: {node: '>= 0.4'} - string-similarity@4.0.4: - resolution: {integrity: sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ==} - deprecated: Package no longer supported. Contact Support at https://www.npmjs.com/support for more info. - string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -3031,7 +3025,7 @@ snapshots: '@next/env@15.5.6': {} - '@next/eslint-plugin-next@15.1.5': + '@next/eslint-plugin-next@15.5.6': dependencies: fast-glob: 3.3.1 @@ -3216,8 +3210,6 @@ snapshots: dependencies: csstype: 3.1.3 - '@types/string-similarity@4.0.2': {} - '@types/unist@2.0.11': {} '@types/unist@3.0.3': {} @@ -3610,6 +3602,8 @@ snapshots: clsx@2.1.1: {} + cmpstr@3.0.4: {} + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -3884,9 +3878,9 @@ snapshots: escape-string-regexp@5.0.0: {} - eslint-config-next@15.1.5(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3): + eslint-config-next@15.5.6(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3): dependencies: - '@next/eslint-plugin-next': 15.1.5 + '@next/eslint-plugin-next': 15.5.6 '@rushstack/eslint-patch': 1.14.1 '@typescript-eslint/eslint-plugin': 8.46.3(@typescript-eslint/parser@8.46.3(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3))(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) '@typescript-eslint/parser': 8.46.3(eslint@9.39.1(jiti@1.21.7))(typescript@5.9.3) @@ -5608,8 +5602,6 @@ snapshots: es-errors: 1.3.0 internal-slot: 1.1.0 - string-similarity@4.0.4: {} - string-width@4.2.3: dependencies: emoji-regex: 8.0.0 diff --git a/src/utils/pdf.ts b/src/utils/pdf.ts index e0059e5..ae8cd9a 100644 --- a/src/utils/pdf.ts +++ b/src/utils/pdf.ts @@ -1,9 +1,11 @@ import { pdfjs } from 'react-pdf'; -import stringSimilarity from 'string-similarity'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; import type { PDFDocumentProxy } from 'pdfjs-dist'; import "core-js/proposals/promise-with-resolvers"; import { processTextToSentences } from '@/utils/nlp'; +import { CmpStr } from 'cmpstr'; + +const cmp = CmpStr.create().setMetric( 'levenshtein' ).setFlags( 'i' ); // Function to detect if we need to use legacy build function shouldUseLegacyBuild() { @@ -207,7 +209,7 @@ export function findBestTextMatch( combinedText = newText; currentElements.push(node.element); - const similarity = stringSimilarity.compareTwoStrings(targetText, combinedText); + const similarity = cmp.compare(combinedText, targetText); const lengthDiff = Math.abs(combinedText.length - targetText.length); const lengthPenalty = lengthDiff / targetText.length; const adjustedRating = similarity * (1 - lengthPenalty * 0.5); @@ -331,7 +333,7 @@ export function handleTextClick( let bestSentenceMatch = { sentence: '', rating: 0 }; for (const sentence of sentences) { - const rating = stringSimilarity.compareTwoStrings(matchText, sentence); + const rating = cmp.compare(matchText, sentence); if (rating > bestSentenceMatch.rating) { bestSentenceMatch = { sentence, rating }; }