refactor(pdf): implement ONNX-based Docling layout parsing and block-level TTS for PDFs

Migrate PDF parsing to use ONNX Docling layout model for structured block extraction, enabling improved TTS segmentation and chaptering. Add compute backend abstraction for heavy tasks (alignment, layout parsing) with configuration via `OPENREADER_COMPUTE_MODE`. Integrate block-level locators and document settings for per-document PDF parsing options. Update S3 storage for parsed PDF JSON, add migration and schema changes, and extend client and server APIs for parsed data and settings. Remove legacy word highlight flag in favor of compute capability detection.

- Add ONNX model download, local/none compute modes, and `onnxruntime-node` dependency
- Update PDF viewer and TTS pipeline to use parsed blocks and block-level locators
- Add document settings storage and APIs for per-document PDF options
- Update S3 storage layout for parsed PDF JSON
- Add admin/config/docs updates for new compute and parsing features
- Remove obsolete word highlight runtime flag and UI

BREAKING CHANGE: PDF parsing and word highlighting now require `OPENREADER_COMPUTE_MODE=local` and ONNX model; document settings and S3 layout updated; legacy word highlight flag removed.
This commit is contained in:
Richard R 2026-05-17 21:18:51 -06:00
parent d90e48aaf3
commit 766c04d08d
72 changed files with 6623 additions and 263 deletions

View file

@ -80,6 +80,17 @@ IMPORT_LIBRARY_DIRS=
# (Required without Docker) Path to your local whisper.cpp CLI binary for STT timestamp generation
WHISPER_CPP_BIN=/whisper.cpp/build/bin/whisper-cli
# Heavy compute backend mode for whisper alignment + PDF layout parsing.
# local = run compute in-process (default)
# none = disable both capabilities (good for preview/serverless)
# worker = reserved for future external worker mode (not implemented in v1)
OPENREADER_COMPUTE_MODE=local
# OPENREADER_COMPUTE_WORKER_URL=
# OPENREADER_COMPUTE_WORKER_TOKEN=
# Optional override for Docling layout model download URL
# OPENREADER_DOCLING_MODEL_URL=https://huggingface.co/ds4sd/docling-layout-heron/resolve/main/model.onnx
# (Optional) Override ffmpeg binary path used for audiobook processing
FFMPEG_BIN=
@ -95,4 +106,3 @@ FFMPEG_BIN=
# NEXT_PUBLIC_DEFAULT_TTS_PROVIDER=custom-openai
# NEXT_PUBLIC_CHANGELOG_FEED_URL=https://docs.openreader.richardr.dev/changelog/manifest.json
# NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT=true
# NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT=true

View file

@ -11,6 +11,7 @@ RUN git clone --depth 1 https://github.com/ggml-org/whisper.cpp.git && \
cd whisper.cpp && \
cmake -S . -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=OFF $( [ "$TARGETARCH" = "arm64" ] && echo "-DGGML_CPU_ARM_ARCH=armv8-a" || true ) && \
cmake --build build -j
RUN wget -qO /tmp/whisper.cpp-LICENSE.txt "https://raw.githubusercontent.com/ggml-org/whisper.cpp/master/LICENSE"
# Stage 1b: extract seaweedfs weed binary (for optional embedded weed mini)
# Pin to 4.18 because CI observed upload regressions on 4.19.
@ -71,10 +72,13 @@ COPY --from=app-builder /app ./
COPY --from=app-builder /app/THIRD_PARTY_LICENSES /licenses
# Include SeaweedFS license text for the copied weed binary.
COPY --from=seaweedfs-builder /tmp/SeaweedFS-LICENSE.txt /licenses/SeaweedFS-LICENSE.txt
# Include static model notices for runtime-downloaded assets.
COPY src/lib/server/pdf-layout/model/LICENSE.txt /licenses/docling-layout-LICENSE.txt
# Copy the compiled whisper.cpp build output into the runtime image
# (includes whisper-cli and its shared libraries, e.g. libwhisper.so, libggml.so)
COPY --from=whisper-builder /opt/whisper.cpp/build /opt/whisper.cpp/build
COPY --from=whisper-builder /tmp/whisper.cpp-LICENSE.txt /licenses/whisper.cpp-LICENSE.txt
# Copy seaweedfs weed binary for optional embedded local S3.
COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed
RUN chmod +x /usr/local/bin/weed

View file

@ -20,7 +20,8 @@ OpenReader is an open source, self-host-friendly text-to-speech document reader
- 🎯 **Multi-provider TTS** with OpenAI-compatible endpoints and cloud providers (Kokoro-FastAPI, KittenTTS-FastAPI, Orpheus-FastAPI or OpenAI, Replicate, DeepInfra).
- 📖 **Read-along playback** for PDF/EPUB with sentence-aware narration.
- ⏱️ **Word-by-word highlighting** via optional `whisper.cpp` timestamps.
- ⏱️ **Word-by-word highlighting** via optional `whisper.cpp` timestamps (`OPENREADER_COMPUTE_MODE=local` + `WHISPER_CPP_BIN`).
- 🧱 **Layout-aware PDF parsing** (Docling ONNX) with structured blocks for cleaner TTS/chaptering.
- 🛜 **Sync + library import** to bring docs across devices and from server-mounted folders.
- 🗂️ **Flexible storage** with embedded SeaweedFS or external S3-compatible backends.
- 🎧 **Audiobook export** in `m4b`/`mp3` with resumable chapter processing.

View file

@ -74,11 +74,12 @@ Runtime-editable settings, one row per key:
| `restrictUserApiKeys` | Restrict user-supplied API keys/base URLs; when `true`, only admin shared providers are allowed. |
| `enableTtsProvidersTab` | Whether the user-facing TTS Provider tab in Settings is shown. |
| `showAllProviderModels` | When `false`, users are restricted to each provider's default model (shared provider `defaultModel` or built-in provider default). |
| `enableWordHighlight` | Enable whisper.cpp word-by-word highlighting during TTS playback. |
| `enableAudiobookExport` | Show the audiobook export entry points on PDF/EPUB pages. |
| `enableDocxConversion` | Accept .docx uploads (converted to PDF server-side). |
| `enableDestructiveDeleteActions` | Show "Delete all data" buttons in the Documents tab (auth-disabled mode). |
Word-by-word highlighting and PDF layout parsing capability are controlled by `OPENREADER_COMPUTE_MODE` (server env), not an admin runtime flag.
Each row shows a source badge:
- **from env** — the value was migrated from the corresponding `NEXT_PUBLIC_*` env var on first boot. Editing it in the UI flips the source to **admin**.

View file

@ -31,7 +31,7 @@ ADMIN_EMAILS=alice@example.com,bob@example.com
Admins see a new **Admin** tab in **Settings** with two sub-tabs:
- **Shared TTS providers** — server-managed TTS provider instances with encrypted keys, visible to all users.
- **Site features** — runtime overrides for what were previously `NEXT_PUBLIC_*` build-time flags (including account signup availability, default TTS provider/model, word highlighting, audiobook export, etc.).
- **Site features** — runtime overrides for what were previously `NEXT_PUBLIC_*` build-time flags (including account signup availability, default TTS provider/model, audiobook export, etc.).
Admin assignment is reconciled on every session resolution, so removing an email from `ADMIN_EMAILS` demotes the user on next login without a restart. See [Admin Panel](./admin-panel) for the full reference.

View file

@ -156,7 +156,7 @@ If you are not on Debian/Ubuntu, install equivalent packages with your distro pa
- Arch: use `pacman` (`base-devel cmake curl git tar xz`)
:::tip
Set `WHISPER_CPP_BIN` in your `.env` to enable word-by-word highlighting.
Set `OPENREADER_COMPUTE_MODE=local` and `WHISPER_CPP_BIN` in your `.env` to enable word-by-word highlighting.
:::
</details>
@ -194,6 +194,7 @@ Use one of these `.env` mode templates:
```env
API_BASE=http://host.docker.internal:8880/v1
API_KEY=none
OPENREADER_COMPUTE_MODE=local
# Leave BASE_URL and AUTH_SECRET unset to keep auth disabled.
# (Admin panel is unavailable without auth.)
# API_BASE/API_KEY seed a shared default provider if you want shared mode.
@ -205,6 +206,7 @@ API_KEY=none
```env
API_BASE=http://host.docker.internal:8880/v1
API_KEY=none
OPENREADER_COMPUTE_MODE=local
BASE_URL=http://localhost:3003
AUTH_SECRET=<generate-with-openssl-rand-hex-32>
# Optional when you need multiple local origins:
@ -219,6 +221,7 @@ AUTH_SECRET=<generate-with-openssl-rand-hex-32>
# on first boot, then no longer read. Manage them in Settings → Admin afterwards.
API_BASE=http://host.docker.internal:8880/v1
API_KEY=none
OPENREADER_COMPUTE_MODE=local
BASE_URL=http://localhost:3003
AUTH_SECRET=<generate-with-openssl-rand-hex-32>
# Comma-separated emails to auto-promote to admin on signin.
@ -231,6 +234,7 @@ ADMIN_EMAILS=you@example.com
```env
API_BASE=http://host.docker.internal:8880/v1
API_KEY=none
OPENREADER_COMPUTE_MODE=local
USE_EMBEDDED_WEED_MINI=false
S3_BUCKET=your-bucket
S3_REGION=us-east-1
@ -256,6 +260,10 @@ If you want each user to enter personal provider credentials, set `restrictUserA
For all environment variables, see [Environment Variables](../reference/environment-variables).
:::
:::tip Optional model prefetch
To pre-populate the Docling ONNX model cache before first PDF parse, run `pnpm fetch-models`.
:::
See [Auth](../configure/auth) for app/auth behavior.
See [Admin Panel](../configure/admin-panel) for the shared-provider and feature-flag management UI.
Storage configuration details are in [Object / Blob Storage](../configure/object-blob-storage).

View file

@ -35,6 +35,11 @@ BASE_URL=https://your-app.vercel.app
AUTH_SECRET=...
ADMIN_EMAILS=you@example.com # comma-separated; admins manage TTS + features in-app
# Heavy compute (recommended on Vercel in v1)
# local = requires native binaries/models in-process
# none = disable whisper alignment + PDF layout parsing
OPENREADER_COMPUTE_MODE=none
# First-boot seed for the TTS shared provider (optional; manage in-app afterwards)
API_KEY=your_replicate_key
# API_BASE only needed for OpenAI-compatible self-hosted providers
@ -58,7 +63,6 @@ After the first successful deploy and admin login, open **Settings → Admin** a
- `defaultTtsProvider=replicate` (or your preferred shared slug).
- `showAllProviderModels=false` if you want users locked to each provider's default model.
- `enableAudiobookExport=true`.
- `enableWordHighlight=false` unless your timestamp stack is configured.
## 3. Legacy first-boot seed (optional)
@ -126,4 +130,4 @@ Adjust memory per route if your files are larger or your plan differs.
1. Upload and read a PDF/EPUB document.
2. Confirm sync/blob fetch works across refreshes/devices.
3. Generate at least one audiobook chapter and play/download it.
4. If using word highlighting, verify timestamps are produced and rendered.
4. If you later enable compute locally (`OPENREADER_COMPUTE_MODE=local`), verify word highlighting timestamps on a TTS run.

View file

@ -53,7 +53,11 @@ For auth-enabled deployments, use **Settings → Admin** as the primary source o
| `RUN_FS_MIGRATIONS` | Storage migrations | `true` | Set `false` to skip startup filesystem -> S3/DB migration pass |
| `IMPORT_LIBRARY_DIR` | Library import | `docstore/library` fallback | Set a single server library root |
| `IMPORT_LIBRARY_DIRS` | Library import | unset | Set multiple roots (comma/colon/semicolon separated) |
| `WHISPER_CPP_BIN` | Word timing | unset | Set to enable `whisper.cpp` timestamps |
| `OPENREADER_COMPUTE_MODE` | Heavy compute backend | `local` | Set to `none` to disable whisper alignment + PDF layout parsing |
| `OPENREADER_COMPUTE_WORKER_URL` | Heavy compute backend | unset | Reserved for future worker backend mode (`worker`) |
| `OPENREADER_COMPUTE_WORKER_TOKEN` | Heavy compute backend | unset | Reserved for future worker backend mode (`worker`) |
| `OPENREADER_DOCLING_MODEL_URL` | PDF layout model | Docling HF URL | Override ONNX download URL for `ensureModel()` |
| `WHISPER_CPP_BIN` | Word timing (local mode) | unset | Set to enable `whisper.cpp` timestamps in `OPENREADER_COMPUTE_MODE=local` |
| `FFMPEG_BIN` | Audio runtime | auto-detected (`ffmpeg-static`) | Override ffmpeg binary path |
@ -346,6 +350,38 @@ Multiple library roots for server library import.
## Audio Tooling and Alignment
### OPENREADER_COMPUTE_MODE
Selects the backend for heavy compute features (word alignment + PDF layout parsing).
- Default: `local`
- Supported in v1:
- `local`: run compute in-process on the app server
- `none`: disable these features cleanly
- `worker` is reserved for a future external worker backend and currently fails fast at startup if selected
### OPENREADER_COMPUTE_WORKER_URL
Reserved for future external compute worker mode.
- Used only when `OPENREADER_COMPUTE_MODE=worker` (not implemented in v1)
- Leave unset in v1
### OPENREADER_COMPUTE_WORKER_TOKEN
Reserved bearer token for future external compute worker mode.
- Used only when `OPENREADER_COMPUTE_MODE=worker` (not implemented in v1)
- Leave unset in v1
### OPENREADER_DOCLING_MODEL_URL
Override URL for the Docling ONNX layout model downloaded by `ensureModel()`.
- Default: `https://huggingface.co/ds4sd/docling-layout-heron/resolve/main/model.onnx`
- Optional for custom mirrors/air-gapped workflows
- You can also pre-populate the model cache via `pnpm fetch-models`
### WHISPER_CPP_BIN
Absolute path to compiled `whisper.cpp` binary for word-level timestamps.
@ -432,14 +468,3 @@ Controls whether audiobook export UI/actions are shown in the client.
- Default: `true` (enabled)
- Affects export entry points in PDF/EPUB pages and document settings UI
- Runtime key: `enableAudiobookExport`
### NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT
Controls word-by-word highlighting UI and timestamp-alignment behavior.
- Default: `true` (enabled)
- Requires working timestamp generation (for example `WHISPER_CPP_BIN`)
- Affects:
- Word-highlight toggles in document settings
- Alignment requests during TTS playback
- Runtime key: `enableWordHighlight`

View file

@ -0,0 +1,14 @@
CREATE TABLE "document_settings" (
"document_id" text NOT NULL,
"user_id" text NOT NULL,
"data_json" jsonb DEFAULT '{}'::jsonb NOT NULL,
"client_updated_at_ms" bigint DEFAULT 0 NOT NULL,
"created_at" bigint DEFAULT (extract(epoch from now()) * 1000)::bigint,
"updated_at" bigint DEFAULT (extract(epoch from now()) * 1000)::bigint,
CONSTRAINT "document_settings_document_id_user_id_pk" PRIMARY KEY("document_id","user_id")
);
--> statement-breakpoint
ALTER TABLE "documents" ADD COLUMN "parse_status" text;--> statement-breakpoint
ALTER TABLE "documents" ADD COLUMN "parsed_json_key" text;--> statement-breakpoint
ALTER TABLE "document_settings" ADD CONSTRAINT "document_settings_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
CREATE INDEX "idx_document_settings_user_id" ON "document_settings" USING btree ("user_id");

File diff suppressed because it is too large Load diff

View file

@ -36,6 +36,13 @@
"when": 1778644075378,
"tag": "0004_admin_panel",
"breakpoints": true
},
{
"idx": 5,
"version": "7",
"when": 1779041718214,
"tag": "0005_pdf_layout_and_compute",
"breakpoints": true
}
]
}

View file

@ -0,0 +1,14 @@
CREATE TABLE `document_settings` (
`document_id` text NOT NULL,
`user_id` text NOT NULL,
`data_json` text DEFAULT '{}' NOT NULL,
`client_updated_at_ms` integer DEFAULT 0 NOT NULL,
`created_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)),
`updated_at` integer DEFAULT (cast(unixepoch('subsecond') * 1000 as integer)),
PRIMARY KEY(`document_id`, `user_id`),
FOREIGN KEY (`user_id`) REFERENCES `user`(`id`) ON UPDATE no action ON DELETE cascade
);
--> statement-breakpoint
CREATE INDEX `idx_document_settings_user_id` ON `document_settings` (`user_id`);--> statement-breakpoint
ALTER TABLE `documents` ADD `parse_status` text;--> statement-breakpoint
ALTER TABLE `documents` ADD `parsed_json_key` text;

File diff suppressed because it is too large Load diff

View file

@ -36,6 +36,13 @@
"when": 1778644075081,
"tag": "0004_admin_panel",
"breakpoints": true
},
{
"idx": 5,
"version": "6",
"when": 1779041717890,
"tag": "0005_pdf_layout_and_compute",
"breakpoints": true
}
]
}

View file

@ -15,6 +15,7 @@
"migrate-fs": "node scripts/migrate-fs-v2.mjs",
"migrate-fs:dry-run": "node scripts/migrate-fs-v2.mjs --dry-run true",
"generate": "node drizzle/scripts/generate.mjs",
"fetch-models": "node scripts/fetch-models.mjs",
"docs:init": "pnpm --dir docs-site install",
"docs:dev": "pnpm --dir docs-site start",
"docs:build": "pnpm --dir docs-site build",
@ -48,6 +49,7 @@
"jszip": "^3.10.1",
"lru-cache": "^11.3.6",
"next": "^15.5.18",
"onnxruntime-node": "^1.26.0",
"openai": "^6.37.0",
"pdfjs-dist": "4.8.69",
"pg": "^8.20.0",

View file

@ -88,6 +88,9 @@ importers:
next:
specifier: ^15.5.18
version: 15.5.18(@playwright/test@1.60.0)(react-dom@19.2.6(react@19.2.6))(react@19.2.6)
onnxruntime-node:
specifier: ^1.26.0
version: 1.26.0
openai:
specifier: ^6.37.0
version: 6.37.0(zod@4.4.3)
@ -1854,6 +1857,10 @@ packages:
engines: {node: '>=0.4.0'}
hasBin: true
adm-zip@0.5.17:
resolution: {integrity: sha512-+Ut8d9LLqwEvHHJl1+PIHqoyDxFgVN847JTVM3Izi3xHDWPE4UtzzXysMZQs64DMcrJfBeS/uoEP4AD3HQHnQQ==}
engines: {node: '>=12.0'}
agent-base@6.0.2:
resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==}
engines: {node: '>= 6.0.0'}
@ -2843,6 +2850,10 @@ packages:
deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me
hasBin: true
global-agent@4.1.3:
resolution: {integrity: sha512-KUJEViiuFT3I97t+GYMikLPJS2Lfo/S2F+DQuBWzuzaMPnvt5yyZePzArx36fBzpGTxZjIpDbXLeySLgh+k76g==}
engines: {node: '>=10.0'}
globals@14.0.0:
resolution: {integrity: sha512-oahGvuMGQlPw/ivIYBjVSrWAfWLBeku5tpPE2fOPLi+WHffIWbuh2tCjhyQhTBPMf5E9jDEH4FOmTYgYwbKwtQ==}
engines: {node: '>=18'}
@ -3206,6 +3217,10 @@ packages:
marks-pane@1.0.9:
resolution: {integrity: sha512-Ahs4oeG90tbdPWwAJkAAoHg2lRR8lAs9mZXETNPO9hYg3AkjUJBKi1NQ4aaIQZVGrig7c/3NUV1jANl8rFTeMg==}
matcher@4.0.0:
resolution: {integrity: sha512-S6x5wmcDmsDRRU/c2dkccDwQPXoFczc5+HpQ2lON8pnvHlnvHAHj5WlLVvw6n6vNyHuVugYrFohYxbS+pvFpKQ==}
engines: {node: '>=10'}
math-intrinsics@1.1.0:
resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==}
engines: {node: '>= 0.4'}
@ -3488,6 +3503,13 @@ packages:
once@1.4.0:
resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==}
onnxruntime-common@1.26.0:
resolution: {integrity: sha512-qVyMR4lcWgbkc4getFV+GQijsTnbg/siteoqcDwa3sI/LxbrMSNw4ePyvCq/ymdQaRomCA7YuWmhzsswxvymdw==}
onnxruntime-node@1.26.0:
resolution: {integrity: sha512-OHl6PiOEOqxaLHL0N9eFrbzS7IGmu3BtJNH3RTEnRAheCIkfc3gjcjl4sGcjp9C22ZC9YTquDOxSdT/stBQ6BQ==}
os: [win32, darwin, linux]
openai@6.37.0:
resolution: {integrity: sha512-0H5dEGFmmLv6KSd0W1w2nyL8WsLkX6yoLeQpU+dZAOuGcany5qkYQMmj35ZrKgb6yiyYqpUzFOpR8mZQkgqeEQ==}
hasBin: true
@ -3924,6 +3946,10 @@ packages:
engines: {node: '>=10'}
hasBin: true
serialize-error@8.1.0:
resolution: {integrity: sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==}
engines: {node: '>=10'}
set-cookie-parser@3.1.0:
resolution: {integrity: sha512-kjnC1DXBHcxaOaOXBHBeRtltsDG2nUiUni+jP92M9gYdW12rsmx92UsfpH7o5tDRs7I1ZZPSQJQGv3UaRfCiuw==}
@ -4181,6 +4207,10 @@ packages:
resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==}
engines: {node: '>= 0.8.0'}
type-fest@0.20.2:
resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==}
engines: {node: '>=10'}
type@2.7.3:
resolution: {integrity: sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==}
@ -5957,6 +5987,8 @@ snapshots:
acorn@8.16.0: {}
adm-zip@0.5.17: {}
agent-base@6.0.2:
dependencies:
debug: 4.4.3
@ -7097,6 +7129,13 @@ snapshots:
package-json-from-dist: 1.0.1
path-scurry: 1.11.1
global-agent@4.1.3:
dependencies:
globalthis: 1.0.4
matcher: 4.0.0
semver: 7.8.0
serialize-error: 8.1.0
globals@14.0.0: {}
globalthis@1.0.4:
@ -7457,6 +7496,10 @@ snapshots:
marks-pane@1.0.9: {}
matcher@4.0.0:
dependencies:
escape-string-regexp: 4.0.0
math-intrinsics@1.1.0: {}
mdast-util-find-and-replace@3.0.2:
@ -7946,6 +7989,14 @@ snapshots:
dependencies:
wrappy: 1.0.2
onnxruntime-common@1.26.0: {}
onnxruntime-node@1.26.0:
dependencies:
adm-zip: 0.5.17
global-agent: 4.1.3
onnxruntime-common: 1.26.0
openai@6.37.0(zod@4.4.3):
optionalDependencies:
zod: 4.4.3
@ -8439,6 +8490,10 @@ snapshots:
semver@7.8.0: {}
serialize-error@8.1.0:
dependencies:
type-fest: 0.20.2
set-cookie-parser@3.1.0: {}
set-function-length@1.2.2:
@ -8814,6 +8869,8 @@ snapshots:
dependencies:
prelude-ls: 1.2.1
type-fest@0.20.2: {}
type@2.7.3: {}
typed-array-buffer@1.0.3:

View file

@ -1,8 +1,6 @@
packages:
- .
strictDepBuilds: false
allowBuilds:
'@napi-rs/canvas': true
better-sqlite3: true
@ -11,10 +9,13 @@ allowBuilds:
es5-ext: false
esbuild: false
ffmpeg-static: true
onnxruntime-node: true
sharp: false
unrs-resolver: false
overrides:
lodash: ^4.17.23
'@xmldom/xmldom': ^0.9.8
'@types/localforage': npm:empty-module@0.0.2
'@xmldom/xmldom': ^0.9.8
lodash: ^4.17.23
strictDepBuilds: false

31
scripts/fetch-models.mjs Normal file
View file

@ -0,0 +1,31 @@
#!/usr/bin/env node
import { mkdir, writeFile } from 'node:fs/promises';
import path from 'node:path';
const modelDir = path.join(process.cwd(), 'docstore', 'model');
const modelPath = path.join(modelDir, 'docling-layout-heron.onnx');
const configPath = path.join(modelDir, 'docling-layout-heron.config.json');
const licensePath = path.join(modelDir, 'docling-layout-heron.LICENSE.txt');
const staticLicensePath = path.join(process.cwd(), 'src', 'lib', 'server', 'pdf-layout', 'model', 'LICENSE.txt');
const modelUrl = process.env.OPENREADER_DOCLING_MODEL_URL || 'https://huggingface.co/docling-project/docling-layout-heron-onnx/resolve/main/model.onnx';
const configUrl = process.env.OPENREADER_DOCLING_CONFIG_URL || 'https://huggingface.co/docling-project/docling-layout-heron-onnx/resolve/main/config.json';
await mkdir(modelDir, { recursive: true });
const modelRes = await fetch(modelUrl);
if (!modelRes.ok) {
throw new Error(`Failed to fetch model: ${modelRes.status} ${modelRes.statusText}`);
}
await writeFile(modelPath, new Uint8Array(await modelRes.arrayBuffer()));
const configRes = await fetch(configUrl);
if (!configRes.ok) {
throw new Error(`Failed to fetch config: ${configRes.status} ${configRes.statusText}`);
}
await writeFile(configPath, new Uint8Array(await configRes.arrayBuffer()));
const staticLicense = await import('node:fs/promises').then((m) => m.readFile(staticLicensePath));
await writeFile(licensePath, staticLicense);
console.log(`Saved model to ${modelPath}`);

View file

@ -32,6 +32,7 @@ const PDFViewer = dynamic(
export default function PDFViewerPage() {
const canExportAudiobook = useFeatureFlag('enableAudiobookExport');
const computeAvailable = useFeatureFlag('computeAvailable');
const { id } = useParams();
const router = useRouter();
const pdfState = usePdfDocument();
@ -41,6 +42,12 @@ export default function PDFViewerPage() {
clearCurrDoc,
currDocPage,
currDocPages,
parseStatus,
documentSettings,
updateDocumentSettings,
parsedOverlayEnabled,
setParsedOverlayEnabled,
forceReparseParsedPdf,
createFullAudioBook: createPDFAudioBook,
regenerateChapter: regeneratePDFChapter,
} = pdfState;
@ -232,6 +239,40 @@ export default function PDFViewerPage() {
<DocumentSettings
isOpen={activeSidebar === 'settings'}
setIsOpen={(isOpen) => setActiveSidebar((prev) => isOpen ? 'settings' : (prev === 'settings' ? null : prev))}
pdf={{
computeAvailable,
parseStatus,
parsedOverlayEnabled,
skipBlockKinds: documentSettings.pdf?.skipBlockKinds ?? [],
chaptersFromSections: documentSettings.pdf?.chaptersFromSections ?? true,
onToggleOverlay: (enabled) => setParsedOverlayEnabled(enabled),
onToggleSkipKind: (kind, enabled) => {
const current = new Set(documentSettings.pdf?.skipBlockKinds ?? []);
if (enabled) current.add(kind);
else current.delete(kind);
void updateDocumentSettings({
...documentSettings,
schemaVersion: 1,
pdf: {
...(documentSettings.pdf ?? { chaptersFromSections: true }),
skipBlockKinds: Array.from(current),
chaptersFromSections: documentSettings.pdf?.chaptersFromSections ?? true,
},
});
},
onToggleChaptersFromSections: (enabled) => {
void updateDocumentSettings({
...documentSettings,
schemaVersion: 1,
pdf: {
...(documentSettings.pdf ?? { skipBlockKinds: [] }),
skipBlockKinds: documentSettings.pdf?.skipBlockKinds ?? [],
chaptersFromSections: enabled,
},
});
},
onForceReparse: () => forceReparseParsedPdf(),
}}
/>
<SegmentsSidebar
isOpen={activeSidebar === 'segments'}

View file

@ -18,7 +18,13 @@ import {
import type { PDFDocumentProxy } from 'pdfjs-dist';
import { getDocumentMetadata } from '@/lib/client/api/documents';
import {
forceReparsePdfDocument,
getDocumentMetadata,
getDocumentSettings,
getParsedPdfDocument,
putDocumentSettings,
} from '@/lib/client/api/documents';
import { createPdfAudiobookSourceAdapter } from '@/lib/client/audiobooks/adapters/pdf';
import { regenerateAudiobookChapter, runAudiobookGeneration } from '@/lib/client/audiobooks/pipeline';
import { ensureCachedDocument } from '@/lib/client/cache/documents';
@ -31,13 +37,20 @@ import {
clearWordHighlights,
highlightWordIndex,
} from '@/lib/client/pdf';
import type { CanonicalTtsSourceUnit } from '@/lib/shared/tts-segment-plan';
import {
DEFAULT_DOCUMENT_SETTINGS,
type DocumentSettings,
} from '@/types/document-settings';
import { mergeDocumentSettings } from '@/lib/shared/document-settings';
import type { ParsedPdfDocument, ParsedPdfPage, PdfParseStatus } from '@/types/parsed-pdf';
import type {
TTSSentenceAlignment,
TTSAudiobookFormat,
TTSAudiobookChapter,
} from '@/types/tts';
import type { AudiobookGenerationSettings } from '@/types/client';
import type { AudiobookGenerationSettings, TTSSegmentLocator } from '@/types/client';
import { clampSegmentPreloadDepth } from '@/types/config';
/**
@ -52,6 +65,13 @@ export interface PdfDocumentState {
currDocPage: number;
currDocText: string | undefined;
pdfDocument: PDFDocumentProxy | undefined;
parsedDocument: ParsedPdfDocument | null;
parseStatus: PdfParseStatus | null;
documentSettings: DocumentSettings;
updateDocumentSettings: (settings: DocumentSettings) => Promise<void>;
parsedOverlayEnabled: boolean;
setParsedOverlayEnabled: (enabled: boolean) => void;
forceReparseParsedPdf: () => Promise<void>;
setCurrentDocument: (id: string) => Promise<void>;
clearCurrDoc: () => void;
@ -119,18 +139,24 @@ export function usePdfDocument(): PdfDocumentState {
const [currDocName, setCurrDocName] = useState<string>();
const [currDocText, setCurrDocText] = useState<string>();
const [pdfDocument, setPdfDocument] = useState<PDFDocumentProxy>();
const [parsedDocument, setParsedDocument] = useState<ParsedPdfDocument | null>(null);
const [parseStatus, setParseStatus] = useState<PdfParseStatus | null>(null);
const [documentSettings, setDocumentSettings] = useState<DocumentSettings>(DEFAULT_DOCUMENT_SETTINGS);
const [parsedOverlayEnabled, setParsedOverlayEnabled] = useState(false);
const [isAudioCombining] = useState(false);
const audiobookAdapter = useMemo(() => createPdfAudiobookSourceAdapter({
pdfDocument,
parsed: parsedDocument ?? undefined,
settings: documentSettings,
margins: {
header: headerMargin,
footer: footerMargin,
left: leftMargin,
right: rightMargin,
header: documentSettings.pdf?.margins?.header ?? headerMargin,
footer: documentSettings.pdf?.margins?.footer ?? footerMargin,
left: documentSettings.pdf?.margins?.left ?? leftMargin,
right: documentSettings.pdf?.margins?.right ?? rightMargin,
},
smartSentenceSplitting,
maxBlockLength: ttsSegmentMaxBlockLength,
}), [pdfDocument, headerMargin, footerMargin, leftMargin, rightMargin, smartSentenceSplitting, ttsSegmentMaxBlockLength]);
}), [pdfDocument, parsedDocument, documentSettings, headerMargin, footerMargin, leftMargin, rightMargin, smartSentenceSplitting, ttsSegmentMaxBlockLength]);
const pageTextCacheRef = useRef<Map<number, string>>(new Map());
const [currDocPage, setCurrDocPage] = useState<number>(currDocPageNumber);
@ -144,6 +170,66 @@ export function usePdfDocument(): PdfDocumentState {
// Guards for setCurrentDocument to prevent stale loads from overwriting newer selections.
const docLoadSeqRef = useRef(0);
const docLoadAbortRef = useRef<AbortController | null>(null);
const parsePollAbortRef = useRef<AbortController | null>(null);
const fetchParsedDocument = useCallback(async (
documentId: string,
initialStatus: PdfParseStatus | null,
signal: AbortSignal,
): Promise<void> => {
// Legacy PDFs may have null parseStatus; treat as pending so opening the
// document backfills parse output via the parsed endpoint polling path.
const effectiveInitialStatus: PdfParseStatus = initialStatus ?? 'pending';
setParseStatus(effectiveInitialStatus);
if (effectiveInitialStatus === 'unsupported') {
setParsedDocument(null);
return;
}
const maxAttempts = 25;
const delayMs = 1200;
const retryFailed = effectiveInitialStatus === 'failed';
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
if (signal.aborted) return;
const result = await getParsedPdfDocument(documentId, {
signal,
retryFailed: retryFailed && attempt === 0,
});
if (result.status === 'ready') {
setParsedDocument(result.parsed);
setParseStatus('ready');
return;
}
setParseStatus(result.status);
if (result.status === 'failed' || result.status === 'unsupported') {
setParsedDocument(null);
return;
}
await new Promise((resolve) => setTimeout(resolve, delayMs));
}
}, []);
const fetchDocumentSettings = useCallback(async (documentId: string, signal: AbortSignal): Promise<void> => {
try {
const response = await getDocumentSettings(documentId, { signal });
setDocumentSettings(mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, response.settings));
} catch (error) {
if (error instanceof DOMException && error.name === 'AbortError') return;
console.warn('Failed to load document settings, using defaults:', error);
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
}
}, []);
const startParsedPolling = useCallback((documentId: string, initialStatus: PdfParseStatus | null) => {
parsePollAbortRef.current?.abort();
const controller = new AbortController();
parsePollAbortRef.current = controller;
void fetchParsedDocument(documentId, initialStatus, controller.signal).finally(() => {
if (parsePollAbortRef.current === controller) {
parsePollAbortRef.current = null;
}
});
}, [fetchParsedDocument]);
useEffect(() => {
pdfDocumentRef.current = pdfDocument;
@ -189,10 +275,37 @@ export function usePdfDocument(): PdfDocumentState {
: null;
const margins = {
header: headerMargin,
footer: footerMargin,
left: leftMargin,
right: rightMargin
header: documentSettings.pdf?.margins?.header ?? headerMargin,
footer: documentSettings.pdf?.margins?.footer ?? footerMargin,
left: documentSettings.pdf?.margins?.left ?? leftMargin,
right: documentSettings.pdf?.margins?.right ?? rightMargin
};
const pageFromParsed = (pageNum: number): ParsedPdfPage | undefined =>
parsedDocument?.pages.find((page) => page.pageNumber === pageNum);
const hasUsableParsedBlocks = (page: ParsedPdfPage | undefined): page is ParsedPdfPage => {
if (!page) return false;
const skipKinds = new Set(documentSettings.pdf?.skipBlockKinds ?? []);
return page.blocks.some((block) => !skipKinds.has(block.kind) && block.text.trim().length > 0);
};
const sourceUnitsFromParsedPage = (pageNum: number): CanonicalTtsSourceUnit[] => {
const page = pageFromParsed(pageNum);
if (!page) return [];
const skipKinds = new Set(documentSettings.pdf?.skipBlockKinds ?? []);
return page.blocks
.filter((block) => !skipKinds.has(block.kind))
.map((block) => ({
sourceKey: `pdf:${pageNum}:${block.id}`,
text: block.text,
locator: {
readerType: 'pdf',
page: block.fragments[0]?.page ?? pageNum,
blockId: block.id,
} as TTSSegmentLocator,
}))
.filter((unit) => unit.text.trim().length > 0);
};
const getPageText = async (pageNumber: number, shouldCache = false): Promise<string> => {
@ -209,7 +322,15 @@ export function usePdfDocument(): PdfDocumentState {
return cached;
}
const extracted = await extractTextFromPDF(currentPdf, pageNumber, margins);
const parsedPage = pageFromParsed(pageNumber);
const useParsedPage = hasUsableParsedBlocks(parsedPage) ? parsedPage : undefined;
const extracted = await extractTextFromPDF(
currentPdf,
pageNumber,
margins,
useParsedPage,
useParsedPage ? documentSettings.pdf?.skipBlockKinds : undefined,
);
if (generation !== pdfDocGenerationRef.current || pdfDocumentRef.current !== currentPdf) {
throw new DOMException('Stale PDF extraction', 'AbortError');
@ -281,12 +402,17 @@ export function usePdfDocument(): PdfDocumentState {
if (text !== currDocText || text === '') {
setCurrDocText(text);
const sourceUnits = sourceUnitsFromParsedPage(currDocPageNumber);
const useBlockStructuredPlanning = sourceUnits.length > 0;
setTTSText(text, {
location: currDocPageNumber,
previousText: prevText,
nextLocation: nextPageNumber,
nextText: nextText,
upcomingLocations: additionalUpcoming,
...(!useBlockStructuredPlanning ? {
previousText: prevText,
nextLocation: nextPageNumber,
nextText: nextText,
upcomingLocations: additionalUpcoming,
} : {}),
...(sourceUnits.length > 0 ? { sourceUnits } : {}),
});
}
} catch (error) {
@ -305,6 +431,8 @@ export function usePdfDocument(): PdfDocumentState {
leftMargin,
rightMargin,
segmentPreloadDepthPages,
parsedDocument,
documentSettings,
]);
/**
@ -341,6 +469,8 @@ export function usePdfDocument(): PdfDocumentState {
clearTimeout(emptyRetryRef.current.timer);
}
emptyRetryRef.current = null;
parsePollAbortRef.current?.abort();
parsePollAbortRef.current = null;
pageTextCacheRef.current.clear();
setPdfDocument(undefined);
setCurrDocPages(undefined);
@ -348,6 +478,9 @@ export function usePdfDocument(): PdfDocumentState {
setCurrDocId(id);
setCurrDocName(undefined);
setCurrDocData(undefined);
setParsedDocument(null);
setParseStatus(null);
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
const meta = await getDocumentMetadata(id, { signal: controller.signal });
if (seq !== docLoadSeqRef.current) return; // stale
@ -355,6 +488,10 @@ export function usePdfDocument(): PdfDocumentState {
console.error('Document not found on server');
return;
}
if (meta.type === 'pdf') {
startParsedPolling(id, (meta.parseStatus ?? null) as PdfParseStatus | null);
void fetchDocumentSettings(id, controller.signal);
}
const doc = await ensureCachedDocument(meta, { signal: controller.signal });
if (seq !== docLoadSeqRef.current) return; // stale
@ -376,7 +513,39 @@ export function usePdfDocument(): PdfDocumentState {
docLoadAbortRef.current = null;
}
}
}, [setCurrDocId, setCurrDocName, setCurrDocData, setCurrDocPages, setCurrDocText, setPdfDocument]);
}, [
setCurrDocId,
setCurrDocName,
setCurrDocData,
setCurrDocPages,
setCurrDocText,
setPdfDocument,
fetchDocumentSettings,
startParsedPolling,
]);
const updateDocumentSettings = useCallback(async (settings: DocumentSettings): Promise<void> => {
if (!currDocId) return;
setDocumentSettings(settings);
try {
const updated = await putDocumentSettings(currDocId, settings);
setDocumentSettings(mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, updated.settings));
} catch (error) {
console.warn('Failed to persist document settings:', error);
}
}, [currDocId]);
const forceReparseParsedPdf = useCallback(async (): Promise<void> => {
if (!currDocId) return;
try {
await forceReparsePdfDocument(currDocId);
setParsedDocument(null);
setParseStatus('pending');
startParsedPolling(currDocId, 'pending');
} catch (error) {
console.error('Failed to force PDF reparse:', error);
}
}, [currDocId, startParsedPolling]);
/**
* Clears the current document state
@ -390,6 +559,8 @@ export function usePdfDocument(): PdfDocumentState {
docLoadSeqRef.current += 1;
docLoadAbortRef.current?.abort();
docLoadAbortRef.current = null;
parsePollAbortRef.current?.abort();
parsePollAbortRef.current = null;
if (emptyRetryRef.current?.timer) {
clearTimeout(emptyRetryRef.current.timer);
}
@ -400,6 +571,9 @@ export function usePdfDocument(): PdfDocumentState {
setCurrDocText(undefined);
setCurrDocPages(undefined);
setPdfDocument(undefined);
setParsedDocument(null);
setParseStatus(null);
setDocumentSettings(DEFAULT_DOCUMENT_SETTINGS);
pageTextCacheRef.current.clear();
stop();
}, [setCurrDocId, setCurrDocName, setCurrDocData, setCurrDocPages, setCurrDocText, setPdfDocument, stop]);
@ -499,6 +673,13 @@ export function usePdfDocument(): PdfDocumentState {
currDocPages,
currDocPage,
currDocText,
parsedDocument,
parseStatus,
documentSettings,
updateDocumentSettings,
parsedOverlayEnabled,
setParsedOverlayEnabled,
forceReparseParsedPdf,
clearCurrDoc,
highlightPattern,
clearHighlights,
@ -518,6 +699,13 @@ export function usePdfDocument(): PdfDocumentState {
currDocPages,
currDocPage,
currDocText,
parsedDocument,
parseStatus,
documentSettings,
updateDocumentSettings,
parsedOverlayEnabled,
setParsedOverlayEnabled,
forceReparseParsedPdf,
clearCurrDoc,
pdfDocument,
createFullAudioBook,

View file

@ -0,0 +1,185 @@
import { NextRequest, NextResponse } from 'next/server';
import { and, eq, inArray } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { requireAuthContext } from '@/lib/server/auth/auth';
import { getParsedDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace';
import { isS3Configured } from '@/lib/server/storage/s3';
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
export const dynamic = 'force-dynamic';
function s3NotConfiguredResponse(): NextResponse {
return NextResponse.json(
{ error: 'Documents storage is not configured. Set S3_* environment variables.' },
{ status: 503 },
);
}
function hasAnyParsedBlocks(doc: ParsedPdfDocument | null): boolean {
if (!doc || !Array.isArray(doc.pages)) return false;
return doc.pages.some((page) => Array.isArray(page.blocks) && page.blocks.length > 0);
}
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
try {
if (!isS3Configured()) return s3NotConfiguredResponse();
const authCtxOrRes = await requireAuthContext(req);
if (authCtxOrRes instanceof Response) return authCtxOrRes;
const params = await ctx.params;
const id = (params.id || '').trim().toLowerCase();
const retryFailed = req.nextUrl.searchParams.get('retry') === '1';
if (!isValidDocumentId(id)) {
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
}
const testNamespace = getOpenReaderTestNamespace(req.headers);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const storageUserId = authCtxOrRes.userId ?? unclaimedUserId;
const allowedUserIds = authCtxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
const rows = (await db
.select({
id: documents.id,
userId: documents.userId,
parseStatus: documents.parseStatus,
})
.from(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
id: string;
userId: string;
parseStatus: 'pending' | 'running' | 'ready' | 'failed' | 'unsupported' | null;
}>;
const row = rows.find((candidate) => candidate.userId === storageUserId) ?? rows[0];
if (!row) {
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
if (row.parseStatus === 'failed' && retryFailed) {
await db
.update(documents)
.set({ parseStatus: 'pending' })
.where(and(eq(documents.id, id), eq(documents.userId, row.userId)));
enqueueParsePdfJob({
documentId: id,
userId: row.userId,
namespace: testNamespace,
});
return NextResponse.json({ parseStatus: 'pending' }, { status: 202 });
}
if (row.parseStatus !== 'ready') {
if (row.parseStatus === 'pending' || row.parseStatus === 'running' || row.parseStatus === null) {
enqueueParsePdfJob({
documentId: id,
userId: row.userId,
namespace: testNamespace,
});
}
return NextResponse.json({ parseStatus: row.parseStatus ?? 'pending' }, { status: 202 });
}
try {
const json = await getParsedDocumentBlob(id, testNamespace);
let parsedDoc: ParsedPdfDocument | null = null;
try {
parsedDoc = JSON.parse(Buffer.from(json).toString('utf8')) as ParsedPdfDocument;
} catch {
parsedDoc = null;
}
if (!hasAnyParsedBlocks(parsedDoc)) {
await db
.update(documents)
.set({ parseStatus: 'pending' })
.where(and(eq(documents.id, id), eq(documents.userId, row.userId)));
enqueueParsePdfJob({
documentId: id,
userId: row.userId,
namespace: testNamespace,
});
return NextResponse.json({ parseStatus: 'pending' }, { status: 202 });
}
return new NextResponse(new Uint8Array(json), {
status: 200,
headers: {
'Content-Type': 'application/json',
'Cache-Control': 'no-store',
},
});
} catch (error) {
if (isMissingBlobError(error)) {
return NextResponse.json({ parseStatus: 'failed', error: 'Parsed document not found' }, { status: 404 });
}
throw error;
}
} catch (error) {
console.error('Error reading parsed PDF:', error);
return NextResponse.json({ error: 'Failed to read parsed PDF' }, { status: 500 });
}
}
export async function POST(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
try {
if (!isS3Configured()) return s3NotConfiguredResponse();
const authCtxOrRes = await requireAuthContext(req);
if (authCtxOrRes instanceof Response) return authCtxOrRes;
const params = await ctx.params;
const id = (params.id || '').trim().toLowerCase();
if (!isValidDocumentId(id)) {
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
}
const testNamespace = getOpenReaderTestNamespace(req.headers);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const storageUserId = authCtxOrRes.userId ?? unclaimedUserId;
const allowedUserIds = authCtxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
const rows = (await db
.select({
id: documents.id,
userId: documents.userId,
parseStatus: documents.parseStatus,
})
.from(documents)
.where(and(eq(documents.id, id), inArray(documents.userId, allowedUserIds)))) as Array<{
id: string;
userId: string;
parseStatus: 'pending' | 'running' | 'ready' | 'failed' | 'unsupported' | null;
}>;
const row = rows.find((candidate) => candidate.userId === storageUserId) ?? rows[0];
if (!row) {
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
if (row.parseStatus !== 'running') {
await db
.update(documents)
.set({ parseStatus: 'pending' })
.where(and(eq(documents.id, id), eq(documents.userId, row.userId)));
}
enqueueParsePdfJob({
documentId: id,
userId: row.userId,
namespace: testNamespace,
});
return NextResponse.json(
{ parseStatus: row.parseStatus === 'running' ? 'running' : 'pending' },
{ status: 202 },
);
} catch (error) {
console.error('Error forcing parsed PDF refresh:', error);
return NextResponse.json({ error: 'Failed to force parsed PDF refresh' }, { status: 500 });
}
}

View file

@ -0,0 +1,167 @@
import { NextRequest, NextResponse } from 'next/server';
import { and, eq, inArray } from 'drizzle-orm';
import { db } from '@/db';
import { documentSettings, documents } from '@/db/schema';
import { requireAuthContext } from '@/lib/server/auth/auth';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace';
import { mergeDocumentSettings } from '@/lib/shared/document-settings';
import { DEFAULT_DOCUMENT_SETTINGS, type DocumentSettings } from '@/types/document-settings';
import { coerceTimestampMs, nowTimestampMs } from '@/lib/shared/timestamps';
export const dynamic = 'force-dynamic';
function serializeForDb(payload: Record<string, unknown>): Record<string, unknown> | string {
if (process.env.POSTGRES_URL) return payload;
return JSON.stringify(payload);
}
function normalizeClientUpdatedAtMs(value: unknown): number {
const normalized = coerceTimestampMs(value, nowTimestampMs());
if (normalized <= 0) return nowTimestampMs();
return normalized;
}
function parseStored(value: unknown): DocumentSettings {
if (typeof value === 'string') {
try {
return mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, JSON.parse(value));
} catch {
return mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, null);
}
}
return mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, value);
}
async function resolveDocumentAccess(req: NextRequest, documentId: string): Promise<
| { ownerUserId: string; allowedUserIds: string[] }
| Response
> {
const authCtxOrRes = await requireAuthContext(req);
if (authCtxOrRes instanceof Response) return authCtxOrRes;
const testNamespace = getOpenReaderTestNamespace(req.headers);
const unclaimedUserId = getUnclaimedUserIdForNamespace(testNamespace);
const storageUserId = authCtxOrRes.userId ?? unclaimedUserId;
const allowedUserIds = authCtxOrRes.authEnabled ? [storageUserId, unclaimedUserId] : [unclaimedUserId];
const rows = await db
.select({ userId: documents.userId })
.from(documents)
.where(and(eq(documents.id, documentId), inArray(documents.userId, allowedUserIds)))
.limit(1);
if (!rows[0]) {
return NextResponse.json({ error: 'Not found' }, { status: 404 });
}
return {
ownerUserId: rows[0].userId,
allowedUserIds,
};
}
export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
try {
const { id } = await ctx.params;
const documentId = (id || '').trim().toLowerCase();
if (!documentId) {
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
}
const scope = await resolveDocumentAccess(req, documentId);
if (scope instanceof Response) return scope;
const rows = await db
.select({
dataJson: documentSettings.dataJson,
clientUpdatedAtMs: documentSettings.clientUpdatedAtMs,
})
.from(documentSettings)
.where(and(
eq(documentSettings.documentId, documentId),
eq(documentSettings.userId, scope.ownerUserId),
))
.limit(1);
const row = rows[0];
const settings = parseStored(row?.dataJson);
return NextResponse.json({
settings,
clientUpdatedAtMs: Number(row?.clientUpdatedAtMs ?? 0),
hasStoredSettings: Boolean(row),
});
} catch (error) {
console.error('Error loading document settings:', error);
return NextResponse.json({ error: 'Failed to load document settings' }, { status: 500 });
}
}
export async function PUT(req: NextRequest, ctx: { params: Promise<{ id: string }> }) {
try {
const { id } = await ctx.params;
const documentId = (id || '').trim().toLowerCase();
if (!documentId) {
return NextResponse.json({ error: 'Invalid document id' }, { status: 400 });
}
const scope = await resolveDocumentAccess(req, documentId);
if (scope instanceof Response) return scope;
const body = (await req.json().catch(() => null)) as { settings?: unknown; clientUpdatedAtMs?: unknown } | null;
const incoming = mergeDocumentSettings(DEFAULT_DOCUMENT_SETTINGS, body?.settings ?? null);
const clientUpdatedAtMs = normalizeClientUpdatedAtMs(body?.clientUpdatedAtMs);
const existingRows = await db
.select({
dataJson: documentSettings.dataJson,
clientUpdatedAtMs: documentSettings.clientUpdatedAtMs,
})
.from(documentSettings)
.where(and(
eq(documentSettings.documentId, documentId),
eq(documentSettings.userId, scope.ownerUserId),
))
.limit(1);
const existing = existingRows[0];
const existingUpdatedAt = Number(existing?.clientUpdatedAtMs ?? 0);
if (existing && clientUpdatedAtMs < existingUpdatedAt) {
return NextResponse.json({
settings: parseStored(existing.dataJson),
clientUpdatedAtMs: existingUpdatedAt,
applied: false,
});
}
const updatedAt = nowTimestampMs();
const payload = serializeForDb(incoming as unknown as Record<string, unknown>);
await db
.insert(documentSettings)
.values({
documentId,
userId: scope.ownerUserId,
dataJson: payload as never,
clientUpdatedAtMs,
updatedAt,
})
.onConflictDoUpdate({
target: [documentSettings.documentId, documentSettings.userId],
set: {
dataJson: payload as never,
clientUpdatedAtMs,
updatedAt,
},
});
return NextResponse.json({
settings: incoming,
clientUpdatedAtMs,
applied: true,
});
} catch (error) {
console.error('Error updating document settings:', error);
return NextResponse.json({ error: 'Failed to update document settings' }, { status: 500 });
}
}

View file

@ -9,6 +9,7 @@ import {
deleteDocumentPreviewRows,
enqueueDocumentPreview,
} from '@/lib/server/documents/previews';
import { enqueueParsePdfJob } from '@/lib/server/jobs/parsePdfJob';
import { deleteDocumentBlob, headDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore';
import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace';
import { isS3Configured } from '@/lib/server/storage/s3';
@ -129,6 +130,8 @@ export async function POST(req: NextRequest) {
size: headSize,
lastModified: doc.lastModified,
filePath: doc.id,
parseStatus: doc.type === 'pdf' ? 'pending' : null,
parsedJsonKey: null,
})
.onConflictDoUpdate({
target: [documents.id, documents.userId],
@ -138,6 +141,8 @@ export async function POST(req: NextRequest) {
size: headSize,
lastModified: doc.lastModified,
filePath: doc.id,
parseStatus: doc.type === 'pdf' ? 'pending' : null,
parsedJsonKey: null,
},
});
@ -160,6 +165,14 @@ export async function POST(req: NextRequest) {
).catch((error) => {
console.error(`Failed to enqueue preview for document ${doc.id}:`, error);
});
if (doc.type === 'pdf') {
enqueueParsePdfJob({
documentId: doc.id,
userId: storageUserId,
namespace: testNamespace,
});
}
}
return NextResponse.json({ success: true, stored });
@ -206,9 +219,25 @@ export async function GET(req: NextRequest) {
size: number;
lastModified: number;
filePath: string;
parseStatus: 'pending' | 'running' | 'ready' | 'failed' | 'unsupported' | null;
parsedJsonKey: string | null;
}>;
const results: BaseDocument[] = rows.map((doc) => {
const preferredById = new Map<string, (typeof rows)[number]>();
for (const row of rows) {
const existing = preferredById.get(row.id);
if (!existing) {
preferredById.set(row.id, row);
continue;
}
const isRowPrimary = row.userId === storageUserId;
const isExistingPrimary = existing.userId === storageUserId;
if (isRowPrimary && !isExistingPrimary) {
preferredById.set(row.id, row);
}
}
const results: BaseDocument[] = Array.from(preferredById.values()).map((doc) => {
const type = normalizeDocumentType(doc.type, doc.name);
return {
id: doc.id,
@ -216,6 +245,8 @@ export async function GET(req: NextRequest) {
size: Number(doc.size),
lastModified: Number(doc.lastModified),
type,
parseStatus: doc.parseStatus,
parsedJsonKey: doc.parsedJsonKey,
scope: doc.userId === unclaimedUserId ? 'unclaimed' : 'user',
};
});

View file

@ -1,9 +1,6 @@
import { NextRequest, NextResponse } from 'next/server';
import { and, eq } from 'drizzle-orm';
import { db } from '@/db';
import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema';
import { deleteTtsSegmentAudioObjects } from '@/lib/server/tts/segments-blobstore';
import { resolveSegmentDocumentScope } from '@/lib/server/tts/segments-auth';
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
export const runtime = 'nodejs';
export const dynamic = 'force-dynamic';
@ -25,59 +22,16 @@ export async function POST(request: NextRequest) {
const scope = await resolveSegmentDocumentScope(request, parsed.documentId);
if (scope instanceof Response) return scope;
const rows = (await db
.select({
segmentId: ttsSegmentVariants.segmentId,
audioKey: ttsSegmentVariants.audioKey,
})
.from(ttsSegmentVariants)
.innerJoin(ttsSegmentEntries, and(
eq(ttsSegmentEntries.segmentEntryId, ttsSegmentVariants.segmentEntryId),
eq(ttsSegmentEntries.userId, ttsSegmentVariants.userId),
))
.where(and(
eq(ttsSegmentEntries.userId, scope.storageUserId),
eq(ttsSegmentEntries.documentId, parsed.documentId),
eq(ttsSegmentEntries.documentVersion, scope.documentVersion),
))) as Array<{ segmentId: string; audioKey: string | null }>;
await db
.delete(ttsSegmentEntries)
.where(and(
eq(ttsSegmentEntries.userId, scope.storageUserId),
eq(ttsSegmentEntries.documentId, parsed.documentId),
eq(ttsSegmentEntries.documentVersion, scope.documentVersion),
));
const audioKeys = rows
.map((row) => row.audioKey)
.filter((key): key is string => Boolean(key));
const uniqueAudioKeys = Array.from(new Set(audioKeys));
let deletedAudioObjects = 0;
let warning: string | undefined;
if (uniqueAudioKeys.length > 0) {
try {
deletedAudioObjects = await deleteTtsSegmentAudioObjects(uniqueAudioKeys);
if (deletedAudioObjects < uniqueAudioKeys.length) {
warning = `Deleted ${deletedAudioObjects} of ${uniqueAudioKeys.length} audio objects.`;
}
} catch (error) {
warning = error instanceof Error ? error.message : 'Failed deleting some audio objects';
console.warn('Failed clearing some TTS segment audio objects:', {
documentId: parsed.documentId,
userId: scope.storageUserId,
error: warning,
});
}
}
const cleared = await clearTtsSegmentCache({
userId: scope.storageUserId,
documentId: parsed.documentId,
documentVersion: scope.documentVersion,
readerType: scope.readerType,
});
return NextResponse.json({
documentId: parsed.documentId,
deletedSegments: rows.length,
requestedAudioObjects: uniqueAudioKeys.length,
deletedAudioObjects,
...(warning ? { warning } : {}),
...cleared,
});
} catch (error) {
console.error('Error clearing TTS segment cache:', error);

View file

@ -30,7 +30,7 @@ import { getClientIp } from '@/lib/server/rate-limit/request-ip';
import { getOrCreateDeviceId, setDeviceIdCookie } from '@/lib/server/rate-limit/device-id';
import { buildDailyQuotaExceededResponse } from '@/lib/server/rate-limit/problem-response';
import { getUpstreamRetryAfterSeconds, getUpstreamStatus } from '@/lib/server/tts/upstream-response';
import { alignAudioWithText } from '@/lib/server/whisper/alignment';
import { getCompute } from '@/lib/server/compute';
import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config';
import { resolveTtsModelForProvider } from '@/lib/shared/tts-provider-policy';
import { resolveSegmentAudioUrls } from '@/lib/server/tts/segment-audio-urls';
@ -374,12 +374,10 @@ export async function POST(request: NextRequest) {
try {
const audioBuffer = await getTtsSegmentAudioObject(existing.audioKey);
const whisperBytes = Uint8Array.from(audioBuffer);
const aligned = await alignAudioWithText(
whisperBytes.buffer,
segment.text,
undefined,
{ engine: 'whisper.cpp' },
);
const aligned = (await getCompute().alignWords({
audioBuffer: whisperBytes.buffer,
text: segment.text,
})).alignments;
alignment = aligned[0] ? { ...aligned[0], sentenceIndex: segment.original.segmentIndex } : null;
if (alignment) {
@ -527,12 +525,10 @@ export async function POST(request: NextRequest) {
let alignment: TTSSegmentManifestItem['alignment'] = null;
try {
const whisperBytes = Uint8Array.from(persistedBuffer);
const aligned = await alignAudioWithText(
whisperBytes.buffer,
segment.text,
undefined,
{ engine: 'whisper.cpp' },
);
const aligned = (await getCompute().alignWords({
audioBuffer: whisperBytes.buffer,
text: segment.text,
})).alignments;
alignment = aligned[0] ? { ...aligned[0], sentenceIndex: segment.original.segmentIndex } : null;
} catch (alignError) {
console.warn('Whisper alignment unavailable for segment; continuing without word highlights.', {

View file

@ -1,11 +1,8 @@
import { NextRequest, NextResponse } from 'next/server';
import type { TTSSentenceAlignment } from '@/types/tts';
import { auth } from '@/lib/server/auth/auth';
import {
alignAudioWithText,
makeWhisperCacheKey,
type WhisperRequestBody,
} from '@/lib/server/whisper/alignment';
import { makeWhisperCacheKey, type WhisperRequestBody } from '@/lib/server/whisper/alignment';
import { getCompute } from '@/lib/server/compute';
export const runtime = 'nodejs';
@ -29,12 +26,12 @@ export async function POST(req: NextRequest) {
const cacheKey = makeWhisperCacheKey(body);
const audioBuffer = new Uint8Array(audio).buffer;
const alignments: TTSSentenceAlignment[] = await alignAudioWithText(
const alignments: TTSSentenceAlignment[] = (await getCompute().alignWords({
audioBuffer,
text,
cacheKey,
{ engine: 'whisper.cpp', lang }
);
lang,
})).alignments;
return NextResponse.json({ alignments }, { status: 200 });
} catch (error) {

View file

@ -303,14 +303,6 @@ export function AdminFeaturesPanel() {
right={renderSource('enableUserSignups')}
variant="flat"
/>
<ToggleRow
label="Word-level highlighting"
description="Highlight words during TTS playback."
checked={Boolean(draft.enableWordHighlight)}
onChange={(checked) => updateDraft('enableWordHighlight', checked)}
right={renderSource('enableWordHighlight')}
variant="flat"
/>
<ToggleRow
label="Audiobook export"
description='Show "Export audiobook" on PDF/EPUB pages.'

View file

@ -16,7 +16,16 @@ import {
clampTtsSegmentMaxBlockLength,
} from '@/types/config';
import { useFeatureFlag } from '@/contexts/RuntimeConfigContext';
import { Section, ToggleRow, segmentedButtonClass, segmentedGroupClass } from '@/components/formPrimitives';
import { Section, ToggleRow, buttonClass, segmentedButtonClass, segmentedGroupClass } from '@/components/formPrimitives';
import { RefreshIcon } from '@/components/icons/Icons';
import type { ParsedPdfBlockKind, PdfParseStatus } from '@/types/parsed-pdf';
const PDF_SKIP_KIND_OPTIONS: Array<{ kind: ParsedPdfBlockKind; label: string }> = [
{ kind: 'page-header', label: 'Page headers' },
{ kind: 'page-footer', label: 'Page footers' },
{ kind: 'footnote', label: 'Footnotes' },
{ kind: 'caption', label: 'Captions' },
];
const viewTypeTextMapping = [
{ id: 'single', name: 'Single Page' },
@ -71,13 +80,25 @@ function RangeSetting({
);
}
export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
export function DocumentSettings({ isOpen, setIsOpen, epub, html, pdf }: {
isOpen: boolean,
setIsOpen: (isOpen: boolean) => void,
epub?: boolean,
html?: boolean
html?: boolean,
pdf?: {
computeAvailable: boolean;
parseStatus: PdfParseStatus | null;
parsedOverlayEnabled: boolean;
skipBlockKinds: ParsedPdfBlockKind[];
chaptersFromSections: boolean;
onToggleOverlay: (enabled: boolean) => void;
onToggleSkipKind: (kind: ParsedPdfBlockKind, enabled: boolean) => void;
onToggleChaptersFromSections: (enabled: boolean) => void;
onForceReparse: () => void;
}
}) {
const canWordHighlight = useFeatureFlag('enableWordHighlight');
const computeAvailable = useFeatureFlag('computeAvailable');
const canWordHighlight = computeAvailable;
const {
viewType,
skipBlank,
@ -315,7 +336,7 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
/>
<ToggleRow
label="Word-by-word highlighting"
description={`Highlight words using timing data${!canWordHighlight ? ' (disabled by config)' : ''}.`}
description={`Highlight words using timing data${!canWordHighlight ? ' (not available on this server)' : ''}.`}
checked={pdfWordHighlightEnabled && pdfHighlightEnabled}
disabled={!pdfHighlightEnabled || !canWordHighlight}
onChange={(checked) => updateConfigKey('pdfWordHighlightEnabled', checked)}
@ -346,7 +367,7 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
/>
<ToggleRow
label="Word-by-word highlighting"
description={`Highlight words using timing data${!canWordHighlight ? ' (disabled by config)' : ''}.`}
description={`Highlight words using timing data${!canWordHighlight ? ' (not available on this server)' : ''}.`}
checked={epubWordHighlightEnabled && epubHighlightEnabled}
disabled={!epubHighlightEnabled || !canWordHighlight}
onChange={(checked) => updateConfigKey('epubWordHighlightEnabled', checked)}
@ -370,7 +391,7 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
/>
<ToggleRow
label="Word-by-word highlighting"
description={`Highlight words using timing data${!canWordHighlight ? ' (disabled by config)' : ''}.`}
description={`Highlight words using timing data${!canWordHighlight ? ' (not available on this server)' : ''}.`}
checked={htmlWordHighlightEnabled && htmlHighlightEnabled}
disabled={!htmlHighlightEnabled || !canWordHighlight}
onChange={(checked) => updateConfigKey('htmlWordHighlightEnabled', checked)}
@ -378,6 +399,59 @@ export function DocumentSettings({ isOpen, setIsOpen, epub, html }: {
/>
</Section>
)}
{!epub && !html && pdf && (
<Section
title="PDF Structure"
subtitle="Layout-aware parsing controls."
variant="flat"
>
<div className="flex items-center gap-1.5 text-xs text-muted pb-1">
Parse status: <span className="text-foreground font-medium">{pdf.parseStatus ?? 'pending'}</span>
<button
type="button"
className={buttonClass({ variant: 'ghost', size: 'icon', className: '!h-5 !w-5 hover:scale-[1.1] shrink-0' })}
onClick={pdf.onForceReparse}
disabled={!computeAvailable || pdf.parseStatus === 'running' || pdf.parseStatus === 'pending'}
title="Force reparse"
>
<RefreshIcon className="h-3 w-3" />
</button>
</div>
<ToggleRow
label="Show block overlay"
description="Render detected block boxes and labels on the page."
checked={pdf.parsedOverlayEnabled}
onChange={pdf.onToggleOverlay}
disabled={pdf.parseStatus !== 'ready'}
variant="flat"
/>
<ToggleRow
label="Use detected sections as chapters"
description={computeAvailable ? 'Build audiobook chapters from section headers.' : 'Not available on this server.'}
checked={pdf.chaptersFromSections}
onChange={pdf.onToggleChaptersFromSections}
disabled={!computeAvailable}
variant="flat"
/>
<div className="space-y-2 pt-1">
<p className="text-[11px] font-semibold uppercase tracking-wide text-muted">Skip while reading aloud</p>
{PDF_SKIP_KIND_OPTIONS.map((option) => {
const checked = pdf.skipBlockKinds.includes(option.kind);
return (
<ToggleRow
key={option.kind}
label={option.label}
description={`Exclude ${option.label.toLowerCase()} from TTS/audiobook output.`}
checked={checked}
onChange={(enabled) => pdf.onToggleSkipKind(option.kind, enabled)}
variant="flat"
/>
);
})}
</div>
</Section>
)}
</div>
</ReaderSidebarShell>
);

View file

@ -10,6 +10,7 @@ import { useTTS } from '@/contexts/TTSContext';
import { useConfig } from '@/contexts/ConfigContext';
import { usePDFResize } from '@/hooks/pdf/usePDFResize';
import type { PdfDocumentState } from '@/app/(app)/pdf/[id]/usePdfDocument';
import type { ParsedPdfBlock, ParsedPdfPage } from '@/types/parsed-pdf';
interface PDFViewerProps {
zoomLevel: number;
@ -25,6 +26,8 @@ interface PDFViewerProps {
| 'currDocPages'
| 'currDocText'
| 'currDocPage'
| 'parsedDocument'
| 'parsedOverlayEnabled'
>;
}
@ -67,6 +70,8 @@ export function PDFViewer({ zoomLevel, pdfState }: PDFViewerProps) {
currDocPages,
currDocText,
currDocPage,
parsedDocument,
parsedOverlayEnabled,
} = pdfState;
// IMPORTANT:
@ -299,6 +304,107 @@ export function PDFViewer({ zoomLevel, pdfState }: PDFViewerProps) {
return scaleRef.current;
}, [calculateScale]);
const parsedPageByNumber = useMemo(() => {
const map = new Map<number, ParsedPdfPage>();
for (const page of parsedDocument?.pages ?? []) {
map.set(page.pageNumber, page);
}
return map;
}, [parsedDocument]);
const parsedOverlayByPage = useMemo(() => {
const map = new Map<number, Array<{
block: ParsedPdfBlock;
fragment: ParsedPdfBlock['fragments'][number];
isContinuation: boolean;
}>>();
const seen = new Set<string>();
for (const page of parsedDocument?.pages ?? []) {
for (const block of page.blocks) {
for (let fragmentIndex = 0; fragmentIndex < block.fragments.length; fragmentIndex += 1) {
const fragment = block.fragments[fragmentIndex];
if (!fragment) continue;
const key = `${block.id}:${fragment.page}:${fragment.readingOrder}`;
if (seen.has(key)) continue;
seen.add(key);
const list = map.get(fragment.page) ?? [];
list.push({
block,
fragment,
isContinuation: fragmentIndex > 0,
});
map.set(fragment.page, list);
}
}
}
for (const list of map.values()) {
list.sort((a, b) => {
if (a.fragment.readingOrder !== b.fragment.readingOrder) {
return a.fragment.readingOrder - b.fragment.readingOrder;
}
return a.block.id.localeCompare(b.block.id);
});
}
return map;
}, [parsedDocument]);
const colorForKind = (kind: ParsedPdfBlock['kind']): string => {
switch (kind) {
case 'section-header': return 'rgba(34,197,94,0.20)';
case 'title': return 'rgba(16,185,129,0.20)';
case 'caption': return 'rgba(245,158,11,0.20)';
case 'table': return 'rgba(59,130,246,0.20)';
case 'picture': return 'rgba(139,92,246,0.20)';
case 'page-header':
case 'page-footer':
case 'footnote': return 'rgba(239,68,68,0.20)';
default: return 'rgba(14,165,233,0.18)';
}
};
const renderParsedOverlay = (pageNumber: number) => {
if (!parsedOverlayEnabled) return null;
const parsedPage = parsedPageByNumber.get(pageNumber);
if (!parsedPage) return null;
const overlayEntries = parsedOverlayByPage.get(pageNumber) ?? [];
return (
<div className="pointer-events-none absolute inset-0 z-20">
{overlayEntries.map(({ block, fragment, isContinuation }) => {
const [x0, y0, x1, y1] = fragment.bbox;
const width = parsedPage.width || 1;
const height = parsedPage.height || 1;
const leftPct = (x0 / width) * 100;
const boxWidthPct = Math.max(0, ((x1 - x0) / width) * 100);
// Parsed model bboxes are top-left based; use y0 directly.
const topPct = (y0 / height) * 100;
const boxHeightPct = Math.max(0, ((y1 - y0) / height) * 100);
return (
<div
key={`${block.id}:${fragment.page}:${fragment.readingOrder}`}
className="absolute border border-accent/70 rounded-[2px]"
style={{
left: `${leftPct}%`,
top: `${topPct}%`,
width: `${boxWidthPct}%`,
height: `${boxHeightPct}%`,
backgroundColor: colorForKind(block.kind),
}}
>
<span className="absolute -top-4 left-0 bg-black/75 text-white text-[10px] px-1 rounded-sm">
{isContinuation ? `${block.kind} (cont)` : block.kind}
</span>
</div>
);
})}
</div>
);
};
return (
<div
ref={containerRef}
@ -330,62 +436,71 @@ export function PDFViewer({ zoomLevel, pdfState }: PDFViewerProps) {
// Scroll mode: render all pages
<div className="flex flex-col gap-4">
{currDocPages && [...Array(currDocPages)].map((_, i) => (
<Page
key={`page_${i + 1}`}
pageNumber={i + 1}
renderAnnotationLayer={true}
renderTextLayer={i + 1 === currDocPage}
className="shadow-lg"
scale={currentScale()}
onRenderSuccess={() => {
lastRenderedLayoutKeyRef.current = layoutKey;
setIsPageRendering(false);
}}
onLoadSuccess={(page) => {
setPageWidth(page.originalWidth);
setPageHeight(page.originalHeight);
}}
/>
<div key={`page_wrap_${i + 1}`} className="relative">
<Page
key={`page_${i + 1}`}
pageNumber={i + 1}
renderAnnotationLayer={true}
renderTextLayer={i + 1 === currDocPage}
className="shadow-lg"
scale={currentScale()}
onRenderSuccess={() => {
lastRenderedLayoutKeyRef.current = layoutKey;
setIsPageRendering(false);
}}
onLoadSuccess={(page) => {
setPageWidth(page.originalWidth);
setPageHeight(page.originalHeight);
}}
/>
{renderParsedOverlay(i + 1)}
</div>
))}
</div>
) : (
// Single/Dual page mode
<div className="flex justify-center gap-4">
{currDocPages && leftPage > 0 && (
<Page
key={`page_${leftPage}`}
pageNumber={leftPage}
renderAnnotationLayer={true}
renderTextLayer={leftPage === currDocPage}
className="shadow-lg"
scale={currentScale()}
onRenderSuccess={() => {
lastRenderedLayoutKeyRef.current = layoutKey;
setIsPageRendering(false);
}}
onLoadSuccess={(page) => {
setPageWidth(page.originalWidth);
setPageHeight(page.originalHeight);
}}
/>
<div className="relative">
<Page
key={`page_${leftPage}`}
pageNumber={leftPage}
renderAnnotationLayer={true}
renderTextLayer={leftPage === currDocPage}
className="shadow-lg"
scale={currentScale()}
onRenderSuccess={() => {
lastRenderedLayoutKeyRef.current = layoutKey;
setIsPageRendering(false);
}}
onLoadSuccess={(page) => {
setPageWidth(page.originalWidth);
setPageHeight(page.originalHeight);
}}
/>
{renderParsedOverlay(leftPage)}
</div>
)}
{currDocPages && rightPage && rightPage <= currDocPages && viewType === 'dual' && (
<Page
key={`page_${rightPage}`}
pageNumber={rightPage}
renderAnnotationLayer={true}
renderTextLayer={rightPage === currDocPage}
className="shadow-lg"
scale={currentScale()}
onRenderSuccess={() => {
lastRenderedLayoutKeyRef.current = layoutKey;
setIsPageRendering(false);
}}
onLoadSuccess={(page) => {
setPageWidth(page.originalWidth);
setPageHeight(page.originalHeight);
}}
/>
<div className="relative">
<Page
key={`page_${rightPage}`}
pageNumber={rightPage}
renderAnnotationLayer={true}
renderTextLayer={rightPage === currDocPage}
className="shadow-lg"
scale={currentScale()}
onRenderSuccess={() => {
lastRenderedLayoutKeyRef.current = layoutKey;
setIsPageRendering(false);
}}
onLoadSuccess={(page) => {
setPageWidth(page.originalWidth);
setPageHeight(page.originalHeight);
}}
/>
{renderParsedOverlay(rightPage)}
</div>
)}
</div>
)}

View file

@ -18,11 +18,11 @@ export interface RuntimeConfig {
enableUserSignups: boolean;
restrictUserApiKeys: boolean;
enableTtsProvidersTab: boolean;
enableWordHighlight: boolean;
enableAudiobookExport: boolean;
enableDocxConversion: boolean;
enableDestructiveDeleteActions: boolean;
showAllProviderModels: boolean;
computeAvailable: boolean;
}
const RUNTIME_DEFAULTS: RuntimeConfig = {
@ -32,11 +32,11 @@ const RUNTIME_DEFAULTS: RuntimeConfig = {
enableUserSignups: true,
restrictUserApiKeys: true,
enableTtsProvidersTab: true,
enableWordHighlight: true,
enableAudiobookExport: true,
enableDocxConversion: true,
enableDestructiveDeleteActions: true,
showAllProviderModels: true,
computeAvailable: true,
};
declare global {

View file

@ -159,6 +159,7 @@ interface TTSContextType extends TTSPlaybackState {
interface SetTextOptions {
shouldPause?: boolean;
location?: TTSLocation;
sourceUnits?: CanonicalTtsSourceUnit[];
previousLocation?: TTSLocation;
nextLocation?: TTSLocation;
nextText?: string;
@ -218,8 +219,8 @@ const wordHighlightFeatureEnabled = (() => {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const injected = (window as any).__OPENREADER_RUNTIME_CONFIG__;
if (!injected || typeof injected !== 'object') return true;
return typeof injected.enableWordHighlight === 'boolean'
? injected.enableWordHighlight
return typeof injected.computeAvailable === 'boolean'
? injected.computeAvailable
: true;
})();
@ -292,6 +293,11 @@ const buildLocatorRequestKey = (locator: TTSSegmentLocator): string => {
if (typeof locator.location === 'string' && locator.location) {
return normalizeLocationKey(locator.location);
}
if (locator.readerType === 'pdf') {
const page = Number(locator.page || 1);
const block = typeof locator.blockId === 'string' && locator.blockId ? locator.blockId : '';
return normalizeLocationKey(`pdf:${page}:${block}`);
}
return normalizeLocationKey(Number(locator.page || 1));
};
@ -1124,6 +1130,9 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
? normalizedOptions.location
: currDocPage;
const resolvedLocationKey = normalizeLocationKey(resolvedLocation);
const currentUnits = normalizedOptions.sourceUnits && normalizedOptions.sourceUnits.length > 0
? normalizedOptions.sourceUnits
: null;
// Keep currDocPage aligned with whatever the caller declared as the viewport's
// location. This is the canonical entry point for "the rendered page now shows
@ -1142,6 +1151,8 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
text,
locator: locatorForLocation(resolvedLocation, activeReaderType),
};
const effectiveCurrentUnits = currentUnits && currentUnits.length > 0 ? currentUnits : [currentSource];
const currentSourceKeySet = new Set(effectiveCurrentUnits.map((unit) => unit.sourceKey));
const contextSourceUnits: CanonicalTtsSourceUnit[] = [];
if (smartSentenceSplitting && normalizedOptions.previousText?.trim()) {
@ -1156,7 +1167,7 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
: null,
});
}
contextSourceUnits.push(currentSource);
contextSourceUnits.push(...effectiveCurrentUnits);
const sourceUnits: CanonicalTtsSourceUnit[] = [...contextSourceUnits];
plannedSegmentsByLocationRef.current.clear();
@ -1191,14 +1202,17 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
readerType: activeReaderType,
maxBlockLength: ttsSegmentMaxBlockLength,
keyPrefix: buildSegmentKeyPrefix(documentId, activeReaderType),
enforceSourceBoundaries: activeReaderType === 'pdf' && currentUnits !== null && currentUnits.length > 0,
});
const currentSegments = smartSentenceSplitting
? plan.segments.filter((segment) => segment.ownerSourceKey === currentSourceKey)
: planCanonicalTtsSegments([currentSource], {
readerType: activeReaderType,
maxBlockLength: ttsSegmentMaxBlockLength,
keyPrefix: buildSegmentKeyPrefix(documentId, activeReaderType),
}).segments;
? plan.segments.filter((segment) => currentSourceKeySet.has(segment.ownerSourceKey))
: effectiveCurrentUnits.flatMap((unit) =>
planCanonicalTtsSegments([unit], {
readerType: activeReaderType,
maxBlockLength: ttsSegmentMaxBlockLength,
keyPrefix: buildSegmentKeyPrefix(documentId, activeReaderType),
enforceSourceBoundaries: activeReaderType === 'pdf' && currentUnits !== null && currentUnits.length > 0,
}).segments);
const newSentences = currentSegments.map((segment) => segment.text);
for (const item of pendingPrefetches) {
@ -1208,13 +1222,14 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
readerType: activeReaderType,
maxBlockLength: ttsSegmentMaxBlockLength,
keyPrefix: buildSegmentKeyPrefix(documentId, activeReaderType),
enforceSourceBoundaries: activeReaderType === 'pdf' && currentUnits !== null && currentUnits.length > 0,
}).segments;
if (planned.length > 0) {
plannedSegmentsByLocationRef.current.set(normalizeLocationKey(item.location), planned);
}
}
currentSourceUnitRef.current = currentSource;
currentSourceUnitRef.current = effectiveCurrentUnits[0] ?? null;
currentSourceContextUnitsRef.current = contextSourceUnits;
if (handleBlankSection(newSentences.join(' '))) return;
@ -1288,7 +1303,12 @@ export function TTSProvider({ children }: { children: ReactNode }): ReactElement
setCurrentSentenceAlignment(undefined);
setCurrentWordIndex(null);
if (smartSentenceSplitting && !isEPUB && normalizedOptions.nextLocation !== undefined) {
if (
smartSentenceSplitting
&& !isEPUB
&& normalizedOptions.nextLocation !== undefined
&& effectiveCurrentUnits.length === 1
) {
const spanningIndex = currentSegments.findIndex((segment) =>
segment.spansSourceBoundary
&& segment.startAnchor.sourceKey === currentSourceKey

View file

@ -11,6 +11,7 @@ export const audiobooks = usePostgres ? postgresSchema.audiobooks : sqliteSchema
export const audiobookChapters = usePostgres ? postgresSchema.audiobookChapters : sqliteSchema.audiobookChapters;
export const userTtsChars = usePostgres ? postgresSchema.userTtsChars : sqliteSchema.userTtsChars;
export const userPreferences = usePostgres ? postgresSchema.userPreferences : sqliteSchema.userPreferences;
export const documentSettings = usePostgres ? postgresSchema.documentSettings : sqliteSchema.documentSettings;
export const userDocumentProgress = usePostgres ? postgresSchema.userDocumentProgress : sqliteSchema.userDocumentProgress;
export const documentPreviews = usePostgres ? postgresSchema.documentPreviews : sqliteSchema.documentPreviews;
export const ttsSegmentEntries = usePostgres ? postgresSchema.ttsSegmentEntries : sqliteSchema.ttsSegmentEntries;

View file

@ -12,6 +12,8 @@ export const documents = pgTable('documents', {
size: bigint('size', { mode: 'number' }).notNull(),
lastModified: bigint('last_modified', { mode: 'number' }).notNull(),
filePath: text('file_path').notNull(),
parseStatus: text('parse_status'),
parsedJsonKey: text('parsed_json_key'),
createdAt: bigint('created_at', { mode: 'number' }).default(PG_NOW_MS),
}, (table) => [
primaryKey({ columns: [table.id, table.userId] }),
@ -72,6 +74,18 @@ export const userPreferences = pgTable('user_preferences', {
updatedAt: bigint('updated_at', { mode: 'number' }).default(PG_NOW_MS),
});
export const documentSettings = pgTable('document_settings', {
documentId: text('document_id').notNull(),
userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
dataJson: jsonb('data_json').notNull().default({}),
clientUpdatedAtMs: bigint('client_updated_at_ms', { mode: 'number' }).notNull().default(0),
createdAt: bigint('created_at', { mode: 'number' }).default(PG_NOW_MS),
updatedAt: bigint('updated_at', { mode: 'number' }).default(PG_NOW_MS),
}, (table) => [
primaryKey({ columns: [table.documentId, table.userId] }),
index('idx_document_settings_user_id').on(table.userId),
]);
export const userDocumentProgress = pgTable('user_document_progress', {
userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
documentId: text('document_id').notNull(),

View file

@ -12,6 +12,8 @@ export const documents = sqliteTable('documents', {
size: integer('size').notNull(),
lastModified: integer('last_modified').notNull(),
filePath: text('file_path').notNull(),
parseStatus: text('parse_status'),
parsedJsonKey: text('parsed_json_key'),
createdAt: integer('created_at').default(SQLITE_NOW_MS),
}, (table) => [
primaryKey({ columns: [table.id, table.userId] }),
@ -72,6 +74,18 @@ export const userPreferences = sqliteTable('user_preferences', {
updatedAt: integer('updated_at').default(SQLITE_NOW_MS),
});
export const documentSettings = sqliteTable('document_settings', {
documentId: text('document_id').notNull(),
userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
dataJson: text('data_json').notNull().default('{}'),
clientUpdatedAtMs: integer('client_updated_at_ms').notNull().default(0),
createdAt: integer('created_at').default(SQLITE_NOW_MS),
updatedAt: integer('updated_at').default(SQLITE_NOW_MS),
}, (table) => [
primaryKey({ columns: [table.documentId, table.userId] }),
index('idx_document_settings_user_id').on(table.userId),
]);
export const userDocumentProgress = sqliteTable('user_document_progress', {
userId: text('user_id').notNull().references(() => user.id, { onDelete: 'cascade' }),
documentId: text('document_id').notNull(),

View file

@ -1,5 +1,7 @@
import { sha256HexFromArrayBuffer } from '@/lib/client/sha256';
import type { BaseDocument, DocumentType } from '@/types/documents';
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
import type { DocumentSettings } from '@/types/document-settings';
export type UploadSource = {
id: string;
@ -76,6 +78,95 @@ export async function getDocumentMetadata(id: string, options?: { signal?: Abort
return docs[0] ?? null;
}
export async function getParsedPdfDocument(
id: string,
options?: { signal?: AbortSignal; retryFailed?: boolean },
): Promise<{ status: 'ready'; parsed: ParsedPdfDocument } | { status: 'pending' | 'running' | 'failed' | 'unsupported' }> {
const query = options?.retryFailed ? '?retry=1' : '';
const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed${query}`, {
signal: options?.signal,
cache: 'no-store',
});
if (res.status === 202) {
const data = (await res.json().catch(() => null)) as { parseStatus?: string } | null;
const parseStatus = data?.parseStatus;
if (parseStatus === 'pending' || parseStatus === 'running' || parseStatus === 'failed' || parseStatus === 'unsupported') {
return { status: parseStatus };
}
return { status: 'pending' };
}
if (!res.ok) {
const data = (await res.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || 'Failed to load parsed PDF');
}
const parsed = (await res.json()) as ParsedPdfDocument;
return { status: 'ready', parsed };
}
export async function forceReparsePdfDocument(
id: string,
options?: { signal?: AbortSignal },
): Promise<'pending' | 'running'> {
const res = await fetch(`/api/documents/${encodeURIComponent(id)}/parsed`, {
method: 'POST',
signal: options?.signal,
cache: 'no-store',
});
if (!res.ok) {
const data = (await res.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || 'Failed to force PDF reparse');
}
const data = (await res.json().catch(() => null)) as { parseStatus?: string } | null;
return data?.parseStatus === 'running' ? 'running' : 'pending';
}
type DocumentSettingsResponse = {
settings: DocumentSettings;
clientUpdatedAtMs: number;
hasStoredSettings?: boolean;
};
export async function getDocumentSettings(
id: string,
options?: { signal?: AbortSignal },
): Promise<DocumentSettingsResponse> {
const res = await fetch(`/api/documents/${encodeURIComponent(id)}/settings`, {
signal: options?.signal,
cache: 'no-store',
});
if (!res.ok) {
const data = (await res.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || 'Failed to load document settings');
}
return (await res.json()) as DocumentSettingsResponse;
}
export async function putDocumentSettings(
id: string,
settings: DocumentSettings,
options?: { signal?: AbortSignal; clientUpdatedAtMs?: number },
): Promise<DocumentSettingsResponse & { applied: boolean }> {
const res = await fetch(`/api/documents/${encodeURIComponent(id)}/settings`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
settings,
clientUpdatedAtMs: options?.clientUpdatedAtMs ?? Date.now(),
}),
signal: options?.signal,
});
if (!res.ok) {
const data = (await res.json().catch(() => null)) as { error?: string } | null;
throw new Error(data?.error || 'Failed to update document settings');
}
return (await res.json()) as DocumentSettingsResponse & { applied: boolean };
}
export async function uploadDocumentSources(sources: UploadSource[], options?: UploadOptions): Promise<BaseDocument[]> {
if (sources.length === 0) return [];

View file

@ -1,11 +1,15 @@
import type { PDFDocumentProxy } from 'pdfjs-dist';
import { extractTextFromPDF } from '@/lib/client/pdf';
import type { AudiobookSourceAdapter, PreparedAudiobookChapter } from '@/lib/client/audiobooks/pipeline';
import { normalizeTextForTts } from '@/lib/shared/nlp';
import type { ParsedPdfDocument, ParsedPdfBlock } from '@/types/parsed-pdf';
import type { DocumentSettings } from '@/types/document-settings';
import { DEFAULT_DOCUMENT_SETTINGS } from '@/types/document-settings';
interface PdfAudiobookAdapterOptions {
pdfDocument?: PDFDocumentProxy;
parsed?: ParsedPdfDocument;
settings?: DocumentSettings;
margins: {
header: number;
footer: number;
@ -16,16 +20,99 @@ interface PdfAudiobookAdapterOptions {
maxBlockLength?: number;
}
function chapterTextFromBlocks(
blocks: ParsedPdfBlock[],
smartSentenceSplitting: boolean,
maxBlockLength?: number,
): string {
const text = blocks
.map((block) => block.text.trim())
.filter(Boolean)
.join('\n\n');
if (!text) return '';
return smartSentenceSplitting ? normalizeTextForTts(text, { maxBlockLength }) : text;
}
function prepareParsedChapters({
parsed,
settings,
smartSentenceSplitting,
maxBlockLength,
}: {
parsed: ParsedPdfDocument;
settings: DocumentSettings;
smartSentenceSplitting: boolean;
maxBlockLength?: number;
}): PreparedAudiobookChapter[] {
const skip = new Set(settings.pdf?.skipBlockKinds ?? DEFAULT_DOCUMENT_SETTINGS.pdf?.skipBlockKinds ?? []);
const allBlocks = parsed.pages
.flatMap((page) => page.blocks)
.filter((block) => !skip.has(block.kind));
if (!allBlocks.length) return [];
const chaptersFromSections = settings.pdf?.chaptersFromSections ?? true;
if (!chaptersFromSections) {
const text = chapterTextFromBlocks(allBlocks, smartSentenceSplitting, maxBlockLength);
return text ? [{ index: 0, title: 'Document', text }] : [];
}
const chapters: PreparedAudiobookChapter[] = [];
let currentTitle = 'Introduction';
let currentBlocks: ParsedPdfBlock[] = [];
const flush = () => {
if (!currentBlocks.length) return;
const text = chapterTextFromBlocks(currentBlocks, smartSentenceSplitting, maxBlockLength);
if (text) {
chapters.push({
index: chapters.length,
title: currentTitle,
text,
});
}
currentBlocks = [];
};
for (const block of allBlocks) {
if (block.kind === 'section-header') {
flush();
currentTitle = block.text.trim() || `Chapter ${chapters.length + 1}`;
currentBlocks.push(block);
continue;
}
currentBlocks.push(block);
}
flush();
return chapters;
}
async function extractPreparedPdfChapters({
pdfDocument,
parsed,
settings = DEFAULT_DOCUMENT_SETTINGS,
margins,
smartSentenceSplitting,
maxBlockLength,
}: PdfAudiobookAdapterOptions): Promise<PreparedAudiobookChapter[]> {
if (parsed) {
const parsedChapters = prepareParsedChapters({
parsed,
settings,
smartSentenceSplitting,
maxBlockLength,
});
if (parsedChapters.length > 0) {
return parsedChapters;
}
}
if (!pdfDocument) {
throw new Error('No PDF document loaded');
}
const { extractTextFromPDF } = await import('@/lib/client/pdf');
const chapters: PreparedAudiobookChapter[] = [];
for (let pageNum = 1; pageNum <= pdfDocument.numPages; pageNum++) {
const rawText = await extractTextFromPDF(pdfDocument, pageNum, margins);

View file

@ -0,0 +1,20 @@
import type { ParsedPdfBlockKind, ParsedPdfPage } from '@/types/parsed-pdf';
export function buildPageTextFromBlocks(
page: ParsedPdfPage,
skipKinds: ParsedPdfBlockKind[] = [],
): string {
const skip = new Set(skipKinds);
return page.blocks
.filter((block) => !skip.has(block.kind))
.sort((a, b) => {
const aOrder = a.fragments[0]?.readingOrder ?? 0;
const bOrder = b.fragments[0]?.readingOrder ?? 0;
return aOrder - bOrder;
})
.map((block) => block.text.trim())
.filter(Boolean)
.join(' ')
.replace(/\s+/g, ' ')
.trim();
}

View file

@ -3,6 +3,8 @@ import type { TextItem } from 'pdfjs-dist/types/src/display/api';
import { type PDFDocumentProxy, TextLayer } from 'pdfjs-dist';
import "core-js/proposals/promise-with-resolvers";
import type { TTSSentenceAlignment } from '@/types/tts';
import type { ParsedPdfPage, ParsedPdfBlockKind } from '@/types/parsed-pdf';
import { buildPageTextFromBlocks } from '@/lib/client/pdf-block-text';
import { CmpStr } from 'cmpstr';
const cmp = CmpStr.create().setMetric('dice').setFlags('itw');
@ -80,18 +82,16 @@ function runHighlightTokenMatch(
function shouldUseLegacyBuild() {
try {
if (typeof window === 'undefined') return false;
const ua = window.navigator.userAgent;
const isSafari = /^((?!chrome|android).)*safari/i.test(ua);
console.log(isSafari ? 'Running on Safari' : 'Not running on Safari');
if (!isSafari) return false;
// Extract Safari version - matches "Version/18" format
const match = ua.match(/Version\/(\d+)/i);
console.log('Safari version:', match);
if (!match || !match[1]) return true; // If we can't determine version, use legacy to be safe
const version = parseInt(match[1]);
return version < 18; // Use legacy build for Safari versions equal or below 18
} catch (e) {
@ -106,10 +106,9 @@ function initPDFWorker() {
if (typeof window !== 'undefined') {
const useLegacy = shouldUseLegacyBuild();
// Use local worker file instead of unpkg
const workerSrc = useLegacy
const workerSrc = useLegacy
? new URL('pdfjs-dist/legacy/build/pdf.worker.min.mjs', import.meta.url).href
: new URL('pdfjs-dist/build/pdf.worker.min.mjs', import.meta.url).href;
console.log('Setting PDF worker to:', workerSrc);
pdfjs.GlobalWorkerOptions.workerSrc = workerSrc;
pdfjs.GlobalWorkerOptions.workerPort = null;
}
@ -192,35 +191,38 @@ const normalizeWordForMatch = (text: string): string =>
export async function extractTextFromPDF(
pdf: PDFDocumentProxy,
pageNumber: number,
margins = { header: 0.07, footer: 0.07, left: 0.07, right: 0.07 }
margins = { header: 0.07, footer: 0.07, left: 0.07, right: 0.07 },
parsed?: ParsedPdfPage,
skipKinds?: ParsedPdfBlockKind[],
): Promise<string> {
try {
// Log pdf worker version
//console.log('PDF worker version:', pdfjs.GlobalWorkerOptions.workerSrc);
if (parsed) {
return buildPageTextFromBlocks(parsed, skipKinds ?? []);
}
const page = await pdf.getPage(pageNumber);
const textContent = await page.getTextContent();
const viewport = page.getViewport({ scale: 1.0 });
const pageHeight = viewport.height;
const pageWidth = viewport.width;
const textItems = textContent.items.filter((item): item is TextItem => {
if (!('str' in item && 'transform' in item)) return false;
const [scaleX, skewX, skewY, scaleY, x, y] = item.transform;
// Basic text filtering
if (Math.abs(scaleX) < 1 || Math.abs(scaleX) > 20) return false;
if (Math.abs(scaleY) < 1 || Math.abs(scaleY) > 20) return false;
if (Math.abs(skewX) > 0.5 || Math.abs(skewY) > 0.5) return false;
// Calculate margins in PDF coordinate space (y=0 is at bottom)
const headerY = pageHeight * (1 - margins.header); // Convert from top margin to bottom-based Y
const footerY = pageHeight * margins.footer; // Footer Y stays as is since it's already bottom-based
const leftX = pageWidth * margins.left;
const rightX = pageWidth * (1 - margins.right);
// Check margins - remember y=0 is at bottom of page in PDF coordinates
if (y > headerY || y < footerY) { // Y greater than headerY means it's in header area, less than footerY means footer area
return false;
@ -230,15 +232,13 @@ export async function extractTextFromPDF(
if (x < leftX || x > rightX) {
return false;
}
// Sanity check for coordinates
if (x < 0 || x > pageWidth) return false;
return item.str.trim().length > 0;
});
//console.log('Filtered text items:', textItems);
const tolerance = 2;
const lines: TextItem[][] = [];
let currentLine: TextItem[] = [];

View file

@ -82,7 +82,6 @@ export const RUNTIME_CONFIG_SCHEMA = {
// Historically the env semantics were "true unless explicitly 'false'",
// i.e. the feature defaults to ON.
enableTtsProvidersTab: booleanFlag(true, 'NEXT_PUBLIC_ENABLE_TTS_PROVIDERS_TAB'),
enableWordHighlight: booleanFlag(true, 'NEXT_PUBLIC_ENABLE_WORD_HIGHLIGHT'),
enableAudiobookExport: booleanFlag(true, 'NEXT_PUBLIC_ENABLE_AUDIOBOOK_EXPORT'),
enableDocxConversion: booleanFlag(true, 'NEXT_PUBLIC_ENABLE_DOCX_CONVERSION'),
enableDestructiveDeleteActions: booleanFlag(true, 'NEXT_PUBLIC_ENABLE_DESTRUCTIVE_DELETE_ACTIONS'),

View file

@ -139,7 +139,6 @@ const createAuth = () => betterAuth({
},
},
plugins: [
nextCookies(), // Enable Next.js cookie handling
...(isAnonymousAuthSessionsEnabled()
? [
anonymous({
@ -219,6 +218,9 @@ const createAuth = () => betterAuth({
}),
]
: []),
// Better Auth requires cookie integration plugins last so post-hooks can
// still append Set-Cookie headers that are forwarded to Next.js.
nextCookies(),
],
});

View file

@ -0,0 +1,37 @@
import type { ComputeBackend, ComputeMode } from '@/lib/server/compute/types';
import { LocalComputeBackend } from '@/lib/server/compute/local';
import { NoneComputeBackend } from '@/lib/server/compute/none';
let backend: ComputeBackend | null = null;
function readMode(): ComputeMode {
const raw = (process.env.OPENREADER_COMPUTE_MODE || 'local').trim().toLowerCase();
if (raw === 'local' || raw === 'none' || raw === 'worker') return raw;
return 'local';
}
function createBackend(): ComputeBackend {
const mode = readMode();
if (mode === 'none') return new NoneComputeBackend();
if (mode === 'worker') {
throw new Error(
'OPENREADER_COMPUTE_MODE=worker is not implemented yet in v1. Switch to local/none or implement WorkerComputeBackend (v2).',
);
}
return new LocalComputeBackend();
}
export function getCompute(): ComputeBackend {
if (!backend) backend = createBackend();
return backend;
}
export function isComputeAvailable(): boolean {
const mode = readMode();
if (mode === 'worker') {
throw new Error(
'OPENREADER_COMPUTE_MODE=worker is not implemented yet in v1. Switch to local/none or implement WorkerComputeBackend (v2).',
);
}
return mode !== 'none';
}

View file

@ -0,0 +1,21 @@
import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types';
import { alignAudioWithText } from '@/lib/server/whisper/alignment';
import { parsePdf } from '@/lib/server/pdf-layout/parsePdf';
export class LocalComputeBackend implements ComputeBackend {
readonly mode = 'local' as const;
async alignWords(input: WhisperAlignInput): Promise<WhisperAlignResult> {
const alignments = await alignAudioWithText(
input.audioBuffer,
input.text,
input.cacheKey,
{ engine: 'whisper.cpp', lang: input.lang },
);
return { alignments };
}
async parsePdfLayout(input: PdfLayoutInput) {
return parsePdf({ documentId: input.documentId, pdfBytes: input.pdfBytes });
}
}

View file

@ -0,0 +1,17 @@
import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types';
import { UnsupportedComputeError } from '@/lib/server/compute/types';
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
export class NoneComputeBackend implements ComputeBackend {
readonly mode = 'none' as const;
async alignWords(input: WhisperAlignInput): Promise<WhisperAlignResult> {
void input;
throw new UnsupportedComputeError('Word alignment is unavailable: OPENREADER_COMPUTE_MODE=none');
}
async parsePdfLayout(input: PdfLayoutInput): Promise<ParsedPdfDocument> {
void input;
throw new UnsupportedComputeError('PDF layout parsing is unavailable: OPENREADER_COMPUTE_MODE=none');
}
}

View file

@ -0,0 +1,33 @@
import type { TTSAudioBuffer, TTSSentenceAlignment } from '@/types/tts';
import type { ParsedPdfDocument } from '@/types/parsed-pdf';
export type ComputeMode = 'local' | 'worker' | 'none';
export interface WhisperAlignInput {
audioBuffer: TTSAudioBuffer;
text: string;
cacheKey?: string;
lang?: string;
}
export interface WhisperAlignResult {
alignments: TTSSentenceAlignment[];
}
export interface PdfLayoutInput {
documentId: string;
pdfBytes: ArrayBuffer;
}
export interface ComputeBackend {
mode: ComputeMode;
alignWords(input: WhisperAlignInput): Promise<WhisperAlignResult>;
parsePdfLayout(input: PdfLayoutInput): Promise<ParsedPdfDocument>;
}
export class UnsupportedComputeError extends Error {
constructor(message: string) {
super(message);
this.name = 'UnsupportedComputeError';
}
}

View file

@ -80,11 +80,55 @@ export function documentKey(id: string, namespace: string | null): string {
return `${cfg.prefix}/documents_v1/${nsSegment}${id}`;
}
export function documentParsedKey(id: string, namespace: string | null): string {
if (!isValidDocumentId(id)) {
throw new Error(`Invalid document id: ${id}`);
}
const cfg = getS3Config();
const ns = sanitizeNamespace(namespace);
const nsSegment = ns ? `ns/${ns}/` : '';
return `${cfg.prefix}/documents_v1/parsed_v1/${nsSegment}${id}.json`;
}
function legacyDocumentParsedKey(id: string, namespace: string | null): string {
if (!isValidDocumentId(id)) {
throw new Error(`Invalid document id: ${id}`);
}
const cfg = getS3Config();
const ns = sanitizeNamespace(namespace);
const nsSegment = ns ? `ns/${ns}/` : '';
return `${cfg.prefix}/documents_v1/${nsSegment}${id}/parsed.v1.json`;
}
async function cleanupLegacyParsedPathCollision(id: string, namespace: string | null): Promise<void> {
const cfg = getS3Config();
const client = getS3ProxyClient();
const key = documentKey(id, namespace);
const legacyPrefix = `${key}/`;
const legacyParsedKey = legacyDocumentParsedKey(id, namespace);
const legacyObjects = await client.send(
new ListObjectsV2Command({
Bucket: cfg.bucket,
Prefix: legacyPrefix,
MaxKeys: 1,
}),
);
const hasLegacyChildren = (legacyObjects.Contents?.length ?? 0) > 0;
if (!hasLegacyChildren) return;
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey })).catch(() => undefined);
await deleteDocumentPrefix(legacyPrefix).catch(() => undefined);
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: key })).catch(() => undefined);
}
export async function presignPut(
id: string,
contentType: string,
namespace: string | null,
): Promise<{ url: string; headers: Record<string, string> }> {
await cleanupLegacyParsedPathCollision(id, namespace);
const cfg = getS3Config();
const client = getS3Client();
const key = documentKey(id, namespace);
@ -94,7 +138,6 @@ export async function presignPut(
Bucket: cfg.bucket,
Key: key,
ContentType: normalizedType,
IfNoneMatch: '*',
ServerSideEncryption: 'AES256',
});
const url = await getSignedUrl(client, command, { expiresIn: 60 * 5 });
@ -103,7 +146,6 @@ export async function presignPut(
url,
headers: {
'Content-Type': normalizedType,
'If-None-Match': '*',
'x-amz-server-side-encryption': 'AES256',
},
};
@ -169,6 +211,35 @@ export async function getDocumentBlobStream(id: string, namespace: string | null
return res.Body as DocumentBlobBody;
}
export async function getParsedDocumentBlob(id: string, namespace: string | null): Promise<Buffer> {
const cfg = getS3Config();
const client = getS3ProxyClient();
const key = documentParsedKey(id, namespace);
const res = await client.send(
new GetObjectCommand({
Bucket: cfg.bucket,
Key: key,
}),
);
return bodyToBuffer(res.Body);
}
export async function putParsedDocumentBlob(id: string, body: Buffer, namespace: string | null): Promise<string> {
const cfg = getS3Config();
const client = getS3ProxyClient();
const key = documentParsedKey(id, namespace);
await client.send(
new PutObjectCommand({
Bucket: cfg.bucket,
Key: key,
Body: body,
ContentType: 'application/json',
ServerSideEncryption: 'AES256',
}),
);
return key;
}
export async function presignGet(
id: string,
namespace: string | null,
@ -193,6 +264,8 @@ export async function putDocumentBlob(
contentType: string,
namespace: string | null,
): Promise<void> {
await cleanupLegacyParsedPathCollision(id, namespace);
const cfg = getS3Config();
const client = getS3ProxyClient();
const key = documentKey(id, namespace);
@ -202,7 +275,6 @@ export async function putDocumentBlob(
Key: key,
Body: body,
ContentType: contentType,
IfNoneMatch: '*',
ServerSideEncryption: 'AES256',
}),
);
@ -212,7 +284,13 @@ export async function deleteDocumentBlob(id: string, namespace: string | null):
const cfg = getS3Config();
const client = getS3ProxyClient();
const key = documentKey(id, namespace);
const parsedKey = documentParsedKey(id, namespace);
const legacyParsedKey = legacyDocumentParsedKey(id, namespace);
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: key }));
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey })).catch(() => undefined);
await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey })).catch(() => undefined);
await deleteDocumentPrefix(`${key}/`).catch(() => undefined);
}
export function isMissingBlobError(error: unknown): boolean {

View file

@ -0,0 +1,92 @@
import { and, eq } from 'drizzle-orm';
import { db } from '@/db';
import { documents } from '@/db/schema';
import { UnsupportedComputeError } from '@/lib/server/compute/types';
import { getDocumentBlob, putParsedDocumentBlob } from '@/lib/server/documents/blobstore';
import { getCompute } from '@/lib/server/compute';
import { clearTtsSegmentCache } from '@/lib/server/tts/segments-cache';
interface ParsePdfJobInput {
documentId: string;
userId: string;
namespace: string | null;
}
const running = new Set<string>();
function keyFor(input: ParsePdfJobInput): string {
return `${input.userId}:${input.documentId}:${input.namespace || ''}`;
}
export async function parsePdfJob(input: ParsePdfJobInput): Promise<void> {
const key = keyFor(input);
if (running.has(key)) return;
running.add(key);
try {
await db
.update(documents)
.set({ parseStatus: 'running' })
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
const pdfBytes = await getDocumentBlob(input.documentId, input.namespace);
const parsed = await getCompute().parsePdfLayout({
documentId: input.documentId,
pdfBytes: new Uint8Array(pdfBytes).buffer,
});
const parsedJson = Buffer.from(JSON.stringify(parsed));
const parsedJsonKey = await putParsedDocumentBlob(input.documentId, parsedJson, input.namespace);
const cleared = await clearTtsSegmentCache({
userId: input.userId,
documentId: input.documentId,
readerType: 'pdf',
});
if (cleared.warning) {
console.warn('[parsePdfJob] cache invalidation warning', {
documentId: input.documentId,
warning: cleared.warning,
});
}
await db
.update(documents)
.set({ parseStatus: 'ready', parsedJsonKey })
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
const stack = error instanceof Error ? error.stack : undefined;
const cause = error instanceof Error ? error.cause : undefined;
const parseStatus = error instanceof UnsupportedComputeError ? 'unsupported' : 'failed';
try {
await db
.update(documents)
.set({ parseStatus })
.where(and(eq(documents.id, input.documentId), eq(documents.userId, input.userId)));
} catch (statusError) {
console.error('[parsePdfJob] failed to write parse status', {
documentId: input.documentId,
parseStatus,
error: statusError instanceof Error ? statusError.message : String(statusError),
});
}
console.error('[parsePdfJob] failed', {
documentId: input.documentId,
parseStatus,
error: message,
...(stack ? { stack } : {}),
...(cause ? { cause: String(cause) } : {}),
});
} finally {
running.delete(key);
}
}
export function enqueueParsePdfJob(input: ParsePdfJobInput): void {
Promise.resolve()
.then(() => parsePdfJob(input))
.catch((error) => {
console.error('[parsePdfJob] uncaught error', error);
});
}

View file

@ -0,0 +1,99 @@
import path from 'path';
import { createHash } from 'crypto';
import { access, mkdir, rename, writeFile, readFile, unlink, copyFile } from 'fs/promises';
import { DOCSTORE_DIR } from '@/lib/server/storage/library-mount';
import manifest from '@/lib/server/pdf-layout/model/manifest.json';
const DEFAULT_MODEL_URL = 'https://huggingface.co/docling-project/docling-layout-heron-onnx/resolve/main/model.onnx';
const DEFAULT_CONFIG_URL = 'https://huggingface.co/docling-project/docling-layout-heron-onnx/resolve/main/config.json';
const MODEL_DIR = path.join(DOCSTORE_DIR, 'model');
const MODEL_PATH = path.join(MODEL_DIR, 'docling-layout-heron.onnx');
const CONFIG_PATH = path.join(MODEL_DIR, 'docling-layout-heron.config.json');
const LICENSE_PATH = path.join(MODEL_DIR, 'docling-layout-heron.LICENSE.txt');
const STATIC_LICENSE_PATH = path.join(process.cwd(), 'src/lib/server/pdf-layout/model/LICENSE.txt');
let inflight: Promise<string> | null = null;
async function sha256Hex(filePath: string): Promise<string> {
const bytes = await readFile(filePath);
return createHash('sha256').update(bytes).digest('hex');
}
async function downloadToFile(url: string, outPath: string): Promise<void> {
const res = await fetch(url);
if (!res.ok) {
throw new Error(`Download failed for ${url}: ${res.status} ${res.statusText}`);
}
const bytes = new Uint8Array(await res.arrayBuffer());
await writeFile(outPath, bytes);
}
function manifestEntry(filePath: string): { sha256: string; size: number } | null {
const found = manifest.files.find((entry) => entry.path === filePath);
if (!found || !found.sha256) return null;
return {
sha256: found.sha256.toLowerCase(),
size: Number(found.size),
};
}
async function verifyFile(pathToFile: string, manifestPath: string): Promise<boolean> {
const expected = manifestEntry(manifestPath);
if (!expected) return true;
const bytes = await readFile(pathToFile);
if (Number.isFinite(expected.size) && expected.size > 0 && bytes.byteLength !== expected.size) {
return false;
}
const actual = await sha256Hex(pathToFile);
return actual === expected.sha256;
}
async function ensureLicense(): Promise<void> {
await copyFile(STATIC_LICENSE_PATH, LICENSE_PATH);
if (!(await verifyFile(LICENSE_PATH, 'LICENSE.txt'))) {
throw new Error('Docling model license checksum verification failed');
}
}
async function ensureModelInternal(): Promise<string> {
try {
await access(MODEL_PATH);
await access(CONFIG_PATH);
if (await verifyFile(MODEL_PATH, 'model.onnx') && await verifyFile(CONFIG_PATH, 'config.json')) {
await ensureLicense();
return MODEL_PATH;
}
} catch {
// continue
}
await mkdir(MODEL_DIR, { recursive: true });
const tmpPath = `${MODEL_PATH}.tmp`;
const configTmpPath = `${CONFIG_PATH}.tmp`;
const modelUrl = process.env.OPENREADER_DOCLING_MODEL_URL?.trim() || DEFAULT_MODEL_URL;
const configUrl = process.env.OPENREADER_DOCLING_CONFIG_URL?.trim() || DEFAULT_CONFIG_URL;
await downloadToFile(modelUrl, tmpPath);
if (!(await verifyFile(tmpPath, 'model.onnx'))) {
await unlink(tmpPath).catch(() => undefined);
throw new Error('Docling model checksum verification failed');
}
await downloadToFile(configUrl, configTmpPath);
if (!(await verifyFile(configTmpPath, 'config.json'))) {
await unlink(configTmpPath).catch(() => undefined);
throw new Error('Docling model config checksum verification failed');
}
await rename(tmpPath, MODEL_PATH);
await rename(configTmpPath, CONFIG_PATH);
await ensureLicense();
return MODEL_PATH;
}
export async function ensureModel(): Promise<string> {
if (inflight) return inflight;
inflight = ensureModelInternal().finally(() => {
inflight = null;
});
return inflight;
}

View file

@ -0,0 +1,145 @@
import type { LayoutRegion, PdfTextItem } from '@/lib/server/pdf-layout/types';
const NON_TEXT_REGION_LABELS = new Set<LayoutRegion['label']>(['picture', 'table']);
const TEXT_ASSIGNABLE_LABELS = new Set<LayoutRegion['label']>([
'title',
'section-header',
'paragraph',
'list-item',
'caption',
'page-header',
'page-footer',
'footnote',
'formula',
]);
function centroid(item: PdfTextItem): { x: number; y: number } {
return {
x: item.x + item.width / 2,
y: item.y + item.height / 2,
};
}
function contains(region: LayoutRegion, item: PdfTextItem): boolean {
const c = centroid(item);
return c.x >= region.bbox[0] && c.x <= region.bbox[2] && c.y >= region.bbox[1] && c.y <= region.bbox[3];
}
function sortReadingOrder(items: PdfTextItem[]): PdfTextItem[] {
const tolerance = 2;
return [...items].sort((a, b) => {
if (Math.abs(a.y - b.y) <= tolerance) return a.x - b.x;
return a.y - b.y;
});
}
function joinText(items: PdfTextItem[]): string {
let out = '';
let prev: PdfTextItem | null = null;
for (const item of items) {
if (!prev) {
out += item.text;
prev = item;
continue;
}
const prevEndX = prev.x + prev.width;
const gap = item.x - prevEndX;
const lineJump = item.y - prev.y;
const lineBreak = lineJump > Math.max(2, Math.min(prev.height, item.height) * 0.6);
const avgCharWidth = item.width / Math.max(1, item.text.length);
const needsSpace = lineBreak || gap > Math.max(avgCharWidth * 0.3, 2);
out += needsSpace ? ` ${item.text}` : item.text;
prev = item;
}
return out.replace(/\s+/g, ' ').trim();
}
function regionArea(region: LayoutRegion): number {
return Math.max(1, (region.bbox[2] - region.bbox[0]) * (region.bbox[3] - region.bbox[1]));
}
function regionScore(region: LayoutRegion): number {
return Number.isFinite(region.confidence) ? Number(region.confidence) : 0;
}
export interface RegionTextBlock {
region: LayoutRegion;
text: string;
items: PdfTextItem[];
sourceOrder: number;
}
export function mergeTextWithRegions(regions: LayoutRegion[], textItems: PdfTextItem[]): RegionTextBlock[] {
const sourceIndex = new Map<PdfTextItem, number>();
for (let i = 0; i < textItems.length; i += 1) {
sourceIndex.set(textItems[i]!, i);
}
const chunkSourceOrder = (items: PdfTextItem[]): number => {
let min = Number.POSITIVE_INFINITY;
for (const item of items) {
const index = sourceIndex.get(item);
if (typeof index === 'number' && index < min) min = index;
}
return Number.isFinite(min) ? min : Number.MAX_SAFE_INTEGER;
};
const assignableRegions = regions
.map((region, index) => ({ region, index }))
.filter(({ region }) => TEXT_ASSIGNABLE_LABELS.has(region.label));
const assignedByRegion = new Map<number, PdfTextItem[]>();
for (const item of textItems) {
const candidates = assignableRegions.filter(({ region }) => contains(region, item));
if (candidates.length === 0) continue;
candidates.sort((a, b) => {
const scoreDelta = regionScore(b.region) - regionScore(a.region);
if (Math.abs(scoreDelta) > 1e-6) return scoreDelta;
return regionArea(a.region) - regionArea(b.region);
});
const winner = candidates[0];
const list = assignedByRegion.get(winner.index) ?? [];
list.push(item);
assignedByRegion.set(winner.index, list);
}
const out: RegionTextBlock[] = [];
for (const [regionIndex, assignedItems] of assignedByRegion.entries()) {
const region = regions[regionIndex];
if (!region) continue;
if (assignedItems.length === 0) continue;
const ordered = sortReadingOrder(assignedItems);
const text = joinText(ordered);
if (!text) continue;
out.push({
region,
text,
items: ordered,
sourceOrder: chunkSourceOrder(ordered),
});
}
for (const region of regions) {
if (!NON_TEXT_REGION_LABELS.has(region.label)) continue;
out.push({
region,
text: '',
items: [],
sourceOrder: Number.MAX_SAFE_INTEGER,
});
}
out.sort((a, b) => {
if (a.sourceOrder !== b.sourceOrder) return a.sourceOrder - b.sourceOrder;
const ay = a.region.bbox[1];
const by = b.region.bbox[1];
if (Math.abs(ay - by) <= 2) return a.region.bbox[0] - b.region.bbox[0];
return ay - by;
});
return out;
}

View file

@ -0,0 +1,3 @@
Docling layout model assets are distributed under Apache-2.0 / CDLA terms by the model publisher.
See the upstream release for the authoritative license text:
https://huggingface.co/ds4sd/docling-layout-heron

View file

@ -0,0 +1,21 @@
{
"name": "docling-layout-heron",
"version": "docling-project/docling-layout-heron-onnx@main",
"files": [
{
"path": "model.onnx",
"sha256": "59c81a3a2923042d85034ffc487f8f47e4854117e879aef89b2b9f728fb4922a",
"size": 171220471
},
{
"path": "config.json",
"sha256": "c6e67b6cdf64fa245779d0409bb994aa96e8c1790e15ec07a65843628e232180",
"size": 878
},
{
"path": "LICENSE.txt",
"sha256": "8f5030e085c59609746fce1010230f12b4a565abb6bdc19a2c2d3735ba1710d9",
"size": 209
}
]
}

View file

@ -0,0 +1,153 @@
import path from 'path';
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
import type { ParsedPdfDocument, ParsedPdfPage } from '@/types/parsed-pdf';
import type { PdfTextItem } from '@/lib/server/pdf-layout/types';
import { ensureModel } from '@/lib/server/pdf-layout/ensureModel';
import { runLayoutModel } from '@/lib/server/pdf-layout/runLayoutModel';
import { mergeTextWithRegions } from '@/lib/server/pdf-layout/mergeTextWithRegions';
import { stitchCrossPageBlocks } from '@/lib/server/pdf-layout/stitchCrossPageBlocks';
import { renderPage } from '@/lib/server/pdf-layout/renderPage';
interface ParsePdfInput {
documentId: string;
pdfBytes: ArrayBuffer;
}
const LAYOUT_RENDER_SCALE = 1.5;
function normalizeTextItems(items: TextItem[], pageHeight: number): PdfTextItem[] {
return items
.filter((item) => typeof item.str === 'string' && item.str.trim().length > 0)
.map((item) => {
const x = Number(item.transform[4] ?? 0);
const width = Math.max(0, Number(item.width ?? 0));
const height = Math.max(1, Math.abs(Number(item.transform[3] ?? 1)));
const baselineY = Number(item.transform[5] ?? 0);
// pdf.js text transforms are in PDF user-space (origin bottom-left).
// Normalize into top-left page coordinates to match rendered image/model boxes.
const y = Math.max(0, pageHeight - baselineY - height);
return {
text: item.str,
x,
y,
width,
height,
};
});
}
export async function parsePdf(input: ParsePdfInput): Promise<ParsedPdfDocument> {
await ensureModel();
// Keep independent byte copies for text extraction and page rendering. pdf.js
// can detach buffers passed to getDocument().
const pdfBytesForText = new Uint8Array(input.pdfBytes).slice();
const pdfBytesForRender = new Uint8Array(input.pdfBytes).slice();
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
if (pdfjs.GlobalWorkerOptions) {
pdfjs.GlobalWorkerOptions.workerSrc = 'pdfjs-dist/legacy/build/pdf.worker.mjs';
pdfjs.GlobalWorkerOptions.workerPort = null;
}
const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts');
const standardFontDataUrl = `${standardFontDir.replace(/\/?$/, '/')}`;
const loadingTask = pdfjs.getDocument({
data: pdfBytesForText,
useWorkerFetch: false,
standardFontDataUrl,
isEvalSupported: false,
});
const pdf = await loadingTask.promise;
try {
const pages: ParsedPdfPage[] = [];
let nextBlockId = 1;
let sawText = false;
for (let pageNumber = 1; pageNumber <= pdf.numPages; pageNumber += 1) {
const page = await pdf.getPage(pageNumber);
const viewport = page.getViewport({ scale: 1.0 });
const textContent = await page.getTextContent();
const textItems = normalizeTextItems(
textContent.items.filter((item): item is TextItem => 'str' in item && 'transform' in item),
viewport.height,
);
if (textItems.length > 0) sawText = true;
const rendered = await renderPage({
pdfBytes: pdfBytesForRender.buffer.slice(
pdfBytesForRender.byteOffset,
pdfBytesForRender.byteOffset + pdfBytesForRender.byteLength,
),
pageNumber,
scale: LAYOUT_RENDER_SCALE,
});
const scaleX = rendered.width / Math.max(1, viewport.width);
const scaleY = rendered.height / Math.max(1, viewport.height);
const layoutTextItems = textItems.map((item) => ({
...item,
x: item.x * scaleX,
y: item.y * scaleY,
width: item.width * scaleX,
height: item.height * scaleY,
}));
const regions = await runLayoutModel({
pageWidth: rendered.width,
pageHeight: rendered.height,
textItems: layoutTextItems,
pagePng: rendered.png,
});
const merged = mergeTextWithRegions(regions, layoutTextItems);
if (textItems.length > 0 && merged.length === 0) {
throw new Error(`layout-merge-empty: page=${pageNumber} regions=${regions.length}`);
}
const blocks = merged
.map((entry, readingOrder) => ({
id: `b${String(nextBlockId++).padStart(4, '0')}`,
kind: entry.region.label,
fragments: [{
page: pageNumber,
bbox: [
entry.region.bbox[0] / scaleX,
entry.region.bbox[1] / scaleY,
entry.region.bbox[2] / scaleX,
entry.region.bbox[3] / scaleY,
] as [number, number, number, number],
text: entry.text,
readingOrder,
...(typeof entry.region.confidence === 'number' ? { modelConfidence: entry.region.confidence } : {}),
}],
text: entry.text,
}));
pages.push({
pageNumber,
width: viewport.width,
height: viewport.height,
blocks,
});
}
if (!sawText) {
throw new Error('no-text-layer');
}
const doc: ParsedPdfDocument = {
schemaVersion: 1,
documentId: input.documentId,
parserVersion: 'docling-layout-heron@v1+pdfjs@4.8.69',
parsedAt: Date.now(),
pages,
};
return stitchCrossPageBlocks(doc);
} finally {
await pdf.destroy().catch(() => undefined);
await loadingTask.destroy().catch(() => undefined);
}
}

View file

@ -0,0 +1,142 @@
import path from 'path';
import type { Canvas } from '@napi-rs/canvas';
type CanvasRuntime = {
DOMMatrixCtor: unknown;
Path2DCtor: unknown;
createCanvasFn: (width: number, height: number) => Canvas;
};
let canvasRuntimePromise: Promise<CanvasRuntime> | null = null;
async function loadCanvasRuntime(): Promise<CanvasRuntime> {
if (!canvasRuntimePromise) {
canvasRuntimePromise = (async () => {
const mod = await import('@napi-rs/canvas');
const namespace = mod as Record<string, unknown>;
const fallback = (namespace.default ?? {}) as Record<string, unknown>;
const createCanvasFn = (namespace.createCanvas ?? fallback.createCanvas) as
| ((width: number, height: number) => Canvas)
| undefined;
const DOMMatrixCtor = namespace.DOMMatrix ?? fallback.DOMMatrix;
const Path2DCtor = namespace.Path2D ?? fallback.Path2D;
if (typeof createCanvasFn !== 'function') {
throw new Error(
`Canvas runtime missing createCanvas export (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`,
);
}
if (!DOMMatrixCtor || !Path2DCtor) {
throw new Error(
`Canvas runtime missing DOMMatrix/Path2D exports (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`,
);
}
return {
DOMMatrixCtor,
Path2DCtor,
createCanvasFn,
};
})();
}
return canvasRuntimePromise;
}
function ensureNodeCanvasGlobals(runtime: CanvasRuntime): void {
const g = globalThis as Record<string, unknown>;
if (typeof g.DOMMatrix === 'undefined') g.DOMMatrix = runtime.DOMMatrixCtor;
if (typeof g.Path2D === 'undefined') g.Path2D = runtime.Path2DCtor;
}
interface RenderInput {
pdfBytes: ArrayBuffer;
pageNumber: number;
scale?: number;
}
function createPdfjsCanvasFactory(runtime: CanvasRuntime) {
return class OpenReaderCanvasFactory {
create(width: number, height: number) {
const canvas = runtime.createCanvasFn(Math.max(1, Math.floor(width)), Math.max(1, Math.floor(height)));
return {
canvas,
context: canvas.getContext('2d') as unknown as CanvasRenderingContext2D,
};
}
reset(target: { canvas: Canvas; context: CanvasRenderingContext2D }, width: number, height: number): void {
target.canvas.width = Math.max(1, Math.floor(width));
target.canvas.height = Math.max(1, Math.floor(height));
}
destroy(target: { canvas: Canvas; context: CanvasRenderingContext2D }): void {
target.canvas.width = 0;
target.canvas.height = 0;
// @ts-expect-error pdf.js expects these nulled on destroy
target.canvas = null;
// @ts-expect-error pdf.js expects these nulled on destroy
target.context = null;
}
};
}
export async function renderPage({ pdfBytes, pageNumber, scale = 1.5 }: RenderInput): Promise<{
width: number;
height: number;
png: Buffer;
}> {
// pdf.js may detach the provided ArrayBuffer. Work with an isolated copy so
// callers can safely reuse their original bytes across pages/calls.
const isolatedBytes = new Uint8Array(pdfBytes).slice();
const canvasRuntime = await loadCanvasRuntime();
ensureNodeCanvasGlobals(canvasRuntime);
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs');
if (pdfjs.GlobalWorkerOptions) {
pdfjs.GlobalWorkerOptions.workerSrc = 'pdfjs-dist/legacy/build/pdf.worker.mjs';
pdfjs.GlobalWorkerOptions.workerPort = null;
}
const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts');
const standardFontDataUrl = `${standardFontDir.replace(/\/?$/, '/')}`;
const loadingTask = pdfjs.getDocument({
data: isolatedBytes,
useWorkerFetch: false,
standardFontDataUrl,
isEvalSupported: false,
// Ensure pdf.js transport uses our canvas backend in Node/Next runtime.
CanvasFactory: createPdfjsCanvasFactory(canvasRuntime),
});
const pdf = await loadingTask.promise;
try {
const page = await pdf.getPage(pageNumber);
const viewport = page.getViewport({ scale });
const width = Math.max(1, Math.floor(viewport.width));
const height = Math.max(1, Math.floor(viewport.height));
const canvas = canvasRuntime.createCanvasFn(width, height);
const ctx = canvas.getContext('2d');
ctx.fillStyle = '#ffffff';
ctx.fillRect(0, 0, width, height);
const renderTask = page.render({
canvasContext: ctx as unknown as CanvasRenderingContext2D,
viewport,
intent: 'display',
});
await renderTask.promise;
return {
width,
height,
png: canvas.toBuffer('image/png'),
};
} finally {
await pdf.destroy().catch(() => undefined);
await loadingTask.destroy().catch(() => undefined);
}
}

View file

@ -0,0 +1,241 @@
import * as ort from 'onnxruntime-node';
import { readFile } from 'fs/promises';
import path from 'path';
import type { LayoutRegion, PdfTextItem } from '@/lib/server/pdf-layout/types';
import { ensureModel } from '@/lib/server/pdf-layout/ensureModel';
interface RunLayoutInput {
pageWidth: number;
pageHeight: number;
textItems: PdfTextItem[];
pagePng: Buffer;
}
const INPUT_SIZE = 640;
const MIN_SCORE = 0.6;
const LABEL_MAP: Record<string, LayoutRegion['label'] | null> = {
caption: 'caption',
footnote: 'footnote',
formula: 'formula',
list_item: 'list-item',
page_footer: 'page-footer',
page_header: 'page-header',
picture: 'picture',
section_header: 'section-header',
table: 'table',
text: 'paragraph',
title: 'title',
document_index: null,
code: null,
checkbox_selected: null,
checkbox_unselected: null,
form: null,
key_value_region: null,
};
const MIN_REGION_SIZE: Partial<Record<LayoutRegion['label'], { minWidth: number; minHeight: number }>> = {
paragraph: { minWidth: 24, minHeight: 14 },
'section-header': { minWidth: 24, minHeight: 14 },
title: { minWidth: 24, minHeight: 14 },
'list-item': { minWidth: 18, minHeight: 12 },
caption: { minWidth: 18, minHeight: 10 },
footnote: { minWidth: 18, minHeight: 10 },
'page-header': { minWidth: 18, minHeight: 10 },
'page-footer': { minWidth: 18, minHeight: 10 },
};
let sessionPromise: Promise<ort.InferenceSession> | null = null;
let idToLabelPromise: Promise<Record<number, string>> | null = null;
let canvasFnsPromise: Promise<{
createCanvasFn: (width: number, height: number) => { getContext: (kind: '2d') => CanvasRenderingContext2D };
loadImageFn: (src: Buffer) => Promise<{ width: number; height: number } & CanvasImageSource>;
}> | null = null;
async function getCanvasFns(): Promise<{
createCanvasFn: (width: number, height: number) => { getContext: (kind: '2d') => CanvasRenderingContext2D };
loadImageFn: (src: Buffer) => Promise<{ width: number; height: number } & CanvasImageSource>;
}> {
if (!canvasFnsPromise) {
canvasFnsPromise = (async () => {
const mod = await import('@napi-rs/canvas');
const namespace = mod as Record<string, unknown>;
const fallback = (namespace.default ?? {}) as Record<string, unknown>;
const createCanvasFn = (namespace.createCanvas ?? fallback.createCanvas) as
| ((width: number, height: number) => { getContext: (kind: '2d') => CanvasRenderingContext2D })
| undefined;
const loadImageFn = (namespace.loadImage ?? fallback.loadImage) as
| ((src: Buffer) => Promise<{ width: number; height: number } & CanvasImageSource>)
| undefined;
if (typeof createCanvasFn !== 'function' || typeof loadImageFn !== 'function') {
throw new Error(
`Canvas runtime missing createCanvas/loadImage exports (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`,
);
}
return { createCanvasFn, loadImageFn };
})();
}
return canvasFnsPromise;
}
async function getSession(): Promise<ort.InferenceSession> {
if (!sessionPromise) {
sessionPromise = (async () => {
const modelPath = await ensureModel();
return ort.InferenceSession.create(modelPath, {
executionProviders: ['cpu'],
graphOptimizationLevel: 'all',
});
})();
}
return sessionPromise;
}
async function getIdToLabel(): Promise<Record<number, string>> {
if (!idToLabelPromise) {
idToLabelPromise = (async () => {
const modelPath = await ensureModel();
const configPath = path.join(path.dirname(modelPath), 'docling-layout-heron.config.json');
const raw = await readFile(configPath, 'utf8');
const parsed = JSON.parse(raw) as { id2label?: Record<string, string> };
const out: Record<number, string> = {};
for (const [key, value] of Object.entries(parsed.id2label ?? {})) {
const n = Number(key);
if (Number.isFinite(n)) out[n] = value;
}
return out;
})();
}
return idToLabelPromise;
}
function preprocessLetterboxed(
image: CanvasImageSource,
createCanvasFn: (width: number, height: number) => { getContext: (kind: '2d') => CanvasRenderingContext2D },
): {
tensor: ort.Tensor;
scale: number;
padX: number;
padY: number;
} {
const sourceWidth = Math.max(1, Number((image as { width?: number }).width ?? INPUT_SIZE));
const sourceHeight = Math.max(1, Number((image as { height?: number }).height ?? INPUT_SIZE));
const scale = Math.min(INPUT_SIZE / sourceWidth, INPUT_SIZE / sourceHeight);
const drawWidth = Math.max(1, Math.round(sourceWidth * scale));
const drawHeight = Math.max(1, Math.round(sourceHeight * scale));
const padX = Math.floor((INPUT_SIZE - drawWidth) / 2);
const padY = Math.floor((INPUT_SIZE - drawHeight) / 2);
const canvas = createCanvasFn(INPUT_SIZE, INPUT_SIZE);
const ctx = canvas.getContext('2d');
ctx.fillStyle = '#ffffff';
ctx.fillRect(0, 0, INPUT_SIZE, INPUT_SIZE);
ctx.imageSmoothingEnabled = true;
ctx.imageSmoothingQuality = 'high';
ctx.drawImage(image, padX, padY, drawWidth, drawHeight);
const imageData = ctx.getImageData(0, 0, INPUT_SIZE, INPUT_SIZE);
const data = new Uint8Array(1 * 3 * INPUT_SIZE * INPUT_SIZE);
for (let y = 0; y < INPUT_SIZE; y += 1) {
for (let x = 0; x < INPUT_SIZE; x += 1) {
const pixelIndex = (y * INPUT_SIZE + x) * 4;
const chwIndex = y * INPUT_SIZE + x;
data[0 * INPUT_SIZE * INPUT_SIZE + chwIndex] = imageData.data[pixelIndex];
data[1 * INPUT_SIZE * INPUT_SIZE + chwIndex] = imageData.data[pixelIndex + 1];
data[2 * INPUT_SIZE * INPUT_SIZE + chwIndex] = imageData.data[pixelIndex + 2];
}
}
return {
tensor: new ort.Tensor('uint8', data, [1, 3, INPUT_SIZE, INPUT_SIZE]),
scale,
padX,
padY,
};
}
function clampBox(
bbox: [number, number, number, number],
pageWidth: number,
pageHeight: number,
): [number, number, number, number] | null {
const x0 = Math.max(0, Math.min(pageWidth, bbox[0]));
const y0 = Math.max(0, Math.min(pageHeight, bbox[1]));
const x1 = Math.max(0, Math.min(pageWidth, bbox[2]));
const y1 = Math.max(0, Math.min(pageHeight, bbox[3]));
if (x1 <= x0 || y1 <= y0) return null;
return [x0, y0, x1, y1];
}
export async function runLayoutModel(input: RunLayoutInput): Promise<LayoutRegion[]> {
const { pageWidth, pageHeight, textItems, pagePng } = input;
if (!textItems.length) return [];
if (!pagePng || pagePng.byteLength === 0) {
throw new Error('layout-render-missing-page-image');
}
try {
const [session, idToLabel, canvasFns] = await Promise.all([getSession(), getIdToLabel(), getCanvasFns()]);
const pageImage = await canvasFns.loadImageFn(pagePng);
const preprocess = preprocessLetterboxed(pageImage, canvasFns.createCanvasFn);
const targetSizes = new ort.Tensor('int64', new BigInt64Array([BigInt(INPUT_SIZE), BigInt(INPUT_SIZE)]), [1, 2]);
const output = await session.run({
images: preprocess.tensor,
orig_target_sizes: targetSizes,
});
const labels = output.labels?.data as BigInt64Array | Int32Array | undefined;
const boxes = output.boxes?.data as Float32Array | undefined;
const scores = output.scores?.data as Float32Array | undefined;
if (!labels || !boxes || !scores) return [];
const regions: LayoutRegion[] = [];
const count = Math.min(scores.length, Math.floor(boxes.length / 4), labels.length);
for (let i = 0; i < count; i += 1) {
const score = scores[i];
if (!Number.isFinite(score) || score < MIN_SCORE) continue;
const rawLabel = Number(labels[i]);
const labelName = idToLabel[rawLabel];
if (!labelName) continue;
const mapped = LABEL_MAP[labelName];
if (!mapped) continue;
const modelBox: [number, number, number, number] = [
boxes[i * 4 + 0],
boxes[i * 4 + 1],
boxes[i * 4 + 2],
boxes[i * 4 + 3],
];
const rawBox: [number, number, number, number] = [
(modelBox[0] - preprocess.padX) / preprocess.scale,
(modelBox[1] - preprocess.padY) / preprocess.scale,
(modelBox[2] - preprocess.padX) / preprocess.scale,
(modelBox[3] - preprocess.padY) / preprocess.scale,
];
const clamped = clampBox(rawBox, pageWidth, pageHeight);
if (!clamped) continue;
const sizeRule = MIN_REGION_SIZE[mapped];
if (sizeRule) {
const width = clamped[2] - clamped[0];
const height = clamped[3] - clamped[1];
if (width < sizeRule.minWidth || height < sizeRule.minHeight) continue;
}
regions.push({
bbox: clamped,
label: mapped,
confidence: score,
});
}
return regions.sort((a, b) => (b.confidence ?? 0) - (a.confidence ?? 0));
} catch (error) {
throw new Error(
`layout-model-inference-failed: ${error instanceof Error ? error.message : String(error)}`,
);
}
}

View file

@ -0,0 +1,80 @@
import type { ParsedPdfDocument, ParsedPdfBlock } from '@/types/parsed-pdf';
function stripTrailingClosers(text: string): string {
return text.trim().replace(/[\"'”’\]\)]+$/g, '');
}
function isSentenceTerminal(text: string): boolean {
return /[.!?]$/.test(stripTrailingClosers(text));
}
function canStitch(a: ParsedPdfBlock, b: ParsedPdfBlock): boolean {
if (!['paragraph', 'list-item'].includes(a.kind)) return false;
if (a.kind !== b.kind) return false;
if (isSentenceTerminal(a.text)) return false;
const next = b.text.trim();
if (!next) return false;
if (/^[A-Z]/.test(next)) return false;
return true;
}
const HARD_BOUNDARY_KINDS = new Set<ParsedPdfBlock['kind']>(['section-header', 'title']);
function findTailCandidateIndex(blocks: ParsedPdfBlock[]): number {
for (let i = blocks.length - 1; i >= 0; i -= 1) {
const block = blocks[i];
if (!block || !block.text.trim()) continue;
if (block.kind === 'paragraph' || block.kind === 'list-item') return i;
}
return -1;
}
function findHeadCandidateIndex(blocks: ParsedPdfBlock[]): number {
for (let i = 0; i < blocks.length; i += 1) {
const block = blocks[i];
if (!block || !block.text.trim()) continue;
if (block.kind === 'paragraph' || block.kind === 'list-item') return i;
}
return -1;
}
function hasHardBoundaryBetween(
pageBlocks: ParsedPdfBlock[],
startInclusive: number,
endExclusive: number,
): boolean {
for (let i = startInclusive; i < endExclusive; i += 1) {
const block = pageBlocks[i];
if (block && HARD_BOUNDARY_KINDS.has(block.kind)) return true;
}
return false;
}
export function stitchCrossPageBlocks(doc: ParsedPdfDocument): ParsedPdfDocument {
const pages = doc.pages.map((page) => ({ ...page, blocks: page.blocks.map((b) => ({ ...b, fragments: b.fragments.map((f) => ({ ...f })) })) }));
for (let i = 0; i < pages.length - 1; i += 1) {
const page = pages[i];
const next = pages[i + 1];
const tailIndex = findTailCandidateIndex(page.blocks);
const headIndex = findHeadCandidateIndex(next.blocks);
if (tailIndex < 0 || headIndex < 0) continue;
if (hasHardBoundaryBetween(page.blocks, tailIndex + 1, page.blocks.length)) continue;
if (hasHardBoundaryBetween(next.blocks, 0, headIndex)) continue;
const tail = page.blocks[tailIndex];
const head = next.blocks[headIndex];
if (!tail || !head) continue;
if (!canStitch(tail, head)) continue;
tail.fragments.push(...head.fragments);
tail.text = `${tail.text} ${head.text}`.replace(/\s+/g, ' ').trim();
next.blocks.splice(headIndex, 1);
}
return {
...doc,
pages,
};
}

View file

@ -0,0 +1,15 @@
import type { ParsedPdfBlockKind } from '@/types/parsed-pdf';
export interface PdfTextItem {
text: string;
x: number;
y: number;
width: number;
height: number;
}
export interface LayoutRegion {
bbox: [number, number, number, number];
label: ParsedPdfBlockKind;
confidence?: number;
}

View file

@ -6,8 +6,11 @@ import {
type RuntimeConfigKey,
type RuntimeConfigSource,
} from '@/lib/server/admin/settings';
import { isComputeAvailable } from '@/lib/server/compute';
export type ResolvedRuntimeConfig = RuntimeConfig;
export type ResolvedRuntimeConfig = RuntimeConfig & {
computeAvailable: boolean;
};
function assertServerRuntime(caller: string): void {
if (typeof window !== 'undefined') {
@ -23,7 +26,11 @@ function assertServerRuntime(caller: string): void {
export async function getResolvedRuntimeConfig(): Promise<ResolvedRuntimeConfig> {
assertServerRuntime('getResolvedRuntimeConfig');
await ensureAdminSeed();
return getRuntimeConfig();
const values = await getRuntimeConfig();
return {
...values,
computeAvailable: isComputeAvailable(),
};
}
export async function getResolvedRuntimeConfigWithSources(): Promise<{

View file

@ -90,6 +90,8 @@ export function getS3Client(): S3Client {
region: config.region,
endpoint: config.endpoint,
forcePathStyle: config.forcePathStyle,
requestChecksumCalculation: 'WHEN_REQUIRED',
responseChecksumValidation: 'WHEN_REQUIRED',
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,
@ -111,6 +113,8 @@ export function getS3ProxyClient(): S3Client {
region: config.region,
endpoint: loopbackEndpoint(config.endpoint),
forcePathStyle: config.forcePathStyle,
requestChecksumCalculation: 'WHEN_REQUIRED',
responseChecksumValidation: 'WHEN_REQUIRED',
credentials: {
accessKeyId: config.accessKeyId,
secretAccessKey: config.secretAccessKey,

View file

@ -0,0 +1,82 @@
import { and, eq } from 'drizzle-orm';
import { db } from '@/db';
import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema';
import { deleteTtsSegmentAudioObjects } from '@/lib/server/tts/segments-blobstore';
import type { ReaderType } from '@/types/user-state';
type ClearTtsSegmentCacheInput = {
userId: string;
documentId: string;
documentVersion?: number;
readerType?: ReaderType;
};
export type ClearTtsSegmentCacheResult = {
deletedSegments: number;
requestedAudioObjects: number;
deletedAudioObjects: number;
warning?: string;
};
export async function clearTtsSegmentCache(
input: ClearTtsSegmentCacheInput,
): Promise<ClearTtsSegmentCacheResult> {
const conditions = [
eq(ttsSegmentEntries.userId, input.userId),
eq(ttsSegmentEntries.documentId, input.documentId),
];
if (typeof input.documentVersion === 'number' && Number.isFinite(input.documentVersion)) {
conditions.push(eq(ttsSegmentEntries.documentVersion, Math.floor(input.documentVersion)));
}
if (input.readerType) {
conditions.push(eq(ttsSegmentEntries.readerType, input.readerType));
}
const rows = (await db
.select({
segmentId: ttsSegmentVariants.segmentId,
audioKey: ttsSegmentVariants.audioKey,
})
.from(ttsSegmentVariants)
.innerJoin(
ttsSegmentEntries,
and(
eq(ttsSegmentEntries.segmentEntryId, ttsSegmentVariants.segmentEntryId),
eq(ttsSegmentEntries.userId, ttsSegmentVariants.userId),
),
)
.where(and(...conditions))) as Array<{ segmentId: string; audioKey: string | null }>;
await db.delete(ttsSegmentEntries).where(and(...conditions));
const audioKeys = rows
.map((row) => row.audioKey)
.filter((key): key is string => Boolean(key));
const uniqueAudioKeys = Array.from(new Set(audioKeys));
let deletedAudioObjects = 0;
let warning: string | undefined;
if (uniqueAudioKeys.length > 0) {
try {
deletedAudioObjects = await deleteTtsSegmentAudioObjects(uniqueAudioKeys);
if (deletedAudioObjects < uniqueAudioKeys.length) {
warning = `Deleted ${deletedAudioObjects} of ${uniqueAudioKeys.length} audio objects.`;
}
} catch (error) {
warning = error instanceof Error ? error.message : 'Failed deleting some audio objects';
console.warn('Failed clearing some TTS segment audio objects:', {
documentId: input.documentId,
userId: input.userId,
error: warning,
});
}
}
return {
deletedSegments: rows.length,
requestedAudioObjects: uniqueAudioKeys.length,
deletedAudioObjects,
...(warning ? { warning } : {}),
};
}

View file

@ -81,6 +81,9 @@ export function normalizeLocator(locator: TTSSegmentLocator | undefined): TTSSeg
return {
readerType: 'pdf',
page: Math.max(1, Math.floor(locator.page)),
...(typeof locator.blockId === 'string' && locator.blockId.trim()
? { blockId: locator.blockId.trim() }
: {}),
};
}
if (locator.readerType === 'html') {

View file

@ -0,0 +1,76 @@
import {
DEFAULT_DOCUMENT_SETTINGS,
type DocumentSettings,
} from '@/types/document-settings';
import type { ParsedPdfBlockKind } from '@/types/parsed-pdf';
const PDF_BLOCK_KINDS: ParsedPdfBlockKind[] = [
'title',
'section-header',
'paragraph',
'list-item',
'caption',
'table',
'picture',
'page-header',
'page-footer',
'footnote',
'formula',
];
function normalizeSkipKinds(value: unknown): ParsedPdfBlockKind[] {
if (!Array.isArray(value)) return [...(DEFAULT_DOCUMENT_SETTINGS.pdf?.skipBlockKinds ?? [])];
const allow = new Set(PDF_BLOCK_KINDS);
const out: ParsedPdfBlockKind[] = [];
for (const entry of value) {
if (typeof entry !== 'string') continue;
if (!allow.has(entry as ParsedPdfBlockKind)) continue;
out.push(entry as ParsedPdfBlockKind);
}
return Array.from(new Set(out));
}
type PdfMargins = { header: number; footer: number; left: number; right: number };
function normalizeMargins(value: unknown): PdfMargins | undefined {
if (!value || typeof value !== 'object') return undefined;
const rec = value as Record<string, unknown>;
const header = Number(rec.header);
const footer = Number(rec.footer);
const left = Number(rec.left);
const right = Number(rec.right);
if (![header, footer, left, right].every((n) => Number.isFinite(n))) return undefined;
return { header, footer, left, right };
}
export function mergeDocumentSettings(
defaults: DocumentSettings = DEFAULT_DOCUMENT_SETTINGS,
stored: unknown,
): DocumentSettings {
const base: DocumentSettings = {
schemaVersion: 1,
pdf: {
skipBlockKinds: [...(defaults.pdf?.skipBlockKinds ?? [])],
chaptersFromSections: defaults.pdf?.chaptersFromSections ?? true,
...(defaults.pdf?.margins ? { margins: defaults.pdf.margins } : {}),
},
};
if (!stored || typeof stored !== 'object') return base;
const rec = stored as Record<string, unknown>;
const pdf = rec.pdf;
if (!pdf || typeof pdf !== 'object') return base;
const pdfRec = pdf as Record<string, unknown>;
return {
schemaVersion: 1,
pdf: {
skipBlockKinds: normalizeSkipKinds(pdfRec.skipBlockKinds),
chaptersFromSections:
typeof pdfRec.chaptersFromSections === 'boolean'
? pdfRec.chaptersFromSections
: (base.pdf?.chaptersFromSections ?? true),
...(normalizeMargins(pdfRec.margins) ? { margins: normalizeMargins(pdfRec.margins) } : {}),
},
};
}

View file

@ -70,7 +70,10 @@ export function locatorIdentityKey(locator: TTSSegmentLocator | null): string {
return `epub:${locator.spineIndex}:${locator.spineHref}:${locator.charOffset}`;
}
if (isPdfLocator(locator)) {
return `pdf:${Math.floor(locator.page)}`;
const blockPart = typeof locator.blockId === 'string' && locator.blockId.trim()
? `:${locator.blockId.trim()}`
: '';
return `pdf:${Math.floor(locator.page)}${blockPart}`;
}
if (isHtmlLocator(locator)) {
return `html:${locator.location}`;
@ -112,7 +115,11 @@ export function compareSegmentLocators(
return a.spineHref.localeCompare(b.spineHref);
}
if (isPdfLocator(a) && isPdfLocator(b)) {
return Math.floor(a.page) - Math.floor(b.page);
const pageCmp = Math.floor(a.page) - Math.floor(b.page);
if (pageCmp !== 0) return pageCmp;
const aBlock = typeof a.blockId === 'string' ? a.blockId : '';
const bBlock = typeof b.blockId === 'string' ? b.blockId : '';
return aBlock.localeCompare(bBlock);
}
if (isHtmlLocator(a) && isHtmlLocator(b)) {
// When both locations look like positive integers (HTML reader blocks),

View file

@ -37,6 +37,7 @@ export interface CanonicalTtsSegmentPlanOptions {
readerType?: ReaderType;
maxBlockLength?: number;
keyPrefix?: string;
enforceSourceBoundaries?: boolean;
}
interface PreparedSourceUnit {
@ -185,6 +186,7 @@ export function planCanonicalTtsSegments(
options: CanonicalTtsSegmentPlanOptions = {},
): CanonicalTtsSegmentPlan {
const readerType = options.readerType ?? 'pdf';
const enforceSourceBoundaries = Boolean(options.enforceSourceBoundaries);
const keyPrefix = options.keyPrefix ?? TTS_SEGMENT_PLAN_VERSION;
const preparedSources: PreparedSourceUnit[] = [];
const textParts: string[] = [];
@ -263,6 +265,41 @@ export function planCanonicalTtsSegments(
spansSourceBoundary: false,
});
} else {
const splitAcrossSourceBoundaries = () => {
const ownerIdx = preparedSources.indexOf(ownerSource);
const endIdx = preparedSources.indexOf(endSource);
if (ownerIdx < 0 || endIdx < 0) return;
let subStart = startOffset;
for (let srcIdx = ownerIdx; srcIdx <= endIdx; srcIdx += 1) {
const source = preparedSources[srcIdx];
const subEnd = srcIdx < endIdx ? source.endOffset : endOffset;
if (subEnd <= subStart) continue;
const subText = canonicalText.slice(subStart, subEnd).trim();
if (!subText) {
subStart = subEnd;
continue;
}
const nextSource = srcIdx < endIdx ? preparedSources[srcIdx + 1] : null;
const subStartAnchor = anchorForOffset(source, subStart);
const subEndAnchor = anchorForOffset(source, subEnd);
const ordinal = segments.length;
segments.push({
key: buildSegmentKey(keyPrefix, subText),
ordinal,
text: subText,
ownerSourceKey: source.sourceKey,
ownerLocator: source.locator,
startAnchor: subStartAnchor,
endAnchor: subEndAnchor,
spansSourceBoundary: false,
});
subStart = nextSource ? nextSource.startOffset : subEnd;
}
};
// Block spans one or more source boundaries. Decide whether to keep it
// as a single boundary-spanning segment or to split it at each boundary.
//
@ -286,6 +323,12 @@ export function planCanonicalTtsSegments(
// continuation, so it should be filtered out of the current page's
// segments.
if (ownerSource.locator !== null) {
if (enforceSourceBoundaries) {
// PDF source units are logical layout blocks; never allow a segment
// to cross a block boundary.
splitAcrossSourceBoundaries();
continue;
}
// Both sources carry locators → original unified boundary behavior.
const ordinal = segments.length;
const startAnchor = anchorForOffset(ownerSource, startOffset);
@ -318,34 +361,7 @@ export function planCanonicalTtsSegments(
if (isCleanBoundary) {
// Clean boundary → split at each source boundary.
let subStart = startOffset;
for (let srcIdx = ownerIdx; srcIdx <= endIdx; srcIdx += 1) {
const source = preparedSources[srcIdx];
const subEnd = srcIdx < endIdx ? source.endOffset : endOffset;
if (subEnd <= subStart) continue;
const subText = canonicalText.slice(subStart, subEnd).trim();
if (!subText) {
subStart = subEnd;
continue;
}
const nextSource = srcIdx < endIdx ? preparedSources[srcIdx + 1] : null;
const subStartAnchor = anchorForOffset(source, subStart);
const subEndAnchor = anchorForOffset(source, subEnd);
const ordinal = segments.length;
segments.push({
key: buildSegmentKey(keyPrefix, subText),
ordinal,
text: subText,
ownerSourceKey: source.sourceKey,
ownerLocator: source.locator,
startAnchor: subStartAnchor,
endAnchor: subEndAnchor,
spansSourceBoundary: false,
});
subStart = nextSource ? nextSource.startOffset : subEnd;
}
splitAcrossSourceBoundaries();
} else {
// Overlapping sentence → keep unified, owned by the context-only
// source. This segment will be filtered out of the current page's

View file

@ -107,6 +107,8 @@ export interface TTSSegmentLocator {
readerType?: TTSReaderType;
// PDF / legacy
page?: number;
// PDF block-level locator (structured parser path)
blockId?: string;
// HTML / legacy EPUB CFI (kept for in-flight drafts; not persisted for EPUB)
location?: string;
// Stable EPUB coordinates

View file

@ -8,15 +8,6 @@ import { defaultModelForProviderType } from '@/lib/shared/tts-provider-policy';
// `window.__OPENREADER_RUNTIME_CONFIG__` (SSR-injected) on the client, and
// from the built-in defaults during SSR.
function readRuntimeFlag(key: string, defaultValue: boolean): boolean {
if (typeof window === 'undefined') return defaultValue;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const injected = (window as any).__OPENREADER_RUNTIME_CONFIG__;
if (!injected || typeof injected !== 'object') return defaultValue;
const value = injected[key];
return typeof value === 'boolean' ? value : defaultValue;
}
function readRuntimeString(key: string, defaultValue: string): string {
if (typeof window === 'undefined') return defaultValue;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
@ -96,7 +87,6 @@ export interface AppConfigValues {
* that reads through it.
*/
export function getAppConfigDefaults(): AppConfigValues {
const wordHighlightEnabledByDefault = readRuntimeFlag('enableWordHighlight', true);
const runtimeProviderRef = readRuntimeString('defaultTtsProvider', 'custom-openai');
const defaultProviderRef = runtimeProviderRef.trim();
const defaultProviderType = isBuiltInTtsProviderId(defaultProviderRef) ? defaultProviderRef : 'unknown';
@ -126,11 +116,11 @@ export function getAppConfigDefaults(): AppConfigValues {
segmentPreloadSentenceLookahead: 3,
ttsSegmentMaxBlockLength: 450,
pdfHighlightEnabled: true,
pdfWordHighlightEnabled: wordHighlightEnabledByDefault,
pdfWordHighlightEnabled: true,
epubHighlightEnabled: true,
epubWordHighlightEnabled: wordHighlightEnabledByDefault,
epubWordHighlightEnabled: true,
htmlHighlightEnabled: true,
htmlWordHighlightEnabled: wordHighlightEnabledByDefault,
htmlWordHighlightEnabled: true,
firstVisit: false,
documentListState: {
sortBy: 'name',

View file

@ -0,0 +1,18 @@
import type { ParsedPdfBlockKind } from '@/types/parsed-pdf';
export interface DocumentSettings {
schemaVersion: 1;
pdf?: {
skipBlockKinds: ParsedPdfBlockKind[];
margins?: { header: number; footer: number; left: number; right: number };
chaptersFromSections: boolean;
};
}
export const DEFAULT_DOCUMENT_SETTINGS: DocumentSettings = {
schemaVersion: 1,
pdf: {
skipBlockKinds: ['page-header', 'page-footer', 'footnote'],
chaptersFromSections: true,
},
};

View file

@ -6,6 +6,8 @@ export interface BaseDocument {
size: number;
lastModified: number;
type: DocumentType;
parseStatus?: 'pending' | 'running' | 'ready' | 'failed' | 'unsupported' | null;
parsedJsonKey?: string | null;
scope?: 'user' | 'unclaimed';
folderId?: string;
isConverting?: boolean;

45
src/types/parsed-pdf.ts Normal file
View file

@ -0,0 +1,45 @@
export type ParsedPdfBlockKind =
| 'title'
| 'section-header'
| 'paragraph'
| 'list-item'
| 'caption'
| 'table'
| 'picture'
| 'page-header'
| 'page-footer'
| 'footnote'
| 'formula';
export interface ParsedPdfBlockFragment {
page: number;
bbox: [number, number, number, number];
text: string;
readingOrder: number;
modelConfidence?: number;
}
export interface ParsedPdfBlock {
id: string;
kind: ParsedPdfBlockKind;
fragments: ParsedPdfBlockFragment[];
text: string;
parentSectionId?: string;
}
export interface ParsedPdfPage {
pageNumber: number;
width: number;
height: number;
blocks: ParsedPdfBlock[];
}
export interface ParsedPdfDocument {
schemaVersion: 1;
documentId: string;
parserVersion: string;
parsedAt: number;
pages: ParsedPdfPage[];
}
export type PdfParseStatus = 'pending' | 'running' | 'ready' | 'failed' | 'unsupported';

View file

@ -0,0 +1,77 @@
import { expect, test } from '@playwright/test';
import { createPdfAudiobookSourceAdapter } from '../../src/lib/client/audiobooks/adapters/pdf';
import type { ParsedPdfDocument } from '../../src/types/parsed-pdf';
import type { DocumentSettings } from '../../src/types/document-settings';
test.describe('pdf audiobook adapter', () => {
test('builds chapters from section headers and filters skipped kinds', async () => {
const parsed: ParsedPdfDocument = {
schemaVersion: 1,
documentId: 'doc-1',
parserVersion: 'test',
parsedAt: Date.now(),
pages: [
{
pageNumber: 1,
width: 100,
height: 100,
blocks: [
{
id: 'b1',
kind: 'section-header',
text: 'Intro',
fragments: [{ page: 1, bbox: [0, 80, 100, 90], text: 'Intro', readingOrder: 0 }],
},
{
id: 'b2',
kind: 'paragraph',
text: 'Welcome text.',
fragments: [{ page: 1, bbox: [0, 60, 100, 79], text: 'Welcome text.', readingOrder: 1 }],
},
{
id: 'b3',
kind: 'page-header',
text: 'Header line',
fragments: [{ page: 1, bbox: [0, 95, 100, 100], text: 'Header line', readingOrder: 2 }],
},
{
id: 'b4',
kind: 'section-header',
text: 'Second',
fragments: [{ page: 1, bbox: [0, 40, 100, 50], text: 'Second', readingOrder: 3 }],
},
{
id: 'b5',
kind: 'paragraph',
text: 'More body.',
fragments: [{ page: 1, bbox: [0, 20, 100, 39], text: 'More body.', readingOrder: 4 }],
},
],
},
],
};
const settings: DocumentSettings = {
schemaVersion: 1,
pdf: {
skipBlockKinds: ['page-header'],
chaptersFromSections: true,
},
};
const adapter = createPdfAudiobookSourceAdapter({
parsed,
settings,
margins: { header: 0.07, footer: 0.07, left: 0.07, right: 0.07 },
smartSentenceSplitting: false,
});
const chapters = await adapter.prepareChapters();
expect(chapters).toHaveLength(2);
expect(chapters[0].title).toBe('Intro');
expect(chapters[0].text).toContain('Welcome text.');
expect(chapters[0].text).not.toContain('Header line');
expect(chapters[1].title).toBe('Second');
expect(chapters[1].text).toContain('More body.');
});
});

View file

@ -0,0 +1,36 @@
import { expect, test } from '@playwright/test';
import { buildPageTextFromBlocks } from '../../src/lib/client/pdf-block-text';
import type { ParsedPdfPage } from '../../src/types/parsed-pdf';
test.describe('buildPageTextFromBlocks', () => {
test('filters skipped kinds and preserves reading order', () => {
const page: ParsedPdfPage = {
pageNumber: 1,
width: 100,
height: 100,
blocks: [
{
id: 'b2',
kind: 'page-header',
text: 'Copyright Header',
fragments: [{ page: 1, bbox: [0, 90, 100, 100], text: 'Copyright Header', readingOrder: 0 }],
},
{
id: 'b1',
kind: 'paragraph',
text: 'Body text',
fragments: [{ page: 1, bbox: [0, 20, 100, 80], text: 'Body text', readingOrder: 1 }],
},
{
id: 'b3',
kind: 'caption',
text: 'Figure caption',
fragments: [{ page: 1, bbox: [0, 5, 100, 19], text: 'Figure caption', readingOrder: 2 }],
},
],
};
expect(buildPageTextFromBlocks(page, ['page-header'])).toBe('Body text Figure caption');
expect(buildPageTextFromBlocks(page, ['page-header', 'caption'])).toBe('Body text');
});
});

View file

@ -0,0 +1,23 @@
import { expect, test } from '@playwright/test';
import { mergeTextWithRegions } from '../../src/lib/server/pdf-layout/mergeTextWithRegions';
test.describe('mergeTextWithRegions', () => {
test('assigns text items to containing regions by centroid and joins in reading order', () => {
const regions = [
{ bbox: [0, 0, 100, 50] as [number, number, number, number], label: 'paragraph' as const },
{ bbox: [0, 50, 100, 100] as [number, number, number, number], label: 'caption' as const },
];
const textItems = [
{ text: 'world', x: 40, y: 20, width: 20, height: 8 },
{ text: 'hello', x: 10, y: 20, width: 20, height: 8 },
{ text: 'Figure', x: 10, y: 70, width: 24, height: 8 },
{ text: '1.2', x: 40, y: 70, width: 10, height: 8 },
];
const merged = mergeTextWithRegions(regions, textItems);
expect(merged).toHaveLength(2);
expect(merged[0].text).toBe('hello world');
expect(merged[1].text).toBe('Figure 1.2');
});
});

View file

@ -0,0 +1,91 @@
import { expect, test } from '@playwright/test';
import { stitchCrossPageBlocks } from '../../src/lib/server/pdf-layout/stitchCrossPageBlocks';
import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../src/types/parsed-pdf';
function makeBlock(
id: string,
kind: ParsedPdfBlockKind,
text: string,
page: number,
readingOrder: number,
): ParsedPdfBlock {
return {
id,
kind,
text,
fragments: [{
page,
bbox: [0, 0, 100, 10],
text,
readingOrder,
}],
};
}
function makeDoc(page1Blocks: ParsedPdfBlock[], page2Blocks: ParsedPdfBlock[]): ParsedPdfDocument {
return {
schemaVersion: 1,
documentId: 'doc',
parserVersion: 'test',
parsedAt: 0,
pages: [
{ pageNumber: 1, width: 100, height: 100, blocks: page1Blocks },
{ pageNumber: 2, width: 100, height: 100, blocks: page2Blocks },
],
};
}
test.describe('stitchCrossPageBlocks', () => {
test('stitches paragraph continuation across footer/header noise', () => {
const doc = makeDoc(
[
makeBlock('b1', 'paragraph', 'This sentence continues', 1, 0),
makeBlock('b2', 'page-footer', 'Footer text', 1, 1),
],
[
makeBlock('b3', 'page-header', 'Header text', 2, 0),
makeBlock('b4', 'paragraph', 'into the next page.', 2, 1),
],
);
const stitched = stitchCrossPageBlocks(doc);
const page1 = stitched.pages[0];
const page2 = stitched.pages[1];
expect(page1?.blocks[0]?.text).toBe('This sentence continues into the next page.');
expect(page1?.blocks[0]?.fragments).toHaveLength(2);
expect(page2?.blocks.map((b) => b.id)).toEqual(['b3']);
});
test('does not stitch across section-header boundary', () => {
const doc = makeDoc(
[
makeBlock('b1', 'paragraph', 'This sentence continues', 1, 0),
],
[
makeBlock('b2', 'section-header', '2 New Section', 2, 0),
makeBlock('b3', 'paragraph', 'into the next page.', 2, 1),
],
);
const stitched = stitchCrossPageBlocks(doc);
expect(stitched.pages[0]?.blocks[0]?.fragments).toHaveLength(1);
expect(stitched.pages[1]?.blocks.map((b) => b.id)).toEqual(['b2', 'b3']);
});
test('does not stitch when tail has sentence terminal', () => {
const doc = makeDoc(
[
makeBlock('b1', 'paragraph', 'This sentence is complete.', 1, 0),
],
[
makeBlock('b2', 'paragraph', 'next sentence starts here', 2, 0),
],
);
const stitched = stitchCrossPageBlocks(doc);
expect(stitched.pages[0]?.blocks[0]?.fragments).toHaveLength(1);
expect(stitched.pages[1]?.blocks.map((b) => b.id)).toEqual(['b2']);
});
});

View file

@ -20,6 +20,8 @@ test.describe('transferUserDocuments', () => {
size INTEGER NOT NULL,
last_modified INTEGER NOT NULL,
file_path TEXT NOT NULL,
parse_status TEXT,
parsed_json_key TEXT,
created_at INTEGER,
PRIMARY KEY (id, user_id)
);