feat(build): improve Docker image layering and embed migration/worker tools
Refactor Dockerfile to optimize layering, embed entrypoint migration tools and compute worker as deployable bundles, and merge only required dependencies for runtime scripts. Add docker/entrypoint-migration-tools package and update workspace configuration. Refactor PDF parsing to resolve standard font data path robustly at runtime. Update dependencies and output config for Next.js standalone mode. Enhance entrypoint script to launch embedded compute worker from deployed bundle. - Dockerfile now deploys migration tools and compute worker as separate bundles - compute/core/pdf uses new pdfjs-runtime utility for robust font path resolution - docker/entrypoint-migration-tools added for migration script dependencies - scripts/openreader-entrypoint.mjs launches embedded worker from deployed bundle - next.config.ts enables standalone output and includes standard_fonts in tracing - Dependency updates across package.json and compute/worker/package.json BREAKING CHANGE: Docker image structure and runtime entrypoint logic have changed; custom deployment scripts or Docker overrides may require updates.
This commit is contained in:
Richard R
parent
4317ef03f5
commit
1074a60462
11 changed files with 1103 additions and 2156 deletions
39
Dockerfile
39
Dockerfile
|
|
@ -9,7 +9,6 @@ RUN cp "$(command -v weed)" /tmp/weed && \
|
|||
FROM nats:2.11-alpine AS nats-builder
|
||||
RUN cp "$(command -v nats-server)" /tmp/nats-server
|
||||
|
||||
|
||||
# Stage 2: build the Next.js app
|
||||
FROM node:lts-slim AS app-builder
|
||||
|
||||
|
|
@ -23,6 +22,7 @@ WORKDIR /app
|
|||
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
|
||||
COPY compute/core/package.json ./compute/core/package.json
|
||||
COPY compute/worker/package.json ./compute/worker/package.json
|
||||
COPY docker/entrypoint-migration-tools/package.json ./docker/entrypoint-migration-tools/package.json
|
||||
|
||||
# Install dependencies
|
||||
RUN pnpm install --frozen-lockfile
|
||||
|
|
@ -33,6 +33,8 @@ COPY . .
|
|||
# Build the Next.js application
|
||||
RUN pnpm exec next telemetry disable
|
||||
RUN AUTH_SECRET=build-placeholder-secret-value-32chars!! BASE_URL=http://localhost:3003 pnpm build
|
||||
RUN pnpm --config.inject-workspace-packages=true --filter @openreader/entrypoint-migration-tools deploy /opt/entrypoint-migration-tools
|
||||
RUN pnpm --config.inject-workspace-packages=true --filter @openreader/compute-worker deploy /opt/embedded-compute-worker
|
||||
# Generate third-party dependency license report plus copied license files.
|
||||
RUN mkdir -p /app/THIRD_PARTY_LICENSES && \
|
||||
pnpm dlx license-checker-rseidelsohn@4.3.0 \
|
||||
|
|
@ -53,14 +55,33 @@ RUN apt-get update && \
|
|||
apt-get install -y --no-install-recommends ca-certificates libreoffice-writer && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install pnpm for runtime process commands.
|
||||
RUN npm install -g pnpm@10.33.4
|
||||
|
||||
# App runtime directory
|
||||
WORKDIR /app
|
||||
|
||||
# Copy built app and runtime files from the builder stage (non-standalone runtime).
|
||||
COPY --from=app-builder /app ./
|
||||
# Copy only the standalone Next runtime and assets.
|
||||
COPY --from=app-builder /app/.next/standalone ./
|
||||
COPY --from=app-builder /app/.next/static ./.next/static
|
||||
COPY --from=app-builder /app/public ./public
|
||||
|
||||
# Copy the entrypoint and migration/runtime helper files it invokes directly.
|
||||
COPY --from=app-builder /app/scripts/openreader-entrypoint.mjs ./scripts/openreader-entrypoint.mjs
|
||||
COPY --from=app-builder /app/scripts/migrate-fs-v2.mjs ./scripts/migrate-fs-v2.mjs
|
||||
COPY --from=app-builder /app/drizzle ./drizzle
|
||||
COPY --from=app-builder /app/drizzle.config.pg.ts ./drizzle.config.pg.ts
|
||||
COPY --from=app-builder /app/drizzle.config.sqlite.ts ./drizzle.config.sqlite.ts
|
||||
COPY --from=app-builder /app/src/db ./src/db
|
||||
|
||||
# Merge in the dependency subset needed by the entrypoint migration scripts.
|
||||
COPY --from=app-builder /opt/entrypoint-migration-tools/node_modules /tmp/runtime-tools-node_modules
|
||||
RUN mkdir -p /app/node_modules && \
|
||||
rm -rf /tmp/runtime-tools-node_modules/@aws-sdk \
|
||||
/tmp/runtime-tools-node_modules/better-sqlite3 \
|
||||
/tmp/runtime-tools-node_modules/pg && \
|
||||
cp -an /tmp/runtime-tools-node_modules/. /app/node_modules/ && \
|
||||
rm -rf /tmp/runtime-tools-node_modules
|
||||
|
||||
# Ship the embedded compute worker as a separate deployed bundle.
|
||||
COPY --from=app-builder /opt/embedded-compute-worker ./embedded-compute-worker
|
||||
# Include third-party license report and copied license texts at a stable path in the image.
|
||||
COPY --from=app-builder /app/THIRD_PARTY_LICENSES /licenses
|
||||
# Include SeaweedFS license text for the copied weed binary.
|
||||
|
|
@ -78,9 +99,13 @@ RUN chmod +x /usr/local/bin/nats-server
|
|||
# Include OpenAI Whisper license text for runtime-downloaded ONNX artifacts.
|
||||
COPY --from=app-builder /app/compute/core/src/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt
|
||||
|
||||
# Match the app's historical container port now that standalone server.js
|
||||
# is started directly instead of `next start -p 3003`.
|
||||
ENV PORT=3003
|
||||
|
||||
# Expose the port the app runs on
|
||||
EXPOSE 3003
|
||||
|
||||
# Start the application
|
||||
ENTRYPOINT ["node", "scripts/openreader-entrypoint.mjs", "--"]
|
||||
CMD ["pnpm", "start:raw"]
|
||||
CMD ["node", "server.js"]
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import path from 'path';
|
||||
import type { TextItem } from 'pdfjs-dist/types/src/display/api';
|
||||
import type { ParsedPdfDocument, ParsedPdfPage } from '../types/parsed-pdf';
|
||||
import { ensureModel } from './model';
|
||||
import { runLayoutModel } from './runLayoutModel';
|
||||
import { mergeTextWithRegions } from './merge';
|
||||
import { resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
|
||||
import { stitchCrossPageBlocks } from './stitch';
|
||||
import { renderPage } from './render';
|
||||
import { normalizeTextItemsForLayout } from './normalize-text';
|
||||
|
|
@ -24,11 +24,6 @@ interface ParsePdfInput {
|
|||
|
||||
const LAYOUT_RENDER_SCALE = 1.5;
|
||||
|
||||
function resolvePdfjsStandardFontDataUrl(): string {
|
||||
const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts');
|
||||
return `${standardFontDir.replace(/\/?$/, '/')}`;
|
||||
}
|
||||
|
||||
export async function parsePdf(input: ParsePdfInput): Promise<ParsedPdfDocument> {
|
||||
await ensureModel();
|
||||
|
||||
|
|
|
|||
22
compute/core/src/pdf/pdfjs-runtime.ts
Normal file
22
compute/core/src/pdf/pdfjs-runtime.ts
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createRequire } from 'node:module';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
|
||||
let cachedStandardFontDataUrl: string | null = null;
|
||||
|
||||
export function resolvePdfjsStandardFontDataUrl(): string {
|
||||
if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl;
|
||||
|
||||
const pdfjsEntry = require.resolve('pdfjs-dist/legacy/build/pdf.mjs');
|
||||
const pdfjsPackageRoot = path.resolve(path.dirname(pdfjsEntry), '..', '..');
|
||||
const standardFontDir = path.join(pdfjsPackageRoot, 'standard_fonts');
|
||||
|
||||
if (!fs.existsSync(standardFontDir)) {
|
||||
throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`);
|
||||
}
|
||||
|
||||
cachedStandardFontDataUrl = `${standardFontDir.replace(/\/?$/, '/')}`;
|
||||
return cachedStandardFontDataUrl;
|
||||
}
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
import path from 'path';
|
||||
import type { Canvas } from '@napi-rs/canvas';
|
||||
import { resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime';
|
||||
|
||||
type CanvasRuntime = {
|
||||
DOMMatrixCtor: unknown;
|
||||
|
|
@ -9,11 +9,6 @@ type CanvasRuntime = {
|
|||
|
||||
let canvasRuntimePromise: Promise<CanvasRuntime> | null = null;
|
||||
|
||||
function resolvePdfjsStandardFontDataUrl(): string {
|
||||
const standardFontDir = path.join(process.cwd(), 'node_modules', 'pdfjs-dist', 'standard_fonts');
|
||||
return `${standardFontDir.replace(/\/?$/, '/')}`;
|
||||
}
|
||||
|
||||
async function loadCanvasRuntime(): Promise<CanvasRuntime> {
|
||||
if (!canvasRuntimePromise) {
|
||||
canvasRuntimePromise = (async () => {
|
||||
|
|
|
|||
|
|
@ -8,15 +8,15 @@
|
|||
"start": "tsx src/server.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.1050.0",
|
||||
"@aws-sdk/client-s3": "^3.1061.0",
|
||||
"@nats-io/jetstream": "^3.4.0",
|
||||
"@nats-io/kv": "^3.4.0",
|
||||
"@nats-io/transport-node": "^3.4.0",
|
||||
"@openreader/compute-core": "workspace:*",
|
||||
"fastify": "^5.6.2",
|
||||
"fastify": "^5.8.5",
|
||||
"pino": "^10.3.1",
|
||||
"pino-pretty": "^13.1.2",
|
||||
"tsx": "^4.22.3",
|
||||
"zod": "^4.1.12"
|
||||
"pino-pretty": "^13.1.3",
|
||||
"tsx": "^4.22.4",
|
||||
"zod": "^4.4.3"
|
||||
}
|
||||
}
|
||||
|
|
|
|||
14
docker/entrypoint-migration-tools/package.json
Normal file
14
docker/entrypoint-migration-tools/package.json
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"name": "@openreader/entrypoint-migration-tools",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.1061.0",
|
||||
"better-sqlite3": "^12.10.0",
|
||||
"dotenv": "^17.4.2",
|
||||
"drizzle-kit": "^0.31.10",
|
||||
"drizzle-orm": "^0.45.2",
|
||||
"ffmpeg-static": "^5.3.0",
|
||||
"pg": "^8.21.0"
|
||||
}
|
||||
}
|
||||
|
|
@ -24,6 +24,7 @@ const serverExternalPackages = [
|
|||
];
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
output: 'standalone',
|
||||
async headers() {
|
||||
return [
|
||||
{
|
||||
|
|
@ -52,12 +53,15 @@ const nextConfig: NextConfig = {
|
|||
],
|
||||
'/api/documents/blob/preview/ensure': [
|
||||
'./node_modules/pdfjs-dist/legacy/build/pdf.worker.mjs',
|
||||
'./node_modules/pdfjs-dist/standard_fonts/**/*',
|
||||
],
|
||||
'/api/documents/blob/preview/presign': [
|
||||
'./node_modules/pdfjs-dist/legacy/build/pdf.worker.mjs',
|
||||
'./node_modules/pdfjs-dist/standard_fonts/**/*',
|
||||
],
|
||||
'/api/documents/blob/preview/fallback': [
|
||||
'./node_modules/pdfjs-dist/legacy/build/pdf.worker.mjs',
|
||||
'./node_modules/pdfjs-dist/standard_fonts/**/*',
|
||||
],
|
||||
},
|
||||
outputFileTracingExcludes: {
|
||||
|
|
|
|||
39
package.json
39
package.json
|
|
@ -31,21 +31,21 @@
|
|||
"lint:route-errors": "node scripts/check-route-error-responses.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@aws-sdk/client-s3": "^3.1045.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.1045.0",
|
||||
"@aws-sdk/client-s3": "^3.1061.0",
|
||||
"@aws-sdk/s3-request-presigner": "^3.1061.0",
|
||||
"@headlessui/react": "^2.2.10",
|
||||
"@huggingface/tokenizers": "^0.1.3",
|
||||
"@napi-rs/canvas": "^0.1.100",
|
||||
"@tanstack/react-query": "^5.100.10",
|
||||
"@tanstack/react-query": "^5.101.0",
|
||||
"@types/archiver": "^7.0.0",
|
||||
"@vercel/analytics": "^1.6.1",
|
||||
"archiver": "^7.0.1",
|
||||
"better-auth": "^1.6.11",
|
||||
"better-auth": "^1.6.14",
|
||||
"better-sqlite3": "^12.10.0",
|
||||
"cmpstr": "^3.3.0",
|
||||
"compromise": "^14.15.0",
|
||||
"compromise": "^14.15.1",
|
||||
"core-js": "^3.49.0",
|
||||
"dexie": "^4.4.2",
|
||||
"dexie": "^4.4.3",
|
||||
"dexie-react-hooks": "^4.4.0",
|
||||
"dotenv": "^17.4.2",
|
||||
"drizzle-kit": "^0.31.10",
|
||||
|
|
@ -55,19 +55,19 @@
|
|||
"ffmpeg-static": "^5.3.0",
|
||||
"howler": "^2.2.4",
|
||||
"jszip": "^3.10.1",
|
||||
"lru-cache": "^11.3.6",
|
||||
"next": "^15.5.18",
|
||||
"lru-cache": "^11.5.1",
|
||||
"next": "^15.5.19",
|
||||
"onnxruntime-node": "^1.26.0",
|
||||
"openai": "^6.37.0",
|
||||
"openai": "^6.41.0",
|
||||
"pdfjs-dist": "4.8.69",
|
||||
"pg": "^8.20.0",
|
||||
"pg": "^8.21.0",
|
||||
"pino": "^10.3.1",
|
||||
"pino-pretty": "^13.1.3",
|
||||
"react": "^19.2.6",
|
||||
"react": "^19.2.7",
|
||||
"react-dnd": "^16.0.1",
|
||||
"react-dnd-html5-backend": "^16.0.1",
|
||||
"react-dnd-touch-backend": "^16.0.1",
|
||||
"react-dom": "^19.2.6",
|
||||
"react-dom": "^19.2.7",
|
||||
"react-dropzone": "^14.4.1",
|
||||
"react-hot-toast": "^2.6.0",
|
||||
"react-markdown": "^10.1.0",
|
||||
|
|
@ -82,17 +82,22 @@
|
|||
"@playwright/test": "^1.60.0",
|
||||
"@tailwindcss/typography": "^0.5.19",
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/howler": "^2.2.12",
|
||||
"@types/howler": "^2.2.13",
|
||||
"@types/node": "^20.19.41",
|
||||
"@types/pg": "^8.20.0",
|
||||
"@types/react": "^19.2.14",
|
||||
"@types/react": "^19.2.16",
|
||||
"@types/react-dom": "^19.2.3",
|
||||
"@types/uuid": "^10.0.0",
|
||||
"eslint": "^9.39.4",
|
||||
"eslint-config-next": "^15.5.18",
|
||||
"postcss": "^8.5.14",
|
||||
"eslint-config-next": "^15.5.19",
|
||||
"postcss": "^8.5.15",
|
||||
"tailwindcss": "^3.4.19",
|
||||
"typescript": "^5.9.3",
|
||||
"vitest": "^4.1.7"
|
||||
"vitest": "^4.1.8"
|
||||
},
|
||||
"pnpm": {
|
||||
"overrides": {
|
||||
"kysely": "0.28.17"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
3087
pnpm-lock.yaml
3087
pnpm-lock.yaml
File diff suppressed because it is too large
Load diff
|
|
@ -1,6 +1,7 @@
|
|||
packages:
|
||||
- .
|
||||
- compute/*
|
||||
- docker/*
|
||||
|
||||
allowBuilds:
|
||||
'@napi-rs/canvas': true
|
||||
|
|
|
|||
|
|
@ -219,6 +219,28 @@ function spawnMainCommand(command, env) {
|
|||
return { child, exitPromise };
|
||||
}
|
||||
|
||||
function resolveEmbeddedWorkerLaunch() {
|
||||
const candidateDirs = [
|
||||
path.join(process.cwd(), 'embedded-compute-worker'),
|
||||
path.join(process.cwd(), 'compute', 'worker'),
|
||||
];
|
||||
|
||||
for (const workerDir of candidateDirs) {
|
||||
const serverEntry = path.join(workerDir, 'src', 'server.ts');
|
||||
if (!fs.existsSync(serverEntry)) continue;
|
||||
return {
|
||||
cmd: process.execPath,
|
||||
args: ['--import', 'tsx', 'src/server.ts'],
|
||||
cwd: workerDir,
|
||||
};
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
'Could not find an embedded compute worker runtime. '
|
||||
+ 'Include embedded-compute-worker/src/server.ts in the runtime image or keep compute/worker available locally.',
|
||||
);
|
||||
}
|
||||
|
||||
function runDbMigrations(env) {
|
||||
const migrateScript = path.join(process.cwd(), 'drizzle', 'scripts', 'migrate.mjs');
|
||||
if (!fs.existsSync(migrateScript)) {
|
||||
|
|
@ -541,13 +563,14 @@ async function main() {
|
|||
...runtimeEnv,
|
||||
PORT: String(embeddedWorkerPort),
|
||||
};
|
||||
const workerLaunch = resolveEmbeddedWorkerLaunch();
|
||||
workerProc = spawn(
|
||||
'pnpm',
|
||||
['--filter', '@openreader/compute-worker', 'start'],
|
||||
workerLaunch.cmd,
|
||||
workerLaunch.args,
|
||||
{
|
||||
cwd: workerLaunch.cwd,
|
||||
env: workerEnv,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
shell: process.platform === 'win32',
|
||||
},
|
||||
);
|
||||
stopWorkerStdoutForward = forwardChildStream(workerProc.stdout, process.stdout);
|
||||
|
|
|
|||
Loading…
Reference in a new issue