From a56aaa2be95cf28e4ae34f48946c247069f0c924 Mon Sep 17 00:00:00 2001 From: Richard R Date: Fri, 12 Jun 2026 13:43:33 -0600 Subject: [PATCH 01/11] Refactor PDF parsing and rendering logic; migrate to compute-worker protocol - Removed deprecated functions related to document parsing and blob storage in blobstore.ts. - Introduced new PDF rendering logic in pdf-preview-renderer.ts and pdf-preview-pdfjs-runtime.ts. - Updated previews-render.ts to utilize the new PDF rendering functions. - Refactored user-whisper-align-job.ts to use the compute-worker client for alignment requests. - Enhanced artifact.ts and operation.ts to validate parsed PDF artifacts and resolve current PDF parses. - Updated snapshot.ts to align with new worker operation types. - Adjusted runtime-config.ts to check for compute-worker availability. - Modified types in parsed-pdf.ts and tts.ts to reflect changes in the compute-worker protocol. - Added unit tests for PDF artifact validation and compute-worker client contract. - Removed obsolete pdf-op-key.vitest.spec.ts test file. --- .dockerignore | 1 + .github/workflows/docker-publish.yml | 4 +- .github/workflows/vitest.yml | 2 + Dockerfile | 7 +- .../worker => compute-worker}/.env.example | 0 {compute/worker => compute-worker}/Dockerfile | 7 +- .../docker-compose.yml | 15 +- compute-worker/openapi.json | 1766 +++++++++++++++++ .../worker => compute-worker}/package.json | 11 +- compute-worker/scripts/generate-openapi.ts | 20 + compute-worker/src/api/operation-keys.ts | 56 + compute-worker/src/api/public-operation.ts | 44 + compute-worker/src/api/schemas.ts | 134 ++ .../src/compute}/api-contracts/index.ts | 0 .../src/compute}/config/cpu-budget.ts | 0 .../src/compute}/config/timeout.ts | 0 .../src/compute}/control-plane/index.ts | 0 .../compute}/control-plane/orchestrator.ts | 0 .../src/compute}/control-plane/sse.ts | 0 .../compute}/control-plane/state-machine.ts | 0 .../src/compute}/control-plane/types.ts | 0 .../src/compute}/index.ts | 0 .../src/compute}/local-runtime.ts | 0 .../src/compute}/pdf/assets/LICENSE.txt | 0 .../src/compute}/pdf/assets/manifest.json | 0 .../src/compute}/pdf/merge.ts | 0 .../src/compute}/pdf/model.ts | 0 .../src/compute}/pdf/normalize-text.ts | 0 .../src/compute}/pdf/parse.ts | 0 .../src/compute}/pdf/parser-version-key.ts | 0 .../src/compute}/pdf/parser-version.ts | 0 .../src/compute}/pdf/pdfjs-runtime.ts | 0 .../src/compute}/pdf/render.ts | 0 .../src/compute}/pdf/runLayoutModel.ts | 0 .../src/compute}/pdf/stitch.ts | 0 .../src/compute}/pdf/types.ts | 0 .../src/compute}/platform/docstore.ts | 0 .../src/compute}/platform/ffmpeg.ts | 0 .../src/compute}/types/index.ts | 0 .../src/compute}/types/parsed-pdf.ts | 0 .../src/compute}/types/tts.ts | 0 .../src/compute}/whisper/align.ts | 0 .../src/compute/whisper/alignment-map.ts | 44 + .../src/compute}/whisper/assets/LICENSE.txt | 0 .../src/compute}/whisper/assets/manifest.json | 0 .../compute}/whisper/assets/mel_filters.npz | Bin .../src/compute}/whisper/model.ts | 0 .../src/compute}/whisper/spectral.ts | 0 .../src/compute}/whisper/token-timestamps.ts | 0 .../src/control-plane/jetstream.ts | 4 +- .../src/control-plane/json-codec.ts | 0 .../src/orphan-recovery.ts | 2 +- .../src/pdf-artifact-persistence.ts | 0 .../src/pdf-progress.ts | 2 +- .../worker => compute-worker}/src/runtime.ts | 236 ++- .../worker => compute-worker}/src/server.ts | 0 .../src/storage/artifact-addressing.ts | 21 + .../src/worker-loop-policy.ts | 2 +- .../tests/api/routes.test.ts | 151 +- .../pdf-merge-text-with-regions.test.ts | 2 +- .../pdf-parse-normalize-text-items.test.ts | 2 +- .../pdf-stitch-cross-page-blocks.test.ts | 4 +- .../algorithms/support/document-fixtures.ts | 54 + .../whisper-alignment-mapping.test.ts | 2 +- .../algorithms/whisper-spectral.test.ts | 2 +- .../whisper-token-timestamps.test.ts | 2 +- .../control-plane/orchestrator.test.ts | 4 +- .../control-plane/run-layout-model.test.ts | 8 +- .../tests/compute}/control-plane/sse.test.ts | 2 +- .../control-plane/state-machine.test.ts | 4 +- .../helpers/in-memory-control-plane.ts | 4 +- .../tests/fixtures/fake-control-plane.ts | 12 +- .../tests/setup-env.ts | 0 .../tests/tsconfig.json | 0 .../tests/unit/jetstream-adapters.test.ts | 4 +- .../tests/unit/key-and-progress.test.ts | 15 + .../tests/unit/orphan-recovery.test.ts | 0 .../unit/pdf-artifact-persistence.test.ts | 0 .../tests/unit/worker-loop-policy.test.ts | 0 .../worker => compute-worker}/tsconfig.json | 2 +- compute/core/package.json | 21 - compute/core/src/whisper/alignment-map.ts | 62 - compute/core/tsconfig.json | 7 - docs-site/docs/deploy/compute-worker.md | 5 +- docs-site/docs/deploy/local-development.md | 14 +- docs-site/docs/reference/stack.md | 9 +- eslint.config.mjs | 14 +- next.config.ts | 10 - package.json | 10 +- pnpm-lock.yaml | 316 ++- pnpm-workspace.yaml | 2 +- scripts/check-compute-boundary.mjs | 39 + scripts/check-next-server-bundle.mjs | 3 +- scripts/openreader-entrypoint.mjs | 4 +- .../api/documents/[id]/parsed/events/route.ts | 22 +- src/lib/server/compute-worker/client.ts | 249 +++ src/lib/server/compute-worker/generated.ts | 700 +++++++ src/lib/server/compute-worker/protocol.ts | 57 + src/lib/server/compute/abort-like-error.ts | 9 - src/lib/server/compute/index.ts | 18 - src/lib/server/compute/types.ts | 36 - src/lib/server/compute/worker-op-create.ts | 60 - src/lib/server/compute/worker-op-state.ts | 179 -- src/lib/server/compute/worker.ts | 706 ------- src/lib/server/documents/blobstore.ts | 59 - .../documents/pdf-preview-pdfjs-runtime.ts | 60 + .../server/documents/pdf-preview-renderer.ts | 157 ++ src/lib/server/documents/previews-render.ts | 2 +- src/lib/server/jobs/user-whisper-align-job.ts | 9 +- src/lib/server/pdf-parse/artifact.ts | 32 +- src/lib/server/pdf-parse/operation.ts | 47 +- src/lib/server/pdf-parse/snapshot.ts | 8 +- src/lib/server/runtime-config.ts | 4 +- src/lib/shared/audio-text.ts | 5 +- src/types/parsed-pdf.ts | 19 +- src/types/tts.ts | 7 +- ...pute-worker-client-contract.vitest.spec.ts | 58 + tests/unit/pdf-op-key.vitest.spec.ts | 13 - ...pdf-parse-artifact-envelope.vitest.spec.ts | 72 + ...d-events-route-worker-proxy.vitest.spec.ts | 27 +- tsconfig.json | 9 +- vitest.config.ts | 21 +- 122 files changed, 4285 insertions(+), 1528 deletions(-) rename {compute/worker => compute-worker}/.env.example (100%) rename {compute/worker => compute-worker}/Dockerfile (72%) rename {compute/worker => compute-worker}/docker-compose.yml (75%) create mode 100644 compute-worker/openapi.json rename {compute/worker => compute-worker}/package.json (58%) create mode 100644 compute-worker/scripts/generate-openapi.ts create mode 100644 compute-worker/src/api/operation-keys.ts create mode 100644 compute-worker/src/api/public-operation.ts create mode 100644 compute-worker/src/api/schemas.ts rename {compute/core/src => compute-worker/src/compute}/api-contracts/index.ts (100%) rename {compute/core/src => compute-worker/src/compute}/config/cpu-budget.ts (100%) rename {compute/core/src => compute-worker/src/compute}/config/timeout.ts (100%) rename {compute/core/src => compute-worker/src/compute}/control-plane/index.ts (100%) rename {compute/core/src => compute-worker/src/compute}/control-plane/orchestrator.ts (100%) rename {compute/core/src => compute-worker/src/compute}/control-plane/sse.ts (100%) rename {compute/core/src => compute-worker/src/compute}/control-plane/state-machine.ts (100%) rename {compute/core/src => compute-worker/src/compute}/control-plane/types.ts (100%) rename {compute/core/src => compute-worker/src/compute}/index.ts (100%) rename {compute/core/src => compute-worker/src/compute}/local-runtime.ts (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/assets/LICENSE.txt (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/assets/manifest.json (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/merge.ts (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/model.ts (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/normalize-text.ts (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/parse.ts (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/parser-version-key.ts (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/parser-version.ts (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/pdfjs-runtime.ts (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/render.ts (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/runLayoutModel.ts (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/stitch.ts (100%) rename {compute/core/src => compute-worker/src/compute}/pdf/types.ts (100%) rename {compute/core/src => compute-worker/src/compute}/platform/docstore.ts (100%) rename {compute/core/src => compute-worker/src/compute}/platform/ffmpeg.ts (100%) rename {compute/core/src => compute-worker/src/compute}/types/index.ts (100%) rename {compute/core/src => compute-worker/src/compute}/types/parsed-pdf.ts (100%) rename {compute/core/src => compute-worker/src/compute}/types/tts.ts (100%) rename {compute/core/src => compute-worker/src/compute}/whisper/align.ts (100%) create mode 100644 compute-worker/src/compute/whisper/alignment-map.ts rename {compute/core/src => compute-worker/src/compute}/whisper/assets/LICENSE.txt (100%) rename {compute/core/src => compute-worker/src/compute}/whisper/assets/manifest.json (100%) rename {compute/core/src => compute-worker/src/compute}/whisper/assets/mel_filters.npz (100%) rename {compute/core/src => compute-worker/src/compute}/whisper/model.ts (100%) rename {compute/core/src => compute-worker/src/compute}/whisper/spectral.ts (100%) rename {compute/core/src => compute-worker/src/compute}/whisper/token-timestamps.ts (100%) rename {compute/worker => compute-worker}/src/control-plane/jetstream.ts (99%) rename {compute/worker => compute-worker}/src/control-plane/json-codec.ts (100%) rename {compute/worker => compute-worker}/src/orphan-recovery.ts (98%) rename {compute/worker => compute-worker}/src/pdf-artifact-persistence.ts (100%) rename {compute/worker => compute-worker}/src/pdf-progress.ts (87%) rename {compute/worker => compute-worker}/src/runtime.ts (89%) rename {compute/worker => compute-worker}/src/server.ts (100%) create mode 100644 compute-worker/src/storage/artifact-addressing.ts rename {compute/worker => compute-worker}/src/worker-loop-policy.ts (89%) rename {compute/worker => compute-worker}/tests/api/routes.test.ts (53%) rename tests/unit/pdf-merge-text-with-regions.vitest.spec.ts => compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts (96%) rename tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts => compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts (97%) rename tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts => compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts (96%) create mode 100644 compute-worker/tests/compute/algorithms/support/document-fixtures.ts rename tests/unit/whisper-alignment-mapping.vitest.spec.ts => compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts (97%) rename tests/unit/whisper-spectral.vitest.spec.ts => compute-worker/tests/compute/algorithms/whisper-spectral.test.ts (93%) rename tests/unit/whisper-token-timestamps.vitest.spec.ts => compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts (98%) rename {compute/core/tests => compute-worker/tests/compute}/control-plane/orchestrator.test.ts (96%) rename {compute/core/tests => compute-worker/tests/compute}/control-plane/run-layout-model.test.ts (93%) rename {compute/core/tests => compute-worker/tests/compute}/control-plane/sse.test.ts (96%) rename {compute/core/tests => compute-worker/tests/compute}/control-plane/state-machine.test.ts (93%) rename {compute/core/tests => compute-worker/tests/compute}/helpers/in-memory-control-plane.ts (97%) rename {compute/worker => compute-worker}/tests/fixtures/fake-control-plane.ts (93%) rename {compute/worker => compute-worker}/tests/setup-env.ts (100%) rename {compute/worker => compute-worker}/tests/tsconfig.json (100%) rename {compute/worker => compute-worker}/tests/unit/jetstream-adapters.test.ts (97%) rename {compute/worker => compute-worker}/tests/unit/key-and-progress.test.ts (62%) rename {compute/worker => compute-worker}/tests/unit/orphan-recovery.test.ts (100%) rename {compute/worker => compute-worker}/tests/unit/pdf-artifact-persistence.test.ts (100%) rename {compute/worker => compute-worker}/tests/unit/worker-loop-policy.test.ts (100%) rename {compute/worker => compute-worker}/tsconfig.json (73%) delete mode 100644 compute/core/package.json delete mode 100644 compute/core/src/whisper/alignment-map.ts delete mode 100644 compute/core/tsconfig.json create mode 100644 scripts/check-compute-boundary.mjs create mode 100644 src/lib/server/compute-worker/client.ts create mode 100644 src/lib/server/compute-worker/generated.ts create mode 100644 src/lib/server/compute-worker/protocol.ts delete mode 100644 src/lib/server/compute/abort-like-error.ts delete mode 100644 src/lib/server/compute/index.ts delete mode 100644 src/lib/server/compute/types.ts delete mode 100644 src/lib/server/compute/worker-op-create.ts delete mode 100644 src/lib/server/compute/worker-op-state.ts delete mode 100644 src/lib/server/compute/worker.ts create mode 100644 src/lib/server/documents/pdf-preview-pdfjs-runtime.ts create mode 100644 src/lib/server/documents/pdf-preview-renderer.ts create mode 100644 tests/unit/compute-worker-client-contract.vitest.spec.ts delete mode 100644 tests/unit/pdf-op-key.vitest.spec.ts create mode 100644 tests/unit/pdf-parse-artifact-envelope.vitest.spec.ts diff --git a/.dockerignore b/.dockerignore index 6c309d7..51d3428 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .env .env.* +**/*.creds README.md .next node_modules diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 2c50173..d7f6733 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -62,13 +62,13 @@ jobs: platform: linux/amd64 runner: ubuntu-24.04 context: . - dockerfile: ./compute/worker/Dockerfile + dockerfile: ./compute-worker/Dockerfile - image_target: compute-worker arch: arm64 platform: linux/arm64 runner: ubuntu-24.04-arm context: . - dockerfile: ./compute/worker/Dockerfile + dockerfile: ./compute-worker/Dockerfile steps: - name: Checkout repository diff --git a/.github/workflows/vitest.yml b/.github/workflows/vitest.yml index 27c2102..c659070 100644 --- a/.github/workflows/vitest.yml +++ b/.github/workflows/vitest.yml @@ -25,3 +25,5 @@ jobs: run: pnpm migrate - name: Run Vitest suites run: pnpm test:unit + - name: Verify compute worker contract and boundary + run: pnpm compute:openapi:check && pnpm check:compute-boundary diff --git a/Dockerfile b/Dockerfile index c6b5ac6..93e32ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,8 +20,7 @@ WORKDIR /app # Copy workspace manifests needed for dependency installation COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ -COPY compute/core/package.json ./compute/core/package.json -COPY compute/worker/package.json ./compute/worker/package.json +COPY compute-worker/package.json ./compute-worker/package.json COPY docker/entrypoint-migration-tools/package.json ./docker/entrypoint-migration-tools/package.json # Install dependencies @@ -88,7 +87,7 @@ COPY --from=app-builder /app/THIRD_PARTY_LICENSES /licenses # Include SeaweedFS license text for the copied weed binary. COPY --from=seaweedfs-builder /tmp/SeaweedFS-LICENSE.txt /licenses/SeaweedFS-LICENSE.txt # Include static model notices for runtime-downloaded assets. -COPY --from=app-builder /app/compute/core/src/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt +COPY --from=app-builder /app/compute-worker/src/compute/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt # Copy seaweedfs weed binary for optional embedded local S3. COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed @@ -98,7 +97,7 @@ COPY --from=nats-builder /tmp/nats-server /usr/local/bin/nats-server RUN chmod +x /usr/local/bin/nats-server # Include OpenAI Whisper license text for runtime-downloaded ONNX artifacts. -COPY --from=app-builder /app/compute/core/src/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt +COPY --from=app-builder /app/compute-worker/src/compute/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt # Match the app's historical container port now that standalone server.js # is started directly instead of `next start -p 3003`. diff --git a/compute/worker/.env.example b/compute-worker/.env.example similarity index 100% rename from compute/worker/.env.example rename to compute-worker/.env.example diff --git a/compute/worker/Dockerfile b/compute-worker/Dockerfile similarity index 72% rename from compute/worker/Dockerfile rename to compute-worker/Dockerfile index 1b0eb3a..356190f 100644 --- a/compute/worker/Dockerfile +++ b/compute-worker/Dockerfile @@ -5,11 +5,8 @@ RUN npm install -g pnpm@11.1.2 WORKDIR /workspace COPY .npmrc package.json pnpm-lock.yaml pnpm-workspace.yaml ./ -COPY compute/worker/package.json compute/worker/package.json -COPY compute/core/package.json compute/core/package.json - -COPY compute/worker compute/worker -COPY compute/core compute/core +COPY compute-worker/package.json compute-worker/package.json +COPY compute-worker compute-worker RUN pnpm --config.inject-workspace-packages=true --filter @openreader/compute-worker deploy /opt/compute-worker diff --git a/compute/worker/docker-compose.yml b/compute-worker/docker-compose.yml similarity index 75% rename from compute/worker/docker-compose.yml rename to compute-worker/docker-compose.yml index 1bbf550..115eea8 100644 --- a/compute/worker/docker-compose.yml +++ b/compute-worker/docker-compose.yml @@ -11,13 +11,14 @@ services: compute-worker: build: - context: ../.. - dockerfile: compute/worker/Dockerfile + context: .. + dockerfile: compute-worker/Dockerfile container_name: openreader-compute-worker depends_on: - nats env_file: - - ./.env + - path: ./.env + required: false environment: NATS_URL: ${NATS_URL:-nats://nats:4222} COMPUTE_WORKER_HOST: ${COMPUTE_WORKER_HOST:-0.0.0.0} @@ -32,16 +33,12 @@ services: - action: sync+restart path: . target: /workspace - - action: rebuild - path: ../../compute/core - action: rebuild path: ./package.json - action: rebuild - path: ../../compute/core/package.json + path: ../pnpm-lock.yaml - action: rebuild - path: ../../pnpm-lock.yaml - - action: rebuild - path: ../../pnpm-workspace.yaml + path: ../pnpm-workspace.yaml volumes: nats-data: diff --git a/compute-worker/openapi.json b/compute-worker/openapi.json new file mode 100644 index 0000000..c0973f1 --- /dev/null +++ b/compute-worker/openapi.json @@ -0,0 +1,1766 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "OpenReader Compute Worker API", + "version": "1.0.0" + }, + "components": { + "securitySchemes": { + "bearerAuth": { + "type": "http", + "scheme": "bearer" + } + }, + "schemas": { + "ParsedPdfDocument": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "enum": [ + 1 + ] + }, + "documentId": { + "type": "string" + }, + "parserVersion": { + "type": "string" + }, + "parsedAt": { + "type": "number" + }, + "pages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "pageNumber": { + "type": "number" + }, + "width": { + "type": "number" + }, + "height": { + "type": "number" + }, + "blocks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "abstract", + "algorithm", + "aside_text", + "chart", + "content", + "formula", + "doc_title", + "figure_title", + "footer", + "footnote", + "formula_number", + "header", + "image", + "number", + "paragraph_title", + "reference", + "reference_content", + "seal", + "table", + "text", + "vision_footnote" + ] + }, + "fragments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "page": { + "type": "number" + }, + "bbox": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + }, + { + "type": "number" + }, + { + "type": "number" + } + ] + }, + "text": { + "type": "string" + }, + "readingOrder": { + "type": "number" + }, + "modelConfidence": { + "type": "number" + } + }, + "required": [ + "page", + "bbox", + "text", + "readingOrder" + ], + "additionalProperties": false + } + }, + "text": { + "type": "string" + }, + "parentSectionId": { + "type": "string" + } + }, + "required": [ + "id", + "kind", + "fragments", + "text" + ], + "additionalProperties": false + } + } + }, + "required": [ + "pageNumber", + "width", + "height", + "blocks" + ], + "additionalProperties": false + } + } + }, + "required": [ + "schemaVersion", + "documentId", + "parserVersion", + "parsedAt", + "pages" + ], + "additionalProperties": false + }, + "TTSSentenceAlignment": { + "type": "object", + "properties": { + "sentence": { + "type": "string" + }, + "sentenceIndex": { + "type": "number" + }, + "words": { + "type": "array", + "items": { + "type": "object", + "properties": { + "text": { + "type": "string" + }, + "startSec": { + "type": "number" + }, + "endSec": { + "type": "number" + }, + "charStart": { + "type": "number" + }, + "charEnd": { + "type": "number" + } + }, + "required": [ + "text", + "startSec", + "endSec", + "charStart", + "charEnd" + ], + "additionalProperties": false + } + } + }, + "required": [ + "sentence", + "sentenceIndex", + "words" + ], + "additionalProperties": false + }, + "ArtifactReference": { + "type": "object", + "properties": { + "objectKey": { + "type": "string" + } + }, + "required": [ + "objectKey" + ], + "additionalProperties": false + }, + "ErrorResponse": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + }, + "OperationError": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "PdfLayoutProgress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + }, + "PublicOperation": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + }, + "PublicOperationEvent": { + "type": "object", + "properties": { + "eventId": { + "type": "number" + }, + "snapshot": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + } + }, + "required": [ + "eventId", + "snapshot" + ], + "additionalProperties": false + }, + "PdfLayoutResolution": { + "type": "object", + "properties": { + "artifact": { + "anyOf": [ + { + "type": "object", + "properties": { + "objectKey": { + "type": "string" + } + }, + "required": [ + "objectKey" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "operation": { + "anyOf": [ + { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "artifact", + "operation" + ], + "additionalProperties": false + } + } + }, + "paths": { + "/health/live": { + "get": { + "security": [], + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + } + }, + "required": [ + "ok" + ] + } + } + } + } + } + } + }, + "/health/ready": { + "get": { + "security": [], + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "natsConnected": { + "type": "boolean" + } + }, + "required": [ + "ok", + "natsConnected" + ] + } + } + } + } + } + } + }, + "/v1/whisper-align/operations": { + "post": { + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "minLength": 1 + }, + "lang": { + "type": "string", + "minLength": 1, + "maxLength": 16 + }, + "cacheKey": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "audioObjectKey": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "required": [ + "text", + "audioObjectKey" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + }, + "/v1/pdf-layout/operations": { + "post": { + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "documentId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "namespace": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-zA-Z0-9._-]{1,128}$" + }, + { + "type": "null" + } + ] + }, + "documentObjectKey": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "replaceToken": { + "type": "string", + "minLength": 1, + "maxLength": 256 + } + }, + "required": [ + "documentId", + "namespace", + "documentObjectKey" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + }, + "/v1/pdf-layout/resolve": { + "post": { + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "documentId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "namespace": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-zA-Z0-9._-]{1,128}$" + }, + { + "type": "null" + } + ] + }, + "documentObjectKey": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "required": [ + "documentId", + "namespace", + "documentObjectKey" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "artifact": { + "anyOf": [ + { + "type": "object", + "properties": { + "objectKey": { + "type": "string" + } + }, + "required": [ + "objectKey" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "operation": { + "anyOf": [ + { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "artifact", + "operation" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + }, + "/v1/operations/{opId}": { + "get": { + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1 + }, + "in": "path", + "name": "opId", + "required": true + } + ], + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + }, + "404": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + }, + "/v1/operations/{opId}/events": { + "get": { + "parameters": [ + { + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "in": "query", + "name": "sinceEventId", + "required": false + }, + { + "schema": { + "type": "string", + "minLength": 1 + }, + "in": "path", + "name": "opId", + "required": true + } + ], + "responses": { + "200": { + "description": "Server-sent PublicOperationEvent stream", + "content": { + "application/json": { + "schema": { + "type": "string", + "description": "Server-sent PublicOperationEvent stream" + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + }, + "404": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + } + }, + "security": [ + { + "bearerAuth": [] + } + ] +} diff --git a/compute/worker/package.json b/compute-worker/package.json similarity index 58% rename from compute/worker/package.json rename to compute-worker/package.json index e07eded..ad31e23 100644 --- a/compute/worker/package.json +++ b/compute-worker/package.json @@ -5,15 +5,22 @@ "type": "module", "scripts": { "dev": "tsx watch src/server.ts", - "start": "tsx src/server.ts" + "start": "tsx src/server.ts", + "openapi:generate": "tsx scripts/generate-openapi.ts" }, "dependencies": { "@aws-sdk/client-s3": "^3.1061.0", + "@fastify/swagger": "^9.0.0", + "@huggingface/tokenizers": "^0.1.3", + "@napi-rs/canvas": "^0.1.100", "@nats-io/jetstream": "^3.4.0", "@nats-io/kv": "^3.4.0", "@nats-io/transport-node": "^3.4.0", - "@openreader/compute-core": "workspace:*", + "ffmpeg-static": "^5.3.0", "fastify": "^5.8.5", + "jszip": "^3.10.1", + "onnxruntime-node": "^1.26.0", + "pdfjs-dist": "4.8.69", "pino": "^10.3.1", "pino-pretty": "^13.1.3", "tsx": "^4.22.4", diff --git a/compute-worker/scripts/generate-openapi.ts b/compute-worker/scripts/generate-openapi.ts new file mode 100644 index 0000000..7af98d5 --- /dev/null +++ b/compute-worker/scripts/generate-openapi.ts @@ -0,0 +1,20 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { createComputeWorkerApp } from '../src/runtime'; + +process.env.COMPUTE_WORKER_TOKEN ||= 'openapi-generation-token'; +process.env.NATS_URL ||= 'nats://127.0.0.1:4222'; + +const outputPath = resolve(process.cwd(), 'openapi.json'); +const runtime = await createComputeWorkerApp({ + workerToken: process.env.COMPUTE_WORKER_TOKEN, + disableWorkers: true, +}); + +try { + await runtime.app.ready(); + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, `${JSON.stringify(runtime.app.swagger(), null, 2)}\n`, 'utf8'); +} finally { + await runtime.close(); +} diff --git a/compute-worker/src/api/operation-keys.ts b/compute-worker/src/api/operation-keys.ts new file mode 100644 index 0000000..9c990b5 --- /dev/null +++ b/compute-worker/src/api/operation-keys.ts @@ -0,0 +1,56 @@ +import { createHash } from 'node:crypto'; +import { PDF_PARSER_VERSION } from '../compute/pdf/parser-version'; + +function sha256Hex(input: string): string { + return createHash('sha256').update(input).digest('hex'); +} + +export function buildWhisperOperationKey(input: { + audioObjectKey: string; + text: string; + lang?: string; + cacheKey?: string; +}): string { + const cacheKey = input.cacheKey?.trim(); + if (cacheKey) { + return `whisper_align|v1|cache|${cacheKey}|${input.audioObjectKey}`; + } + return [ + 'whisper_align', + 'v1', + input.audioObjectKey, + input.lang ?? '', + sha256Hex(input.text), + ].join('|'); +} + +export function buildPdfOperationKey(input: { + documentId: string; + namespace: string | null; + documentObjectKey: string; + replaceToken?: string; +}, parserVersion = PDF_PARSER_VERSION): string { + return [ + 'pdf_layout', + 'v1', + parserVersion, + input.documentId, + input.namespace ?? '', + input.documentObjectKey, + input.replaceToken?.trim() || '', + ].join('|'); +} + +export function pdfSubjectFromOperationKey(opKey: string): { + kind: 'pdf_layout'; + documentId: string; + namespace: string | null; +} | null { + const [kind, version, , documentId, namespace] = opKey.split('|'); + if (kind !== 'pdf_layout' || version !== 'v1' || !documentId) return null; + return { + kind: 'pdf_layout', + documentId, + namespace: namespace || null, + }; +} diff --git a/compute-worker/src/api/public-operation.ts b/compute-worker/src/api/public-operation.ts new file mode 100644 index 0000000..9ab98a4 --- /dev/null +++ b/compute-worker/src/api/public-operation.ts @@ -0,0 +1,44 @@ +import type { WorkerOperationState } from '../compute/api-contracts'; +import { pdfSubjectFromOperationKey } from './operation-keys'; + +export type PublicOperationSubject = + | { kind: 'whisper_align' } + | { kind: 'pdf_layout'; documentId: string; namespace: string | null }; + +export interface PublicOperation { + opId: string; + subject: PublicOperationSubject; + status: WorkerOperationState['status']; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: Result; + error?: WorkerOperationState['error']; + timing?: WorkerOperationState['timing']; + progress?: WorkerOperationState['progress']; +} + +export interface PublicOperationEvent { + eventId: number; + snapshot: PublicOperation; +} + +export function toPublicOperation( + state: WorkerOperationState, +): PublicOperation { + const subject = state.kind === 'pdf_layout' + ? (pdfSubjectFromOperationKey(state.opKey) ?? { kind: 'pdf_layout', documentId: '', namespace: null }) + : { kind: 'whisper_align' as const }; + return { + opId: state.opId, + subject, + status: state.status, + queuedAt: state.queuedAt, + updatedAt: state.updatedAt, + ...(state.startedAt === undefined ? {} : { startedAt: state.startedAt }), + ...(state.result === undefined ? {} : { result: state.result }), + ...(state.error === undefined ? {} : { error: state.error }), + ...(state.timing === undefined ? {} : { timing: state.timing }), + ...(state.progress === undefined ? {} : { progress: state.progress }), + }; +} diff --git a/compute-worker/src/api/schemas.ts b/compute-worker/src/api/schemas.ts new file mode 100644 index 0000000..66df5c5 --- /dev/null +++ b/compute-worker/src/api/schemas.ts @@ -0,0 +1,134 @@ +import { z } from 'zod'; + +const bboxSchema = z.tuple([z.number(), z.number(), z.number(), z.number()]); +const parsedPdfBlockKindSchema = z.enum([ + 'abstract', 'algorithm', 'aside_text', 'chart', 'content', 'formula', + 'doc_title', 'figure_title', 'footer', 'footnote', 'formula_number', + 'header', 'image', 'number', 'paragraph_title', 'reference', + 'reference_content', 'seal', 'table', 'text', 'vision_footnote', +]); +const documentIdSchema = z.string().trim().regex(/^[a-f0-9]{64}$/i); +const namespaceSchema = z.string().trim().regex(/^[a-zA-Z0-9._-]{1,128}$/).nullable(); + +export const parsedPdfDocumentSchema = z.object({ + schemaVersion: z.literal(1), + documentId: z.string(), + parserVersion: z.string(), + parsedAt: z.number(), + pages: z.array(z.object({ + pageNumber: z.number(), + width: z.number(), + height: z.number(), + blocks: z.array(z.object({ + id: z.string(), + kind: parsedPdfBlockKindSchema, + fragments: z.array(z.object({ + page: z.number(), + bbox: bboxSchema, + text: z.string(), + readingOrder: z.number(), + modelConfidence: z.number().optional(), + })), + text: z.string(), + parentSectionId: z.string().optional(), + })), + })), +}); + +export const ttsSentenceAlignmentSchema = z.object({ + sentence: z.string(), + sentenceIndex: z.number(), + words: z.array(z.object({ + text: z.string(), + startSec: z.number(), + endSec: z.number(), + charStart: z.number(), + charEnd: z.number(), + })), +}); + +export const whisperOperationCreateSchema = z.object({ + text: z.string().trim().min(1), + lang: z.string().trim().min(1).max(16).optional(), + cacheKey: z.string().trim().min(1).max(256).optional(), + audioObjectKey: z.string().trim().min(1).max(2048), +}); + +export const pdfOperationCreateSchema = z.object({ + documentId: documentIdSchema, + namespace: namespaceSchema, + documentObjectKey: z.string().trim().min(1).max(2048), + replaceToken: z.string().trim().min(1).max(256).optional(), +}); + +export const pdfResolveSchema = z.object({ + documentId: documentIdSchema, + namespace: namespaceSchema, + documentObjectKey: z.string().trim().min(1).max(2048), +}); + +export const operationParamsSchema = z.object({ + opId: z.string().trim().min(1), +}); + +export const operationEventsQuerySchema = z.object({ + sinceEventId: z.union([z.string(), z.number()]).optional(), +}); + +export const apiErrorResponseSchema = z.object({ + error: z.string(), +}).passthrough(); + +export const operationErrorSchema = z.object({ + message: z.string(), + code: z.string().optional(), +}); + +export const artifactReferenceSchema = z.object({ + objectKey: z.string(), +}); + +export const pdfLayoutProgressSchema = z.object({ + totalPages: z.number(), + pagesParsed: z.number(), + currentPage: z.number().optional(), + phase: z.enum(['infer', 'merge']), +}); + +export const publicOperationSchema = z.object({ + opId: z.string(), + subject: z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('whisper_align') }), + z.object({ + kind: z.literal('pdf_layout'), + documentId: z.string(), + namespace: z.string().nullable(), + }), + ]), + status: z.enum(['queued', 'running', 'succeeded', 'failed']), + queuedAt: z.number(), + updatedAt: z.number(), + startedAt: z.number().optional(), + result: z.unknown().optional(), + error: operationErrorSchema.optional(), + timing: z.object({ + queueWaitMs: z.number().optional(), + s3FetchMs: z.number().optional(), + computeMs: z.number().optional(), + }).optional(), + progress: pdfLayoutProgressSchema.optional(), +}); + +export const publicOperationEventSchema = z.object({ + eventId: z.number(), + snapshot: publicOperationSchema, +}); + +export const pdfLayoutResolutionSchema = z.object({ + artifact: artifactReferenceSchema.nullable(), + operation: publicOperationSchema.nullable(), +}); + +export function jsonSchema(schema: z.ZodType): Record { + return z.toJSONSchema(schema, { target: 'draft-7' }) as Record; +} diff --git a/compute/core/src/api-contracts/index.ts b/compute-worker/src/compute/api-contracts/index.ts similarity index 100% rename from compute/core/src/api-contracts/index.ts rename to compute-worker/src/compute/api-contracts/index.ts diff --git a/compute/core/src/config/cpu-budget.ts b/compute-worker/src/compute/config/cpu-budget.ts similarity index 100% rename from compute/core/src/config/cpu-budget.ts rename to compute-worker/src/compute/config/cpu-budget.ts diff --git a/compute/core/src/config/timeout.ts b/compute-worker/src/compute/config/timeout.ts similarity index 100% rename from compute/core/src/config/timeout.ts rename to compute-worker/src/compute/config/timeout.ts diff --git a/compute/core/src/control-plane/index.ts b/compute-worker/src/compute/control-plane/index.ts similarity index 100% rename from compute/core/src/control-plane/index.ts rename to compute-worker/src/compute/control-plane/index.ts diff --git a/compute/core/src/control-plane/orchestrator.ts b/compute-worker/src/compute/control-plane/orchestrator.ts similarity index 100% rename from compute/core/src/control-plane/orchestrator.ts rename to compute-worker/src/compute/control-plane/orchestrator.ts diff --git a/compute/core/src/control-plane/sse.ts b/compute-worker/src/compute/control-plane/sse.ts similarity index 100% rename from compute/core/src/control-plane/sse.ts rename to compute-worker/src/compute/control-plane/sse.ts diff --git a/compute/core/src/control-plane/state-machine.ts b/compute-worker/src/compute/control-plane/state-machine.ts similarity index 100% rename from compute/core/src/control-plane/state-machine.ts rename to compute-worker/src/compute/control-plane/state-machine.ts diff --git a/compute/core/src/control-plane/types.ts b/compute-worker/src/compute/control-plane/types.ts similarity index 100% rename from compute/core/src/control-plane/types.ts rename to compute-worker/src/compute/control-plane/types.ts diff --git a/compute/core/src/index.ts b/compute-worker/src/compute/index.ts similarity index 100% rename from compute/core/src/index.ts rename to compute-worker/src/compute/index.ts diff --git a/compute/core/src/local-runtime.ts b/compute-worker/src/compute/local-runtime.ts similarity index 100% rename from compute/core/src/local-runtime.ts rename to compute-worker/src/compute/local-runtime.ts diff --git a/compute/core/src/pdf/assets/LICENSE.txt b/compute-worker/src/compute/pdf/assets/LICENSE.txt similarity index 100% rename from compute/core/src/pdf/assets/LICENSE.txt rename to compute-worker/src/compute/pdf/assets/LICENSE.txt diff --git a/compute/core/src/pdf/assets/manifest.json b/compute-worker/src/compute/pdf/assets/manifest.json similarity index 100% rename from compute/core/src/pdf/assets/manifest.json rename to compute-worker/src/compute/pdf/assets/manifest.json diff --git a/compute/core/src/pdf/merge.ts b/compute-worker/src/compute/pdf/merge.ts similarity index 100% rename from compute/core/src/pdf/merge.ts rename to compute-worker/src/compute/pdf/merge.ts diff --git a/compute/core/src/pdf/model.ts b/compute-worker/src/compute/pdf/model.ts similarity index 100% rename from compute/core/src/pdf/model.ts rename to compute-worker/src/compute/pdf/model.ts diff --git a/compute/core/src/pdf/normalize-text.ts b/compute-worker/src/compute/pdf/normalize-text.ts similarity index 100% rename from compute/core/src/pdf/normalize-text.ts rename to compute-worker/src/compute/pdf/normalize-text.ts diff --git a/compute/core/src/pdf/parse.ts b/compute-worker/src/compute/pdf/parse.ts similarity index 100% rename from compute/core/src/pdf/parse.ts rename to compute-worker/src/compute/pdf/parse.ts diff --git a/compute/core/src/pdf/parser-version-key.ts b/compute-worker/src/compute/pdf/parser-version-key.ts similarity index 100% rename from compute/core/src/pdf/parser-version-key.ts rename to compute-worker/src/compute/pdf/parser-version-key.ts diff --git a/compute/core/src/pdf/parser-version.ts b/compute-worker/src/compute/pdf/parser-version.ts similarity index 100% rename from compute/core/src/pdf/parser-version.ts rename to compute-worker/src/compute/pdf/parser-version.ts diff --git a/compute/core/src/pdf/pdfjs-runtime.ts b/compute-worker/src/compute/pdf/pdfjs-runtime.ts similarity index 100% rename from compute/core/src/pdf/pdfjs-runtime.ts rename to compute-worker/src/compute/pdf/pdfjs-runtime.ts diff --git a/compute/core/src/pdf/render.ts b/compute-worker/src/compute/pdf/render.ts similarity index 100% rename from compute/core/src/pdf/render.ts rename to compute-worker/src/compute/pdf/render.ts diff --git a/compute/core/src/pdf/runLayoutModel.ts b/compute-worker/src/compute/pdf/runLayoutModel.ts similarity index 100% rename from compute/core/src/pdf/runLayoutModel.ts rename to compute-worker/src/compute/pdf/runLayoutModel.ts diff --git a/compute/core/src/pdf/stitch.ts b/compute-worker/src/compute/pdf/stitch.ts similarity index 100% rename from compute/core/src/pdf/stitch.ts rename to compute-worker/src/compute/pdf/stitch.ts diff --git a/compute/core/src/pdf/types.ts b/compute-worker/src/compute/pdf/types.ts similarity index 100% rename from compute/core/src/pdf/types.ts rename to compute-worker/src/compute/pdf/types.ts diff --git a/compute/core/src/platform/docstore.ts b/compute-worker/src/compute/platform/docstore.ts similarity index 100% rename from compute/core/src/platform/docstore.ts rename to compute-worker/src/compute/platform/docstore.ts diff --git a/compute/core/src/platform/ffmpeg.ts b/compute-worker/src/compute/platform/ffmpeg.ts similarity index 100% rename from compute/core/src/platform/ffmpeg.ts rename to compute-worker/src/compute/platform/ffmpeg.ts diff --git a/compute/core/src/types/index.ts b/compute-worker/src/compute/types/index.ts similarity index 100% rename from compute/core/src/types/index.ts rename to compute-worker/src/compute/types/index.ts diff --git a/compute/core/src/types/parsed-pdf.ts b/compute-worker/src/compute/types/parsed-pdf.ts similarity index 100% rename from compute/core/src/types/parsed-pdf.ts rename to compute-worker/src/compute/types/parsed-pdf.ts diff --git a/compute/core/src/types/tts.ts b/compute-worker/src/compute/types/tts.ts similarity index 100% rename from compute/core/src/types/tts.ts rename to compute-worker/src/compute/types/tts.ts diff --git a/compute/core/src/whisper/align.ts b/compute-worker/src/compute/whisper/align.ts similarity index 100% rename from compute/core/src/whisper/align.ts rename to compute-worker/src/compute/whisper/align.ts diff --git a/compute-worker/src/compute/whisper/alignment-map.ts b/compute-worker/src/compute/whisper/alignment-map.ts new file mode 100644 index 0000000..098374c --- /dev/null +++ b/compute-worker/src/compute/whisper/alignment-map.ts @@ -0,0 +1,44 @@ +import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts'; + +export interface WhisperWord { + start: number; + end: number; + word: string; +} + +export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment { + const lowerSentence = sentence.toLowerCase(); + let cursor = 0; + + const alignedWords: TTSSentenceWord[] = words.map((w) => { + const token = w.word.trim(); + if (!token) { + return { + text: '', + startSec: w.start, + endSec: w.end, + charStart: cursor, + charEnd: cursor, + }; + } + + const idx = lowerSentence.indexOf(token.toLowerCase(), cursor); + const start = idx >= 0 ? idx : cursor; + const end = Math.min(sentence.length, start + token.length); + cursor = Math.max(cursor, end); + + return { + text: token, + startSec: w.start, + endSec: w.end, + charStart: start, + charEnd: end, + }; + }).filter((word) => word.text.length > 0); + + return { + sentence, + sentenceIndex: 0, + words: alignedWords, + }; +} diff --git a/compute/core/src/whisper/assets/LICENSE.txt b/compute-worker/src/compute/whisper/assets/LICENSE.txt similarity index 100% rename from compute/core/src/whisper/assets/LICENSE.txt rename to compute-worker/src/compute/whisper/assets/LICENSE.txt diff --git a/compute/core/src/whisper/assets/manifest.json b/compute-worker/src/compute/whisper/assets/manifest.json similarity index 100% rename from compute/core/src/whisper/assets/manifest.json rename to compute-worker/src/compute/whisper/assets/manifest.json diff --git a/compute/core/src/whisper/assets/mel_filters.npz b/compute-worker/src/compute/whisper/assets/mel_filters.npz similarity index 100% rename from compute/core/src/whisper/assets/mel_filters.npz rename to compute-worker/src/compute/whisper/assets/mel_filters.npz diff --git a/compute/core/src/whisper/model.ts b/compute-worker/src/compute/whisper/model.ts similarity index 100% rename from compute/core/src/whisper/model.ts rename to compute-worker/src/compute/whisper/model.ts diff --git a/compute/core/src/whisper/spectral.ts b/compute-worker/src/compute/whisper/spectral.ts similarity index 100% rename from compute/core/src/whisper/spectral.ts rename to compute-worker/src/compute/whisper/spectral.ts diff --git a/compute/core/src/whisper/token-timestamps.ts b/compute-worker/src/compute/whisper/token-timestamps.ts similarity index 100% rename from compute/core/src/whisper/token-timestamps.ts rename to compute-worker/src/compute/whisper/token-timestamps.ts diff --git a/compute/worker/src/control-plane/jetstream.ts b/compute-worker/src/control-plane/jetstream.ts similarity index 99% rename from compute/worker/src/control-plane/jetstream.ts rename to compute-worker/src/control-plane/jetstream.ts index 9225d96..e5367eb 100644 --- a/compute/worker/src/control-plane/jetstream.ts +++ b/compute-worker/src/control-plane/jetstream.ts @@ -8,12 +8,12 @@ import type { OperationState, OperationStateStore, QueuedOperation, -} from '@openreader/compute-core/control-plane'; +} from '../compute/control-plane'; import type { PdfLayoutJobRequest, WhisperAlignJobRequest, WorkerOperationKind, -} from '@openreader/compute-core/api-contracts'; +} from '../compute/api-contracts'; import { createJsonCodec } from './json-codec'; export interface KvEntryLike { diff --git a/compute/worker/src/control-plane/json-codec.ts b/compute-worker/src/control-plane/json-codec.ts similarity index 100% rename from compute/worker/src/control-plane/json-codec.ts rename to compute-worker/src/control-plane/json-codec.ts diff --git a/compute/worker/src/orphan-recovery.ts b/compute-worker/src/orphan-recovery.ts similarity index 98% rename from compute/worker/src/orphan-recovery.ts rename to compute-worker/src/orphan-recovery.ts index 882ea94..8257b4d 100644 --- a/compute/worker/src/orphan-recovery.ts +++ b/compute-worker/src/orphan-recovery.ts @@ -4,7 +4,7 @@ import type { WorkerJobTiming, WorkerJobState, WorkerOperationState, -} from '@openreader/compute-core/api-contracts'; +} from './compute/api-contracts'; export type StreamedOperationState = WorkerOperationState; diff --git a/compute/worker/src/pdf-artifact-persistence.ts b/compute-worker/src/pdf-artifact-persistence.ts similarity index 100% rename from compute/worker/src/pdf-artifact-persistence.ts rename to compute-worker/src/pdf-artifact-persistence.ts diff --git a/compute/worker/src/pdf-progress.ts b/compute-worker/src/pdf-progress.ts similarity index 87% rename from compute/worker/src/pdf-progress.ts rename to compute-worker/src/pdf-progress.ts index 756a256..cc5c025 100644 --- a/compute/worker/src/pdf-progress.ts +++ b/compute-worker/src/pdf-progress.ts @@ -1,4 +1,4 @@ -import type { PdfLayoutProgress } from '@openreader/compute-core/api-contracts'; +import type { PdfLayoutProgress } from './compute/api-contracts'; export function buildInferProgressForPageStart(input: { pageNumber: number; diff --git a/compute/worker/src/runtime.ts b/compute-worker/src/runtime.ts similarity index 89% rename from compute/worker/src/runtime.ts rename to compute-worker/src/runtime.ts index a5099a6..5cafa7c 100644 --- a/compute/worker/src/runtime.ts +++ b/compute-worker/src/runtime.ts @@ -1,4 +1,5 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify'; +import swagger from '@fastify/swagger'; import { z } from 'zod'; import { connect, @@ -24,18 +25,16 @@ import { ensureComputeModels, runPdfLayoutFromPdfBuffer, runWhisperAlignmentFromAudioBuffer, -} from '@openreader/compute-core/local-runtime'; +} from './compute/local-runtime'; import { getComputeTimeoutConfig, getComputeOpStaleMs, getAvailableCpuCores, getOnnxThreadsPerJob, - PDF_PARSER_VERSION, - encodeParserVersion, withIdleTimeoutAndHardCap, withTimeout, -} from '@openreader/compute-core'; -import { encodeSseFrame, OperationOrchestrator } from '@openreader/compute-core/control-plane'; +} from './compute'; +import { encodeSseFrame, OperationOrchestrator } from './compute/control-plane'; import type { PdfLayoutJobRequest, PdfLayoutJobResult, @@ -48,7 +47,7 @@ import type { WorkerOperationRequest, WorkerOperationState, PdfLayoutProgress, -} from '@openreader/compute-core/api-contracts'; +} from './compute/api-contracts'; import { DeleteObjectCommand, GetObjectCommand, @@ -71,6 +70,26 @@ import { import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy'; import { persistParsedPdfWhileSourceExists } from './pdf-artifact-persistence'; +import { parsedPdfArtifactKey } from './storage/artifact-addressing'; +import { buildPdfOperationKey, buildWhisperOperationKey } from './api/operation-keys'; +import { toPublicOperation, type PublicOperationEvent } from './api/public-operation'; +import { + apiErrorResponseSchema, + artifactReferenceSchema, + jsonSchema, + operationEventsQuerySchema, + operationParamsSchema, + operationErrorSchema, + parsedPdfDocumentSchema, + pdfLayoutProgressSchema, + pdfLayoutResolutionSchema, + pdfOperationCreateSchema, + pdfResolveSchema, + publicOperationEventSchema, + publicOperationSchema, + ttsSentenceAlignmentSchema, + whisperOperationCreateSchema, +} from './api/schemas'; const JOBS_STREAM_NAME = 'compute_jobs'; const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; @@ -88,8 +107,6 @@ const OP_EVENTS_KEEPALIVE_MS = 15_000; // client from immediately reconnecting and re-waking the worker; instead it // reconnects on a slow cadence so the container stays asleep most of the time. const OP_EVENTS_RECONNECT_HINT_MS = 120_000; -const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; -const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; const WHISPER_MAX_DELIVER = 1; const NATS_API_TIMEOUT_MS = 60_000; // Disconnect from NATS after this much continuous idle so the worker stops @@ -182,6 +199,7 @@ export interface ComputeWorkerRouteDeps { orchestrator: OrchestratorLike; operationStateStore: OperationStateStoreLike; operationEventStream: OperationEventStreamLike; + artifactExists?: (key: string) => Promise; } export interface CreateComputeWorkerAppOptions { @@ -254,20 +272,6 @@ function normalizeS3Prefix(prefix: string | undefined): string { return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; } -function sanitizeNamespace(namespace: string | null): string | null { - if (!namespace) return null; - return SAFE_NAMESPACE_REGEX.test(namespace) ? namespace : null; -} - -function documentParsedKey(id: string, namespace: string | null, prefix: string): string { - if (!DOCUMENT_ID_REGEX.test(id)) { - throw new Error(`Invalid document id: ${id}`); - } - const ns = sanitizeNamespace(namespace); - const nsSegment = ns ? `ns/${ns}/` : ''; - return `${prefix}/documents_v1/parsed_v2/${nsSegment}${id}/${encodeParserVersion(PDF_PARSER_VERSION)}.json`; -} - function buildS3Client(): S3Client { const bucket = requireEnv('S3_BUCKET'); const region = requireEnv('S3_REGION'); @@ -357,7 +361,7 @@ function extractOpId(request: FastifyRequest, path: string): string | null { if (params && typeof params.opId === 'string' && params.opId.trim()) { return params.opId.trim(); } - const match = path.match(/^\/ops\/([^/]+)/); + const match = path.match(/^\/v1\/operations\/([^/]+)/); if (!match?.[1]) return null; try { return decodeURIComponent(match[1]); @@ -428,22 +432,7 @@ const layoutSchema = z.object({ documentObjectKey: z.string().trim().min(1).max(2048), }); -const operationCreateSchema = z.discriminatedUnion('kind', [ - z.object({ - kind: z.literal('whisper_align'), - opKey: z.string().trim().min(1).max(1024), - payload: alignSchema, - }), - z.object({ - kind: z.literal('pdf_layout'), - opKey: z.string().trim().min(1).max(1024), - payload: layoutSchema, - }), -]); - -const operationLookupSchema = z.object({ - opKey: z.string().trim().min(1).max(1024), -}); +const errorResponseSchema = jsonSchema(apiErrorResponseSchema); async function ensureJetStreamResources( jsm: JetStreamManager, @@ -567,7 +556,7 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti // Lazy NATS connection lifecycle. The worker connects on demand (first request // that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle // so it stops emitting outbound traffic and Railway can sleep it. Reconnect is - // transparent: any inbound /ops request both wakes the container and re-establishes + // transparent: any inbound operation request both wakes the container and re-establishes // the session via ensureConnected(). let session: NatsSession | null = null; let connecting: Promise | null = null; @@ -785,7 +774,7 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti if (!s3) { throw new Error('S3 access is disabled for this worker app instance'); } - const key = documentParsedKey(documentId, namespace, s3Prefix); + const key = parsedPdfArtifactKey({ documentId, namespace, prefix: s3Prefix }); const body = Buffer.from(JSON.stringify(parsed)); await s3.send(new PutObjectCommand({ Bucket: s3Bucket, @@ -805,6 +794,35 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti logger: buildLoggerConfig(), disableRequestLogging: true, }); + await app.register(swagger, { + openapi: { + openapi: '3.0.3', + info: { + title: 'OpenReader Compute Worker API', + version: '1.0.0', + }, + components: { + securitySchemes: { + bearerAuth: { + type: 'http', + scheme: 'bearer', + }, + }, + schemas: { + ParsedPdfDocument: jsonSchema(parsedPdfDocumentSchema), + TTSSentenceAlignment: jsonSchema(ttsSentenceAlignmentSchema), + ArtifactReference: jsonSchema(artifactReferenceSchema), + ErrorResponse: jsonSchema(apiErrorResponseSchema), + OperationError: jsonSchema(operationErrorSchema), + PdfLayoutProgress: jsonSchema(pdfLayoutProgressSchema), + PublicOperation: jsonSchema(publicOperationSchema), + PublicOperationEvent: jsonSchema(publicOperationEventSchema), + PdfLayoutResolution: jsonSchema(pdfLayoutResolutionSchema), + }, + }, + security: [{ bearerAuth: [] }], + }, + }); app.log.info({ jobConcurrency, whisperTimeoutMs, @@ -956,16 +974,37 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti } }); - app.get('/health/live', async () => ({ ok: true })); + app.get('/health/live', { + schema: { + security: [], + response: { 200: { type: 'object', properties: { ok: { type: 'boolean' } }, required: ['ok'] } }, + }, + }, async () => ({ ok: true })); // Reports readiness without forcing a NATS round-trip. Probing NATS here would // reconnect (and keep) the connection open, defeating idle sleep, so we only // report the current connection state. The worker reconnects lazily on the next - // /ops request regardless of what this returns. - app.get('/health/ready', async () => ({ ok: true, natsConnected: session !== null })); + // operation request regardless of what this returns. + app.get('/health/ready', { + schema: { + security: [], + response: { + 200: { + type: 'object', + properties: { ok: { type: 'boolean' }, natsConnected: { type: 'boolean' } }, + required: ['ok', 'natsConnected'], + }, + }, + }, + }, async () => ({ ok: true, natsConnected: session !== null })); - app.post('/ops', async (request, reply) => { - const parsed = operationCreateSchema.safeParse(request.body); + app.post('/v1/whisper-align/operations', { + schema: { + body: jsonSchema(whisperOperationCreateSchema), + response: { 202: jsonSchema(publicOperationSchema), 400: errorResponseSchema }, + }, + }, async (request, reply) => { + const parsed = whisperOperationCreateSchema.safeParse(request.body); if (!parsed.success) { reply.code(400); return { @@ -974,7 +1013,11 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti }; } - const requestOp = parsed.data as WorkerOperationRequest; + const requestOp: WorkerOperationRequest = { + kind: 'whisper_align', + opKey: buildWhisperOperationKey(parsed.data), + payload: parsed.data, + }; await ensureOrphanedOpRecovery(); const op = await orchestrator.enqueueOrReuse(requestOp); app.log.info({ @@ -985,11 +1028,16 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16), }, 'op.accepted'); reply.code(202); - return op; + return toPublicOperation(op); }); - app.post('/ops/lookup', async (request, reply) => { - const parsed = operationLookupSchema.safeParse(request.body); + app.post('/v1/pdf-layout/operations', { + schema: { + body: jsonSchema(pdfOperationCreateSchema), + response: { 202: jsonSchema(publicOperationSchema), 400: errorResponseSchema }, + }, + }, async (request, reply) => { + const parsed = pdfOperationCreateSchema.safeParse(request.body); if (!parsed.success) { reply.code(400); return { @@ -998,28 +1046,58 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti }; } + const requestOp: WorkerOperationRequest = { + kind: 'pdf_layout', + opKey: buildPdfOperationKey(parsed.data), + payload: { + documentId: parsed.data.documentId, + namespace: parsed.data.namespace, + documentObjectKey: parsed.data.documentObjectKey, + }, + }; await ensureOrphanedOpRecovery(); - if (typeof operationStateStore.getOpIndex !== 'function') { - reply.code(501); - return { error: 'Operation lookup by opKey is not supported by this state store' }; - } - const indexEntry = await operationStateStore.getOpIndex(parsed.data.opKey); - if (!indexEntry?.opId) { - reply.code(404); - return { error: 'Operation not found' }; - } - - const state = await operationStateStore.getOpState(indexEntry.opId); - if (!state) { - reply.code(404); - return { error: 'Operation not found' }; - } - - return state; + const op = await orchestrator.enqueueOrReuse(requestOp); + reply.code(202); + return toPublicOperation(op); }); - app.get('/ops/:opId', async (request, reply) => { - const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); + app.post('/v1/pdf-layout/resolve', { + schema: { + body: jsonSchema(pdfResolveSchema), + response: { + 200: jsonSchema(pdfLayoutResolutionSchema), + 400: errorResponseSchema, + }, + }, + }, async (request, reply) => { + const parsed = pdfResolveSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { error: 'Invalid request body', issues: parsed.error.issues }; + } + await ensureOrphanedOpRecovery(); + const artifactKey = parsedPdfArtifactKey({ + documentId: parsed.data.documentId, + namespace: parsed.data.namespace, + prefix: s3Prefix, + }); + const hasArtifact = await (options.routeDeps?.artifactExists ?? objectExists)(artifactKey); + const opKey = buildPdfOperationKey(parsed.data); + const index = await operationStateStore.getOpIndex?.(opKey); + const operation = index?.opId ? await operationStateStore.getOpState(index.opId) : null; + return { + artifact: hasArtifact ? { objectKey: artifactKey } : null, + operation: operation ? toPublicOperation(operation) : null, + }; + }); + + app.get('/v1/operations/:opId', { + schema: { + params: jsonSchema(operationParamsSchema), + response: { 200: jsonSchema(publicOperationSchema), 400: errorResponseSchema, 404: errorResponseSchema }, + }, + }, async (request, reply) => { + const params = operationParamsSchema.safeParse(request.params); if (!params.success) { reply.code(400); return { error: 'Invalid op id' }; @@ -1031,11 +1109,21 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti return { error: 'Operation not found' }; } - return state; + return toPublicOperation(state); }); - app.get('/ops/:opId/events', async (request, reply) => { - const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); + app.get('/v1/operations/:opId/events', { + schema: { + params: jsonSchema(operationParamsSchema), + querystring: jsonSchema(operationEventsQuerySchema), + response: { + 200: { type: 'string', description: 'Server-sent PublicOperationEvent stream' }, + 400: errorResponseSchema, + 404: errorResponseSchema, + }, + }, + }, async (request, reply) => { + const params = operationParamsSchema.safeParse(request.params); if (!params.success) { reply.code(400); return { error: 'Invalid op id' }; @@ -1080,9 +1168,9 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => { if (closed || reply.raw.writableEnded) return; - const frameEvent: WorkerOperationEvent = { + const frameEvent: PublicOperationEvent = { eventId, - snapshot, + snapshot: toPublicOperation(snapshot), }; reply.raw.write(encodeSseFrame({ id: eventId, diff --git a/compute/worker/src/server.ts b/compute-worker/src/server.ts similarity index 100% rename from compute/worker/src/server.ts rename to compute-worker/src/server.ts diff --git a/compute-worker/src/storage/artifact-addressing.ts b/compute-worker/src/storage/artifact-addressing.ts new file mode 100644 index 0000000..a4d1ee6 --- /dev/null +++ b/compute-worker/src/storage/artifact-addressing.ts @@ -0,0 +1,21 @@ +import { PDF_PARSER_VERSION } from '../compute/pdf/parser-version'; +import { encodeParserVersion } from '../compute/pdf/parser-version-key'; + +const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; +const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; + +export function parsedPdfArtifactKey(input: { + documentId: string; + namespace: string | null; + prefix: string; + parserVersion?: string; +}): string { + if (!DOCUMENT_ID_REGEX.test(input.documentId)) { + throw new Error(`Invalid document id: ${input.documentId}`); + } + const namespace = input.namespace && SAFE_NAMESPACE_REGEX.test(input.namespace) + ? input.namespace + : null; + const namespaceSegment = namespace ? `ns/${namespace}/` : ''; + return `${input.prefix}/documents_v1/parsed_v2/${namespaceSegment}${input.documentId}/${encodeParserVersion(input.parserVersion ?? PDF_PARSER_VERSION)}.json`; +} diff --git a/compute/worker/src/worker-loop-policy.ts b/compute-worker/src/worker-loop-policy.ts similarity index 89% rename from compute/worker/src/worker-loop-policy.ts rename to compute-worker/src/worker-loop-policy.ts index 7462b4c..bce8653 100644 --- a/compute/worker/src/worker-loop-policy.ts +++ b/compute-worker/src/worker-loop-policy.ts @@ -1,4 +1,4 @@ -import type { WorkerOperationKind } from '@openreader/compute-core/api-contracts'; +import type { WorkerOperationKind } from './compute/api-contracts'; export type RetryAction = 'nak_retry' | 'term_fail'; diff --git a/compute/worker/tests/api/routes.test.ts b/compute-worker/tests/api/routes.test.ts similarity index 53% rename from compute/worker/tests/api/routes.test.ts rename to compute-worker/tests/api/routes.test.ts index dad89d9..e7336ee 100644 --- a/compute/worker/tests/api/routes.test.ts +++ b/compute-worker/tests/api/routes.test.ts @@ -25,53 +25,51 @@ describe('compute worker API routes', () => { const live = await runtime.app.inject({ method: 'GET', url: '/health/live' }); expect(live.statusCode).toBe(200); - const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/ops/op-1' }); + const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/v1/operations/op-1' }); expect(protectedRoute.statusCode).toBe(401); }); test('validates operation creation body and returns 400 for invalid payload', async () => { const response = await runtime.app.inject({ method: 'POST', - url: '/ops', + url: '/v1/pdf-layout/operations', headers: AUTH, payload: { - kind: 'pdf_layout', - opKey: '', - payload: { - documentId: 'd1', - namespace: null, - documentObjectKey: 's3://bucket/doc.pdf', - }, + documentId: '', + namespace: null, + documentObjectKey: 'openreader/doc.pdf', }, }); expect(response.statusCode).toBe(400); - expect(response.json()).toMatchObject({ error: 'Invalid request body' }); + expect(response.json()).toMatchObject({ error: 'Bad Request' }); }); test('creates operation and fetches state by op id', async () => { + const documentId = 'c'.repeat(64); const create = await runtime.app.inject({ method: 'POST', - url: '/ops', + url: '/v1/pdf-layout/operations', headers: AUTH, payload: { - kind: 'pdf_layout', - opKey: 'doc-1:layout', - payload: { - documentId: 'doc-1', - namespace: null, - documentObjectKey: 'openreader/doc-1.pdf', - }, + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, }, }); expect(create.statusCode).toBe(202); const created = create.json(); - expect(created).toMatchObject({ kind: 'pdf_layout', status: 'queued' }); + expect(created).toMatchObject({ + subject: { kind: 'pdf_layout', documentId, namespace: null }, + status: 'queued', + }); + expect(created).not.toHaveProperty('opKey'); + expect(created).not.toHaveProperty('jobId'); const fetch = await runtime.app.inject({ method: 'GET', - url: `/ops/${created.opId}`, + url: `/v1/operations/${created.opId}`, headers: AUTH, }); @@ -79,17 +77,120 @@ describe('compute worker API routes', () => { expect(fetch.json()).toMatchObject({ opId: created.opId, status: 'queued' }); }); + test('reuses idempotent PDF requests and replaces them only with an explicit token', async () => { + const documentId = 'e'.repeat(64); + const payload = { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }; + const create = (body: typeof payload & { replaceToken?: string }) => runtime.app.inject({ + method: 'POST', + url: '/v1/pdf-layout/operations', + headers: AUTH, + payload: body, + }); + + const initial = (await create(payload)).json(); + const reused = (await create(payload)).json(); + const replacement = (await create({ ...payload, replaceToken: 'replace-1' })).json(); + + expect(reused.opId).toBe(initial.opId); + expect(replacement.opId).not.toBe(initial.opId); + expect(replacement.subject).toEqual(initial.subject); + }); + + test('creates whisper alignment operations without exposing internal keys', async () => { + const response = await runtime.app.inject({ + method: 'POST', + url: '/v1/whisper-align/operations', + headers: AUTH, + payload: { + text: 'Canonical worker text', + audioObjectKey: 'openreader/audio.mp3', + lang: 'en', + }, + }); + + expect(response.statusCode).toBe(202); + expect(response.json()).toMatchObject({ + subject: { kind: 'whisper_align' }, + status: 'queued', + }); + expect(response.json()).not.toHaveProperty('opKey'); + }); + + test('resolves the current PDF artifact and operation without exposing parser identity', async () => { + const documentId = 'a'.repeat(64); + const create = await runtime.app.inject({ + method: 'POST', + url: '/v1/pdf-layout/operations', + headers: AUTH, + payload: { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }, + }); + expect(create.statusCode).toBe(202); + + const artifactKey = `openreader/documents_v1/parsed_v2/${documentId}/pp-doclayoutv3-onnx%40800%2Bpdfjs%404.8.69.json`; + fake.seedArtifact(artifactKey); + const resolve = await runtime.app.inject({ + method: 'POST', + url: '/v1/pdf-layout/resolve', + headers: AUTH, + payload: { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }, + }); + + expect(resolve.statusCode).toBe(200); + expect(resolve.json()).toMatchObject({ + artifact: { objectKey: artifactKey }, + operation: { + subject: { kind: 'pdf_layout', documentId, namespace: null }, + }, + }); + expect(resolve.json().operation).not.toHaveProperty('opKey'); + }); + + test('resolves a current PDF artifact after operation state has expired', async () => { + const documentId = 'd'.repeat(64); + const artifactKey = `openreader/documents_v1/parsed_v2/${documentId}/pp-doclayoutv3-onnx%40800%2Bpdfjs%404.8.69.json`; + fake.seedArtifact(artifactKey); + + const resolve = await runtime.app.inject({ + method: 'POST', + url: '/v1/pdf-layout/resolve', + headers: AUTH, + payload: { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }, + }); + + expect(resolve.statusCode).toBe(200); + expect(resolve.json()).toEqual({ + artifact: { objectKey: artifactKey }, + operation: null, + }); + }); + test('returns not found for unknown operation and event stream lookups', async () => { const opResponse = await runtime.app.inject({ method: 'GET', - url: '/ops/missing', + url: '/v1/operations/missing', headers: AUTH, }); expect(opResponse.statusCode).toBe(404); const eventsResponse = await runtime.app.inject({ method: 'GET', - url: '/ops/missing/events', + url: '/v1/operations/missing/events', headers: AUTH, }); expect(eventsResponse.statusCode).toBe(404); @@ -109,7 +210,7 @@ describe('compute worker API routes', () => { const stream = await runtime.app.inject({ method: 'GET', - url: '/ops/op-terminal/events?sinceEventId=7', + url: '/v1/operations/op-terminal/events?sinceEventId=7', headers: AUTH, }); @@ -159,11 +260,11 @@ describe('compute worker API routes', () => { updatedAt: now - 310_000, }); - // GET /ops/:opId resolves via getOpState(), which first awaits the shared + // GET /v1/operations/:opId resolves via getOpState(), which first awaits the shared // orphanRecoveryPromise path through ensureOrphanedOpRecovery(). const fetch = await runtime.app.inject({ method: 'GET', - url: '/ops/op-stale-whisper-running', + url: '/v1/operations/op-stale-whisper-running', headers: AUTH, }); diff --git a/tests/unit/pdf-merge-text-with-regions.vitest.spec.ts b/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts similarity index 96% rename from tests/unit/pdf-merge-text-with-regions.vitest.spec.ts rename to compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts index 449824e..71436d0 100644 --- a/tests/unit/pdf-merge-text-with-regions.vitest.spec.ts +++ b/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { mergeTextWithRegions } from '@openreader/compute-core'; +import { mergeTextWithRegions } from '../../../src/compute'; describe('mergeTextWithRegions', () => { test('assigns text items to containing regions by centroid and joins in reading order', () => { diff --git a/tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts b/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts similarity index 97% rename from tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts rename to compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts index b61bd26..f11272d 100644 --- a/tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts +++ b/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { normalizeTextItemsForLayout } from '@openreader/compute-core'; +import { normalizeTextItemsForLayout } from '../../../src/compute'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; function makeTextItem( diff --git a/tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts b/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts similarity index 96% rename from tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts rename to compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts index 0aa37e6..fb7b602 100644 --- a/tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts +++ b/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { stitchCrossPageBlocks } from '@openreader/compute-core'; -import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../src/types/parsed-pdf'; +import { stitchCrossPageBlocks } from '../../../src/compute'; +import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/compute/types'; import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures'; function makeBlock( diff --git a/compute-worker/tests/compute/algorithms/support/document-fixtures.ts b/compute-worker/tests/compute/algorithms/support/document-fixtures.ts new file mode 100644 index 0000000..ffcde2d --- /dev/null +++ b/compute-worker/tests/compute/algorithms/support/document-fixtures.ts @@ -0,0 +1,54 @@ +import type { BaseDocument } from '../../../src/types/documents'; +import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/compute/types'; + +export function makeBaseDocument(overrides: Partial = {}): BaseDocument { + return { + id: 'doc-1', + name: 'document.pdf', + size: 1_024, + lastModified: 1_700_000_000_000, + type: 'pdf', + ...overrides, + }; +} + +export function makeParsedPdfBlock(input: { + id: string; + kind: ParsedPdfBlockKind; + text: string; + page: number; + readingOrder: number; +}): ParsedPdfBlock { + return { + id: input.id, + kind: input.kind, + text: input.text, + fragments: [ + { + page: input.page, + bbox: [0, 0, 100, 10], + text: input.text, + readingOrder: input.readingOrder, + }, + ], + }; +} + +export function makeParsedPdfPage(pageNumber: number, blocks: ParsedPdfBlock[]): ParsedPdfPage { + return { + pageNumber, + width: 800, + height: 1_200, + blocks, + }; +} + +export function makeParsedPdfDocument(pages: ParsedPdfPage[]): ParsedPdfDocument { + return { + schemaVersion: 1, + documentId: 'doc-fixture', + parserVersion: 'test', + parsedAt: 1_700_000_000_000, + pages, + }; +} diff --git a/tests/unit/whisper-alignment-mapping.vitest.spec.ts b/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts similarity index 97% rename from tests/unit/whisper-alignment-mapping.vitest.spec.ts rename to compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts index e5c9575..2eea68f 100644 --- a/tests/unit/whisper-alignment-mapping.vitest.spec.ts +++ b/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; import { mapWordsToSentenceOffsets, -} from '@openreader/compute-core'; +} from '../../../src/compute'; describe('whisper alignment mapping', () => { test('maps words to sentence offsets with punctuation and repeated spaces', () => { diff --git a/tests/unit/whisper-spectral.vitest.spec.ts b/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts similarity index 93% rename from tests/unit/whisper-spectral.vitest.spec.ts rename to compute-worker/tests/compute/algorithms/whisper-spectral.test.ts index 2b87a04..ac4a3d8 100644 --- a/tests/unit/whisper-spectral.vitest.spec.ts +++ b/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { buildGoertzelCoefficients, goertzelPower } from '@openreader/compute-core'; +import { buildGoertzelCoefficients, goertzelPower } from '../../../src/compute'; function dftPower(samples: Float32Array, k: number): number { const n = samples.length; diff --git a/tests/unit/whisper-token-timestamps.vitest.spec.ts b/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts similarity index 98% rename from tests/unit/whisper-token-timestamps.vitest.spec.ts rename to compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts index c2aded0..a6ff75d 100644 --- a/tests/unit/whisper-token-timestamps.vitest.spec.ts +++ b/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts @@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-node'; import { buildWordsFromTimestampedTokens, extractTokenStartTimestamps, -} from '@openreader/compute-core'; +} from '../../../src/compute'; describe('whisper token timestamp alignment', () => { test('extracts monotonic token timestamps from cross-attention maps', () => { diff --git a/compute/core/tests/control-plane/orchestrator.test.ts b/compute-worker/tests/compute/control-plane/orchestrator.test.ts similarity index 96% rename from compute/core/tests/control-plane/orchestrator.test.ts rename to compute-worker/tests/compute/control-plane/orchestrator.test.ts index 80bd9fb..6c9bbab 100644 --- a/compute/core/tests/control-plane/orchestrator.test.ts +++ b/compute-worker/tests/compute/control-plane/orchestrator.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import type { WorkerOperationRequest } from '../../src/api-contracts'; -import { OperationOrchestrator } from '../../src/control-plane'; +import type { WorkerOperationRequest } from '../../../src/compute/api-contracts'; +import { OperationOrchestrator } from '../../../src/compute/control-plane'; import { InMemoryOperationEventStream, InMemoryOperationQueue, diff --git a/compute/core/tests/control-plane/run-layout-model.test.ts b/compute-worker/tests/compute/control-plane/run-layout-model.test.ts similarity index 93% rename from compute/core/tests/control-plane/run-layout-model.test.ts rename to compute-worker/tests/compute/control-plane/run-layout-model.test.ts index e951bf3..27ef706 100644 --- a/compute/core/tests/control-plane/run-layout-model.test.ts +++ b/compute-worker/tests/compute/control-plane/run-layout-model.test.ts @@ -70,13 +70,13 @@ vi.mock('@napi-rs/canvas', () => { }; }); -vi.mock('../../src/pdf/model', () => ({ +vi.mock('../../../src/compute/pdf/model', () => ({ ensureModel: vi.fn(async () => '/tmp/model.onnx'), MODEL_CONFIG_PATH: '/tmp/model-config.json', MODEL_PREPROCESSOR_PATH: '/tmp/model-preprocessor.json', })); -vi.mock('../../src/config/cpu-budget', () => ({ +vi.mock('../../../src/compute/config/cpu-budget', () => ({ getOnnxThreadsPerJob: vi.fn(() => 1), })); @@ -107,7 +107,7 @@ describe('runLayoutModel', () => { }, }; - const { runLayoutModel } = await import('../../src/pdf/runLayoutModel'); + const { runLayoutModel } = await import('../../../src/compute/pdf/runLayoutModel'); const regions = await runLayoutModel({ pageWidth: 100, pageHeight: 100, @@ -154,7 +154,7 @@ describe('runLayoutModel', () => { }, }; - const { runLayoutModel } = await import('../../src/pdf/runLayoutModel'); + const { runLayoutModel } = await import('../../../src/compute/pdf/runLayoutModel'); const regions = await runLayoutModel({ pageWidth: 100, pageHeight: 100, diff --git a/compute/core/tests/control-plane/sse.test.ts b/compute-worker/tests/compute/control-plane/sse.test.ts similarity index 96% rename from compute/core/tests/control-plane/sse.test.ts rename to compute-worker/tests/compute/control-plane/sse.test.ts index 9b8e5b9..3461229 100644 --- a/compute/core/tests/control-plane/sse.test.ts +++ b/compute-worker/tests/compute/control-plane/sse.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../src/control-plane/sse'; +import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../../src/compute/control-plane/sse'; describe('sse codec', () => { test('encodes event id and payload and decodes both reliably', () => { diff --git a/compute/core/tests/control-plane/state-machine.test.ts b/compute-worker/tests/compute/control-plane/state-machine.test.ts similarity index 93% rename from compute/core/tests/control-plane/state-machine.test.ts rename to compute-worker/tests/compute/control-plane/state-machine.test.ts index e9617cb..a9d5390 100644 --- a/compute/core/tests/control-plane/state-machine.test.ts +++ b/compute-worker/tests/compute/control-plane/state-machine.test.ts @@ -1,11 +1,11 @@ import { describe, expect, test } from 'vitest'; -import type { WorkerOperationState } from '../../src/api-contracts'; +import type { WorkerOperationState } from '../../../src/compute/api-contracts'; import { explainReplacementReason, isInflightStatus, isTerminalStatus, shouldReuseExistingOperation, -} from '../../src/control-plane/state-machine'; +} from '../../../src/compute/control-plane/state-machine'; function runningState(overrides: Partial = {}): WorkerOperationState { return { diff --git a/compute/core/tests/helpers/in-memory-control-plane.ts b/compute-worker/tests/compute/helpers/in-memory-control-plane.ts similarity index 97% rename from compute/core/tests/helpers/in-memory-control-plane.ts rename to compute-worker/tests/compute/helpers/in-memory-control-plane.ts index cc9b9ce..7395b2b 100644 --- a/compute/core/tests/helpers/in-memory-control-plane.ts +++ b/compute-worker/tests/compute/helpers/in-memory-control-plane.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events'; -import type { WorkerOperationKind } from '../../src/api-contracts'; +import type { WorkerOperationKind } from '../../../src/compute/api-contracts'; import type { OperationEvent, OperationEventStream, @@ -8,7 +8,7 @@ import type { OperationState, OperationStateStore, QueuedOperation, -} from '../../src/control-plane/types'; +} from '../../../src/compute/control-plane/types'; function topicFor(opId: string): string { return `op.${opId}`; diff --git a/compute/worker/tests/fixtures/fake-control-plane.ts b/compute-worker/tests/fixtures/fake-control-plane.ts similarity index 93% rename from compute/worker/tests/fixtures/fake-control-plane.ts rename to compute-worker/tests/fixtures/fake-control-plane.ts index 9822523..40ab638 100644 --- a/compute/worker/tests/fixtures/fake-control-plane.ts +++ b/compute-worker/tests/fixtures/fake-control-plane.ts @@ -4,7 +4,7 @@ import type { WorkerOperationEvent, WorkerOperationRequest, WorkerOperationState, -} from '@openreader/compute-core/api-contracts'; +} from '../../src/compute/api-contracts'; import type { ComputeWorkerRouteDeps } from '../../src/runtime'; type ComputeResult = WhisperAlignJobResult | PdfLayoutJobResult; @@ -16,6 +16,7 @@ export class FakeControlPlane { private readonly revisionByOpId = new Map(); private readonly opIdByOpKey = new Map(); private readonly eventsByOpId = new Map(); + private readonly artifactKeys = new Set(); private nextOpId = 1; readonly deps: ComputeWorkerRouteDeps = { @@ -58,6 +59,10 @@ export class FakeControlPlane { }; }, listOpStates: async () => Array.from(this.stateByOpId.values()), + getOpIndex: async (opKey) => { + const opId = this.opIdByOpKey.get(opKey); + return opId ? { opId } : null; + }, }, operationEventStream: { subscribe: async ({ opId, sinceEventId, onEvent }) => { @@ -69,6 +74,7 @@ export class FakeControlPlane { return () => undefined; }, }, + artifactExists: async (key) => this.artifactKeys.has(key), }; seedState(state: ComputeState): void { @@ -83,6 +89,10 @@ export class FakeControlPlane { this.eventsByOpId.set(opId, list); } + seedArtifact(key: string): void { + this.artifactKeys.add(key); + } + getState(opId: string): ComputeState | null { return this.stateByOpId.get(opId) ?? null; } diff --git a/compute/worker/tests/setup-env.ts b/compute-worker/tests/setup-env.ts similarity index 100% rename from compute/worker/tests/setup-env.ts rename to compute-worker/tests/setup-env.ts diff --git a/compute/worker/tests/tsconfig.json b/compute-worker/tests/tsconfig.json similarity index 100% rename from compute/worker/tests/tsconfig.json rename to compute-worker/tests/tsconfig.json diff --git a/compute/worker/tests/unit/jetstream-adapters.test.ts b/compute-worker/tests/unit/jetstream-adapters.test.ts similarity index 97% rename from compute/worker/tests/unit/jetstream-adapters.test.ts rename to compute-worker/tests/unit/jetstream-adapters.test.ts index 9f30ccc..ae86c06 100644 --- a/compute/worker/tests/unit/jetstream-adapters.test.ts +++ b/compute-worker/tests/unit/jetstream-adapters.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { OperationOrchestrator } from '@openreader/compute-core/control-plane'; -import type { WorkerOperationRequest } from '@openreader/compute-core/api-contracts'; +import { OperationOrchestrator } from '../../src/compute/control-plane'; +import type { WorkerOperationRequest } from '../../src/compute/api-contracts'; import { JetStreamOperationEventStream, JetStreamOperationQueue, diff --git a/compute/worker/tests/unit/key-and-progress.test.ts b/compute-worker/tests/unit/key-and-progress.test.ts similarity index 62% rename from compute/worker/tests/unit/key-and-progress.test.ts rename to compute-worker/tests/unit/key-and-progress.test.ts index 6887cad..cbaf5ac 100644 --- a/compute/worker/tests/unit/key-and-progress.test.ts +++ b/compute-worker/tests/unit/key-and-progress.test.ts @@ -4,6 +4,8 @@ import { buildInferProgressForPageParsed, buildInferProgressForPageStart, } from '../../src/pdf-progress'; +import { buildPdfOperationKey } from '../../src/api/operation-keys'; +import { parsedPdfArtifactKey } from '../../src/storage/artifact-addressing'; describe('compute worker helpers', () => { test('hash and kv key helpers are stable and deterministic', () => { @@ -34,4 +36,17 @@ describe('compute worker helpers', () => { phase: 'infer', }); }); + + test('parser-version changes rotate worker-owned operation and artifact identities', () => { + const request = { + documentId: 'a'.repeat(64), + namespace: null, + documentObjectKey: `openreader/${'a'.repeat(64)}.pdf`, + }; + + expect(buildPdfOperationKey(request, 'parser-v1')) + .not.toBe(buildPdfOperationKey(request, 'parser-v2')); + expect(parsedPdfArtifactKey({ ...request, prefix: 'openreader', parserVersion: 'parser-v1' })) + .not.toBe(parsedPdfArtifactKey({ ...request, prefix: 'openreader', parserVersion: 'parser-v2' })); + }); }); diff --git a/compute/worker/tests/unit/orphan-recovery.test.ts b/compute-worker/tests/unit/orphan-recovery.test.ts similarity index 100% rename from compute/worker/tests/unit/orphan-recovery.test.ts rename to compute-worker/tests/unit/orphan-recovery.test.ts diff --git a/compute/worker/tests/unit/pdf-artifact-persistence.test.ts b/compute-worker/tests/unit/pdf-artifact-persistence.test.ts similarity index 100% rename from compute/worker/tests/unit/pdf-artifact-persistence.test.ts rename to compute-worker/tests/unit/pdf-artifact-persistence.test.ts diff --git a/compute/worker/tests/unit/worker-loop-policy.test.ts b/compute-worker/tests/unit/worker-loop-policy.test.ts similarity index 100% rename from compute/worker/tests/unit/worker-loop-policy.test.ts rename to compute-worker/tests/unit/worker-loop-policy.test.ts diff --git a/compute/worker/tsconfig.json b/compute-worker/tsconfig.json similarity index 73% rename from compute/worker/tsconfig.json rename to compute-worker/tsconfig.json index 428e252..820e03b 100644 --- a/compute/worker/tsconfig.json +++ b/compute-worker/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../tsconfig.json", + "extends": "../tsconfig.json", "compilerOptions": { "noEmit": true, }, diff --git a/compute/core/package.json b/compute/core/package.json deleted file mode 100644 index 4f27980..0000000 --- a/compute/core/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "@openreader/compute-core", - "version": "0.0.0", - "private": true, - "type": "module", - "dependencies": { - "@huggingface/tokenizers": "^0.1.3", - "@napi-rs/canvas": "^0.1.100", - "ffmpeg-static": "^5.3.0", - "jszip": "^3.10.1", - "onnxruntime-node": "^1.26.0", - "pdfjs-dist": "4.8.69" - }, - "exports": { - ".": "./src/index.ts", - "./local-runtime": "./src/local-runtime.ts", - "./api-contracts": "./src/api-contracts/index.ts", - "./control-plane": "./src/control-plane/index.ts", - "./types": "./src/types/index.ts" - } -} diff --git a/compute/core/src/whisper/alignment-map.ts b/compute/core/src/whisper/alignment-map.ts deleted file mode 100644 index a246cec..0000000 --- a/compute/core/src/whisper/alignment-map.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts'; - -// Worker-side mirror of the app's canonical audio-text cleaning rules in -// `src/lib/shared/audio-text.ts`. This is a separate build target and cannot -// import from `@/lib`, so the rules are duplicated here on purpose. The word -// `charStart`/`charEnd` offsets this module emits are consumed against text -// normalized by that shared module, so any divergence shifts viewer highlights -// off-word — keep this byte-for-byte in sync with `audio-text.ts`. -const STRIPPED_GLYPHS = /[*•◦‣⁃∙▪▫■□●○◆◇★☆▶▸►▹➤➢❖]/g; - -function preprocessSentenceForAudio(text: string): string { - return text - .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') - .replace(/([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu, '$1$2') - .replace(STRIPPED_GLYPHS, '') - .replace(/\s+/g, ' ') - .trim(); -} - -export interface WhisperWord { - start: number; - end: number; - word: string; -} - -export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment { - const normalizedSentence = preprocessSentenceForAudio(sentence); - const lowerSentence = normalizedSentence.toLowerCase(); - let cursor = 0; - - const alignedWords: TTSSentenceWord[] = words.map((w) => { - const token = w.word.trim(); - if (!token) { - return { - text: '', - startSec: w.start, - endSec: w.end, - charStart: cursor, - charEnd: cursor, - }; - } - - const idx = lowerSentence.indexOf(token.toLowerCase(), cursor); - const start = idx >= 0 ? idx : cursor; - const end = Math.min(normalizedSentence.length, start + token.length); - cursor = Math.max(cursor, end); - - return { - text: token, - startSec: w.start, - endSec: w.end, - charStart: start, - charEnd: end, - }; - }).filter((word) => word.text.length > 0); - - return { - sentence, - sentenceIndex: 0, - words: alignedWords, - }; -} diff --git a/compute/core/tsconfig.json b/compute/core/tsconfig.json deleted file mode 100644 index 3e68ae5..0000000 --- a/compute/core/tsconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "noEmit": true - }, - "include": ["src/**/*.ts"] -} diff --git a/docs-site/docs/deploy/compute-worker.md b/docs-site/docs/deploy/compute-worker.md index cd279db..604c7e1 100644 --- a/docs-site/docs/deploy/compute-worker.md +++ b/docs-site/docs/deploy/compute-worker.md @@ -11,7 +11,8 @@ Use this guide when OpenReader runs compute as a separate service. For the defau - Runs PDF layout parsing jobs - Stores durable job state in NATS JetStream and NATS KV -The app server submits work to `POST /ops` and listens for updates on `GET /ops/:opId/events`. +The app server submits resource-specific operations under `/v1` and listens for updates on +`GET /v1/operations/:opId/events`. ## When to use it @@ -37,7 +38,7 @@ S3_SECRET_ACCESS_KEY=... ``` :::important -`compute/worker/.env*` is only for standalone worker deployments. +`compute-worker/.env*` is only for standalone worker deployments. - Embedded/local mode: configure the root `.env` only. - External worker mode: set `COMPUTE_WORKER_URL` and `COMPUTE_WORKER_TOKEN` on the app, and worker runtime values on the worker service. diff --git a/docs-site/docs/deploy/local-development.md b/docs-site/docs/deploy/local-development.md index b14a087..6098225 100644 --- a/docs-site/docs/deploy/local-development.md +++ b/docs-site/docs/deploy/local-development.md @@ -163,17 +163,17 @@ If you need mirrors or pinned artifact locations, set `WHISPER_MODEL_BASE_URL` i External compute worker dev stack (optional) Use this only when you intentionally run compute-worker as a separate service. -Default local flow does not need `compute/worker/.env`; embedded worker startup reads root `.env`. +Default local flow does not need `compute-worker/.env`; embedded worker startup reads root `.env`. Full worker deployment details are in [Compute Worker (NATS JetStream)](./compute-worker). Start only NATS + compute-worker via compose watch: ```bash -docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --watch -# or: pnpm compute:dev:watch +docker compose --env-file compute-worker/.env -f compute-worker/docker-compose.yml up --watch +# or: pnpm compute:dev:compose ``` -`compute/worker/.env.example` contains a starter config for standalone worker service deployments. +`compute-worker/.env.example` contains a starter config for standalone worker service deployments. Run the main app separately on the host: @@ -190,9 +190,9 @@ COMPUTE_WORKER_TOKEN= Ownership in external worker mode: - root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale/retry overrides such as `COMPUTE_PDF_JOB_ATTEMPTS` -- `compute/worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning) +- `compute-worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning) -For embedded worker startup (`COMPUTE_WORKER_URL` unset), worker tuning values such as `COMPUTE_PDF_JOB_ATTEMPTS` must be set in the root `.env` because `compute/worker/.env*` is ignored in that mode. +For embedded worker startup (`COMPUTE_WORKER_URL` unset), worker tuning values such as `COMPUTE_PDF_JOB_ATTEMPTS` must be set in the root `.env` because `compute-worker/.env*` is ignored in that mode. Worker mode requires worker-reachable shared object storage (S3-compatible endpoint). For external worker mode, object storage must be shared/reachable by both app and worker services. @@ -240,7 +240,7 @@ COMPUTE_WORKER_TOKEN= Use the same ownership split: - root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale overrides -- `compute/worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning) +- `compute-worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning) Use one of these `.env` mode templates: diff --git a/docs-site/docs/reference/stack.md b/docs-site/docs/reference/stack.md index ea847e6..f260e31 100644 --- a/docs-site/docs/reference/stack.md +++ b/docs-site/docs/reference/stack.md @@ -45,21 +45,20 @@ title: Stack ## External compute worker (optional) -Monorepo packages under `compute/`: +Standalone worker package: -- **`@openreader/compute-core`** — ONNX runtime lifecycle, model management, and inference logic shared by compute worker runtime + app/worker contracts +- **`@openreader/compute-worker`** — standalone Node.js compute service containing its private inference and queue runtime - ONNX runtime: `onnxruntime-node` with `@huggingface/tokenizers` - Whisper alignment: `onnx-community/whisper-base_timestamped` (q4) for word-level timestamps - PDF layout: `Bei0001/PP-DocLayoutV3-ONNX` for document block detection and layout parsing - PDF rendering: `pdfjs-dist`, `@napi-rs/canvas` for server-side page rasterization - Utilities: `jszip`, `ffmpeg-static` -- **`@openreader/compute-worker`** — standalone Node.js worker service - - HTTP server: [Fastify](https://fastify.dev/) v5 +- HTTP server: [Fastify](https://fastify.dev/) v5 with a versioned OpenAPI contract - Job queue + state: [NATS](https://nats.io/) JetStream WorkQueue pull consumers + NATS KV (`jobs.whisper`, `jobs.layout`) - Storage: AWS SDK v3 S3 client for reading/writing blobs - Logging: [Pino](https://getpino.io/) - Validation: [Zod](https://zod.dev/) -- Heavy compute is worker-backed via `COMPUTE_WORKER_URL` + `COMPUTE_WORKER_TOKEN` (remote queue via HTTP + NATS) +- The Next.js app communicates with the worker only through the versioned HTTP API generated from OpenAPI. ## Tooling and testing diff --git a/eslint.config.mjs b/eslint.config.mjs index 2a0310d..e7af21a 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -34,16 +34,14 @@ const UI_ARCHITECTURE_FILES = [ "src/components/player/**/*.{ts,tsx}", "src/components/reader/**/*.{ts,tsx}", ]; -const COMPUTE_CORE_IMPORT_PATTERNS = [ +const COMPUTE_WORKER_IMPORT_PATTERNS = [ { group: [ - "@openreader/compute-core/*", - "!@openreader/compute-core/local-runtime", - "!@openreader/compute-core/types", - "!@openreader/compute-core/api-contracts", + "@openreader/compute-worker", + "@openreader/compute-worker/*", ], message: - "Use '@openreader/compute-core' root imports for light APIs. Allowed subpaths are '@openreader/compute-core/local-runtime', '@openreader/compute-core/types', and '@openreader/compute-core/api-contracts'.", + "The app must communicate with compute-worker through the app-owned HTTP client and generated protocol types.", }, ]; const UI_ARCHITECTURE_IMPORT_PATHS = [ @@ -124,7 +122,7 @@ const eslintConfig = [ "no-restricted-imports": [ "error", { - patterns: COMPUTE_CORE_IMPORT_PATTERNS, + patterns: COMPUTE_WORKER_IMPORT_PATTERNS, }, ], }, @@ -142,7 +140,7 @@ const eslintConfig = [ "error", { paths: UI_ARCHITECTURE_IMPORT_PATHS, - patterns: COMPUTE_CORE_IMPORT_PATTERNS, + patterns: COMPUTE_WORKER_IMPORT_PATTERNS, }, ], }, diff --git a/next.config.ts b/next.config.ts index 1cf8c6b..a5908d7 100644 --- a/next.config.ts +++ b/next.config.ts @@ -15,7 +15,6 @@ const securityHeaders = [ }, ]; -const bundleWorkerCompute = true; const pdfjsTraceFiles = [ './node_modules/pdfjs-dist/package.json', './node_modules/pdfjs-dist/legacy/build/pdf.mjs', @@ -29,7 +28,6 @@ const serverExternalPackages = [ // Keep pdfjs-dist as a real package in node_modules. Server-side preview // rendering resolves pdf.js runtime assets from the filesystem at runtime. 'pdfjs-dist', - ...(!bundleWorkerCompute ? ['onnxruntime-node', '@huggingface/tokenizers'] : []), ]; const nextConfig: NextConfig = { @@ -88,14 +86,6 @@ const nextConfig: NextConfig = { }, ]; } - if (isServer && bundleWorkerCompute) { - config.resolve.alias = { - ...(config.resolve.alias || {}), - '@openreader/compute-core/local-runtime$': false, - 'onnxruntime-node$': false, - '@huggingface/tokenizers$': false, - }; - } return config; }, }; diff --git a/package.json b/package.json index de098e0..25d915d 100644 --- a/package.json +++ b/package.json @@ -8,13 +8,16 @@ "dev:raw": "next dev --turbopack -p 3003", "build": "next build", "build:bundle-guard": "node scripts/check-next-server-bundle.mjs", + "check:compute-boundary": "node scripts/check-compute-boundary.mjs", "start": "node scripts/openreader-entrypoint.mjs -- pnpm start:raw", "start:raw": "next start -p 3003", "lint": "next lint", "test": "playwright test", "test:e2e": "playwright test", "test:unit": "vitest run", - "test:compute": "vitest run --project compute-core --project compute-worker", + "test:compute": "vitest run --project compute-worker", + "compute:openapi:generate": "pnpm --dir compute-worker openapi:generate && openapi-typescript compute-worker/openapi.json -o src/lib/server/compute-worker/generated.ts", + "compute:openapi:check": "pnpm compute:openapi:generate && git diff --exit-code -- compute-worker/openapi.json src/lib/server/compute-worker/generated.ts", "migrate": "node drizzle/scripts/migrate.mjs", "migrate-fs": "node scripts/migrate-fs-v2.mjs", "migrate-fs:dry-run": "node scripts/migrate-fs-v2.mjs --dry-run true", @@ -27,14 +30,13 @@ "docs:clear": "pnpm --dir docs-site clear", "compute:worker:dev": "pnpm --filter @openreader/compute-worker dev", "compute:worker:start": "pnpm --filter @openreader/compute-worker start", - "compute:dev:compose": "docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --build", + "compute:dev:compose": "docker compose --env-file compute-worker/.env -f compute-worker/docker-compose.yml up --build", "lint:route-errors": "node scripts/check-route-error-responses.mjs" }, "dependencies": { "@aws-sdk/client-s3": "^3.1061.0", "@aws-sdk/s3-request-presigner": "^3.1061.0", "@headlessui/react": "^2.2.10", - "@huggingface/tokenizers": "^0.1.3", "@microsoft/antissrf": "^1.0.0", "@mozilla/readability": "^0.6.0", "@napi-rs/canvas": "^0.1.100", @@ -61,7 +63,6 @@ "linkedom": "^0.18.12", "lru-cache": "^11.5.1", "next": "^15.5.19", - "onnxruntime-node": "^1.26.0", "openai": "^6.41.0", "pdfjs-dist": "4.8.69", "pg": "^8.21.0", @@ -98,6 +99,7 @@ "eslint": "^9.39.4", "eslint-config-next": "^15.5.19", "postcss": "^8.5.15", + "openapi-typescript": "^7.10.1", "tailwindcss": "^3.4.19", "typescript": "^5.9.3", "vitest": "^4.1.8" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e6e0608..32a7d38 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,9 +20,6 @@ importers: '@headlessui/react': specifier: ^2.2.10 version: 2.2.10(react-dom@19.2.7(react@19.2.7))(react@19.2.7) - '@huggingface/tokenizers': - specifier: ^0.1.3 - version: 0.1.3 '@microsoft/antissrf': specifier: ^1.0.0 version: 1.0.0 @@ -49,7 +46,7 @@ importers: version: 7.0.1 better-auth: specifier: ^1.6.14 - version: 1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))) + version: 1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))) better-sqlite3: specifier: ^12.10.0 version: 12.10.0 @@ -101,9 +98,6 @@ importers: next: specifier: ^15.5.19 version: 15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) - onnxruntime-node: - specifier: ^1.26.0 - version: 1.26.0 openai: specifier: ^6.41.0 version: 6.41.0(zod@4.4.3) @@ -170,7 +164,7 @@ importers: version: 1.60.0 '@tailwindcss/typography': specifier: ^0.5.19 - version: 0.5.19(tailwindcss@3.4.19(tsx@4.22.4)) + version: 0.5.19(tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0)) '@types/better-sqlite3': specifier: ^7.6.13 version: 7.6.13 @@ -204,27 +198,48 @@ importers: eslint-config-next: specifier: ^15.5.19 version: 15.5.19(eslint@9.39.4(jiti@1.21.7))(typescript@5.9.3) + openapi-typescript: + specifier: ^7.10.1 + version: 7.13.0(typescript@5.9.3) postcss: specifier: ^8.5.15 version: 8.5.15 tailwindcss: specifier: ^3.4.19 - version: 3.4.19(tsx@4.22.4) + version: 3.4.19(tsx@4.22.4)(yaml@2.9.0) typescript: specifier: ^5.9.3 version: 5.9.3 vitest: specifier: ^4.1.8 - version: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)) + version: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)) - compute/core: + compute-worker: dependencies: + '@aws-sdk/client-s3': + specifier: ^3.1061.0 + version: 3.1061.0 + '@fastify/swagger': + specifier: ^9.0.0 + version: 9.7.0 '@huggingface/tokenizers': specifier: ^0.1.3 version: 0.1.3 '@napi-rs/canvas': specifier: ^0.1.100 version: 0.1.100 + '@nats-io/jetstream': + specifier: ^3.4.0 + version: 3.4.0 + '@nats-io/kv': + specifier: ^3.4.0 + version: 3.4.0 + '@nats-io/transport-node': + specifier: ^3.4.0 + version: 3.4.0 + fastify: + specifier: ^5.8.5 + version: 5.8.5 ffmpeg-static: specifier: ^5.3.0 version: 5.3.0 @@ -237,27 +252,6 @@ importers: pdfjs-dist: specifier: 4.8.69 version: 4.8.69 - - compute/worker: - dependencies: - '@aws-sdk/client-s3': - specifier: ^3.1061.0 - version: 3.1061.0 - '@nats-io/jetstream': - specifier: ^3.4.0 - version: 3.4.0 - '@nats-io/kv': - specifier: ^3.4.0 - version: 3.4.0 - '@nats-io/transport-node': - specifier: ^3.4.0 - version: 3.4.0 - '@openreader/compute-core': - specifier: workspace:* - version: link:../core - fastify: - specifier: ^5.8.5 - version: 5.8.5 pino: specifier: ^10.3.1 version: 10.3.1 @@ -424,6 +418,14 @@ packages: resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==} engines: {node: '>=18.0.0'} + '@babel/code-frame@7.29.7': + resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + '@babel/runtime@7.29.7': resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} engines: {node: '>=6.9.0'} @@ -1031,6 +1033,9 @@ packages: '@fastify/proxy-addr@5.1.0': resolution: {integrity: sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==} + '@fastify/swagger@9.7.0': + resolution: {integrity: sha512-Vp1SC1GC2Hrkd3faFILv86BzUNyFz5N4/xdExqtCgkGASOzn/x+eMe4qXIGq7cdT6wif/P/oa6r1Ruqx19paZA==} + '@floating-ui/core@1.7.5': resolution: {integrity: sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==} @@ -1508,6 +1513,16 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1 + '@redocly/ajv@8.11.2': + resolution: {integrity: sha512-io1JpnwtIcvojV7QKDUSIuMN/ikdOUd1ReEnUnMKGfDVridQZ31J0MmIuqwuRjWDZfmvr+Q0MqCcfHM2gTivOg==} + + '@redocly/config@0.22.0': + resolution: {integrity: sha512-gAy93Ddo01Z3bHuVdPWfCwzgfaYgMdaZPcfL7JZ7hWJoK9V0lXDbigTWkhiPFAaLWzbOJ+kbUQG1+XwIm0KRGQ==} + + '@redocly/openapi-core@1.34.15': + resolution: {integrity: sha512-HAwCnNyKcs5XGQqms+9t7OdAPM/5TDstmhF+0i7tdCFato2QKuYIlyWETwkXd8c5zbltr1oB+6y9NTeQLr2d6Q==} + engines: {node: '>=18.17.0', npm: '>=9.5.0'} + '@rolldown/binding-android-arm64@1.0.2': resolution: {integrity: sha512-ZS4D1JPGn/MYQN/SYDWftIE/nVsM8j/AFOYEzAoOE2O3NktQOZru+/vYXGbR/qtdLdIfGCP0lcoJiYVzsEz+iQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2036,6 +2051,10 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + ajv-formats@3.0.1: resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} peerDependencies: @@ -2050,6 +2069,10 @@ packages: ajv@8.20.0: resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + ansi-colors@4.1.3: + resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} + engines: {node: '>=6'} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -2384,6 +2407,9 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + change-case@5.4.4: + resolution: {integrity: sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==} + character-entities-html4@2.1.0: resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==} @@ -2421,6 +2447,9 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + colorette@1.4.0: + resolution: {integrity: sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==} + colorette@2.0.20: resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} @@ -3011,6 +3040,9 @@ packages: resolution: {integrity: sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==} hasBin: true + fastify-plugin@5.1.0: + resolution: {integrity: sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw==} + fastify@5.8.5: resolution: {integrity: sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==} @@ -3212,6 +3244,10 @@ packages: resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} engines: {node: '>= 6'} + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} @@ -3234,6 +3270,10 @@ packages: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} + index-to-position@1.2.0: + resolution: {integrity: sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==} + engines: {node: '>=18'} + inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} @@ -3413,9 +3453,17 @@ packages: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} + js-levenshtein@1.1.6: + resolution: {integrity: sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==} + engines: {node: '>=0.10.0'} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + js-yaml@4.1.1: + resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} + hasBin: true + js-yaml@4.2.0: resolution: {integrity: sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==} hasBin: true @@ -3426,6 +3474,10 @@ packages: json-schema-ref-resolver@3.0.0: resolution: {integrity: sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==} + json-schema-resolver@3.0.0: + resolution: {integrity: sha512-HqMnbz0tz2DaEJ3ntsqtx3ezzZyDE7G56A/pPY/NGmrPu76UzsWquOpHFRAf5beTNXoH2LU5cQePVvRli1nchA==} + engines: {node: '>=20'} + json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} @@ -3927,6 +3979,15 @@ packages: zod: optional: true + openapi-types@12.1.3: + resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==} + + openapi-typescript@7.13.0: + resolution: {integrity: sha512-EFP392gcqXS7ntPvbhBzbF8TyBA+baIYEm791Hy5YkjDYKTnk/Tn5OQeKm5BIZvJihpp8Zzr4hzx0Irde1LNGQ==} + hasBin: true + peerDependencies: + typescript: ^5.x + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -3963,6 +4024,10 @@ packages: parse-entities@4.0.2: resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} + parse-json@8.3.0: + resolution: {integrity: sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==} + engines: {node: '>=18'} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -4073,6 +4138,10 @@ packages: engines: {node: '>=18'} hasBin: true + pluralize@8.0.0: + resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} + engines: {node: '>=4'} + possible-typed-array-names@1.1.0: resolution: {integrity: sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==} engines: {node: '>= 0.4'} @@ -4615,6 +4684,10 @@ packages: suffix-thumb@5.0.2: resolution: {integrity: sha512-I5PWXAFKx3FYnI9a+dQMWNqTxoRt6vdBdb0O+BJ1sxXCWtSoQCusc13E58f+9p4MYx/qCnEMkD5jac6K2j3dgA==} + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -4728,6 +4801,10 @@ packages: resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==} engines: {node: '>=10'} + type-fest@4.41.0: + resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} + engines: {node: '>=16'} + type@2.7.3: resolution: {integrity: sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==} @@ -4786,6 +4863,9 @@ packages: unrs-resolver@1.12.2: resolution: {integrity: sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==} + uri-js-replace@1.0.1: + resolution: {integrity: sha512-W+C9NWNLFOoBI2QWDp4UT9pv65r2w5Cx+3sTYFvtMdDBxkKt1syCqsUdSFAChbEe1uK5TfS04wt/nGwmaeIQ0g==} + uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} @@ -4943,6 +5023,18 @@ packages: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} + yaml-ast-parser@0.0.43: + resolution: {integrity: sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A==} + + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} + engines: {node: '>= 14.6'} + hasBin: true + + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -5224,6 +5316,14 @@ snapshots: '@aws/lambda-invoke-store@0.2.4': {} + '@babel/code-frame@7.29.7': + dependencies: + '@babel/helper-validator-identifier': 7.29.7 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/helper-validator-identifier@7.29.7': {} + '@babel/runtime@7.29.7': {} '@better-auth/core@1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0)': @@ -5546,7 +5646,7 @@ snapshots: '@eslint/config-array@0.21.2': dependencies: '@eslint/object-schema': 2.1.7 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) minimatch: 3.1.5 transitivePeerDependencies: - supports-color @@ -5562,7 +5662,7 @@ snapshots: '@eslint/eslintrc@3.3.5': dependencies: ajv: 6.15.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) espree: 10.4.0 globals: 14.0.0 ignore: 5.3.2 @@ -5605,6 +5705,16 @@ snapshots: '@fastify/forwarded': 3.0.1 ipaddr.js: 2.4.0 + '@fastify/swagger@9.7.0': + dependencies: + fastify-plugin: 5.1.0 + json-schema-resolver: 3.0.0 + openapi-types: 12.1.3 + rfdc: 1.4.1 + yaml: 2.9.0 + transitivePeerDependencies: + - supports-color + '@floating-ui/core@1.7.5': dependencies: '@floating-ui/utils': 0.2.11 @@ -5968,6 +6078,29 @@ snapshots: dependencies: react: 19.2.7 + '@redocly/ajv@8.11.2': + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js-replace: 1.0.1 + + '@redocly/config@0.22.0': {} + + '@redocly/openapi-core@1.34.15(supports-color@10.2.2)': + dependencies: + '@redocly/ajv': 8.11.2 + '@redocly/config': 0.22.0 + colorette: 1.4.0 + https-proxy-agent: 7.0.6(supports-color@10.2.2) + js-levenshtein: 1.1.6 + js-yaml: 4.1.1 + minimatch: 5.1.9 + pluralize: 8.0.0 + yaml-ast-parser: 0.0.43 + transitivePeerDependencies: + - supports-color + '@rolldown/binding-android-arm64@1.0.2': optional: true @@ -6088,10 +6221,10 @@ snapshots: dependencies: tslib: 2.8.1 - '@tailwindcss/typography@0.5.19(tailwindcss@3.4.19(tsx@4.22.4))': + '@tailwindcss/typography@0.5.19(tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0))': dependencies: postcss-selector-parser: 6.0.10 - tailwindcss: 3.4.19(tsx@4.22.4) + tailwindcss: 3.4.19(tsx@4.22.4)(yaml@2.9.0) '@tanstack/query-core@5.101.0': {} @@ -6220,7 +6353,7 @@ snapshots: '@typescript-eslint/types': 8.60.1 '@typescript-eslint/typescript-estree': 8.60.1(typescript@5.9.3) '@typescript-eslint/visitor-keys': 8.60.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) eslint: 9.39.4(jiti@1.21.7) typescript: 5.9.3 transitivePeerDependencies: @@ -6230,7 +6363,7 @@ snapshots: dependencies: '@typescript-eslint/tsconfig-utils': 8.60.1(typescript@5.9.3) '@typescript-eslint/types': 8.60.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -6249,7 +6382,7 @@ snapshots: '@typescript-eslint/types': 8.60.1 '@typescript-eslint/typescript-estree': 8.60.1(typescript@5.9.3) '@typescript-eslint/utils': 8.60.1(eslint@9.39.4(jiti@1.21.7))(typescript@5.9.3) - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) eslint: 9.39.4(jiti@1.21.7) ts-api-utils: 2.5.0(typescript@5.9.3) typescript: 5.9.3 @@ -6264,7 +6397,7 @@ snapshots: '@typescript-eslint/tsconfig-utils': 8.60.1(typescript@5.9.3) '@typescript-eslint/types': 8.60.1 '@typescript-eslint/visitor-keys': 8.60.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) minimatch: 10.2.5 semver: 7.8.1 tinyglobby: 0.2.17 @@ -6375,13 +6508,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))': + '@vitest/mocker@4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.8 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4) + vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0) '@vitest/pretty-format@4.1.8': dependencies: @@ -6425,10 +6558,12 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color + agent-base@7.1.4: {} + ajv-formats@3.0.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -6447,6 +6582,8 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 + ansi-colors@4.1.3: {} + ansi-regex@5.0.1: {} ansi-regex@6.2.2: {} @@ -6632,7 +6769,7 @@ snapshots: base64-js@1.5.1: {} - better-auth@1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))): + better-auth@1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))): dependencies: '@better-auth/core': 1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/drizzle-adapter': 1.6.14(@better-auth/core@1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.1)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0)) @@ -6659,7 +6796,7 @@ snapshots: pg: 8.21.0 react: 19.2.7 react-dom: 19.2.7(react@19.2.7) - vitest: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)) + vitest: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)) transitivePeerDependencies: - '@cloudflare/workers-types' - '@opentelemetry/api' @@ -6765,6 +6902,8 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + change-case@5.4.4: {} + character-entities-html4@2.1.0: {} character-entities-legacy@3.0.0: {} @@ -6799,6 +6938,8 @@ snapshots: color-name@1.1.4: {} + colorette@1.4.0: {} + colorette@2.0.20: {} comma-separated-tokens@2.0.3: {} @@ -6896,9 +7037,11 @@ snapshots: dependencies: ms: 2.1.3 - debug@4.4.3: + debug@4.4.3(supports-color@10.2.2): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 10.2.2 decode-named-character-reference@1.3.0: dependencies: @@ -7265,7 +7408,7 @@ snapshots: eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@1.21.7)): dependencies: '@nolyfill/is-core-module': 1.0.39 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) eslint: 9.39.4(jiti@1.21.7) get-tsconfig: 4.14.0 is-bun-module: 2.0.0 @@ -7390,7 +7533,7 @@ snapshots: ajv: 6.15.0 chalk: 4.1.2 cross-spawn: 7.0.6 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) escape-string-regexp: 4.0.0 eslint-scope: 8.4.0 eslint-visitor-keys: 4.2.1 @@ -7535,6 +7678,8 @@ snapshots: strnum: 2.3.0 xml-naming: 0.1.0 + fastify-plugin@5.1.0: {} + fastify@5.8.5: dependencies: '@fastify/ajv-compiler': 4.0.5 @@ -7777,7 +7922,14 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + https-proxy-agent@7.0.6(supports-color@10.2.2): + dependencies: + agent-base: 7.1.4 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -7796,6 +7948,8 @@ snapshots: imurmurhash@0.1.4: {} + index-to-position@1.2.0: {} + inherits@2.0.4: {} ini@1.3.8: {} @@ -7972,8 +8126,14 @@ snapshots: joycon@3.1.1: {} + js-levenshtein@1.1.6: {} + js-tokens@4.0.0: {} + js-yaml@4.1.1: + dependencies: + argparse: 2.0.1 + js-yaml@4.2.0: dependencies: argparse: 2.0.1 @@ -7984,6 +8144,14 @@ snapshots: dependencies: dequal: 2.0.3 + json-schema-resolver@3.0.0: + dependencies: + debug: 4.4.3(supports-color@10.2.2) + fast-uri: 3.1.2 + rfdc: 1.4.1 + transitivePeerDependencies: + - supports-color + json-schema-traverse@0.4.1: {} json-schema-traverse@1.0.0: {} @@ -8482,7 +8650,7 @@ snapshots: micromark@4.0.2: dependencies: '@types/debug': 4.1.13 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) decode-named-character-reference: 1.3.0 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 @@ -8658,6 +8826,18 @@ snapshots: optionalDependencies: zod: 4.4.3 + openapi-types@12.1.3: {} + + openapi-typescript@7.13.0(typescript@5.9.3): + dependencies: + '@redocly/openapi-core': 1.34.15(supports-color@10.2.2) + ansi-colors: 4.1.3 + change-case: 5.4.4 + parse-json: 8.3.0 + supports-color: 10.2.2 + typescript: 5.9.3 + yargs-parser: 21.1.1 + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -8705,6 +8885,12 @@ snapshots: is-decimal: 2.0.1 is-hexadecimal: 2.0.1 + parse-json@8.3.0: + dependencies: + '@babel/code-frame': 7.29.7 + index-to-position: 1.2.0 + type-fest: 4.41.0 + path-exists@4.0.0: {} path-expression-matcher@1.5.0: {} @@ -8819,6 +9005,8 @@ snapshots: optionalDependencies: fsevents: 2.3.2 + pluralize@8.0.0: {} + possible-typed-array-names@1.1.0: {} postcss-import@15.1.0(postcss@8.5.15): @@ -8833,13 +9021,14 @@ snapshots: camelcase-css: 2.0.1 postcss: 8.5.15 - postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4): + postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4)(yaml@2.9.0): dependencies: lilconfig: 3.1.3 optionalDependencies: jiti: 1.21.7 postcss: 8.5.15 tsx: 4.22.4 + yaml: 2.9.0 postcss-nested@6.2.0(postcss@8.5.15): dependencies: @@ -9504,6 +9693,8 @@ snapshots: suffix-thumb@5.0.2: {} + supports-color@10.2.2: {} + supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -9512,7 +9703,7 @@ snapshots: tabbable@6.4.0: {} - tailwindcss@3.4.19(tsx@4.22.4): + tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0): dependencies: '@alloc/quick-lru': 5.2.0 arg: 5.0.2 @@ -9531,7 +9722,7 @@ snapshots: postcss: 8.5.15 postcss-import: 15.1.0(postcss@8.5.15) postcss-js: 4.1.0(postcss@8.5.15) - postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4) + postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4)(yaml@2.9.0) postcss-nested: 6.2.0(postcss@8.5.15) postcss-selector-parser: 6.1.2 resolve: 1.22.12 @@ -9651,6 +9842,8 @@ snapshots: type-fest@0.20.2: {} + type-fest@4.41.0: {} + type@2.7.3: {} typed-array-buffer@1.0.3: @@ -9761,6 +9954,8 @@ snapshots: '@unrs/resolver-binding-win32-ia32-msvc': 1.12.2 '@unrs/resolver-binding-win32-x64-msvc': 1.12.2 + uri-js-replace@1.0.1: {} + uri-js@4.4.1: dependencies: punycode: 2.3.1 @@ -9783,7 +9978,7 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 - vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4): + vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -9796,11 +9991,12 @@ snapshots: fsevents: 2.3.3 jiti: 1.21.7 tsx: 4.22.4 + yaml: 2.9.0 - vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)): + vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.8 - '@vitest/mocker': 4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)) + '@vitest/mocker': 4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)) '@vitest/pretty-format': 4.1.8 '@vitest/runner': 4.1.8 '@vitest/snapshot': 4.1.8 @@ -9817,7 +10013,7 @@ snapshots: tinyexec: 1.2.4 tinyglobby: 0.2.17 tinyrainbow: 3.1.0 - vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4) + vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 20.19.41 @@ -9898,6 +10094,12 @@ snapshots: xtend@4.0.2: {} + yaml-ast-parser@0.0.43: {} + + yaml@2.9.0: {} + + yargs-parser@21.1.1: {} + yocto-queue@0.1.0: {} zip-stream@6.0.1: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index fcc6ecc..e94fdc1 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,6 @@ packages: - . - - compute/* + - compute-worker - docker/* allowBuilds: diff --git a/scripts/check-compute-boundary.mjs b/scripts/check-compute-boundary.mjs new file mode 100644 index 0000000..8cc6a20 --- /dev/null +++ b/scripts/check-compute-boundary.mjs @@ -0,0 +1,39 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +const root = process.cwd(); +const forbidden = [ + '@openreader/compute-core', + '@openreader/compute-worker', + 'compute/core', + 'compute/worker', + 'compute-worker/src', +]; +const failures = []; + +function walk(dir) { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + walk(fullPath); + continue; + } + if (!entry.isFile() || !/\.(ts|tsx|js|mjs|cjs)$/.test(entry.name)) continue; + const text = fs.readFileSync(fullPath, 'utf8'); + for (const pattern of forbidden) { + if (text.includes(pattern)) failures.push({ file: fullPath, pattern }); + } + } +} + +walk(path.join(root, 'src')); + +if (failures.length > 0) { + console.error('[compute-boundary] Forbidden app-to-worker source coupling detected:'); + for (const failure of failures) { + console.error(`- ${failure.pattern} in ${path.relative(root, failure.file)}`); + } + process.exit(1); +} + +console.info('[compute-boundary] OK: app communicates with compute worker through protocol/client only'); diff --git a/scripts/check-next-server-bundle.mjs b/scripts/check-next-server-bundle.mjs index 9621cb4..2b3cbcb 100644 --- a/scripts/check-next-server-bundle.mjs +++ b/scripts/check-next-server-bundle.mjs @@ -12,7 +12,8 @@ if (!fs.existsSync(serverDir)) { const forbidden = [ 'onnxruntime-node', '@huggingface/tokenizers', - '@openreader/compute-core/local-runtime', + '@openreader/compute-worker', + '/compute-worker/src/compute/', ]; const includeExt = new Set(['.js', '.mjs', '.cjs']); diff --git a/scripts/openreader-entrypoint.mjs b/scripts/openreader-entrypoint.mjs index 44588e1..957ad63 100644 --- a/scripts/openreader-entrypoint.mjs +++ b/scripts/openreader-entrypoint.mjs @@ -222,7 +222,7 @@ function spawnMainCommand(command, env) { function resolveEmbeddedWorkerLaunch() { const candidateDirs = [ path.join(process.cwd(), 'embedded-compute-worker'), - path.join(process.cwd(), 'compute', 'worker'), + path.join(process.cwd(), 'compute-worker'), ]; for (const workerDir of candidateDirs) { @@ -237,7 +237,7 @@ function resolveEmbeddedWorkerLaunch() { throw new Error( 'Could not find an embedded compute worker runtime. ' - + 'Include embedded-compute-worker/src/server.ts in the runtime image or keep compute/worker available locally.', + + 'Include embedded-compute-worker/src/server.ts in the runtime image or keep compute-worker available locally.', ); } diff --git a/src/app/api/documents/[id]/parsed/events/route.ts b/src/app/api/documents/[id]/parsed/events/route.ts index a0d3dfd..fd8e08d 100644 --- a/src/app/api/documents/[id]/parsed/events/route.ts +++ b/src/app/api/documents/[id]/parsed/events/route.ts @@ -3,7 +3,7 @@ import { and, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker'; +import { getComputeWorkerClient } from '@/lib/server/compute-worker/client'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { fetchPdfParseOperation, @@ -45,14 +45,6 @@ async function loadOwnedDocumentRow(input: { return rows[0] ?? null; } -function workerEventsUrl(baseUrl: string, opId: string, sinceEventId: string | null): string { - const url = new URL(`${baseUrl}/ops/${encodeURIComponent(opId)}/events`); - if (sinceEventId) { - url.searchParams.set('sinceEventId', sinceEventId); - } - return url.toString(); -} - export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) { const { logger } = createRequestLogger({ route: '/api/documents/[id]/parsed/events', @@ -92,17 +84,11 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string return NextResponse.json({ error: 'Operation not found' }, { status: 404 }); } - const cfg = getWorkerClientConfigFromEnv(); const lastEventId = req.headers.get('last-event-id'); const sinceEventId = req.nextUrl.searchParams.get('sinceEventId') || lastEventId; - const upstream = await fetch(workerEventsUrl(cfg.baseUrl, opId, sinceEventId), { - method: 'GET', - headers: { - Authorization: `Bearer ${cfg.token}`, - Accept: 'text/event-stream', - ...(lastEventId ? { 'Last-Event-ID': lastEventId } : {}), - }, - cache: 'no-store', + const upstream = await getComputeWorkerClient().openOperationEvents(opId, { + sinceEventId, + lastEventId, signal: req.signal, }); diff --git a/src/lib/server/compute-worker/client.ts b/src/lib/server/compute-worker/client.ts new file mode 100644 index 0000000..2f8c0b6 --- /dev/null +++ b/src/lib/server/compute-worker/client.ts @@ -0,0 +1,249 @@ +import type { + PdfLayoutRequest, + PdfLayoutResolution, + PdfLayoutResult, + WhisperAlignRequest, + WhisperAlignResult, + WorkerOperation, + WorkerOperationEvent, +} from './protocol'; +import { isWorkerOperation } from './protocol'; + +class WorkerHttpError extends Error { + constructor( + message: string, + readonly status: number, + readonly retryAfterMs: number | null = null, + ) { + super(message); + this.name = 'WorkerHttpError'; + } +} + +function readRequiredEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required for compute worker client`); + return value; +} + +function normalizeWorkerBaseUrl(raw: string): string { + const trimmed = raw.trim(); + if (!trimmed) throw new Error('COMPUTE_WORKER_URL is empty'); + const withScheme = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(trimmed) + ? trimmed + : (/^(localhost|127(?:\.\d{1,3}){3})(:\d+)?(\/|$)/.test(trimmed) + ? `http://${trimmed}` + : `https://${trimmed}`); + const parsed = new URL(withScheme); + parsed.pathname = parsed.pathname.replace(/\/+$/, ''); + return parsed.toString().replace(/\/+$/, ''); +} + +export function getComputeWorkerConfigFromEnv(): { baseUrl: string; token: string } { + return { + baseUrl: normalizeWorkerBaseUrl(readRequiredEnv('COMPUTE_WORKER_URL')), + token: readRequiredEnv('COMPUTE_WORKER_TOKEN'), + }; +} + +export function isComputeWorkerAvailable(): boolean { + try { + getComputeWorkerConfigFromEnv(); + return true; + } catch { + return false; + } +} + +function readPositiveIntEnv(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const parsed = Number(raw); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback; +} + +function alignmentWaitTimeoutMs(): number { + return Math.max(60_000, readPositiveIntEnv('COMPUTE_WHISPER_TIMEOUT_MS', 30_000) + 15_000); +} + +function parseRetryAfterMs(value: string | null): number | null { + if (!value) return null; + const seconds = Number(value); + if (Number.isFinite(seconds)) return Math.max(0, Math.floor(seconds * 1000)); + const date = Date.parse(value); + return Number.isNaN(date) ? null : Math.max(0, date - Date.now()); +} + +function parseSseFrame(frame: string): { id: number | null; data: string | null } { + let id: number | null = null; + const data: string[] = []; + for (const line of frame.split(/\r?\n/)) { + if (line.startsWith('id:')) { + const parsed = Number(line.slice(3).trim()); + if (Number.isFinite(parsed) && parsed >= 0) id = Math.floor(parsed); + } else if (line.startsWith('data:')) { + data.push(line.slice(5).trimStart()); + } + } + return { id, data: data.length > 0 ? data.join('\n') : null }; +} + +function isTerminal(operation: WorkerOperation): boolean { + return operation.status === 'succeeded' || operation.status === 'failed'; +} + +export class ComputeWorkerClient { + private readonly baseUrl: string; + private readonly token: string; + + constructor(config = getComputeWorkerConfigFromEnv()) { + this.baseUrl = config.baseUrl; + this.token = config.token; + } + + async alignWords(input: WhisperAlignRequest): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= 2; attempt += 1) { + try { + const operation = await this.requestJson>( + 'POST', + '/v1/whisper-align/operations', + input, + ); + const final = isTerminal(operation) + ? operation + : await this.waitForOperation(operation.opId, alignmentWaitTimeoutMs()); + if (final.status !== 'succeeded' || !final.result) { + throw new Error(final.error?.message || 'Whisper worker operation did not complete'); + } + return final.result; + } catch (error) { + lastError = error; + if (attempt >= 2 || !this.shouldRetry(error)) break; + const delay = error instanceof WorkerHttpError && error.retryAfterMs !== null + ? error.retryAfterMs + : attempt * 250; + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + throw lastError instanceof Error ? lastError : new Error('Unknown compute worker failure'); + } + + createPdfLayoutOperation(input: PdfLayoutRequest): Promise> { + return this.requestJson('POST', '/v1/pdf-layout/operations', input); + } + + resolvePdfLayout(input: PdfLayoutRequest): Promise { + return this.requestJson('POST', '/v1/pdf-layout/resolve', { + documentId: input.documentId, + namespace: input.namespace, + documentObjectKey: input.documentObjectKey, + }); + } + + async getOperation(opId: string): Promise | null> { + try { + return await this.requestJson('GET', `/v1/operations/${encodeURIComponent(opId)}`); + } catch (error) { + if (error instanceof WorkerHttpError && error.status === 404) return null; + throw error; + } + } + + async openOperationEvents(opId: string, input?: { + sinceEventId?: string | null; + lastEventId?: string | null; + signal?: AbortSignal; + }): Promise { + const url = new URL(`${this.baseUrl}/v1/operations/${encodeURIComponent(opId)}/events`); + if (input?.sinceEventId) url.searchParams.set('sinceEventId', input.sinceEventId); + return fetch(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${this.token}`, + Accept: 'text/event-stream', + ...(input?.lastEventId ? { 'Last-Event-ID': input.lastEventId } : {}), + }, + cache: 'no-store', + signal: input?.signal, + }); + } + + private async requestJson(method: 'GET' | 'POST', path: string, body?: unknown): Promise { + const response = await fetch(`${this.baseUrl}${path}`, { + method, + headers: { + Authorization: `Bearer ${this.token}`, + Accept: 'application/json', + ...(method === 'POST' ? { 'Content-Type': 'application/json' } : {}), + }, + ...(method === 'POST' ? { body: JSON.stringify(body ?? {}) } : {}), + cache: 'no-store', + }); + if (!response.ok) { + const detail = await response.text().catch(() => ''); + throw new WorkerHttpError( + `Worker request failed (${method} ${path}): ${response.status}${detail ? ` ${detail}` : ''}`, + response.status, + parseRetryAfterMs(response.headers.get('retry-after')), + ); + } + const parsed = await response.json() as T; + return parsed; + } + + private shouldRetry(error: unknown): boolean { + if (error instanceof WorkerHttpError) { + return error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504; + } + return error instanceof Error && /network|timeout|fetch failed/i.test(error.message); + } + + private async waitForOperation(opId: string, timeoutMs: number): Promise> { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + let lastEventId: number | null = null; + try { + while (!controller.signal.aborted) { + const response = await this.openOperationEvents(opId, { + sinceEventId: lastEventId === null ? null : String(lastEventId), + lastEventId: lastEventId === null ? null : String(lastEventId), + signal: controller.signal, + }); + if (!response.ok || !response.body) { + throw new WorkerHttpError(`Worker event stream failed: ${response.status}`, response.status); + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + while (true) { + const end = buffer.indexOf('\n\n'); + if (end < 0) break; + const frame = parseSseFrame(buffer.slice(0, end)); + buffer = buffer.slice(end + 2); + if (frame.id !== null) lastEventId = frame.id; + if (!frame.data) continue; + const event = JSON.parse(frame.data) as WorkerOperationEvent; + if (!event?.snapshot || !isWorkerOperation(event.snapshot) || event.snapshot.opId !== opId) continue; + if (isTerminal(event.snapshot)) return event.snapshot; + } + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error(`Operation stream timed out for ${opId}`); + } finally { + clearTimeout(timeout); + } + } +} + +let client: ComputeWorkerClient | null = null; + +export function getComputeWorkerClient(): ComputeWorkerClient { + client ??= new ComputeWorkerClient(); + return client; +} diff --git a/src/lib/server/compute-worker/generated.ts b/src/lib/server/compute-worker/generated.ts new file mode 100644 index 0000000..feb2060 --- /dev/null +++ b/src/lib/server/compute-worker/generated.ts @@ -0,0 +1,700 @@ +/** + * This file was auto-generated by openapi-typescript. + * Do not make direct changes to the file. + */ + +export interface paths { + "/health/live": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + ok: boolean; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/health/ready": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + ok: boolean; + natsConnected: boolean; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/whisper-align/operations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + text: string; + lang?: string; + cacheKey?: string; + audioObjectKey: string; + }; + }; + }; + responses: { + /** @description Default Response */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/pdf-layout/operations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + documentId: string; + namespace: string | null; + documentObjectKey: string; + replaceToken?: string; + }; + }; + }; + responses: { + /** @description Default Response */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/pdf-layout/resolve": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + documentId: string; + namespace: string | null; + documentObjectKey: string; + }; + }; + }; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + artifact: { + objectKey: string; + } | null; + operation: { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + } | null; + }; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/operations/{opId}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header?: never; + path: { + opId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + /** @description Default Response */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/operations/{opId}/events": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: { + sinceEventId?: string | number; + }; + header?: never; + path: { + opId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Server-sent PublicOperationEvent stream */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": string; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + /** @description Default Response */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: { + ParsedPdfDocument: { + /** @enum {number} */ + schemaVersion: 1; + documentId: string; + parserVersion: string; + parsedAt: number; + pages: { + pageNumber: number; + width: number; + height: number; + blocks: { + id: string; + /** @enum {string} */ + kind: "abstract" | "algorithm" | "aside_text" | "chart" | "content" | "formula" | "doc_title" | "figure_title" | "footer" | "footnote" | "formula_number" | "header" | "image" | "number" | "paragraph_title" | "reference" | "reference_content" | "seal" | "table" | "text" | "vision_footnote"; + fragments: { + page: number; + bbox: [ + number, + number, + number, + number + ]; + text: string; + readingOrder: number; + modelConfidence?: number; + }[]; + text: string; + parentSectionId?: string; + }[]; + }[]; + }; + TTSSentenceAlignment: { + sentence: string; + sentenceIndex: number; + words: { + text: string; + startSec: number; + endSec: number; + charStart: number; + charEnd: number; + }[]; + }; + ArtifactReference: { + objectKey: string; + }; + ErrorResponse: { + error: string; + } & { + [key: string]: unknown; + }; + OperationError: { + message: string; + code?: string; + }; + PdfLayoutProgress: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + PublicOperation: { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + PublicOperationEvent: { + eventId: number; + snapshot: { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + }; + PdfLayoutResolution: { + artifact: { + objectKey: string; + } | null; + operation: { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + } | null; + }; + }; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export type operations = Record; diff --git a/src/lib/server/compute-worker/protocol.ts b/src/lib/server/compute-worker/protocol.ts new file mode 100644 index 0000000..73b196c --- /dev/null +++ b/src/lib/server/compute-worker/protocol.ts @@ -0,0 +1,57 @@ +import type { components, paths } from './generated'; + +export type ParsedPdfDocument = components['schemas']['ParsedPdfDocument']; +export type ParsedPdfPage = ParsedPdfDocument['pages'][number]; +export type ParsedPdfBlock = ParsedPdfPage['blocks'][number]; +export type ParsedPdfBlockKind = ParsedPdfBlock['kind']; +export type ParsedPdfBlockFragment = ParsedPdfBlock['fragments'][number]; + +export type TTSSentenceAlignment = components['schemas']['TTSSentenceAlignment']; +export type TTSSentenceWord = TTSSentenceAlignment['words'][number]; + +export type PdfLayoutProgress = NonNullable; +export type WorkerOperationStatus = components['schemas']['PublicOperation']['status']; +export type WorkerOperationSubject = components['schemas']['PublicOperation']['subject']; + +export type WorkerOperation = + Omit + & { result?: Result }; + +export type WorkerOperationEvent = { + eventId: number; + snapshot: WorkerOperation; +}; + +export type WhisperAlignRequest = + paths['/v1/whisper-align/operations']['post']['requestBody']['content']['application/json']; +export type PdfLayoutRequest = + paths['/v1/pdf-layout/operations']['post']['requestBody']['content']['application/json']; +export type PdfLayoutResolveRequest = + paths['/v1/pdf-layout/resolve']['post']['requestBody']['content']['application/json']; + +export type WhisperAlignResult = { + alignments: TTSSentenceAlignment[]; + timing?: components['schemas']['PublicOperation']['timing']; +}; + +export type PdfLayoutResult = { + parsedObjectKey: string; + timing?: components['schemas']['PublicOperation']['timing']; +}; + +export type PdfLayoutResolution = { + artifact: { objectKey: string } | null; + operation: WorkerOperation | null; +}; + +export function isWorkerOperation(value: unknown): value is WorkerOperation { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + return typeof record.opId === 'string' + && !!record.subject + && typeof record.subject === 'object' + && typeof (record.subject as Record).kind === 'string' + && typeof record.status === 'string' + && typeof record.queuedAt === 'number' + && typeof record.updatedAt === 'number'; +} diff --git a/src/lib/server/compute/abort-like-error.ts b/src/lib/server/compute/abort-like-error.ts deleted file mode 100644 index 2ce848b..0000000 --- a/src/lib/server/compute/abort-like-error.ts +++ /dev/null @@ -1,9 +0,0 @@ -export function isAbortLikeError(error: unknown): boolean { - if (!error) return false; - if (error instanceof DOMException) return error.name === 'AbortError'; - if (error instanceof Error) return error.name === 'AbortError' || error.message === 'This operation was aborted'; - if (typeof error === 'object' && error !== null && 'name' in error) { - return (error as { name?: unknown }).name === 'AbortError'; - } - return false; -} diff --git a/src/lib/server/compute/index.ts b/src/lib/server/compute/index.ts deleted file mode 100644 index ac62a56..0000000 --- a/src/lib/server/compute/index.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { ComputeBackend } from '@/lib/server/compute/types'; -import { isWorkerClientConfigAvailable, WorkerComputeBackend } from '@/lib/server/compute/worker'; - -let backendPromise: Promise | null = null; - -export async function getCompute(): Promise { - if (!backendPromise) { - backendPromise = Promise.resolve(new WorkerComputeBackend()).catch((error) => { - backendPromise = null; - throw error; - }); - } - return backendPromise; -} - -export function isComputeAvailable(): boolean { - return isWorkerClientConfigAvailable(); -} diff --git a/src/lib/server/compute/types.ts b/src/lib/server/compute/types.ts deleted file mode 100644 index ffd0951..0000000 --- a/src/lib/server/compute/types.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { - TTSAudioBuffer, - TTSSentenceAlignment, - ParsedPdfDocument, -} from '@openreader/compute-core/types'; -import type { PdfLayoutProgress, WhisperAlignJobBase } from '@openreader/compute-core/api-contracts'; -import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; - -export interface WhisperAlignInput extends WhisperAlignJobBase { - audioBuffer?: TTSAudioBuffer; - audioObjectKey?: string; -} - -export interface WhisperAlignResult { - alignments: TTSSentenceAlignment[]; -} - -export interface PdfLayoutInput { - documentId: string; - namespace?: string | null; - documentObjectKey?: string; - pdfBytes?: ArrayBuffer; - forceToken?: string; - onProgress?: (progress: PdfLayoutProgress) => void | Promise; - onWorkerSnapshot?: (snapshot: WorkerOperationState) => void | Promise; -} - -export type PdfLayoutResult = - | { parsed: ParsedPdfDocument; parsedObjectKey?: never } - | { parsed?: never; parsedObjectKey: string }; - -export interface ComputeBackend { - mode: 'worker'; - alignWords(input: WhisperAlignInput): Promise; - parsePdfLayout(input: PdfLayoutInput): Promise; -} diff --git a/src/lib/server/compute/worker-op-create.ts b/src/lib/server/compute/worker-op-create.ts deleted file mode 100644 index 433b68e..0000000 --- a/src/lib/server/compute/worker-op-create.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { getWorkerClientConfigFromEnv, buildPdfOpKey } from '@/lib/server/compute/worker'; -import type { PdfLayoutInput } from '@/lib/server/compute/types'; -import type { - PdfLayoutJobResult, - WorkerOperationState, -} from '@openreader/compute-core/api-contracts'; - -type CreatePdfWorkerOpInput = - Pick - & { documentObjectKey: string }; - -const CREATE_OP_TIMEOUT_MS = 10_000; - -export async function createOrReusePdfWorkerOperation( - input: CreatePdfWorkerOpInput, -): Promise> { - const cfg = getWorkerClientConfigFromEnv(); - const opKey = buildPdfOpKey({ - documentId: input.documentId, - namespace: input.namespace, - documentObjectKey: input.documentObjectKey, - forceToken: input.forceToken, - }); - - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), CREATE_OP_TIMEOUT_MS); - try { - const res = await fetch(`${cfg.baseUrl}/ops`, { - method: 'POST', - headers: { - Authorization: `Bearer ${cfg.token}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - kind: 'pdf_layout', - opKey, - payload: { - documentId: input.documentId, - namespace: input.namespace, - documentObjectKey: input.documentObjectKey, - }, - }), - cache: 'no-store', - signal: controller.signal, - }); - - if (!res.ok) { - const detail = await res.text().catch(() => ''); - throw new Error(`Worker op create failed: ${res.status}${detail ? ` ${detail}` : ''}`); - } - - const parsed = await res.json() as WorkerOperationState; - if (!parsed || typeof parsed !== 'object' || typeof parsed.opId !== 'string' || !parsed.opId.trim()) { - throw new Error('Worker op create returned invalid response'); - } - return parsed; - } finally { - clearTimeout(timeout); - } -} diff --git a/src/lib/server/compute/worker-op-state.ts b/src/lib/server/compute/worker-op-state.ts deleted file mode 100644 index a6debf0..0000000 --- a/src/lib/server/compute/worker-op-state.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker'; -import type { WorkerOperationState } from '@openreader/compute-core/api-contracts'; -import { serverLogger } from '@/lib/server/logger'; -import { logDegraded } from '@/lib/server/errors/logging'; - -const WORKER_OP_REQUEST_TIMEOUT_MS = 2_500; - -export async function fetchWorkerOperationState( - opId: string | null | undefined, -): Promise | null> { - const normalized = opId?.trim(); - if (!normalized) return null; - - let cfg: { baseUrl: string; token: string }; - try { - cfg = getWorkerClientConfigFromEnv(); - } catch (error) { - logDegraded(serverLogger, { - event: 'compute.worker_op_state.config.invalid', - msg: 'Worker client env missing/invalid', - step: 'read_worker_config', - context: { opId: normalized }, - error, - }); - return null; - } - - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), WORKER_OP_REQUEST_TIMEOUT_MS); - - try { - const res = await fetch(`${cfg.baseUrl}/ops/${encodeURIComponent(normalized)}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${cfg.token}`, - Accept: 'application/json', - }, - cache: 'no-store', - signal: controller.signal, - }); - - if (!res.ok) { - const upstreamResponseBody = await res.text().catch(() => ''); - logDegraded(serverLogger, { - event: 'compute.worker_op_state.fetch.failed', - msg: 'Worker op request failed', - step: 'fetch_worker_op', - context: { - opId: normalized, - status: res.status, - upstreamResponseBody, - }, - error: { - name: 'WorkerOpStateFetchFailed', - message: `Worker op request failed with status ${res.status}`, - }, - }); - return null; - } - const parsed = await res.json() as WorkerOperationState; - if (!parsed || typeof parsed !== 'object' || parsed.opId !== normalized) { - logDegraded(serverLogger, { - event: 'compute.worker_op_state.response.invalid', - msg: 'Worker op response invalid', - step: 'validate_worker_op_response', - context: { opId: normalized }, - }); - return null; - } - return parsed; - } catch (error) { - logDegraded(serverLogger, { - event: 'compute.worker_op_state.fetch.error', - msg: 'Worker op request threw', - step: 'fetch_worker_op', - context: { opId: normalized }, - error, - }); - return null; - } finally { - clearTimeout(timeout); - } -} - -export async function fetchWorkerOperationStateByKey( - opKey: string | null | undefined, -): Promise | null> { - const normalized = opKey?.trim(); - if (!normalized) return null; - - let cfg: { baseUrl: string; token: string }; - try { - cfg = getWorkerClientConfigFromEnv(); - } catch (error) { - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.config.invalid', - msg: 'Worker client env missing/invalid', - step: 'read_worker_config', - context: { opKey: normalized }, - error, - }); - return null; - } - - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), WORKER_OP_REQUEST_TIMEOUT_MS); - - try { - const res = await fetch(`${cfg.baseUrl}/ops/lookup`, { - method: 'POST', - headers: { - Authorization: `Bearer ${cfg.token}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ opKey: normalized }), - cache: 'no-store', - signal: controller.signal, - }); - - if (res.status === 404) { - return null; - } - if (!res.ok) { - const upstreamResponseBody = await res.text().catch(() => ''); - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.fetch.failed', - msg: 'Worker op lookup failed', - step: 'fetch_worker_op_by_key', - context: { - opKey: normalized, - status: res.status, - upstreamResponseBody, - }, - error: { - name: 'WorkerOpLookupFailed', - message: `Worker op lookup failed with status ${res.status}`, - }, - }); - return null; - } - - const parsed = await res.json() as WorkerOperationState; - if (!parsed || typeof parsed !== 'object' || typeof parsed.opId !== 'string' || !parsed.opId.trim()) { - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.response.invalid', - msg: 'Worker op lookup response invalid', - step: 'validate_worker_op_lookup_response', - context: { opKey: normalized }, - }); - return null; - } - if (typeof parsed.opKey !== 'string' || parsed.opKey.trim() !== normalized) { - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.response.mismatch', - msg: 'Worker op lookup response did not match requested op key', - step: 'validate_worker_op_lookup_response', - context: { - opKey: normalized, - responseOpId: parsed.opId, - responseOpKey: typeof parsed.opKey === 'string' ? parsed.opKey : null, - }, - }); - return null; - } - return parsed; - } catch (error) { - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.fetch.error', - msg: 'Worker op lookup threw', - step: 'fetch_worker_op_by_key', - context: { opKey: normalized }, - error, - }); - return null; - } finally { - clearTimeout(timeout); - } -} diff --git a/src/lib/server/compute/worker.ts b/src/lib/server/compute/worker.ts deleted file mode 100644 index d97e34e..0000000 --- a/src/lib/server/compute/worker.ts +++ /dev/null @@ -1,706 +0,0 @@ -import { createHash, randomUUID } from 'node:crypto'; -import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types'; -import { parseSseEventId, parseSsePayload } from '@openreader/compute-core'; -import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core'; -import { PDF_PARSER_VERSION } from '@openreader/compute-core/api-contracts'; -import { errorToLog, serverLogger } from '@/lib/server/logger'; -import { logDegraded, logServerError } from '@/lib/server/errors/logging'; -import type { - PdfLayoutJobRequest, - PdfLayoutJobResult, - WorkerOperationEvent, - WhisperAlignJobRequest, - WhisperAlignJobResult, - WorkerOperationState, -} from '@openreader/compute-core/api-contracts'; - -class WorkerHttpError extends Error { - status: number; - retryAfterMs: number | null; - - constructor(message: string, status: number, retryAfterMs: number | null = null) { - super(message); - this.name = 'WorkerHttpError'; - this.status = status; - this.retryAfterMs = retryAfterMs; - } -} - -const DEFAULT_RETRIES = 2; -const MAX_LOG_DETAIL_CHARS = 600; -const LOG_EVENTS = new Set([ - 'align.request.failed', - 'align.request.attempt_error', - 'pdf_layout.request.failed', - 'pdf_layout.request.attempt_error', - 'http.request.failed', - 'sse.wait.http_failed', - 'sse.wait.ended_without_terminal', - 'sse.wait.failed', -]); - -type WorkerLogLevel = 'info' | 'warn' | 'error'; - -function truncateForLog(value: string, maxChars = MAX_LOG_DETAIL_CHARS): string { - if (value.length <= maxChars) return value; - return `${value.slice(0, maxChars)}...`; -} - -function logWorker(level: WorkerLogLevel, event: string, fields: Record): void { - if (!LOG_EVENTS.has(event)) return; - const eventName = `compute.worker_client.${event}`; - const msg = `Worker client ${event}`; - if (level === 'error') { - logServerError(serverLogger, { - event: eventName, - msg, - error: fields.error ?? new Error(msg), - context: { - operation: 'compute_worker_client', - ...fields, - }, - normalize: { code: 'COMPUTE_WORKER_CLIENT_EVENT_FAILED', errorClass: 'upstream' }, - }); - return; - } - if (level === 'warn') { - logDegraded(serverLogger, { - event: eventName, - msg, - step: 'worker_client', - context: { - operation: 'compute_worker_client', - ...fields, - }, - error: fields.error, - }); - return; - } - serverLogger.info({ - event: eventName, - operation: 'compute_worker_client', - ...fields, - }, `Worker client ${event}`); -} - -function opSummary(value: unknown): Record { - if (!value || typeof value !== 'object') return {}; - const record = value as Record; - const summary: Record = {}; - if (typeof record.opId === 'string') summary.opId = record.opId; - if (typeof record.status === 'string') summary.status = record.status; - if (typeof record.jobId === 'string') summary.jobId = record.jobId; - if (typeof record.updatedAt === 'string') summary.updatedAt = record.updatedAt; - return summary; -} - -function readRequiredEnv(name: string): string { - const value = process.env[name]?.trim(); - if (!value) throw new Error(`${name} is required for compute worker client`); - return value; -} - -function normalizeWorkerBaseUrl(raw: string): string { - const trimmed = raw.trim(); - if (!trimmed) throw new Error('COMPUTE_WORKER_URL is empty'); - - const withScheme = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(trimmed) - ? trimmed - : (/^(localhost|127(?:\.\d{1,3}){3})(:\d+)?(\/|$)/.test(trimmed) - ? `http://${trimmed}` - : `https://${trimmed}`); - - let parsed: URL; - try { - parsed = new URL(withScheme); - } catch { - throw new Error( - `Invalid COMPUTE_WORKER_URL="${raw}". Expected full URL like https://example.com (or http://localhost:4000).`, - ); - } - - parsed.pathname = parsed.pathname.replace(/\/+$/, ''); - return parsed.toString().replace(/\/+$/, ''); -} - -export function getWorkerClientConfigFromEnv(): { baseUrl: string; token: string } { - return { - baseUrl: normalizeWorkerBaseUrl(readRequiredEnv('COMPUTE_WORKER_URL')), - token: readRequiredEnv('COMPUTE_WORKER_TOKEN'), - }; -} - -export function isWorkerClientConfigAvailable(): boolean { - try { - getWorkerClientConfigFromEnv(); - return true; - } catch { - return false; - } -} - -function parseRetryAfterMs(value: string | null): number | null { - if (!value) return null; - const asNum = Number(value); - if (Number.isFinite(asNum)) { - return Math.max(0, Math.floor(asNum * 1000)); - } - const when = Date.parse(value); - if (Number.isNaN(when)) return null; - return Math.max(0, when - Date.now()); -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function shouldRetry(error: unknown): boolean { - if (error instanceof WorkerHttpError) { - return error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504; - } - if (error instanceof Error) { - const msg = error.message.toLowerCase(); - return msg.includes('network') || msg.includes('timeout') || msg.includes('fetch failed'); - } - return false; -} - -function isTerminalStatus(status: string): boolean { - return status === 'succeeded' || status === 'failed'; -} - -function sha256Hex(input: string): string { - return createHash('sha256').update(input).digest('hex'); -} - -function buildWhisperOpKey(input: WhisperAlignInput): string { - const cacheKey = input.cacheKey?.trim(); - if (cacheKey) { - return `whisper_align|v1|cache|${cacheKey}|${input.audioObjectKey}`; - } - return [ - 'whisper_align', - 'v1', - input.audioObjectKey, - input.lang ?? '', - sha256Hex(input.text), - ].join('|'); -} - -export function buildPdfOpKey(input: PdfLayoutInput): string { - return [ - 'pdf_layout', - 'v1', - PDF_PARSER_VERSION, - input.documentId, - input.namespace ?? '', - input.documentObjectKey ?? '', - input.forceToken?.trim() || '', - ].join('|'); -} - -type RetryMeta = { - attempt: number, - maxAttempts: number, - willRetry: boolean, - delayMs: number | null, - error: unknown, -}; - -async function withRetries( - attempts: number, - operation: (attempt: number) => Promise, - onAttemptError?: (meta: RetryMeta) => void, -): Promise { - let lastError: unknown = null; - for (let attemptIndex = 0; attemptIndex < attempts; attemptIndex += 1) { - const attempt = attemptIndex + 1; - try { - return await operation(attempt); - } catch (error) { - lastError = error; - const willRetry = attemptIndex < attempts - 1 && shouldRetry(error); - let delayMs: number | null = null; - if (willRetry) { - if (error instanceof WorkerHttpError && typeof error.retryAfterMs === 'number') { - delayMs = error.retryAfterMs; - } else { - delayMs = attempt * 250; - } - } - onAttemptError?.({ - attempt, - maxAttempts: attempts, - willRetry, - delayMs, - error, - }); - if (!willRetry) break; - await sleep(delayMs ?? 0); - } - } - throw lastError instanceof Error ? lastError : new Error('Unknown worker compute failure'); -} - -export class WorkerComputeBackend implements ComputeBackend { - readonly mode = 'worker' as const; - private readonly baseUrl: string; - private readonly token: string; - private readonly waitTimeoutMsByKind: Record<'whisper_align' | 'pdf_layout', number>; - private readonly retries: number; - - constructor() { - const cfg = getWorkerClientConfigFromEnv(); - this.baseUrl = cfg.baseUrl; - this.token = cfg.token; - this.waitTimeoutMsByKind = { - whisper_align: getWorkerClientWaitTimeoutMs('whisper_align'), - pdf_layout: getWorkerClientWaitTimeoutMs('pdf_layout'), - }; - this.retries = DEFAULT_RETRIES; - } - - async alignWords(input: WhisperAlignInput): Promise { - if (!input.audioObjectKey) { - throw new Error('Worker compute alignment requires audioObjectKey'); - } - const payload: WhisperAlignJobRequest = { - text: input.text, - lang: input.lang, - cacheKey: input.cacheKey, - audioObjectKey: input.audioObjectKey, - }; - const traceId = randomUUID(); - const opKey = buildWhisperOpKey(input); - const opKeyHash = sha256Hex(opKey).slice(0, 16); - const startedAt = Date.now(); - logWorker('info', 'align.request.start', { - traceId, - kind: 'whisper_align', - opKeyHash, - audioObjectKey: input.audioObjectKey, - cacheKey: input.cacheKey ?? null, - lang: input.lang ?? null, - textLength: input.text.length, - waitTimeoutMs: this.waitTimeoutMsByKind.whisper_align, - maxRetries: this.retries, - }); - - try { - const result = await withRetries(this.retries, async (attempt) => { - const op = await this.requestJson>('POST', '/ops', { - kind: 'whisper_align', - opKey, - payload, - }, { - traceId, - kind: 'whisper_align', - opKeyHash, - attempt, - }); - - const final = isTerminalStatus(op.status) - ? op - : await this.waitForOperation(op.opId, { - traceId, - kind: 'whisper_align', - opKeyHash, - attempt, - waitTimeoutMs: this.waitTimeoutMsByKind.whisper_align, - }); - - if (final.status !== 'succeeded' || !final.result) { - throw new Error(final.error?.message || 'Whisper worker operation did not complete'); - } - return { alignments: final.result.alignments }; - }, ({ attempt, maxAttempts, willRetry, delayMs, error }) => { - logWorker(willRetry ? 'warn' : 'error', 'align.request.attempt_error', { - traceId, - kind: 'whisper_align', - opKeyHash, - attempt, - maxAttempts, - willRetry, - delayMs, - error: errorToLog(error), - }); - }); - - logWorker('info', 'align.request.succeeded', { - traceId, - kind: 'whisper_align', - opKeyHash, - durationMs: Date.now() - startedAt, - }); - return result; - } catch (error) { - logWorker('error', 'align.request.failed', { - traceId, - kind: 'whisper_align', - opKeyHash, - durationMs: Date.now() - startedAt, - error: errorToLog(error), - }); - throw error; - } - } - - async parsePdfLayout(input: PdfLayoutInput) { - if (!input.documentObjectKey) { - throw new Error('Worker compute PDF layout requires documentObjectKey'); - } - const payload: PdfLayoutJobRequest = { - documentId: input.documentId, - namespace: input.namespace ?? null, - documentObjectKey: input.documentObjectKey, - }; - const traceId = randomUUID(); - const opKey = buildPdfOpKey(input); - const opKeyHash = sha256Hex(opKey).slice(0, 16); - const startedAt = Date.now(); - logWorker('info', 'pdf_layout.request.start', { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - namespace: input.namespace ?? null, - documentObjectKey: input.documentObjectKey, - waitTimeoutMs: this.waitTimeoutMsByKind.pdf_layout, - maxRetries: this.retries, - }); - - try { - const result = await withRetries(this.retries, async (attempt) => { - const op = await this.requestJson>('POST', '/ops', { - kind: 'pdf_layout', - opKey, - payload, - }, { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - attempt, - }); - await input.onWorkerSnapshot?.(op); - - const final = isTerminalStatus(op.status) - ? op - : await this.waitForOperation(op.opId, { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - attempt, - waitTimeoutMs: this.waitTimeoutMsByKind.pdf_layout, - onSnapshot: (snapshot) => { - void input.onWorkerSnapshot?.(snapshot); - if (snapshot.progress) { - void input.onProgress?.(snapshot.progress); - } - }, - }); - - if (final.status !== 'succeeded' || !final.result) { - throw new Error(final.error?.message || 'PDF layout worker operation did not complete'); - } - if (final.result.parsedObjectKey) { - return { parsedObjectKey: final.result.parsedObjectKey }; - } - if (final.result.parsed) { - return { parsed: final.result.parsed }; - } - throw new Error('PDF layout worker operation completed without parsed output'); - }, ({ attempt, maxAttempts, willRetry, delayMs, error }) => { - logWorker(willRetry ? 'warn' : 'error', 'pdf_layout.request.attempt_error', { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - attempt, - maxAttempts, - willRetry, - delayMs, - error: errorToLog(error), - }); - }); - - logWorker('info', 'pdf_layout.request.succeeded', { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - durationMs: Date.now() - startedAt, - }); - return result; - } catch (error) { - logWorker('error', 'pdf_layout.request.failed', { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - durationMs: Date.now() - startedAt, - error: errorToLog(error), - }); - throw error; - } - } - - private async requestJson( - method: 'GET' | 'POST', - path: string, - body?: unknown, - context: Record = {}, - ): Promise { - const startedAt = Date.now(); - const traceId = typeof context.traceId === 'string' ? context.traceId : randomUUID(); - logWorker('info', 'http.request.start', { - ...context, - traceId, - method, - path, - }); - const res = await fetch(`${this.baseUrl}${path}`, { - method, - headers: { - Authorization: `Bearer ${this.token}`, - 'x-openreader-trace-id': traceId, - ...(method === 'POST' ? { 'Content-Type': 'application/json' } : {}), - }, - ...(method === 'POST' ? { body: JSON.stringify(body ?? {}) } : {}), - }); - - if (!res.ok) { - const retryAfterMs = parseRetryAfterMs(res.headers.get('retry-after')); - const upstreamResponseBody = await res.text().catch(() => ''); - const requestError = new WorkerHttpError( - `Worker request failed (${method} ${path}): ${res.status}${upstreamResponseBody ? ` ${upstreamResponseBody}` : ''}`, - res.status, - retryAfterMs, - ); - logWorker(res.status >= 500 ? 'warn' : 'error', 'http.request.failed', { - ...context, - traceId, - method, - path, - status: res.status, - retryAfterMs, - durationMs: Date.now() - startedAt, - upstreamResponseBody: truncateForLog(upstreamResponseBody), - error: errorToLog(requestError), - }); - throw requestError; - } - - const parsed = await res.json() as T; - const operationSummary = opSummary(parsed); - logWorker('info', 'http.request.succeeded', { - ...context, - traceId, - method, - path, - httpStatus: res.status, - durationMs: Date.now() - startedAt, - ...operationSummary, - }); - return parsed; - } - - private async waitForOperation( - opId: string, - context: Record & { - waitTimeoutMs?: number; - onSnapshot?: (snapshot: WorkerOperationState) => void - } = {}, - ): Promise> { - const waitTimeoutMs = typeof context.waitTimeoutMs === 'number' - ? context.waitTimeoutMs - : this.waitTimeoutMsByKind.whisper_align; - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), waitTimeoutMs); - const startedAt = Date.now(); - const traceId = typeof context.traceId === 'string' ? context.traceId : randomUUID(); - logWorker('info', 'sse.wait.start', { - ...context, - traceId, - opId, - waitTimeoutMs, - }); - - try { - let latest: WorkerOperationState | null = null; - let lastEventId: number | null = null; - let eventCount = 0; - let lastStatus: string | null = null; - - while (!controller.signal.aborted) { - const query = lastEventId && lastEventId > 0 - ? `?sinceEventId=${encodeURIComponent(String(lastEventId))}` - : ''; - const res = await fetch(`${this.baseUrl}/ops/${encodeURIComponent(opId)}/events${query}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${this.token}`, - Accept: 'text/event-stream', - 'x-openreader-trace-id': traceId, - ...(lastEventId && lastEventId > 0 ? { 'Last-Event-ID': String(lastEventId) } : {}), - }, - signal: controller.signal, - }); - - if (!res.ok) { - const retryAfterMs = parseRetryAfterMs(res.headers.get('retry-after')); - const upstreamResponseBody = await res.text().catch(() => ''); - const requestError = new WorkerHttpError( - `Worker request failed (GET /ops/${encodeURIComponent(opId)}/events): ${res.status}${upstreamResponseBody ? ` ${upstreamResponseBody}` : ''}`, - res.status, - retryAfterMs, - ); - logWorker(res.status >= 500 ? 'warn' : 'error', 'sse.wait.http_failed', { - ...context, - traceId, - opId, - status: res.status, - retryAfterMs, - durationMs: Date.now() - startedAt, - upstreamResponseBody: truncateForLog(upstreamResponseBody), - error: errorToLog(requestError), - }); - throw requestError; - } - - if (!res.body) { - logWorker('error', 'sse.wait.no_body', { - ...context, - traceId, - opId, - durationMs: Date.now() - startedAt, - }); - throw new Error('Worker operation stream response has no body'); - } - - logWorker('info', 'sse.wait.connected', { - ...context, - traceId, - opId, - status: res.status, - }); - const reader = res.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ''; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - buffer += decoder.decode(value, { stream: true }); - - while (true) { - const frameEnd = buffer.indexOf('\n\n'); - if (frameEnd < 0) break; - const frame = buffer.slice(0, frameEnd); - buffer = buffer.slice(frameEnd + 2); - - const eventId = parseSseEventId(frame); - if (eventId && eventId > 0) { - lastEventId = eventId; - } - const payload = parseSsePayload(frame); - if (!payload) continue; - - let event: WorkerOperationEvent | WorkerOperationState; - try { - event = JSON.parse(payload) as WorkerOperationEvent | WorkerOperationState; - } catch { - logWorker('warn', 'sse.wait.json_parse_skipped', { - ...context, - traceId, - opId, - sample: truncateForLog(payload), - }); - continue; - } - - const snapshot: WorkerOperationState = ( - event && typeof event === 'object' && 'snapshot' in event - ? (event as WorkerOperationEvent).snapshot - : (event as WorkerOperationState) - ); - if (!snapshot || typeof snapshot !== 'object') continue; - if (snapshot.opId !== opId) continue; - - eventCount += 1; - latest = snapshot; - context.onSnapshot?.(snapshot); - if (snapshot.status !== lastStatus) { - lastStatus = snapshot.status; - logWorker('info', 'sse.wait.status', { - ...context, - traceId, - opId, - eventCount, - status: snapshot.status, - jobId: snapshot.jobId ?? null, - updatedAt: snapshot.updatedAt ?? null, - }); - } - if (isTerminalStatus(snapshot.status)) { - logWorker('info', 'sse.wait.terminal', { - ...context, - traceId, - opId, - eventCount, - status: snapshot.status, - durationMs: Date.now() - startedAt, - }); - return snapshot; - } - } - } - - if (latest && isTerminalStatus(latest.status)) { - logWorker('info', 'sse.wait.terminal_after_close', { - ...context, - traceId, - opId, - eventCount, - status: latest.status, - durationMs: Date.now() - startedAt, - }); - return latest; - } - - if (controller.signal.aborted) break; - await sleep(250); - } - - if (latest && isTerminalStatus(latest.status)) { - return latest; - } - - logWorker('error', 'sse.wait.ended_without_terminal', { - ...context, - traceId, - opId, - eventCount, - latestStatus: latest?.status ?? null, - durationMs: Date.now() - startedAt, - error: { - name: 'WorkerSseWaitEndedWithoutTerminal', - message: `Operation stream ended before terminal state for op ${opId}`, - }, - }); - throw new Error(`Operation stream ended before terminal state for op ${opId}`); - } catch (error) { - logWorker('error', 'sse.wait.failed', { - ...context, - traceId, - opId, - durationMs: Date.now() - startedAt, - error: errorToLog(error), - }); - throw error; - } finally { - clearTimeout(timeout); - } - } -} diff --git a/src/lib/server/documents/blobstore.ts b/src/lib/server/documents/blobstore.ts index 230bbed..42799da 100644 --- a/src/lib/server/documents/blobstore.ts +++ b/src/lib/server/documents/blobstore.ts @@ -8,7 +8,6 @@ import { PutObjectCommand, } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; -import { PDF_PARSER_VERSION, encodeParserVersion } from '@openreader/compute-core/api-contracts'; import { getS3Client, getS3Config, getS3ProxyClient } from '@/lib/server/storage/s3'; import { serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; @@ -104,24 +103,6 @@ export function documentKey(id: string, namespace: string | null): string { return `${cfg.prefix}/documents_v1/${nsSegment}${id}`; } -export function documentParsedKey(id: string, namespace: string | null): string { - return documentParsedKeyForVersion(id, namespace, PDF_PARSER_VERSION); -} - -export function documentParsedKeyForVersion( - id: string, - namespace: string | null, - parserVersion: string, -): string { - if (!isValidDocumentId(id)) { - throw new Error(`Invalid document id: ${id}`); - } - const cfg = getS3Config(); - const ns = sanitizeNamespace(namespace); - const nsSegment = ns ? `ns/${ns}/` : ''; - return `${cfg.prefix}/documents_v1/parsed_v2/${nsSegment}${id}/${encodeParserVersion(parserVersion)}.json`; -} - export function tempDocumentUploadPrefix(userId: string, namespace: string | null): string { const cfg = getS3Config(); const ns = sanitizeNamespace(namespace); @@ -342,19 +323,6 @@ export async function getTempDocumentFinalizeReceipt( } } -export async function getParsedDocumentBlob(id: string, namespace: string | null): Promise { - const cfg = getS3Config(); - const client = getS3ProxyClient(); - const key = documentParsedKey(id, namespace); - const res = await client.send( - new GetObjectCommand({ - Bucket: cfg.bucket, - Key: key, - }), - ); - return bodyToBuffer(res.Body); -} - export async function getParsedDocumentBlobByKey(key: string): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); @@ -369,31 +337,6 @@ export async function getParsedDocumentBlobByKey(key: string): Promise { return bodyToBuffer(res.Body); } -export async function putParsedDocumentBlob(id: string, body: Buffer, namespace: string | null): Promise { - return putParsedDocumentBlobForVersion(id, body, namespace, PDF_PARSER_VERSION); -} - -export async function putParsedDocumentBlobForVersion( - id: string, - body: Buffer, - namespace: string | null, - parserVersion: string, -): Promise { - const cfg = getS3Config(); - const client = getS3ProxyClient(); - const key = documentParsedKeyForVersion(id, namespace, parserVersion); - await client.send( - new PutObjectCommand({ - Bucket: cfg.bucket, - Key: key, - Body: body, - ContentType: 'application/json', - ServerSideEncryption: 'AES256', - }), - ); - return key; -} - export async function putTempDocumentFinalizeReceipt( token: string, userId: string, @@ -502,7 +445,6 @@ export async function deleteDocumentBlob(id: string, namespace: string | null): const cfg = getS3Config(); const client = getS3ProxyClient(); const key = documentKey(id, namespace); - const parsedKey = documentParsedKey(id, namespace); const legacyParsedKey = legacyDocumentParsedKey(id, namespace); const ns = sanitizeNamespace(namespace); const nsSegment = ns ? `ns/${ns}/` : ''; @@ -510,7 +452,6 @@ export async function deleteDocumentBlob(id: string, namespace: string | null): await deleteDocumentPrefix(parsedPrefix); await deleteDocumentPrefix(`${key}/`); - await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey })); await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey })); // Delete the source after the initial derived-artifact cleanup, then sweep // parsed output once more to catch a worker that finished during deletion. diff --git a/src/lib/server/documents/pdf-preview-pdfjs-runtime.ts b/src/lib/server/documents/pdf-preview-pdfjs-runtime.ts new file mode 100644 index 0000000..4f041c5 --- /dev/null +++ b/src/lib/server/documents/pdf-preview-pdfjs-runtime.ts @@ -0,0 +1,60 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +let cachedStandardFontDataUrl: string | null = null; +const pdfjsPackageName = 'pdfjs-dist'; + +function candidatePackageRoots(): string[] { + const roots = new Set(); + let dir = process.cwd(); + + for (let i = 0; i < 8; i += 1) { + roots.add(path.join(dir, 'node_modules', pdfjsPackageName)); + const next = path.dirname(dir); + if (next === dir) break; + dir = next; + } + + return [...roots]; +} + +function resolvePdfjsPackageFile(relativePath: string): string { + const normalizedRelativePath = relativePath.replace(/^\/+/, ''); + const candidates = candidatePackageRoots().map((root) => path.join(root, normalizedRelativePath)); + const found = candidates.find((candidate) => fs.existsSync(candidate)); + if (found) return found; + + throw new Error( + `pdfjs-dist file not found: ${normalizedRelativePath}; checked ${candidates.join(', ')}`, + ); +} + +export function resolvePdfjsStandardFontDataUrl(): string { + if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl; + + const pdfjsPackageDir = path.dirname(resolvePdfjsPackageFile('package.json')); + const standardFontDir = path.join(pdfjsPackageDir, 'standard_fonts'); + + if (!fs.existsSync(standardFontDir)) { + throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`); + } + + cachedStandardFontDataUrl = `${standardFontDir.replace(/\/?$/, '/')}`; + return cachedStandardFontDataUrl; +} + +export function resolvePdfjsWorkerSrc(): string { + return pathToFileURL(resolvePdfjsPackageFile('legacy/build/pdf.worker.mjs')).href; +} + +export function configurePdfjsNodeRuntime(pdfjs: { + GlobalWorkerOptions?: { + workerSrc?: string; + workerPort?: unknown; + }; +}): void { + if (!pdfjs.GlobalWorkerOptions) return; + pdfjs.GlobalWorkerOptions.workerSrc = resolvePdfjsWorkerSrc(); + pdfjs.GlobalWorkerOptions.workerPort = null; +} diff --git a/src/lib/server/documents/pdf-preview-renderer.ts b/src/lib/server/documents/pdf-preview-renderer.ts new file mode 100644 index 0000000..4ef78ac --- /dev/null +++ b/src/lib/server/documents/pdf-preview-renderer.ts @@ -0,0 +1,157 @@ +import type { Canvas } from '@napi-rs/canvas'; +import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdf-preview-pdfjs-runtime'; + +type CanvasRuntime = { + DOMMatrixCtor: unknown; + Path2DCtor: unknown; + createCanvasFn: (width: number, height: number) => Canvas; +}; + +let canvasRuntimePromise: Promise | null = null; + +async function loadCanvasRuntime(): Promise { + if (!canvasRuntimePromise) { + canvasRuntimePromise = (async () => { + const mod = await import('@napi-rs/canvas'); + const namespace = mod as Record; + const fallback = (namespace.default ?? {}) as Record; + + const createCanvasFn = (namespace.createCanvas ?? fallback.createCanvas) as + | ((width: number, height: number) => Canvas) + | undefined; + const DOMMatrixCtor = namespace.DOMMatrix ?? fallback.DOMMatrix; + const Path2DCtor = namespace.Path2D ?? fallback.Path2D; + + if (typeof createCanvasFn !== 'function') { + throw new Error( + `Canvas runtime missing createCanvas export (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`, + ); + } + if (!DOMMatrixCtor || !Path2DCtor) { + throw new Error( + `Canvas runtime missing DOMMatrix/Path2D exports (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`, + ); + } + + return { + DOMMatrixCtor, + Path2DCtor, + createCanvasFn, + }; + })(); + } + return canvasRuntimePromise; +} + +function ensureNodeCanvasGlobals(runtime: CanvasRuntime): void { + const g = globalThis as Record; + if (typeof g.DOMMatrix === 'undefined') g.DOMMatrix = runtime.DOMMatrixCtor; + if (typeof g.Path2D === 'undefined') g.Path2D = runtime.Path2DCtor; +} + +interface RenderInput { + pdfBytes: ArrayBuffer; + pageNumber: number; + scale?: number; + targetWidth?: number; + format?: 'png' | 'jpeg'; + jpegQuality?: number; +} + +function createPdfjsCanvasFactory(runtime: CanvasRuntime) { + return class OpenReaderCanvasFactory { + create(width: number, height: number) { + const canvas = runtime.createCanvasFn(Math.max(1, Math.floor(width)), Math.max(1, Math.floor(height))); + return { + canvas, + context: canvas.getContext('2d') as unknown as CanvasRenderingContext2D, + }; + } + + reset(target: { canvas: Canvas; context: CanvasRenderingContext2D }, width: number, height: number): void { + target.canvas.width = Math.max(1, Math.floor(width)); + target.canvas.height = Math.max(1, Math.floor(height)); + } + + destroy(target: { canvas: Canvas; context: CanvasRenderingContext2D }): void { + target.canvas.width = 0; + target.canvas.height = 0; + // @ts-expect-error pdf.js expects these nulled on destroy + target.canvas = null; + // @ts-expect-error pdf.js expects these nulled on destroy + target.context = null; + } + }; +} + +export async function renderPage({ + pdfBytes, + pageNumber, + scale = 1.5, + targetWidth, + format = 'png', + jpegQuality = 82, +}: RenderInput): Promise<{ + width: number; + height: number; + image: Buffer; + contentType: 'image/png' | 'image/jpeg'; +}> { + // pdf.js may detach the provided ArrayBuffer. Work with an isolated copy so + // callers can safely reuse their original bytes across pages/calls. + const isolatedBytes = new Uint8Array(pdfBytes).slice(); + + const canvasRuntime = await loadCanvasRuntime(); + ensureNodeCanvasGlobals(canvasRuntime); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs'); + configurePdfjsNodeRuntime(pdfjs); + + const standardFontDataUrl = resolvePdfjsStandardFontDataUrl(); + + const loadingTask = pdfjs.getDocument({ + data: isolatedBytes, + useWorkerFetch: false, + standardFontDataUrl, + isEvalSupported: false, + // Ensure pdf.js transport uses our canvas backend in Node/Next runtime. + CanvasFactory: createPdfjsCanvasFactory(canvasRuntime), + }); + + const pdf = await loadingTask.promise; + try { + const page = await pdf.getPage(pageNumber); + const baseViewport = page.getViewport({ scale: 1.0 }); + const effectiveScale = typeof targetWidth === 'number' && Number.isFinite(targetWidth) && targetWidth > 0 + ? (Math.max(1, Math.round(targetWidth)) / Math.max(1, baseViewport.width)) + : scale; + const viewport = page.getViewport({ scale: effectiveScale }); + const width = Math.max(1, Math.floor(viewport.width)); + const height = Math.max(1, Math.floor(viewport.height)); + const canvas = canvasRuntime.createCanvasFn(width, height); + const ctx = canvas.getContext('2d'); + ctx.fillStyle = '#ffffff'; + ctx.fillRect(0, 0, width, height); + + const renderTask = page.render({ + canvasContext: ctx as unknown as CanvasRenderingContext2D, + viewport, + intent: 'display', + }); + await renderTask.promise; + const contentType = format === 'jpeg' ? 'image/jpeg' : 'image/png'; + const image = format === 'jpeg' + ? canvas.toBuffer('image/jpeg', jpegQuality) + : canvas.toBuffer('image/png'); + return { + width, + height, + image, + contentType, + }; + } finally { + await pdf.destroy().catch(() => undefined); + await loadingTask.destroy().catch(() => undefined); + } +} diff --git a/src/lib/server/documents/previews-render.ts b/src/lib/server/documents/previews-render.ts index c32bbce..033e8fc 100644 --- a/src/lib/server/documents/previews-render.ts +++ b/src/lib/server/documents/previews-render.ts @@ -2,7 +2,7 @@ import path from 'path'; import { createCanvas, loadImage } from '@napi-rs/canvas'; import JSZip from 'jszip'; import { XMLParser } from 'fast-xml-parser'; -import { renderPage } from '@openreader/compute-core'; +import { renderPage } from '@/lib/server/documents/pdf-preview-renderer'; export type RenderedDocumentPreview = { bytes: Buffer; diff --git a/src/lib/server/jobs/user-whisper-align-job.ts b/src/lib/server/jobs/user-whisper-align-job.ts index afe0127..022ee92 100644 --- a/src/lib/server/jobs/user-whisper-align-job.ts +++ b/src/lib/server/jobs/user-whisper-align-job.ts @@ -1,14 +1,13 @@ -import { getCompute } from '@/lib/server/compute'; -import type { WhisperAlignJobRequest } from '@openreader/compute-core/api-contracts'; +import { getComputeWorkerClient } from '@/lib/server/compute-worker/client'; +import type { WhisperAlignRequest } from '@/lib/server/compute-worker/protocol'; import type { TTSSentenceAlignment } from '@/types/tts'; -export type UserWhisperAlignJobRequest = WhisperAlignJobRequest & { +export type UserWhisperAlignJobRequest = WhisperAlignRequest & { sentenceIndex?: number; }; export async function userWhisperAlignJob(input: UserWhisperAlignJobRequest): Promise { - const compute = await getCompute(); - const { alignments } = await compute.alignWords({ + const { alignments } = await getComputeWorkerClient().alignWords({ audioObjectKey: input.audioObjectKey, text: input.text, cacheKey: input.cacheKey, diff --git a/src/lib/server/pdf-parse/artifact.ts b/src/lib/server/pdf-parse/artifact.ts index d195480..7e49b92 100644 --- a/src/lib/server/pdf-parse/artifact.ts +++ b/src/lib/server/pdf-parse/artifact.ts @@ -1,9 +1,8 @@ -import { PDF_PARSER_VERSION } from '@openreader/compute-core/api-contracts'; import { - documentParsedKeyForVersion, getParsedDocumentBlobByKey, isMissingBlobError, } from '@/lib/server/documents/blobstore'; +import { resolveCurrentPdfParse } from '@/lib/server/pdf-parse/operation'; import type { ParsedPdfDocument } from '@/types/parsed-pdf'; export interface ParsedPdfArtifact { @@ -13,7 +12,25 @@ export interface ParsedPdfArtifact { } function parseArtifact(bytes: Buffer): ParsedPdfDocument { - return JSON.parse(bytes.toString('utf8')) as ParsedPdfDocument; + const parsed = JSON.parse(bytes.toString('utf8')) as Partial; + if ( + parsed.schemaVersion !== 1 + || typeof parsed.documentId !== 'string' + || typeof parsed.parserVersion !== 'string' + || typeof parsed.parsedAt !== 'number' + || !Array.isArray(parsed.pages) + || parsed.pages.some((page) => ( + !page + || typeof page !== 'object' + || typeof page.pageNumber !== 'number' + || typeof page.width !== 'number' + || typeof page.height !== 'number' + || !Array.isArray(page.blocks) + )) + ) { + throw new Error('Parsed PDF artifact envelope is invalid'); + } + return parsed as ParsedPdfDocument; } export async function readParsedPdfArtifactByKey(key: string): Promise { @@ -34,6 +51,11 @@ export async function readCurrentParsedPdfArtifact(input: { documentId: string; namespace: string | null; }): Promise { - const key = documentParsedKeyForVersion(input.documentId, input.namespace, PDF_PARSER_VERSION); - return readParsedPdfArtifactByKey(key); + const resolved = await resolveCurrentPdfParse(input); + if (!resolved.artifact) return null; + const artifact = await readParsedPdfArtifactByKey(resolved.artifact.objectKey); + if (artifact && artifact.parsed.documentId !== input.documentId) { + throw new Error('Parsed PDF artifact document identity mismatch'); + } + return artifact; } diff --git a/src/lib/server/pdf-parse/operation.ts b/src/lib/server/pdf-parse/operation.ts index f00aa19..60a924a 100644 --- a/src/lib/server/pdf-parse/operation.ts +++ b/src/lib/server/pdf-parse/operation.ts @@ -1,65 +1,66 @@ -import { buildPdfOpKey } from '@/lib/server/compute/worker'; -import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create'; -import { - fetchWorkerOperationState, - fetchWorkerOperationStateByKey, -} from '@/lib/server/compute/worker-op-state'; +import { getComputeWorkerClient } from '@/lib/server/compute-worker/client'; import { documentKey } from '@/lib/server/documents/blobstore'; -import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; +import type { + PdfLayoutResolution, + PdfLayoutResult, + WorkerOperation, +} from '@/lib/server/compute-worker/protocol'; function currentPdfOperationInput(documentId: string, namespace: string | null, forceToken?: string): { documentId: string; namespace: string | null; documentObjectKey: string; - forceToken?: string; + replaceToken?: string; } { return { documentId, namespace, documentObjectKey: documentKey(documentId, namespace), - ...(forceToken ? { forceToken } : {}), + ...(forceToken ? { replaceToken: forceToken } : {}), }; } -export function buildCurrentPdfParseOpKeyPrefix(input: { +export async function resolveCurrentPdfParse(input: { documentId: string; namespace: string | null; -}): string { - return buildPdfOpKey(currentPdfOperationInput(input.documentId, input.namespace)); +}): Promise { + return getComputeWorkerClient().resolvePdfLayout(currentPdfOperationInput( + input.documentId, + input.namespace, + )); } export async function lookupCurrentPdfParseOperation(input: { documentId: string; namespace: string | null; -}): Promise | null> { - return fetchWorkerOperationStateByKey( - buildCurrentPdfParseOpKeyPrefix(input), - ); +}): Promise | null> { + return (await resolveCurrentPdfParse(input)).operation; } export async function createOrReuseCurrentPdfParseOperation(input: { documentId: string; namespace: string | null; forceToken?: string; -}): Promise> { - return createOrReusePdfWorkerOperation(currentPdfOperationInput( +}): Promise> { + return getComputeWorkerClient().createPdfLayoutOperation(currentPdfOperationInput( input.documentId, input.namespace, input.forceToken, )); } -export async function fetchPdfParseOperation(opId: string): Promise | null> { - return fetchWorkerOperationState(opId); +export async function fetchPdfParseOperation(opId: string): Promise | null> { + return getComputeWorkerClient().getOperation(opId); } export function isPdfParseOperationForDocument( - state: WorkerOperationState, + state: WorkerOperation, input: { documentId: string; namespace: string | null; }, ): boolean { - if (state.kind !== 'pdf_layout') return false; - return state.opKey.startsWith(buildCurrentPdfParseOpKeyPrefix(input)); + return state.subject.kind === 'pdf_layout' + && state.subject.documentId === input.documentId + && state.subject.namespace === input.namespace; } diff --git a/src/lib/server/pdf-parse/snapshot.ts b/src/lib/server/pdf-parse/snapshot.ts index 49c6f61..2fba7f9 100644 --- a/src/lib/server/pdf-parse/snapshot.ts +++ b/src/lib/server/pdf-parse/snapshot.ts @@ -1,8 +1,8 @@ -import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; +import type { PdfLayoutResult, WorkerOperation } from '@/lib/server/compute-worker/protocol'; import type { PdfParseStatus } from '@/types/parsed-pdf'; import type { PdfParseSnapshot } from '@/lib/server/pdf-parse/types'; -function mapWorkerStatusToParseStatus(status: WorkerOperationState['status']): PdfParseStatus { +function mapWorkerStatusToParseStatus(status: WorkerOperation['status']): PdfParseStatus { switch (status) { case 'queued': return 'pending'; @@ -19,7 +19,7 @@ function mapWorkerStatusToParseStatus(status: WorkerOperationState['status']): P } export function parsedObjectKeyFromWorkerState( - state: WorkerOperationState, + state: WorkerOperation, ): string | null { const result = state.result; if (!result || typeof result !== 'object' || !('parsedObjectKey' in result)) return null; @@ -30,7 +30,7 @@ export function parsedObjectKeyFromWorkerState( } export function pdfParseSnapshotFromWorkerState( - state: WorkerOperationState, + state: WorkerOperation, ): PdfParseSnapshot { const parseStatus = mapWorkerStatusToParseStatus(state.status); return { diff --git a/src/lib/server/runtime-config.ts b/src/lib/server/runtime-config.ts index 87889d8..61bcfb6 100644 --- a/src/lib/server/runtime-config.ts +++ b/src/lib/server/runtime-config.ts @@ -6,7 +6,7 @@ import { type RuntimeConfigKey, type RuntimeConfigSource, } from '@/lib/server/admin/settings'; -import { isComputeAvailable } from '@/lib/server/compute'; +import { isComputeWorkerAvailable } from '@/lib/server/compute-worker/client'; export type ResolvedRuntimeConfig = RuntimeConfig & { computeAvailable: boolean; @@ -29,7 +29,7 @@ export async function getResolvedRuntimeConfig(): Promise const values = await getRuntimeConfig(); return { ...values, - computeAvailable: isComputeAvailable(), + computeAvailable: isComputeWorkerAvailable(), }; } diff --git a/src/lib/shared/audio-text.ts b/src/lib/shared/audio-text.ts index 78d8797..74fdbae 100644 --- a/src/lib/shared/audio-text.ts +++ b/src/lib/shared/audio-text.ts @@ -9,9 +9,8 @@ * position-preserving variant that reuses the same patterns/glyph set so * viewer highlight offsets line up with the audio text. * - * The compute/worker package keeps its own mirror in - * `compute/core/src/whisper/alignment-map.ts` (a separate build target that - * cannot import from `@/lib`). Keep that copy in lock-step with these rules. + * Compute alignment receives this already-canonical text over the worker API, + * so its returned character offsets are measured against exactly this value. */ /** Matches a bare or explicit URL token (http(s):// or www.). Capture 1 = domain. */ diff --git a/src/types/parsed-pdf.ts b/src/types/parsed-pdf.ts index 2c7bdf7..7da5cd5 100644 --- a/src/types/parsed-pdf.ts +++ b/src/types/parsed-pdf.ts @@ -1,12 +1,12 @@ -import type { ParsedPdfBlockKind } from '@openreader/compute-core/types'; - export { type ParsedPdfBlock, type ParsedPdfBlockFragment, type ParsedPdfBlockKind, type ParsedPdfDocument, type ParsedPdfPage, -} from '@openreader/compute-core/types'; +} from '@/lib/server/compute-worker/protocol'; + +import type { ParsedPdfBlockKind } from '@/lib/server/compute-worker/protocol'; export const PARSED_PDF_BLOCK_KINDS: ParsedPdfBlockKind[] = [ 'abstract', @@ -32,8 +32,11 @@ export const PARSED_PDF_BLOCK_KINDS: ParsedPdfBlockKind[] = [ 'vision_footnote', ]; -export type { - PdfParsePhase, - PdfParseProgress, - PdfParseStatus, -} from '@openreader/compute-core/types'; +export type PdfParseStatus = 'pending' | 'running' | 'ready' | 'failed'; +export type PdfParsePhase = 'infer' | 'merge'; +export interface PdfParseProgress { + totalPages: number; + pagesParsed: number; + currentPage?: number; + phase: PdfParsePhase; +} diff --git a/src/types/tts.ts b/src/types/tts.ts index c5f42d5..cc3073c 100644 --- a/src/types/tts.ts +++ b/src/types/tts.ts @@ -1,10 +1,11 @@ import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; export type { - TTSAudioBuffer, - TTSAudioBytes, TTSSentenceAlignment, TTSSentenceWord, -} from '@openreader/compute-core/types'; +} from '@/lib/server/compute-worker/protocol'; + +export type TTSAudioBuffer = ArrayBuffer; +export type TTSAudioBytes = number[]; export type TTSLocation = string | number; diff --git a/tests/unit/compute-worker-client-contract.vitest.spec.ts b/tests/unit/compute-worker-client-contract.vitest.spec.ts new file mode 100644 index 0000000..e37f234 --- /dev/null +++ b/tests/unit/compute-worker-client-contract.vitest.spec.ts @@ -0,0 +1,58 @@ +import { afterEach, describe, expect, test } from 'vitest'; +import { createComputeWorkerApp, type ComputeWorkerApp } from '../../compute-worker/src/runtime'; +import { FakeControlPlane } from '../../compute-worker/tests/fixtures/fake-control-plane'; +import { ComputeWorkerClient } from '../../src/lib/server/compute-worker/client'; + +describe('ComputeWorkerClient contract', () => { + let runtime: ComputeWorkerApp | null = null; + + afterEach(async () => { + await runtime?.close(); + runtime = null; + }); + + test('exercises the real Fastify API through the app-owned HTTP client', async () => { + process.env.NATS_URL = 'nats://127.0.0.1:4222'; + const fake = new FakeControlPlane(); + runtime = await createComputeWorkerApp({ + host: '127.0.0.1', + port: 0, + workerToken: 'contract-token', + disableWorkers: true, + routeDeps: fake.deps, + }); + await runtime.start(); + + const address = runtime.app.server.address(); + if (!address || typeof address === 'string') { + throw new Error('Compute worker did not bind a TCP port'); + } + const client = new ComputeWorkerClient({ + baseUrl: `http://127.0.0.1:${address.port}`, + token: 'contract-token', + }); + const documentId = 'b'.repeat(64); + const request = { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }; + + const created = await client.createPdfLayoutOperation(request); + expect(created).toMatchObject({ + subject: { kind: 'pdf_layout', documentId, namespace: null }, + status: 'queued', + }); + expect(created).not.toHaveProperty('opKey'); + expect(created).not.toHaveProperty('jobId'); + + await expect(client.getOperation(created.opId)).resolves.toMatchObject({ + opId: created.opId, + subject: { kind: 'pdf_layout', documentId, namespace: null }, + }); + await expect(client.resolvePdfLayout(request)).resolves.toMatchObject({ + artifact: null, + operation: { opId: created.opId }, + }); + }); +}); diff --git a/tests/unit/pdf-op-key.vitest.spec.ts b/tests/unit/pdf-op-key.vitest.spec.ts deleted file mode 100644 index fbe746e..0000000 --- a/tests/unit/pdf-op-key.vitest.spec.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, test } from 'vitest'; -import { PDF_PARSER_VERSION } from '@openreader/compute-core'; -import { buildPdfOpKey } from '../../src/lib/server/compute/worker'; - -describe('pdf worker op key', () => { - test('includes parser version in the reuse boundary', () => { - expect(buildPdfOpKey({ - documentId: 'doc-1', - namespace: null, - documentObjectKey: 'documents/doc-1.pdf', - })).toBe(`pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1||documents/doc-1.pdf|`); - }); -}); diff --git a/tests/unit/pdf-parse-artifact-envelope.vitest.spec.ts b/tests/unit/pdf-parse-artifact-envelope.vitest.spec.ts new file mode 100644 index 0000000..c9276ea --- /dev/null +++ b/tests/unit/pdf-parse-artifact-envelope.vitest.spec.ts @@ -0,0 +1,72 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const hoisted = vi.hoisted(() => ({ + getParsedDocumentBlobByKey: vi.fn(), + resolveCurrentPdfParse: vi.fn(), +})); + +vi.mock('@/lib/server/documents/blobstore', () => ({ + getParsedDocumentBlobByKey: hoisted.getParsedDocumentBlobByKey, + isMissingBlobError: vi.fn(() => false), +})); + +vi.mock('@/lib/server/pdf-parse/operation', () => ({ + resolveCurrentPdfParse: hoisted.resolveCurrentPdfParse, +})); + +describe('parsed PDF artifact envelope validation', () => { + beforeEach(() => { + hoisted.resolveCurrentPdfParse.mockReset(); + hoisted.resolveCurrentPdfParse.mockResolvedValue({ + artifact: { objectKey: 'openreader/parsed.json' }, + operation: null, + }); + hoisted.getParsedDocumentBlobByKey.mockReset(); + }); + + test('accepts a valid current artifact envelope', async () => { + hoisted.getParsedDocumentBlobByKey.mockResolvedValue(Buffer.from(JSON.stringify({ + schemaVersion: 1, + documentId: 'doc-1', + parserVersion: 'parser-v1', + parsedAt: 1, + pages: [{ pageNumber: 1, width: 100, height: 200, blocks: [] }], + }))); + + const { readCurrentParsedPdfArtifact } = await import('../../src/lib/server/pdf-parse/artifact'); + await expect(readCurrentParsedPdfArtifact({ documentId: 'doc-1', namespace: null })) + .resolves.toMatchObject({ key: 'openreader/parsed.json' }); + }); + + test('rejects malformed envelopes and document identity mismatches', async () => { + const { readCurrentParsedPdfArtifact } = await import('../../src/lib/server/pdf-parse/artifact'); + + hoisted.getParsedDocumentBlobByKey.mockResolvedValueOnce(Buffer.from(JSON.stringify({ + schemaVersion: 2, + documentId: 'doc-1', + pages: [], + }))); + await expect(readCurrentParsedPdfArtifact({ documentId: 'doc-1', namespace: null })) + .rejects.toThrow('envelope is invalid'); + + hoisted.getParsedDocumentBlobByKey.mockResolvedValueOnce(Buffer.from(JSON.stringify({ + schemaVersion: 1, + documentId: 'other-doc', + parserVersion: 'parser-v1', + parsedAt: 1, + pages: [], + }))); + await expect(readCurrentParsedPdfArtifact({ documentId: 'doc-1', namespace: null })) + .rejects.toThrow('document identity mismatch'); + + hoisted.getParsedDocumentBlobByKey.mockResolvedValueOnce(Buffer.from(JSON.stringify({ + schemaVersion: 1, + documentId: 'doc-1', + parserVersion: 'parser-v1', + parsedAt: 1, + pages: [{ pageNumber: 1, width: 100, height: 200 }], + }))); + await expect(readCurrentParsedPdfArtifact({ documentId: 'doc-1', namespace: null })) + .rejects.toThrow('envelope is invalid'); + }); +}); diff --git a/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts b/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts index 98ceb4d..85b49cb 100644 --- a/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts +++ b/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts @@ -12,7 +12,7 @@ const hoisted = vi.hoisted(() => ({ requireAuthContext: vi.fn(), fetchPdfParseOperation: vi.fn(), isPdfParseOperationForDocument: vi.fn(), - getWorkerClientConfigFromEnv: vi.fn(), + openOperationEvents: vi.fn(), })); vi.mock('@/db', () => ({ @@ -30,8 +30,10 @@ vi.mock('@/lib/server/pdf-parse/operation', () => ({ isPdfParseOperationForDocument: hoisted.isPdfParseOperationForDocument, })); -vi.mock('@/lib/server/compute/worker', () => ({ - getWorkerClientConfigFromEnv: hoisted.getWorkerClientConfigFromEnv, +vi.mock('@/lib/server/compute-worker/client', () => ({ + getComputeWorkerClient: () => ({ + openOperationEvents: hoisted.openOperationEvents, + }), })); vi.mock('@/lib/server/documents/blobstore', () => ({ @@ -84,12 +86,8 @@ describe('GET /api/documents/[id]/parsed/events worker event proxy', () => { }); hoisted.isPdfParseOperationForDocument.mockReset(); hoisted.isPdfParseOperationForDocument.mockReturnValue(true); - hoisted.getWorkerClientConfigFromEnv.mockReset(); - hoisted.getWorkerClientConfigFromEnv.mockReturnValue({ - baseUrl: 'http://worker.local', - token: 'worker-token', - }); - global.fetch = vi.fn(async () => new Response( + hoisted.openOperationEvents.mockReset(); + hoisted.openOperationEvents.mockResolvedValue(new Response( 'event: snapshot\ndata: {"eventId":1,"snapshot":{"opId":"op-1","status":"running"}}\n\n', { status: 200, @@ -97,7 +95,7 @@ describe('GET /api/documents/[id]/parsed/events worker event proxy', () => { 'Content-Type': 'text/event-stream; charset=utf-8', }, }, - )) as typeof fetch; + )); }); test('requires an opId', async () => { @@ -123,10 +121,9 @@ describe('GET /api/documents/[id]/parsed/events worker event proxy', () => { expect(text).toContain('"opId":"op-1"'); expect(hoisted.fetchPdfParseOperation).toHaveBeenCalledWith('op-1'); expect(hoisted.isPdfParseOperationForDocument).toHaveBeenCalled(); - expect(global.fetch).toHaveBeenCalledWith( - 'http://worker.local/ops/op-1/events', - expect.anything(), - ); + expect(hoisted.openOperationEvents).toHaveBeenCalledWith('op-1', expect.objectContaining({ + signal: expect.any(AbortSignal), + })); }); test('denies proxying when the op does not belong to the document', async () => { @@ -140,7 +137,7 @@ describe('GET /api/documents/[id]/parsed/events worker event proxy', () => { expect(response.status).toBe(404); await expect(response.json()).resolves.toMatchObject({ error: 'Operation not found' }); expect(hoisted.fetchPdfParseOperation).toHaveBeenCalledWith('op-1'); - expect(global.fetch).not.toHaveBeenCalled(); + expect(hoisted.openOperationEvents).not.toHaveBeenCalled(); }); afterEach(() => { diff --git a/tsconfig.json b/tsconfig.json index 2c7a43b..9cd3715 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -20,11 +20,6 @@ ], "paths": { "@/*": ["./src/*"], - "@openreader/compute-core": ["./compute/core/src/index.ts"], - "@openreader/compute-core/api-contracts": ["./compute/core/src/api-contracts/index.ts"], - "@openreader/compute-core/control-plane": ["./compute/core/src/control-plane/index.ts"], - "@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"], - "@openreader/compute-core/types": ["./compute/core/src/types/index.ts"] } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], @@ -32,7 +27,7 @@ "node_modules", "docs-site", "docs-site/**", - "compute/worker", - "compute/worker/**" + "compute-worker", + "compute-worker/**" ] } diff --git a/vitest.config.ts b/vitest.config.ts index 3489ba3..99417d3 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -10,17 +10,7 @@ if (!/^https?:\/\//.test(process.env.BASE_URL ?? '')) { } const srcDir = fileURLToPath(new URL('./src/', import.meta.url)); -const computeCoreIndex = fileURLToPath(new URL('./compute/core/src/index.ts', import.meta.url)); -const computeCoreApiContracts = fileURLToPath(new URL('./compute/core/src/api-contracts/index.ts', import.meta.url)); -const computeCoreControlPlane = fileURLToPath(new URL('./compute/core/src/control-plane/index.ts', import.meta.url)); -const computeCoreLocalRuntime = fileURLToPath(new URL('./compute/core/src/local-runtime.ts', import.meta.url)); -const computeCoreTypes = fileURLToPath(new URL('./compute/core/src/types/index.ts', import.meta.url)); const alias = [ - { find: /^@openreader\/compute-core$/, replacement: computeCoreIndex }, - { find: /^@openreader\/compute-core\/api-contracts$/, replacement: computeCoreApiContracts }, - { find: /^@openreader\/compute-core\/control-plane$/, replacement: computeCoreControlPlane }, - { find: /^@openreader\/compute-core\/local-runtime$/, replacement: computeCoreLocalRuntime }, - { find: /^@openreader\/compute-core\/types$/, replacement: computeCoreTypes }, { find: /^@\//, replacement: `${srcDir}` }, { find: '@', replacement: srcDir }, ]; @@ -44,19 +34,12 @@ export default defineConfig({ setupFiles: ['tests/unit/setup-env.ts'], }, }, - { - test: { - name: 'compute-core', - environment: 'node', - include: ['compute/core/tests/control-plane/**/*.test.ts'], - }, - }, { test: { name: 'compute-worker', environment: 'node', - include: ['compute/worker/tests/{unit,api}/**/*.test.ts'], - setupFiles: ['compute/worker/tests/setup-env.ts'], + include: ['compute-worker/tests/{unit,api,compute}/**/*.test.ts'], + setupFiles: ['compute-worker/tests/setup-env.ts'], }, }, ], From 3d5dfd2a88a76aaa96fe3b34e1dad80d249b1aa0 Mon Sep 17 00:00:00 2001 From: Richard R Date: Fri, 12 Jun 2026 14:07:15 -0600 Subject: [PATCH 02/11] refactor(compute-worker): migrate compute logic to modular architecture - Remove legacy compute-worker monolith under `src/compute/`, including PDF and Whisper inference, control-plane, and runtime orchestration - Move PDF and Whisper inference logic to new `src/inference/` module - Move control-plane, orchestrator, and state machine logic to `src/operations/` - Move NATS/JetStream adapters to `src/infrastructure/` - Move job orchestration, progress, and artifact persistence to `src/jobs/` - Update imports throughout tests and main app to reference new module structure - Update Dockerfile and scripts to use new asset paths - Add new entrypoints: `src/api/app.ts` and `src/api/contracts.ts` - Remove obsolete files and update `.gitignore` for new dev artifacts This refactor modularizes the compute-worker, improving maintainability and separation of concerns. No functional changes to inference or orchestration logic. BREAKING CHANGE: compute-worker internal APIs, directory structure, and imports have changed; downstream code must update imports and integration points. --- .gitignore | 4 +- Dockerfile | 4 +- compute-worker/scripts/generate-openapi.ts | 2 +- compute-worker/src/{runtime.ts => api/app.ts} | 881 ++---------------- .../index.ts => api/contracts.ts} | 12 +- compute-worker/src/api/operation-keys.ts | 2 +- compute-worker/src/api/public-operation.ts | 2 +- .../config/cpu-budget.ts | 0 .../{compute => inference}/config/timeout.ts | 0 .../src/{compute => inference}/index.ts | 4 +- .../{compute => inference}/local-runtime.ts | 0 .../pdf/assets/LICENSE.txt | 0 .../pdf/assets/manifest.json | 0 .../src/{compute => inference}/pdf/merge.ts | 0 .../src/{compute => inference}/pdf/model.ts | 0 .../pdf/normalize-text.ts | 0 .../src/{compute => inference}/pdf/parse.ts | 0 .../pdf/parser-version-key.ts | 0 .../pdf/parser-version.ts | 0 .../pdf/pdfjs-runtime.ts | 0 .../src/{compute => inference}/pdf/render.ts | 0 .../pdf/runLayoutModel.ts | 0 .../src/{compute => inference}/pdf/stitch.ts | 0 .../src/{compute => inference}/pdf/types.ts | 0 .../platform/docstore.ts | 0 .../{compute => inference}/platform/ffmpeg.ts | 0 .../src/{compute => inference}/types/index.ts | 0 .../types/parsed-pdf.ts | 0 .../src/{compute => inference}/types/tts.ts | 0 .../{compute => inference}/whisper/align.ts | 0 .../whisper/alignment-map.ts | 0 .../whisper/assets/LICENSE.txt | 0 .../whisper/assets/manifest.json | 0 .../whisper/assets/mel_filters.npz | Bin .../{compute => inference}/whisper/model.ts | 0 .../whisper/spectral.ts | 0 .../whisper/token-timestamps.ts | 0 .../json-codec.ts | 0 .../nats-adapters.ts} | 4 +- compute-worker/src/infrastructure/nats.ts | 104 +++ compute-worker/src/infrastructure/storage.ts | 127 +++ compute-worker/src/jobs/handlers.ts | 126 +++ .../{ => jobs}/pdf-artifact-persistence.ts | 0 compute-worker/src/{ => jobs}/pdf-progress.ts | 2 +- .../src/{ => jobs}/worker-loop-policy.ts | 2 +- compute-worker/src/jobs/worker-loop.ts | 312 +++++++ .../control-plane => operations}/index.ts | 2 +- .../recovery.ts} | 2 +- .../orchestrator.ts => operations/service.ts} | 2 +- .../control-plane => operations}/sse.ts | 0 .../state-machine.ts | 2 +- .../control-plane => operations}/types.ts | 2 +- compute-worker/src/server.ts | 2 +- .../src/storage/artifact-addressing.ts | 4 +- compute-worker/tests/api/routes.test.ts | 2 +- .../pdf-merge-text-with-regions.test.ts | 2 +- .../pdf-parse-normalize-text-items.test.ts | 2 +- .../pdf-stitch-cross-page-blocks.test.ts | 4 +- .../algorithms/support/document-fixtures.ts | 4 +- .../whisper-alignment-mapping.test.ts | 2 +- .../algorithms/whisper-spectral.test.ts | 2 +- .../whisper-token-timestamps.test.ts | 2 +- .../control-plane/orchestrator.test.ts | 4 +- .../control-plane/run-layout-model.test.ts | 8 +- .../tests/compute/control-plane/sse.test.ts | 2 +- .../control-plane/state-machine.test.ts | 4 +- .../helpers/in-memory-control-plane.ts | 4 +- .../tests/fixtures/fake-control-plane.ts | 4 +- .../tests/unit/jetstream-adapters.test.ts | 6 +- .../tests/unit/key-and-progress.test.ts | 4 +- .../tests/unit/orphan-recovery.test.ts | 2 +- .../unit/pdf-artifact-persistence.test.ts | 2 +- .../tests/unit/worker-loop-policy.test.ts | 2 +- scripts/check-next-server-bundle.mjs | 2 +- ...pute-worker-client-contract.vitest.spec.ts | 2 +- 75 files changed, 810 insertions(+), 860 deletions(-) rename compute-worker/src/{runtime.ts => api/app.ts} (56%) rename compute-worker/src/{compute/api-contracts/index.ts => api/contracts.ts} (87%) rename compute-worker/src/{compute => inference}/config/cpu-budget.ts (100%) rename compute-worker/src/{compute => inference}/config/timeout.ts (100%) rename compute-worker/src/{compute => inference}/index.ts (93%) rename compute-worker/src/{compute => inference}/local-runtime.ts (100%) rename compute-worker/src/{compute => inference}/pdf/assets/LICENSE.txt (100%) rename compute-worker/src/{compute => inference}/pdf/assets/manifest.json (100%) rename compute-worker/src/{compute => inference}/pdf/merge.ts (100%) rename compute-worker/src/{compute => inference}/pdf/model.ts (100%) rename compute-worker/src/{compute => inference}/pdf/normalize-text.ts (100%) rename compute-worker/src/{compute => inference}/pdf/parse.ts (100%) rename compute-worker/src/{compute => inference}/pdf/parser-version-key.ts (100%) rename compute-worker/src/{compute => inference}/pdf/parser-version.ts (100%) rename compute-worker/src/{compute => inference}/pdf/pdfjs-runtime.ts (100%) rename compute-worker/src/{compute => inference}/pdf/render.ts (100%) rename compute-worker/src/{compute => inference}/pdf/runLayoutModel.ts (100%) rename compute-worker/src/{compute => inference}/pdf/stitch.ts (100%) rename compute-worker/src/{compute => inference}/pdf/types.ts (100%) rename compute-worker/src/{compute => inference}/platform/docstore.ts (100%) rename compute-worker/src/{compute => inference}/platform/ffmpeg.ts (100%) rename compute-worker/src/{compute => inference}/types/index.ts (100%) rename compute-worker/src/{compute => inference}/types/parsed-pdf.ts (100%) rename compute-worker/src/{compute => inference}/types/tts.ts (100%) rename compute-worker/src/{compute => inference}/whisper/align.ts (100%) rename compute-worker/src/{compute => inference}/whisper/alignment-map.ts (100%) rename compute-worker/src/{compute => inference}/whisper/assets/LICENSE.txt (100%) rename compute-worker/src/{compute => inference}/whisper/assets/manifest.json (100%) rename compute-worker/src/{compute => inference}/whisper/assets/mel_filters.npz (100%) rename compute-worker/src/{compute => inference}/whisper/model.ts (100%) rename compute-worker/src/{compute => inference}/whisper/spectral.ts (100%) rename compute-worker/src/{compute => inference}/whisper/token-timestamps.ts (100%) rename compute-worker/src/{control-plane => infrastructure}/json-codec.ts (100%) rename compute-worker/src/{control-plane/jetstream.ts => infrastructure/nats-adapters.ts} (99%) create mode 100644 compute-worker/src/infrastructure/nats.ts create mode 100644 compute-worker/src/infrastructure/storage.ts create mode 100644 compute-worker/src/jobs/handlers.ts rename compute-worker/src/{ => jobs}/pdf-artifact-persistence.ts (100%) rename compute-worker/src/{ => jobs}/pdf-progress.ts (89%) rename compute-worker/src/{ => jobs}/worker-loop-policy.ts (91%) create mode 100644 compute-worker/src/jobs/worker-loop.ts rename compute-worker/src/{compute/control-plane => operations}/index.ts (71%) rename compute-worker/src/{orphan-recovery.ts => operations/recovery.ts} (98%) rename compute-worker/src/{compute/control-plane/orchestrator.ts => operations/service.ts} (99%) rename compute-worker/src/{compute/control-plane => operations}/sse.ts (100%) rename compute-worker/src/{compute/control-plane => operations}/state-machine.ts (98%) rename compute-worker/src/{compute/control-plane => operations}/types.ts (98%) diff --git a/.gitignore b/.gitignore index 55b2755..466d577 100644 --- a/.gitignore +++ b/.gitignore @@ -56,5 +56,7 @@ node_modules/ # vscode .vscode -# .agents +# Agents .agents +.codex +.claude diff --git a/Dockerfile b/Dockerfile index 93e32ab..f4ebc3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -87,7 +87,7 @@ COPY --from=app-builder /app/THIRD_PARTY_LICENSES /licenses # Include SeaweedFS license text for the copied weed binary. COPY --from=seaweedfs-builder /tmp/SeaweedFS-LICENSE.txt /licenses/SeaweedFS-LICENSE.txt # Include static model notices for runtime-downloaded assets. -COPY --from=app-builder /app/compute-worker/src/compute/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt +COPY --from=app-builder /app/compute-worker/src/inference/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt # Copy seaweedfs weed binary for optional embedded local S3. COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed @@ -97,7 +97,7 @@ COPY --from=nats-builder /tmp/nats-server /usr/local/bin/nats-server RUN chmod +x /usr/local/bin/nats-server # Include OpenAI Whisper license text for runtime-downloaded ONNX artifacts. -COPY --from=app-builder /app/compute-worker/src/compute/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt +COPY --from=app-builder /app/compute-worker/src/inference/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt # Match the app's historical container port now that standalone server.js # is started directly instead of `next start -p 3003`. diff --git a/compute-worker/scripts/generate-openapi.ts b/compute-worker/scripts/generate-openapi.ts index 7af98d5..0cfb141 100644 --- a/compute-worker/scripts/generate-openapi.ts +++ b/compute-worker/scripts/generate-openapi.ts @@ -1,6 +1,6 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { dirname, resolve } from 'node:path'; -import { createComputeWorkerApp } from '../src/runtime'; +import { createComputeWorkerApp } from '../src/api/app'; process.env.COMPUTE_WORKER_TOKEN ||= 'openapi-generation-token'; process.env.NATS_URL ||= 'nats://127.0.0.1:4222'; diff --git a/compute-worker/src/runtime.ts b/compute-worker/src/api/app.ts similarity index 56% rename from compute-worker/src/runtime.ts rename to compute-worker/src/api/app.ts index 5cafa7c..1775f4e 100644 --- a/compute-worker/src/runtime.ts +++ b/compute-worker/src/api/app.ts @@ -1,78 +1,72 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify'; import swagger from '@fastify/swagger'; -import { z } from 'zod'; import { connect, - nanos, credsAuthenticator, type NatsConnection, } from '@nats-io/transport-node'; import { - AckPolicy, - DeliverPolicy, - ReplayPolicy, - RetentionPolicy, - StorageType, jetstream, jetstreamManager, type Consumer, type JetStreamClient, type JetStreamManager, - type JsMsg, } from '@nats-io/jetstream'; import { Kvm } from '@nats-io/kv'; import { ensureComputeModels, - runPdfLayoutFromPdfBuffer, - runWhisperAlignmentFromAudioBuffer, -} from './compute/local-runtime'; +} from '../inference/local-runtime'; import { getComputeTimeoutConfig, getComputeOpStaleMs, getAvailableCpuCores, getOnnxThreadsPerJob, - withIdleTimeoutAndHardCap, - withTimeout, -} from './compute'; -import { encodeSseFrame, OperationOrchestrator } from './compute/control-plane'; +} from '../inference'; +import { encodeSseFrame, OperationOrchestrator } from '../operations'; import type { PdfLayoutJobRequest, PdfLayoutJobResult, WorkerOperationEvent, WhisperAlignJobRequest, WhisperAlignJobResult, - WorkerJobState, WorkerJobTiming, - WorkerOperationKind, WorkerOperationRequest, WorkerOperationState, - PdfLayoutProgress, -} from './compute/api-contracts'; -import { - DeleteObjectCommand, - GetObjectCommand, - HeadObjectCommand, - PutObjectCommand, - S3Client, -} from '@aws-sdk/client-s3'; +} from '../api/contracts'; import { JetStreamOperationEventStream, - OP_EVENTS_SUBJECT_WILDCARD, JetStreamOperationQueue, JetStreamOperationStateStore, hashOpKey, -} from './control-plane/jetstream'; -import { type JsonCodec, createJsonCodec } from './control-plane/json-codec'; +} from '../infrastructure/nats-adapters'; +import { createJsonCodec } from '../infrastructure/json-codec'; import { recoverOrphanedOperations, type StreamedOperationState, -} from './orphan-recovery'; -import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; -import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy'; -import { persistParsedPdfWhileSourceExists } from './pdf-artifact-persistence'; -import { parsedPdfArtifactKey } from './storage/artifact-addressing'; -import { buildPdfOperationKey, buildWhisperOperationKey } from './api/operation-keys'; -import { toPublicOperation, type PublicOperationEvent } from './api/public-operation'; +} from '../operations/recovery'; +import { parsedPdfArtifactKey } from '../storage/artifact-addressing'; +import { + createArtifactStorage, + createS3ClientFromEnv, + normalizeS3Prefix, + type ArtifactStorage, +} from '../infrastructure/storage'; +import { createJobHandlers } from '../jobs/handlers'; +import { createWorkerLoopController, type QueuedJob } from '../jobs/worker-loop'; +import { + COMPUTE_STATE_BUCKET, + COMPUTE_STATE_TTL_MS, + EVENTS_STREAM_NAME, + JOBS_STREAM_NAME, + LAYOUT_CONSUMER_NAME, + LAYOUT_JOBS_SUBJECT, + NATS_API_TIMEOUT_MS, + WHISPER_CONSUMER_NAME, + WHISPER_JOBS_SUBJECT, + ensureJetStreamResources, +} from '../infrastructure/nats'; +import { buildPdfOperationKey, buildWhisperOperationKey } from './operation-keys'; +import { toPublicOperation, type PublicOperationEvent } from './public-operation'; import { apiErrorResponseSchema, artifactReferenceSchema, @@ -89,26 +83,14 @@ import { publicOperationSchema, ttsSentenceAlignmentSchema, whisperOperationCreateSchema, -} from './api/schemas'; +} from './schemas'; -const JOBS_STREAM_NAME = 'compute_jobs'; -const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; -const LAYOUT_JOBS_SUBJECT = 'jobs.layout'; -const WHISPER_CONSUMER_NAME = 'compute_whisper'; -const LAYOUT_CONSUMER_NAME = 'compute_layout'; -const EVENTS_STREAM_NAME = 'compute_events'; -const COMPUTE_STATE_BUCKET = 'compute_state'; -const COMPUTE_STATE_TTL_MS = 24 * 60 * 60 * 1000; -const LOOP_ERROR_BACKOFF_MS = 500; -const RUNNING_HEARTBEAT_MS = 5000; const OP_EVENTS_KEEPALIVE_MS = 15_000; // Reconnection delay handed to the browser EventSource via the SSE `retry:` // directive. When a silent stream is torn down for idle sleep, this keeps the // client from immediately reconnecting and re-waking the worker; instead it // reconnects on a slow cadence so the container stays asleep most of the time. const OP_EVENTS_RECONNECT_HINT_MS = 120_000; -const WHISPER_MAX_DELIVER = 1; -const NATS_API_TIMEOUT_MS = 60_000; // Disconnect from NATS after this much continuous idle so the worker stops // generating outbound traffic (pull polling + keepalive PINGs) and Railway can // put it to sleep. Reconnect happens lazily on the next inbound request. @@ -118,22 +100,9 @@ const IDLE_STATUS_LOG_INTERVAL_MS = 60_000; const ORPHAN_SWEEP_INTERVAL_MS = 15_000; // Bounded pull window so consumer loops yield periodically and can be stopped // cleanly when going idle, instead of blocking on a long-lived pull. -const PULL_EXPIRES_MS = 5_000; const REQUEST_STARTED_AT_MS_KEY = Symbol('request-started-at-ms'); const REQUEST_COUNTED_KEY = Symbol('request-activity-counted'); -const SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND: Record = { - whisper_align: 15_000, - pdf_layout: 120_000, -}; - -interface QueuedJob { - jobId: string; - opId: string; - opKey: string; - kind: WorkerOperationKind; - queuedAt: number; - payload: TPayload; -} +const WHISPER_MAX_DELIVER = 1; interface NatsSession { nc: NatsConnection; @@ -267,64 +236,6 @@ function buildLoggerConfig(): boolean | Record { }; } -function normalizeS3Prefix(prefix: string | undefined): string { - const value = (prefix || 'openreader').trim(); - return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; -} - -function buildS3Client(): S3Client { - const bucket = requireEnv('S3_BUCKET'); - const region = requireEnv('S3_REGION'); - const accessKeyId = requireEnv('S3_ACCESS_KEY_ID'); - const secretAccessKey = requireEnv('S3_SECRET_ACCESS_KEY'); - const endpoint = process.env.S3_ENDPOINT?.trim() || undefined; - const forcePathStyle = parseBoolEnv('S3_FORCE_PATH_STYLE', false); - - void bucket; - - return new S3Client({ - region, - endpoint, - forcePathStyle, - requestChecksumCalculation: 'WHEN_REQUIRED', - responseChecksumValidation: 'WHEN_REQUIRED', - credentials: { - accessKeyId, - secretAccessKey, - }, - }); -} - -async function bodyToBuffer(body: unknown): Promise { - if (!body) return Buffer.alloc(0); - if (body instanceof Uint8Array) return Buffer.from(body); - if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); - if (body instanceof ArrayBuffer) return Buffer.from(body); - if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { - const maybe = body as { transformToByteArray?: () => Promise }; - if (typeof maybe.transformToByteArray === 'function') { - return Buffer.from(await maybe.transformToByteArray()); - } - } - if (typeof body === 'object' && body !== null && 'on' in body) { - const stream = body as NodeJS.ReadableStream; - const chunks: Buffer[] = []; - for await (const chunk of stream) { - if (Buffer.isBuffer(chunk)) chunks.push(chunk); - else if (typeof chunk === 'string') chunks.push(Buffer.from(chunk)); - else chunks.push(Buffer.from(chunk as Uint8Array)); - } - return Buffer.concat(chunks); - } - throw new Error('Unsupported S3 response body type'); -} - -function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { - const copy = new Uint8Array(bytes.byteLength); - copy.set(bytes); - return copy.buffer; -} - function isAuthed(request: FastifyRequest, expectedToken: string): boolean { const auth = request.headers.authorization; if (!auth?.startsWith('Bearer ')) return false; @@ -332,11 +243,6 @@ function isAuthed(request: FastifyRequest, expectedToken: string): boolean { return token === expectedToken; } -function safeDurationMs(start: number | undefined, end: number | undefined): number | undefined { - if (!Number.isFinite(start) || !Number.isFinite(end)) return undefined; - return Math.max(0, Math.floor((end as number) - (start as number))); -} - function toErrorMessage(error: unknown): string { if (error instanceof Error && error.message) return error.message; return String(error); @@ -370,155 +276,12 @@ function extractOpId(request: FastifyRequest, path: string): string | null { } } -function isAlreadyExistsError(error: unknown): boolean { - const message = toErrorMessage(error).toLowerCase(); - return message.includes('already in use') || message.includes('already exists'); -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -class ConcurrencyGate { - private readonly maxInFlight: number; - private inFlight = 0; - private readonly queue: Array<() => void> = []; - - constructor(limit: number) { - this.maxInFlight = Number.isFinite(limit) && limit >= 1 ? Math.floor(limit) : 1; - } - - async acquire(): Promise { - if (this.inFlight < this.maxInFlight) { - this.inFlight += 1; - return; - } - - await new Promise((resolve) => { - this.queue.push(() => { - this.inFlight += 1; - resolve(); - }); - }); - } - - release(): void { - this.inFlight = Math.max(0, this.inFlight - 1); - const next = this.queue.shift(); - if (next) next(); - } -} - -function isTerminalStatus(status: WorkerJobState): boolean { +function isTerminalStatus(status: import('../api/contracts').WorkerJobState): boolean { return status === 'succeeded' || status === 'failed'; } -function extractResultRef(kind: WorkerOperationKind, result: unknown): string | undefined { - if (kind !== 'pdf_layout' || !result || typeof result !== 'object') return undefined; - const maybe = result as { parsedObjectKey?: unknown }; - return typeof maybe.parsedObjectKey === 'string' ? maybe.parsedObjectKey : undefined; -} - -const alignSchema = z.object({ - text: z.string().trim().min(1), - lang: z.string().trim().min(1).max(16).optional(), - cacheKey: z.string().trim().min(1).max(256).optional(), - audioObjectKey: z.string().trim().min(1).max(2048), -}); - -const layoutSchema = z.object({ - documentId: z.string().trim().min(1), - namespace: z.string().trim().min(1).max(128).nullable(), - documentObjectKey: z.string().trim().min(1).max(2048), -}); - const errorResponseSchema = jsonSchema(apiErrorResponseSchema); -async function ensureJetStreamResources( - jsm: JetStreamManager, - whisperTimeoutMs: number, - pdfTimeoutMs: number, - pdfAttempts: number, - jobsMaxBytes: number, - eventsMaxBytes: number, - natsReplicas: number, -): Promise { - const streamConfig = { - name: JOBS_STREAM_NAME, - subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], - retention: RetentionPolicy.Workqueue, - storage: StorageType.File, - max_bytes: jobsMaxBytes, - num_replicas: natsReplicas, - }; - - try { - await jsm.streams.add(streamConfig); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.streams.update(JOBS_STREAM_NAME, { - subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], - max_bytes: jobsMaxBytes, - num_replicas: natsReplicas, - }); - } - - const eventsStreamConfig = { - name: EVENTS_STREAM_NAME, - subjects: [OP_EVENTS_SUBJECT_WILDCARD], - retention: RetentionPolicy.Limits, - storage: StorageType.File, - max_bytes: eventsMaxBytes, - max_age: nanos(COMPUTE_STATE_TTL_MS), - num_replicas: natsReplicas, - }; - - try { - await jsm.streams.add(eventsStreamConfig); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.streams.update(EVENTS_STREAM_NAME, { - subjects: [OP_EVENTS_SUBJECT_WILDCARD], - max_bytes: eventsMaxBytes, - max_age: nanos(COMPUTE_STATE_TTL_MS), - num_replicas: natsReplicas, - }); - } - - const ensureConsumer = async ( - name: string, - subject: string, - ackWaitMs: number, - maxDeliver: number, - ): Promise => { - const config = { - durable_name: name, - ack_policy: AckPolicy.Explicit, - deliver_policy: DeliverPolicy.All, - replay_policy: ReplayPolicy.Instant, - filter_subject: subject, - ack_wait: nanos(Math.max(ackWaitMs, 1_000)), - max_deliver: maxDeliver, - }; - - try { - await jsm.consumers.add(JOBS_STREAM_NAME, config); - } catch (error) { - if (!isAlreadyExistsError(error)) throw error; - await jsm.consumers.update(JOBS_STREAM_NAME, name, { - filter_subject: subject, - ack_wait: nanos(Math.max(ackWaitMs, 1_000)), - max_deliver: maxDeliver, - }); - } - }; - - await Promise.all([ - ensureConsumer(WHISPER_CONSUMER_NAME, WHISPER_JOBS_SUBJECT, whisperTimeoutMs + 15_000, WHISPER_MAX_DELIVER), - ensureConsumer(LAYOUT_CONSUMER_NAME, LAYOUT_JOBS_SUBJECT, pdfTimeoutMs + 15_000, pdfAttempts), - ]); -} - export async function createComputeWorkerApp(options: CreateComputeWorkerAppOptions = {}): Promise { const port = options.port ?? readIntEnv('PORT', 8081); const host = options.host ?? (process.env.COMPUTE_WORKER_HOST?.trim() || '0.0.0.0'); @@ -560,11 +323,9 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti // the session via ensureConnected(). let session: NatsSession | null = null; let connecting: Promise | null = null; - let workerLoops: Promise[] = []; let idleTimer: NodeJS.Timeout | null = null; let orphanSweepTimer: NodeJS.Timeout | null = null; let stopping = false; - let loopStopRequested = false; // Activity accounting feeding the idle detector. The worker is considered idle // only when no HTTP request is in flight, no SSE stream is open, no job is @@ -575,7 +336,6 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti let lastActivityAt = Date.now(); let lastActivityReason = 'startup'; let lastIdleStatusLogAt = 0; - const jobGate = new ConcurrencyGate(jobConcurrency); const markActivity = (reason: string): void => { lastActivityAt = Date.now(); @@ -641,7 +401,6 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti // Clear synchronously (before any await) so concurrent requests reconnect a // fresh session instead of using the connection we're about to close. session = null; - loopStopRequested = true; if (idleTimer) { clearInterval(idleTimer); idleTimer = null; @@ -655,9 +414,7 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti } catch { // ignore close errors } - await Promise.allSettled(workerLoops); - workerLoops = []; - loopStopRequested = false; + await workerLoops.stop(); app.log.info({ reason }, 'nats disconnected'); } @@ -668,15 +425,15 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti const nc: NatsConnection = await connect(connectOpts); const js: JetStreamClient = jetstream(nc, { timeout: NATS_API_TIMEOUT_MS }); const jsm: JetStreamManager = await jetstreamManager(nc, { timeout: NATS_API_TIMEOUT_MS }); - await ensureJetStreamResources( + await ensureJetStreamResources({ jsm, whisperTimeoutMs, pdfTimeoutMs, pdfAttempts, - jobsStreamMaxBytes, - eventsStreamMaxBytes, + jobsMaxBytes: jobsStreamMaxBytes, + eventsMaxBytes: eventsStreamMaxBytes, natsReplicas, - ); + }); const kv = await new Kvm(js).create(COMPUTE_STATE_BUCKET, { replicas: natsReplicas, history: 1, @@ -690,7 +447,12 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti sessionGeneration += 1; orphanRecoveryDoneForGeneration = -1; markActivity('nats_connected'); - startWorkerLoops(next); + if (!disableWorkers) { + workerLoops.start(next, { + whisper: next.whisperConsumer, + pdfLayout: next.layoutConsumer, + }); + } startIdleTimer(); startOrphanSweepTimer(); // Safety net: if the connection closes for any reason (network drop after @@ -709,82 +471,22 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti } } - const s3 = disableWorkers ? null : buildS3Client(); - const s3Bucket = disableWorkers ? '' : requireEnv('S3_BUCKET'); const s3Prefix = normalizeS3Prefix(process.env.S3_PREFIX); - - const ensureSafeKey = (key: string): string => { - const trimmed = key.trim(); - if (!trimmed.startsWith(`${s3Prefix}/`)) { - throw new Error('Object key prefix mismatch'); - } - return trimmed; + const storageDisabled = async (): Promise => { + throw new Error('S3 access is disabled for this worker app instance'); }; - - const readObjectByKey = async (key: string): Promise => { - if (!s3) { - throw new Error('S3 access is disabled for this worker app instance'); + const storage: ArtifactStorage = disableWorkers + ? { + readObject: storageDisabled, + objectExists: storageDisabled, + deleteObject: storageDisabled, + putParsedPdf: storageDisabled, } - const safeKey = ensureSafeKey(key); - const response = await s3.send(new GetObjectCommand({ - Bucket: s3Bucket, - Key: safeKey, - })); - const bytes = await bodyToBuffer(response.Body); - return toArrayBuffer(new Uint8Array(bytes)); - }; - - const objectExists = async (key: string): Promise => { - if (!s3) { - throw new Error('S3 access is disabled for this worker app instance'); - } - const safeKey = ensureSafeKey(key); - try { - await s3.send(new HeadObjectCommand({ - Bucket: s3Bucket, - Key: safeKey, - })); - return true; - } catch (error) { - const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } }; - if ( - maybe.$metadata?.httpStatusCode === 404 - || maybe.name === 'NotFound' - || maybe.name === 'NoSuchKey' - || maybe.Code === 'NotFound' - || maybe.Code === 'NoSuchKey' - ) { - return false; - } - throw error; - } - }; - - const deleteObjectByKey = async (key: string): Promise => { - if (!s3) { - throw new Error('S3 access is disabled for this worker app instance'); - } - await s3.send(new DeleteObjectCommand({ - Bucket: s3Bucket, - Key: ensureSafeKey(key), - })); - }; - - const putParsedObject = async (documentId: string, namespace: string | null, parsed: unknown): Promise => { - if (!s3) { - throw new Error('S3 access is disabled for this worker app instance'); - } - const key = parsedPdfArtifactKey({ documentId, namespace, prefix: s3Prefix }); - const body = Buffer.from(JSON.stringify(parsed)); - await s3.send(new PutObjectCommand({ - Bucket: s3Bucket, - Key: key, - Body: body, - ContentType: 'application/json', - ServerSideEncryption: 'AES256', - })); - return key; - }; + : createArtifactStorage({ + bucket: requireEnv('S3_BUCKET'), + prefix: s3Prefix, + client: createS3ClientFromEnv(requireEnv), + }); if (prewarmModels && !disableWorkers) { await ensureComputeModels(); @@ -1081,7 +783,7 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti namespace: parsed.data.namespace, prefix: s3Prefix, }); - const hasArtifact = await (options.routeDeps?.artifactExists ?? objectExists)(artifactKey); + const hasArtifact = await (options.routeDeps?.artifactExists ?? storage.objectExists)(artifactKey); const opKey = buildPdfOperationKey(parsed.data); const index = await operationStateStore.getOpIndex?.(opKey); const operation = index?.opId ? await operationStateStore.getOpState(index.opId) : null; @@ -1257,451 +959,28 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti return reply; }); - const runWhisper = async ( - payload: WhisperAlignJobRequest, - queueWaitMs: number, - ): Promise => { - const parsed = alignSchema.parse(payload); + const jobHandlers = createJobHandlers({ + storage, + whisperTimeoutMs, + pdfTimeoutMs, + pdfHardCapMs, + }); - const s3FetchStartedAt = Date.now(); - const audioBuffer = await withTimeout( - readObjectByKey(parsed.audioObjectKey), - whisperTimeoutMs, - 'whisper s3 fetch', - ); - const s3FetchMs = Date.now() - s3FetchStartedAt; - - const computeStartedAt = Date.now(); - const result = await withTimeout( - runWhisperAlignmentFromAudioBuffer({ - audioBuffer, - text: parsed.text, - cacheKey: parsed.cacheKey, - lang: parsed.lang, - }), - whisperTimeoutMs, - 'whisper alignment job', - ); - - const computeMs = Date.now() - computeStartedAt; - return { - ...result, - timing: { - queueWaitMs, - s3FetchMs, - computeMs, - }, - }; - }; - - const runLayout = async ( - payload: PdfLayoutJobRequest, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ): Promise => { - const parsed = layoutSchema.parse(payload); - - const s3FetchStartedAt = Date.now(); - const pdfBytes = await withTimeout( - readObjectByKey(parsed.documentObjectKey), - Math.max(pdfTimeoutMs, 1_000), - 'pdf s3 fetch', - ); - const s3FetchMs = Date.now() - s3FetchStartedAt; - - let lastTotalPages = 0; - let lastPagesParsed = 0; - const computeStartedAt = Date.now(); - const result = await withIdleTimeoutAndHardCap({ - idleTimeoutMs: Math.max(pdfTimeoutMs, 1_000), - hardCapMs: pdfHardCapMs, - label: 'pdf layout job', - run: async (touchProgress) => runPdfLayoutFromPdfBuffer({ - documentId: parsed.documentId, - pdfBytes, - onPageStarted: async ({ pageNumber, totalPages }) => { - touchProgress(); - lastTotalPages = totalPages; - if (!hooks?.onProgress) return; - await hooks.onProgress(buildInferProgressForPageStart({ - pageNumber, - totalPages, - })); - }, - onPageParsed: async ({ pageNumber, totalPages }) => { - touchProgress(); - lastTotalPages = totalPages; - lastPagesParsed = pageNumber; - if (!hooks?.onProgress) return; - await hooks.onProgress(buildInferProgressForPageParsed({ - pageNumber, - totalPages, - })); - }, - }), - }); - - const computeMs = Date.now() - computeStartedAt; - if (hooks?.onProgress && lastTotalPages > 0) { - await hooks.onProgress({ - totalPages: lastTotalPages, - pagesParsed: lastPagesParsed, - currentPage: lastPagesParsed || undefined, - phase: 'merge', - }); - } - const parsedObjectKey = await persistParsedPdfWhileSourceExists({ - sourceObjectKey: parsed.documentObjectKey, - sourceExists: objectExists, - putParsedObject: () => putParsedObject(parsed.documentId, parsed.namespace, result.parsed), - deleteParsedObject: deleteObjectByKey, - }); - return { - parsedObjectKey, - timing: { - queueWaitMs, - s3FetchMs, - computeMs, - }, - }; - }; - - type ProcessMessageInput = { - msg: JsMsg; - codec: JsonCodec>; - run: ( - payload: TPayload, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ) => Promise; - workerLabel: string; - }; - - type ProcessMessageContext = { - decoded: QueuedJob; - workerLabel: string; - startedAt: number; - queueWaitTiming?: { queueWaitMs: number }; - latestProgress?: PdfLayoutProgress; - }; - - function extractTiming(result: unknown): WorkerJobTiming | undefined { - if (!result || typeof result !== 'object' || !('timing' in result)) return undefined; - return (result as { timing?: WorkerJobTiming }).timing; - } - - async function markRunning( - context: ProcessMessageContext, - updatedAt: number, - options?: { includeStartedAt?: boolean }, - ): Promise { - if (context.latestProgress) { - await orchestrator.markProgress({ - opId: context.decoded.opId, - progress: context.latestProgress, - updatedAt, - ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), - }); - return; - } - - await orchestrator.markRunning({ - opId: context.decoded.opId, - ...(options?.includeStartedAt === false ? {} : { startedAt: context.startedAt }), - updatedAt, - ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), - }); - } - - async function markProgress( - context: ProcessMessageContext, - progress: PdfLayoutProgress, - updatedAt: number, - ): Promise { - context.latestProgress = progress; - await markRunning(context, updatedAt); - } - - async function markTerminal(input: { - context: ProcessMessageContext; - status: 'succeeded' | 'failed'; - result?: TResult; - errorMessage?: string; - timing?: WorkerJobTiming; - updatedAt: number; - }): Promise { - if (input.status === 'succeeded') { - await orchestrator.markSucceeded({ - opId: input.context.decoded.opId, - result: input.result as WhisperAlignJobResult | PdfLayoutJobResult, - updatedAt: input.updatedAt, - ...(input.timing ? { timing: input.timing } : {}), - }); - return; - } - - await orchestrator.markFailed({ - opId: input.context.decoded.opId, - error: { message: input.errorMessage ?? 'unknown worker failure' }, - updatedAt: input.updatedAt, - ...(input.timing ? { timing: input.timing } : {}), - }); - } - - async function handleRetry(input: { - context: ProcessMessageContext | null; - msg: JsMsg; - errorMessage: string; - }): Promise { - const deliveryCount = input.msg.info.deliveryCount; - const kind = input.context?.decoded.kind ?? 'pdf_layout'; - const retryAction = decideRetryAction({ - kind, - deliveryCount, - pdfAttempts, - whisperMaxDeliver: WHISPER_MAX_DELIVER, - }); - const hasRetriesLeft = retryAction === 'nak_retry'; - const maxAttempts = kind === 'whisper_align' ? WHISPER_MAX_DELIVER : pdfAttempts; - - if (input.context) { - const now = Date.now(); - const retryTiming = buildQueueWaitTiming(input.context.decoded.queuedAt, now); - const persistOpUpdate = hasRetriesLeft - ? (input.context.latestProgress - ? orchestrator.markProgress({ - opId: input.context.decoded.opId, - progress: input.context.latestProgress, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - }) - : orchestrator.markRunning({ - opId: input.context.decoded.opId, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - })) - : markTerminal({ - context: input.context, - status: 'failed', - errorMessage: input.errorMessage, - updatedAt: now, - ...(retryTiming ? { timing: retryTiming } : {}), - }); - - await persistOpUpdate.catch((stateError) => { - app.log.error({ - worker: input.context?.workerLabel, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - error: toErrorMessage(stateError), - }, 'failed to persist operation state'); - }); - } - - if (hasRetriesLeft) { - input.msg.nak(); - app.log.error({ - worker: input.context?.workerLabel, - kind: input.context?.decoded.kind, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - status: 'running', - error: input.errorMessage, - deliveryCount, - maxAttempts, - retryAction: 'nack_retry', - }, 'job.terminal'); - return; - } - - input.msg.term(input.errorMessage); - app.log.error({ - worker: input.context?.workerLabel, - kind: input.context?.decoded.kind, - opId: input.context?.decoded.opId, - jobId: input.context?.decoded.jobId, - status: 'failed', - error: input.errorMessage, - deliveryCount, - maxAttempts, - retrySuppressed: kind === 'whisper_align' ? 'whisper_align' : undefined, - retryAction: 'term', - }, 'job.terminal'); - } - - async function processMessage(input: ProcessMessageInput): Promise { - let context: ProcessMessageContext | null = null; - let heartbeat: NodeJS.Timeout | null = null; - try { - const decoded = input.codec.decode(input.msg.data); - const startedAt = Date.now(); - context = { - decoded, - workerLabel: input.workerLabel, - startedAt, - queueWaitTiming: buildQueueWaitTiming(decoded.queuedAt, startedAt), - }; - - await markRunning(context, startedAt); - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - queueWaitMs: context.queueWaitTiming?.queueWaitMs ?? null, - deliveryCount: input.msg.info.deliveryCount, - }, 'job.started'); - - heartbeat = setInterval(() => { - const now = Date.now(); - void markRunning(context!, now).catch((stateError) => { - app.log.error({ - worker: input.workerLabel, - opId: context?.decoded.opId, - jobId: context?.decoded.jobId, - error: toErrorMessage(stateError), - }, 'failed to persist operation heartbeat state'); - }); - }, RUNNING_HEARTBEAT_MS); - - const result = await input.run(decoded.payload, context.queueWaitTiming?.queueWaitMs ?? 0, { - onProgress: async (progress) => { - try { - input.msg.working(); - } catch (ackError) { - app.log.warn({ - worker: input.workerLabel, - kind: context?.decoded.kind, - opId: context?.decoded.opId, - jobId: context?.decoded.jobId, - error: toErrorMessage(ackError), - }, 'failed to extend JetStream ack wait on progress'); - } - await markProgress(context!, progress, Date.now()); - }, - }); - const resultTiming = extractTiming(result); - const now = Date.now(); - - await markTerminal({ - context, - status: 'succeeded', - result, - timing: resultTiming, - updatedAt: now, - }); - - input.msg.ack(); - const terminalDurationMs = safeDurationMs(context.startedAt, now); - const slowJobLogThresholdMs = SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND[decoded.kind]; - if ((terminalDurationMs ?? 0) >= slowJobLogThresholdMs) { - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - durationMs: terminalDurationMs ?? null, - timing: resultTiming ?? null, - }, 'job.stage'); - } - app.log.info({ - worker: input.workerLabel, - kind: decoded.kind, - opId: decoded.opId, - jobId: decoded.jobId, - status: 'succeeded', - durationMs: terminalDurationMs ?? null, - resultRef: extractResultRef(decoded.kind, result), - timing: resultTiming ?? null, - }, 'job.terminal'); - } catch (error) { - await handleRetry({ - context, - msg: input.msg, - errorMessage: toErrorMessage(error), - }); - } finally { - if (heartbeat) clearInterval(heartbeat); - } - } - - async function createWorkerLoop(input: { - owner: NatsSession; - consumer: Consumer; - codec: JsonCodec>; - run: ( - payload: TPayload, - queueWaitMs: number, - hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, - ) => Promise; - workerLabel: string; - }): Promise { - // Exit when the loop's connection is no longer the active session (idle - // disconnect, unexpected close, or replaced by a reconnect). - const detached = (): boolean => stopping || loopStopRequested || session !== input.owner; - while (!detached()) { - let msg: JsMsg | null = null; - try { - try { - // Bounded pull so the loop yields periodically and exits promptly when - // the session is torn down for idle (nc.close() rejects the pending pull). - msg = await input.consumer.next({ expires: PULL_EXPIRES_MS }); - } catch (error) { - if (detached()) return; - app.log.error({ error: toErrorMessage(error), worker: input.workerLabel }, 'worker pull failed'); - await sleep(LOOP_ERROR_BACKOFF_MS); - continue; - } - - // An empty pull is not activity; let the idle window advance. - if (!msg) continue; - markActivity(`job_received:${input.workerLabel}`); - inFlightJobs += 1; - await jobGate.acquire(); - if (detached()) { - return; - } - await processMessage({ - msg, - codec: input.codec, - run: input.run, - workerLabel: input.workerLabel, - }); - } finally { - if (msg) { - jobGate.release(); - inFlightJobs = Math.max(0, inFlightJobs - 1); - markActivity(`job_completed:${input.workerLabel}`); - } - } - } - } - - function startWorkerLoops(active: NatsSession): void { - if (disableWorkers) return; - // Always starts a fresh set bound to the new session. Any loops from a prior - // session self-terminate once they observe session !== their owner. - workerLoops = []; - loopStopRequested = false; - for (let i = 0; i < jobConcurrency; i += 1) { - workerLoops.push(createWorkerLoop({ - owner: active, - consumer: active.whisperConsumer, - codec: whisperJobCodec, - run: runWhisper, - workerLabel: `whisper-${i + 1}`, - })); - } - for (let i = 0; i < jobConcurrency; i += 1) { - workerLoops.push(createWorkerLoop({ - owner: active, - consumer: active.layoutConsumer, - codec: layoutJobCodec, - run: runLayout, - workerLabel: `layout-${i + 1}`, - })); - } - } + const workerLoops = createWorkerLoopController({ + orchestrator, + handlers: jobHandlers, + logger: app.log, + jobConcurrency, + pdfAttempts, + whisperCodec: whisperJobCodec, + pdfCodec: layoutJobCodec, + isOwnerActive: (owner) => session === owner, + isStopping: () => stopping, + markActivity, + onInFlightJobsChanged: (delta) => { + inFlightJobs = Math.max(0, inFlightJobs + delta); + }, + }); const close = async (): Promise => { if (stopping) return; @@ -1715,7 +994,7 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti orphanSweepTimer = null; } await app.close(); - await Promise.allSettled(workerLoops); + await workerLoops.stop(); const current = session; session = null; if (current) { diff --git a/compute-worker/src/compute/api-contracts/index.ts b/compute-worker/src/api/contracts.ts similarity index 87% rename from compute-worker/src/compute/api-contracts/index.ts rename to compute-worker/src/api/contracts.ts index 487a618..d6e87f3 100644 --- a/compute-worker/src/compute/api-contracts/index.ts +++ b/compute-worker/src/api/contracts.ts @@ -1,24 +1,24 @@ -import type { TTSSentenceAlignment } from '../types/tts'; -import type { ParsedPdfDocument } from '../types/parsed-pdf'; +import type { TTSSentenceAlignment } from '../inference/types/tts'; +import type { ParsedPdfDocument } from '../inference/types/parsed-pdf'; export type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment, TTSSentenceWord, -} from '../types/tts'; +} from '../inference/types/tts'; export type { ParsedPdfBlockKind, ParsedPdfBlockFragment, ParsedPdfBlock, ParsedPdfPage, ParsedPdfDocument, -} from '../types/parsed-pdf'; +} from '../inference/types/parsed-pdf'; export const ALIGN_QUEUE_NAME = 'whisper-align'; export const PDF_LAYOUT_QUEUE_NAME = 'pdf-layout'; -export { PDF_PARSER_VERSION } from '../pdf/parser-version'; -export { encodeParserVersion } from '../pdf/parser-version-key'; +export { PDF_PARSER_VERSION } from '../inference/pdf/parser-version'; +export { encodeParserVersion } from '../inference/pdf/parser-version-key'; export interface WhisperAlignJobBase { text: string; diff --git a/compute-worker/src/api/operation-keys.ts b/compute-worker/src/api/operation-keys.ts index 9c990b5..a6763fe 100644 --- a/compute-worker/src/api/operation-keys.ts +++ b/compute-worker/src/api/operation-keys.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { PDF_PARSER_VERSION } from '../compute/pdf/parser-version'; +import { PDF_PARSER_VERSION } from '../inference/pdf/parser-version'; function sha256Hex(input: string): string { return createHash('sha256').update(input).digest('hex'); diff --git a/compute-worker/src/api/public-operation.ts b/compute-worker/src/api/public-operation.ts index 9ab98a4..ac9b43b 100644 --- a/compute-worker/src/api/public-operation.ts +++ b/compute-worker/src/api/public-operation.ts @@ -1,4 +1,4 @@ -import type { WorkerOperationState } from '../compute/api-contracts'; +import type { WorkerOperationState } from '../api/contracts'; import { pdfSubjectFromOperationKey } from './operation-keys'; export type PublicOperationSubject = diff --git a/compute-worker/src/compute/config/cpu-budget.ts b/compute-worker/src/inference/config/cpu-budget.ts similarity index 100% rename from compute-worker/src/compute/config/cpu-budget.ts rename to compute-worker/src/inference/config/cpu-budget.ts diff --git a/compute-worker/src/compute/config/timeout.ts b/compute-worker/src/inference/config/timeout.ts similarity index 100% rename from compute-worker/src/compute/config/timeout.ts rename to compute-worker/src/inference/config/timeout.ts diff --git a/compute-worker/src/compute/index.ts b/compute-worker/src/inference/index.ts similarity index 93% rename from compute-worker/src/compute/index.ts rename to compute-worker/src/inference/index.ts index ea7b118..255fdce 100644 --- a/compute-worker/src/compute/index.ts +++ b/compute-worker/src/inference/index.ts @@ -1,4 +1,4 @@ -export * from './api-contracts'; +export * from '../api/contracts'; export { getComputeJobConcurrency, getAvailableCpuCores, @@ -23,4 +23,4 @@ export { normalizeTextItemsForLayout } from './pdf/normalize-text'; export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map'; export { buildGoertzelCoefficients, goertzelPower } from './whisper/spectral'; export { buildWordsFromTimestampedTokens, extractTokenStartTimestamps } from './whisper/token-timestamps'; -export * from './control-plane'; +export * from '../operations'; diff --git a/compute-worker/src/compute/local-runtime.ts b/compute-worker/src/inference/local-runtime.ts similarity index 100% rename from compute-worker/src/compute/local-runtime.ts rename to compute-worker/src/inference/local-runtime.ts diff --git a/compute-worker/src/compute/pdf/assets/LICENSE.txt b/compute-worker/src/inference/pdf/assets/LICENSE.txt similarity index 100% rename from compute-worker/src/compute/pdf/assets/LICENSE.txt rename to compute-worker/src/inference/pdf/assets/LICENSE.txt diff --git a/compute-worker/src/compute/pdf/assets/manifest.json b/compute-worker/src/inference/pdf/assets/manifest.json similarity index 100% rename from compute-worker/src/compute/pdf/assets/manifest.json rename to compute-worker/src/inference/pdf/assets/manifest.json diff --git a/compute-worker/src/compute/pdf/merge.ts b/compute-worker/src/inference/pdf/merge.ts similarity index 100% rename from compute-worker/src/compute/pdf/merge.ts rename to compute-worker/src/inference/pdf/merge.ts diff --git a/compute-worker/src/compute/pdf/model.ts b/compute-worker/src/inference/pdf/model.ts similarity index 100% rename from compute-worker/src/compute/pdf/model.ts rename to compute-worker/src/inference/pdf/model.ts diff --git a/compute-worker/src/compute/pdf/normalize-text.ts b/compute-worker/src/inference/pdf/normalize-text.ts similarity index 100% rename from compute-worker/src/compute/pdf/normalize-text.ts rename to compute-worker/src/inference/pdf/normalize-text.ts diff --git a/compute-worker/src/compute/pdf/parse.ts b/compute-worker/src/inference/pdf/parse.ts similarity index 100% rename from compute-worker/src/compute/pdf/parse.ts rename to compute-worker/src/inference/pdf/parse.ts diff --git a/compute-worker/src/compute/pdf/parser-version-key.ts b/compute-worker/src/inference/pdf/parser-version-key.ts similarity index 100% rename from compute-worker/src/compute/pdf/parser-version-key.ts rename to compute-worker/src/inference/pdf/parser-version-key.ts diff --git a/compute-worker/src/compute/pdf/parser-version.ts b/compute-worker/src/inference/pdf/parser-version.ts similarity index 100% rename from compute-worker/src/compute/pdf/parser-version.ts rename to compute-worker/src/inference/pdf/parser-version.ts diff --git a/compute-worker/src/compute/pdf/pdfjs-runtime.ts b/compute-worker/src/inference/pdf/pdfjs-runtime.ts similarity index 100% rename from compute-worker/src/compute/pdf/pdfjs-runtime.ts rename to compute-worker/src/inference/pdf/pdfjs-runtime.ts diff --git a/compute-worker/src/compute/pdf/render.ts b/compute-worker/src/inference/pdf/render.ts similarity index 100% rename from compute-worker/src/compute/pdf/render.ts rename to compute-worker/src/inference/pdf/render.ts diff --git a/compute-worker/src/compute/pdf/runLayoutModel.ts b/compute-worker/src/inference/pdf/runLayoutModel.ts similarity index 100% rename from compute-worker/src/compute/pdf/runLayoutModel.ts rename to compute-worker/src/inference/pdf/runLayoutModel.ts diff --git a/compute-worker/src/compute/pdf/stitch.ts b/compute-worker/src/inference/pdf/stitch.ts similarity index 100% rename from compute-worker/src/compute/pdf/stitch.ts rename to compute-worker/src/inference/pdf/stitch.ts diff --git a/compute-worker/src/compute/pdf/types.ts b/compute-worker/src/inference/pdf/types.ts similarity index 100% rename from compute-worker/src/compute/pdf/types.ts rename to compute-worker/src/inference/pdf/types.ts diff --git a/compute-worker/src/compute/platform/docstore.ts b/compute-worker/src/inference/platform/docstore.ts similarity index 100% rename from compute-worker/src/compute/platform/docstore.ts rename to compute-worker/src/inference/platform/docstore.ts diff --git a/compute-worker/src/compute/platform/ffmpeg.ts b/compute-worker/src/inference/platform/ffmpeg.ts similarity index 100% rename from compute-worker/src/compute/platform/ffmpeg.ts rename to compute-worker/src/inference/platform/ffmpeg.ts diff --git a/compute-worker/src/compute/types/index.ts b/compute-worker/src/inference/types/index.ts similarity index 100% rename from compute-worker/src/compute/types/index.ts rename to compute-worker/src/inference/types/index.ts diff --git a/compute-worker/src/compute/types/parsed-pdf.ts b/compute-worker/src/inference/types/parsed-pdf.ts similarity index 100% rename from compute-worker/src/compute/types/parsed-pdf.ts rename to compute-worker/src/inference/types/parsed-pdf.ts diff --git a/compute-worker/src/compute/types/tts.ts b/compute-worker/src/inference/types/tts.ts similarity index 100% rename from compute-worker/src/compute/types/tts.ts rename to compute-worker/src/inference/types/tts.ts diff --git a/compute-worker/src/compute/whisper/align.ts b/compute-worker/src/inference/whisper/align.ts similarity index 100% rename from compute-worker/src/compute/whisper/align.ts rename to compute-worker/src/inference/whisper/align.ts diff --git a/compute-worker/src/compute/whisper/alignment-map.ts b/compute-worker/src/inference/whisper/alignment-map.ts similarity index 100% rename from compute-worker/src/compute/whisper/alignment-map.ts rename to compute-worker/src/inference/whisper/alignment-map.ts diff --git a/compute-worker/src/compute/whisper/assets/LICENSE.txt b/compute-worker/src/inference/whisper/assets/LICENSE.txt similarity index 100% rename from compute-worker/src/compute/whisper/assets/LICENSE.txt rename to compute-worker/src/inference/whisper/assets/LICENSE.txt diff --git a/compute-worker/src/compute/whisper/assets/manifest.json b/compute-worker/src/inference/whisper/assets/manifest.json similarity index 100% rename from compute-worker/src/compute/whisper/assets/manifest.json rename to compute-worker/src/inference/whisper/assets/manifest.json diff --git a/compute-worker/src/compute/whisper/assets/mel_filters.npz b/compute-worker/src/inference/whisper/assets/mel_filters.npz similarity index 100% rename from compute-worker/src/compute/whisper/assets/mel_filters.npz rename to compute-worker/src/inference/whisper/assets/mel_filters.npz diff --git a/compute-worker/src/compute/whisper/model.ts b/compute-worker/src/inference/whisper/model.ts similarity index 100% rename from compute-worker/src/compute/whisper/model.ts rename to compute-worker/src/inference/whisper/model.ts diff --git a/compute-worker/src/compute/whisper/spectral.ts b/compute-worker/src/inference/whisper/spectral.ts similarity index 100% rename from compute-worker/src/compute/whisper/spectral.ts rename to compute-worker/src/inference/whisper/spectral.ts diff --git a/compute-worker/src/compute/whisper/token-timestamps.ts b/compute-worker/src/inference/whisper/token-timestamps.ts similarity index 100% rename from compute-worker/src/compute/whisper/token-timestamps.ts rename to compute-worker/src/inference/whisper/token-timestamps.ts diff --git a/compute-worker/src/control-plane/json-codec.ts b/compute-worker/src/infrastructure/json-codec.ts similarity index 100% rename from compute-worker/src/control-plane/json-codec.ts rename to compute-worker/src/infrastructure/json-codec.ts diff --git a/compute-worker/src/control-plane/jetstream.ts b/compute-worker/src/infrastructure/nats-adapters.ts similarity index 99% rename from compute-worker/src/control-plane/jetstream.ts rename to compute-worker/src/infrastructure/nats-adapters.ts index e5367eb..00d2526 100644 --- a/compute-worker/src/control-plane/jetstream.ts +++ b/compute-worker/src/infrastructure/nats-adapters.ts @@ -8,12 +8,12 @@ import type { OperationState, OperationStateStore, QueuedOperation, -} from '../compute/control-plane'; +} from '../operations'; import type { PdfLayoutJobRequest, WhisperAlignJobRequest, WorkerOperationKind, -} from '../compute/api-contracts'; +} from '../api/contracts'; import { createJsonCodec } from './json-codec'; export interface KvEntryLike { diff --git a/compute-worker/src/infrastructure/nats.ts b/compute-worker/src/infrastructure/nats.ts new file mode 100644 index 0000000..429a03d --- /dev/null +++ b/compute-worker/src/infrastructure/nats.ts @@ -0,0 +1,104 @@ +import { + AckPolicy, + DeliverPolicy, + ReplayPolicy, + RetentionPolicy, + StorageType, + type JetStreamManager, +} from '@nats-io/jetstream'; +import { nanos } from '@nats-io/transport-node'; +import { OP_EVENTS_SUBJECT_WILDCARD } from './nats-adapters'; + +export const JOBS_STREAM_NAME = 'compute_jobs'; +export const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; +export const LAYOUT_JOBS_SUBJECT = 'jobs.layout'; +export const WHISPER_CONSUMER_NAME = 'compute_whisper'; +export const LAYOUT_CONSUMER_NAME = 'compute_layout'; +export const EVENTS_STREAM_NAME = 'compute_events'; +export const COMPUTE_STATE_BUCKET = 'compute_state'; +export const COMPUTE_STATE_TTL_MS = 24 * 60 * 60 * 1000; +export const NATS_API_TIMEOUT_MS = 60_000; + +const WHISPER_MAX_DELIVER = 1; + +function isAlreadyExistsError(error: unknown): boolean { + const message = (error instanceof Error ? error.message : String(error)).toLowerCase(); + return message.includes('already in use') || message.includes('already exists'); +} + +export async function ensureJetStreamResources(input: { + jsm: JetStreamManager; + whisperTimeoutMs: number; + pdfTimeoutMs: number; + pdfAttempts: number; + jobsMaxBytes: number; + eventsMaxBytes: number; + natsReplicas: number; +}): Promise { + const streamConfig = { + name: JOBS_STREAM_NAME, + subjects: [WHISPER_JOBS_SUBJECT, LAYOUT_JOBS_SUBJECT], + retention: RetentionPolicy.Workqueue, + storage: StorageType.File, + max_bytes: input.jobsMaxBytes, + num_replicas: input.natsReplicas, + }; + try { + await input.jsm.streams.add(streamConfig); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await input.jsm.streams.update(JOBS_STREAM_NAME, { + subjects: streamConfig.subjects, + max_bytes: input.jobsMaxBytes, + num_replicas: input.natsReplicas, + }); + } + + const eventsStreamConfig = { + name: EVENTS_STREAM_NAME, + subjects: [OP_EVENTS_SUBJECT_WILDCARD], + retention: RetentionPolicy.Limits, + storage: StorageType.File, + max_bytes: input.eventsMaxBytes, + max_age: nanos(COMPUTE_STATE_TTL_MS), + num_replicas: input.natsReplicas, + }; + try { + await input.jsm.streams.add(eventsStreamConfig); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await input.jsm.streams.update(EVENTS_STREAM_NAME, { + subjects: eventsStreamConfig.subjects, + max_bytes: input.eventsMaxBytes, + max_age: eventsStreamConfig.max_age, + num_replicas: input.natsReplicas, + }); + } + + const ensureConsumer = async (name: string, subject: string, ackWaitMs: number, maxDeliver: number) => { + const config = { + durable_name: name, + ack_policy: AckPolicy.Explicit, + deliver_policy: DeliverPolicy.All, + replay_policy: ReplayPolicy.Instant, + filter_subject: subject, + ack_wait: nanos(Math.max(ackWaitMs, 1_000)), + max_deliver: maxDeliver, + }; + try { + await input.jsm.consumers.add(JOBS_STREAM_NAME, config); + } catch (error) { + if (!isAlreadyExistsError(error)) throw error; + await input.jsm.consumers.update(JOBS_STREAM_NAME, name, { + filter_subject: subject, + ack_wait: config.ack_wait, + max_deliver: maxDeliver, + }); + } + }; + + await Promise.all([ + ensureConsumer(WHISPER_CONSUMER_NAME, WHISPER_JOBS_SUBJECT, input.whisperTimeoutMs + 15_000, WHISPER_MAX_DELIVER), + ensureConsumer(LAYOUT_CONSUMER_NAME, LAYOUT_JOBS_SUBJECT, input.pdfTimeoutMs + 15_000, input.pdfAttempts), + ]); +} diff --git a/compute-worker/src/infrastructure/storage.ts b/compute-worker/src/infrastructure/storage.ts new file mode 100644 index 0000000..1190da3 --- /dev/null +++ b/compute-worker/src/infrastructure/storage.ts @@ -0,0 +1,127 @@ +import { + DeleteObjectCommand, + GetObjectCommand, + HeadObjectCommand, + PutObjectCommand, + S3Client, +} from '@aws-sdk/client-s3'; +import { parsedPdfArtifactKey } from '../storage/artifact-addressing'; + +export interface ArtifactStorage { + readObject(key: string): Promise; + objectExists(key: string): Promise; + deleteObject(key: string): Promise; + putParsedPdf(documentId: string, namespace: string | null, parsed: unknown): Promise; +} + +export interface ArtifactStorageConfig { + bucket: string; + prefix: string; + client: S3Client; +} + +function bodyToBuffer(body: unknown): Promise | Buffer { + if (!body) return Buffer.alloc(0); + if (body instanceof Uint8Array) return Buffer.from(body); + if (ArrayBuffer.isView(body)) return Buffer.from(body.buffer, body.byteOffset, body.byteLength); + if (body instanceof ArrayBuffer) return Buffer.from(body); + if (typeof body === 'object' && body !== null && 'transformToByteArray' in body) { + const maybe = body as { transformToByteArray?: () => Promise }; + if (typeof maybe.transformToByteArray === 'function') { + return maybe.transformToByteArray().then((bytes) => Buffer.from(bytes)); + } + } + if (typeof body === 'object' && body !== null && 'on' in body) { + return (async () => { + const chunks: Buffer[] = []; + for await (const chunk of body as NodeJS.ReadableStream) { + if (Buffer.isBuffer(chunk)) chunks.push(chunk); + else if (typeof chunk === 'string') chunks.push(Buffer.from(chunk)); + else chunks.push(Buffer.from(chunk as Uint8Array)); + } + return Buffer.concat(chunks); + })(); + } + throw new Error('Unsupported S3 response body type'); +} + +function toArrayBuffer(bytes: Uint8Array): ArrayBuffer { + const copy = new Uint8Array(bytes.byteLength); + copy.set(bytes); + return copy.buffer; +} + +function isNotFound(error: unknown): boolean { + const maybe = error as { name?: string; Code?: string; $metadata?: { httpStatusCode?: number } }; + return maybe.$metadata?.httpStatusCode === 404 + || maybe.name === 'NotFound' + || maybe.name === 'NoSuchKey' + || maybe.Code === 'NotFound' + || maybe.Code === 'NoSuchKey'; +} + +export function normalizeS3Prefix(prefix: string | undefined): string { + const value = (prefix || 'openreader').trim(); + return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; +} + +export function createS3ClientFromEnv(requireEnv: (name: string) => string): S3Client { + return new S3Client({ + region: requireEnv('S3_REGION'), + endpoint: process.env.S3_ENDPOINT?.trim() || undefined, + forcePathStyle: ['1', 'true', 'yes', 'on'].includes(process.env.S3_FORCE_PATH_STYLE?.trim().toLowerCase() ?? ''), + requestChecksumCalculation: 'WHEN_REQUIRED', + responseChecksumValidation: 'WHEN_REQUIRED', + credentials: { + accessKeyId: requireEnv('S3_ACCESS_KEY_ID'), + secretAccessKey: requireEnv('S3_SECRET_ACCESS_KEY'), + }, + }); +} + +export function createArtifactStorage(config: ArtifactStorageConfig): ArtifactStorage { + const safeKey = (key: string): string => { + const trimmed = key.trim(); + if (!trimmed.startsWith(`${config.prefix}/`)) throw new Error('Object key prefix mismatch'); + return trimmed; + }; + + return { + async readObject(key) { + const response = await config.client.send(new GetObjectCommand({ + Bucket: config.bucket, + Key: safeKey(key), + })); + return toArrayBuffer(new Uint8Array(await bodyToBuffer(response.Body))); + }, + async objectExists(key) { + try { + await config.client.send(new HeadObjectCommand({ + Bucket: config.bucket, + Key: safeKey(key), + })); + return true; + } catch (error) { + if (isNotFound(error)) return false; + throw error; + } + }, + async deleteObject(key) { + await config.client.send(new DeleteObjectCommand({ + Bucket: config.bucket, + Key: safeKey(key), + })); + }, + async putParsedPdf(documentId, namespace, parsed) { + const key = parsedPdfArtifactKey({ documentId, namespace, prefix: config.prefix }); + await config.client.send(new PutObjectCommand({ + Bucket: config.bucket, + Key: key, + Body: Buffer.from(JSON.stringify(parsed)), + ContentType: 'application/json', + ServerSideEncryption: 'AES256', + })); + return key; + }, + }; +} diff --git a/compute-worker/src/jobs/handlers.ts b/compute-worker/src/jobs/handlers.ts new file mode 100644 index 0000000..90a4068 --- /dev/null +++ b/compute-worker/src/jobs/handlers.ts @@ -0,0 +1,126 @@ +import { z } from 'zod'; +import { + runPdfLayoutFromPdfBuffer, + runWhisperAlignmentFromAudioBuffer, +} from '../inference/local-runtime'; +import { withIdleTimeoutAndHardCap, withTimeout } from '../inference'; +import type { + PdfLayoutJobRequest, + PdfLayoutJobResult, + PdfLayoutProgress, + WhisperAlignJobRequest, + WhisperAlignJobResult, +} from '../api/contracts'; +import type { ArtifactStorage } from '../infrastructure/storage'; +import { persistParsedPdfWhileSourceExists } from './pdf-artifact-persistence'; +import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; + +const whisperRequestSchema = z.object({ + text: z.string().trim().min(1), + lang: z.string().trim().min(1).max(16).optional(), + cacheKey: z.string().trim().min(1).max(256).optional(), + audioObjectKey: z.string().trim().min(1).max(2048), +}); + +const pdfRequestSchema = z.object({ + documentId: z.string().trim().min(1), + namespace: z.string().trim().min(1).max(128).nullable(), + documentObjectKey: z.string().trim().min(1).max(2048), +}); + +export interface JobHandlers { + runWhisper(payload: WhisperAlignJobRequest, queueWaitMs: number): Promise; + runPdfLayout( + payload: PdfLayoutJobRequest, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ): Promise; +} + +export function createJobHandlers(input: { + storage: ArtifactStorage; + whisperTimeoutMs: number; + pdfTimeoutMs: number; + pdfHardCapMs: number; +}): JobHandlers { + return { + async runWhisper(payload, queueWaitMs) { + const parsed = whisperRequestSchema.parse(payload); + const s3FetchStartedAt = Date.now(); + const audioBuffer = await withTimeout( + input.storage.readObject(parsed.audioObjectKey), + input.whisperTimeoutMs, + 'whisper s3 fetch', + ); + const s3FetchMs = Date.now() - s3FetchStartedAt; + const computeStartedAt = Date.now(); + const result = await withTimeout( + runWhisperAlignmentFromAudioBuffer({ + audioBuffer, + text: parsed.text, + cacheKey: parsed.cacheKey, + lang: parsed.lang, + }), + input.whisperTimeoutMs, + 'whisper alignment job', + ); + return { + ...result, + timing: { queueWaitMs, s3FetchMs, computeMs: Date.now() - computeStartedAt }, + }; + }, + + async runPdfLayout(payload, queueWaitMs, hooks) { + const parsed = pdfRequestSchema.parse(payload); + const s3FetchStartedAt = Date.now(); + const pdfBytes = await withTimeout( + input.storage.readObject(parsed.documentObjectKey), + Math.max(input.pdfTimeoutMs, 1_000), + 'pdf s3 fetch', + ); + const s3FetchMs = Date.now() - s3FetchStartedAt; + let lastTotalPages = 0; + let lastPagesParsed = 0; + const computeStartedAt = Date.now(); + const result = await withIdleTimeoutAndHardCap({ + idleTimeoutMs: Math.max(input.pdfTimeoutMs, 1_000), + hardCapMs: input.pdfHardCapMs, + label: 'pdf layout job', + run: async (touchProgress) => runPdfLayoutFromPdfBuffer({ + documentId: parsed.documentId, + pdfBytes, + onPageStarted: async ({ pageNumber, totalPages }) => { + touchProgress(); + lastTotalPages = totalPages; + await hooks?.onProgress?.(buildInferProgressForPageStart({ pageNumber, totalPages })); + }, + onPageParsed: async ({ pageNumber, totalPages }) => { + touchProgress(); + lastTotalPages = totalPages; + lastPagesParsed = pageNumber; + await hooks?.onProgress?.(buildInferProgressForPageParsed({ pageNumber, totalPages })); + }, + }), + }); + const computeMs = Date.now() - computeStartedAt; + if (hooks?.onProgress && lastTotalPages > 0) { + await hooks.onProgress({ + totalPages: lastTotalPages, + pagesParsed: lastPagesParsed, + currentPage: lastPagesParsed || undefined, + phase: 'merge', + }); + } + const parsedObjectKey = await persistParsedPdfWhileSourceExists({ + sourceObjectKey: parsed.documentObjectKey, + sourceExists: input.storage.objectExists, + putParsedObject: () => input.storage.putParsedPdf(parsed.documentId, parsed.namespace, result.parsed), + deleteParsedObject: input.storage.deleteObject, + }); + return { + parsedObjectKey, + timing: { queueWaitMs, s3FetchMs, computeMs }, + }; + }, + }; +} diff --git a/compute-worker/src/pdf-artifact-persistence.ts b/compute-worker/src/jobs/pdf-artifact-persistence.ts similarity index 100% rename from compute-worker/src/pdf-artifact-persistence.ts rename to compute-worker/src/jobs/pdf-artifact-persistence.ts diff --git a/compute-worker/src/pdf-progress.ts b/compute-worker/src/jobs/pdf-progress.ts similarity index 89% rename from compute-worker/src/pdf-progress.ts rename to compute-worker/src/jobs/pdf-progress.ts index cc5c025..4aefe08 100644 --- a/compute-worker/src/pdf-progress.ts +++ b/compute-worker/src/jobs/pdf-progress.ts @@ -1,4 +1,4 @@ -import type { PdfLayoutProgress } from './compute/api-contracts'; +import type { PdfLayoutProgress } from '../api/contracts'; export function buildInferProgressForPageStart(input: { pageNumber: number; diff --git a/compute-worker/src/worker-loop-policy.ts b/compute-worker/src/jobs/worker-loop-policy.ts similarity index 91% rename from compute-worker/src/worker-loop-policy.ts rename to compute-worker/src/jobs/worker-loop-policy.ts index bce8653..ce3547a 100644 --- a/compute-worker/src/worker-loop-policy.ts +++ b/compute-worker/src/jobs/worker-loop-policy.ts @@ -1,4 +1,4 @@ -import type { WorkerOperationKind } from './compute/api-contracts'; +import type { WorkerOperationKind } from '../api/contracts'; export type RetryAction = 'nak_retry' | 'term_fail'; diff --git a/compute-worker/src/jobs/worker-loop.ts b/compute-worker/src/jobs/worker-loop.ts new file mode 100644 index 0000000..194bf62 --- /dev/null +++ b/compute-worker/src/jobs/worker-loop.ts @@ -0,0 +1,312 @@ +import type { Consumer, JsMsg } from '@nats-io/jetstream'; +import type { + PdfLayoutJobRequest, + PdfLayoutJobResult, + PdfLayoutProgress, + WhisperAlignJobRequest, + WhisperAlignJobResult, + WorkerJobTiming, + WorkerOperationKind, +} from '../api/contracts'; +import type { JsonCodec } from '../infrastructure/json-codec'; +import type { JobHandlers } from './handlers'; +import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy'; + +const LOOP_ERROR_BACKOFF_MS = 500; +const RUNNING_HEARTBEAT_MS = 5000; +const PULL_EXPIRES_MS = 5_000; +const WHISPER_MAX_DELIVER = 1; +const SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND: Record = { + whisper_align: 15_000, + pdf_layout: 120_000, +}; + +export interface QueuedJob { + jobId: string; + opId: string; + opKey: string; + kind: WorkerOperationKind; + queuedAt: number; + payload: TPayload; +} + +interface WorkerLoopOrchestrator { + markRunning(input: { opId: string; startedAt?: number; updatedAt?: number; timing?: WorkerJobTiming }): Promise; + markProgress(input: { + opId: string; + progress: PdfLayoutProgress; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markSucceeded(input: { opId: string; result: unknown; updatedAt?: number; timing?: WorkerJobTiming }): Promise; + markFailed(input: { + opId: string; + error: { message: string; code?: string } | string; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; +} + +interface WorkerLogger { + info(data: unknown, message?: string): void; + warn(data: unknown, message?: string): void; + error(data: unknown, message?: string): void; +} + +class ConcurrencyGate { + private inFlight = 0; + private readonly queue: Array<() => void> = []; + + constructor(private readonly limit: number) {} + + async acquire(): Promise { + if (this.inFlight < this.limit) { + this.inFlight += 1; + return; + } + await new Promise((resolve) => { + this.queue.push(() => { + this.inFlight += 1; + resolve(); + }); + }); + } + + release(): void { + this.inFlight = Math.max(0, this.inFlight - 1); + this.queue.shift()?.(); + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function toErrorMessage(error: unknown): string { + return error instanceof Error && error.message ? error.message : String(error); +} + +function safeDurationMs(start: number, end: number): number { + return Math.max(0, Math.floor(end - start)); +} + +function extractTiming(result: unknown): WorkerJobTiming | undefined { + if (!result || typeof result !== 'object' || !('timing' in result)) return undefined; + return (result as { timing?: WorkerJobTiming }).timing; +} + +function extractResultRef(kind: WorkerOperationKind, result: unknown): string | undefined { + if (kind !== 'pdf_layout' || !result || typeof result !== 'object') return undefined; + const maybe = result as { parsedObjectKey?: unknown }; + return typeof maybe.parsedObjectKey === 'string' ? maybe.parsedObjectKey : undefined; +} + +export function createWorkerLoopController(input: { + orchestrator: WorkerLoopOrchestrator; + handlers: JobHandlers; + logger: WorkerLogger; + jobConcurrency: number; + pdfAttempts: number; + whisperCodec: JsonCodec>; + pdfCodec: JsonCodec>; + isOwnerActive: (owner: object) => boolean; + isStopping: () => boolean; + markActivity: (reason: string) => void; + onInFlightJobsChanged: (delta: number) => void; +}) { + const gate = new ConcurrencyGate(Math.max(1, Math.floor(input.jobConcurrency))); + let loops: Promise[] = []; + let stopRequested = false; + + type Context = { + decoded: QueuedJob; + workerLabel: string; + startedAt: number; + queueWaitTiming?: { queueWaitMs: number }; + latestProgress?: PdfLayoutProgress; + }; + + const markRunning = async (context: Context, updatedAt: number): Promise => { + if (context.latestProgress) { + await input.orchestrator.markProgress({ + opId: context.decoded.opId, + progress: context.latestProgress, + updatedAt, + ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), + }); + return; + } + await input.orchestrator.markRunning({ + opId: context.decoded.opId, + startedAt: context.startedAt, + updatedAt, + ...(context.queueWaitTiming ? { timing: context.queueWaitTiming } : {}), + }); + }; + + const processMessage = async (work: { + msg: JsMsg; + codec: JsonCodec>; + run: JobHandlers['runWhisper'] | JobHandlers['runPdfLayout']; + workerLabel: string; + }): Promise => { + let context: Context | null = null; + let heartbeat: NodeJS.Timeout | null = null; + try { + const decoded = work.codec.decode(work.msg.data); + const startedAt = Date.now(); + context = { + decoded, + workerLabel: work.workerLabel, + startedAt, + queueWaitTiming: buildQueueWaitTiming(decoded.queuedAt, startedAt), + }; + await markRunning(context, startedAt); + input.logger.info({ + worker: work.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + queueWaitMs: context.queueWaitTiming?.queueWaitMs ?? null, + deliveryCount: work.msg.info.deliveryCount, + }, 'job.started'); + heartbeat = setInterval(() => { + void markRunning(context!, Date.now()).catch((error) => { + input.logger.error({ + worker: work.workerLabel, + opId: context?.decoded.opId, + jobId: context?.decoded.jobId, + error: toErrorMessage(error), + }, 'failed to persist operation heartbeat state'); + }); + }, RUNNING_HEARTBEAT_MS); + const result = await work.run(decoded.payload as never, context.queueWaitTiming?.queueWaitMs ?? 0, { + onProgress: async (progress) => { + try { + work.msg.working(); + } catch (error) { + input.logger.warn({ + worker: work.workerLabel, + kind: context?.decoded.kind, + opId: context?.decoded.opId, + jobId: context?.decoded.jobId, + error: toErrorMessage(error), + }, 'failed to extend JetStream ack wait on progress'); + } + context!.latestProgress = progress; + await markRunning(context!, Date.now()); + }, + }); + const timing = extractTiming(result); + const now = Date.now(); + await input.orchestrator.markSucceeded({ + opId: decoded.opId, + result, + updatedAt: now, + ...(timing ? { timing } : {}), + }); + work.msg.ack(); + const durationMs = safeDurationMs(startedAt, now); + if (durationMs >= SLOW_JOB_LOG_THRESHOLD_MS_BY_KIND[decoded.kind]) { + input.logger.info({ worker: work.workerLabel, kind: decoded.kind, opId: decoded.opId, jobId: decoded.jobId, durationMs, timing: timing ?? null }, 'job.stage'); + } + input.logger.info({ + worker: work.workerLabel, + kind: decoded.kind, + opId: decoded.opId, + jobId: decoded.jobId, + status: 'succeeded', + durationMs, + resultRef: extractResultRef(decoded.kind, result), + timing: timing ?? null, + }, 'job.terminal'); + } catch (error) { + const errorMessage = toErrorMessage(error); + const deliveryCount = work.msg.info.deliveryCount; + const kind = context?.decoded.kind ?? 'pdf_layout'; + const action = decideRetryAction({ kind, deliveryCount, pdfAttempts: input.pdfAttempts, whisperMaxDeliver: WHISPER_MAX_DELIVER }); + const timing = context ? buildQueueWaitTiming(context.decoded.queuedAt, Date.now()) : undefined; + if (context) { + const update = action === 'nak_retry' + ? markRunning(context, Date.now()) + : input.orchestrator.markFailed({ + opId: context.decoded.opId, + error: { message: errorMessage }, + updatedAt: Date.now(), + ...(timing ? { timing } : {}), + }); + await update.catch((stateError) => input.logger.error({ + worker: context?.workerLabel, + opId: context?.decoded.opId, + jobId: context?.decoded.jobId, + error: toErrorMessage(stateError), + }, 'failed to persist operation state')); + } + if (action === 'nak_retry') work.msg.nak(); + else work.msg.term(errorMessage); + input.logger.error({ + worker: context?.workerLabel, + kind: context?.decoded.kind, + opId: context?.decoded.opId, + jobId: context?.decoded.jobId, + status: action === 'nak_retry' ? 'running' : 'failed', + error: errorMessage, + deliveryCount, + retryAction: action === 'nak_retry' ? 'nack_retry' : 'term', + }, 'job.terminal'); + } finally { + if (heartbeat) clearInterval(heartbeat); + } + }; + + const runLoop = async (work: { + owner: object; + consumer: Consumer; + codec: JsonCodec>; + run: JobHandlers['runWhisper'] | JobHandlers['runPdfLayout']; + workerLabel: string; + }): Promise => { + const detached = () => input.isStopping() || stopRequested || !input.isOwnerActive(work.owner); + while (!detached()) { + let msg: JsMsg | null = null; + try { + try { + msg = await work.consumer.next({ expires: PULL_EXPIRES_MS }); + } catch (error) { + if (detached()) return; + input.logger.error({ error: toErrorMessage(error), worker: work.workerLabel }, 'worker pull failed'); + await sleep(LOOP_ERROR_BACKOFF_MS); + continue; + } + if (!msg) continue; + input.markActivity(`job_received:${work.workerLabel}`); + input.onInFlightJobsChanged(1); + await gate.acquire(); + if (detached()) return; + await processMessage({ ...work, msg }); + } finally { + if (msg) { + gate.release(); + input.onInFlightJobsChanged(-1); + input.markActivity(`job_completed:${work.workerLabel}`); + } + } + } + }; + + return { + start(owner: object, consumers: { whisper: Consumer; pdfLayout: Consumer }): void { + stopRequested = false; + loops = []; + for (let i = 0; i < input.jobConcurrency; i += 1) { + loops.push(runLoop({ owner, consumer: consumers.whisper, codec: input.whisperCodec, run: input.handlers.runWhisper, workerLabel: `whisper-${i + 1}` })); + loops.push(runLoop({ owner, consumer: consumers.pdfLayout, codec: input.pdfCodec, run: input.handlers.runPdfLayout, workerLabel: `layout-${i + 1}` })); + } + }, + async stop(): Promise { + stopRequested = true; + await Promise.allSettled(loops); + loops = []; + }, + }; +} diff --git a/compute-worker/src/compute/control-plane/index.ts b/compute-worker/src/operations/index.ts similarity index 71% rename from compute-worker/src/compute/control-plane/index.ts rename to compute-worker/src/operations/index.ts index 5b8255d..35699ba 100644 --- a/compute-worker/src/compute/control-plane/index.ts +++ b/compute-worker/src/operations/index.ts @@ -1,4 +1,4 @@ export * from './types'; export * from './state-machine'; -export * from './orchestrator'; +export * from './service'; export * from './sse'; diff --git a/compute-worker/src/orphan-recovery.ts b/compute-worker/src/operations/recovery.ts similarity index 98% rename from compute-worker/src/orphan-recovery.ts rename to compute-worker/src/operations/recovery.ts index 8257b4d..5aaaa10 100644 --- a/compute-worker/src/orphan-recovery.ts +++ b/compute-worker/src/operations/recovery.ts @@ -4,7 +4,7 @@ import type { WorkerJobTiming, WorkerJobState, WorkerOperationState, -} from './compute/api-contracts'; +} from '../api/contracts'; export type StreamedOperationState = WorkerOperationState; diff --git a/compute-worker/src/compute/control-plane/orchestrator.ts b/compute-worker/src/operations/service.ts similarity index 99% rename from compute-worker/src/compute/control-plane/orchestrator.ts rename to compute-worker/src/operations/service.ts index 7f621a4..1469277 100644 --- a/compute-worker/src/compute/control-plane/orchestrator.ts +++ b/compute-worker/src/operations/service.ts @@ -5,7 +5,7 @@ import type { WorkerOperationKind, WorkerOperationRequest, WorkerOperationState, -} from '../api-contracts'; +} from '../api/contracts'; import { buildQueuedState, createErrorShape, diff --git a/compute-worker/src/compute/control-plane/sse.ts b/compute-worker/src/operations/sse.ts similarity index 100% rename from compute-worker/src/compute/control-plane/sse.ts rename to compute-worker/src/operations/sse.ts diff --git a/compute-worker/src/compute/control-plane/state-machine.ts b/compute-worker/src/operations/state-machine.ts similarity index 98% rename from compute-worker/src/compute/control-plane/state-machine.ts rename to compute-worker/src/operations/state-machine.ts index d280215..3144e8c 100644 --- a/compute-worker/src/compute/control-plane/state-machine.ts +++ b/compute-worker/src/operations/state-machine.ts @@ -4,7 +4,7 @@ import type { WorkerOperationKind, WorkerOperationRequest, WorkerOperationState, -} from '../api-contracts'; +} from '../api/contracts'; export function isTerminalStatus(status: WorkerJobState): boolean { return status === 'succeeded' || status === 'failed'; diff --git a/compute-worker/src/compute/control-plane/types.ts b/compute-worker/src/operations/types.ts similarity index 98% rename from compute-worker/src/compute/control-plane/types.ts rename to compute-worker/src/operations/types.ts index 9fc66ff..1d60200 100644 --- a/compute-worker/src/compute/control-plane/types.ts +++ b/compute-worker/src/operations/types.ts @@ -2,7 +2,7 @@ import type { WorkerOperationEvent, WorkerOperationKind, WorkerOperationState, -} from '../api-contracts'; +} from '../api/contracts'; export type OperationState = WorkerOperationState; export type OperationEvent = WorkerOperationEvent; diff --git a/compute-worker/src/server.ts b/compute-worker/src/server.ts index 1bf176c..3a110cc 100644 --- a/compute-worker/src/server.ts +++ b/compute-worker/src/server.ts @@ -1,4 +1,4 @@ -import { startComputeWorkerFromEnv } from './runtime'; +import { startComputeWorkerFromEnv } from './api/app'; void startComputeWorkerFromEnv().catch((error) => { console.error('[compute-worker] fatal startup error', error); diff --git a/compute-worker/src/storage/artifact-addressing.ts b/compute-worker/src/storage/artifact-addressing.ts index a4d1ee6..e341f23 100644 --- a/compute-worker/src/storage/artifact-addressing.ts +++ b/compute-worker/src/storage/artifact-addressing.ts @@ -1,5 +1,5 @@ -import { PDF_PARSER_VERSION } from '../compute/pdf/parser-version'; -import { encodeParserVersion } from '../compute/pdf/parser-version-key'; +import { PDF_PARSER_VERSION } from '../inference/pdf/parser-version'; +import { encodeParserVersion } from '../inference/pdf/parser-version-key'; const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; diff --git a/compute-worker/tests/api/routes.test.ts b/compute-worker/tests/api/routes.test.ts index e7336ee..4dea817 100644 --- a/compute-worker/tests/api/routes.test.ts +++ b/compute-worker/tests/api/routes.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from 'vitest'; -import { createComputeWorkerApp } from '../../src/runtime'; +import { createComputeWorkerApp } from '../../src/api/app'; import { FakeControlPlane } from '../fixtures/fake-control-plane'; const AUTH = { authorization: 'Bearer test-token' }; diff --git a/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts b/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts index 71436d0..d035613 100644 --- a/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts +++ b/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { mergeTextWithRegions } from '../../../src/compute'; +import { mergeTextWithRegions } from '../../../src/inference'; describe('mergeTextWithRegions', () => { test('assigns text items to containing regions by centroid and joins in reading order', () => { diff --git a/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts b/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts index f11272d..167dc46 100644 --- a/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts +++ b/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { normalizeTextItemsForLayout } from '../../../src/compute'; +import { normalizeTextItemsForLayout } from '../../../src/inference'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; function makeTextItem( diff --git a/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts b/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts index fb7b602..97d4b19 100644 --- a/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts +++ b/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { stitchCrossPageBlocks } from '../../../src/compute'; -import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/compute/types'; +import { stitchCrossPageBlocks } from '../../../src/inference'; +import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/inference/types'; import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures'; function makeBlock( diff --git a/compute-worker/tests/compute/algorithms/support/document-fixtures.ts b/compute-worker/tests/compute/algorithms/support/document-fixtures.ts index ffcde2d..f8db418 100644 --- a/compute-worker/tests/compute/algorithms/support/document-fixtures.ts +++ b/compute-worker/tests/compute/algorithms/support/document-fixtures.ts @@ -1,5 +1,5 @@ -import type { BaseDocument } from '../../../src/types/documents'; -import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/compute/types'; +import type { BaseDocument } from '../../../../../src/types/documents'; +import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/inference/types'; export function makeBaseDocument(overrides: Partial = {}): BaseDocument { return { diff --git a/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts b/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts index 2eea68f..fedea8b 100644 --- a/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts +++ b/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; import { mapWordsToSentenceOffsets, -} from '../../../src/compute'; +} from '../../../src/inference'; describe('whisper alignment mapping', () => { test('maps words to sentence offsets with punctuation and repeated spaces', () => { diff --git a/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts b/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts index ac4a3d8..6b51d62 100644 --- a/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts +++ b/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { buildGoertzelCoefficients, goertzelPower } from '../../../src/compute'; +import { buildGoertzelCoefficients, goertzelPower } from '../../../src/inference'; function dftPower(samples: Float32Array, k: number): number { const n = samples.length; diff --git a/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts b/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts index a6ff75d..6f8bced 100644 --- a/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts +++ b/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts @@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-node'; import { buildWordsFromTimestampedTokens, extractTokenStartTimestamps, -} from '../../../src/compute'; +} from '../../../src/inference'; describe('whisper token timestamp alignment', () => { test('extracts monotonic token timestamps from cross-attention maps', () => { diff --git a/compute-worker/tests/compute/control-plane/orchestrator.test.ts b/compute-worker/tests/compute/control-plane/orchestrator.test.ts index 6c9bbab..66964d3 100644 --- a/compute-worker/tests/compute/control-plane/orchestrator.test.ts +++ b/compute-worker/tests/compute/control-plane/orchestrator.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import type { WorkerOperationRequest } from '../../../src/compute/api-contracts'; -import { OperationOrchestrator } from '../../../src/compute/control-plane'; +import type { WorkerOperationRequest } from '../../../src/api/contracts'; +import { OperationOrchestrator } from '../../../src/operations'; import { InMemoryOperationEventStream, InMemoryOperationQueue, diff --git a/compute-worker/tests/compute/control-plane/run-layout-model.test.ts b/compute-worker/tests/compute/control-plane/run-layout-model.test.ts index 27ef706..c8278c9 100644 --- a/compute-worker/tests/compute/control-plane/run-layout-model.test.ts +++ b/compute-worker/tests/compute/control-plane/run-layout-model.test.ts @@ -70,13 +70,13 @@ vi.mock('@napi-rs/canvas', () => { }; }); -vi.mock('../../../src/compute/pdf/model', () => ({ +vi.mock('../../../src/inference/pdf/model', () => ({ ensureModel: vi.fn(async () => '/tmp/model.onnx'), MODEL_CONFIG_PATH: '/tmp/model-config.json', MODEL_PREPROCESSOR_PATH: '/tmp/model-preprocessor.json', })); -vi.mock('../../../src/compute/config/cpu-budget', () => ({ +vi.mock('../../../src/inference/config/cpu-budget', () => ({ getOnnxThreadsPerJob: vi.fn(() => 1), })); @@ -107,7 +107,7 @@ describe('runLayoutModel', () => { }, }; - const { runLayoutModel } = await import('../../../src/compute/pdf/runLayoutModel'); + const { runLayoutModel } = await import('../../../src/inference/pdf/runLayoutModel'); const regions = await runLayoutModel({ pageWidth: 100, pageHeight: 100, @@ -154,7 +154,7 @@ describe('runLayoutModel', () => { }, }; - const { runLayoutModel } = await import('../../../src/compute/pdf/runLayoutModel'); + const { runLayoutModel } = await import('../../../src/inference/pdf/runLayoutModel'); const regions = await runLayoutModel({ pageWidth: 100, pageHeight: 100, diff --git a/compute-worker/tests/compute/control-plane/sse.test.ts b/compute-worker/tests/compute/control-plane/sse.test.ts index 3461229..2697d6d 100644 --- a/compute-worker/tests/compute/control-plane/sse.test.ts +++ b/compute-worker/tests/compute/control-plane/sse.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../../src/compute/control-plane/sse'; +import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../../src/operations/sse'; describe('sse codec', () => { test('encodes event id and payload and decodes both reliably', () => { diff --git a/compute-worker/tests/compute/control-plane/state-machine.test.ts b/compute-worker/tests/compute/control-plane/state-machine.test.ts index a9d5390..8fba44a 100644 --- a/compute-worker/tests/compute/control-plane/state-machine.test.ts +++ b/compute-worker/tests/compute/control-plane/state-machine.test.ts @@ -1,11 +1,11 @@ import { describe, expect, test } from 'vitest'; -import type { WorkerOperationState } from '../../../src/compute/api-contracts'; +import type { WorkerOperationState } from '../../../src/api/contracts'; import { explainReplacementReason, isInflightStatus, isTerminalStatus, shouldReuseExistingOperation, -} from '../../../src/compute/control-plane/state-machine'; +} from '../../../src/operations/state-machine'; function runningState(overrides: Partial = {}): WorkerOperationState { return { diff --git a/compute-worker/tests/compute/helpers/in-memory-control-plane.ts b/compute-worker/tests/compute/helpers/in-memory-control-plane.ts index 7395b2b..811521b 100644 --- a/compute-worker/tests/compute/helpers/in-memory-control-plane.ts +++ b/compute-worker/tests/compute/helpers/in-memory-control-plane.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events'; -import type { WorkerOperationKind } from '../../../src/compute/api-contracts'; +import type { WorkerOperationKind } from '../../../src/api/contracts'; import type { OperationEvent, OperationEventStream, @@ -8,7 +8,7 @@ import type { OperationState, OperationStateStore, QueuedOperation, -} from '../../../src/compute/control-plane/types'; +} from '../../../src/operations/types'; function topicFor(opId: string): string { return `op.${opId}`; diff --git a/compute-worker/tests/fixtures/fake-control-plane.ts b/compute-worker/tests/fixtures/fake-control-plane.ts index 40ab638..23f2760 100644 --- a/compute-worker/tests/fixtures/fake-control-plane.ts +++ b/compute-worker/tests/fixtures/fake-control-plane.ts @@ -4,8 +4,8 @@ import type { WorkerOperationEvent, WorkerOperationRequest, WorkerOperationState, -} from '../../src/compute/api-contracts'; -import type { ComputeWorkerRouteDeps } from '../../src/runtime'; +} from '../../src/api/contracts'; +import type { ComputeWorkerRouteDeps } from '../../src/api/app'; type ComputeResult = WhisperAlignJobResult | PdfLayoutJobResult; type ComputeState = WorkerOperationState; diff --git a/compute-worker/tests/unit/jetstream-adapters.test.ts b/compute-worker/tests/unit/jetstream-adapters.test.ts index ae86c06..a238996 100644 --- a/compute-worker/tests/unit/jetstream-adapters.test.ts +++ b/compute-worker/tests/unit/jetstream-adapters.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { OperationOrchestrator } from '../../src/compute/control-plane'; -import type { WorkerOperationRequest } from '../../src/compute/api-contracts'; +import { OperationOrchestrator } from '../../src/operations'; +import type { WorkerOperationRequest } from '../../src/api/contracts'; import { JetStreamOperationEventStream, JetStreamOperationQueue, @@ -11,7 +11,7 @@ import { opStateKvKey, type KvEntryLike, type KvStoreLike, -} from '../../src/control-plane/jetstream'; +} from '../../src/infrastructure/nats-adapters'; class FakeKvStore implements KvStoreLike { private readonly data = new Map(); diff --git a/compute-worker/tests/unit/key-and-progress.test.ts b/compute-worker/tests/unit/key-and-progress.test.ts index cbaf5ac..d6274db 100644 --- a/compute-worker/tests/unit/key-and-progress.test.ts +++ b/compute-worker/tests/unit/key-and-progress.test.ts @@ -1,9 +1,9 @@ import { describe, expect, test } from 'vitest'; -import { hashOpKey, opIndexKvKey, opStateKvKey } from '../../src/control-plane/jetstream'; +import { hashOpKey, opIndexKvKey, opStateKvKey } from '../../src/infrastructure/nats-adapters'; import { buildInferProgressForPageParsed, buildInferProgressForPageStart, -} from '../../src/pdf-progress'; +} from '../../src/jobs/pdf-progress'; import { buildPdfOperationKey } from '../../src/api/operation-keys'; import { parsedPdfArtifactKey } from '../../src/storage/artifact-addressing'; diff --git a/compute-worker/tests/unit/orphan-recovery.test.ts b/compute-worker/tests/unit/orphan-recovery.test.ts index 712fdf6..7ec771d 100644 --- a/compute-worker/tests/unit/orphan-recovery.test.ts +++ b/compute-worker/tests/unit/orphan-recovery.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test, vi } from 'vitest'; -import { recoverOrphanedOperations } from '../../src/orphan-recovery'; +import { recoverOrphanedOperations } from '../../src/operations/recovery'; import { FakeControlPlane } from '../fixtures/fake-control-plane'; describe('orphan recovery', () => { diff --git a/compute-worker/tests/unit/pdf-artifact-persistence.test.ts b/compute-worker/tests/unit/pdf-artifact-persistence.test.ts index 9378435..68f3de5 100644 --- a/compute-worker/tests/unit/pdf-artifact-persistence.test.ts +++ b/compute-worker/tests/unit/pdf-artifact-persistence.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test, vi } from 'vitest'; -import { persistParsedPdfWhileSourceExists } from '../../src/pdf-artifact-persistence'; +import { persistParsedPdfWhileSourceExists } from '../../src/jobs/pdf-artifact-persistence'; describe('PDF artifact persistence', () => { test('does not write parsed output after the source was deleted', async () => { diff --git a/compute-worker/tests/unit/worker-loop-policy.test.ts b/compute-worker/tests/unit/worker-loop-policy.test.ts index 818e459..c200e70 100644 --- a/compute-worker/tests/unit/worker-loop-policy.test.ts +++ b/compute-worker/tests/unit/worker-loop-policy.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { buildQueueWaitTiming, decideRetryAction } from '../../src/worker-loop-policy'; +import { buildQueueWaitTiming, decideRetryAction } from '../../src/jobs/worker-loop-policy'; describe('worker loop policy', () => { test('returns queue wait timing with non-negative clamped duration', () => { diff --git a/scripts/check-next-server-bundle.mjs b/scripts/check-next-server-bundle.mjs index 2b3cbcb..8174d40 100644 --- a/scripts/check-next-server-bundle.mjs +++ b/scripts/check-next-server-bundle.mjs @@ -13,7 +13,7 @@ const forbidden = [ 'onnxruntime-node', '@huggingface/tokenizers', '@openreader/compute-worker', - '/compute-worker/src/compute/', + '/compute-worker/src/inference/', ]; const includeExt = new Set(['.js', '.mjs', '.cjs']); diff --git a/tests/unit/compute-worker-client-contract.vitest.spec.ts b/tests/unit/compute-worker-client-contract.vitest.spec.ts index e37f234..39adac6 100644 --- a/tests/unit/compute-worker-client-contract.vitest.spec.ts +++ b/tests/unit/compute-worker-client-contract.vitest.spec.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, test } from 'vitest'; -import { createComputeWorkerApp, type ComputeWorkerApp } from '../../compute-worker/src/runtime'; +import { createComputeWorkerApp, type ComputeWorkerApp } from '../../compute-worker/src/api/app'; import { FakeControlPlane } from '../../compute-worker/tests/fixtures/fake-control-plane'; import { ComputeWorkerClient } from '../../src/lib/server/compute-worker/client'; From 6e69b481034aa7eb58b88ee7a59697e2cdac78b4 Mon Sep 17 00:00:00 2001 From: Richard R Date: Fri, 12 Jun 2026 14:16:13 -0600 Subject: [PATCH 03/11] chore(worker): restructure inference modules and centralize config - Remove legacy inference and platform files from `src/inference/` and migrate responsibilities to new modular locations under `src/infrastructure/`, `src/inference/pdf/`, and `src/inference/whisper/` - Consolidate environment/config logic into `src/infrastructure/config.ts` - Move docstore and ffmpeg platform utilities to `src/infrastructure/platform.ts` - Refactor PDF and Whisper inference code to use new document layout, layout model, and timestamp utilities - Update API contracts to define and export types and constants previously scattered in inference/types - Adjust all imports in jobs, storage, and tests to reference new module structure and type locations - Inline parser version and encoding logic into API contracts - Remove obsolete files and update test fixtures for new type locations This update improves codebase modularity, maintainability, and separation of concerns. No changes to inference or orchestration functionality. BREAKING CHANGE: inference module structure, type imports, and config utilities have changed; downstream code must update imports and integration points. --- compute-worker/src/api/app.ts | 4 +- compute-worker/src/api/contracts.ts | 14 +++--- compute-worker/src/api/operation-keys.ts | 2 +- .../types/parsed-pdf.ts => api/types.ts} | 17 +++++++ .../src/inference/config/cpu-budget.ts | 28 ------------ compute-worker/src/inference/index.ts | 26 ----------- .../pdf/{merge.ts => document-layout.ts} | 16 ++++++- .../{runLayoutModel.ts => layout-model.ts} | 4 +- compute-worker/src/inference/pdf/model.ts | 2 +- .../src/inference/pdf/normalize-text.ts | 2 +- compute-worker/src/inference/pdf/parse.ts | 10 ++--- .../src/inference/pdf/parser-version-key.ts | 9 ---- .../src/inference/pdf/parser-version.ts | 1 - .../pdf/{pdfjs-runtime.ts => pdfjs.ts} | 0 compute-worker/src/inference/pdf/render.ts | 2 +- compute-worker/src/inference/pdf/stitch.ts | 2 +- compute-worker/src/inference/pdf/types.ts | 15 ------- .../src/inference/platform/docstore.ts | 21 --------- .../{local-runtime.ts => runtime.ts} | 0 compute-worker/src/inference/types/index.ts | 17 ------- compute-worker/src/inference/types/tts.ts | 16 ------- compute-worker/src/inference/whisper/align.ts | 33 +++++++++++--- .../src/inference/whisper/alignment-map.ts | 44 ------------------- compute-worker/src/inference/whisper/model.ts | 2 +- .../src/inference/whisper/spectral.ts | 21 --------- .../{token-timestamps.ts => timestamps.ts} | 24 ++++++++++ .../timeout.ts => infrastructure/config.ts} | 20 +++++++++ .../ffmpeg.ts => infrastructure/platform.ts} | 29 +++++++++--- compute-worker/src/jobs/handlers.ts | 4 +- .../src/storage/artifact-addressing.ts | 4 +- .../pdf-merge-text-with-regions.test.ts | 2 +- .../pdf-parse-normalize-text-items.test.ts | 2 +- .../pdf-stitch-cross-page-blocks.test.ts | 4 +- .../algorithms/support/document-fixtures.ts | 2 +- .../whisper-alignment-mapping.test.ts | 2 +- .../algorithms/whisper-spectral.test.ts | 2 +- .../whisper-token-timestamps.test.ts | 2 +- .../control-plane/run-layout-model.test.ts | 6 +-- 38 files changed, 163 insertions(+), 248 deletions(-) rename compute-worker/src/{inference/types/parsed-pdf.ts => api/types.ts} (78%) delete mode 100644 compute-worker/src/inference/config/cpu-budget.ts delete mode 100644 compute-worker/src/inference/index.ts rename compute-worker/src/inference/pdf/{merge.ts => document-layout.ts} (94%) rename compute-worker/src/inference/pdf/{runLayoutModel.ts => layout-model.ts} (98%) delete mode 100644 compute-worker/src/inference/pdf/parser-version-key.ts delete mode 100644 compute-worker/src/inference/pdf/parser-version.ts rename compute-worker/src/inference/pdf/{pdfjs-runtime.ts => pdfjs.ts} (100%) delete mode 100644 compute-worker/src/inference/pdf/types.ts delete mode 100644 compute-worker/src/inference/platform/docstore.ts rename compute-worker/src/inference/{local-runtime.ts => runtime.ts} (100%) delete mode 100644 compute-worker/src/inference/types/index.ts delete mode 100644 compute-worker/src/inference/types/tts.ts delete mode 100644 compute-worker/src/inference/whisper/alignment-map.ts delete mode 100644 compute-worker/src/inference/whisper/spectral.ts rename compute-worker/src/inference/whisper/{token-timestamps.ts => timestamps.ts} (93%) rename compute-worker/src/{inference/config/timeout.ts => infrastructure/config.ts} (84%) rename compute-worker/src/{inference/platform/ffmpeg.ts => infrastructure/platform.ts} (60%) diff --git a/compute-worker/src/api/app.ts b/compute-worker/src/api/app.ts index 1775f4e..77b5a55 100644 --- a/compute-worker/src/api/app.ts +++ b/compute-worker/src/api/app.ts @@ -15,13 +15,13 @@ import { import { Kvm } from '@nats-io/kv'; import { ensureComputeModels, -} from '../inference/local-runtime'; +} from '../inference/runtime'; import { getComputeTimeoutConfig, getComputeOpStaleMs, getAvailableCpuCores, getOnnxThreadsPerJob, -} from '../inference'; +} from '../infrastructure/config'; import { encodeSseFrame, OperationOrchestrator } from '../operations'; import type { PdfLayoutJobRequest, diff --git a/compute-worker/src/api/contracts.ts b/compute-worker/src/api/contracts.ts index d6e87f3..4f78782 100644 --- a/compute-worker/src/api/contracts.ts +++ b/compute-worker/src/api/contracts.ts @@ -1,24 +1,26 @@ -import type { TTSSentenceAlignment } from '../inference/types/tts'; -import type { ParsedPdfDocument } from '../inference/types/parsed-pdf'; +import type { TTSSentenceAlignment, ParsedPdfDocument } from './types'; export type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment, TTSSentenceWord, -} from '../inference/types/tts'; +} from './types'; export type { ParsedPdfBlockKind, ParsedPdfBlockFragment, ParsedPdfBlock, ParsedPdfPage, ParsedPdfDocument, -} from '../inference/types/parsed-pdf'; +} from './types'; export const ALIGN_QUEUE_NAME = 'whisper-align'; export const PDF_LAYOUT_QUEUE_NAME = 'pdf-layout'; -export { PDF_PARSER_VERSION } from '../inference/pdf/parser-version'; -export { encodeParserVersion } from '../inference/pdf/parser-version-key'; +export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69'; + +export function encodeParserVersion(parserVersion: string, defaultVersion = PDF_PARSER_VERSION): string { + return encodeURIComponent(parserVersion.trim() || defaultVersion); +} export interface WhisperAlignJobBase { text: string; diff --git a/compute-worker/src/api/operation-keys.ts b/compute-worker/src/api/operation-keys.ts index a6763fe..d814fb2 100644 --- a/compute-worker/src/api/operation-keys.ts +++ b/compute-worker/src/api/operation-keys.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { PDF_PARSER_VERSION } from '../inference/pdf/parser-version'; +import { PDF_PARSER_VERSION } from '../api/contracts'; function sha256Hex(input: string): string { return createHash('sha256').update(input).digest('hex'); diff --git a/compute-worker/src/inference/types/parsed-pdf.ts b/compute-worker/src/api/types.ts similarity index 78% rename from compute-worker/src/inference/types/parsed-pdf.ts rename to compute-worker/src/api/types.ts index 01d7067..16ad911 100644 --- a/compute-worker/src/inference/types/parsed-pdf.ts +++ b/compute-worker/src/api/types.ts @@ -61,3 +61,20 @@ export interface PdfParseProgress { currentPage?: number; phase: PdfParsePhase; } + +export type TTSAudioBuffer = ArrayBuffer; +export type TTSAudioBytes = number[]; + +export interface TTSSentenceWord { + text: string; + startSec: number; + endSec: number; + charStart: number; + charEnd: number; +} + +export interface TTSSentenceAlignment { + sentence: string; + sentenceIndex: number; + words: TTSSentenceWord[]; +} diff --git a/compute-worker/src/inference/config/cpu-budget.ts b/compute-worker/src/inference/config/cpu-budget.ts deleted file mode 100644 index 50d161b..0000000 --- a/compute-worker/src/inference/config/cpu-budget.ts +++ /dev/null @@ -1,28 +0,0 @@ -import os from 'node:os'; - -function readPositiveInt(name: string, fallback: number): number { - const raw = process.env[name]?.trim(); - if (!raw) return fallback; - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) return fallback; - return Math.floor(parsed); -} - -export function getComputeJobConcurrency(): number { - return readPositiveInt('COMPUTE_JOB_CONCURRENCY', 1); -} - -export function getAvailableCpuCores(): number { - if (typeof os.availableParallelism === 'function') { - const value = os.availableParallelism(); - if (Number.isFinite(value) && value >= 1) return Math.floor(value); - } - const fallback = os.cpus().length; - return Number.isFinite(fallback) && fallback >= 1 ? Math.floor(fallback) : 1; -} - -export function getOnnxThreadsPerJob(): number { - const concurrency = getComputeJobConcurrency(); - const usableCores = Math.max(1, getAvailableCpuCores() - 1); - return Math.max(1, Math.floor(usableCores / concurrency)); -} diff --git a/compute-worker/src/inference/index.ts b/compute-worker/src/inference/index.ts deleted file mode 100644 index 255fdce..0000000 --- a/compute-worker/src/inference/index.ts +++ /dev/null @@ -1,26 +0,0 @@ -export * from '../api/contracts'; -export { - getComputeJobConcurrency, - getAvailableCpuCores, - getOnnxThreadsPerJob, -} from './config/cpu-budget'; -export { - getComputeTimeoutConfig, - getComputeOpStaleMs, - getWorkerClientWaitTimeoutMs, - withTimeout, - withIdleTimeoutAndHardCap, - type ComputeTimeoutConfig, - type ComputeOperationKind, - type IdleTimeoutAndHardCapInput, -} from './config/timeout'; -export { renderPage } from './pdf/render'; -export { mergeTextWithRegions } from './pdf/merge'; -export { PDF_PARSER_VERSION } from './pdf/parser-version'; -export { encodeParserVersion } from './pdf/parser-version-key'; -export { stitchCrossPageBlocks } from './pdf/stitch'; -export { normalizeTextItemsForLayout } from './pdf/normalize-text'; -export { mapWordsToSentenceOffsets, type WhisperWord } from './whisper/alignment-map'; -export { buildGoertzelCoefficients, goertzelPower } from './whisper/spectral'; -export { buildWordsFromTimestampedTokens, extractTokenStartTimestamps } from './whisper/token-timestamps'; -export * from '../operations'; diff --git a/compute-worker/src/inference/pdf/merge.ts b/compute-worker/src/inference/pdf/document-layout.ts similarity index 94% rename from compute-worker/src/inference/pdf/merge.ts rename to compute-worker/src/inference/pdf/document-layout.ts index e1f71b2..8d48fd9 100644 --- a/compute-worker/src/inference/pdf/merge.ts +++ b/compute-worker/src/inference/pdf/document-layout.ts @@ -1,4 +1,18 @@ -import type { LayoutRegion, PdfTextItem } from './types'; +import type { ParsedPdfBlockKind } from '../../api/types'; + +export interface PdfTextItem { + text: string; + x: number; + y: number; + width: number; + height: number; +} + +export interface LayoutRegion { + bbox: [number, number, number, number]; + label: ParsedPdfBlockKind; + confidence?: number; +} const NON_TEXT_REGION_LABELS = new Set(['chart', 'image', 'table', 'seal']); const TEXT_ASSIGNABLE_LABELS = new Set([ diff --git a/compute-worker/src/inference/pdf/runLayoutModel.ts b/compute-worker/src/inference/pdf/layout-model.ts similarity index 98% rename from compute-worker/src/inference/pdf/runLayoutModel.ts rename to compute-worker/src/inference/pdf/layout-model.ts index 5c726e0..4e1e6e2 100644 --- a/compute-worker/src/inference/pdf/runLayoutModel.ts +++ b/compute-worker/src/inference/pdf/layout-model.ts @@ -1,8 +1,8 @@ import * as ort from 'onnxruntime-node'; import { readFile } from 'fs/promises'; -import type { LayoutRegion, PdfTextItem } from './types'; +import type { LayoutRegion, PdfTextItem } from './document-layout'; import { ensureModel, MODEL_CONFIG_PATH, MODEL_PREPROCESSOR_PATH } from './model'; -import { getOnnxThreadsPerJob } from '../config/cpu-budget'; +import { getOnnxThreadsPerJob } from '../../infrastructure/config'; interface RunLayoutInput { pageWidth: number; diff --git a/compute-worker/src/inference/pdf/model.ts b/compute-worker/src/inference/pdf/model.ts index ca09540..2d13cda 100644 --- a/compute-worker/src/inference/pdf/model.ts +++ b/compute-worker/src/inference/pdf/model.ts @@ -3,7 +3,7 @@ import { fileURLToPath } from 'url'; import { createHash } from 'crypto'; import { readFileSync } from 'fs'; import { access, mkdir, rename, writeFile, readFile, unlink, copyFile } from 'fs/promises'; -import { DOCSTORE_DIR } from '../platform/docstore'; +import { DOCSTORE_DIR } from '../../infrastructure/platform'; const DEFAULT_MODEL_BASE_URL = 'https://huggingface.co/Bei0001/PP-DocLayoutV3-ONNX/resolve/main'; const PDF_LAYOUT_MODEL_BASE_URL_ENV = 'PDF_LAYOUT_MODEL_BASE_URL'; diff --git a/compute-worker/src/inference/pdf/normalize-text.ts b/compute-worker/src/inference/pdf/normalize-text.ts index 1e91101..f8e5dc0 100644 --- a/compute-worker/src/inference/pdf/normalize-text.ts +++ b/compute-worker/src/inference/pdf/normalize-text.ts @@ -1,5 +1,5 @@ import type { TextItem } from 'pdfjs-dist/types/src/display/api'; -import type { PdfTextItem } from './types'; +import type { PdfTextItem } from './document-layout'; interface ViewportLike { height: number; diff --git a/compute-worker/src/inference/pdf/parse.ts b/compute-worker/src/inference/pdf/parse.ts index 69d10d2..b4491b2 100644 --- a/compute-worker/src/inference/pdf/parse.ts +++ b/compute-worker/src/inference/pdf/parse.ts @@ -1,10 +1,10 @@ import type { TextItem } from 'pdfjs-dist/types/src/display/api'; -import type { ParsedPdfDocument, ParsedPdfPage } from '../types/parsed-pdf'; +import type { ParsedPdfDocument, ParsedPdfPage } from '../../api/types'; import { ensureModel } from './model'; -import { runLayoutModel } from './runLayoutModel'; -import { mergeTextWithRegions } from './merge'; -import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime'; -import { PDF_PARSER_VERSION } from './parser-version'; +import { runLayoutModel } from './layout-model'; +import { mergeTextWithRegions } from './document-layout'; +import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs'; +import { PDF_PARSER_VERSION } from '../../api/contracts'; import { stitchCrossPageBlocks } from './stitch'; import { renderPage } from './render'; import { normalizeTextItemsForLayout } from './normalize-text'; diff --git a/compute-worker/src/inference/pdf/parser-version-key.ts b/compute-worker/src/inference/pdf/parser-version-key.ts deleted file mode 100644 index f9f82b2..0000000 --- a/compute-worker/src/inference/pdf/parser-version-key.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { PDF_PARSER_VERSION } from './parser-version'; - -export function encodeParserVersion( - parserVersion: string, - defaultVersion = PDF_PARSER_VERSION, -): string { - const normalized = parserVersion.trim() || defaultVersion; - return encodeURIComponent(normalized); -} diff --git a/compute-worker/src/inference/pdf/parser-version.ts b/compute-worker/src/inference/pdf/parser-version.ts deleted file mode 100644 index b5fa7d7..0000000 --- a/compute-worker/src/inference/pdf/parser-version.ts +++ /dev/null @@ -1 +0,0 @@ -export const PDF_PARSER_VERSION = 'pp-doclayoutv3-onnx@800+pdfjs@4.8.69'; diff --git a/compute-worker/src/inference/pdf/pdfjs-runtime.ts b/compute-worker/src/inference/pdf/pdfjs.ts similarity index 100% rename from compute-worker/src/inference/pdf/pdfjs-runtime.ts rename to compute-worker/src/inference/pdf/pdfjs.ts diff --git a/compute-worker/src/inference/pdf/render.ts b/compute-worker/src/inference/pdf/render.ts index 33ec1a1..eb23993 100644 --- a/compute-worker/src/inference/pdf/render.ts +++ b/compute-worker/src/inference/pdf/render.ts @@ -1,5 +1,5 @@ import type { Canvas } from '@napi-rs/canvas'; -import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs-runtime'; +import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs'; type CanvasRuntime = { DOMMatrixCtor: unknown; diff --git a/compute-worker/src/inference/pdf/stitch.ts b/compute-worker/src/inference/pdf/stitch.ts index 98a3d77..d3da6f6 100644 --- a/compute-worker/src/inference/pdf/stitch.ts +++ b/compute-worker/src/inference/pdf/stitch.ts @@ -1,4 +1,4 @@ -import type { ParsedPdfDocument, ParsedPdfBlock } from '../types/parsed-pdf'; +import type { ParsedPdfDocument, ParsedPdfBlock } from '../../api/types'; const STITCHABLE_KINDS = new Set([ 'text', diff --git a/compute-worker/src/inference/pdf/types.ts b/compute-worker/src/inference/pdf/types.ts deleted file mode 100644 index 38089a6..0000000 --- a/compute-worker/src/inference/pdf/types.ts +++ /dev/null @@ -1,15 +0,0 @@ -import type { ParsedPdfBlockKind } from '../types/parsed-pdf'; - -export interface PdfTextItem { - text: string; - x: number; - y: number; - width: number; - height: number; -} - -export interface LayoutRegion { - bbox: [number, number, number, number]; - label: ParsedPdfBlockKind; - confidence?: number; -} diff --git a/compute-worker/src/inference/platform/docstore.ts b/compute-worker/src/inference/platform/docstore.ts deleted file mode 100644 index 0b62401..0000000 --- a/compute-worker/src/inference/platform/docstore.ts +++ /dev/null @@ -1,21 +0,0 @@ -import fs from 'fs'; -import path from 'path'; - -function findMonorepoRoot(startDir: string): string | null { - let current = path.resolve(startDir); - for (;;) { - const marker = path.join(current, 'pnpm-workspace.yaml'); - if (fs.existsSync(marker)) return current; - const parent = path.dirname(current); - if (parent === current) return null; - current = parent; - } -} - -function resolveDocstoreDir(): string { - const repoRoot = findMonorepoRoot(process.cwd()); - if (repoRoot) return path.join(repoRoot, 'docstore'); - return path.join(process.cwd(), 'docstore'); -} - -export const DOCSTORE_DIR = resolveDocstoreDir(); diff --git a/compute-worker/src/inference/local-runtime.ts b/compute-worker/src/inference/runtime.ts similarity index 100% rename from compute-worker/src/inference/local-runtime.ts rename to compute-worker/src/inference/runtime.ts diff --git a/compute-worker/src/inference/types/index.ts b/compute-worker/src/inference/types/index.ts deleted file mode 100644 index 7408914..0000000 --- a/compute-worker/src/inference/types/index.ts +++ /dev/null @@ -1,17 +0,0 @@ -export type { - TTSAudioBuffer, - TTSAudioBytes, - TTSSentenceAlignment, - TTSSentenceWord, -} from './tts'; - -export type { - ParsedPdfBlock, - ParsedPdfBlockFragment, - ParsedPdfBlockKind, - ParsedPdfDocument, - ParsedPdfPage, - PdfParsePhase, - PdfParseProgress, - PdfParseStatus, -} from './parsed-pdf'; diff --git a/compute-worker/src/inference/types/tts.ts b/compute-worker/src/inference/types/tts.ts deleted file mode 100644 index 73db759..0000000 --- a/compute-worker/src/inference/types/tts.ts +++ /dev/null @@ -1,16 +0,0 @@ -export type TTSAudioBuffer = ArrayBuffer; -export type TTSAudioBytes = number[]; - -export interface TTSSentenceWord { - text: string; - startSec: number; - endSec: number; - charStart: number; - charEnd: number; -} - -export interface TTSSentenceAlignment { - sentence: string; - sentenceIndex: number; - words: TTSSentenceWord[]; -} diff --git a/compute-worker/src/inference/whisper/align.ts b/compute-worker/src/inference/whisper/align.ts index d18fa07..073afc4 100644 --- a/compute-worker/src/inference/whisper/align.ts +++ b/compute-worker/src/inference/whisper/align.ts @@ -7,19 +7,18 @@ import { fileURLToPath } from 'url'; import * as ort from 'onnxruntime-node'; import { Tokenizer } from '@huggingface/tokenizers'; import JSZip from 'jszip'; -import type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment } from '../types/tts'; -import { getFFmpegPath } from '../platform/ffmpeg'; -import { getOnnxThreadsPerJob } from '../config/cpu-budget'; -import { getComputeTimeoutConfig } from '../config/timeout'; +import type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment } from '../../api/types'; +import { getFFmpegPath } from '../../infrastructure/platform'; +import { getOnnxThreadsPerJob } from '../../infrastructure/config'; +import { getComputeTimeoutConfig } from '../../infrastructure/config'; import { mapWordsToSentenceOffsets, type WhisperWord, -} from './alignment-map'; -import { buildGoertzelCoefficients, goertzelPower } from './spectral'; +} from './timestamps'; import { buildWordsFromTimestampedTokens, extractTokenStartTimestamps, -} from './token-timestamps'; +} from './timestamps'; import { ensureWhisperModel, WHISPER_CONFIG_PATH, @@ -31,6 +30,26 @@ import { WHISPER_DECODER_WITH_PAST_MODEL_PATH, } from './model'; +export function buildGoertzelCoefficients(freqBins: number, fftSize: number): Float64Array { + const coeffs = new Float64Array(freqBins); + for (let k = 0; k < freqBins; k += 1) { + coeffs[k] = 2 * Math.cos((2 * Math.PI * k) / fftSize); + } + return coeffs; +} + +export function goertzelPower(samples: Float32Array, coeff: number): number { + let s1 = 0; + let s2 = 0; + for (let i = 0; i < samples.length; i += 1) { + const s0 = samples[i] + (coeff * s1) - s2; + s2 = s1; + s1 = s0; + } + const power = (s1 * s1) + (s2 * s2) - (coeff * s1 * s2); + return !Number.isFinite(power) || power < 0 ? 0 : power; +} + interface WhisperAlignmentOptions { lang?: string; textHint?: string; diff --git a/compute-worker/src/inference/whisper/alignment-map.ts b/compute-worker/src/inference/whisper/alignment-map.ts deleted file mode 100644 index 098374c..0000000 --- a/compute-worker/src/inference/whisper/alignment-map.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts'; - -export interface WhisperWord { - start: number; - end: number; - word: string; -} - -export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment { - const lowerSentence = sentence.toLowerCase(); - let cursor = 0; - - const alignedWords: TTSSentenceWord[] = words.map((w) => { - const token = w.word.trim(); - if (!token) { - return { - text: '', - startSec: w.start, - endSec: w.end, - charStart: cursor, - charEnd: cursor, - }; - } - - const idx = lowerSentence.indexOf(token.toLowerCase(), cursor); - const start = idx >= 0 ? idx : cursor; - const end = Math.min(sentence.length, start + token.length); - cursor = Math.max(cursor, end); - - return { - text: token, - startSec: w.start, - endSec: w.end, - charStart: start, - charEnd: end, - }; - }).filter((word) => word.text.length > 0); - - return { - sentence, - sentenceIndex: 0, - words: alignedWords, - }; -} diff --git a/compute-worker/src/inference/whisper/model.ts b/compute-worker/src/inference/whisper/model.ts index 4c5a021..9cf0c4f 100644 --- a/compute-worker/src/inference/whisper/model.ts +++ b/compute-worker/src/inference/whisper/model.ts @@ -3,7 +3,7 @@ import { fileURLToPath } from 'url'; import { createHash } from 'crypto'; import { readFileSync } from 'fs'; import { access, copyFile, mkdir, readFile, rename, unlink, writeFile } from 'fs/promises'; -import { DOCSTORE_DIR } from '../platform/docstore'; +import { DOCSTORE_DIR } from '../../infrastructure/platform'; const MODULE_DIR = path.dirname(fileURLToPath(import.meta.url)); const MODEL_DIR = path.join(DOCSTORE_DIR, 'model', 'whisper-base_timestamped'); diff --git a/compute-worker/src/inference/whisper/spectral.ts b/compute-worker/src/inference/whisper/spectral.ts deleted file mode 100644 index b7223cc..0000000 --- a/compute-worker/src/inference/whisper/spectral.ts +++ /dev/null @@ -1,21 +0,0 @@ -export function buildGoertzelCoefficients(freqBins: number, fftSize: number): Float64Array { - const coeffs = new Float64Array(freqBins); - for (let k = 0; k < freqBins; k += 1) { - coeffs[k] = 2 * Math.cos((2 * Math.PI * k) / fftSize); - } - return coeffs; -} - -export function goertzelPower(samples: Float32Array, coeff: number): number { - let s1 = 0; - let s2 = 0; - for (let i = 0; i < samples.length; i += 1) { - const s0 = samples[i] + (coeff * s1) - s2; - s2 = s1; - s1 = s0; - } - - const power = (s1 * s1) + (s2 * s2) - (coeff * s1 * s2); - if (!Number.isFinite(power) || power < 0) return 0; - return power; -} diff --git a/compute-worker/src/inference/whisper/token-timestamps.ts b/compute-worker/src/inference/whisper/timestamps.ts similarity index 93% rename from compute-worker/src/inference/whisper/token-timestamps.ts rename to compute-worker/src/inference/whisper/timestamps.ts index 47edc1d..e6e2b96 100644 --- a/compute-worker/src/inference/whisper/token-timestamps.ts +++ b/compute-worker/src/inference/whisper/timestamps.ts @@ -1,5 +1,6 @@ import type { Tokenizer } from '@huggingface/tokenizers'; import type * as ort from 'onnxruntime-node'; +import type { TTSSentenceAlignment, TTSSentenceWord } from '../../api/types'; const PUNCTUATION_REGEX = '\\p{P}\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E'; const PUNCTUATION_ONLY_REGEX = new RegExp(`^[${PUNCTUATION_REGEX}]+$`, 'gu'); @@ -12,6 +13,29 @@ export interface WhisperWordTiming { endSec: number; } +export interface WhisperWord { + start: number; + end: number; + word: string; +} + +export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment { + const lowerSentence = sentence.toLowerCase(); + let cursor = 0; + const alignedWords: TTSSentenceWord[] = words.map((word) => { + const token = word.word.trim(); + if (!token) { + return { text: '', startSec: word.start, endSec: word.end, charStart: cursor, charEnd: cursor }; + } + const index = lowerSentence.indexOf(token.toLowerCase(), cursor); + const start = index >= 0 ? index : cursor; + const end = Math.min(sentence.length, start + token.length); + cursor = Math.max(cursor, end); + return { text: token, startSec: word.start, endSec: word.end, charStart: start, charEnd: end }; + }).filter((word) => word.text.length > 0); + return { sentence, sentenceIndex: 0, words: alignedWords }; +} + function medianFilter(data: Float32Array, windowSize: number): Float32Array { if (windowSize % 2 === 0 || windowSize <= 0) { throw new Error('Window size must be a positive odd number'); diff --git a/compute-worker/src/inference/config/timeout.ts b/compute-worker/src/infrastructure/config.ts similarity index 84% rename from compute-worker/src/inference/config/timeout.ts rename to compute-worker/src/infrastructure/config.ts index 40fd406..2547f3b 100644 --- a/compute-worker/src/inference/config/timeout.ts +++ b/compute-worker/src/infrastructure/config.ts @@ -117,3 +117,23 @@ export async function withIdleTimeoutAndHardCap(input: IdleTimeoutAndHardCapI throw error; } } + +export function getComputeJobConcurrency(): number { + return readPositiveIntEnv('COMPUTE_JOB_CONCURRENCY', 1); +} + +export function getAvailableCpuCores(): number { + if (typeof os.availableParallelism === 'function') { + const value = os.availableParallelism(); + if (Number.isFinite(value) && value >= 1) return Math.floor(value); + } + const fallback = os.cpus().length; + return Number.isFinite(fallback) && fallback >= 1 ? Math.floor(fallback) : 1; +} + +export function getOnnxThreadsPerJob(): number { + const concurrency = getComputeJobConcurrency(); + const usableCores = Math.max(1, getAvailableCpuCores() - 1); + return Math.max(1, Math.floor(usableCores / concurrency)); +} +import os from 'node:os'; diff --git a/compute-worker/src/inference/platform/ffmpeg.ts b/compute-worker/src/infrastructure/platform.ts similarity index 60% rename from compute-worker/src/inference/platform/ffmpeg.ts rename to compute-worker/src/infrastructure/platform.ts index b4ed669..9f029e4 100644 --- a/compute-worker/src/inference/platform/ffmpeg.ts +++ b/compute-worker/src/infrastructure/platform.ts @@ -1,6 +1,26 @@ -import { existsSync } from 'fs'; +import fs from 'fs'; +import path from 'path'; import ffmpegStatic from 'ffmpeg-static'; +function findMonorepoRoot(startDir: string): string | null { + let current = path.resolve(startDir); + for (;;) { + const marker = path.join(current, 'pnpm-workspace.yaml'); + if (fs.existsSync(marker)) return current; + const parent = path.dirname(current); + if (parent === current) return null; + current = parent; + } +} + +function resolveDocstoreDir(): string { + const repoRoot = findMonorepoRoot(process.cwd()); + if (repoRoot) return path.join(repoRoot, 'docstore'); + return path.join(process.cwd(), 'docstore'); +} + +export const DOCSTORE_DIR = resolveDocstoreDir(); + function normalizePath(value: unknown): string | null { if (typeof value !== 'string') return null; const trimmed = value.trim(); @@ -9,20 +29,17 @@ function normalizePath(value: unknown): string | null { function resolveBinary(envValue: string | null, bundledValue: string | null, envVarName: string, packageName: string): string { if (envValue) { - if ((envValue.includes('/') || envValue.includes('\\')) && !existsSync(envValue)) { + if ((envValue.includes('/') || envValue.includes('\\')) && !fs.existsSync(envValue)) { throw new Error(`${envVarName} points to a missing binary: ${envValue}`); } return envValue; } - if (!bundledValue) { throw new Error(`${packageName} binary is unavailable on this platform. Set ${envVarName} to an installed binary path.`); } - - if ((bundledValue.includes('/') || bundledValue.includes('\\')) && !existsSync(bundledValue)) { + if ((bundledValue.includes('/') || bundledValue.includes('\\')) && !fs.existsSync(bundledValue)) { throw new Error(`${packageName} resolved to a missing binary path: ${bundledValue}`); } - return bundledValue; } diff --git a/compute-worker/src/jobs/handlers.ts b/compute-worker/src/jobs/handlers.ts index 90a4068..a013602 100644 --- a/compute-worker/src/jobs/handlers.ts +++ b/compute-worker/src/jobs/handlers.ts @@ -2,8 +2,8 @@ import { z } from 'zod'; import { runPdfLayoutFromPdfBuffer, runWhisperAlignmentFromAudioBuffer, -} from '../inference/local-runtime'; -import { withIdleTimeoutAndHardCap, withTimeout } from '../inference'; +} from '../inference/runtime'; +import { withIdleTimeoutAndHardCap, withTimeout } from '../infrastructure/config'; import type { PdfLayoutJobRequest, PdfLayoutJobResult, diff --git a/compute-worker/src/storage/artifact-addressing.ts b/compute-worker/src/storage/artifact-addressing.ts index e341f23..bf4e312 100644 --- a/compute-worker/src/storage/artifact-addressing.ts +++ b/compute-worker/src/storage/artifact-addressing.ts @@ -1,5 +1,5 @@ -import { PDF_PARSER_VERSION } from '../inference/pdf/parser-version'; -import { encodeParserVersion } from '../inference/pdf/parser-version-key'; +import { PDF_PARSER_VERSION } from '../api/contracts'; +import { encodeParserVersion } from '../api/contracts'; const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; diff --git a/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts b/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts index d035613..cb18eee 100644 --- a/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts +++ b/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { mergeTextWithRegions } from '../../../src/inference'; +import { mergeTextWithRegions } from '../../../src/inference/pdf/document-layout'; describe('mergeTextWithRegions', () => { test('assigns text items to containing regions by centroid and joins in reading order', () => { diff --git a/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts b/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts index 167dc46..f93cf25 100644 --- a/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts +++ b/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { normalizeTextItemsForLayout } from '../../../src/inference'; +import { normalizeTextItemsForLayout } from '../../../src/inference/pdf/normalize-text'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; function makeTextItem( diff --git a/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts b/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts index 97d4b19..3fd35a6 100644 --- a/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts +++ b/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { stitchCrossPageBlocks } from '../../../src/inference'; -import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/inference/types'; +import { stitchCrossPageBlocks } from '../../../src/inference/pdf/stitch'; +import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/api/types'; import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures'; function makeBlock( diff --git a/compute-worker/tests/compute/algorithms/support/document-fixtures.ts b/compute-worker/tests/compute/algorithms/support/document-fixtures.ts index f8db418..5a3d697 100644 --- a/compute-worker/tests/compute/algorithms/support/document-fixtures.ts +++ b/compute-worker/tests/compute/algorithms/support/document-fixtures.ts @@ -1,5 +1,5 @@ import type { BaseDocument } from '../../../../../src/types/documents'; -import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/inference/types'; +import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/api/types'; export function makeBaseDocument(overrides: Partial = {}): BaseDocument { return { diff --git a/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts b/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts index fedea8b..201c9c6 100644 --- a/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts +++ b/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; import { mapWordsToSentenceOffsets, -} from '../../../src/inference'; +} from '../../../src/inference/whisper/timestamps'; describe('whisper alignment mapping', () => { test('maps words to sentence offsets with punctuation and repeated spaces', () => { diff --git a/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts b/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts index 6b51d62..1794290 100644 --- a/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts +++ b/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { buildGoertzelCoefficients, goertzelPower } from '../../../src/inference'; +import { buildGoertzelCoefficients, goertzelPower } from '../../../src/inference/whisper/align'; function dftPower(samples: Float32Array, k: number): number { const n = samples.length; diff --git a/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts b/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts index 6f8bced..e1cdc18 100644 --- a/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts +++ b/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts @@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-node'; import { buildWordsFromTimestampedTokens, extractTokenStartTimestamps, -} from '../../../src/inference'; +} from '../../../src/inference/whisper/timestamps'; describe('whisper token timestamp alignment', () => { test('extracts monotonic token timestamps from cross-attention maps', () => { diff --git a/compute-worker/tests/compute/control-plane/run-layout-model.test.ts b/compute-worker/tests/compute/control-plane/run-layout-model.test.ts index c8278c9..cc6a34d 100644 --- a/compute-worker/tests/compute/control-plane/run-layout-model.test.ts +++ b/compute-worker/tests/compute/control-plane/run-layout-model.test.ts @@ -76,7 +76,7 @@ vi.mock('../../../src/inference/pdf/model', () => ({ MODEL_PREPROCESSOR_PATH: '/tmp/model-preprocessor.json', })); -vi.mock('../../../src/inference/config/cpu-budget', () => ({ +vi.mock('../../../src/infrastructure/config', () => ({ getOnnxThreadsPerJob: vi.fn(() => 1), })); @@ -107,7 +107,7 @@ describe('runLayoutModel', () => { }, }; - const { runLayoutModel } = await import('../../../src/inference/pdf/runLayoutModel'); + const { runLayoutModel } = await import('../../../src/inference/pdf/layout-model'); const regions = await runLayoutModel({ pageWidth: 100, pageHeight: 100, @@ -154,7 +154,7 @@ describe('runLayoutModel', () => { }, }; - const { runLayoutModel } = await import('../../../src/inference/pdf/runLayoutModel'); + const { runLayoutModel } = await import('../../../src/inference/pdf/layout-model'); const regions = await runLayoutModel({ pageWidth: 100, pageHeight: 100, From 5403aa00cc592751d69fce0351adf4e04d5fbbf8 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 13 Jun 2026 05:04:24 -0600 Subject: [PATCH 04/11] refactor(compute-worker): modularize operation contracts and API, update protocol naming - Move operation contracts from api/contracts to operations/contracts for clearer separation - Delete legacy API modules: contracts.ts, operation-keys.ts, public-operation.ts - Add new modular files: compute-operation.ts, http-hooks.ts, routes.ts, operations/keys.ts, operations/reconciliation.ts - Refactor all imports to use new operations/contracts and keys modules - Rename protocol types from PublicOperation to ComputeOperation for clarity - Update OpenAPI schema and all references to use ComputeOperation/Event naming - Extract whisper decoder helpers to inference/whisper/decoder.ts - Add new infra: nats-session.ts for NATS session management - Update tests and storage logic to use new contracts and keys modules - Refactor worker loop and orchestrator to use modular job definitions - Update client and protocol in src/lib/server/compute-worker to use new ComputeOperation types This refactor modularizes compute-worker operation logic, clarifies type naming, and improves maintainability by separating contracts, keys, and reconciliation logic. No breaking changes to external API, but all internal references and tests are updated for the new structure. --- compute-worker/openapi.json | 8 +- compute-worker/src/api/app.ts | 865 ++---------------- ...blic-operation.ts => compute-operation.ts} | 18 +- compute-worker/src/api/http-hooks.ts | 80 ++ compute-worker/src/api/routes.ts | 356 +++++++ compute-worker/src/api/schemas.ts | 8 +- compute-worker/src/inference/pdf/parse.ts | 2 +- compute-worker/src/inference/whisper/align.ts | 117 +-- .../src/inference/whisper/decoder.ts | 111 +++ compute-worker/src/infrastructure/config.ts | 36 +- .../src/infrastructure/nats-adapters.ts | 2 +- .../src/infrastructure/nats-session.ts | 204 +++++ compute-worker/src/jobs/handlers.ts | 2 +- compute-worker/src/jobs/pdf-progress.ts | 2 +- compute-worker/src/jobs/worker-loop-policy.ts | 2 +- compute-worker/src/jobs/worker-loop.ts | 39 +- .../src/{api => operations}/contracts.ts | 6 +- .../operation-keys.ts => operations/keys.ts} | 2 +- .../src/operations/reconciliation.ts | 95 ++ compute-worker/src/operations/recovery.ts | 2 +- compute-worker/src/operations/service.ts | 2 +- .../src/operations/state-machine.ts | 2 +- compute-worker/src/operations/types.ts | 2 +- .../src/storage/artifact-addressing.ts | 4 +- .../algorithms/whisper-decoder.test.ts | 61 ++ .../control-plane/orchestrator.test.ts | 2 +- .../control-plane/state-machine.test.ts | 2 +- .../helpers/in-memory-control-plane.ts | 2 +- .../tests/fixtures/fake-control-plane.ts | 2 +- .../tests/unit/jetstream-adapters.test.ts | 2 +- .../tests/unit/key-and-progress.test.ts | 2 +- compute-worker/tests/unit/worker-loop.test.ts | 170 ++++ src/lib/server/compute-worker/client.ts | 20 +- src/lib/server/compute-worker/generated.ts | 6 +- src/lib/server/compute-worker/protocol.ts | 22 +- src/lib/server/pdf-parse/operation.ts | 10 +- src/lib/server/pdf-parse/snapshot.ts | 8 +- 37 files changed, 1313 insertions(+), 963 deletions(-) rename compute-worker/src/api/{public-operation.ts => compute-operation.ts} (72%) create mode 100644 compute-worker/src/api/http-hooks.ts create mode 100644 compute-worker/src/api/routes.ts create mode 100644 compute-worker/src/inference/whisper/decoder.ts create mode 100644 compute-worker/src/infrastructure/nats-session.ts rename compute-worker/src/{api => operations}/contracts.ts (95%) rename compute-worker/src/{api/operation-keys.ts => operations/keys.ts} (96%) create mode 100644 compute-worker/src/operations/reconciliation.ts create mode 100644 compute-worker/tests/compute/algorithms/whisper-decoder.test.ts create mode 100644 compute-worker/tests/unit/worker-loop.test.ts diff --git a/compute-worker/openapi.json b/compute-worker/openapi.json index c0973f1..5b1f63a 100644 --- a/compute-worker/openapi.json +++ b/compute-worker/openapi.json @@ -272,7 +272,7 @@ ], "additionalProperties": false }, - "PublicOperation": { + "ComputeOperation": { "type": "object", "properties": { "opId": { @@ -413,7 +413,7 @@ ], "additionalProperties": false }, - "PublicOperationEvent": { + "ComputeOperationEvent": { "type": "object", "properties": { "eventId": { @@ -1706,12 +1706,12 @@ ], "responses": { "200": { - "description": "Server-sent PublicOperationEvent stream", + "description": "Server-sent ComputeOperationEvent stream", "content": { "application/json": { "schema": { "type": "string", - "description": "Server-sent PublicOperationEvent stream" + "description": "Server-sent ComputeOperationEvent stream" } } } diff --git a/compute-worker/src/api/app.ts b/compute-worker/src/api/app.ts index 77b5a55..7ea327c 100644 --- a/compute-worker/src/api/app.ts +++ b/compute-worker/src/api/app.ts @@ -1,18 +1,9 @@ -import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify'; +import Fastify, { type FastifyInstance } from 'fastify'; import swagger from '@fastify/swagger'; import { - connect, credsAuthenticator, - type NatsConnection, + type ConnectionOptions, } from '@nats-io/transport-node'; -import { - jetstream, - jetstreamManager, - type Consumer, - type JetStreamClient, - type JetStreamManager, -} from '@nats-io/jetstream'; -import { Kvm } from '@nats-io/kv'; import { ensureComputeModels, } from '../inference/runtime'; @@ -21,30 +12,26 @@ import { getComputeOpStaleMs, getAvailableCpuCores, getOnnxThreadsPerJob, + buildLoggerConfig, + normalizeNatsReplicas, + readBoolEnv, + readPositiveIntEnv, + requireEnv, } from '../infrastructure/config'; -import { encodeSseFrame, OperationOrchestrator } from '../operations'; +import { OperationOrchestrator } from '../operations'; import type { PdfLayoutJobRequest, PdfLayoutJobResult, - WorkerOperationEvent, WhisperAlignJobRequest, WhisperAlignJobResult, - WorkerJobTiming, - WorkerOperationRequest, - WorkerOperationState, -} from '../api/contracts'; +} from '../operations/contracts'; import { JetStreamOperationEventStream, JetStreamOperationQueue, JetStreamOperationStateStore, - hashOpKey, } from '../infrastructure/nats-adapters'; import { createJsonCodec } from '../infrastructure/json-codec'; -import { - recoverOrphanedOperations, - type StreamedOperationState, -} from '../operations/recovery'; -import { parsedPdfArtifactKey } from '../storage/artifact-addressing'; +import { createOperationReconciler } from '../operations/reconciliation'; import { createArtifactStorage, createS3ClientFromEnv, @@ -53,44 +40,31 @@ import { } from '../infrastructure/storage'; import { createJobHandlers } from '../jobs/handlers'; import { createWorkerLoopController, type QueuedJob } from '../jobs/worker-loop'; +import { createNatsSessionManager } from '../infrastructure/nats-session'; import { - COMPUTE_STATE_BUCKET, - COMPUTE_STATE_TTL_MS, EVENTS_STREAM_NAME, - JOBS_STREAM_NAME, - LAYOUT_CONSUMER_NAME, LAYOUT_JOBS_SUBJECT, NATS_API_TIMEOUT_MS, - WHISPER_CONSUMER_NAME, WHISPER_JOBS_SUBJECT, - ensureJetStreamResources, } from '../infrastructure/nats'; -import { buildPdfOperationKey, buildWhisperOperationKey } from './operation-keys'; -import { toPublicOperation, type PublicOperationEvent } from './public-operation'; +import { registerHttpHooks } from './http-hooks'; +import { + registerComputeWorkerRoutes, + type ComputeWorkerRouteDeps, +} from './routes'; import { apiErrorResponseSchema, artifactReferenceSchema, jsonSchema, - operationEventsQuerySchema, - operationParamsSchema, operationErrorSchema, parsedPdfDocumentSchema, pdfLayoutProgressSchema, pdfLayoutResolutionSchema, - pdfOperationCreateSchema, - pdfResolveSchema, - publicOperationEventSchema, - publicOperationSchema, + computeOperationEventSchema, + computeOperationSchema, ttsSentenceAlignmentSchema, - whisperOperationCreateSchema, } from './schemas'; -const OP_EVENTS_KEEPALIVE_MS = 15_000; -// Reconnection delay handed to the browser EventSource via the SSE `retry:` -// directive. When a silent stream is torn down for idle sleep, this keeps the -// client from immediately reconnecting and re-waking the worker; instead it -// reconnects on a slow cadence so the container stays asleep most of the time. -const OP_EVENTS_RECONNECT_HINT_MS = 120_000; // Disconnect from NATS after this much continuous idle so the worker stops // generating outbound traffic (pull polling + keepalive PINGs) and Railway can // put it to sleep. Reconnect happens lazily on the next inbound request. @@ -100,76 +74,9 @@ const IDLE_STATUS_LOG_INTERVAL_MS = 60_000; const ORPHAN_SWEEP_INTERVAL_MS = 15_000; // Bounded pull window so consumer loops yield periodically and can be stopped // cleanly when going idle, instead of blocking on a long-lived pull. -const REQUEST_STARTED_AT_MS_KEY = Symbol('request-started-at-ms'); -const REQUEST_COUNTED_KEY = Symbol('request-activity-counted'); const WHISPER_MAX_DELIVER = 1; -interface NatsSession { - nc: NatsConnection; - js: JetStreamClient; - jsm: JetStreamManager; - kv: Awaited>; - whisperConsumer: Consumer; - layoutConsumer: Consumer; -} - -interface OperationEventStreamLike { - subscribe(input: { - opId: string; - sinceEventId?: number; - onEvent: (event: WorkerOperationEvent) => void | Promise; - onError?: (error: unknown) => void; - }): Promise<() => void>; -} - -interface OperationStateStoreLike { - getOpState(opId: string): Promise; - getOpStateRecord?(opId: string): Promise<{ state: StreamedOperationState; revision: number } | null>; - getOpIndex?(opKey: string): Promise<{ opId: string } | null>; - listOpStates?(): Promise; -} - -interface OrchestratorLike { - enqueueOrReuse(request: WorkerOperationRequest): Promise; - markRunning(input: { - opId: string; - startedAt?: number; - updatedAt?: number; - timing?: WorkerJobTiming; - }): Promise; - markProgress(input: { - opId: string; - progress: WorkerOperationState['progress']; - updatedAt?: number; - timing?: WorkerJobTiming; - }): Promise; - markSucceeded(input: { - opId: string; - result: unknown; - updatedAt?: number; - timing?: WorkerJobTiming; - }): Promise; - markFailed(input: { - opId: string; - error: { message: string; code?: string } | string; - updatedAt?: number; - timing?: WorkerJobTiming; - }): Promise; - markFailedIfUnchanged?(input: { - current: StreamedOperationState; - expectedRevision: number; - error: { message: string; code?: string } | string; - updatedAt?: number; - timing?: WorkerJobTiming; - }): Promise; -} - -export interface ComputeWorkerRouteDeps { - orchestrator: OrchestratorLike; - operationStateStore: OperationStateStoreLike; - operationEventStream: OperationEventStreamLike; - artifactExists?: (key: string) => Promise; -} +export type { ComputeWorkerRouteDeps } from './routes'; export interface CreateComputeWorkerAppOptions { host?: string; @@ -187,122 +94,27 @@ export interface ComputeWorkerApp { close(): Promise; } -function requireEnv(name: string): string { - const value = process.env[name]?.trim(); - if (!value) throw new Error(`${name} is required`); - return value; -} - -function readIntEnv(name: string, fallback: number): number { - const raw = process.env[name]?.trim(); - if (!raw) return fallback; - const parsed = Number(raw); - if (!Number.isFinite(parsed) || parsed <= 0) return fallback; - return Math.floor(parsed); -} - -function normalizeNatsReplicas(value: number): number { - if (value === 3 || value === 5) return value; - return 1; -} - -function parseBoolEnv(name: string, fallback: boolean): boolean { - const raw = process.env[name]?.trim(); - if (!raw) return fallback; - const normalized = raw.toLowerCase(); - return normalized === '1' || normalized === 'true' || normalized === 'yes' || normalized === 'on'; -} - -function buildLoggerConfig(): boolean | Record { - const format = (process.env.LOG_FORMAT?.trim().toLowerCase() || 'pretty'); - const level = process.env.COMPUTE_LOG_LEVEL?.trim() || 'info'; - if (format === 'json') { - return { - level, - base: null, - }; - } - return { - level, - base: null, - transport: { - target: 'pino-pretty', - options: { - colorize: true, - translateTime: 'SYS:standard', - ignore: 'pid,hostname', - }, - }, - }; -} - -function isAuthed(request: FastifyRequest, expectedToken: string): boolean { - const auth = request.headers.authorization; - if (!auth?.startsWith('Bearer ')) return false; - const token = auth.slice('Bearer '.length).trim(); - return token === expectedToken; -} - -function toErrorMessage(error: unknown): string { - if (error instanceof Error && error.message) return error.message; - return String(error); -} - -function requestPath(request: FastifyRequest): string { - return request.url.split('?')[0] ?? request.url; -} - -function isHealthPath(path: string): boolean { - return path === '/health/live' || path === '/health/ready'; -} - -function extractTraceId(request: FastifyRequest): string | null { - const header = request.headers['x-openreader-trace-id']; - if (Array.isArray(header)) return header[0] ?? null; - return typeof header === 'string' ? header : null; -} - -function extractOpId(request: FastifyRequest, path: string): string | null { - const params = request.params as { opId?: unknown } | undefined; - if (params && typeof params.opId === 'string' && params.opId.trim()) { - return params.opId.trim(); - } - const match = path.match(/^\/v1\/operations\/([^/]+)/); - if (!match?.[1]) return null; - try { - return decodeURIComponent(match[1]); - } catch { - return match[1]; - } -} - -function isTerminalStatus(status: import('../api/contracts').WorkerJobState): boolean { - return status === 'succeeded' || status === 'failed'; -} - -const errorResponseSchema = jsonSchema(apiErrorResponseSchema); - export async function createComputeWorkerApp(options: CreateComputeWorkerAppOptions = {}): Promise { - const port = options.port ?? readIntEnv('PORT', 8081); + const port = options.port ?? readPositiveIntEnv('PORT', 8081); const host = options.host ?? (process.env.COMPUTE_WORKER_HOST?.trim() || '0.0.0.0'); const workerToken = options.workerToken ?? requireEnv('COMPUTE_WORKER_TOKEN'); const disableWorkers = options.disableWorkers ?? false; const natsUrl = requireEnv('NATS_URL'); const timeoutConfig = getComputeTimeoutConfig(); - const jobConcurrency = readIntEnv('COMPUTE_JOB_CONCURRENCY', 1); + const jobConcurrency = readPositiveIntEnv('COMPUTE_JOB_CONCURRENCY', 1); const whisperTimeoutMs = timeoutConfig.whisperTimeoutMs; const pdfTimeoutMs = timeoutConfig.pdfTimeoutMs; const pdfHardCapMs = timeoutConfig.pdfHardCapMs; - const pdfAttempts = readIntEnv('COMPUTE_PDF_JOB_ATTEMPTS', 1); - const prewarmModels = parseBoolEnv('COMPUTE_PREWARM_MODELS', false); - const jobsStreamMaxBytes = readIntEnv('COMPUTE_JOBS_STREAM_MAX_BYTES', 256 * 1024 * 1024); - const eventsStreamMaxBytes = readIntEnv('COMPUTE_EVENTS_STREAM_MAX_BYTES', 128 * 1024 * 1024); - const jobStatesMaxBytes = readIntEnv('COMPUTE_JOB_STATES_MAX_BYTES', 64 * 1024 * 1024); - const natsReplicas = normalizeNatsReplicas(readIntEnv('COMPUTE_NATS_REPLICAS', 1)); + const pdfAttempts = readPositiveIntEnv('COMPUTE_PDF_JOB_ATTEMPTS', 1); + const prewarmModels = readBoolEnv('COMPUTE_PREWARM_MODELS', false); + const jobsStreamMaxBytes = readPositiveIntEnv('COMPUTE_JOBS_STREAM_MAX_BYTES', 256 * 1024 * 1024); + const eventsStreamMaxBytes = readPositiveIntEnv('COMPUTE_EVENTS_STREAM_MAX_BYTES', 128 * 1024 * 1024); + const jobStatesMaxBytes = readPositiveIntEnv('COMPUTE_JOB_STATES_MAX_BYTES', 64 * 1024 * 1024); + const natsReplicas = normalizeNatsReplicas(readPositiveIntEnv('COMPUTE_NATS_REPLICAS', 1)); const opStaleMs = getComputeOpStaleMs(); - const connectOpts: Parameters[0] = { servers: natsUrl }; + const connectOpts: ConnectionOptions = { servers: natsUrl }; const natsCreds = process.env.NATS_CREDS?.trim(); const natsCredsFile = process.env.NATS_CREDS_FILE?.trim(); @@ -316,160 +128,18 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti connectOpts.authenticator = credsAuthenticator(credsData); } - // Lazy NATS connection lifecycle. The worker connects on demand (first request - // that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle - // so it stops emitting outbound traffic and Railway can sleep it. Reconnect is - // transparent: any inbound operation request both wakes the container and re-establishes - // the session via ensureConnected(). - let session: NatsSession | null = null; - let connecting: Promise | null = null; - let idleTimer: NodeJS.Timeout | null = null; - let orphanSweepTimer: NodeJS.Timeout | null = null; let stopping = false; - - // Activity accounting feeding the idle detector. The worker is considered idle - // only when no HTTP request is in flight, no SSE stream is open, no job is - // processing, and nothing has happened for IDLE_DISCONNECT_MS. let inFlightHttp = 0; let activeSse = 0; let inFlightJobs = 0; let lastActivityAt = Date.now(); - let lastActivityReason = 'startup'; - let lastIdleStatusLogAt = 0; - + let lastActivityReason = "startup"; const markActivity = (reason: string): void => { lastActivityAt = Date.now(); lastActivityReason = reason; }; - - function startIdleTimer(): void { - if (idleTimer) return; - idleTimer = setInterval(() => { - if (!session || stopping) return; - const now = Date.now(); - const idleForMs = now - lastActivityAt; - if (now - lastIdleStatusLogAt >= IDLE_STATUS_LOG_INTERVAL_MS) { - lastIdleStatusLogAt = now; - app.log.info({ - activeSse, - idleForMs, - inFlightHttp, - inFlightJobs, - lastActivityReason, - disconnectEligible: inFlightHttp === 0 - && inFlightJobs === 0 - && idleForMs >= IDLE_DISCONNECT_MS, - }, 'nats idle status'); - } - // Hard work in flight always blocks idle. An open SSE no longer blocks just - // by existing — only by delivering events, which refresh lastActivityAt via - // markActivity() in the stream's onEvent. So a silent/stuck-op stream lets - // the idle window elapse and is torn down by disconnect() below. - if (inFlightHttp > 0 || inFlightJobs > 0) return; - if (idleForMs < IDLE_DISCONNECT_MS) return; - void disconnect('idle'); - }, IDLE_CHECK_INTERVAL_MS); - // Don't let the idle checker keep the process alive on its own. - idleTimer.unref?.(); - } - - function startOrphanSweepTimer(): void { - if (orphanSweepTimer) return; - orphanSweepTimer = setInterval(() => { - if (!session || stopping) return; - void runOrphanedOpRecovery({ force: true }).catch((error) => { - app.log.error({ - error: toErrorMessage(error), - }, 'periodic orphaned operation recovery failed'); - }); - }, ORPHAN_SWEEP_INTERVAL_MS); - orphanSweepTimer.unref?.(); - } - - async function disconnect(reason: string): Promise { - const current = session; - if (!current) return; - // Snapshot what's still attached so a dropped connection is visible in logs - // (e.g. SSE streams torn down because we went idle while they were open). - app.log.info({ - reason, - activeSse, - inFlightHttp, - inFlightJobs, - idleForMs: Date.now() - lastActivityAt, - }, 'nats dropping connection'); - // Clear synchronously (before any await) so concurrent requests reconnect a - // fresh session instead of using the connection we're about to close. - session = null; - if (idleTimer) { - clearInterval(idleTimer); - idleTimer = null; - } - if (orphanSweepTimer) { - clearInterval(orphanSweepTimer); - orphanSweepTimer = null; - } - try { - await current.nc.close(); - } catch { - // ignore close errors - } - await workerLoops.stop(); - app.log.info({ reason }, 'nats disconnected'); - } - - async function ensureConnected(): Promise { - if (session) return session; - if (connecting) return connecting; - connecting = (async () => { - const nc: NatsConnection = await connect(connectOpts); - const js: JetStreamClient = jetstream(nc, { timeout: NATS_API_TIMEOUT_MS }); - const jsm: JetStreamManager = await jetstreamManager(nc, { timeout: NATS_API_TIMEOUT_MS }); - await ensureJetStreamResources({ - jsm, - whisperTimeoutMs, - pdfTimeoutMs, - pdfAttempts, - jobsMaxBytes: jobsStreamMaxBytes, - eventsMaxBytes: eventsStreamMaxBytes, - natsReplicas, - }); - const kv = await new Kvm(js).create(COMPUTE_STATE_BUCKET, { - replicas: natsReplicas, - history: 1, - ttl: COMPUTE_STATE_TTL_MS, - max_bytes: jobStatesMaxBytes, - }); - const whisperConsumer = await js.consumers.get(JOBS_STREAM_NAME, WHISPER_CONSUMER_NAME); - const layoutConsumer = await js.consumers.get(JOBS_STREAM_NAME, LAYOUT_CONSUMER_NAME); - const next: NatsSession = { nc, js, jsm, kv, whisperConsumer, layoutConsumer }; - session = next; - sessionGeneration += 1; - orphanRecoveryDoneForGeneration = -1; - markActivity('nats_connected'); - if (!disableWorkers) { - workerLoops.start(next, { - whisper: next.whisperConsumer, - pdfLayout: next.layoutConsumer, - }); - } - startIdleTimer(); - startOrphanSweepTimer(); - // Safety net: if the connection closes for any reason (network drop after - // exhausting reconnects, or our own disconnect), drop the stale session so - // the next request reconnects cleanly. - void nc.closed().then(() => { - if (session?.nc === nc) session = null; - }); - app.log.info('nats connected'); - return next; - })(); - try { - return await connecting; - } finally { - connecting = null; - } - } + let sessionManager!: ReturnType; + const ensureConnected = () => sessionManager.ensureConnected(); const s3Prefix = normalizeS3Prefix(process.env.S3_PREFIX); const storageDisabled = async (): Promise => { @@ -517,8 +187,8 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti ErrorResponse: jsonSchema(apiErrorResponseSchema), OperationError: jsonSchema(operationErrorSchema), PdfLayoutProgress: jsonSchema(pdfLayoutProgressSchema), - PublicOperation: jsonSchema(publicOperationSchema), - PublicOperationEvent: jsonSchema(publicOperationEventSchema), + ComputeOperation: jsonSchema(computeOperationSchema), + ComputeOperationEvent: jsonSchema(computeOperationEventSchema), PdfLayoutResolution: jsonSchema(pdfLayoutResolutionSchema), }, }, @@ -571,392 +241,44 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti const operationStateStore = options.routeDeps?.operationStateStore ?? defaultOperationStateStore; const operationEventStream = options.routeDeps?.operationEventStream ?? defaultOperationEventStream; const orchestrator = options.routeDeps?.orchestrator ?? defaultOrchestrator; - let orphanRecoveryPromise: Promise | null = null; - let orphanRecoveryDoneForGeneration = -1; - let sessionGeneration = options.routeDeps ? 0 : -1; + const reconciler = createOperationReconciler({ + stateStore: operationStateStore, + orchestrator, + whisperTimeoutMs, + pdfTimeoutMs, + opStaleMs, + getGeneration: () => options.routeDeps ? 0 : sessionManager.getGeneration(), + logger: app.log, + }); + const ensureOrphanedOpRecovery = () => reconciler.run(); + const getOpState = (opId: string) => reconciler.getOpState(opId); - const runOrphanedOpRecovery = async (options?: { force?: boolean }): Promise => { - if (typeof operationStateStore.listOpStates !== 'function') return; - if (typeof operationStateStore.getOpStateRecord !== 'function') return; - if (typeof orchestrator.markFailedIfUnchanged !== 'function') return; - if (!options?.force && orphanRecoveryDoneForGeneration === sessionGeneration) return; - if (orphanRecoveryPromise) { - await orphanRecoveryPromise; - return; - } - - orphanRecoveryPromise = (async () => { - const recoveredStates = await recoverOrphanedOperations({ - operationStateStore: operationStateStore as typeof operationStateStore & { - getOpStateRecord(opId: string): Promise<{ state: StreamedOperationState; revision: number } | null>; - listOpStates(): Promise; - }, - orchestrator: orchestrator as typeof orchestrator & { - markFailedIfUnchanged(input: { - current: StreamedOperationState; - expectedRevision: number; - error: { message: string; code?: string } | string; - updatedAt?: number; - timing?: WorkerJobTiming; - }): Promise; - }, - whisperTimeoutMs, - pdfTimeoutMs, - opStaleMs, - }); - - if (recoveredStates.length > 0) { - app.log.warn({ - recoveredCount: recoveredStates.length, - ops: recoveredStates.map((state) => ({ - opId: state.opId, - kind: state.kind, - status: state.status, - })), - }, 'recovered stale in-flight operations during reconciliation'); - } - - orphanRecoveryDoneForGeneration = sessionGeneration; - })().finally(() => { - orphanRecoveryPromise = null; - }); - - await orphanRecoveryPromise; - }; - - const ensureOrphanedOpRecovery = async (): Promise => { - await runOrphanedOpRecovery(); - }; - - const getOpState = async (opId: string): Promise => { - await ensureOrphanedOpRecovery(); - return await operationStateStore.getOpState(opId); - }; - - const releaseHttp = (request: FastifyRequest): void => { - const counted = request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean }; - if (!counted[REQUEST_COUNTED_KEY]) return; - counted[REQUEST_COUNTED_KEY] = false; - inFlightHttp = Math.max(0, inFlightHttp - 1); - markActivity('http_completed'); - }; - - app.addHook('onRequest', async (request, reply) => { - const path = requestPath(request); - (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY] = Date.now(); - // Count every request as in-flight activity so the idle detector never - // disconnects mid-request. Released in onResponse, or manually after hijack - // for SSE streams (where onResponse does not fire). - (request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean })[REQUEST_COUNTED_KEY] = true; - inFlightHttp += 1; - markActivity(`http_started:${path}`); - if (isHealthPath(path)) return; - if (!isAuthed(request, workerToken)) { - return reply.code(401).send({ error: 'Unauthorized' }); - } - return; + const { releaseHttp } = registerHttpHooks({ + app, + workerToken, + markActivity, + onInFlightHttpChanged: (delta) => { + inFlightHttp = Math.max(0, inFlightHttp + delta); + }, }); - app.addHook('onResponse', async (request, reply) => { - releaseHttp(request); - const path = requestPath(request); - if (isHealthPath(path)) return; - if (reply.statusCode >= 500) { - const startedAt = (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY]; - const durationMs = Number.isFinite(startedAt) ? Math.max(0, Date.now() - (startedAt as number)) : -1; - app.log.error({ - reqId: request.id, - method: request.method, - path, - statusCode: reply.statusCode, - durationMs, - traceId: extractTraceId(request) ?? null, - opId: extractOpId(request, path), - }, 'http.error'); - } - }); - - app.get('/health/live', { - schema: { - security: [], - response: { 200: { type: 'object', properties: { ok: { type: 'boolean' } }, required: ['ok'] } }, + registerComputeWorkerRoutes({ + app, + deps: { + orchestrator, + operationStateStore, + operationEventStream, + artifactExists: options.routeDeps?.artifactExists ?? storage.objectExists, }, - }, async () => ({ ok: true })); - - // Reports readiness without forcing a NATS round-trip. Probing NATS here would - // reconnect (and keep) the connection open, defeating idle sleep, so we only - // report the current connection state. The worker reconnects lazily on the next - // operation request regardless of what this returns. - app.get('/health/ready', { - schema: { - security: [], - response: { - 200: { - type: 'object', - properties: { ok: { type: 'boolean' }, natsConnected: { type: 'boolean' } }, - required: ['ok', 'natsConnected'], - }, - }, + s3Prefix, + ensureOrphanedOpRecovery, + getOpState, + getNatsConnected: () => sessionManager.isConnected(), + releaseHttp, + markActivity, + onActiveSseChanged: (delta) => { + activeSse = Math.max(0, activeSse + delta); }, - }, async () => ({ ok: true, natsConnected: session !== null })); - - app.post('/v1/whisper-align/operations', { - schema: { - body: jsonSchema(whisperOperationCreateSchema), - response: { 202: jsonSchema(publicOperationSchema), 400: errorResponseSchema }, - }, - }, async (request, reply) => { - const parsed = whisperOperationCreateSchema.safeParse(request.body); - if (!parsed.success) { - reply.code(400); - return { - error: 'Invalid request body', - issues: parsed.error.issues, - }; - } - - const requestOp: WorkerOperationRequest = { - kind: 'whisper_align', - opKey: buildWhisperOperationKey(parsed.data), - payload: parsed.data, - }; - await ensureOrphanedOpRecovery(); - const op = await orchestrator.enqueueOrReuse(requestOp); - app.log.info({ - kind: requestOp.kind, - opId: op.opId, - jobId: op.jobId, - status: op.status, - opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16), - }, 'op.accepted'); - reply.code(202); - return toPublicOperation(op); - }); - - app.post('/v1/pdf-layout/operations', { - schema: { - body: jsonSchema(pdfOperationCreateSchema), - response: { 202: jsonSchema(publicOperationSchema), 400: errorResponseSchema }, - }, - }, async (request, reply) => { - const parsed = pdfOperationCreateSchema.safeParse(request.body); - if (!parsed.success) { - reply.code(400); - return { - error: 'Invalid request body', - issues: parsed.error.issues, - }; - } - - const requestOp: WorkerOperationRequest = { - kind: 'pdf_layout', - opKey: buildPdfOperationKey(parsed.data), - payload: { - documentId: parsed.data.documentId, - namespace: parsed.data.namespace, - documentObjectKey: parsed.data.documentObjectKey, - }, - }; - await ensureOrphanedOpRecovery(); - const op = await orchestrator.enqueueOrReuse(requestOp); - reply.code(202); - return toPublicOperation(op); - }); - - app.post('/v1/pdf-layout/resolve', { - schema: { - body: jsonSchema(pdfResolveSchema), - response: { - 200: jsonSchema(pdfLayoutResolutionSchema), - 400: errorResponseSchema, - }, - }, - }, async (request, reply) => { - const parsed = pdfResolveSchema.safeParse(request.body); - if (!parsed.success) { - reply.code(400); - return { error: 'Invalid request body', issues: parsed.error.issues }; - } - await ensureOrphanedOpRecovery(); - const artifactKey = parsedPdfArtifactKey({ - documentId: parsed.data.documentId, - namespace: parsed.data.namespace, - prefix: s3Prefix, - }); - const hasArtifact = await (options.routeDeps?.artifactExists ?? storage.objectExists)(artifactKey); - const opKey = buildPdfOperationKey(parsed.data); - const index = await operationStateStore.getOpIndex?.(opKey); - const operation = index?.opId ? await operationStateStore.getOpState(index.opId) : null; - return { - artifact: hasArtifact ? { objectKey: artifactKey } : null, - operation: operation ? toPublicOperation(operation) : null, - }; - }); - - app.get('/v1/operations/:opId', { - schema: { - params: jsonSchema(operationParamsSchema), - response: { 200: jsonSchema(publicOperationSchema), 400: errorResponseSchema, 404: errorResponseSchema }, - }, - }, async (request, reply) => { - const params = operationParamsSchema.safeParse(request.params); - if (!params.success) { - reply.code(400); - return { error: 'Invalid op id' }; - } - - const state = await getOpState(params.data.opId); - if (!state) { - reply.code(404); - return { error: 'Operation not found' }; - } - - return toPublicOperation(state); - }); - - app.get('/v1/operations/:opId/events', { - schema: { - params: jsonSchema(operationParamsSchema), - querystring: jsonSchema(operationEventsQuerySchema), - response: { - 200: { type: 'string', description: 'Server-sent PublicOperationEvent stream' }, - 400: errorResponseSchema, - 404: errorResponseSchema, - }, - }, - }, async (request, reply) => { - const params = operationParamsSchema.safeParse(request.params); - if (!params.success) { - reply.code(400); - return { error: 'Invalid op id' }; - } - - const initial = await getOpState(params.data.opId); - if (!initial) { - reply.code(404); - return { error: 'Operation not found' }; - } - - const cursorQueryRaw = request.query as { sinceEventId?: string | number | null } | undefined; - const cursorFromQuery = Number(cursorQueryRaw?.sinceEventId ?? 0); - const lastEventIdHeader = request.headers['last-event-id']; - const cursorFromHeader = Number( - Array.isArray(lastEventIdHeader) ? (lastEventIdHeader[0] ?? 0) : (lastEventIdHeader ?? 0), - ); - const sinceEventId = Math.max( - 0, - Number.isFinite(cursorFromQuery) ? Math.floor(cursorFromQuery) : 0, - Number.isFinite(cursorFromHeader) ? Math.floor(cursorFromHeader) : 0, - ); - - reply.hijack(); - // onResponse will not fire for a hijacked reply, so release the HTTP in-flight - // count here and track the long-lived stream via activeSse instead. - releaseHttp(request); - activeSse += 1; - markActivity('sse_started'); - reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); - reply.raw.setHeader('Cache-Control', 'no-cache, no-transform'); - reply.raw.setHeader('Connection', 'keep-alive'); - reply.raw.setHeader('X-Accel-Buffering', 'no'); - // Tell the browser EventSource to back off before reconnecting. If this stream - // is torn down because the worker went idle (NATS dropped), we don't want the - // client to reconnect immediately and re-wake the container. - reply.raw.write(encodeSseFrame({ retry: OP_EVENTS_RECONNECT_HINT_MS })); - - let closed = false; - let unsubscribe: (() => void) | null = null; - let keepalive: NodeJS.Timeout | null = null; - - const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => { - if (closed || reply.raw.writableEnded) return; - const frameEvent: PublicOperationEvent = { - eventId, - snapshot: toPublicOperation(snapshot), - }; - reply.raw.write(encodeSseFrame({ - id: eventId, - event: 'snapshot', - data: frameEvent, - })); - }; - - const closeStream = (): void => { - if (closed) return; - closed = true; - if (unsubscribe) { - unsubscribe(); - unsubscribe = null; - } - if (keepalive) { - clearInterval(keepalive); - keepalive = null; - } - activeSse = Math.max(0, activeSse - 1); - markActivity('sse_closed'); - if (!reply.raw.writableEnded) { - reply.raw.end(); - } - }; - - request.raw.on('close', () => { - closeStream(); - }); - - try { - let current = initial; - let signature = JSON.stringify(current); - writeSnapshot(current, sinceEventId > 0 ? sinceEventId : 0); - if (isTerminalStatus(current.status)) { - return reply; - } - - keepalive = setInterval(() => { - if (closed || reply.raw.writableEnded) return; - reply.raw.write(': keepalive\n\n'); - }, OP_EVENTS_KEEPALIVE_MS); - - unsubscribe = await operationEventStream.subscribe({ - opId: params.data.opId, - sinceEventId, - onEvent: (event) => { - if (closed) return; - if (event.snapshot.opId !== params.data.opId) return; - const nextSignature = JSON.stringify(event.snapshot); - if (nextSignature !== signature) { - current = event.snapshot; - signature = nextSignature; - // A real event is progress: refresh the idle window so an actively - // streaming op keeps the worker awake. A silent stream does not. - markActivity('sse_event'); - writeSnapshot(current, event.eventId); - } - if (isTerminalStatus(event.snapshot.status)) { - closeStream(); - } - }, - onError: (error) => { - app.log.warn({ - opId: params.data.opId, - error: toErrorMessage(error), - }, 'op events stream loop error'); - closeStream(); - }, - }); - - await new Promise((resolve) => { - request.raw.once('close', () => resolve()); - }); - } catch (error) { - app.log.warn({ - opId: params.data.opId, - error: toErrorMessage(error), - }, 'op events stream loop error'); - } finally { - closeStream(); - } - - return reply; }); const jobHandlers = createJobHandlers({ @@ -974,7 +296,7 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti pdfAttempts, whisperCodec: whisperJobCodec, pdfCodec: layoutJobCodec, - isOwnerActive: (owner) => session === owner, + isOwnerActive: (owner) => sessionManager.isOwnerActive(owner), isStopping: () => stopping, markActivity, onInFlightJobsChanged: (delta) => { @@ -982,32 +304,41 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti }, }); + sessionManager = createNatsSessionManager({ + connectOptions: connectOpts, + logger: app.log, + whisperTimeoutMs, + pdfTimeoutMs, + pdfAttempts, + jobsStreamMaxBytes, + eventsStreamMaxBytes, + jobStatesMaxBytes, + natsReplicas, + isStopping: () => stopping, + getActivity: () => ({ + activeSse, + inFlightHttp, + inFlightJobs, + lastActivityAt, + lastActivityReason, + }), + markActivity, + startWorkers: (session) => { + if (disableWorkers) return; + workerLoops.start(session, { + whisper: session.whisperConsumer, + pdfLayout: session.layoutConsumer, + }); + }, + stopWorkers: () => workerLoops.stop(), + runReconciliation: () => reconciler.run({ force: true }), + }); + const close = async (): Promise => { if (stopping) return; stopping = true; - if (idleTimer) { - clearInterval(idleTimer); - idleTimer = null; - } - if (orphanSweepTimer) { - clearInterval(orphanSweepTimer); - orphanSweepTimer = null; - } await app.close(); - await workerLoops.stop(); - const current = session; - session = null; - if (current) { - try { - await current.nc.drain(); - } catch { - try { - await current.nc.close(); - } catch { - // ignore close errors - } - } - } + await sessionManager.close(); }; const registerSignalHandlers = (): void => { diff --git a/compute-worker/src/api/public-operation.ts b/compute-worker/src/api/compute-operation.ts similarity index 72% rename from compute-worker/src/api/public-operation.ts rename to compute-worker/src/api/compute-operation.ts index ac9b43b..03c6360 100644 --- a/compute-worker/src/api/public-operation.ts +++ b/compute-worker/src/api/compute-operation.ts @@ -1,13 +1,13 @@ -import type { WorkerOperationState } from '../api/contracts'; -import { pdfSubjectFromOperationKey } from './operation-keys'; +import type { WorkerOperationState } from '../operations/contracts'; +import { pdfSubjectFromOperationKey } from '../operations/keys'; -export type PublicOperationSubject = +export type ComputeOperationSubject = | { kind: 'whisper_align' } | { kind: 'pdf_layout'; documentId: string; namespace: string | null }; -export interface PublicOperation { +export interface ComputeOperation { opId: string; - subject: PublicOperationSubject; + subject: ComputeOperationSubject; status: WorkerOperationState['status']; queuedAt: number; updatedAt: number; @@ -18,14 +18,14 @@ export interface PublicOperation { progress?: WorkerOperationState['progress']; } -export interface PublicOperationEvent { +export interface ComputeOperationEvent { eventId: number; - snapshot: PublicOperation; + snapshot: ComputeOperation; } -export function toPublicOperation( +export function toComputeOperation( state: WorkerOperationState, -): PublicOperation { +): ComputeOperation { const subject = state.kind === 'pdf_layout' ? (pdfSubjectFromOperationKey(state.opKey) ?? { kind: 'pdf_layout', documentId: '', namespace: null }) : { kind: 'whisper_align' as const }; diff --git a/compute-worker/src/api/http-hooks.ts b/compute-worker/src/api/http-hooks.ts new file mode 100644 index 0000000..d633da3 --- /dev/null +++ b/compute-worker/src/api/http-hooks.ts @@ -0,0 +1,80 @@ +import type { FastifyInstance, FastifyRequest } from 'fastify'; + +const REQUEST_STARTED_AT_MS_KEY = Symbol('request-started-at-ms'); +const REQUEST_COUNTED_KEY = Symbol('request-activity-counted'); + +function requestPath(request: FastifyRequest): string { + return request.url.split('?')[0] ?? request.url; +} + +function isHealthPath(path: string): boolean { + return path === '/health/live' || path === '/health/ready'; +} + +function isAuthed(request: FastifyRequest, expectedToken: string): boolean { + const auth = request.headers.authorization; + return auth?.startsWith('Bearer ') === true + && auth.slice('Bearer '.length).trim() === expectedToken; +} + +function extractTraceId(request: FastifyRequest): string | null { + const header = request.headers['x-openreader-trace-id']; + if (Array.isArray(header)) return header[0] ?? null; + return typeof header === 'string' ? header : null; +} + +function extractOpId(request: FastifyRequest, path: string): string | null { + const params = request.params as { opId?: unknown } | undefined; + if (typeof params?.opId === 'string' && params.opId.trim()) return params.opId.trim(); + const match = path.match(/^\/v1\/operations\/([^/]+)/); + if (!match?.[1]) return null; + try { + return decodeURIComponent(match[1]); + } catch { + return match[1]; + } +} + +export function registerHttpHooks(input: { + app: FastifyInstance; + workerToken: string; + markActivity: (reason: string) => void; + onInFlightHttpChanged: (delta: number) => void; +}) { + const releaseHttp = (request: FastifyRequest): void => { + const counted = request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean }; + if (!counted[REQUEST_COUNTED_KEY]) return; + counted[REQUEST_COUNTED_KEY] = false; + input.onInFlightHttpChanged(-1); + input.markActivity('http_completed'); + }; + + input.app.addHook('onRequest', async (request, reply) => { + const path = requestPath(request); + (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY] = Date.now(); + (request as FastifyRequest & { [REQUEST_COUNTED_KEY]?: boolean })[REQUEST_COUNTED_KEY] = true; + input.onInFlightHttpChanged(1); + input.markActivity(`http_started:${path}`); + if (!isHealthPath(path) && !isAuthed(request, input.workerToken)) { + return reply.code(401).send({ error: 'Unauthorized' }); + } + }); + + input.app.addHook('onResponse', async (request, reply) => { + releaseHttp(request); + const path = requestPath(request); + if (isHealthPath(path) || reply.statusCode < 500) return; + const startedAt = (request as FastifyRequest & { [REQUEST_STARTED_AT_MS_KEY]?: number })[REQUEST_STARTED_AT_MS_KEY]; + input.app.log.error({ + reqId: request.id, + method: request.method, + path, + statusCode: reply.statusCode, + durationMs: Number.isFinite(startedAt) ? Math.max(0, Date.now() - (startedAt as number)) : -1, + traceId: extractTraceId(request), + opId: extractOpId(request, path), + }, 'http.error'); + }); + + return { releaseHttp }; +} diff --git a/compute-worker/src/api/routes.ts b/compute-worker/src/api/routes.ts new file mode 100644 index 0000000..481e965 --- /dev/null +++ b/compute-worker/src/api/routes.ts @@ -0,0 +1,356 @@ +import type { FastifyInstance, FastifyRequest } from 'fastify'; +import { encodeSseFrame } from '../operations'; +import type { + PdfLayoutJobResult, + WorkerJobTiming, + WorkerOperationEvent, + WorkerOperationRequest, + WorkerOperationState, + WhisperAlignJobResult, +} from '../operations/contracts'; +import { hashOpKey } from '../infrastructure/nats-adapters'; +import type { StreamedOperationState } from '../operations/recovery'; +import type { ReconciliationStateStore } from '../operations/reconciliation'; +import { parsedPdfArtifactKey } from '../storage/artifact-addressing'; +import { buildPdfOperationKey, buildWhisperOperationKey } from '../operations/keys'; +import { toComputeOperation, type ComputeOperationEvent } from './compute-operation'; +import { + apiErrorResponseSchema, + jsonSchema, + operationEventsQuerySchema, + operationParamsSchema, + pdfLayoutResolutionSchema, + pdfOperationCreateSchema, + pdfResolveSchema, + computeOperationSchema, + whisperOperationCreateSchema, +} from './schemas'; + +const OP_EVENTS_KEEPALIVE_MS = 15_000; +const OP_EVENTS_RECONNECT_HINT_MS = 120_000; +const errorResponseSchema = jsonSchema(apiErrorResponseSchema); + +interface OperationEventStreamLike { + subscribe(input: { + opId: string; + sinceEventId?: number; + onEvent: (event: WorkerOperationEvent) => void | Promise; + onError?: (error: unknown) => void; + }): Promise<() => void>; +} + +interface OperationStateStoreLike extends ReconciliationStateStore {} + +interface OrchestratorLike { + enqueueOrReuse(request: WorkerOperationRequest): Promise; + markRunning(input: { + opId: string; + startedAt?: number; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markProgress(input: { + opId: string; + progress: WorkerOperationState['progress']; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markSucceeded(input: { + opId: string; + result: unknown; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markFailed(input: { + opId: string; + error: { message: string; code?: string } | string; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; + markFailedIfUnchanged?(input: { + current: StreamedOperationState; + expectedRevision: number; + error: { message: string; code?: string } | string; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; +} + +export interface ComputeWorkerRouteDeps { + orchestrator: OrchestratorLike; + operationStateStore: OperationStateStoreLike; + operationEventStream: OperationEventStreamLike; + artifactExists?: (key: string) => Promise; +} + +function toErrorMessage(error: unknown): string { + if (error instanceof Error && error.message) return error.message; + return String(error); +} + +function isTerminalStatus(status: import('../operations/contracts').WorkerJobState): boolean { + return status === 'succeeded' || status === 'failed'; +} + +export function registerComputeWorkerRoutes(input: { + app: FastifyInstance; + deps: ComputeWorkerRouteDeps; + s3Prefix: string; + ensureOrphanedOpRecovery: () => Promise; + getOpState: (opId: string) => Promise; + getNatsConnected: () => boolean; + releaseHttp: (request: FastifyRequest) => void; + markActivity: (reason: string) => void; + onActiveSseChanged: (delta: number) => void; +}) { + const { + app, + deps, + s3Prefix, + ensureOrphanedOpRecovery, + getOpState, + getNatsConnected, + releaseHttp, + markActivity, + onActiveSseChanged, + } = input; + + app.get('/health/live', { + schema: { + security: [], + response: { 200: { type: 'object', properties: { ok: { type: 'boolean' } }, required: ['ok'] } }, + }, + }, async () => ({ ok: true })); + + app.get('/health/ready', { + schema: { + security: [], + response: { + 200: { + type: 'object', + properties: { ok: { type: 'boolean' }, natsConnected: { type: 'boolean' } }, + required: ['ok', 'natsConnected'], + }, + }, + }, + }, async () => ({ ok: true, natsConnected: getNatsConnected() })); + + app.post('/v1/whisper-align/operations', { + schema: { + body: jsonSchema(whisperOperationCreateSchema), + response: { 202: jsonSchema(computeOperationSchema), 400: errorResponseSchema }, + }, + }, async (request, reply) => { + const parsed = whisperOperationCreateSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { error: 'Invalid request body', issues: parsed.error.issues }; + } + + const requestOp: WorkerOperationRequest = { + kind: 'whisper_align', + opKey: buildWhisperOperationKey(parsed.data), + payload: parsed.data, + }; + await ensureOrphanedOpRecovery(); + const op = await deps.orchestrator.enqueueOrReuse(requestOp); + app.log.info({ + kind: requestOp.kind, + opId: op.opId, + jobId: op.jobId, + status: op.status, + opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16), + }, 'op.accepted'); + reply.code(202); + return toComputeOperation(op); + }); + + app.post('/v1/pdf-layout/operations', { + schema: { + body: jsonSchema(pdfOperationCreateSchema), + response: { 202: jsonSchema(computeOperationSchema), 400: errorResponseSchema }, + }, + }, async (request, reply) => { + const parsed = pdfOperationCreateSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { error: 'Invalid request body', issues: parsed.error.issues }; + } + + const requestOp: WorkerOperationRequest = { + kind: 'pdf_layout', + opKey: buildPdfOperationKey(parsed.data), + payload: { + documentId: parsed.data.documentId, + namespace: parsed.data.namespace, + documentObjectKey: parsed.data.documentObjectKey, + }, + }; + await ensureOrphanedOpRecovery(); + const op = await deps.orchestrator.enqueueOrReuse(requestOp); + reply.code(202); + return toComputeOperation(op); + }); + + app.post('/v1/pdf-layout/resolve', { + schema: { + body: jsonSchema(pdfResolveSchema), + response: { 200: jsonSchema(pdfLayoutResolutionSchema), 400: errorResponseSchema }, + }, + }, async (request, reply) => { + const parsed = pdfResolveSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { error: 'Invalid request body', issues: parsed.error.issues }; + } + await ensureOrphanedOpRecovery(); + const artifactKey = parsedPdfArtifactKey({ + documentId: parsed.data.documentId, + namespace: parsed.data.namespace, + prefix: s3Prefix, + }); + const hasArtifact = await deps.artifactExists?.(artifactKey) ?? false; + const opKey = buildPdfOperationKey(parsed.data); + const index = await deps.operationStateStore.getOpIndex?.(opKey); + const operation = index?.opId ? await deps.operationStateStore.getOpState(index.opId) : null; + return { + artifact: hasArtifact ? { objectKey: artifactKey } : null, + operation: operation ? toComputeOperation(operation) : null, + }; + }); + + app.get('/v1/operations/:opId', { + schema: { + params: jsonSchema(operationParamsSchema), + response: { 200: jsonSchema(computeOperationSchema), 400: errorResponseSchema, 404: errorResponseSchema }, + }, + }, async (request, reply) => { + const params = operationParamsSchema.safeParse(request.params); + if (!params.success) { + reply.code(400); + return { error: 'Invalid op id' }; + } + + const state = await getOpState(params.data.opId); + if (!state) { + reply.code(404); + return { error: 'Operation not found' }; + } + return toComputeOperation(state); + }); + + app.get('/v1/operations/:opId/events', { + schema: { + params: jsonSchema(operationParamsSchema), + querystring: jsonSchema(operationEventsQuerySchema), + response: { + 200: { type: 'string', description: 'Server-sent ComputeOperationEvent stream' }, + 400: errorResponseSchema, + 404: errorResponseSchema, + }, + }, + }, async (request, reply) => { + const params = operationParamsSchema.safeParse(request.params); + if (!params.success) { + reply.code(400); + return { error: 'Invalid op id' }; + } + + const initial = await getOpState(params.data.opId); + if (!initial) { + reply.code(404); + return { error: 'Operation not found' }; + } + + const cursorQueryRaw = request.query as { sinceEventId?: string | number | null } | undefined; + const cursorFromQuery = Number(cursorQueryRaw?.sinceEventId ?? 0); + const lastEventIdHeader = request.headers['last-event-id']; + const cursorFromHeader = Number( + Array.isArray(lastEventIdHeader) ? (lastEventIdHeader[0] ?? 0) : (lastEventIdHeader ?? 0), + ); + const sinceEventId = Math.max( + 0, + Number.isFinite(cursorFromQuery) ? Math.floor(cursorFromQuery) : 0, + Number.isFinite(cursorFromHeader) ? Math.floor(cursorFromHeader) : 0, + ); + + reply.hijack(); + releaseHttp(request); + onActiveSseChanged(1); + markActivity('sse_started'); + reply.raw.setHeader('Content-Type', 'text/event-stream; charset=utf-8'); + reply.raw.setHeader('Cache-Control', 'no-cache, no-transform'); + reply.raw.setHeader('Connection', 'keep-alive'); + reply.raw.setHeader('X-Accel-Buffering', 'no'); + reply.raw.write(encodeSseFrame({ retry: OP_EVENTS_RECONNECT_HINT_MS })); + + let closed = false; + let unsubscribe: (() => void) | null = null; + let keepalive: NodeJS.Timeout | null = null; + + const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => { + if (closed || reply.raw.writableEnded) return; + const frameEvent: ComputeOperationEvent = { + eventId, + snapshot: toComputeOperation(snapshot), + }; + reply.raw.write(encodeSseFrame({ id: eventId, event: 'snapshot', data: frameEvent })); + }; + + const closeStream = (): void => { + if (closed) return; + closed = true; + if (unsubscribe) { + unsubscribe(); + unsubscribe = null; + } + if (keepalive) { + clearInterval(keepalive); + keepalive = null; + } + onActiveSseChanged(-1); + markActivity('sse_closed'); + if (!reply.raw.writableEnded) reply.raw.end(); + }; + + request.raw.on('close', closeStream); + + try { + let current = initial; + let signature = JSON.stringify(current); + writeSnapshot(current, sinceEventId > 0 ? sinceEventId : 0); + if (isTerminalStatus(current.status)) return reply; + + keepalive = setInterval(() => { + if (!closed && !reply.raw.writableEnded) reply.raw.write(': keepalive\n\n'); + }, OP_EVENTS_KEEPALIVE_MS); + + unsubscribe = await deps.operationEventStream.subscribe({ + opId: params.data.opId, + sinceEventId, + onEvent: (event) => { + if (closed || event.snapshot.opId !== params.data.opId) return; + const nextSignature = JSON.stringify(event.snapshot); + if (nextSignature !== signature) { + current = event.snapshot; + signature = nextSignature; + markActivity('sse_event'); + writeSnapshot(current, event.eventId); + } + if (isTerminalStatus(event.snapshot.status)) closeStream(); + }, + onError: (error) => { + app.log.warn({ opId: params.data.opId, error: toErrorMessage(error) }, 'op events stream loop error'); + closeStream(); + }, + }); + + await new Promise((resolve) => request.raw.once('close', resolve)); + } catch (error) { + app.log.warn({ opId: params.data.opId, error: toErrorMessage(error) }, 'op events stream loop error'); + } finally { + closeStream(); + } + return reply; + }); +} diff --git a/compute-worker/src/api/schemas.ts b/compute-worker/src/api/schemas.ts index 66df5c5..0175d86 100644 --- a/compute-worker/src/api/schemas.ts +++ b/compute-worker/src/api/schemas.ts @@ -95,7 +95,7 @@ export const pdfLayoutProgressSchema = z.object({ phase: z.enum(['infer', 'merge']), }); -export const publicOperationSchema = z.object({ +export const computeOperationSchema = z.object({ opId: z.string(), subject: z.discriminatedUnion('kind', [ z.object({ kind: z.literal('whisper_align') }), @@ -119,14 +119,14 @@ export const publicOperationSchema = z.object({ progress: pdfLayoutProgressSchema.optional(), }); -export const publicOperationEventSchema = z.object({ +export const computeOperationEventSchema = z.object({ eventId: z.number(), - snapshot: publicOperationSchema, + snapshot: computeOperationSchema, }); export const pdfLayoutResolutionSchema = z.object({ artifact: artifactReferenceSchema.nullable(), - operation: publicOperationSchema.nullable(), + operation: computeOperationSchema.nullable(), }); export function jsonSchema(schema: z.ZodType): Record { diff --git a/compute-worker/src/inference/pdf/parse.ts b/compute-worker/src/inference/pdf/parse.ts index b4491b2..4f70f53 100644 --- a/compute-worker/src/inference/pdf/parse.ts +++ b/compute-worker/src/inference/pdf/parse.ts @@ -4,7 +4,7 @@ import { ensureModel } from './model'; import { runLayoutModel } from './layout-model'; import { mergeTextWithRegions } from './document-layout'; import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdfjs'; -import { PDF_PARSER_VERSION } from '../../api/contracts'; +import { PDF_PARSER_VERSION } from '../../operations/contracts'; import { stitchCrossPageBlocks } from './stitch'; import { renderPage } from './render'; import { normalizeTextItemsForLayout } from './normalize-text'; diff --git a/compute-worker/src/inference/whisper/align.ts b/compute-worker/src/inference/whisper/align.ts index 073afc4..a271623 100644 --- a/compute-worker/src/inference/whisper/align.ts +++ b/compute-worker/src/inference/whisper/align.ts @@ -29,6 +29,11 @@ import { WHISPER_DECODER_MERGED_MODEL_PATH, WHISPER_DECODER_WITH_PAST_MODEL_PATH, } from './model'; +import { + applyTokenSuppression, + applyWhisperTimestampLogitsRules, + argmax, +} from './decoder'; export function buildGoertzelCoefficients(freqBins: number, fftSize: number): Float64Array { const coeffs = new Float64Array(freqBins); @@ -448,118 +453,6 @@ function buildEmptyPastFeeds() { return state.emptyPastFeedsTemplate; } -function argmax(values: Float32Array): number | null { - let bestIdx = 0; - let bestScore = Number.NEGATIVE_INFINITY; - - for (let i = 0; i < values.length; i += 1) { - const score = values[i]; - if (score > bestScore) { - bestScore = score; - bestIdx = i; - } - } - - return Number.isFinite(bestScore) ? bestIdx : null; -} - -function applyTokenSuppression(logits: Float32Array, tokens: Set) { - for (const tokenId of tokens) { - if (tokenId >= 0 && tokenId < logits.length) { - logits[tokenId] = Number.NEGATIVE_INFINITY; - } - } -} - -function logSoftmax(input: Float32Array): Float32Array { - let max = Number.NEGATIVE_INFINITY; - for (let i = 0; i < input.length; i += 1) { - if (input[i] > max) max = input[i]; - } - if (!Number.isFinite(max)) { - return new Float32Array(input.length).fill(Number.NEGATIVE_INFINITY); - } - - let sum = 0; - for (let i = 0; i < input.length; i += 1) { - sum += Math.exp(input[i] - max); - } - const logSum = Math.log(sum); - - const out = new Float32Array(input.length); - for (let i = 0; i < input.length; i += 1) { - out[i] = input[i] - max - logSum; - } - return out; -} - -function applyWhisperTimestampLogitsRules(input: { - logits: Float32Array; - generated: number[]; - beginIndex: number; - eosTokenId: number; - noTimestampsTokenId: number; - timestampBeginTokenId: number; - maxInitialTimestampIndex: number; -}) { - const { - logits, - generated, - beginIndex, - eosTokenId, - noTimestampsTokenId, - timestampBeginTokenId, - maxInitialTimestampIndex, - } = input; - - if (noTimestampsTokenId >= 0 && noTimestampsTokenId < logits.length) { - logits[noTimestampsTokenId] = Number.NEGATIVE_INFINITY; - } - - if (generated.length === beginIndex) { - const upper = Math.min(timestampBeginTokenId, logits.length); - for (let i = 0; i < upper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; - } - - const seq = generated.slice(beginIndex); - const lastWasTimestamp = seq.length >= 1 && seq[seq.length - 1] >= timestampBeginTokenId; - const penultimateWasTimestamp = seq.length < 2 || seq[seq.length - 2] >= timestampBeginTokenId; - - if (lastWasTimestamp) { - if (penultimateWasTimestamp) { - for (let i = timestampBeginTokenId; i < logits.length; i += 1) logits[i] = Number.NEGATIVE_INFINITY; - } else { - const upper = Math.min(eosTokenId, logits.length); - for (let i = 0; i < upper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; - } - } - - if (generated.length === beginIndex && Number.isFinite(maxInitialTimestampIndex)) { - const lastAllowed = timestampBeginTokenId + maxInitialTimestampIndex; - for (let i = lastAllowed + 1; i < logits.length; i += 1) logits[i] = Number.NEGATIVE_INFINITY; - } - - const textUpper = Math.min(timestampBeginTokenId, logits.length); - if (textUpper <= 0 || textUpper >= logits.length) return; - - const logprobs = logSoftmax(logits); - - let maxTextTokenLogprob = Number.NEGATIVE_INFINITY; - for (let i = 0; i < textUpper; i += 1) { - if (logprobs[i] > maxTextTokenLogprob) maxTextTokenLogprob = logprobs[i]; - } - - let timestampProbMass = 0; - for (let i = textUpper; i < logprobs.length; i += 1) { - timestampProbMass += Math.exp(logprobs[i]); - } - const timestampLogprob = timestampProbMass > 0 ? Math.log(timestampProbMass) : Number.NEGATIVE_INFINITY; - - if (timestampLogprob > maxTextTokenLogprob) { - for (let i = 0; i < textUpper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; - } -} - async function getRuntime(): Promise { if (state.runtimePromise) return state.runtimePromise; diff --git a/compute-worker/src/inference/whisper/decoder.ts b/compute-worker/src/inference/whisper/decoder.ts new file mode 100644 index 0000000..777047e --- /dev/null +++ b/compute-worker/src/inference/whisper/decoder.ts @@ -0,0 +1,111 @@ +export function argmax(values: Float32Array): number | null { + let bestIdx = 0; + let bestScore = Number.NEGATIVE_INFINITY; + + for (let i = 0; i < values.length; i += 1) { + const score = values[i]; + if (score > bestScore) { + bestScore = score; + bestIdx = i; + } + } + + return Number.isFinite(bestScore) ? bestIdx : null; +} + +export function applyTokenSuppression(logits: Float32Array, tokens: Set) { + for (const tokenId of tokens) { + if (tokenId >= 0 && tokenId < logits.length) { + logits[tokenId] = Number.NEGATIVE_INFINITY; + } + } +} + +function logSoftmax(input: Float32Array): Float32Array { + let max = Number.NEGATIVE_INFINITY; + for (let i = 0; i < input.length; i += 1) { + if (input[i] > max) max = input[i]; + } + if (!Number.isFinite(max)) { + return new Float32Array(input.length).fill(Number.NEGATIVE_INFINITY); + } + + let sum = 0; + for (let i = 0; i < input.length; i += 1) { + sum += Math.exp(input[i] - max); + } + const logSum = Math.log(sum); + + const out = new Float32Array(input.length); + for (let i = 0; i < input.length; i += 1) { + out[i] = input[i] - max - logSum; + } + return out; +} + +export function applyWhisperTimestampLogitsRules(input: { + logits: Float32Array; + generated: number[]; + beginIndex: number; + eosTokenId: number; + noTimestampsTokenId: number; + timestampBeginTokenId: number; + maxInitialTimestampIndex: number; +}) { + const { + logits, + generated, + beginIndex, + eosTokenId, + noTimestampsTokenId, + timestampBeginTokenId, + maxInitialTimestampIndex, + } = input; + + if (noTimestampsTokenId >= 0 && noTimestampsTokenId < logits.length) { + logits[noTimestampsTokenId] = Number.NEGATIVE_INFINITY; + } + + if (generated.length === beginIndex) { + const upper = Math.min(timestampBeginTokenId, logits.length); + for (let i = 0; i < upper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } + + const seq = generated.slice(beginIndex); + const lastWasTimestamp = seq.length >= 1 && seq[seq.length - 1] >= timestampBeginTokenId; + const penultimateWasTimestamp = seq.length < 2 || seq[seq.length - 2] >= timestampBeginTokenId; + + if (lastWasTimestamp) { + if (penultimateWasTimestamp) { + for (let i = timestampBeginTokenId; i < logits.length; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } else { + const upper = Math.min(eosTokenId, logits.length); + for (let i = 0; i < upper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } + } + + if (generated.length === beginIndex && Number.isFinite(maxInitialTimestampIndex)) { + const lastAllowed = timestampBeginTokenId + maxInitialTimestampIndex; + for (let i = lastAllowed + 1; i < logits.length; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } + + const textUpper = Math.min(timestampBeginTokenId, logits.length); + if (textUpper <= 0 || textUpper >= logits.length) return; + + const logprobs = logSoftmax(logits); + + let maxTextTokenLogprob = Number.NEGATIVE_INFINITY; + for (let i = 0; i < textUpper; i += 1) { + if (logprobs[i] > maxTextTokenLogprob) maxTextTokenLogprob = logprobs[i]; + } + + let timestampProbMass = 0; + for (let i = textUpper; i < logprobs.length; i += 1) { + timestampProbMass += Math.exp(logprobs[i]); + } + const timestampLogprob = timestampProbMass > 0 ? Math.log(timestampProbMass) : Number.NEGATIVE_INFINITY; + + if (timestampLogprob > maxTextTokenLogprob) { + for (let i = 0; i < textUpper; i += 1) logits[i] = Number.NEGATIVE_INFINITY; + } +} diff --git a/compute-worker/src/infrastructure/config.ts b/compute-worker/src/infrastructure/config.ts index 2547f3b..791ac4e 100644 --- a/compute-worker/src/infrastructure/config.ts +++ b/compute-worker/src/infrastructure/config.ts @@ -19,7 +19,13 @@ export type IdleTimeoutAndHardCapInput = { label: string; }; -function readPositiveIntEnv(name: string, fallback: number): number { +export function requireEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +export function readPositiveIntEnv(name: string, fallback: number): number { const raw = process.env[name]?.trim(); if (!raw) return fallback; const parsed = Number(raw); @@ -27,6 +33,34 @@ function readPositiveIntEnv(name: string, fallback: number): number { return Math.floor(parsed); } +export function readBoolEnv(name: string, fallback: boolean): boolean { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + return ['1', 'true', 'yes', 'on'].includes(raw.toLowerCase()); +} + +export function normalizeNatsReplicas(value: number): number { + return value === 3 || value === 5 ? value : 1; +} + +export function buildLoggerConfig(): boolean | Record { + const format = process.env.LOG_FORMAT?.trim().toLowerCase() || 'pretty'; + const level = process.env.COMPUTE_LOG_LEVEL?.trim() || 'info'; + if (format === 'json') return { level, base: null }; + return { + level, + base: null, + transport: { + target: 'pino-pretty', + options: { + colorize: true, + translateTime: 'SYS:standard', + ignore: 'pid,hostname', + }, + }, + }; +} + let timeoutConfigCache: ComputeTimeoutConfig | null = null; let opStaleMsCache: number | null = null; diff --git a/compute-worker/src/infrastructure/nats-adapters.ts b/compute-worker/src/infrastructure/nats-adapters.ts index 00d2526..d6e4ddb 100644 --- a/compute-worker/src/infrastructure/nats-adapters.ts +++ b/compute-worker/src/infrastructure/nats-adapters.ts @@ -13,7 +13,7 @@ import type { PdfLayoutJobRequest, WhisperAlignJobRequest, WorkerOperationKind, -} from '../api/contracts'; +} from '../operations/contracts'; import { createJsonCodec } from './json-codec'; export interface KvEntryLike { diff --git a/compute-worker/src/infrastructure/nats-session.ts b/compute-worker/src/infrastructure/nats-session.ts new file mode 100644 index 0000000..356a0de --- /dev/null +++ b/compute-worker/src/infrastructure/nats-session.ts @@ -0,0 +1,204 @@ +import type { Consumer, JetStreamClient, JetStreamManager } from '@nats-io/jetstream'; +import { jetstream, jetstreamManager } from '@nats-io/jetstream'; +import { Kvm } from '@nats-io/kv'; +import { connect, type ConnectionOptions, type NatsConnection } from '@nats-io/transport-node'; +import { + COMPUTE_STATE_BUCKET, + COMPUTE_STATE_TTL_MS, + JOBS_STREAM_NAME, + LAYOUT_CONSUMER_NAME, + NATS_API_TIMEOUT_MS, + WHISPER_CONSUMER_NAME, + ensureJetStreamResources, +} from './nats'; + +const IDLE_DISCONNECT_MS = 120_000; +const IDLE_CHECK_INTERVAL_MS = 5_000; +const IDLE_STATUS_LOG_INTERVAL_MS = 60_000; +const ORPHAN_SWEEP_INTERVAL_MS = 15_000; + +export interface NatsSession { + nc: NatsConnection; + js: JetStreamClient; + jsm: JetStreamManager; + kv: Awaited>; + whisperConsumer: Consumer; + layoutConsumer: Consumer; +} + +interface NatsSessionLogger { + info(data: unknown, message?: string): void; + info(message: string): void; + error(data: unknown, message?: string): void; +} + +export interface NatsActivitySnapshot { + activeSse: number; + inFlightHttp: number; + inFlightJobs: number; + lastActivityAt: number; + lastActivityReason: string; +} + +function toErrorMessage(error: unknown): string { + return error instanceof Error && error.message ? error.message : String(error); +} + +export function createNatsSessionManager(input: { + connectOptions: ConnectionOptions; + logger: NatsSessionLogger; + whisperTimeoutMs: number; + pdfTimeoutMs: number; + pdfAttempts: number; + jobsStreamMaxBytes: number; + eventsStreamMaxBytes: number; + jobStatesMaxBytes: number; + natsReplicas: number; + isStopping: () => boolean; + getActivity: () => NatsActivitySnapshot; + markActivity: (reason: string) => void; + startWorkers: (session: NatsSession) => void; + stopWorkers: () => Promise; + runReconciliation: () => Promise; +}) { + let session: NatsSession | null = null; + let connecting: Promise | null = null; + let idleTimer: NodeJS.Timeout | null = null; + let orphanSweepTimer: NodeJS.Timeout | null = null; + let lastIdleStatusLogAt = 0; + let generation = -1; + + const clearTimers = () => { + if (idleTimer) clearInterval(idleTimer); + if (orphanSweepTimer) clearInterval(orphanSweepTimer); + idleTimer = null; + orphanSweepTimer = null; + }; + + const disconnect = async (reason: string): Promise => { + const current = session; + if (!current) return; + const activity = input.getActivity(); + input.logger.info({ + reason, + activeSse: activity.activeSse, + inFlightHttp: activity.inFlightHttp, + inFlightJobs: activity.inFlightJobs, + idleForMs: Date.now() - activity.lastActivityAt, + }, 'nats dropping connection'); + session = null; + clearTimers(); + try { + await current.nc.close(); + } catch { + // Closing an already-closed session is harmless. + } + await input.stopWorkers(); + input.logger.info({ reason }, 'nats disconnected'); + }; + + const startTimers = () => { + if (!idleTimer) { + idleTimer = setInterval(() => { + if (!session || input.isStopping()) return; + const activity = input.getActivity(); + const now = Date.now(); + const idleForMs = now - activity.lastActivityAt; + if (now - lastIdleStatusLogAt >= IDLE_STATUS_LOG_INTERVAL_MS) { + lastIdleStatusLogAt = now; + input.logger.info({ + ...activity, + idleForMs, + disconnectEligible: activity.inFlightHttp === 0 + && activity.inFlightJobs === 0 + && idleForMs >= IDLE_DISCONNECT_MS, + }, 'nats idle status'); + } + if (activity.inFlightHttp > 0 || activity.inFlightJobs > 0 || idleForMs < IDLE_DISCONNECT_MS) return; + void disconnect('idle'); + }, IDLE_CHECK_INTERVAL_MS); + idleTimer.unref?.(); + } + if (!orphanSweepTimer) { + orphanSweepTimer = setInterval(() => { + if (!session || input.isStopping()) return; + void input.runReconciliation().catch((error) => { + input.logger.error({ error: toErrorMessage(error) }, 'periodic orphaned operation recovery failed'); + }); + }, ORPHAN_SWEEP_INTERVAL_MS); + orphanSweepTimer.unref?.(); + } + }; + + const ensureConnected = async (): Promise => { + if (session) return session; + if (connecting) return connecting; + connecting = (async () => { + const nc = await connect(input.connectOptions); + const js = jetstream(nc, { timeout: NATS_API_TIMEOUT_MS }); + const jsm = await jetstreamManager(nc, { timeout: NATS_API_TIMEOUT_MS }); + await ensureJetStreamResources({ + jsm, + whisperTimeoutMs: input.whisperTimeoutMs, + pdfTimeoutMs: input.pdfTimeoutMs, + pdfAttempts: input.pdfAttempts, + jobsMaxBytes: input.jobsStreamMaxBytes, + eventsMaxBytes: input.eventsStreamMaxBytes, + natsReplicas: input.natsReplicas, + }); + const kv = await new Kvm(js).create(COMPUTE_STATE_BUCKET, { + replicas: input.natsReplicas, + history: 1, + ttl: COMPUTE_STATE_TTL_MS, + max_bytes: input.jobStatesMaxBytes, + }); + const next: NatsSession = { + nc, + js, + jsm, + kv, + whisperConsumer: await js.consumers.get(JOBS_STREAM_NAME, WHISPER_CONSUMER_NAME), + layoutConsumer: await js.consumers.get(JOBS_STREAM_NAME, LAYOUT_CONSUMER_NAME), + }; + session = next; + generation += 1; + input.markActivity('nats_connected'); + input.startWorkers(next); + startTimers(); + void nc.closed().then(() => { + if (session?.nc === nc) session = null; + }); + input.logger.info('nats connected'); + return next; + })(); + try { + return await connecting; + } finally { + connecting = null; + } + }; + + return { + ensureConnected, + disconnect, + isConnected: () => session !== null, + isOwnerActive: (owner: object) => session === owner, + getGeneration: () => generation, + async close(): Promise { + clearTimers(); + await input.stopWorkers(); + const current = session; + session = null; + if (!current) return; + try { + await current.nc.drain(); + } catch { + try { + await current.nc.close(); + } catch { + // Closing an already-closed session is harmless. + } + } + }, + }; +} diff --git a/compute-worker/src/jobs/handlers.ts b/compute-worker/src/jobs/handlers.ts index a013602..0f64c24 100644 --- a/compute-worker/src/jobs/handlers.ts +++ b/compute-worker/src/jobs/handlers.ts @@ -10,7 +10,7 @@ import type { PdfLayoutProgress, WhisperAlignJobRequest, WhisperAlignJobResult, -} from '../api/contracts'; +} from '../operations/contracts'; import type { ArtifactStorage } from '../infrastructure/storage'; import { persistParsedPdfWhileSourceExists } from './pdf-artifact-persistence'; import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; diff --git a/compute-worker/src/jobs/pdf-progress.ts b/compute-worker/src/jobs/pdf-progress.ts index 4aefe08..acc3867 100644 --- a/compute-worker/src/jobs/pdf-progress.ts +++ b/compute-worker/src/jobs/pdf-progress.ts @@ -1,4 +1,4 @@ -import type { PdfLayoutProgress } from '../api/contracts'; +import type { PdfLayoutProgress } from '../operations/contracts'; export function buildInferProgressForPageStart(input: { pageNumber: number; diff --git a/compute-worker/src/jobs/worker-loop-policy.ts b/compute-worker/src/jobs/worker-loop-policy.ts index ce3547a..c5f9c4e 100644 --- a/compute-worker/src/jobs/worker-loop-policy.ts +++ b/compute-worker/src/jobs/worker-loop-policy.ts @@ -1,4 +1,4 @@ -import type { WorkerOperationKind } from '../api/contracts'; +import type { WorkerOperationKind } from '../operations/contracts'; export type RetryAction = 'nak_retry' | 'term_fail'; diff --git a/compute-worker/src/jobs/worker-loop.ts b/compute-worker/src/jobs/worker-loop.ts index 194bf62..27c5afe 100644 --- a/compute-worker/src/jobs/worker-loop.ts +++ b/compute-worker/src/jobs/worker-loop.ts @@ -7,7 +7,7 @@ import type { WhisperAlignJobResult, WorkerJobTiming, WorkerOperationKind, -} from '../api/contracts'; +} from '../operations/contracts'; import type { JsonCodec } from '../infrastructure/json-codec'; import type { JobHandlers } from './handlers'; import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy'; @@ -30,7 +30,7 @@ export interface QueuedJob { payload: TPayload; } -interface WorkerLoopOrchestrator { +export interface WorkerLoopOrchestrator { markRunning(input: { opId: string; startedAt?: number; updatedAt?: number; timing?: WorkerJobTiming }): Promise; markProgress(input: { opId: string; @@ -47,7 +47,7 @@ interface WorkerLoopOrchestrator { }): Promise; } -interface WorkerLogger { +export interface WorkerLogger { info(data: unknown, message?: string): void; warn(data: unknown, message?: string): void; error(data: unknown, message?: string): void; @@ -126,6 +126,17 @@ export function createWorkerLoopController(input: { latestProgress?: PdfLayoutProgress; }; + type JobRunner = ( + payload: TPayload, + queueWaitMs: number, + hooks?: { onProgress?: (progress: PdfLayoutProgress) => Promise }, + ) => Promise; + + type WorkDefinition = { + codec: JsonCodec>; + run: JobRunner; + }; + const markRunning = async (context: Context, updatedAt: number): Promise => { if (context.latestProgress) { await input.orchestrator.markProgress({ @@ -144,10 +155,8 @@ export function createWorkerLoopController(input: { }); }; - const processMessage = async (work: { + const processMessage = async (work: WorkDefinition & { msg: JsMsg; - codec: JsonCodec>; - run: JobHandlers['runWhisper'] | JobHandlers['runPdfLayout']; workerLabel: string; }): Promise => { let context: Context | null = null; @@ -180,7 +189,7 @@ export function createWorkerLoopController(input: { }, 'failed to persist operation heartbeat state'); }); }, RUNNING_HEARTBEAT_MS); - const result = await work.run(decoded.payload as never, context.queueWaitTiming?.queueWaitMs ?? 0, { + const result = await work.run(decoded.payload, context.queueWaitTiming?.queueWaitMs ?? 0, { onProgress: async (progress) => { try { work.msg.working(); @@ -259,11 +268,9 @@ export function createWorkerLoopController(input: { } }; - const runLoop = async (work: { + const runLoop = async (work: WorkDefinition & { owner: object; consumer: Consumer; - codec: JsonCodec>; - run: JobHandlers['runWhisper'] | JobHandlers['runPdfLayout']; workerLabel: string; }): Promise => { const detached = () => input.isStopping() || stopRequested || !input.isOwnerActive(work.owner); @@ -298,9 +305,17 @@ export function createWorkerLoopController(input: { start(owner: object, consumers: { whisper: Consumer; pdfLayout: Consumer }): void { stopRequested = false; loops = []; + const whisperWork: WorkDefinition = { + codec: input.whisperCodec, + run: input.handlers.runWhisper, + }; + const pdfWork: WorkDefinition = { + codec: input.pdfCodec, + run: input.handlers.runPdfLayout, + }; for (let i = 0; i < input.jobConcurrency; i += 1) { - loops.push(runLoop({ owner, consumer: consumers.whisper, codec: input.whisperCodec, run: input.handlers.runWhisper, workerLabel: `whisper-${i + 1}` })); - loops.push(runLoop({ owner, consumer: consumers.pdfLayout, codec: input.pdfCodec, run: input.handlers.runPdfLayout, workerLabel: `layout-${i + 1}` })); + loops.push(runLoop({ owner, consumer: consumers.whisper, ...whisperWork, workerLabel: `whisper-${i + 1}` })); + loops.push(runLoop({ owner, consumer: consumers.pdfLayout, ...pdfWork, workerLabel: `layout-${i + 1}` })); } }, async stop(): Promise { diff --git a/compute-worker/src/api/contracts.ts b/compute-worker/src/operations/contracts.ts similarity index 95% rename from compute-worker/src/api/contracts.ts rename to compute-worker/src/operations/contracts.ts index 4f78782..5180972 100644 --- a/compute-worker/src/api/contracts.ts +++ b/compute-worker/src/operations/contracts.ts @@ -1,18 +1,18 @@ -import type { TTSSentenceAlignment, ParsedPdfDocument } from './types'; +import type { TTSSentenceAlignment, ParsedPdfDocument } from '../api/types'; export type { TTSAudioBuffer, TTSAudioBytes, TTSSentenceAlignment, TTSSentenceWord, -} from './types'; +} from '../api/types'; export type { ParsedPdfBlockKind, ParsedPdfBlockFragment, ParsedPdfBlock, ParsedPdfPage, ParsedPdfDocument, -} from './types'; +} from '../api/types'; export const ALIGN_QUEUE_NAME = 'whisper-align'; export const PDF_LAYOUT_QUEUE_NAME = 'pdf-layout'; diff --git a/compute-worker/src/api/operation-keys.ts b/compute-worker/src/operations/keys.ts similarity index 96% rename from compute-worker/src/api/operation-keys.ts rename to compute-worker/src/operations/keys.ts index d814fb2..02fc2c8 100644 --- a/compute-worker/src/api/operation-keys.ts +++ b/compute-worker/src/operations/keys.ts @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { PDF_PARSER_VERSION } from '../api/contracts'; +import { PDF_PARSER_VERSION } from './contracts'; function sha256Hex(input: string): string { return createHash('sha256').update(input).digest('hex'); diff --git a/compute-worker/src/operations/reconciliation.ts b/compute-worker/src/operations/reconciliation.ts new file mode 100644 index 0000000..df8dc9e --- /dev/null +++ b/compute-worker/src/operations/reconciliation.ts @@ -0,0 +1,95 @@ +import type { WorkerJobTiming } from './contracts'; +import { + recoverOrphanedOperations, + type StreamedOperationState, +} from './recovery'; + +export interface ReconciliationStateStore { + getOpState(opId: string): Promise; + getOpStateRecord?(opId: string): Promise<{ state: StreamedOperationState; revision: number } | null>; + getOpIndex?(opKey: string): Promise<{ opId: string } | null>; + listOpStates?(): Promise; +} + +export interface ReconciliationOrchestrator { + markFailedIfUnchanged?(input: { + current: StreamedOperationState; + expectedRevision: number; + error: { message: string; code?: string } | string; + updatedAt?: number; + timing?: WorkerJobTiming; + }): Promise; +} + +interface ReconciliationLogger { + warn(data: unknown, message?: string): void; +} + +function supportsRecovery( + stateStore: ReconciliationStateStore, + orchestrator: ReconciliationOrchestrator, +): boolean { + return typeof stateStore.listOpStates === 'function' + && typeof stateStore.getOpStateRecord === 'function' + && typeof orchestrator.markFailedIfUnchanged === 'function'; +} + +export function createOperationReconciler(input: { + stateStore: ReconciliationStateStore; + orchestrator: ReconciliationOrchestrator; + whisperTimeoutMs: number; + pdfTimeoutMs: number; + opStaleMs: number; + getGeneration: () => number; + logger: ReconciliationLogger; +}) { + let recoveryPromise: Promise | null = null; + let recoveredGeneration = -1; + + const run = async (options?: { force?: boolean }): Promise => { + if (!supportsRecovery(input.stateStore, input.orchestrator)) return; + const generation = input.getGeneration(); + if (!options?.force && recoveredGeneration === generation) return; + if (recoveryPromise) return await recoveryPromise; + + recoveryPromise = (async () => { + const recoveredStates = await recoverOrphanedOperations({ + operationStateStore: { + getOpStateRecord: input.stateStore.getOpStateRecord!, + listOpStates: input.stateStore.listOpStates!, + }, + orchestrator: { + markFailedIfUnchanged: async (request) => { + const result = await input.orchestrator.markFailedIfUnchanged!(request); + return result as StreamedOperationState | null; + }, + }, + whisperTimeoutMs: input.whisperTimeoutMs, + pdfTimeoutMs: input.pdfTimeoutMs, + opStaleMs: input.opStaleMs, + }); + if (recoveredStates.length > 0) { + input.logger.warn({ + recoveredCount: recoveredStates.length, + ops: recoveredStates.map((state) => ({ + opId: state.opId, + kind: state.kind, + status: state.status, + })), + }, 'recovered stale in-flight operations during reconciliation'); + } + recoveredGeneration = generation; + })().finally(() => { + recoveryPromise = null; + }); + await recoveryPromise; + }; + + return { + run, + async getOpState(opId: string): Promise { + await run(); + return await input.stateStore.getOpState(opId); + }, + }; +} diff --git a/compute-worker/src/operations/recovery.ts b/compute-worker/src/operations/recovery.ts index 5aaaa10..b453a9a 100644 --- a/compute-worker/src/operations/recovery.ts +++ b/compute-worker/src/operations/recovery.ts @@ -4,7 +4,7 @@ import type { WorkerJobTiming, WorkerJobState, WorkerOperationState, -} from '../api/contracts'; +} from '../operations/contracts'; export type StreamedOperationState = WorkerOperationState; diff --git a/compute-worker/src/operations/service.ts b/compute-worker/src/operations/service.ts index 1469277..67430e9 100644 --- a/compute-worker/src/operations/service.ts +++ b/compute-worker/src/operations/service.ts @@ -5,7 +5,7 @@ import type { WorkerOperationKind, WorkerOperationRequest, WorkerOperationState, -} from '../api/contracts'; +} from '../operations/contracts'; import { buildQueuedState, createErrorShape, diff --git a/compute-worker/src/operations/state-machine.ts b/compute-worker/src/operations/state-machine.ts index 3144e8c..cd4076f 100644 --- a/compute-worker/src/operations/state-machine.ts +++ b/compute-worker/src/operations/state-machine.ts @@ -4,7 +4,7 @@ import type { WorkerOperationKind, WorkerOperationRequest, WorkerOperationState, -} from '../api/contracts'; +} from '../operations/contracts'; export function isTerminalStatus(status: WorkerJobState): boolean { return status === 'succeeded' || status === 'failed'; diff --git a/compute-worker/src/operations/types.ts b/compute-worker/src/operations/types.ts index 1d60200..d3f24c2 100644 --- a/compute-worker/src/operations/types.ts +++ b/compute-worker/src/operations/types.ts @@ -2,7 +2,7 @@ import type { WorkerOperationEvent, WorkerOperationKind, WorkerOperationState, -} from '../api/contracts'; +} from '../operations/contracts'; export type OperationState = WorkerOperationState; export type OperationEvent = WorkerOperationEvent; diff --git a/compute-worker/src/storage/artifact-addressing.ts b/compute-worker/src/storage/artifact-addressing.ts index bf4e312..10bb820 100644 --- a/compute-worker/src/storage/artifact-addressing.ts +++ b/compute-worker/src/storage/artifact-addressing.ts @@ -1,5 +1,5 @@ -import { PDF_PARSER_VERSION } from '../api/contracts'; -import { encodeParserVersion } from '../api/contracts'; +import { PDF_PARSER_VERSION } from '../operations/contracts'; +import { encodeParserVersion } from '../operations/contracts'; const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; diff --git a/compute-worker/tests/compute/algorithms/whisper-decoder.test.ts b/compute-worker/tests/compute/algorithms/whisper-decoder.test.ts new file mode 100644 index 0000000..07f405e --- /dev/null +++ b/compute-worker/tests/compute/algorithms/whisper-decoder.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from 'vitest'; +import { + applyTokenSuppression, + applyWhisperTimestampLogitsRules, + argmax, +} from '../../../src/inference/whisper/decoder'; + +describe('whisper decoder logits', () => { + test('selects finite maximums and suppresses valid token ids', () => { + const logits = new Float32Array([1, 5, 3]); + + applyTokenSuppression(logits, new Set([-1, 1, 10])); + + expect(argmax(logits)).toBe(2); + expect(logits[1]).toBe(Number.NEGATIVE_INFINITY); + expect(argmax(new Float32Array([Number.NEGATIVE_INFINITY]))).toBeNull(); + }); + + test('forces an initial timestamp within the configured range', () => { + const logits = new Float32Array([4, 3, 2, 1, 8, 7, 6, 5]); + + applyWhisperTimestampLogitsRules({ + logits, + generated: [10, 11], + beginIndex: 2, + eosTokenId: 3, + noTimestampsTokenId: 2, + timestampBeginTokenId: 4, + maxInitialTimestampIndex: 1, + }); + + expect(Array.from(logits.slice(0, 4))).toEqual([ + Number.NEGATIVE_INFINITY, + Number.NEGATIVE_INFINITY, + Number.NEGATIVE_INFINITY, + Number.NEGATIVE_INFINITY, + ]); + expect(logits[4]).toBe(8); + expect(logits[5]).toBe(7); + expect(logits[6]).toBe(Number.NEGATIVE_INFINITY); + expect(logits[7]).toBe(Number.NEGATIVE_INFINITY); + }); + + test('requires text after a timestamp pair', () => { + const logits = new Float32Array([4, 3, 2, 1, 8, 7]); + + applyWhisperTimestampLogitsRules({ + logits, + generated: [10, 4, 5], + beginIndex: 1, + eosTokenId: 3, + noTimestampsTokenId: -1, + timestampBeginTokenId: 4, + maxInitialTimestampIndex: Number.POSITIVE_INFINITY, + }); + + expect(logits[4]).toBe(Number.NEGATIVE_INFINITY); + expect(logits[5]).toBe(Number.NEGATIVE_INFINITY); + expect(logits[0]).toBe(4); + }); +}); diff --git a/compute-worker/tests/compute/control-plane/orchestrator.test.ts b/compute-worker/tests/compute/control-plane/orchestrator.test.ts index 66964d3..092cfdb 100644 --- a/compute-worker/tests/compute/control-plane/orchestrator.test.ts +++ b/compute-worker/tests/compute/control-plane/orchestrator.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import type { WorkerOperationRequest } from '../../../src/api/contracts'; +import type { WorkerOperationRequest } from '../../../src/operations/contracts'; import { OperationOrchestrator } from '../../../src/operations'; import { InMemoryOperationEventStream, diff --git a/compute-worker/tests/compute/control-plane/state-machine.test.ts b/compute-worker/tests/compute/control-plane/state-machine.test.ts index 8fba44a..667084f 100644 --- a/compute-worker/tests/compute/control-plane/state-machine.test.ts +++ b/compute-worker/tests/compute/control-plane/state-machine.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import type { WorkerOperationState } from '../../../src/api/contracts'; +import type { WorkerOperationState } from '../../../src/operations/contracts'; import { explainReplacementReason, isInflightStatus, diff --git a/compute-worker/tests/compute/helpers/in-memory-control-plane.ts b/compute-worker/tests/compute/helpers/in-memory-control-plane.ts index 811521b..0348bff 100644 --- a/compute-worker/tests/compute/helpers/in-memory-control-plane.ts +++ b/compute-worker/tests/compute/helpers/in-memory-control-plane.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events'; -import type { WorkerOperationKind } from '../../../src/api/contracts'; +import type { WorkerOperationKind } from '../../../src/operations/contracts'; import type { OperationEvent, OperationEventStream, diff --git a/compute-worker/tests/fixtures/fake-control-plane.ts b/compute-worker/tests/fixtures/fake-control-plane.ts index 23f2760..0cd1c86 100644 --- a/compute-worker/tests/fixtures/fake-control-plane.ts +++ b/compute-worker/tests/fixtures/fake-control-plane.ts @@ -4,7 +4,7 @@ import type { WorkerOperationEvent, WorkerOperationRequest, WorkerOperationState, -} from '../../src/api/contracts'; +} from '../../src/operations/contracts'; import type { ComputeWorkerRouteDeps } from '../../src/api/app'; type ComputeResult = WhisperAlignJobResult | PdfLayoutJobResult; diff --git a/compute-worker/tests/unit/jetstream-adapters.test.ts b/compute-worker/tests/unit/jetstream-adapters.test.ts index a238996..8425061 100644 --- a/compute-worker/tests/unit/jetstream-adapters.test.ts +++ b/compute-worker/tests/unit/jetstream-adapters.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; import { OperationOrchestrator } from '../../src/operations'; -import type { WorkerOperationRequest } from '../../src/api/contracts'; +import type { WorkerOperationRequest } from '../../src/operations/contracts'; import { JetStreamOperationEventStream, JetStreamOperationQueue, diff --git a/compute-worker/tests/unit/key-and-progress.test.ts b/compute-worker/tests/unit/key-and-progress.test.ts index d6274db..dc1656a 100644 --- a/compute-worker/tests/unit/key-and-progress.test.ts +++ b/compute-worker/tests/unit/key-and-progress.test.ts @@ -4,7 +4,7 @@ import { buildInferProgressForPageParsed, buildInferProgressForPageStart, } from '../../src/jobs/pdf-progress'; -import { buildPdfOperationKey } from '../../src/api/operation-keys'; +import { buildPdfOperationKey } from '../../src/operations/keys'; import { parsedPdfArtifactKey } from '../../src/storage/artifact-addressing'; describe('compute worker helpers', () => { diff --git a/compute-worker/tests/unit/worker-loop.test.ts b/compute-worker/tests/unit/worker-loop.test.ts new file mode 100644 index 0000000..3b9ba07 --- /dev/null +++ b/compute-worker/tests/unit/worker-loop.test.ts @@ -0,0 +1,170 @@ +import type { Consumer, JsMsg } from '@nats-io/jetstream'; +import { describe, expect, test, vi } from 'vitest'; +import type { PdfLayoutProgress } from '../../src/operations/contracts'; +import { createJsonCodec } from '../../src/infrastructure/json-codec'; +import { + createWorkerLoopController, + type QueuedJob, + type WorkerLoopOrchestrator, +} from '../../src/jobs/worker-loop'; + +function createMessage(job: QueuedJob, deliveryCount = 1) { + const codec = createJsonCodec>(); + const ack = vi.fn(); + const nak = vi.fn(); + const term = vi.fn(); + const working = vi.fn(); + return { + codec, + msg: { + data: codec.encode(job), + info: { deliveryCount }, + ack, + nak, + term, + working, + } as unknown as JsMsg, + ack, + nak, + term, + working, + }; +} + +function createConsumer(message?: JsMsg): Consumer { + let delivered = false; + return { + next: async () => { + if (!delivered && message) { + delivered = true; + return message; + } + await new Promise((resolve) => setTimeout(resolve, 1)); + return null; + }, + } as unknown as Consumer; +} + +function createOrchestrator() { + const calls: Array<{ method: string; input: unknown }> = []; + const record = (method: string) => async (input: unknown) => { + calls.push({ method, input }); + return input; + }; + const orchestrator: WorkerLoopOrchestrator = { + markRunning: record('running'), + markProgress: record('progress'), + markSucceeded: record('succeeded'), + markFailed: record('failed'), + }; + return { orchestrator, calls }; +} + +describe('worker loop controller', () => { + test('processes PDF progress, persists success, ACKs, and releases in-flight activity', async () => { + const owner = {}; + let active = true; + let inFlight = 0; + const { orchestrator, calls } = createOrchestrator(); + let complete!: () => void; + const completed = new Promise((resolve) => { complete = resolve; }); + const progress: PdfLayoutProgress = { totalPages: 2, pagesParsed: 1, currentPage: 1, phase: 'infer' }; + const pdf = createMessage({ + jobId: 'job-pdf', + opId: 'op-pdf', + opKey: 'pdf-key', + kind: 'pdf_layout', + queuedAt: Date.now() - 10, + payload: { + documentId: 'a'.repeat(64), + namespace: null, + documentObjectKey: 'openreader/doc.pdf', + }, + }); + const whisperCodec = createJsonCodec>(); + const controller = createWorkerLoopController({ + orchestrator, + handlers: { + runWhisper: async () => ({ alignments: [] }), + runPdfLayout: async (_payload, _queueWaitMs, hooks) => { + await hooks?.onProgress?.(progress); + active = false; + complete(); + return { parsedObjectKey: 'openreader/parsed.json' }; + }, + }, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + jobConcurrency: 1, + pdfAttempts: 2, + whisperCodec, + pdfCodec: pdf.codec, + isOwnerActive: () => active, + isStopping: () => false, + markActivity: vi.fn(), + onInFlightJobsChanged: (delta) => { inFlight += delta; }, + }); + + controller.start(owner, { whisper: createConsumer(), pdfLayout: createConsumer(pdf.msg) }); + await completed; + await controller.stop(); + + expect(pdf.working).toHaveBeenCalledOnce(); + expect(pdf.ack).toHaveBeenCalledOnce(); + expect(pdf.nak).not.toHaveBeenCalled(); + expect(inFlight).toBe(0); + expect(calls.map((call) => call.method)).toContain('progress'); + expect(calls.map((call) => call.method)).toContain('succeeded'); + }); + + test('NAKs a retryable PDF failure and does not mark it terminal', async () => { + const owner = {}; + let active = true; + const { orchestrator, calls } = createOrchestrator(); + let attempted!: () => void; + const attemptCompleted = new Promise((resolve) => { attempted = resolve; }); + const pdf = createMessage({ + jobId: 'job-pdf', + opId: 'op-pdf', + opKey: 'pdf-key', + kind: 'pdf_layout', + queuedAt: Date.now(), + payload: { + documentId: 'a'.repeat(64), + namespace: null, + documentObjectKey: 'openreader/doc.pdf', + }, + }); + const controller = createWorkerLoopController({ + orchestrator, + handlers: { + runWhisper: async () => ({ alignments: [] }), + runPdfLayout: async () => { + active = false; + attempted(); + throw new Error('retry me'); + }, + }, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + jobConcurrency: 1, + pdfAttempts: 2, + whisperCodec: createJsonCodec(), + pdfCodec: pdf.codec, + isOwnerActive: () => active, + isStopping: () => false, + markActivity: vi.fn(), + onInFlightJobsChanged: vi.fn(), + }); + + controller.start(owner, { whisper: createConsumer(), pdfLayout: createConsumer(pdf.msg) }); + await attemptCompleted; + await new Promise((resolve) => setTimeout(resolve, 0)); + await controller.stop(); + + expect(pdf.nak).toHaveBeenCalledOnce(); + expect(pdf.term).not.toHaveBeenCalled(); + expect(calls.map((call) => call.method)).not.toContain('failed'); + }); +}); diff --git a/src/lib/server/compute-worker/client.ts b/src/lib/server/compute-worker/client.ts index 2f8c0b6..a8f35bf 100644 --- a/src/lib/server/compute-worker/client.ts +++ b/src/lib/server/compute-worker/client.ts @@ -4,10 +4,10 @@ import type { PdfLayoutResult, WhisperAlignRequest, WhisperAlignResult, - WorkerOperation, - WorkerOperationEvent, + ComputeOperation, + ComputeOperationEvent, } from './protocol'; -import { isWorkerOperation } from './protocol'; +import { isComputeOperation } from './protocol'; class WorkerHttpError extends Error { constructor( @@ -88,7 +88,7 @@ function parseSseFrame(frame: string): { id: number | null; data: string | null return { id, data: data.length > 0 ? data.join('\n') : null }; } -function isTerminal(operation: WorkerOperation): boolean { +function isTerminal(operation: ComputeOperation): boolean { return operation.status === 'succeeded' || operation.status === 'failed'; } @@ -105,7 +105,7 @@ export class ComputeWorkerClient { let lastError: unknown; for (let attempt = 1; attempt <= 2; attempt += 1) { try { - const operation = await this.requestJson>( + const operation = await this.requestJson>( 'POST', '/v1/whisper-align/operations', input, @@ -129,7 +129,7 @@ export class ComputeWorkerClient { throw lastError instanceof Error ? lastError : new Error('Unknown compute worker failure'); } - createPdfLayoutOperation(input: PdfLayoutRequest): Promise> { + createPdfLayoutOperation(input: PdfLayoutRequest): Promise> { return this.requestJson('POST', '/v1/pdf-layout/operations', input); } @@ -141,7 +141,7 @@ export class ComputeWorkerClient { }); } - async getOperation(opId: string): Promise | null> { + async getOperation(opId: string): Promise | null> { try { return await this.requestJson('GET', `/v1/operations/${encodeURIComponent(opId)}`); } catch (error) { @@ -199,7 +199,7 @@ export class ComputeWorkerClient { return error instanceof Error && /network|timeout|fetch failed/i.test(error.message); } - private async waitForOperation(opId: string, timeoutMs: number): Promise> { + private async waitForOperation(opId: string, timeoutMs: number): Promise> { const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); let lastEventId: number | null = null; @@ -227,8 +227,8 @@ export class ComputeWorkerClient { buffer = buffer.slice(end + 2); if (frame.id !== null) lastEventId = frame.id; if (!frame.data) continue; - const event = JSON.parse(frame.data) as WorkerOperationEvent; - if (!event?.snapshot || !isWorkerOperation(event.snapshot) || event.snapshot.opId !== opId) continue; + const event = JSON.parse(frame.data) as ComputeOperationEvent; + if (!event?.snapshot || !isComputeOperation(event.snapshot) || event.snapshot.opId !== opId) continue; if (isTerminal(event.snapshot)) return event.snapshot; } } diff --git a/src/lib/server/compute-worker/generated.ts b/src/lib/server/compute-worker/generated.ts index feb2060..cdac950 100644 --- a/src/lib/server/compute-worker/generated.ts +++ b/src/lib/server/compute-worker/generated.ts @@ -469,7 +469,7 @@ export interface paths { }; requestBody?: never; responses: { - /** @description Server-sent PublicOperationEvent stream */ + /** @description Server-sent ComputeOperationEvent stream */ 200: { headers: { [name: string]: unknown; @@ -579,7 +579,7 @@ export interface components { /** @enum {string} */ phase: "infer" | "merge"; }; - PublicOperation: { + ComputeOperation: { opId: string; subject: { /** @enum {string} */ @@ -613,7 +613,7 @@ export interface components { phase: "infer" | "merge"; }; }; - PublicOperationEvent: { + ComputeOperationEvent: { eventId: number; snapshot: { opId: string; diff --git a/src/lib/server/compute-worker/protocol.ts b/src/lib/server/compute-worker/protocol.ts index 73b196c..2fa0c83 100644 --- a/src/lib/server/compute-worker/protocol.ts +++ b/src/lib/server/compute-worker/protocol.ts @@ -9,17 +9,17 @@ export type ParsedPdfBlockFragment = ParsedPdfBlock['fragments'][number]; export type TTSSentenceAlignment = components['schemas']['TTSSentenceAlignment']; export type TTSSentenceWord = TTSSentenceAlignment['words'][number]; -export type PdfLayoutProgress = NonNullable; -export type WorkerOperationStatus = components['schemas']['PublicOperation']['status']; -export type WorkerOperationSubject = components['schemas']['PublicOperation']['subject']; +export type PdfLayoutProgress = NonNullable; +export type ComputeOperationStatus = components['schemas']['ComputeOperation']['status']; +export type ComputeOperationSubject = components['schemas']['ComputeOperation']['subject']; -export type WorkerOperation = - Omit +export type ComputeOperation = + Omit & { result?: Result }; -export type WorkerOperationEvent = { +export type ComputeOperationEvent = { eventId: number; - snapshot: WorkerOperation; + snapshot: ComputeOperation; }; export type WhisperAlignRequest = @@ -31,20 +31,20 @@ export type PdfLayoutResolveRequest = export type WhisperAlignResult = { alignments: TTSSentenceAlignment[]; - timing?: components['schemas']['PublicOperation']['timing']; + timing?: components['schemas']['ComputeOperation']['timing']; }; export type PdfLayoutResult = { parsedObjectKey: string; - timing?: components['schemas']['PublicOperation']['timing']; + timing?: components['schemas']['ComputeOperation']['timing']; }; export type PdfLayoutResolution = { artifact: { objectKey: string } | null; - operation: WorkerOperation | null; + operation: ComputeOperation | null; }; -export function isWorkerOperation(value: unknown): value is WorkerOperation { +export function isComputeOperation(value: unknown): value is ComputeOperation { if (!value || typeof value !== 'object') return false; const record = value as Record; return typeof record.opId === 'string' diff --git a/src/lib/server/pdf-parse/operation.ts b/src/lib/server/pdf-parse/operation.ts index 60a924a..a10136b 100644 --- a/src/lib/server/pdf-parse/operation.ts +++ b/src/lib/server/pdf-parse/operation.ts @@ -3,7 +3,7 @@ import { documentKey } from '@/lib/server/documents/blobstore'; import type { PdfLayoutResolution, PdfLayoutResult, - WorkerOperation, + ComputeOperation, } from '@/lib/server/compute-worker/protocol'; function currentPdfOperationInput(documentId: string, namespace: string | null, forceToken?: string): { @@ -33,7 +33,7 @@ export async function resolveCurrentPdfParse(input: { export async function lookupCurrentPdfParseOperation(input: { documentId: string; namespace: string | null; -}): Promise | null> { +}): Promise | null> { return (await resolveCurrentPdfParse(input)).operation; } @@ -41,7 +41,7 @@ export async function createOrReuseCurrentPdfParseOperation(input: { documentId: string; namespace: string | null; forceToken?: string; -}): Promise> { +}): Promise> { return getComputeWorkerClient().createPdfLayoutOperation(currentPdfOperationInput( input.documentId, input.namespace, @@ -49,12 +49,12 @@ export async function createOrReuseCurrentPdfParseOperation(input: { )); } -export async function fetchPdfParseOperation(opId: string): Promise | null> { +export async function fetchPdfParseOperation(opId: string): Promise | null> { return getComputeWorkerClient().getOperation(opId); } export function isPdfParseOperationForDocument( - state: WorkerOperation, + state: ComputeOperation, input: { documentId: string; namespace: string | null; diff --git a/src/lib/server/pdf-parse/snapshot.ts b/src/lib/server/pdf-parse/snapshot.ts index 2fba7f9..b4e8924 100644 --- a/src/lib/server/pdf-parse/snapshot.ts +++ b/src/lib/server/pdf-parse/snapshot.ts @@ -1,8 +1,8 @@ -import type { PdfLayoutResult, WorkerOperation } from '@/lib/server/compute-worker/protocol'; +import type { PdfLayoutResult, ComputeOperation } from '@/lib/server/compute-worker/protocol'; import type { PdfParseStatus } from '@/types/parsed-pdf'; import type { PdfParseSnapshot } from '@/lib/server/pdf-parse/types'; -function mapWorkerStatusToParseStatus(status: WorkerOperation['status']): PdfParseStatus { +function mapWorkerStatusToParseStatus(status: ComputeOperation['status']): PdfParseStatus { switch (status) { case 'queued': return 'pending'; @@ -19,7 +19,7 @@ function mapWorkerStatusToParseStatus(status: WorkerOperation['status']): PdfPar } export function parsedObjectKeyFromWorkerState( - state: WorkerOperation, + state: ComputeOperation, ): string | null { const result = state.result; if (!result || typeof result !== 'object' || !('parsedObjectKey' in result)) return null; @@ -30,7 +30,7 @@ export function parsedObjectKeyFromWorkerState( } export function pdfParseSnapshotFromWorkerState( - state: WorkerOperation, + state: ComputeOperation, ): PdfParseSnapshot { const parseStatus = mapWorkerStatusToParseStatus(state.status); return { From 2abadde949ccc8fb3c719cc2d03129a6936c2468 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 13 Jun 2026 13:21:18 -0600 Subject: [PATCH 05/11] docs(deploy): add unified Docker Compose guide and update dev workflow Introduce a comprehensive Docker Compose documentation page covering slim, full, and local-build stack examples. Add example Compose files under docker/examples/ for streamlined deployment. Update local development docs to reference the new Compose guide. Revise package.json scripts to support the new Compose workflows and remove legacy compute-worker compose commands. Update sidebar to include the Docker Compose documentation for improved discoverability. --- Dockerfile | 6 +- compute-worker/docker-compose.yml | 44 ------ docker/examples/compose.full.yml | 124 +++++++++++++++++ docker/examples/compose.local.yml | 130 +++++++++++++++++ docker/examples/compose.yml | 49 +++++++ docs-site/docs/deploy/docker-compose.md | 153 +++++++++++++++++++++ docs-site/docs/deploy/local-development.md | 43 +----- docs-site/sidebars.ts | 7 +- package.json | 12 +- 9 files changed, 477 insertions(+), 91 deletions(-) delete mode 100644 compute-worker/docker-compose.yml create mode 100644 docker/examples/compose.full.yml create mode 100644 docker/examples/compose.local.yml create mode 100644 docker/examples/compose.yml create mode 100644 docs-site/docs/deploy/docker-compose.md diff --git a/Dockerfile b/Dockerfile index f4ebc3e..83230da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -73,12 +73,14 @@ COPY --from=app-builder /app/src/db ./src/db # Merge in the dependency subset needed by the entrypoint migration scripts. COPY --from=app-builder /opt/entrypoint-migration-tools/node_modules /tmp/runtime-tools-node_modules RUN mkdir -p /app/node_modules && \ + rm -rf /app/node_modules/pg && \ rm -rf /tmp/runtime-tools-node_modules/@aws-sdk \ /tmp/runtime-tools-node_modules/better-sqlite3 \ - /tmp/runtime-tools-node_modules/ffmpeg-static \ - /tmp/runtime-tools-node_modules/pg && \ + /tmp/runtime-tools-node_modules/ffmpeg-static && \ cp -an /tmp/runtime-tools-node_modules/. /app/node_modules/ && \ rm -rf /tmp/runtime-tools-node_modules +# Drizzle Kit discovers the Postgres driver through ESM import during startup migrations. +RUN node --input-type=module -e "await import('pg')" # Ship the embedded compute worker as a separate deployed bundle. COPY --from=app-builder /opt/embedded-compute-worker ./embedded-compute-worker diff --git a/compute-worker/docker-compose.yml b/compute-worker/docker-compose.yml deleted file mode 100644 index 115eea8..0000000 --- a/compute-worker/docker-compose.yml +++ /dev/null @@ -1,44 +0,0 @@ -services: - nats: - image: nats:2.14-alpine - container_name: openreader-compute-nats - command: ["-js", "-sd", "/data"] - ports: - - "4222:4222" - - "8222:8222" - volumes: - - nats-data:/data - - compute-worker: - build: - context: .. - dockerfile: compute-worker/Dockerfile - container_name: openreader-compute-worker - depends_on: - - nats - env_file: - - path: ./.env - required: false - environment: - NATS_URL: ${NATS_URL:-nats://nats:4222} - COMPUTE_WORKER_HOST: ${COMPUTE_WORKER_HOST:-0.0.0.0} - PORT: ${PORT:-8081} - COMPUTE_WORKER_TOKEN: ${COMPUTE_WORKER_TOKEN:-local-compute-token} - S3_PREFIX: ${S3_PREFIX:-openreader} - COMPUTE_PREWARM_MODELS: ${COMPUTE_PREWARM_MODELS:-false} - ports: - - "8081:8081" - develop: - watch: - - action: sync+restart - path: . - target: /workspace - - action: rebuild - path: ./package.json - - action: rebuild - path: ../pnpm-lock.yaml - - action: rebuild - path: ../pnpm-workspace.yaml - -volumes: - nats-data: diff --git a/docker/examples/compose.full.yml b/docker/examples/compose.full.yml new file mode 100644 index 0000000..bdf2270 --- /dev/null +++ b/docker/examples/compose.full.yml @@ -0,0 +1,124 @@ +name: openreader-example + +services: + openreader: + image: ghcr.io/richardr1126/openreader:latest + # Keep localhost:8333 valid for both app requests and browser-facing presigned URLs. + network_mode: service:seaweedfs + depends_on: + kokoro-tts: + condition: service_started + postgres: + condition: service_healthy + seaweedfs: + condition: service_healthy + environment: + BASE_URL: ${BASE_URL:-http://localhost:3003} + AUTH_SECRET: ${AUTH_SECRET:-local-openreader-auth-secret-change-me} + POSTGRES_URL: postgres://openreader:openreader@postgres:5432/openreader + COMPUTE_WORKER_URL: http://compute-worker:8081 + COMPUTE_WORKER_TOKEN: ${COMPUTE_WORKER_TOKEN:-local-compute-token} + USE_EMBEDDED_WEED_MINI: "false" + S3_ENDPOINT: ${S3_ENDPOINT:-http://localhost:8333} # Used for internal endpoint and public facing presigned URLs. + S3_BUCKET: ${S3_BUCKET:-openreader-documents} + S3_REGION: ${S3_REGION:-us-east-1} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-devkey} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-devsecret} + S3_FORCE_PATH_STYLE: "true" + S3_PREFIX: ${S3_PREFIX:-openreader} + RUNTIME_SEED_JSON: |- + { + "version": 1, + "runtimeConfig": { + "defaultTtsProvider": "kokoro" + }, + "providers": [ + { + "slug": "kokoro", + "displayName": "Kokoro", + "providerType": "custom-openai", + "baseUrl": "http://kokoro-tts:8880/v1", + "defaultModel": "kokoro", + "enabled": true + } + ] + } + volumes: + - openreader-docstore:/app/docstore + + kokoro-tts: + image: ghcr.io/remsky/kokoro-fastapi-cpu:v0.2.4 + environment: + ONNX_NUM_THREADS: 8 + ONNX_INTER_OP_THREADS: 4 + ONNX_EXECUTION_MODE: parallel + ONNX_OPTIMIZATION_LEVEL: all + ONNX_MEMORY_PATTERN: "true" + ONNX_ARENA_EXTEND_STRATEGY: kNextPowerOfTwo + API_LOG_LEVEL: DEBUG + ports: + - "8880:8880" + + seaweedfs: + image: chrislusf/seaweedfs:4.18 + command: ["mini", "-dir=/data"] + environment: + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-devkey} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-devsecret} + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9333/cluster/status >/dev/null"] + interval: 2s + timeout: 2s + retries: 30 + ports: + - "3003:3003" + - "8333:8333" + volumes: + - seaweedfs-data:/data + + nats: + image: nats:2.14-alpine + command: ["-js", "-sd", "/data"] + volumes: + - nats-data:/data + + postgres: + image: postgres:17-alpine + environment: + POSTGRES_USER: openreader + POSTGRES_PASSWORD: openreader + POSTGRES_DB: openreader + healthcheck: + test: ["CMD-SHELL", "pg_isready -U openreader -d openreader"] + interval: 2s + timeout: 2s + retries: 30 + volumes: + - postgres-data:/var/lib/postgresql/data + + compute-worker: + image: ghcr.io/richardr1126/openreader-compute-worker:latest + depends_on: + nats: + condition: service_started + seaweedfs: + condition: service_healthy + environment: + NATS_URL: nats://nats:4222 + COMPUTE_WORKER_HOST: 0.0.0.0 + PORT: 8081 + COMPUTE_WORKER_TOKEN: ${COMPUTE_WORKER_TOKEN:-local-compute-token} + S3_ENDPOINT: http://seaweedfs:8333 + S3_BUCKET: ${S3_BUCKET:-openreader-documents} + S3_REGION: ${S3_REGION:-us-east-1} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-devkey} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-devsecret} + S3_FORCE_PATH_STYLE: "true" + S3_PREFIX: ${S3_PREFIX:-openreader} + COMPUTE_PREWARM_MODELS: ${COMPUTE_PREWARM_MODELS:-false} + +volumes: + openreader-docstore: + seaweedfs-data: + nats-data: + postgres-data: diff --git a/docker/examples/compose.local.yml b/docker/examples/compose.local.yml new file mode 100644 index 0000000..a50b667 --- /dev/null +++ b/docker/examples/compose.local.yml @@ -0,0 +1,130 @@ +name: openreader-local-build + +services: + openreader: + image: openreader:local + build: + context: ../.. + dockerfile: Dockerfile + # Keep localhost:8333 valid for both app requests and browser-facing presigned URLs. + network_mode: service:seaweedfs + depends_on: + kokoro-tts: + condition: service_started + postgres: + condition: service_healthy + seaweedfs: + condition: service_healthy + environment: + BASE_URL: ${BASE_URL:-http://localhost:3003} + AUTH_SECRET: ${AUTH_SECRET:-local-openreader-auth-secret-change-me} + POSTGRES_URL: postgres://openreader:openreader@postgres:5432/openreader + COMPUTE_WORKER_URL: http://compute-worker:8081 + COMPUTE_WORKER_TOKEN: ${COMPUTE_WORKER_TOKEN:-local-compute-token} + USE_EMBEDDED_WEED_MINI: "false" + S3_ENDPOINT: ${S3_ENDPOINT:-http://localhost:8333} # Used for internal endpoint and public facing presigned URLs. + S3_BUCKET: ${S3_BUCKET:-openreader-documents} + S3_REGION: ${S3_REGION:-us-east-1} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-devkey} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-devsecret} + S3_FORCE_PATH_STYLE: "true" + S3_PREFIX: ${S3_PREFIX:-openreader} + RUNTIME_SEED_JSON: |- + { + "version": 1, + "runtimeConfig": { + "defaultTtsProvider": "kokoro" + }, + "providers": [ + { + "slug": "kokoro", + "displayName": "Kokoro", + "providerType": "custom-openai", + "baseUrl": "http://kokoro-tts:8880/v1", + "defaultModel": "kokoro", + "enabled": true + } + ] + } + volumes: + - openreader-docstore:/app/docstore + + kokoro-tts: + image: ghcr.io/remsky/kokoro-fastapi-cpu:v0.2.4 + environment: + ONNX_NUM_THREADS: 8 + ONNX_INTER_OP_THREADS: 4 + ONNX_EXECUTION_MODE: parallel + ONNX_OPTIMIZATION_LEVEL: all + ONNX_MEMORY_PATTERN: "true" + ONNX_ARENA_EXTEND_STRATEGY: kNextPowerOfTwo + API_LOG_LEVEL: DEBUG + ports: + - "8880:8880" + + seaweedfs: + image: chrislusf/seaweedfs:4.18 + command: ["mini", "-dir=/data"] + environment: + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-devkey} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-devsecret} + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:9333/cluster/status >/dev/null"] + interval: 2s + timeout: 2s + retries: 30 + ports: + - "3003:3003" + - "8333:8333" + volumes: + - seaweedfs-data:/data + + nats: + image: nats:2.14-alpine + command: ["-js", "-sd", "/data"] + volumes: + - nats-data:/data + + postgres: + image: postgres:17-alpine + environment: + POSTGRES_USER: openreader + POSTGRES_PASSWORD: openreader + POSTGRES_DB: openreader + healthcheck: + test: ["CMD-SHELL", "pg_isready -U openreader -d openreader"] + interval: 2s + timeout: 2s + retries: 30 + volumes: + - postgres-data:/var/lib/postgresql/data + + compute-worker: + image: openreader-compute-worker:local + build: + context: ../.. + dockerfile: compute-worker/Dockerfile + depends_on: + nats: + condition: service_started + seaweedfs: + condition: service_healthy + environment: + NATS_URL: nats://nats:4222 + COMPUTE_WORKER_HOST: 0.0.0.0 + PORT: 8081 + COMPUTE_WORKER_TOKEN: ${COMPUTE_WORKER_TOKEN:-local-compute-token} + S3_ENDPOINT: http://seaweedfs:8333 + S3_BUCKET: ${S3_BUCKET:-openreader-documents} + S3_REGION: ${S3_REGION:-us-east-1} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:-devkey} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:-devsecret} + S3_FORCE_PATH_STYLE: "true" + S3_PREFIX: ${S3_PREFIX:-openreader} + COMPUTE_PREWARM_MODELS: ${COMPUTE_PREWARM_MODELS:-false} + +volumes: + openreader-docstore: + seaweedfs-data: + nats-data: + postgres-data: diff --git a/docker/examples/compose.yml b/docker/examples/compose.yml new file mode 100644 index 0000000..4745d0e --- /dev/null +++ b/docker/examples/compose.yml @@ -0,0 +1,49 @@ +name: openreader-slim-example + +services: + openreader: + image: ghcr.io/richardr1126/openreader:latest + depends_on: + kokoro-tts: + condition: service_started + environment: + BASE_URL: ${BASE_URL:-http://localhost:3003} + AUTH_SECRET: ${AUTH_SECRET:-local-openreader-auth-secret-change-me} + RUNTIME_SEED_JSON: |- + { + "version": 1, + "runtimeConfig": { + "defaultTtsProvider": "kokoro" + }, + "providers": [ + { + "slug": "kokoro", + "displayName": "Kokoro", + "providerType": "custom-openai", + "baseUrl": "http://kokoro-tts:8880/v1", + "defaultModel": "kokoro", + "enabled": true + } + ] + } + ports: + - "3003:3003" + - "8333:8333" + volumes: + - openreader-docstore:/app/docstore + + kokoro-tts: + image: ghcr.io/remsky/kokoro-fastapi-cpu:v0.2.4 + environment: + ONNX_NUM_THREADS: 8 + ONNX_INTER_OP_THREADS: 4 + ONNX_EXECUTION_MODE: parallel + ONNX_OPTIMIZATION_LEVEL: all + ONNX_MEMORY_PATTERN: "true" + ONNX_ARENA_EXTEND_STRATEGY: kNextPowerOfTwo + API_LOG_LEVEL: DEBUG + ports: + - "8880:8880" + +volumes: + openreader-docstore: diff --git a/docs-site/docs/deploy/docker-compose.md b/docs-site/docs/deploy/docker-compose.md new file mode 100644 index 0000000..78786d3 --- /dev/null +++ b/docs-site/docs/deploy/docker-compose.md @@ -0,0 +1,153 @@ +--- +title: Docker Compose +description: Run OpenReader with the slim, full, or local-build Docker Compose examples. +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Use these examples to run OpenReader with Kokoro-FastAPI and persistent storage. Choose the slim +stack for the simplest deployment, or the full stack when you want PostgreSQL, SeaweedFS, NATS, +and the compute worker as separate containers. + +## Prerequisites + +- A recent Docker version with Docker Compose +- A clone of the OpenReader repository + +```bash +git clone https://github.com/richardr1126/openreader.git +cd openreader +``` + +## Choose a stack + + + + +The default slim example runs: + +- OpenReader with embedded SeaweedFS, NATS, compute worker, and SQLite +- Kokoro-FastAPI as a companion container + +```bash +docker compose -f docker/examples/compose.yml up +# Repository convenience command: pnpm compose +``` + +Compose file: [`docker/examples/compose.yml`](https://github.com/richardr1126/openreader/blob/main/docker/examples/compose.yml) + + + + +The full example runs OpenReader, Kokoro-FastAPI, PostgreSQL, SeaweedFS, NATS, and the compute +worker as separate containers using published images. + +```bash +docker compose -f docker/examples/compose.full.yml up +# Repository convenience command: pnpm compose:full +``` + +Compose file: [`docker/examples/compose.full.yml`](https://github.com/richardr1126/openreader/blob/main/docker/examples/compose.full.yml) + +For details about running the worker separately, see +[Compute Worker](./compute-worker). + + + + +The local-build example uses the full multi-container layout, but builds the OpenReader app and +compute-worker images from the current checkout. + +```bash +docker compose -f docker/examples/compose.local.yml up --build +# Repository convenience command: pnpm compose:local +``` + +Compose file: [`docker/examples/compose.local.yml`](https://github.com/richardr1126/openreader/blob/main/docker/examples/compose.local.yml) + + + + +## Included services + +| Service | Slim | Full | Local Build | +| --- | --- | --- | --- | +| OpenReader | Published image | Published image | Local build | +| Kokoro-FastAPI | Container | Container | Container | +| Database | Embedded SQLite | PostgreSQL container | PostgreSQL container | +| SeaweedFS | Embedded | Container | Container | +| NATS | Embedded | Container | Container | +| Compute worker | Embedded | Published image | Local build | + +On first boot, `RUNTIME_SEED_JSON` creates an enabled Kokoro shared provider and selects it as the +default TTS provider. + +## Endpoints + +- OpenReader: `http://localhost:3003` +- SeaweedFS S3: `http://localhost:8333` +- Kokoro-FastAPI: `http://localhost:8880` + +In the full examples, PostgreSQL, the compute worker, and NATS remain internal to the Compose +network. + +## LAN access + +Set `BASE_URL` and `S3_ENDPOINT` to the Docker host's LAN IP so browser-facing app and presigned +S3 URLs are reachable from other devices: + + + + +```bash +BASE_URL=http://192.168.0.XXX:3003 \ +S3_ENDPOINT=http://192.168.0.XXX:8333 \ +docker compose -f docker/examples/compose.yml up +# Repository convenience command: pnpm compose +``` + + + + +```bash +BASE_URL=http://192.168.0.XXX:3003 \ +S3_ENDPOINT=http://192.168.0.XXX:8333 \ +docker compose -f docker/examples/compose.full.yml up +# Repository convenience command: pnpm compose:full +``` + + + + +```bash +BASE_URL=http://192.168.0.XXX:3003 \ +S3_ENDPOINT=http://192.168.0.XXX:8333 \ +docker compose -f docker/examples/compose.local.yml up --build +# Repository convenience command: pnpm compose:local +``` + + + + +Replace `192.168.0.XXX` with your Docker host's LAN IP and allow inbound TCP ports `3003` and +`8333` through its firewall. + +:::info Internal full-stack endpoint +The full and local-build compute workers continue using `http://seaweedfs:8333` internally. +`S3_ENDPOINT` configures the app endpoint and browser-facing presigned URLs. +::: + +## Configuration + +The examples use local-only default credentials. Override existing `${VARIABLE}` values through +your shell environment before using them beyond local development. + +:::warning Protect public deployments +Replace the default `AUTH_SECRET`, PostgreSQL credentials, S3 credentials, and compute-worker +token before exposing a stack outside your trusted local network. +::: + +For the complete configuration reference, see +[Environment Variables](../reference/environment-variables). See [Database](../configure/database) +for PostgreSQL and SQLite behavior. diff --git a/docs-site/docs/deploy/local-development.md b/docs-site/docs/deploy/local-development.md index 6098225..7d81811 100644 --- a/docs-site/docs/deploy/local-development.md +++ b/docs-site/docs/deploy/local-development.md @@ -159,45 +159,10 @@ If you need mirrors or pinned artifact locations, set `WHISPER_MODEL_BASE_URL` i -
-External compute worker dev stack (optional) - -Use this only when you intentionally run compute-worker as a separate service. -Default local flow does not need `compute-worker/.env`; embedded worker startup reads root `.env`. -Full worker deployment details are in [Compute Worker (NATS JetStream)](./compute-worker). - -Start only NATS + compute-worker via compose watch: - -```bash -docker compose --env-file compute-worker/.env -f compute-worker/docker-compose.yml up --watch -# or: pnpm compute:dev:compose -``` - -`compute-worker/.env.example` contains a starter config for standalone worker service deployments. - -Run the main app separately on the host: - -```bash -pnpm dev -``` - -For app -> external worker routing, set in root `.env`: - -```env -COMPUTE_WORKER_URL=http://localhost:8081 -COMPUTE_WORKER_TOKEN= -``` - -Ownership in external worker mode: -- root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale/retry overrides such as `COMPUTE_PDF_JOB_ATTEMPTS` -- `compute-worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning) - -For embedded worker startup (`COMPUTE_WORKER_URL` unset), worker tuning values such as `COMPUTE_PDF_JOB_ATTEMPTS` must be set in the root `.env` because `compute-worker/.env*` is ignored in that mode. - -Worker mode requires worker-reachable shared object storage (S3-compatible endpoint). -For external worker mode, object storage must be shared/reachable by both app and worker services. - -
+:::tip Docker Compose +To run OpenReader and Kokoro-FastAPI with Docker Compose, including slim, full, and local-build +options, see [Docker Compose](./docker-compose). +::: ## Steps diff --git a/docs-site/sidebars.ts b/docs-site/sidebars.ts index 63041f9..45597dc 100644 --- a/docs-site/sidebars.ts +++ b/docs-site/sidebars.ts @@ -55,7 +55,12 @@ const sidebars: SidebarsConfig = { { type: 'category', label: '🚀 Deploy', - items: ['deploy/local-development', 'deploy/compute-worker', 'deploy/vercel-deployment'], + items: [ + 'deploy/local-development', + 'deploy/docker-compose', + 'deploy/compute-worker', + 'deploy/vercel-deployment', + ], }, { type: 'category', diff --git a/package.json b/package.json index 25d915d..37ad06a 100644 --- a/package.json +++ b/package.json @@ -16,8 +16,9 @@ "test:e2e": "playwright test", "test:unit": "vitest run", "test:compute": "vitest run --project compute-worker", - "compute:openapi:generate": "pnpm --dir compute-worker openapi:generate && openapi-typescript compute-worker/openapi.json -o src/lib/server/compute-worker/generated.ts", - "compute:openapi:check": "pnpm compute:openapi:generate && git diff --exit-code -- compute-worker/openapi.json src/lib/server/compute-worker/generated.ts", + "compose": "docker compose -f docker/examples/compose.yml up", + "compose:full": "docker compose -f docker/examples/compose.full.yml up", + "compose:local": "docker compose -f docker/examples/compose.local.yml up --build", "migrate": "node drizzle/scripts/migrate.mjs", "migrate-fs": "node scripts/migrate-fs-v2.mjs", "migrate-fs:dry-run": "node scripts/migrate-fs-v2.mjs --dry-run true", @@ -28,9 +29,10 @@ "docs:serve": "pnpm --dir docs-site serve", "docs:version": "pnpm --dir docs-site docusaurus docs:version", "docs:clear": "pnpm --dir docs-site clear", - "compute:worker:dev": "pnpm --filter @openreader/compute-worker dev", - "compute:worker:start": "pnpm --filter @openreader/compute-worker start", - "compute:dev:compose": "docker compose --env-file compute-worker/.env -f compute-worker/docker-compose.yml up --build", + "compute:dev": "pnpm --filter @openreader/compute-worker dev", + "compute:start": "pnpm --filter @openreader/compute-worker start", + "compute:openapi:generate": "pnpm --dir compute-worker openapi:generate && openapi-typescript compute-worker/openapi.json -o src/lib/server/compute-worker/generated.ts", + "compute:openapi:check": "pnpm compute:openapi:generate && git diff --exit-code -- compute-worker/openapi.json src/lib/server/compute-worker/generated.ts", "lint:route-errors": "node scripts/check-route-error-responses.mjs" }, "dependencies": { From 358a56cc9a57337ed858bebf1474e2e652bd5414 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 13 Jun 2026 13:51:18 -0600 Subject: [PATCH 06/11] refactor: implement modular monorepo packages - Extract database schema and migrations into @openreader/database - Extract startup and orchestration scripts into @openreader/bootstrap - Move compute-worker into packages/compute-worker - Remove runtime dependency on drizzle-kit - Update Dockerfile to deploy isolated packages without merging node_modules --- .github/workflows/docker-publish.yml | 4 +- Dockerfile | 41 ++-- .../entrypoint-migration-tools/package.json | 14 -- docker/examples/compose.local.yml | 2 +- docs-site/docs/configure/migrations.md | 57 +++--- docs-site/docs/deploy/vercel-deployment.md | 2 +- drizzle/scripts/migrate.mjs | 56 ----- next.config.ts | 2 +- package.json | 22 +- packages/bootstrap/package.json | 25 +++ .../bootstrap/src/cli.mjs | 192 ++---------------- packages/bootstrap/src/embedded-nats.mjs | 5 + packages/bootstrap/src/embedded-seaweedfs.mjs | 95 +++++++++ packages/bootstrap/src/embedded-worker.mjs | 30 +++ .../bootstrap/src/storage-migration.mjs | 0 .../compute-worker}/.env.example | 0 .../compute-worker}/Dockerfile | 8 +- .../compute-worker}/openapi.json | 0 .../compute-worker}/package.json | 0 .../scripts/generate-openapi.ts | 0 .../compute-worker}/src/api/app.ts | 0 .../src/api/compute-operation.ts | 0 .../compute-worker}/src/api/http-hooks.ts | 0 .../compute-worker}/src/api/routes.ts | 0 .../compute-worker}/src/api/schemas.ts | 0 .../compute-worker}/src/api/types.ts | 0 .../src/inference/pdf/assets/LICENSE.txt | 0 .../src/inference/pdf/assets/manifest.json | 0 .../src/inference/pdf/document-layout.ts | 0 .../src/inference/pdf/layout-model.ts | 0 .../src/inference/pdf/model.ts | 0 .../src/inference/pdf/normalize-text.ts | 0 .../src/inference/pdf/parse.ts | 0 .../src/inference/pdf/pdfjs.ts | 0 .../src/inference/pdf/render.ts | 0 .../src/inference/pdf/stitch.ts | 0 .../compute-worker}/src/inference/runtime.ts | 0 .../src/inference/whisper/align.ts | 0 .../src/inference/whisper/assets/LICENSE.txt | 0 .../inference/whisper/assets/manifest.json | 0 .../inference/whisper/assets/mel_filters.npz | Bin .../src/inference/whisper/decoder.ts | 0 .../src/inference/whisper/model.ts | 0 .../src/inference/whisper/timestamps.ts | 0 .../src/infrastructure/config.ts | 0 .../src/infrastructure/json-codec.ts | 0 .../src/infrastructure/nats-adapters.ts | 0 .../src/infrastructure/nats-session.ts | 0 .../src/infrastructure/nats.ts | 0 .../src/infrastructure/platform.ts | 0 .../src/infrastructure/storage.ts | 0 .../compute-worker}/src/jobs/handlers.ts | 0 .../src/jobs/pdf-artifact-persistence.ts | 0 .../compute-worker}/src/jobs/pdf-progress.ts | 0 .../src/jobs/worker-loop-policy.ts | 0 .../compute-worker}/src/jobs/worker-loop.ts | 0 .../src/operations/contracts.ts | 0 .../compute-worker}/src/operations/index.ts | 0 .../compute-worker}/src/operations/keys.ts | 0 .../src/operations/reconciliation.ts | 0 .../src/operations/recovery.ts | 0 .../compute-worker}/src/operations/service.ts | 0 .../compute-worker}/src/operations/sse.ts | 0 .../src/operations/state-machine.ts | 0 .../compute-worker}/src/operations/types.ts | 0 .../compute-worker}/src/server.ts | 0 .../src/storage/artifact-addressing.ts | 0 .../compute-worker}/tests/api/routes.test.ts | 0 .../pdf-merge-text-with-regions.test.ts | 0 .../pdf-parse-normalize-text-items.test.ts | 0 .../pdf-stitch-cross-page-blocks.test.ts | 0 .../algorithms/support/document-fixtures.ts | 2 +- .../whisper-alignment-mapping.test.ts | 0 .../algorithms/whisper-decoder.test.ts | 0 .../algorithms/whisper-spectral.test.ts | 0 .../whisper-token-timestamps.test.ts | 0 .../control-plane/orchestrator.test.ts | 0 .../control-plane/run-layout-model.test.ts | 0 .../tests/compute/control-plane/sse.test.ts | 0 .../control-plane/state-machine.test.ts | 0 .../helpers/in-memory-control-plane.ts | 0 .../tests/fixtures/fake-control-plane.ts | 0 .../compute-worker}/tests/setup-env.ts | 0 .../compute-worker}/tests/tsconfig.json | 0 .../tests/unit/jetstream-adapters.test.ts | 11 + .../tests/unit/key-and-progress.test.ts | 0 .../tests/unit/orphan-recovery.test.ts | 8 +- .../unit/pdf-artifact-persistence.test.ts | 0 .../tests/unit/worker-loop-policy.test.ts | 0 .../tests/unit/worker-loop.test.ts | 0 .../compute-worker}/tsconfig.json | 2 +- packages/database/bin/migrate.mjs | 9 + .../database/drizzle.config.pg.ts | 4 +- .../database/drizzle.config.sqlite.ts | 4 +- .../postgres/0000_black_lucky_pierre.sql | 0 .../postgres/0001_tts_segments.sql | 0 .../0002_add_segment_key_to_tts_segments.sql | 0 .../postgres/0003_tts_segments_v2_split.sql | 0 .../migrations}/postgres/0004_admin_panel.sql | 0 .../postgres/0005_pdf_layout_and_compute.sql | 0 .../0006_preview-defaults-cleanup.sql | 0 .../postgres/0007_pdf_parse_progress.sql | 0 .../postgres/0008_user_job_events.sql | 0 .../postgres/0009_drop_pdf_parse.sql | 0 .../0010_user-data-cleanup-cascades.sql | 0 .../postgres/0011_scheduled-tasks.sql | 0 .../postgres/meta/0000_snapshot.json | 0 .../postgres/meta/0001_snapshot.json | 0 .../postgres/meta/0002_snapshot.json | 0 .../postgres/meta/0003_snapshot.json | 0 .../postgres/meta/0004_snapshot.json | 0 .../postgres/meta/0005_snapshot.json | 0 .../postgres/meta/0006_snapshot.json | 0 .../postgres/meta/0007_snapshot.json | 0 .../postgres/meta/0008_snapshot.json | 0 .../postgres/meta/0009_snapshot.json | 0 .../postgres/meta/0010_snapshot.json | 0 .../postgres/meta/0011_snapshot.json | 0 .../migrations}/postgres/meta/_journal.json | 0 .../sqlite/0000_familiar_caretaker.sql | 0 .../migrations}/sqlite/0001_tts_segments.sql | 0 .../0002_add_segment_key_to_tts_segments.sql | 0 .../sqlite/0003_tts_segments_v2_split.sql | 0 .../migrations}/sqlite/0004_admin_panel.sql | 0 .../sqlite/0005_pdf_layout_and_compute.sql | 0 .../sqlite/0006_preview-defaults-cleanup.sql | 0 .../sqlite/0007_pdf_parse_progress.sql | 0 .../sqlite/0008_user_job_events.sql | 0 .../sqlite/0009_drop_pdf_parse.sql | 0 .../0010_user-data-cleanup-cascades.sql | 0 .../sqlite/0011_scheduled-tasks.sql | 0 .../sqlite/meta/0000_snapshot.json | 0 .../sqlite/meta/0001_snapshot.json | 0 .../sqlite/meta/0002_snapshot.json | 0 .../sqlite/meta/0003_snapshot.json | 0 .../sqlite/meta/0004_snapshot.json | 0 .../sqlite/meta/0005_snapshot.json | 0 .../sqlite/meta/0006_snapshot.json | 0 .../sqlite/meta/0007_snapshot.json | 0 .../sqlite/meta/0008_snapshot.json | 0 .../sqlite/meta/0009_snapshot.json | 0 .../sqlite/meta/0010_snapshot.json | 0 .../sqlite/meta/0011_snapshot.json | 0 .../migrations}/sqlite/meta/_journal.json | 0 packages/database/package.json | 33 +++ .../database}/scripts/generate.mjs | 9 +- {src/db => packages/database/src}/index.ts | 0 packages/database/src/migrate.mjs | 51 +++++ .../database/src}/run-in-transaction.ts | 2 +- {src/db => packages/database/src}/schema.ts | 0 .../database/src}/schema_auth_postgres.ts | 0 .../database/src}/schema_auth_sqlite.ts | 0 .../database/src}/schema_postgres.ts | 0 .../database/src}/schema_sqlite.ts | 0 pnpm-lock.yaml | 52 +++-- pnpm-workspace.yaml | 3 +- scripts/check-compute-boundary.mjs | 2 +- scripts/check-next-server-bundle.mjs | 2 +- src/app/api/audiobook/chapter/route.ts | 4 +- src/app/api/audiobook/route.ts | 4 +- src/app/api/audiobook/status/route.ts | 4 +- .../api/documents/[id]/parsed/events/route.ts | 4 +- src/app/api/documents/[id]/parsed/route.ts | 4 +- src/app/api/documents/[id]/settings/route.ts | 4 +- .../api/documents/blob/get/fallback/route.ts | 4 +- .../api/documents/blob/get/presign/route.ts | 4 +- .../documents/blob/preview/fallback/route.ts | 4 +- src/app/api/documents/blob/preview/utils.ts | 4 +- src/app/api/documents/route.ts | 4 +- src/app/api/tts/segments/ensure/route.ts | 4 +- src/app/api/tts/segments/manifest/route.ts | 4 +- src/app/api/user/claim/route.ts | 4 +- src/app/api/user/export/route.ts | 8 +- .../state/changelog/version-check/route.ts | 4 +- src/app/api/user/state/preferences/route.ts | 4 +- src/app/api/user/state/progress/route.ts | 4 +- src/lib/server/admin/email-sync.ts | 6 +- src/lib/server/admin/providers.ts | 4 +- src/lib/server/admin/seed.ts | 4 +- src/lib/server/admin/settings.ts | 4 +- src/lib/server/audiobooks/prune.ts | 4 +- src/lib/server/auth/auth.ts | 6 +- src/lib/server/documents/blob-lease.ts | 4 +- src/lib/server/documents/delete-owned.ts | 4 +- src/lib/server/documents/previews.ts | 4 +- src/lib/server/documents/register-upload.ts | 4 +- src/lib/server/rate-limit/job-rate-limiter.ts | 4 +- src/lib/server/rate-limit/rate-limiter.ts | 6 +- src/lib/server/tasks/engine.ts | 4 +- .../server/tasks/handlers/prune-job-events.ts | 4 +- .../tasks/handlers/reap-orphaned-blobs.ts | 4 +- src/lib/server/testing/test-namespace.ts | 2 +- src/lib/server/tts/segments-audio.ts | 4 +- src/lib/server/tts/segments-auth.ts | 4 +- src/lib/server/tts/segments-cache.ts | 4 +- src/lib/server/user/claim-data.ts | 4 +- src/lib/server/user/data-cleanup.ts | 8 +- tests/global-teardown.ts | 8 +- .../admin-providers-validation.vitest.spec.ts | 4 +- ...pute-worker-client-contract.vitest.spec.ts | 4 +- tests/unit/document-blob-lease.vitest.spec.ts | 4 +- .../document-delete-cleanup.vitest.spec.ts | 2 +- ...d-events-route-worker-proxy.vitest.spec.ts | 2 +- ...df-parsed-route-worker-flow.vitest.spec.ts | 2 +- tests/unit/reap-orphaned-blobs.vitest.spec.ts | 2 +- tests/unit/runtime-seed-json.vitest.spec.ts | 4 +- .../scheduled-tasks-engine.vitest.spec.ts | 4 +- .../transfer-user-documents.vitest.spec.ts | 2 +- tests/unit/tts-segments-cache.vitest.spec.ts | 2 +- tests/unit/user-data-cleanup.vitest.spec.ts | 2 +- vitest.config.ts | 4 +- 211 files changed, 497 insertions(+), 466 deletions(-) delete mode 100644 docker/entrypoint-migration-tools/package.json delete mode 100644 drizzle/scripts/migrate.mjs create mode 100644 packages/bootstrap/package.json rename scripts/openreader-entrypoint.mjs => packages/bootstrap/src/cli.mjs (77%) create mode 100644 packages/bootstrap/src/embedded-nats.mjs create mode 100644 packages/bootstrap/src/embedded-seaweedfs.mjs create mode 100644 packages/bootstrap/src/embedded-worker.mjs rename scripts/migrate-fs-v2.mjs => packages/bootstrap/src/storage-migration.mjs (100%) rename {compute-worker => packages/compute-worker}/.env.example (100%) rename {compute-worker => packages/compute-worker}/Dockerfile (72%) rename {compute-worker => packages/compute-worker}/openapi.json (100%) rename {compute-worker => packages/compute-worker}/package.json (100%) rename {compute-worker => packages/compute-worker}/scripts/generate-openapi.ts (100%) rename {compute-worker => packages/compute-worker}/src/api/app.ts (100%) rename {compute-worker => packages/compute-worker}/src/api/compute-operation.ts (100%) rename {compute-worker => packages/compute-worker}/src/api/http-hooks.ts (100%) rename {compute-worker => packages/compute-worker}/src/api/routes.ts (100%) rename {compute-worker => packages/compute-worker}/src/api/schemas.ts (100%) rename {compute-worker => packages/compute-worker}/src/api/types.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/pdf/assets/LICENSE.txt (100%) rename {compute-worker => packages/compute-worker}/src/inference/pdf/assets/manifest.json (100%) rename {compute-worker => packages/compute-worker}/src/inference/pdf/document-layout.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/pdf/layout-model.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/pdf/model.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/pdf/normalize-text.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/pdf/parse.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/pdf/pdfjs.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/pdf/render.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/pdf/stitch.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/runtime.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/whisper/align.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/whisper/assets/LICENSE.txt (100%) rename {compute-worker => packages/compute-worker}/src/inference/whisper/assets/manifest.json (100%) rename {compute-worker => packages/compute-worker}/src/inference/whisper/assets/mel_filters.npz (100%) rename {compute-worker => packages/compute-worker}/src/inference/whisper/decoder.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/whisper/model.ts (100%) rename {compute-worker => packages/compute-worker}/src/inference/whisper/timestamps.ts (100%) rename {compute-worker => packages/compute-worker}/src/infrastructure/config.ts (100%) rename {compute-worker => packages/compute-worker}/src/infrastructure/json-codec.ts (100%) rename {compute-worker => packages/compute-worker}/src/infrastructure/nats-adapters.ts (100%) rename {compute-worker => packages/compute-worker}/src/infrastructure/nats-session.ts (100%) rename {compute-worker => packages/compute-worker}/src/infrastructure/nats.ts (100%) rename {compute-worker => packages/compute-worker}/src/infrastructure/platform.ts (100%) rename {compute-worker => packages/compute-worker}/src/infrastructure/storage.ts (100%) rename {compute-worker => packages/compute-worker}/src/jobs/handlers.ts (100%) rename {compute-worker => packages/compute-worker}/src/jobs/pdf-artifact-persistence.ts (100%) rename {compute-worker => packages/compute-worker}/src/jobs/pdf-progress.ts (100%) rename {compute-worker => packages/compute-worker}/src/jobs/worker-loop-policy.ts (100%) rename {compute-worker => packages/compute-worker}/src/jobs/worker-loop.ts (100%) rename {compute-worker => packages/compute-worker}/src/operations/contracts.ts (100%) rename {compute-worker => packages/compute-worker}/src/operations/index.ts (100%) rename {compute-worker => packages/compute-worker}/src/operations/keys.ts (100%) rename {compute-worker => packages/compute-worker}/src/operations/reconciliation.ts (100%) rename {compute-worker => packages/compute-worker}/src/operations/recovery.ts (100%) rename {compute-worker => packages/compute-worker}/src/operations/service.ts (100%) rename {compute-worker => packages/compute-worker}/src/operations/sse.ts (100%) rename {compute-worker => packages/compute-worker}/src/operations/state-machine.ts (100%) rename {compute-worker => packages/compute-worker}/src/operations/types.ts (100%) rename {compute-worker => packages/compute-worker}/src/server.ts (100%) rename {compute-worker => packages/compute-worker}/src/storage/artifact-addressing.ts (100%) rename {compute-worker => packages/compute-worker}/tests/api/routes.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/compute/algorithms/support/document-fixtures.ts (93%) rename {compute-worker => packages/compute-worker}/tests/compute/algorithms/whisper-alignment-mapping.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/compute/algorithms/whisper-decoder.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/compute/algorithms/whisper-spectral.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/compute/algorithms/whisper-token-timestamps.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/compute/control-plane/orchestrator.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/compute/control-plane/run-layout-model.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/compute/control-plane/sse.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/compute/control-plane/state-machine.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/compute/helpers/in-memory-control-plane.ts (100%) rename {compute-worker => packages/compute-worker}/tests/fixtures/fake-control-plane.ts (100%) rename {compute-worker => packages/compute-worker}/tests/setup-env.ts (100%) rename {compute-worker => packages/compute-worker}/tests/tsconfig.json (100%) rename {compute-worker => packages/compute-worker}/tests/unit/jetstream-adapters.test.ts (95%) rename {compute-worker => packages/compute-worker}/tests/unit/key-and-progress.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/unit/orphan-recovery.test.ts (88%) rename {compute-worker => packages/compute-worker}/tests/unit/pdf-artifact-persistence.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/unit/worker-loop-policy.test.ts (100%) rename {compute-worker => packages/compute-worker}/tests/unit/worker-loop.test.ts (100%) rename {compute-worker => packages/compute-worker}/tsconfig.json (73%) create mode 100755 packages/database/bin/migrate.mjs rename drizzle.config.pg.ts => packages/database/drizzle.config.pg.ts (72%) rename drizzle.config.sqlite.ts => packages/database/drizzle.config.sqlite.ts (57%) rename {drizzle => packages/database/migrations}/postgres/0000_black_lucky_pierre.sql (100%) rename {drizzle => packages/database/migrations}/postgres/0001_tts_segments.sql (100%) rename {drizzle => packages/database/migrations}/postgres/0002_add_segment_key_to_tts_segments.sql (100%) rename {drizzle => packages/database/migrations}/postgres/0003_tts_segments_v2_split.sql (100%) rename {drizzle => packages/database/migrations}/postgres/0004_admin_panel.sql (100%) rename {drizzle => packages/database/migrations}/postgres/0005_pdf_layout_and_compute.sql (100%) rename {drizzle => packages/database/migrations}/postgres/0006_preview-defaults-cleanup.sql (100%) rename {drizzle => packages/database/migrations}/postgres/0007_pdf_parse_progress.sql (100%) rename {drizzle => packages/database/migrations}/postgres/0008_user_job_events.sql (100%) rename {drizzle => packages/database/migrations}/postgres/0009_drop_pdf_parse.sql (100%) rename {drizzle => packages/database/migrations}/postgres/0010_user-data-cleanup-cascades.sql (100%) rename {drizzle => packages/database/migrations}/postgres/0011_scheduled-tasks.sql (100%) rename {drizzle => packages/database/migrations}/postgres/meta/0000_snapshot.json (100%) rename {drizzle => packages/database/migrations}/postgres/meta/0001_snapshot.json (100%) rename {drizzle => packages/database/migrations}/postgres/meta/0002_snapshot.json (100%) rename {drizzle => packages/database/migrations}/postgres/meta/0003_snapshot.json (100%) rename {drizzle => packages/database/migrations}/postgres/meta/0004_snapshot.json (100%) rename {drizzle => packages/database/migrations}/postgres/meta/0005_snapshot.json (100%) rename {drizzle => packages/database/migrations}/postgres/meta/0006_snapshot.json (100%) rename {drizzle => packages/database/migrations}/postgres/meta/0007_snapshot.json (100%) rename {drizzle => packages/database/migrations}/postgres/meta/0008_snapshot.json (100%) rename {drizzle => packages/database/migrations}/postgres/meta/0009_snapshot.json (100%) rename {drizzle => packages/database/migrations}/postgres/meta/0010_snapshot.json (100%) rename {drizzle => packages/database/migrations}/postgres/meta/0011_snapshot.json (100%) rename {drizzle => packages/database/migrations}/postgres/meta/_journal.json (100%) rename {drizzle => packages/database/migrations}/sqlite/0000_familiar_caretaker.sql (100%) rename {drizzle => packages/database/migrations}/sqlite/0001_tts_segments.sql (100%) rename {drizzle => packages/database/migrations}/sqlite/0002_add_segment_key_to_tts_segments.sql (100%) rename {drizzle => packages/database/migrations}/sqlite/0003_tts_segments_v2_split.sql (100%) rename {drizzle => packages/database/migrations}/sqlite/0004_admin_panel.sql (100%) rename {drizzle => packages/database/migrations}/sqlite/0005_pdf_layout_and_compute.sql (100%) rename {drizzle => packages/database/migrations}/sqlite/0006_preview-defaults-cleanup.sql (100%) rename {drizzle => packages/database/migrations}/sqlite/0007_pdf_parse_progress.sql (100%) rename {drizzle => packages/database/migrations}/sqlite/0008_user_job_events.sql (100%) rename {drizzle => packages/database/migrations}/sqlite/0009_drop_pdf_parse.sql (100%) rename {drizzle => packages/database/migrations}/sqlite/0010_user-data-cleanup-cascades.sql (100%) rename {drizzle => packages/database/migrations}/sqlite/0011_scheduled-tasks.sql (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/0000_snapshot.json (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/0001_snapshot.json (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/0002_snapshot.json (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/0003_snapshot.json (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/0004_snapshot.json (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/0005_snapshot.json (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/0006_snapshot.json (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/0007_snapshot.json (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/0008_snapshot.json (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/0009_snapshot.json (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/0010_snapshot.json (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/0011_snapshot.json (100%) rename {drizzle => packages/database/migrations}/sqlite/meta/_journal.json (100%) create mode 100644 packages/database/package.json rename {drizzle => packages/database}/scripts/generate.mjs (91%) rename {src/db => packages/database/src}/index.ts (100%) create mode 100644 packages/database/src/migrate.mjs rename {src/db => packages/database/src}/run-in-transaction.ts (96%) rename {src/db => packages/database/src}/schema.ts (100%) rename {src/db => packages/database/src}/schema_auth_postgres.ts (100%) rename {src/db => packages/database/src}/schema_auth_sqlite.ts (100%) rename {src/db => packages/database/src}/schema_postgres.ts (100%) rename {src/db => packages/database/src}/schema_sqlite.ts (100%) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d7f6733..fedd0d3 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -62,13 +62,13 @@ jobs: platform: linux/amd64 runner: ubuntu-24.04 context: . - dockerfile: ./compute-worker/Dockerfile + dockerfile: ./packages/compute-worker/Dockerfile - image_target: compute-worker arch: arm64 platform: linux/arm64 runner: ubuntu-24.04-arm context: . - dockerfile: ./compute-worker/Dockerfile + dockerfile: ./packages/compute-worker/Dockerfile steps: - name: Checkout repository diff --git a/Dockerfile b/Dockerfile index 83230da..f3396d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,8 +20,9 @@ WORKDIR /app # Copy workspace manifests needed for dependency installation COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ -COPY compute-worker/package.json ./compute-worker/package.json -COPY docker/entrypoint-migration-tools/package.json ./docker/entrypoint-migration-tools/package.json +COPY packages/bootstrap/package.json ./packages/bootstrap/package.json +COPY packages/compute-worker/package.json ./packages/compute-worker/package.json +COPY packages/database/package.json ./packages/database/package.json # Install dependencies RUN pnpm install --frozen-lockfile @@ -32,8 +33,9 @@ COPY . . # Build the Next.js application RUN pnpm exec next telemetry disable RUN AUTH_SECRET=build-placeholder-secret-value-32chars!! BASE_URL=http://localhost:3003 pnpm build -RUN pnpm --config.inject-workspace-packages=true --filter @openreader/entrypoint-migration-tools deploy /opt/entrypoint-migration-tools -RUN pnpm --config.inject-workspace-packages=true --filter @openreader/compute-worker deploy /opt/embedded-compute-worker +RUN pnpm --config.inject-workspace-packages=true --filter @openreader/bootstrap deploy /opt/openreader/bootstrap +RUN pnpm --dir /opt/openreader/bootstrap rebuild better-sqlite3 ffmpeg-static +RUN pnpm --config.inject-workspace-packages=true --filter @openreader/compute-worker deploy /opt/openreader/embedded-compute-worker # Generate third-party dependency license report plus copied license files. RUN mkdir -p /app/THIRD_PARTY_LICENSES && \ pnpm dlx license-checker-rseidelsohn@4.3.0 \ @@ -62,34 +64,15 @@ COPY --from=app-builder /app/.next/standalone ./ COPY --from=app-builder /app/.next/static ./.next/static COPY --from=app-builder /app/public ./public -# Copy the entrypoint and migration/runtime helper files it invokes directly. -COPY --from=app-builder /app/scripts/openreader-entrypoint.mjs ./scripts/openreader-entrypoint.mjs -COPY --from=app-builder /app/scripts/migrate-fs-v2.mjs ./scripts/migrate-fs-v2.mjs -COPY --from=app-builder /app/drizzle ./drizzle -COPY --from=app-builder /app/drizzle.config.pg.ts ./drizzle.config.pg.ts -COPY --from=app-builder /app/drizzle.config.sqlite.ts ./drizzle.config.sqlite.ts -COPY --from=app-builder /app/src/db ./src/db - -# Merge in the dependency subset needed by the entrypoint migration scripts. -COPY --from=app-builder /opt/entrypoint-migration-tools/node_modules /tmp/runtime-tools-node_modules -RUN mkdir -p /app/node_modules && \ - rm -rf /app/node_modules/pg && \ - rm -rf /tmp/runtime-tools-node_modules/@aws-sdk \ - /tmp/runtime-tools-node_modules/better-sqlite3 \ - /tmp/runtime-tools-node_modules/ffmpeg-static && \ - cp -an /tmp/runtime-tools-node_modules/. /app/node_modules/ && \ - rm -rf /tmp/runtime-tools-node_modules -# Drizzle Kit discovers the Postgres driver through ESM import during startup migrations. -RUN node --input-type=module -e "await import('pg')" - -# Ship the embedded compute worker as a separate deployed bundle. -COPY --from=app-builder /opt/embedded-compute-worker ./embedded-compute-worker +# Ship startup orchestration and the embedded worker as independent deployed bundles. +COPY --from=app-builder /opt/openreader/bootstrap /opt/openreader/bootstrap +COPY --from=app-builder /opt/openreader/embedded-compute-worker /opt/openreader/embedded-compute-worker # Include third-party license report and copied license texts at a stable path in the image. COPY --from=app-builder /app/THIRD_PARTY_LICENSES /licenses # Include SeaweedFS license text for the copied weed binary. COPY --from=seaweedfs-builder /tmp/SeaweedFS-LICENSE.txt /licenses/SeaweedFS-LICENSE.txt # Include static model notices for runtime-downloaded assets. -COPY --from=app-builder /app/compute-worker/src/inference/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt +COPY --from=app-builder /app/packages/compute-worker/src/inference/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt # Copy seaweedfs weed binary for optional embedded local S3. COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed @@ -99,7 +82,7 @@ COPY --from=nats-builder /tmp/nats-server /usr/local/bin/nats-server RUN chmod +x /usr/local/bin/nats-server # Include OpenAI Whisper license text for runtime-downloaded ONNX artifacts. -COPY --from=app-builder /app/compute-worker/src/inference/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt +COPY --from=app-builder /app/packages/compute-worker/src/inference/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt # Match the app's historical container port now that standalone server.js # is started directly instead of `next start -p 3003`. @@ -109,5 +92,5 @@ ENV PORT=3003 EXPOSE 3003 # Start the application -ENTRYPOINT ["node", "scripts/openreader-entrypoint.mjs", "--"] +ENTRYPOINT ["node", "/opt/openreader/bootstrap/src/cli.mjs", "--"] CMD ["node", "server.js"] diff --git a/docker/entrypoint-migration-tools/package.json b/docker/entrypoint-migration-tools/package.json deleted file mode 100644 index db7e2a3..0000000 --- a/docker/entrypoint-migration-tools/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "name": "@openreader/entrypoint-migration-tools", - "version": "0.0.0", - "private": true, - "dependencies": { - "@aws-sdk/client-s3": "^3.1061.0", - "better-sqlite3": "^12.10.0", - "dotenv": "^17.4.2", - "drizzle-kit": "^0.31.10", - "drizzle-orm": "^0.45.2", - "ffmpeg-static": "^5.3.0", - "pg": "^8.21.0" - } -} diff --git a/docker/examples/compose.local.yml b/docker/examples/compose.local.yml index a50b667..9c0a6d6 100644 --- a/docker/examples/compose.local.yml +++ b/docker/examples/compose.local.yml @@ -103,7 +103,7 @@ services: image: openreader-compute-worker:local build: context: ../.. - dockerfile: compute-worker/Dockerfile + dockerfile: packages/compute-worker/Dockerfile depends_on: nats: condition: service_started diff --git a/docs-site/docs/configure/migrations.md b/docs-site/docs/configure/migrations.md index 43df512..71a51ed 100644 --- a/docs-site/docs/configure/migrations.md +++ b/docs-site/docs/configure/migrations.md @@ -7,6 +7,18 @@ import TabItem from '@theme/TabItem'; This page covers migration behavior for both database schema and storage data in OpenReader. +## Runtime ownership + +- `@openreader/database` owns database clients, schemas, SQL migration files, and programmatic + migration execution for SQLite and PostgreSQL. +- `@openreader/bootstrap` owns startup orchestration, storage migration, and optional embedded + SeaweedFS, NATS, and compute-worker processes. +- The Next.js app imports `@openreader/database` directly, but does not orchestrate migrations or + child processes. + +Docker deploys bootstrap as an isolated runtime bundle under `/opt/openreader/bootstrap`; it does +not merge migration dependencies into the standalone Next.js app under `/app`. + ## Startup migration behavior By default, the shared entrypoint runs migrations automatically before app startup in: @@ -53,11 +65,6 @@ In most cases, you do not need manual migration commands because startup runs mi - Postgres when `POSTGRES_URL` is set - SQLite when `POSTGRES_URL` is unset -You can always override the target explicitly with `--config`. - - - - ```bash # Run pending migrations for one target: # - Postgres if POSTGRES_URL is set @@ -71,26 +78,15 @@ pnpm migrate-fs pnpm migrate-fs:dry-run ``` - - - -```bash -# Migrate SQLite -pnpm exec drizzle-kit migrate --config drizzle.config.sqlite.ts - -# Migrate Postgres -pnpm exec drizzle-kit migrate --config drizzle.config.pg.ts -``` - - - +`pnpm migrate` uses the programmatic Drizzle migrator from `@openreader/database`. Drizzle Kit is +not a production or startup dependency; it is used only to generate new migration files. ## Generate migrations `pnpm generate` is a two-phase script for contributors and schema changes: 1. **Better Auth schema generation** — runs the Better Auth CLI twice (once for SQLite, once for Postgres) to produce auto-generated Drizzle schema files for auth tables (`user`, `session`, `account`, `verification`). -2. **Drizzle migration generation** — runs `drizzle-kit generate` for both `drizzle.config.sqlite.ts` and `drizzle.config.pg.ts`, producing SQL migration files from all schema files (app + auth). +2. **Drizzle migration generation** — runs `drizzle-kit generate` for both configs in `packages/database`, producing SQL migration files from all schema files (app + auth). :::note Most users do not need to run `pnpm generate`. Use it when contributing or when you have changed Drizzle schema files and need new migration files. @@ -100,22 +96,23 @@ Most users do not need to run `pnpm generate`. Use it when contributing or when Auth tables are owned by Better Auth. Their Drizzle schema definitions are auto-generated and should **not** be hand-edited: -- `src/db/schema_auth_sqlite.ts` -- `src/db/schema_auth_postgres.ts` +- `packages/database/src/schema_auth_sqlite.ts` +- `packages/database/src/schema_auth_postgres.ts` App-specific tables are manually maintained in the standard Drizzle schema files: -- `src/db/schema_sqlite.ts` -- `src/db/schema_postgres.ts` +- `packages/database/src/schema_sqlite.ts` +- `packages/database/src/schema_postgres.ts` -Both sets of schema files are included in the Drizzle configs, so `drizzle-kit generate` and `drizzle-kit migrate` handle all tables together. +Both sets of schema files are included in the Drizzle generation configs. Runtime migration +execution is owned by `@openreader/database`. When app schema changes (for example `tts_segments`), keep these in sync: -- `src/db/schema_sqlite.ts` -- `src/db/schema_postgres.ts` -- `drizzle/sqlite/*.sql` + `drizzle/sqlite/meta/_journal.json` -- `drizzle/postgres/*.sql` + `drizzle/postgres/meta/_journal.json` +- `packages/database/src/schema_sqlite.ts` +- `packages/database/src/schema_postgres.ts` +- `packages/database/migrations/sqlite/*.sql` + `packages/database/migrations/sqlite/meta/_journal.json` +- `packages/database/migrations/postgres/*.sql` + `packages/database/migrations/postgres/meta/_journal.json` @@ -130,10 +127,10 @@ pnpm generate ```bash # Generate SQLite migrations only (skips Better Auth CLI) -pnpm exec drizzle-kit generate --config drizzle.config.sqlite.ts +pnpm exec drizzle-kit generate --config packages/database/drizzle.config.sqlite.ts # Generate Postgres migrations only (skips Better Auth CLI) -pnpm exec drizzle-kit generate --config drizzle.config.pg.ts +pnpm exec drizzle-kit generate --config packages/database/drizzle.config.pg.ts ``` :::warning diff --git a/docs-site/docs/deploy/vercel-deployment.md b/docs-site/docs/deploy/vercel-deployment.md index fc634c8..298851c 100644 --- a/docs-site/docs/deploy/vercel-deployment.md +++ b/docs-site/docs/deploy/vercel-deployment.md @@ -106,7 +106,7 @@ For all variables and defaults, see [Environment Variables](../reference/environ ## 4. Database and data migrations -Vercel deployments do not run `scripts/openreader-entrypoint.mjs`, so automatic startup migrations do not run there. +Vercel deployments do not run the `@openreader/bootstrap` process, so automatic startup migrations do not run there. - Run `pnpm migrate` in a controlled environment to apply Drizzle schema migrations to your Postgres DB. - Run `pnpm migrate-fs` only when migrating legacy local filesystem data (`docstore/documents_v1`, `docstore/audiobooks_v1`) into object storage + DB rows. Fresh Vercel deployments usually do not need this. diff --git a/drizzle/scripts/migrate.mjs b/drizzle/scripts/migrate.mjs deleted file mode 100644 index d7c9e8e..0000000 --- a/drizzle/scripts/migrate.mjs +++ /dev/null @@ -1,56 +0,0 @@ -import { spawnSync } from 'node:child_process'; -import path from 'node:path'; -import fs from 'node:fs'; -import * as dotenv from 'dotenv'; - -function loadEnvFiles() { - // Approximate Next.js behavior enough for server-side scripts. - // Load .env first, then .env.local (local overrides). - const cwd = process.cwd(); - const envPath = path.join(cwd, '.env'); - const envLocalPath = path.join(cwd, '.env.local'); - - if (fs.existsSync(envPath)) { - dotenv.config({ path: envPath }); - } - if (fs.existsSync(envLocalPath)) { - dotenv.config({ path: envLocalPath, override: true }); - } -} - -loadEnvFiles(); - -const extraArgs = process.argv.slice(2); - -const hasConfigArg = extraArgs.includes('--config'); -const configFile = process.env.POSTGRES_URL ? 'drizzle.config.pg.ts' : 'drizzle.config.sqlite.ts'; -const configArgs = hasConfigArg ? [] : ['--config', configFile]; - -// Ensure the docstore directory exists for SQLite migrations. -// drizzle-kit opens the database file directly and will fail if the parent -// directory is missing (e.g. in a fresh CI checkout). -if (!process.env.POSTGRES_URL) { - const dbDir = path.join(process.cwd(), 'docstore'); - if (!fs.existsSync(dbDir)) { - fs.mkdirSync(dbDir, { recursive: true }); - } -} - -function resolveDrizzleKitBin() { - const binName = process.platform === 'win32' ? 'drizzle-kit.cmd' : 'drizzle-kit'; - const localBin = path.join(process.cwd(), 'node_modules', '.bin', binName); - if (fs.existsSync(localBin)) return localBin; - return 'drizzle-kit'; -} - -const result = spawnSync(resolveDrizzleKitBin(), ['migrate', ...configArgs, ...extraArgs], { - stdio: 'inherit', - env: process.env, -}); - -if (result.error) { - console.error(result.error.message); - process.exit(1); -} - -process.exit(result.status ?? 1); diff --git a/next.config.ts b/next.config.ts index a5908d7..4417b46 100644 --- a/next.config.ts +++ b/next.config.ts @@ -46,7 +46,7 @@ const nextConfig: NextConfig = { canvas: '@napi-rs/canvas', }, }, - transpilePackages: [], + transpilePackages: ['@openreader/database'], serverExternalPackages, outputFileTracingIncludes: { '/api/audiobook': [ diff --git a/package.json b/package.json index 37ad06a..9c9b62d 100644 --- a/package.json +++ b/package.json @@ -4,12 +4,12 @@ "private": true, "packageManager": "pnpm@10.33.4", "scripts": { - "dev": "node scripts/openreader-entrypoint.mjs -- pnpm dev:raw", + "dev": "pnpm --filter @openreader/bootstrap start pnpm dev:raw", "dev:raw": "next dev --turbopack -p 3003", "build": "next build", "build:bundle-guard": "node scripts/check-next-server-bundle.mjs", "check:compute-boundary": "node scripts/check-compute-boundary.mjs", - "start": "node scripts/openreader-entrypoint.mjs -- pnpm start:raw", + "start": "pnpm --filter @openreader/bootstrap start pnpm start:raw", "start:raw": "next start -p 3003", "lint": "next lint", "test": "playwright test", @@ -19,10 +19,10 @@ "compose": "docker compose -f docker/examples/compose.yml up", "compose:full": "docker compose -f docker/examples/compose.full.yml up", "compose:local": "docker compose -f docker/examples/compose.local.yml up --build", - "migrate": "node drizzle/scripts/migrate.mjs", - "migrate-fs": "node scripts/migrate-fs-v2.mjs", - "migrate-fs:dry-run": "node scripts/migrate-fs-v2.mjs --dry-run true", - "generate": "node drizzle/scripts/generate.mjs", + "migrate": "pnpm --filter @openreader/database migrate", + "migrate-fs": "pnpm --filter @openreader/bootstrap migrate-storage", + "migrate-fs:dry-run": "pnpm --filter @openreader/bootstrap migrate-storage --dry-run true", + "generate": "node packages/database/scripts/generate.mjs", "docs:init": "pnpm --dir docs-site install", "docs:dev": "pnpm --dir docs-site start", "docs:build": "pnpm --dir docs-site build", @@ -31,8 +31,8 @@ "docs:clear": "pnpm --dir docs-site clear", "compute:dev": "pnpm --filter @openreader/compute-worker dev", "compute:start": "pnpm --filter @openreader/compute-worker start", - "compute:openapi:generate": "pnpm --dir compute-worker openapi:generate && openapi-typescript compute-worker/openapi.json -o src/lib/server/compute-worker/generated.ts", - "compute:openapi:check": "pnpm compute:openapi:generate && git diff --exit-code -- compute-worker/openapi.json src/lib/server/compute-worker/generated.ts", + "compute:openapi:generate": "pnpm --dir packages/compute-worker openapi:generate && openapi-typescript packages/compute-worker/openapi.json -o src/lib/server/compute-worker/generated.ts", + "compute:openapi:check": "pnpm compute:openapi:generate && git diff --exit-code -- packages/compute-worker/openapi.json src/lib/server/compute-worker/generated.ts", "lint:route-errors": "node scripts/check-route-error-responses.mjs" }, "dependencies": { @@ -42,20 +42,19 @@ "@microsoft/antissrf": "^1.0.0", "@mozilla/readability": "^0.6.0", "@napi-rs/canvas": "^0.1.100", + "@openreader/database": "workspace:*", "@speech-sdk/core": "^0.15.0", "@tanstack/react-query": "^5.101.0", "@types/archiver": "^7.0.0", "@vercel/analytics": "^1.6.1", "archiver": "^7.0.1", "better-auth": "^1.6.14", - "better-sqlite3": "^12.10.0", "cmpstr": "^3.3.0", "compromise": "^14.15.1", "core-js": "^3.49.0", "dexie": "^4.4.3", "dexie-react-hooks": "^4.4.0", "dotenv": "^17.4.2", - "drizzle-kit": "^0.31.10", "drizzle-orm": "^0.45.2", "epubjs": "^0.3.93", "fast-xml-parser": "^5.8.0", @@ -67,7 +66,6 @@ "next": "^15.5.19", "openai": "^6.41.0", "pdfjs-dist": "4.8.69", - "pg": "^8.21.0", "pino": "^10.3.1", "pino-pretty": "^13.1.3", "react": "^19.2.7", @@ -98,6 +96,8 @@ "@types/react-dom": "^19.2.3", "@types/turndown": "^5.0.6", "@types/uuid": "^10.0.0", + "better-sqlite3": "^12.10.0", + "drizzle-kit": "^0.31.10", "eslint": "^9.39.4", "eslint-config-next": "^15.5.19", "postcss": "^8.5.15", diff --git a/packages/bootstrap/package.json b/packages/bootstrap/package.json new file mode 100644 index 0000000..0f7a8fa --- /dev/null +++ b/packages/bootstrap/package.json @@ -0,0 +1,25 @@ +{ + "name": "@openreader/bootstrap", + "version": "0.0.0", + "private": true, + "type": "module", + "files": [ + "src" + ], + "bin": { + "openreader": "./src/cli.mjs", + "openreader-migrate-storage": "./src/storage-migration.mjs" + }, + "scripts": { + "start": "node src/cli.mjs --", + "migrate-storage": "node src/storage-migration.mjs" + }, + "dependencies": { + "@aws-sdk/client-s3": "^3.1061.0", + "@openreader/database": "workspace:*", + "better-sqlite3": "^12.10.0", + "dotenv": "^17.4.2", + "ffmpeg-static": "^5.3.0", + "pg": "^8.21.0" + } +} diff --git a/scripts/openreader-entrypoint.mjs b/packages/bootstrap/src/cli.mjs similarity index 77% rename from scripts/openreader-entrypoint.mjs rename to packages/bootstrap/src/cli.mjs index 957ad63..887ecd7 100644 --- a/scripts/openreader-entrypoint.mjs +++ b/packages/bootstrap/src/cli.mjs @@ -3,11 +3,23 @@ import { spawn, spawnSync } from 'node:child_process'; import { randomBytes } from 'node:crypto'; import { once } from 'node:events'; import fs from 'node:fs'; -import os from 'node:os'; import path from 'node:path'; import process from 'node:process'; import { setTimeout as delay } from 'node:timers/promises'; +import { fileURLToPath } from 'node:url'; import * as dotenv from 'dotenv'; +import { runMigrations } from '@openreader/database/migrate'; +import { hasNatsBinary } from './embedded-nats.mjs'; +import { + detectHostForDefaultEndpoint, + hasWeedBinary, + isRunningInDocker, + loopbackS3Endpoint, + parseS3Endpoint, + parseUrlHost, + waitForEndpoint, +} from './embedded-seaweedfs.mjs'; +import { resolveEmbeddedWorkerLaunch } from './embedded-worker.mjs'; function loadEnvFiles() { const cwd = process.cwd(); @@ -49,83 +61,6 @@ function requireAuthEnv(env) { } } -function isPrivateIPv4(address) { - if (!address) return false; - if (address.startsWith('10.')) return true; - if (address.startsWith('192.168.')) return true; - const m = /^172\.(\d+)\./.exec(address); - if (m) { - const second = Number.parseInt(m[1], 10); - if (second >= 16 && second <= 31) return true; - } - return false; -} - -function detectHostForDefaultEndpoint() { - const interfaces = os.networkInterfaces(); - const ipv4 = []; - - for (const entries of Object.values(interfaces)) { - for (const entry of entries || []) { - if (!entry) continue; - const family = typeof entry.family === 'string' ? entry.family : String(entry.family); - if (family !== 'IPv4') continue; - if (entry.internal) continue; - ipv4.push(entry.address); - } - } - - const privateAddr = ipv4.find(isPrivateIPv4); - if (privateAddr) return privateAddr; - if (ipv4[0]) return ipv4[0]; - return '127.0.0.1'; -} - -function parseS3Endpoint(endpoint) { - let url; - try { - url = new URL(endpoint); - } catch { - throw new Error(`Invalid S3_ENDPOINT: ${endpoint}`); - } - - if (!url.hostname) { - throw new Error(`Invalid S3_ENDPOINT host: ${endpoint}`); - } - - const port = Number.parseInt(url.port || '8333', 10); - if (!Number.isFinite(port) || port < 1 || port > 65535) { - throw new Error(`Invalid S3_ENDPOINT port: ${endpoint}`); - } - - return { - hostname: url.hostname, - port, - normalized: `${url.protocol}//${url.hostname}:${port}`, - }; -} - -function loopbackS3Endpoint(endpoint) { - const parsed = parseS3Endpoint(endpoint); - const url = new URL(parsed.normalized); - return `${url.protocol}//127.0.0.1:${parsed.port}`; -} - -function parseUrlHost(urlValue, fieldName) { - let url; - try { - url = new URL(urlValue); - } catch { - throw new Error(`Invalid ${fieldName}: ${urlValue}`); - } - - if (!url.hostname) { - throw new Error(`Invalid ${fieldName} host: ${urlValue}`); - } - - return url.hostname; -} - function parseCommandFromArgs(argv) { const marker = argv.indexOf('--'); if (marker >= 0) return argv.slice(marker + 1); @@ -143,51 +78,6 @@ function forwardChildStream(stream, target) { }; } -function hasWeedBinary() { - const probe = spawnSync('weed', ['version'], { stdio: 'ignore' }); - if (probe.error) return false; - return true; -} - -function hasNatsBinary() { - const probe = spawnSync('nats-server', ['-v'], { stdio: 'ignore' }); - if (probe.error) return false; - return true; -} - -async function waitForEndpoint(url, timeoutSeconds) { - const waitMs = Math.max(1, timeoutSeconds) * 1000; - const deadline = Date.now() + waitMs; - - while (Date.now() < deadline) { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), 2000); - - try { - const res = await fetch(url, { method: 'GET', signal: controller.signal }); - if (res) return; - } catch { - // retry - } finally { - clearTimeout(timeoutId); - } - await delay(1000); - } - - throw new Error(`Embedded weed mini did not become ready at ${url} within ${timeoutSeconds}s.`); -} - -function isRunningInDocker() { - if (process.platform !== 'linux') return false; - if (fs.existsSync('/.dockerenv')) return true; - try { - const cgroup = fs.readFileSync('/proc/1/cgroup', 'utf8'); - return /(docker|containerd|kubepods|podman)/i.test(cgroup); - } catch { - return false; - } -} - function spawnMainCommand(command, env) { const [cmd, ...args] = command; const child = spawn(cmd, args, { @@ -219,53 +109,13 @@ function spawnMainCommand(command, env) { return { child, exitPromise }; } -function resolveEmbeddedWorkerLaunch() { - const candidateDirs = [ - path.join(process.cwd(), 'embedded-compute-worker'), - path.join(process.cwd(), 'compute-worker'), - ]; - - for (const workerDir of candidateDirs) { - const serverEntry = path.join(workerDir, 'src', 'server.ts'); - if (!fs.existsSync(serverEntry)) continue; - return { - cmd: process.execPath, - args: ['--import', 'tsx', 'src/server.ts'], - cwd: workerDir, - }; - } - - throw new Error( - 'Could not find an embedded compute worker runtime. ' - + 'Include embedded-compute-worker/src/server.ts in the runtime image or keep compute-worker available locally.', - ); -} - -function runDbMigrations(env) { - const migrateScript = path.join(process.cwd(), 'drizzle', 'scripts', 'migrate.mjs'); - if (!fs.existsSync(migrateScript)) { - throw new Error(`Could not find migration script at ${migrateScript}`); - } - +async function runDbMigrations(env) { console.log('Running database migrations...'); - const migration = spawnSync(process.execPath, [migrateScript], { - env, - stdio: 'inherit', - }); - - if (migration.error) { - throw migration.error; - } - if (typeof migration.status === 'number' && migration.status !== 0) { - throw new Error(`Database migrations failed with exit code ${migration.status}.`); - } + await runMigrations({ cwd: process.cwd(), env }); } function runStorageMigrations(env) { - const migrateScript = path.join(process.cwd(), 'scripts', 'migrate-fs-v2.mjs'); - if (!fs.existsSync(migrateScript)) { - throw new Error(`Could not find storage migration script at ${migrateScript}`); - } + const migrateScript = fileURLToPath(new URL('./storage-migration.mjs', import.meta.url)); console.log('Running storage migrations (v2)...'); const migration = spawnSync(process.execPath, [migrateScript, '--dry-run', 'false', '--delete-local', 'false'], { @@ -335,7 +185,7 @@ async function main() { const command = parseCommandFromArgs(process.argv.slice(2)); if (command.length === 0) { - console.error('Usage: node scripts/openreader-entrypoint.mjs -- [args]'); + console.error('Usage: openreader -- [args]'); process.exit(2); } @@ -424,7 +274,7 @@ async function main() { try { const shouldRunDbMigrations = resolveBooleanEnv(runtimeEnv, 'RUN_DRIZZLE_MIGRATIONS', true); if (shouldRunDbMigrations) { - runDbMigrations(runtimeEnv); + await runDbMigrations(runtimeEnv); } if (useEmbeddedWeed) { @@ -482,7 +332,7 @@ async function main() { console.log('Starting embedded SeaweedFS weed mini...'); launchWeed(runtimeEnv.S3_ENDPOINT); const startupEndpoint = parseS3Endpoint(runtimeEnv.S3_ENDPOINT); - await waitForEndpoint(`http://127.0.0.1:${startupEndpoint.port}`, waitTimeout); + await waitForEndpoint(`http://127.0.0.1:${startupEndpoint.port}`, waitTimeout, 'Embedded SeaweedFS'); console.log(`Embedded SeaweedFS is ready at ${runtimeEnv.S3_ENDPOINT}`); } @@ -555,7 +405,7 @@ async function main() { console.error(`Embedded nats-server failed to start: ${error instanceof Error ? error.message : String(error)}`); scheduleFatalShutdown('nats-server', null, null, 'failed to start'); }); - await waitForEndpoint(`http://127.0.0.1:${embeddedNatsMonitorPort}/healthz`, 20); + await waitForEndpoint(`http://127.0.0.1:${embeddedNatsMonitorPort}/healthz`, 20, 'Embedded nats-server'); console.log(`Embedded nats-server is ready at nats://127.0.0.1:${embeddedNatsPort}`); console.log(`Starting embedded compute-worker on 127.0.0.1:${embeddedWorkerPort}...`); @@ -589,7 +439,7 @@ async function main() { console.error(`Embedded compute-worker failed to start: ${error instanceof Error ? error.message : String(error)}`); scheduleFatalShutdown('compute-worker', null, null, 'failed to start'); }); - await waitForEndpoint(`http://127.0.0.1:${embeddedWorkerPort}/health/ready`, 30); + await waitForEndpoint(`http://127.0.0.1:${embeddedWorkerPort}/health/ready`, 30, 'Embedded compute-worker'); console.log(`Embedded compute-worker is ready at http://127.0.0.1:${embeddedWorkerPort}`); } else if (!runtimeEnv.COMPUTE_WORKER_URL?.trim() || !runtimeEnv.COMPUTE_WORKER_TOKEN?.trim()) { throw new Error('COMPUTE_WORKER_URL and COMPUTE_WORKER_TOKEN are required when embedded compute worker startup is disabled.'); diff --git a/packages/bootstrap/src/embedded-nats.mjs b/packages/bootstrap/src/embedded-nats.mjs new file mode 100644 index 0000000..d9d1238 --- /dev/null +++ b/packages/bootstrap/src/embedded-nats.mjs @@ -0,0 +1,5 @@ +import { spawnSync } from 'node:child_process'; + +export function hasNatsBinary() { + return !spawnSync('nats-server', ['-v'], { stdio: 'ignore' }).error; +} diff --git a/packages/bootstrap/src/embedded-seaweedfs.mjs b/packages/bootstrap/src/embedded-seaweedfs.mjs new file mode 100644 index 0000000..4fc40d8 --- /dev/null +++ b/packages/bootstrap/src/embedded-seaweedfs.mjs @@ -0,0 +1,95 @@ +import { spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import process from 'node:process'; +import { setTimeout as delay } from 'node:timers/promises'; + +function isPrivateIPv4(address) { + if (!address) return false; + if (address.startsWith('10.')) return true; + if (address.startsWith('192.168.')) return true; + const match = /^172\.(\d+)\./.exec(address); + if (!match) return false; + const second = Number.parseInt(match[1], 10); + return second >= 16 && second <= 31; +} + +export function hasWeedBinary() { + return !spawnSync('weed', ['version'], { stdio: 'ignore' }).error; +} + +export function detectHostForDefaultEndpoint() { + const ipv4 = Object.values(os.networkInterfaces()) + .flatMap((entries) => entries || []) + .filter((entry) => entry && !entry.internal && String(entry.family) === 'IPv4') + .map((entry) => entry.address); + + return ipv4.find(isPrivateIPv4) || ipv4[0] || '127.0.0.1'; +} + +export function parseS3Endpoint(endpoint) { + let url; + try { + url = new URL(endpoint); + } catch { + throw new Error(`Invalid S3_ENDPOINT: ${endpoint}`); + } + + if (!url.hostname) throw new Error(`Invalid S3_ENDPOINT host: ${endpoint}`); + const port = Number.parseInt(url.port || '8333', 10); + if (!Number.isFinite(port) || port < 1 || port > 65535) { + throw new Error(`Invalid S3_ENDPOINT port: ${endpoint}`); + } + + return { + hostname: url.hostname, + port, + normalized: `${url.protocol}//${url.hostname}:${port}`, + }; +} + +export function loopbackS3Endpoint(endpoint) { + const parsed = parseS3Endpoint(endpoint); + const url = new URL(parsed.normalized); + return `${url.protocol}//127.0.0.1:${parsed.port}`; +} + +export function parseUrlHost(urlValue, fieldName) { + let url; + try { + url = new URL(urlValue); + } catch { + throw new Error(`Invalid ${fieldName}: ${urlValue}`); + } + + if (!url.hostname) throw new Error(`Invalid ${fieldName} host: ${urlValue}`); + return url.hostname; +} + +export async function waitForEndpoint(url, timeoutSeconds, serviceName = 'service') { + const deadline = Date.now() + Math.max(1, timeoutSeconds) * 1000; + while (Date.now() < deadline) { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), 2000); + try { + const response = await fetch(url, { method: 'GET', signal: controller.signal }); + if (response) return; + } catch { + // Retry until the startup deadline. + } finally { + clearTimeout(timeoutId); + } + await delay(1000); + } + throw new Error(`${serviceName} did not become ready at ${url} within ${timeoutSeconds}s.`); +} + +export function isRunningInDocker() { + if (process.platform !== 'linux') return false; + if (fs.existsSync('/.dockerenv')) return true; + try { + return /(docker|containerd|kubepods|podman)/i.test(fs.readFileSync('/proc/1/cgroup', 'utf8')); + } catch { + return false; + } +} diff --git a/packages/bootstrap/src/embedded-worker.mjs b/packages/bootstrap/src/embedded-worker.mjs new file mode 100644 index 0000000..f4ea4b6 --- /dev/null +++ b/packages/bootstrap/src/embedded-worker.mjs @@ -0,0 +1,30 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const bootstrapDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +export function resolveEmbeddedWorkerLaunch() { + const candidateDirs = [ + path.resolve(bootstrapDir, '..', 'embedded-compute-worker'), + path.resolve(bootstrapDir, '..', 'compute-worker'), + path.join(process.cwd(), 'embedded-compute-worker'), + path.join(process.cwd(), 'packages', 'compute-worker'), + path.join(process.cwd(), 'compute-worker'), + ]; + + for (const workerDir of candidateDirs) { + if (!fs.existsSync(path.join(workerDir, 'src', 'server.ts'))) continue; + return { + cmd: process.execPath, + args: ['--import', 'tsx', 'src/server.ts'], + cwd: workerDir, + }; + } + + throw new Error( + 'Could not find an embedded compute worker runtime. ' + + 'Include embedded-compute-worker/src/server.ts in the runtime image or keep packages/compute-worker available locally.', + ); +} diff --git a/scripts/migrate-fs-v2.mjs b/packages/bootstrap/src/storage-migration.mjs similarity index 100% rename from scripts/migrate-fs-v2.mjs rename to packages/bootstrap/src/storage-migration.mjs diff --git a/compute-worker/.env.example b/packages/compute-worker/.env.example similarity index 100% rename from compute-worker/.env.example rename to packages/compute-worker/.env.example diff --git a/compute-worker/Dockerfile b/packages/compute-worker/Dockerfile similarity index 72% rename from compute-worker/Dockerfile rename to packages/compute-worker/Dockerfile index 356190f..bcec9ac 100644 --- a/compute-worker/Dockerfile +++ b/packages/compute-worker/Dockerfile @@ -1,19 +1,17 @@ FROM node:lts AS deploy-stage -RUN npm install -g pnpm@11.1.2 +RUN npm install -g pnpm@10.33.4 WORKDIR /workspace COPY .npmrc package.json pnpm-lock.yaml pnpm-workspace.yaml ./ -COPY compute-worker/package.json compute-worker/package.json -COPY compute-worker compute-worker +COPY packages/compute-worker/package.json packages/compute-worker/package.json +COPY packages/compute-worker packages/compute-worker RUN pnpm --config.inject-workspace-packages=true --filter @openreader/compute-worker deploy /opt/compute-worker FROM node:lts -RUN npm install -g pnpm@11.1.2 - WORKDIR /workspace COPY --from=deploy-stage /opt/compute-worker ./ diff --git a/compute-worker/openapi.json b/packages/compute-worker/openapi.json similarity index 100% rename from compute-worker/openapi.json rename to packages/compute-worker/openapi.json diff --git a/compute-worker/package.json b/packages/compute-worker/package.json similarity index 100% rename from compute-worker/package.json rename to packages/compute-worker/package.json diff --git a/compute-worker/scripts/generate-openapi.ts b/packages/compute-worker/scripts/generate-openapi.ts similarity index 100% rename from compute-worker/scripts/generate-openapi.ts rename to packages/compute-worker/scripts/generate-openapi.ts diff --git a/compute-worker/src/api/app.ts b/packages/compute-worker/src/api/app.ts similarity index 100% rename from compute-worker/src/api/app.ts rename to packages/compute-worker/src/api/app.ts diff --git a/compute-worker/src/api/compute-operation.ts b/packages/compute-worker/src/api/compute-operation.ts similarity index 100% rename from compute-worker/src/api/compute-operation.ts rename to packages/compute-worker/src/api/compute-operation.ts diff --git a/compute-worker/src/api/http-hooks.ts b/packages/compute-worker/src/api/http-hooks.ts similarity index 100% rename from compute-worker/src/api/http-hooks.ts rename to packages/compute-worker/src/api/http-hooks.ts diff --git a/compute-worker/src/api/routes.ts b/packages/compute-worker/src/api/routes.ts similarity index 100% rename from compute-worker/src/api/routes.ts rename to packages/compute-worker/src/api/routes.ts diff --git a/compute-worker/src/api/schemas.ts b/packages/compute-worker/src/api/schemas.ts similarity index 100% rename from compute-worker/src/api/schemas.ts rename to packages/compute-worker/src/api/schemas.ts diff --git a/compute-worker/src/api/types.ts b/packages/compute-worker/src/api/types.ts similarity index 100% rename from compute-worker/src/api/types.ts rename to packages/compute-worker/src/api/types.ts diff --git a/compute-worker/src/inference/pdf/assets/LICENSE.txt b/packages/compute-worker/src/inference/pdf/assets/LICENSE.txt similarity index 100% rename from compute-worker/src/inference/pdf/assets/LICENSE.txt rename to packages/compute-worker/src/inference/pdf/assets/LICENSE.txt diff --git a/compute-worker/src/inference/pdf/assets/manifest.json b/packages/compute-worker/src/inference/pdf/assets/manifest.json similarity index 100% rename from compute-worker/src/inference/pdf/assets/manifest.json rename to packages/compute-worker/src/inference/pdf/assets/manifest.json diff --git a/compute-worker/src/inference/pdf/document-layout.ts b/packages/compute-worker/src/inference/pdf/document-layout.ts similarity index 100% rename from compute-worker/src/inference/pdf/document-layout.ts rename to packages/compute-worker/src/inference/pdf/document-layout.ts diff --git a/compute-worker/src/inference/pdf/layout-model.ts b/packages/compute-worker/src/inference/pdf/layout-model.ts similarity index 100% rename from compute-worker/src/inference/pdf/layout-model.ts rename to packages/compute-worker/src/inference/pdf/layout-model.ts diff --git a/compute-worker/src/inference/pdf/model.ts b/packages/compute-worker/src/inference/pdf/model.ts similarity index 100% rename from compute-worker/src/inference/pdf/model.ts rename to packages/compute-worker/src/inference/pdf/model.ts diff --git a/compute-worker/src/inference/pdf/normalize-text.ts b/packages/compute-worker/src/inference/pdf/normalize-text.ts similarity index 100% rename from compute-worker/src/inference/pdf/normalize-text.ts rename to packages/compute-worker/src/inference/pdf/normalize-text.ts diff --git a/compute-worker/src/inference/pdf/parse.ts b/packages/compute-worker/src/inference/pdf/parse.ts similarity index 100% rename from compute-worker/src/inference/pdf/parse.ts rename to packages/compute-worker/src/inference/pdf/parse.ts diff --git a/compute-worker/src/inference/pdf/pdfjs.ts b/packages/compute-worker/src/inference/pdf/pdfjs.ts similarity index 100% rename from compute-worker/src/inference/pdf/pdfjs.ts rename to packages/compute-worker/src/inference/pdf/pdfjs.ts diff --git a/compute-worker/src/inference/pdf/render.ts b/packages/compute-worker/src/inference/pdf/render.ts similarity index 100% rename from compute-worker/src/inference/pdf/render.ts rename to packages/compute-worker/src/inference/pdf/render.ts diff --git a/compute-worker/src/inference/pdf/stitch.ts b/packages/compute-worker/src/inference/pdf/stitch.ts similarity index 100% rename from compute-worker/src/inference/pdf/stitch.ts rename to packages/compute-worker/src/inference/pdf/stitch.ts diff --git a/compute-worker/src/inference/runtime.ts b/packages/compute-worker/src/inference/runtime.ts similarity index 100% rename from compute-worker/src/inference/runtime.ts rename to packages/compute-worker/src/inference/runtime.ts diff --git a/compute-worker/src/inference/whisper/align.ts b/packages/compute-worker/src/inference/whisper/align.ts similarity index 100% rename from compute-worker/src/inference/whisper/align.ts rename to packages/compute-worker/src/inference/whisper/align.ts diff --git a/compute-worker/src/inference/whisper/assets/LICENSE.txt b/packages/compute-worker/src/inference/whisper/assets/LICENSE.txt similarity index 100% rename from compute-worker/src/inference/whisper/assets/LICENSE.txt rename to packages/compute-worker/src/inference/whisper/assets/LICENSE.txt diff --git a/compute-worker/src/inference/whisper/assets/manifest.json b/packages/compute-worker/src/inference/whisper/assets/manifest.json similarity index 100% rename from compute-worker/src/inference/whisper/assets/manifest.json rename to packages/compute-worker/src/inference/whisper/assets/manifest.json diff --git a/compute-worker/src/inference/whisper/assets/mel_filters.npz b/packages/compute-worker/src/inference/whisper/assets/mel_filters.npz similarity index 100% rename from compute-worker/src/inference/whisper/assets/mel_filters.npz rename to packages/compute-worker/src/inference/whisper/assets/mel_filters.npz diff --git a/compute-worker/src/inference/whisper/decoder.ts b/packages/compute-worker/src/inference/whisper/decoder.ts similarity index 100% rename from compute-worker/src/inference/whisper/decoder.ts rename to packages/compute-worker/src/inference/whisper/decoder.ts diff --git a/compute-worker/src/inference/whisper/model.ts b/packages/compute-worker/src/inference/whisper/model.ts similarity index 100% rename from compute-worker/src/inference/whisper/model.ts rename to packages/compute-worker/src/inference/whisper/model.ts diff --git a/compute-worker/src/inference/whisper/timestamps.ts b/packages/compute-worker/src/inference/whisper/timestamps.ts similarity index 100% rename from compute-worker/src/inference/whisper/timestamps.ts rename to packages/compute-worker/src/inference/whisper/timestamps.ts diff --git a/compute-worker/src/infrastructure/config.ts b/packages/compute-worker/src/infrastructure/config.ts similarity index 100% rename from compute-worker/src/infrastructure/config.ts rename to packages/compute-worker/src/infrastructure/config.ts diff --git a/compute-worker/src/infrastructure/json-codec.ts b/packages/compute-worker/src/infrastructure/json-codec.ts similarity index 100% rename from compute-worker/src/infrastructure/json-codec.ts rename to packages/compute-worker/src/infrastructure/json-codec.ts diff --git a/compute-worker/src/infrastructure/nats-adapters.ts b/packages/compute-worker/src/infrastructure/nats-adapters.ts similarity index 100% rename from compute-worker/src/infrastructure/nats-adapters.ts rename to packages/compute-worker/src/infrastructure/nats-adapters.ts diff --git a/compute-worker/src/infrastructure/nats-session.ts b/packages/compute-worker/src/infrastructure/nats-session.ts similarity index 100% rename from compute-worker/src/infrastructure/nats-session.ts rename to packages/compute-worker/src/infrastructure/nats-session.ts diff --git a/compute-worker/src/infrastructure/nats.ts b/packages/compute-worker/src/infrastructure/nats.ts similarity index 100% rename from compute-worker/src/infrastructure/nats.ts rename to packages/compute-worker/src/infrastructure/nats.ts diff --git a/compute-worker/src/infrastructure/platform.ts b/packages/compute-worker/src/infrastructure/platform.ts similarity index 100% rename from compute-worker/src/infrastructure/platform.ts rename to packages/compute-worker/src/infrastructure/platform.ts diff --git a/compute-worker/src/infrastructure/storage.ts b/packages/compute-worker/src/infrastructure/storage.ts similarity index 100% rename from compute-worker/src/infrastructure/storage.ts rename to packages/compute-worker/src/infrastructure/storage.ts diff --git a/compute-worker/src/jobs/handlers.ts b/packages/compute-worker/src/jobs/handlers.ts similarity index 100% rename from compute-worker/src/jobs/handlers.ts rename to packages/compute-worker/src/jobs/handlers.ts diff --git a/compute-worker/src/jobs/pdf-artifact-persistence.ts b/packages/compute-worker/src/jobs/pdf-artifact-persistence.ts similarity index 100% rename from compute-worker/src/jobs/pdf-artifact-persistence.ts rename to packages/compute-worker/src/jobs/pdf-artifact-persistence.ts diff --git a/compute-worker/src/jobs/pdf-progress.ts b/packages/compute-worker/src/jobs/pdf-progress.ts similarity index 100% rename from compute-worker/src/jobs/pdf-progress.ts rename to packages/compute-worker/src/jobs/pdf-progress.ts diff --git a/compute-worker/src/jobs/worker-loop-policy.ts b/packages/compute-worker/src/jobs/worker-loop-policy.ts similarity index 100% rename from compute-worker/src/jobs/worker-loop-policy.ts rename to packages/compute-worker/src/jobs/worker-loop-policy.ts diff --git a/compute-worker/src/jobs/worker-loop.ts b/packages/compute-worker/src/jobs/worker-loop.ts similarity index 100% rename from compute-worker/src/jobs/worker-loop.ts rename to packages/compute-worker/src/jobs/worker-loop.ts diff --git a/compute-worker/src/operations/contracts.ts b/packages/compute-worker/src/operations/contracts.ts similarity index 100% rename from compute-worker/src/operations/contracts.ts rename to packages/compute-worker/src/operations/contracts.ts diff --git a/compute-worker/src/operations/index.ts b/packages/compute-worker/src/operations/index.ts similarity index 100% rename from compute-worker/src/operations/index.ts rename to packages/compute-worker/src/operations/index.ts diff --git a/compute-worker/src/operations/keys.ts b/packages/compute-worker/src/operations/keys.ts similarity index 100% rename from compute-worker/src/operations/keys.ts rename to packages/compute-worker/src/operations/keys.ts diff --git a/compute-worker/src/operations/reconciliation.ts b/packages/compute-worker/src/operations/reconciliation.ts similarity index 100% rename from compute-worker/src/operations/reconciliation.ts rename to packages/compute-worker/src/operations/reconciliation.ts diff --git a/compute-worker/src/operations/recovery.ts b/packages/compute-worker/src/operations/recovery.ts similarity index 100% rename from compute-worker/src/operations/recovery.ts rename to packages/compute-worker/src/operations/recovery.ts diff --git a/compute-worker/src/operations/service.ts b/packages/compute-worker/src/operations/service.ts similarity index 100% rename from compute-worker/src/operations/service.ts rename to packages/compute-worker/src/operations/service.ts diff --git a/compute-worker/src/operations/sse.ts b/packages/compute-worker/src/operations/sse.ts similarity index 100% rename from compute-worker/src/operations/sse.ts rename to packages/compute-worker/src/operations/sse.ts diff --git a/compute-worker/src/operations/state-machine.ts b/packages/compute-worker/src/operations/state-machine.ts similarity index 100% rename from compute-worker/src/operations/state-machine.ts rename to packages/compute-worker/src/operations/state-machine.ts diff --git a/compute-worker/src/operations/types.ts b/packages/compute-worker/src/operations/types.ts similarity index 100% rename from compute-worker/src/operations/types.ts rename to packages/compute-worker/src/operations/types.ts diff --git a/compute-worker/src/server.ts b/packages/compute-worker/src/server.ts similarity index 100% rename from compute-worker/src/server.ts rename to packages/compute-worker/src/server.ts diff --git a/compute-worker/src/storage/artifact-addressing.ts b/packages/compute-worker/src/storage/artifact-addressing.ts similarity index 100% rename from compute-worker/src/storage/artifact-addressing.ts rename to packages/compute-worker/src/storage/artifact-addressing.ts diff --git a/compute-worker/tests/api/routes.test.ts b/packages/compute-worker/tests/api/routes.test.ts similarity index 100% rename from compute-worker/tests/api/routes.test.ts rename to packages/compute-worker/tests/api/routes.test.ts diff --git a/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts b/packages/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts similarity index 100% rename from compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts rename to packages/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts diff --git a/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts b/packages/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts similarity index 100% rename from compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts rename to packages/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts diff --git a/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts b/packages/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts similarity index 100% rename from compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts rename to packages/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts diff --git a/compute-worker/tests/compute/algorithms/support/document-fixtures.ts b/packages/compute-worker/tests/compute/algorithms/support/document-fixtures.ts similarity index 93% rename from compute-worker/tests/compute/algorithms/support/document-fixtures.ts rename to packages/compute-worker/tests/compute/algorithms/support/document-fixtures.ts index 5a3d697..4d8d23d 100644 --- a/compute-worker/tests/compute/algorithms/support/document-fixtures.ts +++ b/packages/compute-worker/tests/compute/algorithms/support/document-fixtures.ts @@ -1,4 +1,4 @@ -import type { BaseDocument } from '../../../../../src/types/documents'; +import type { BaseDocument } from '../../../../../../src/types/documents'; import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/api/types'; export function makeBaseDocument(overrides: Partial = {}): BaseDocument { diff --git a/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts b/packages/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts similarity index 100% rename from compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts rename to packages/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts diff --git a/compute-worker/tests/compute/algorithms/whisper-decoder.test.ts b/packages/compute-worker/tests/compute/algorithms/whisper-decoder.test.ts similarity index 100% rename from compute-worker/tests/compute/algorithms/whisper-decoder.test.ts rename to packages/compute-worker/tests/compute/algorithms/whisper-decoder.test.ts diff --git a/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts b/packages/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts similarity index 100% rename from compute-worker/tests/compute/algorithms/whisper-spectral.test.ts rename to packages/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts diff --git a/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts b/packages/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts similarity index 100% rename from compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts rename to packages/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts diff --git a/compute-worker/tests/compute/control-plane/orchestrator.test.ts b/packages/compute-worker/tests/compute/control-plane/orchestrator.test.ts similarity index 100% rename from compute-worker/tests/compute/control-plane/orchestrator.test.ts rename to packages/compute-worker/tests/compute/control-plane/orchestrator.test.ts diff --git a/compute-worker/tests/compute/control-plane/run-layout-model.test.ts b/packages/compute-worker/tests/compute/control-plane/run-layout-model.test.ts similarity index 100% rename from compute-worker/tests/compute/control-plane/run-layout-model.test.ts rename to packages/compute-worker/tests/compute/control-plane/run-layout-model.test.ts diff --git a/compute-worker/tests/compute/control-plane/sse.test.ts b/packages/compute-worker/tests/compute/control-plane/sse.test.ts similarity index 100% rename from compute-worker/tests/compute/control-plane/sse.test.ts rename to packages/compute-worker/tests/compute/control-plane/sse.test.ts diff --git a/compute-worker/tests/compute/control-plane/state-machine.test.ts b/packages/compute-worker/tests/compute/control-plane/state-machine.test.ts similarity index 100% rename from compute-worker/tests/compute/control-plane/state-machine.test.ts rename to packages/compute-worker/tests/compute/control-plane/state-machine.test.ts diff --git a/compute-worker/tests/compute/helpers/in-memory-control-plane.ts b/packages/compute-worker/tests/compute/helpers/in-memory-control-plane.ts similarity index 100% rename from compute-worker/tests/compute/helpers/in-memory-control-plane.ts rename to packages/compute-worker/tests/compute/helpers/in-memory-control-plane.ts diff --git a/compute-worker/tests/fixtures/fake-control-plane.ts b/packages/compute-worker/tests/fixtures/fake-control-plane.ts similarity index 100% rename from compute-worker/tests/fixtures/fake-control-plane.ts rename to packages/compute-worker/tests/fixtures/fake-control-plane.ts diff --git a/compute-worker/tests/setup-env.ts b/packages/compute-worker/tests/setup-env.ts similarity index 100% rename from compute-worker/tests/setup-env.ts rename to packages/compute-worker/tests/setup-env.ts diff --git a/compute-worker/tests/tsconfig.json b/packages/compute-worker/tests/tsconfig.json similarity index 100% rename from compute-worker/tests/tsconfig.json rename to packages/compute-worker/tests/tsconfig.json diff --git a/compute-worker/tests/unit/jetstream-adapters.test.ts b/packages/compute-worker/tests/unit/jetstream-adapters.test.ts similarity index 95% rename from compute-worker/tests/unit/jetstream-adapters.test.ts rename to packages/compute-worker/tests/unit/jetstream-adapters.test.ts index 8425061..d694594 100644 --- a/compute-worker/tests/unit/jetstream-adapters.test.ts +++ b/packages/compute-worker/tests/unit/jetstream-adapters.test.ts @@ -45,6 +45,17 @@ class FakeKvStore implements KvStoreLike { this.revision += 1; this.data.set(key, { operation: 'PUT', value: data.slice(), revision: this.revision }); } + + async keys(filter?: string | string[]): Promise> { + const keys = Array.from(this.data.keys()); + return { + async *[Symbol.asyncIterator]() { + for (const key of keys) { + yield key; + } + } + }; + } } class FakeJetStream { diff --git a/compute-worker/tests/unit/key-and-progress.test.ts b/packages/compute-worker/tests/unit/key-and-progress.test.ts similarity index 100% rename from compute-worker/tests/unit/key-and-progress.test.ts rename to packages/compute-worker/tests/unit/key-and-progress.test.ts diff --git a/compute-worker/tests/unit/orphan-recovery.test.ts b/packages/compute-worker/tests/unit/orphan-recovery.test.ts similarity index 88% rename from compute-worker/tests/unit/orphan-recovery.test.ts rename to packages/compute-worker/tests/unit/orphan-recovery.test.ts index 7ec771d..54db314 100644 --- a/compute-worker/tests/unit/orphan-recovery.test.ts +++ b/packages/compute-worker/tests/unit/orphan-recovery.test.ts @@ -24,8 +24,8 @@ describe('orphan recovery', () => { }); const firstSweep = await recoverOrphanedOperations({ - operationStateStore: fake.deps.operationStateStore, - orchestrator: fake.deps.orchestrator, + operationStateStore: fake.deps.operationStateStore!, + orchestrator: fake.deps.orchestrator!, whisperTimeoutMs: 30_000, pdfTimeoutMs: 300_000, opStaleMs: 1_800_000, @@ -38,8 +38,8 @@ describe('orphan recovery', () => { vi.advanceTimersByTime(31_000); const secondSweep = await recoverOrphanedOperations({ - operationStateStore: fake.deps.operationStateStore, - orchestrator: fake.deps.orchestrator, + operationStateStore: fake.deps.operationStateStore!, + orchestrator: fake.deps.orchestrator!, whisperTimeoutMs: 30_000, pdfTimeoutMs: 300_000, opStaleMs: 1_800_000, diff --git a/compute-worker/tests/unit/pdf-artifact-persistence.test.ts b/packages/compute-worker/tests/unit/pdf-artifact-persistence.test.ts similarity index 100% rename from compute-worker/tests/unit/pdf-artifact-persistence.test.ts rename to packages/compute-worker/tests/unit/pdf-artifact-persistence.test.ts diff --git a/compute-worker/tests/unit/worker-loop-policy.test.ts b/packages/compute-worker/tests/unit/worker-loop-policy.test.ts similarity index 100% rename from compute-worker/tests/unit/worker-loop-policy.test.ts rename to packages/compute-worker/tests/unit/worker-loop-policy.test.ts diff --git a/compute-worker/tests/unit/worker-loop.test.ts b/packages/compute-worker/tests/unit/worker-loop.test.ts similarity index 100% rename from compute-worker/tests/unit/worker-loop.test.ts rename to packages/compute-worker/tests/unit/worker-loop.test.ts diff --git a/compute-worker/tsconfig.json b/packages/compute-worker/tsconfig.json similarity index 73% rename from compute-worker/tsconfig.json rename to packages/compute-worker/tsconfig.json index 820e03b..428e252 100644 --- a/compute-worker/tsconfig.json +++ b/packages/compute-worker/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../tsconfig.json", + "extends": "../../tsconfig.json", "compilerOptions": { "noEmit": true, }, diff --git a/packages/database/bin/migrate.mjs b/packages/database/bin/migrate.mjs new file mode 100755 index 0000000..39fd805 --- /dev/null +++ b/packages/database/bin/migrate.mjs @@ -0,0 +1,9 @@ +#!/usr/bin/env node +import { runMigrations } from '../src/migrate.mjs'; + +try { + await runMigrations(); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +} diff --git a/drizzle.config.pg.ts b/packages/database/drizzle.config.pg.ts similarity index 72% rename from drizzle.config.pg.ts rename to packages/database/drizzle.config.pg.ts index 663a9f7..b2ff9f6 100644 --- a/drizzle.config.pg.ts +++ b/packages/database/drizzle.config.pg.ts @@ -10,8 +10,8 @@ if (!url) { } export default { - schema: ['./src/db/schema_postgres.ts', './src/db/schema_auth_postgres.ts'], - out: './drizzle/postgres', + schema: ['./packages/database/src/schema_postgres.ts', './packages/database/src/schema_auth_postgres.ts'], + out: './packages/database/migrations/postgres', dialect: 'postgresql', dbCredentials: { url, diff --git a/drizzle.config.sqlite.ts b/packages/database/drizzle.config.sqlite.ts similarity index 57% rename from drizzle.config.sqlite.ts rename to packages/database/drizzle.config.sqlite.ts index 74cc605..ec64ec4 100644 --- a/drizzle.config.sqlite.ts +++ b/packages/database/drizzle.config.sqlite.ts @@ -4,8 +4,8 @@ import * as dotenv from 'dotenv'; dotenv.config(); export default { - schema: ['./src/db/schema_sqlite.ts', './src/db/schema_auth_sqlite.ts'], - out: './drizzle/sqlite', + schema: ['./packages/database/src/schema_sqlite.ts', './packages/database/src/schema_auth_sqlite.ts'], + out: './packages/database/migrations/sqlite', dialect: 'sqlite', dbCredentials: { url: 'docstore/sqlite3.db', diff --git a/drizzle/postgres/0000_black_lucky_pierre.sql b/packages/database/migrations/postgres/0000_black_lucky_pierre.sql similarity index 100% rename from drizzle/postgres/0000_black_lucky_pierre.sql rename to packages/database/migrations/postgres/0000_black_lucky_pierre.sql diff --git a/drizzle/postgres/0001_tts_segments.sql b/packages/database/migrations/postgres/0001_tts_segments.sql similarity index 100% rename from drizzle/postgres/0001_tts_segments.sql rename to packages/database/migrations/postgres/0001_tts_segments.sql diff --git a/drizzle/postgres/0002_add_segment_key_to_tts_segments.sql b/packages/database/migrations/postgres/0002_add_segment_key_to_tts_segments.sql similarity index 100% rename from drizzle/postgres/0002_add_segment_key_to_tts_segments.sql rename to packages/database/migrations/postgres/0002_add_segment_key_to_tts_segments.sql diff --git a/drizzle/postgres/0003_tts_segments_v2_split.sql b/packages/database/migrations/postgres/0003_tts_segments_v2_split.sql similarity index 100% rename from drizzle/postgres/0003_tts_segments_v2_split.sql rename to packages/database/migrations/postgres/0003_tts_segments_v2_split.sql diff --git a/drizzle/postgres/0004_admin_panel.sql b/packages/database/migrations/postgres/0004_admin_panel.sql similarity index 100% rename from drizzle/postgres/0004_admin_panel.sql rename to packages/database/migrations/postgres/0004_admin_panel.sql diff --git a/drizzle/postgres/0005_pdf_layout_and_compute.sql b/packages/database/migrations/postgres/0005_pdf_layout_and_compute.sql similarity index 100% rename from drizzle/postgres/0005_pdf_layout_and_compute.sql rename to packages/database/migrations/postgres/0005_pdf_layout_and_compute.sql diff --git a/drizzle/postgres/0006_preview-defaults-cleanup.sql b/packages/database/migrations/postgres/0006_preview-defaults-cleanup.sql similarity index 100% rename from drizzle/postgres/0006_preview-defaults-cleanup.sql rename to packages/database/migrations/postgres/0006_preview-defaults-cleanup.sql diff --git a/drizzle/postgres/0007_pdf_parse_progress.sql b/packages/database/migrations/postgres/0007_pdf_parse_progress.sql similarity index 100% rename from drizzle/postgres/0007_pdf_parse_progress.sql rename to packages/database/migrations/postgres/0007_pdf_parse_progress.sql diff --git a/drizzle/postgres/0008_user_job_events.sql b/packages/database/migrations/postgres/0008_user_job_events.sql similarity index 100% rename from drizzle/postgres/0008_user_job_events.sql rename to packages/database/migrations/postgres/0008_user_job_events.sql diff --git a/drizzle/postgres/0009_drop_pdf_parse.sql b/packages/database/migrations/postgres/0009_drop_pdf_parse.sql similarity index 100% rename from drizzle/postgres/0009_drop_pdf_parse.sql rename to packages/database/migrations/postgres/0009_drop_pdf_parse.sql diff --git a/drizzle/postgres/0010_user-data-cleanup-cascades.sql b/packages/database/migrations/postgres/0010_user-data-cleanup-cascades.sql similarity index 100% rename from drizzle/postgres/0010_user-data-cleanup-cascades.sql rename to packages/database/migrations/postgres/0010_user-data-cleanup-cascades.sql diff --git a/drizzle/postgres/0011_scheduled-tasks.sql b/packages/database/migrations/postgres/0011_scheduled-tasks.sql similarity index 100% rename from drizzle/postgres/0011_scheduled-tasks.sql rename to packages/database/migrations/postgres/0011_scheduled-tasks.sql diff --git a/drizzle/postgres/meta/0000_snapshot.json b/packages/database/migrations/postgres/meta/0000_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0000_snapshot.json rename to packages/database/migrations/postgres/meta/0000_snapshot.json diff --git a/drizzle/postgres/meta/0001_snapshot.json b/packages/database/migrations/postgres/meta/0001_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0001_snapshot.json rename to packages/database/migrations/postgres/meta/0001_snapshot.json diff --git a/drizzle/postgres/meta/0002_snapshot.json b/packages/database/migrations/postgres/meta/0002_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0002_snapshot.json rename to packages/database/migrations/postgres/meta/0002_snapshot.json diff --git a/drizzle/postgres/meta/0003_snapshot.json b/packages/database/migrations/postgres/meta/0003_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0003_snapshot.json rename to packages/database/migrations/postgres/meta/0003_snapshot.json diff --git a/drizzle/postgres/meta/0004_snapshot.json b/packages/database/migrations/postgres/meta/0004_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0004_snapshot.json rename to packages/database/migrations/postgres/meta/0004_snapshot.json diff --git a/drizzle/postgres/meta/0005_snapshot.json b/packages/database/migrations/postgres/meta/0005_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0005_snapshot.json rename to packages/database/migrations/postgres/meta/0005_snapshot.json diff --git a/drizzle/postgres/meta/0006_snapshot.json b/packages/database/migrations/postgres/meta/0006_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0006_snapshot.json rename to packages/database/migrations/postgres/meta/0006_snapshot.json diff --git a/drizzle/postgres/meta/0007_snapshot.json b/packages/database/migrations/postgres/meta/0007_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0007_snapshot.json rename to packages/database/migrations/postgres/meta/0007_snapshot.json diff --git a/drizzle/postgres/meta/0008_snapshot.json b/packages/database/migrations/postgres/meta/0008_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0008_snapshot.json rename to packages/database/migrations/postgres/meta/0008_snapshot.json diff --git a/drizzle/postgres/meta/0009_snapshot.json b/packages/database/migrations/postgres/meta/0009_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0009_snapshot.json rename to packages/database/migrations/postgres/meta/0009_snapshot.json diff --git a/drizzle/postgres/meta/0010_snapshot.json b/packages/database/migrations/postgres/meta/0010_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0010_snapshot.json rename to packages/database/migrations/postgres/meta/0010_snapshot.json diff --git a/drizzle/postgres/meta/0011_snapshot.json b/packages/database/migrations/postgres/meta/0011_snapshot.json similarity index 100% rename from drizzle/postgres/meta/0011_snapshot.json rename to packages/database/migrations/postgres/meta/0011_snapshot.json diff --git a/drizzle/postgres/meta/_journal.json b/packages/database/migrations/postgres/meta/_journal.json similarity index 100% rename from drizzle/postgres/meta/_journal.json rename to packages/database/migrations/postgres/meta/_journal.json diff --git a/drizzle/sqlite/0000_familiar_caretaker.sql b/packages/database/migrations/sqlite/0000_familiar_caretaker.sql similarity index 100% rename from drizzle/sqlite/0000_familiar_caretaker.sql rename to packages/database/migrations/sqlite/0000_familiar_caretaker.sql diff --git a/drizzle/sqlite/0001_tts_segments.sql b/packages/database/migrations/sqlite/0001_tts_segments.sql similarity index 100% rename from drizzle/sqlite/0001_tts_segments.sql rename to packages/database/migrations/sqlite/0001_tts_segments.sql diff --git a/drizzle/sqlite/0002_add_segment_key_to_tts_segments.sql b/packages/database/migrations/sqlite/0002_add_segment_key_to_tts_segments.sql similarity index 100% rename from drizzle/sqlite/0002_add_segment_key_to_tts_segments.sql rename to packages/database/migrations/sqlite/0002_add_segment_key_to_tts_segments.sql diff --git a/drizzle/sqlite/0003_tts_segments_v2_split.sql b/packages/database/migrations/sqlite/0003_tts_segments_v2_split.sql similarity index 100% rename from drizzle/sqlite/0003_tts_segments_v2_split.sql rename to packages/database/migrations/sqlite/0003_tts_segments_v2_split.sql diff --git a/drizzle/sqlite/0004_admin_panel.sql b/packages/database/migrations/sqlite/0004_admin_panel.sql similarity index 100% rename from drizzle/sqlite/0004_admin_panel.sql rename to packages/database/migrations/sqlite/0004_admin_panel.sql diff --git a/drizzle/sqlite/0005_pdf_layout_and_compute.sql b/packages/database/migrations/sqlite/0005_pdf_layout_and_compute.sql similarity index 100% rename from drizzle/sqlite/0005_pdf_layout_and_compute.sql rename to packages/database/migrations/sqlite/0005_pdf_layout_and_compute.sql diff --git a/drizzle/sqlite/0006_preview-defaults-cleanup.sql b/packages/database/migrations/sqlite/0006_preview-defaults-cleanup.sql similarity index 100% rename from drizzle/sqlite/0006_preview-defaults-cleanup.sql rename to packages/database/migrations/sqlite/0006_preview-defaults-cleanup.sql diff --git a/drizzle/sqlite/0007_pdf_parse_progress.sql b/packages/database/migrations/sqlite/0007_pdf_parse_progress.sql similarity index 100% rename from drizzle/sqlite/0007_pdf_parse_progress.sql rename to packages/database/migrations/sqlite/0007_pdf_parse_progress.sql diff --git a/drizzle/sqlite/0008_user_job_events.sql b/packages/database/migrations/sqlite/0008_user_job_events.sql similarity index 100% rename from drizzle/sqlite/0008_user_job_events.sql rename to packages/database/migrations/sqlite/0008_user_job_events.sql diff --git a/drizzle/sqlite/0009_drop_pdf_parse.sql b/packages/database/migrations/sqlite/0009_drop_pdf_parse.sql similarity index 100% rename from drizzle/sqlite/0009_drop_pdf_parse.sql rename to packages/database/migrations/sqlite/0009_drop_pdf_parse.sql diff --git a/drizzle/sqlite/0010_user-data-cleanup-cascades.sql b/packages/database/migrations/sqlite/0010_user-data-cleanup-cascades.sql similarity index 100% rename from drizzle/sqlite/0010_user-data-cleanup-cascades.sql rename to packages/database/migrations/sqlite/0010_user-data-cleanup-cascades.sql diff --git a/drizzle/sqlite/0011_scheduled-tasks.sql b/packages/database/migrations/sqlite/0011_scheduled-tasks.sql similarity index 100% rename from drizzle/sqlite/0011_scheduled-tasks.sql rename to packages/database/migrations/sqlite/0011_scheduled-tasks.sql diff --git a/drizzle/sqlite/meta/0000_snapshot.json b/packages/database/migrations/sqlite/meta/0000_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0000_snapshot.json rename to packages/database/migrations/sqlite/meta/0000_snapshot.json diff --git a/drizzle/sqlite/meta/0001_snapshot.json b/packages/database/migrations/sqlite/meta/0001_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0001_snapshot.json rename to packages/database/migrations/sqlite/meta/0001_snapshot.json diff --git a/drizzle/sqlite/meta/0002_snapshot.json b/packages/database/migrations/sqlite/meta/0002_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0002_snapshot.json rename to packages/database/migrations/sqlite/meta/0002_snapshot.json diff --git a/drizzle/sqlite/meta/0003_snapshot.json b/packages/database/migrations/sqlite/meta/0003_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0003_snapshot.json rename to packages/database/migrations/sqlite/meta/0003_snapshot.json diff --git a/drizzle/sqlite/meta/0004_snapshot.json b/packages/database/migrations/sqlite/meta/0004_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0004_snapshot.json rename to packages/database/migrations/sqlite/meta/0004_snapshot.json diff --git a/drizzle/sqlite/meta/0005_snapshot.json b/packages/database/migrations/sqlite/meta/0005_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0005_snapshot.json rename to packages/database/migrations/sqlite/meta/0005_snapshot.json diff --git a/drizzle/sqlite/meta/0006_snapshot.json b/packages/database/migrations/sqlite/meta/0006_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0006_snapshot.json rename to packages/database/migrations/sqlite/meta/0006_snapshot.json diff --git a/drizzle/sqlite/meta/0007_snapshot.json b/packages/database/migrations/sqlite/meta/0007_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0007_snapshot.json rename to packages/database/migrations/sqlite/meta/0007_snapshot.json diff --git a/drizzle/sqlite/meta/0008_snapshot.json b/packages/database/migrations/sqlite/meta/0008_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0008_snapshot.json rename to packages/database/migrations/sqlite/meta/0008_snapshot.json diff --git a/drizzle/sqlite/meta/0009_snapshot.json b/packages/database/migrations/sqlite/meta/0009_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0009_snapshot.json rename to packages/database/migrations/sqlite/meta/0009_snapshot.json diff --git a/drizzle/sqlite/meta/0010_snapshot.json b/packages/database/migrations/sqlite/meta/0010_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0010_snapshot.json rename to packages/database/migrations/sqlite/meta/0010_snapshot.json diff --git a/drizzle/sqlite/meta/0011_snapshot.json b/packages/database/migrations/sqlite/meta/0011_snapshot.json similarity index 100% rename from drizzle/sqlite/meta/0011_snapshot.json rename to packages/database/migrations/sqlite/meta/0011_snapshot.json diff --git a/drizzle/sqlite/meta/_journal.json b/packages/database/migrations/sqlite/meta/_journal.json similarity index 100% rename from drizzle/sqlite/meta/_journal.json rename to packages/database/migrations/sqlite/meta/_journal.json diff --git a/packages/database/package.json b/packages/database/package.json new file mode 100644 index 0000000..ee40dda --- /dev/null +++ b/packages/database/package.json @@ -0,0 +1,33 @@ +{ + "name": "@openreader/database", + "version": "0.0.0", + "private": true, + "type": "module", + "files": [ + "bin", + "migrations", + "src" + ], + "exports": { + ".": "./src/index.ts", + "./migrate": "./src/migrate.mjs", + "./schema": "./src/schema.ts", + "./schema-postgres": "./src/schema_postgres.ts", + "./schema-sqlite": "./src/schema_sqlite.ts", + "./schema-auth-postgres": "./src/schema_auth_postgres.ts", + "./schema-auth-sqlite": "./src/schema_auth_sqlite.ts", + "./run-in-transaction": "./src/run-in-transaction.ts" + }, + "bin": { + "openreader-migrate": "./bin/migrate.mjs" + }, + "scripts": { + "migrate": "node bin/migrate.mjs" + }, + "dependencies": { + "better-sqlite3": "^12.10.0", + "dotenv": "^17.4.2", + "drizzle-orm": "^0.45.2", + "pg": "^8.21.0" + } +} diff --git a/drizzle/scripts/generate.mjs b/packages/database/scripts/generate.mjs similarity index 91% rename from drizzle/scripts/generate.mjs rename to packages/database/scripts/generate.mjs index 320c334..8e56aa3 100644 --- a/drizzle/scripts/generate.mjs +++ b/packages/database/scripts/generate.mjs @@ -43,7 +43,7 @@ console.log('\n--- Generating Better Auth schema (SQLite) ---'); delete envSqlite.POSTGRES_URL; // force SQLite adapter const result = spawnSync('npx', [ '@better-auth/cli', 'generate', - '--output', 'src/db/schema_auth_sqlite.ts', + '--output', 'packages/database/src/schema_auth_sqlite.ts', '--yes', ], { stdio: 'inherit', env: envSqlite }); if ((result.status ?? 1) !== 0) process.exit(result.status ?? 1); @@ -58,7 +58,7 @@ console.log('\n--- Generating Better Auth schema (Postgres) ---'); } const result = spawnSync('npx', [ '@better-auth/cli', 'generate', - '--output', 'src/db/schema_auth_postgres.ts', + '--output', 'packages/database/src/schema_auth_postgres.ts', '--yes', ], { stdio: 'inherit', env: envPg }); if ((result.status ?? 1) !== 0) process.exit(result.status ?? 1); @@ -67,7 +67,10 @@ console.log('\n--- Generating Better Auth schema (Postgres) ---'); // --------------------------------------------------------------------------- // Step 2: Generate Drizzle migrations for both dialects. // --------------------------------------------------------------------------- -for (const configFile of ['drizzle.config.sqlite.ts', 'drizzle.config.pg.ts']) { +for (const configFile of [ + 'packages/database/drizzle.config.sqlite.ts', + 'packages/database/drizzle.config.pg.ts', +]) { console.log(`\n--- Drizzle generate (${configFile}) ---`); const result = spawnSync('drizzle-kit', ['generate', '--config', configFile, ...extraArgs], { stdio: 'inherit', diff --git a/src/db/index.ts b/packages/database/src/index.ts similarity index 100% rename from src/db/index.ts rename to packages/database/src/index.ts diff --git a/packages/database/src/migrate.mjs b/packages/database/src/migrate.mjs new file mode 100644 index 0000000..8aba886 --- /dev/null +++ b/packages/database/src/migrate.mjs @@ -0,0 +1,51 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as dotenv from 'dotenv'; + +const packageDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); + +function loadEnvFiles(cwd) { + const envPath = path.join(cwd, '.env'); + const envLocalPath = path.join(cwd, '.env.local'); + + if (fs.existsSync(envPath)) dotenv.config({ path: envPath }); + if (fs.existsSync(envLocalPath)) dotenv.config({ path: envLocalPath, override: true }); +} + +export async function runMigrations({ cwd = process.cwd(), env = process.env } = {}) { + loadEnvFiles(cwd); + + if (env.POSTGRES_URL?.trim()) { + const [{ drizzle }, { migrate }, { Pool }] = await Promise.all([ + import('drizzle-orm/node-postgres'), + import('drizzle-orm/node-postgres/migrator'), + import('pg'), + ]); + const pool = new Pool({ connectionString: env.POSTGRES_URL }); + try { + await migrate(drizzle(pool), { + migrationsFolder: path.join(packageDir, 'migrations', 'postgres'), + }); + } finally { + await pool.end(); + } + return; + } + + const dbPath = path.join(cwd, 'docstore', 'sqlite3.db'); + fs.mkdirSync(path.dirname(dbPath), { recursive: true }); + const [{ drizzle }, { migrate }, { default: BetterSqlite3 }] = await Promise.all([ + import('drizzle-orm/better-sqlite3'), + import('drizzle-orm/better-sqlite3/migrator'), + import('better-sqlite3'), + ]); + const sqlite = new BetterSqlite3(dbPath); + try { + migrate(drizzle(sqlite), { + migrationsFolder: path.join(packageDir, 'migrations', 'sqlite'), + }); + } finally { + sqlite.close(); + } +} diff --git a/src/db/run-in-transaction.ts b/packages/database/src/run-in-transaction.ts similarity index 96% rename from src/db/run-in-transaction.ts rename to packages/database/src/run-in-transaction.ts index 1ec6ecd..8813e88 100644 --- a/src/db/run-in-transaction.ts +++ b/packages/database/src/run-in-transaction.ts @@ -1,4 +1,4 @@ -import { db } from '@/db'; +import { db } from './index'; /** * Run `fn` with a database connection, wrapped in a transaction on Postgres. diff --git a/src/db/schema.ts b/packages/database/src/schema.ts similarity index 100% rename from src/db/schema.ts rename to packages/database/src/schema.ts diff --git a/src/db/schema_auth_postgres.ts b/packages/database/src/schema_auth_postgres.ts similarity index 100% rename from src/db/schema_auth_postgres.ts rename to packages/database/src/schema_auth_postgres.ts diff --git a/src/db/schema_auth_sqlite.ts b/packages/database/src/schema_auth_sqlite.ts similarity index 100% rename from src/db/schema_auth_sqlite.ts rename to packages/database/src/schema_auth_sqlite.ts diff --git a/src/db/schema_postgres.ts b/packages/database/src/schema_postgres.ts similarity index 100% rename from src/db/schema_postgres.ts rename to packages/database/src/schema_postgres.ts diff --git a/src/db/schema_sqlite.ts b/packages/database/src/schema_sqlite.ts similarity index 100% rename from src/db/schema_sqlite.ts rename to packages/database/src/schema_sqlite.ts diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 32a7d38..0ca9ff3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -29,6 +29,9 @@ importers: '@napi-rs/canvas': specifier: ^0.1.100 version: 0.1.100 + '@openreader/database': + specifier: workspace:* + version: link:packages/database '@speech-sdk/core': specifier: ^0.15.0 version: 0.15.0 @@ -47,9 +50,6 @@ importers: better-auth: specifier: ^1.6.14 version: 1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))) - better-sqlite3: - specifier: ^12.10.0 - version: 12.10.0 cmpstr: specifier: ^3.3.0 version: 3.3.0 @@ -68,9 +68,6 @@ importers: dotenv: specifier: ^17.4.2 version: 17.4.2 - drizzle-kit: - specifier: ^0.31.10 - version: 0.31.10 drizzle-orm: specifier: ^0.45.2 version: 0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0) @@ -104,9 +101,6 @@ importers: pdfjs-dist: specifier: 4.8.69 version: 4.8.69 - pg: - specifier: ^8.21.0 - version: 8.21.0 pino: specifier: ^10.3.1 version: 10.3.1 @@ -192,6 +186,12 @@ importers: '@types/uuid': specifier: ^10.0.0 version: 10.0.0 + better-sqlite3: + specifier: ^12.10.0 + version: 12.10.0 + drizzle-kit: + specifier: ^0.31.10 + version: 0.31.10 eslint: specifier: ^9.39.4 version: 9.39.4(jiti@1.21.7) @@ -214,7 +214,28 @@ importers: specifier: ^4.1.8 version: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)) - compute-worker: + packages/bootstrap: + dependencies: + '@aws-sdk/client-s3': + specifier: ^3.1061.0 + version: 3.1061.0 + '@openreader/database': + specifier: workspace:* + version: link:../database + better-sqlite3: + specifier: ^12.10.0 + version: 12.10.0 + dotenv: + specifier: ^17.4.2 + version: 17.4.2 + ffmpeg-static: + specifier: ^5.3.0 + version: 5.3.0 + pg: + specifier: ^8.21.0 + version: 8.21.0 + + packages/compute-worker: dependencies: '@aws-sdk/client-s3': specifier: ^3.1061.0 @@ -265,26 +286,17 @@ importers: specifier: ^4.4.3 version: 4.4.3 - docker/entrypoint-migration-tools: + packages/database: dependencies: - '@aws-sdk/client-s3': - specifier: ^3.1061.0 - version: 3.1061.0 better-sqlite3: specifier: ^12.10.0 version: 12.10.0 dotenv: specifier: ^17.4.2 version: 17.4.2 - drizzle-kit: - specifier: ^0.31.10 - version: 0.31.10 drizzle-orm: specifier: ^0.45.2 version: 0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0) - ffmpeg-static: - specifier: ^5.3.0 - version: 5.3.0 pg: specifier: ^8.21.0 version: 8.21.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index e94fdc1..d7d7a9f 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,7 +1,6 @@ packages: - . - - compute-worker - - docker/* + - packages/* allowBuilds: '@napi-rs/canvas': true diff --git a/scripts/check-compute-boundary.mjs b/scripts/check-compute-boundary.mjs index 8cc6a20..62ad85b 100644 --- a/scripts/check-compute-boundary.mjs +++ b/scripts/check-compute-boundary.mjs @@ -7,7 +7,7 @@ const forbidden = [ '@openreader/compute-worker', 'compute/core', 'compute/worker', - 'compute-worker/src', + 'packages/compute-worker/src', ]; const failures = []; diff --git a/scripts/check-next-server-bundle.mjs b/scripts/check-next-server-bundle.mjs index 8174d40..795c60b 100644 --- a/scripts/check-next-server-bundle.mjs +++ b/scripts/check-next-server-bundle.mjs @@ -13,7 +13,7 @@ const forbidden = [ 'onnxruntime-node', '@huggingface/tokenizers', '@openreader/compute-worker', - '/compute-worker/src/inference/', + '/packages/compute-worker/src/inference/', ]; const includeExt = new Set(['.js', '.mjs', '.cjs']); diff --git a/src/app/api/audiobook/chapter/route.ts b/src/app/api/audiobook/chapter/route.ts index 7aa881a..f080fbe 100644 --- a/src/app/api/audiobook/chapter/route.ts +++ b/src/app/api/audiobook/chapter/route.ts @@ -5,8 +5,8 @@ import { tmpdir } from 'os'; import { join } from 'path'; import { randomUUID } from 'crypto'; import { and, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { audiobooks, audiobookChapters } from '@/db/schema'; +import { db } from '@openreader/database'; +import { audiobooks, audiobookChapters } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { rateLimiter, resolveRateLimitThresholds } from '@/lib/server/rate-limit/rate-limiter'; import { getClientIp } from '@/lib/server/rate-limit/request-ip'; diff --git a/src/app/api/audiobook/route.ts b/src/app/api/audiobook/route.ts index 336c772..a2955dc 100644 --- a/src/app/api/audiobook/route.ts +++ b/src/app/api/audiobook/route.ts @@ -4,8 +4,8 @@ import { mkdtemp, readFile, rm, writeFile } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; import { and, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { audiobooks, audiobookChapters } from '@/db/schema'; +import { db } from '@openreader/database'; +import { audiobooks, audiobookChapters } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; diff --git a/src/app/api/audiobook/status/route.ts b/src/app/api/audiobook/status/route.ts index 6416dc3..14519ca 100644 --- a/src/app/api/audiobook/status/route.ts +++ b/src/app/api/audiobook/status/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { audiobooks, audiobookChapters } from '@/db/schema'; +import { db } from '@openreader/database'; +import { audiobooks, audiobookChapters } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { getAudiobookObjectBuffer, isMissingBlobError, listAudiobookObjects } from '@/lib/server/audiobooks/blobstore'; import { decodeChapterFileName } from '@/lib/server/audiobooks/chapters'; diff --git a/src/app/api/documents/[id]/parsed/events/route.ts b/src/app/api/documents/[id]/parsed/events/route.ts index fd8e08d..2a49029 100644 --- a/src/app/api/documents/[id]/parsed/events/route.ts +++ b/src/app/api/documents/[id]/parsed/events/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { getComputeWorkerClient } from '@/lib/server/compute-worker/client'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; diff --git a/src/app/api/documents/[id]/parsed/route.ts b/src/app/api/documents/[id]/parsed/route.ts index 0234ade..898dac7 100644 --- a/src/app/api/documents/[id]/parsed/route.ts +++ b/src/app/api/documents/[id]/parsed/route.ts @@ -1,8 +1,8 @@ import { randomUUID } from 'node:crypto'; import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { diff --git a/src/app/api/documents/[id]/settings/route.ts b/src/app/api/documents/[id]/settings/route.ts index 0b2b282..f618022 100644 --- a/src/app/api/documents/[id]/settings/route.ts +++ b/src/app/api/documents/[id]/settings/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { documentSettings, documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documentSettings, documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { mergeDocumentSettings } from '@/lib/shared/document-settings'; import { DEFAULT_DOCUMENT_SETTINGS, type DocumentSettings } from '@/types/document-settings'; diff --git a/src/app/api/documents/blob/get/fallback/route.ts b/src/app/api/documents/blob/get/fallback/route.ts index 0db67e0..7ca9006 100644 --- a/src/app/api/documents/blob/get/fallback/route.ts +++ b/src/app/api/documents/blob/get/fallback/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { contentTypeForName } from '@/lib/server/storage/library-mount'; import { getDocumentBlob, isMissingBlobError, isValidDocumentId } from '@/lib/server/documents/blobstore'; diff --git a/src/app/api/documents/blob/get/presign/route.ts b/src/app/api/documents/blob/get/presign/route.ts index 4992354..685e4dd 100644 --- a/src/app/api/documents/blob/get/presign/route.ts +++ b/src/app/api/documents/blob/get/presign/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { isValidDocumentId, presignGet } from '@/lib/server/documents/blobstore'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; diff --git a/src/app/api/documents/blob/preview/fallback/route.ts b/src/app/api/documents/blob/preview/fallback/route.ts index 004aa74..4341a08 100644 --- a/src/app/api/documents/blob/preview/fallback/route.ts +++ b/src/app/api/documents/blob/preview/fallback/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { errorToLog, serverLogger } from '@/lib/server/logger'; import { errorResponse } from '@/lib/server/errors/next-response'; diff --git a/src/app/api/documents/blob/preview/utils.ts b/src/app/api/documents/blob/preview/utils.ts index 937942f..ad19bce 100644 --- a/src/app/api/documents/blob/preview/utils.ts +++ b/src/app/api/documents/blob/preview/utils.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { isPreviewableDocumentType } from '@/lib/server/documents/previews'; diff --git a/src/app/api/documents/route.ts b/src/app/api/documents/route.ts index c7d21d8..328591f 100644 --- a/src/app/api/documents/route.ts +++ b/src/app/api/documents/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { toDocumentTypeFromName } from '@/lib/server/documents/utils'; import { errorToLog, serverLogger } from '@/lib/server/logger'; diff --git a/src/app/api/tts/segments/ensure/route.ts b/src/app/api/tts/segments/ensure/route.ts index 28d7a0a..f93e551 100644 --- a/src/app/api/tts/segments/ensure/route.ts +++ b/src/app/api/tts/segments/ensure/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents, ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents, ttsSegmentEntries, ttsSegmentVariants } from '@openreader/database/schema'; import { isS3Configured, getS3Config } from '@/lib/server/storage/s3'; import { generateTTSBuffer } from '@/lib/server/tts/generate'; import { diff --git a/src/app/api/tts/segments/manifest/route.ts b/src/app/api/tts/segments/manifest/route.ts index f33155d..89621f3 100644 --- a/src/app/api/tts/segments/manifest/route.ts +++ b/src/app/api/tts/segments/manifest/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, asc, eq, gt, inArray, or } from 'drizzle-orm'; -import { db } from '@/db'; -import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema'; +import { db } from '@openreader/database'; +import { ttsSegmentEntries, ttsSegmentVariants } from '@openreader/database/schema'; import { resolveSegmentDocumentScope } from '@/lib/server/tts/segments-auth'; import { decodeManifestCursor, diff --git a/src/app/api/user/claim/route.ts b/src/app/api/user/claim/route.ts index 5745cd7..8d4b918 100644 --- a/src/app/api/user/claim/route.ts +++ b/src/app/api/user/claim/route.ts @@ -1,8 +1,8 @@ import { NextRequest, NextResponse } from 'next/server'; import { claimAnonymousData } from '@/lib/server/user/claim-data'; import { auth } from '@/lib/server/auth/auth'; -import { db } from '@/db'; -import { audiobooks, documentSettings, documents, userDocumentProgress, userPreferences } from '@/db/schema'; +import { db } from '@openreader/database'; +import { audiobooks, documentSettings, documents, userDocumentProgress, userPreferences } from '@openreader/database/schema'; import { count, eq, ne } from 'drizzle-orm'; import { getOpenReaderTestNamespace, getUnclaimedUserIdForNamespace } from '@/lib/server/testing/test-namespace'; import { errorToLog, serverLogger } from '@/lib/server/logger'; diff --git a/src/app/api/user/export/route.ts b/src/app/api/user/export/route.ts index e65b035..eae1047 100644 --- a/src/app/api/user/export/route.ts +++ b/src/app/api/user/export/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { PassThrough, Readable } from 'stream'; import { auth } from '@/lib/server/auth/auth'; -import { db } from '@/db'; +import { db } from '@openreader/database'; import { documents, audiobooks, @@ -13,9 +13,9 @@ import { userJobEvents, userPreferences, userTtsChars, -} from '@/db/schema'; -import * as authSchemaSqlite from '@/db/schema_auth_sqlite'; -import * as authSchemaPostgres from '@/db/schema_auth_postgres'; +} from '@openreader/database/schema'; +import * as authSchemaSqlite from '@openreader/database/schema-auth-sqlite'; +import * as authSchemaPostgres from '@openreader/database/schema-auth-postgres'; import { and, desc, eq, inArray } from 'drizzle-orm'; import archiver from 'archiver'; import { appendUserExportArchive } from '@/lib/server/user/data-export'; diff --git a/src/app/api/user/state/changelog/version-check/route.ts b/src/app/api/user/state/changelog/version-check/route.ts index b36d74f..498467b 100644 --- a/src/app/api/user/state/changelog/version-check/route.ts +++ b/src/app/api/user/state/changelog/version-check/route.ts @@ -1,7 +1,7 @@ import { eq } from 'drizzle-orm'; import { NextRequest, NextResponse } from 'next/server'; -import { db } from '@/db'; -import { userPreferences } from '@/db/schema'; +import { db } from '@openreader/database'; +import { userPreferences } from '@openreader/database/schema'; import { normalizeVersion, shouldOpenChangelogForVersionChange } from '@/lib/shared/changelog'; import { nowTimestampMs } from '@/lib/shared/timestamps'; import { resolveUserStateScope } from '@/lib/server/user/resolve-state-scope'; diff --git a/src/app/api/user/state/preferences/route.ts b/src/app/api/user/state/preferences/route.ts index 9eeae34..4c81de2 100644 --- a/src/app/api/user/state/preferences/route.ts +++ b/src/app/api/user/state/preferences/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { userPreferences } from '@/db/schema'; +import { db } from '@openreader/database'; +import { userPreferences } from '@openreader/database/schema'; import { type SyncedPreferencesPatch } from '@/types/user-state'; import { resolveUserStateScope } from '@/lib/server/user/resolve-state-scope'; import { coerceTimestampMs, nowTimestampMs } from '@/lib/shared/timestamps'; diff --git a/src/app/api/user/state/progress/route.ts b/src/app/api/user/state/progress/route.ts index 1fef9f5..b365588 100644 --- a/src/app/api/user/state/progress/route.ts +++ b/src/app/api/user/state/progress/route.ts @@ -1,7 +1,7 @@ import { NextRequest, NextResponse } from 'next/server'; import { and, eq, sql } from 'drizzle-orm'; -import { db } from '@/db'; -import { userDocumentProgress } from '@/db/schema'; +import { db } from '@openreader/database'; +import { userDocumentProgress } from '@openreader/database/schema'; import type { ReaderType } from '@/types/user-state'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { resolveUserStateScope } from '@/lib/server/user/resolve-state-scope'; diff --git a/src/lib/server/admin/email-sync.ts b/src/lib/server/admin/email-sync.ts index 105ec17..1c9696b 100644 --- a/src/lib/server/admin/email-sync.ts +++ b/src/lib/server/admin/email-sync.ts @@ -1,5 +1,5 @@ import { eq } from 'drizzle-orm'; -import { db } from '@/db'; +import { db } from '@openreader/database'; import { hashForLog, serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; @@ -7,9 +7,9 @@ import { logDegraded } from '@/lib/server/errors/logging'; // we import them lazily to avoid coupling this module to a single dialect. function getUserTable() { // eslint-disable-next-line @typescript-eslint/no-require-imports - const sqlite = require('@/db/schema_auth_sqlite'); + const sqlite = require('@openreader/database/schema-auth-sqlite'); // eslint-disable-next-line @typescript-eslint/no-require-imports - const pg = require('@/db/schema_auth_postgres'); + const pg = require('@openreader/database/schema-auth-postgres'); return process.env.POSTGRES_URL ? pg.user : sqlite.user; } diff --git a/src/lib/server/admin/providers.ts b/src/lib/server/admin/providers.ts index a562c9c..9befbb9 100644 --- a/src/lib/server/admin/providers.ts +++ b/src/lib/server/admin/providers.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; import { and, asc, desc, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { adminProviders } from '@/db/schema'; +import { db } from '@openreader/database'; +import { adminProviders } from '@openreader/database/schema'; import { apiKeyLast4, decryptSecret, encryptSecret } from '@/lib/server/crypto/secrets'; import { type TtsProviderId } from '@/lib/shared/tts-provider-catalog'; import { resolveTtsProviderModelPolicy } from '@/lib/shared/tts-provider-policy'; diff --git a/src/lib/server/admin/seed.ts b/src/lib/server/admin/seed.ts index 0409299..f120f62 100644 --- a/src/lib/server/admin/seed.ts +++ b/src/lib/server/admin/seed.ts @@ -1,5 +1,5 @@ -import { db } from '@/db'; -import { adminProviders, adminSettings } from '@/db/schema'; +import { db } from '@openreader/database'; +import { adminProviders, adminSettings } from '@openreader/database/schema'; import { encryptSecret, apiKeyLast4 } from '@/lib/server/crypto/secrets'; import { randomUUID } from 'node:crypto'; import { and, eq } from 'drizzle-orm'; diff --git a/src/lib/server/admin/settings.ts b/src/lib/server/admin/settings.ts index 99e1b87..4da8900 100644 --- a/src/lib/server/admin/settings.ts +++ b/src/lib/server/admin/settings.ts @@ -1,6 +1,6 @@ import { asc, desc, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { adminProviders, adminSettings } from '@/db/schema'; +import { db } from '@openreader/database'; +import { adminProviders, adminSettings } from '@openreader/database/schema'; import { serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; diff --git a/src/lib/server/audiobooks/prune.ts b/src/lib/server/audiobooks/prune.ts index 35d6669..6f1e3bb 100644 --- a/src/lib/server/audiobooks/prune.ts +++ b/src/lib/server/audiobooks/prune.ts @@ -1,7 +1,7 @@ import { and, eq, notInArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { audiobooks, audiobookChapters } from '@/db/schema'; +import { db } from '@openreader/database'; +import { audiobooks, audiobookChapters } from '@openreader/database/schema'; export async function pruneAudiobookIfMissingDir(bookId: string, userId: string, intermediateDirExists: boolean): Promise { if (intermediateDirExists) return; diff --git a/src/lib/server/auth/auth.ts b/src/lib/server/auth/auth.ts index d0359d1..c9345c5 100644 --- a/src/lib/server/auth/auth.ts +++ b/src/lib/server/auth/auth.ts @@ -4,13 +4,13 @@ import { anonymous } from "better-auth/plugins"; import { drizzleAdapter } from "better-auth/adapters/drizzle"; import { NextResponse } from 'next/server'; import type { NextRequest } from 'next/server'; -import { db } from "@/db"; +import { db } from "@openreader/database"; import { getRequiredAuthEnv, isAnonymousAuthSessionsEnabled } from "@/lib/server/auth/config"; import { isAdminEmail, syncAdminFlag } from "@/lib/server/admin/email-sync"; import { getResolvedRuntimeConfig } from '@/lib/server/runtime-config'; import { assertUserSignupAllowed } from '@/lib/server/auth/signup-policy'; -import * as authSchemaSqlite from "@/db/schema_auth_sqlite"; -import * as authSchemaPostgres from "@/db/schema_auth_postgres"; +import * as authSchemaSqlite from "@openreader/database/schema-auth-sqlite"; +import * as authSchemaPostgres from "@openreader/database/schema-auth-postgres"; import { hashForLog, serverLogger } from '@/lib/server/logger'; import { logDegraded, logServerError } from '@/lib/server/errors/logging'; import { tryGetOrigin } from "@/lib/shared/urls"; diff --git a/src/lib/server/documents/blob-lease.ts b/src/lib/server/documents/blob-lease.ts index c5b82bf..162bd99 100644 --- a/src/lib/server/documents/blob-lease.ts +++ b/src/lib/server/documents/blob-lease.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; import { and, eq, lt } from 'drizzle-orm'; -import { db } from '@/db'; -import { documentBlobLeases } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documentBlobLeases } from '@openreader/database/schema'; import { errorToLog, serverLogger } from '@/lib/server/logger'; const DEFAULT_LEASE_MS = 15 * 60 * 1000; diff --git a/src/lib/server/documents/delete-owned.ts b/src/lib/server/documents/delete-owned.ts index 4b2d14e..1686a51 100644 --- a/src/lib/server/documents/delete-owned.ts +++ b/src/lib/server/documents/delete-owned.ts @@ -1,6 +1,6 @@ import { and, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { deleteDocumentTtsSegmentCache } from '@/lib/server/tts/segments-cache'; import { hashForLog, serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; diff --git a/src/lib/server/documents/previews.ts b/src/lib/server/documents/previews.ts index 7c60c98..c127605 100644 --- a/src/lib/server/documents/previews.ts +++ b/src/lib/server/documents/previews.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'crypto'; import { and, eq, inArray, lt, or, sql } from 'drizzle-orm'; -import { db } from '@/db'; -import { documentPreviews } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documentPreviews } from '@openreader/database/schema'; import { serverLogger } from '@/lib/server/logger'; import { logServerError } from '@/lib/server/errors/logging'; import { diff --git a/src/lib/server/documents/register-upload.ts b/src/lib/server/documents/register-upload.ts index a29feb8..073c048 100644 --- a/src/lib/server/documents/register-upload.ts +++ b/src/lib/server/documents/register-upload.ts @@ -1,5 +1,5 @@ -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { and, eq } from 'drizzle-orm'; import { enqueueDocumentPreview, diff --git a/src/lib/server/rate-limit/job-rate-limiter.ts b/src/lib/server/rate-limit/job-rate-limiter.ts index 9a886ce..962d46e 100644 --- a/src/lib/server/rate-limit/job-rate-limiter.ts +++ b/src/lib/server/rate-limit/job-rate-limiter.ts @@ -1,6 +1,6 @@ import { and, eq, gte, sql } from 'drizzle-orm'; -import { db } from '@/db'; -import { userJobEvents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { userJobEvents } from '@openreader/database/schema'; import { nowTimestampMs } from '@/lib/shared/timestamps'; import type { RuntimeConfig } from '@/lib/server/admin/settings'; diff --git a/src/lib/server/rate-limit/rate-limiter.ts b/src/lib/server/rate-limit/rate-limiter.ts index a526d0d..cbbe335 100644 --- a/src/lib/server/rate-limit/rate-limiter.ts +++ b/src/lib/server/rate-limit/rate-limiter.ts @@ -1,6 +1,6 @@ -import { db } from '@/db'; -import { runInDbTransaction } from '@/db/run-in-transaction'; -import { userTtsChars } from '@/db/schema'; +import { db } from '@openreader/database'; +import { runInDbTransaction } from '@openreader/database/run-in-transaction'; +import { userTtsChars } from '@openreader/database/schema'; import { eq, and, lt, sql } from 'drizzle-orm'; import { nextUtcMidnightTimestampMs, nowTimestampMs } from '@/lib/shared/timestamps'; diff --git a/src/lib/server/tasks/engine.ts b/src/lib/server/tasks/engine.ts index 93822df..24ffa62 100644 --- a/src/lib/server/tasks/engine.ts +++ b/src/lib/server/tasks/engine.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; import { and, eq, lt, ne, or, sql } from 'drizzle-orm'; -import { db } from '@/db'; -import { scheduledTasks } from '@/db/schema'; +import { db } from '@openreader/database'; +import { scheduledTasks } from '@openreader/database/schema'; import { serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; import { TASK_REGISTRY } from './registry'; diff --git a/src/lib/server/tasks/handlers/prune-job-events.ts b/src/lib/server/tasks/handlers/prune-job-events.ts index f0ddf18..03938dc 100644 --- a/src/lib/server/tasks/handlers/prune-job-events.ts +++ b/src/lib/server/tasks/handlers/prune-job-events.ts @@ -1,6 +1,6 @@ import { lt } from 'drizzle-orm'; -import { db } from '@/db'; -import { userJobEvents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { userJobEvents } from '@openreader/database/schema'; import type { TaskResult } from '../types'; // Retention far exceeds the largest rate-limit window, so pruning never removes diff --git a/src/lib/server/tasks/handlers/reap-orphaned-blobs.ts b/src/lib/server/tasks/handlers/reap-orphaned-blobs.ts index a692370..0f88454 100644 --- a/src/lib/server/tasks/handlers/reap-orphaned-blobs.ts +++ b/src/lib/server/tasks/handlers/reap-orphaned-blobs.ts @@ -1,6 +1,6 @@ import { inArray } from 'drizzle-orm'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { isS3Configured } from '@/lib/server/storage/s3'; import { deleteDocumentBlob, listDocumentSourceBlobs } from '@/lib/server/documents/blobstore'; import { deleteDocumentPreviewArtifacts } from '@/lib/server/documents/previews-blobstore'; diff --git a/src/lib/server/testing/test-namespace.ts b/src/lib/server/testing/test-namespace.ts index 044e46f..834d7cd 100644 --- a/src/lib/server/testing/test-namespace.ts +++ b/src/lib/server/testing/test-namespace.ts @@ -1,6 +1,6 @@ import path from 'path'; import { UNCLAIMED_USER_ID } from '@/lib/server/storage/docstore-legacy'; -import { ensureSystemUserExists } from '@/db'; +import { ensureSystemUserExists } from '@openreader/database'; const TEST_NAMESPACE_HEADER = 'x-openreader-test-namespace'; const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; diff --git a/src/lib/server/tts/segments-audio.ts b/src/lib/server/tts/segments-audio.ts index 779c79e..1959baa 100644 --- a/src/lib/server/tts/segments-audio.ts +++ b/src/lib/server/tts/segments-audio.ts @@ -1,8 +1,8 @@ import { and, eq } from 'drizzle-orm'; import type { NextRequest } from 'next/server'; import { NextResponse } from 'next/server'; -import { db } from '@/db'; -import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema'; +import { db } from '@openreader/database'; +import { ttsSegmentEntries, ttsSegmentVariants } from '@openreader/database/schema'; export type ResolvedSegmentAudio = { documentId: string; diff --git a/src/lib/server/tts/segments-auth.ts b/src/lib/server/tts/segments-auth.ts index ff19de3..cd00f71 100644 --- a/src/lib/server/tts/segments-auth.ts +++ b/src/lib/server/tts/segments-auth.ts @@ -1,7 +1,7 @@ import { and, eq, inArray } from 'drizzle-orm'; import type { NextRequest } from 'next/server'; -import { db } from '@/db'; -import { documents } from '@/db/schema'; +import { db } from '@openreader/database'; +import { documents } from '@openreader/database/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; import { getOpenReaderTestNamespace } from '@/lib/server/testing/test-namespace'; import type { ReaderType } from '@/types/user-state'; diff --git a/src/lib/server/tts/segments-cache.ts b/src/lib/server/tts/segments-cache.ts index cc63791..f0f7c0a 100644 --- a/src/lib/server/tts/segments-cache.ts +++ b/src/lib/server/tts/segments-cache.ts @@ -1,6 +1,6 @@ import { and, eq } from 'drizzle-orm'; -import { db } from '@/db'; -import { ttsSegmentEntries, ttsSegmentVariants } from '@/db/schema'; +import { db } from '@openreader/database'; +import { ttsSegmentEntries, ttsSegmentVariants } from '@openreader/database/schema'; import { deleteTtsSegmentAudioObjects, deleteTtsSegmentPrefix, diff --git a/src/lib/server/user/claim-data.ts b/src/lib/server/user/claim-data.ts index 47a52c9..259425e 100644 --- a/src/lib/server/user/claim-data.ts +++ b/src/lib/server/user/claim-data.ts @@ -1,4 +1,4 @@ -import { db } from '@/db'; +import { db } from '@openreader/database'; import { documents, audiobooks, @@ -8,7 +8,7 @@ import { ttsSegmentVariants, userPreferences, userDocumentProgress, -} from '@/db/schema'; +} from '@openreader/database/schema'; import { eq, and, inArray } from 'drizzle-orm'; import { UNCLAIMED_USER_ID } from '../storage/docstore-legacy'; import { cleanupClaimedLegacyFsSources } from './legacy-fs-claim-cleanup'; diff --git a/src/lib/server/user/data-cleanup.ts b/src/lib/server/user/data-cleanup.ts index fd3845f..b096ab2 100644 --- a/src/lib/server/user/data-cleanup.ts +++ b/src/lib/server/user/data-cleanup.ts @@ -10,10 +10,10 @@ * and failures block the deletion. */ -import { db } from '@/db'; -import { documents, audiobooks, userJobEvents, userTtsChars } from '@/db/schema'; -import * as authSchemaSqlite from '@/db/schema_auth_sqlite'; -import * as authSchemaPostgres from '@/db/schema_auth_postgres'; +import { db } from '@openreader/database'; +import { documents, audiobooks, userJobEvents, userTtsChars } from '@openreader/database/schema'; +import * as authSchemaSqlite from '@openreader/database/schema-auth-sqlite'; +import * as authSchemaPostgres from '@openreader/database/schema-auth-postgres'; import { eq } from 'drizzle-orm'; import { getS3Config, isS3Configured } from '@/lib/server/storage/s3'; import { diff --git a/tests/global-teardown.ts b/tests/global-teardown.ts index 826a85e..9a4d339 100644 --- a/tests/global-teardown.ts +++ b/tests/global-teardown.ts @@ -1,13 +1,13 @@ import { ListObjectsV2Command } from '@aws-sdk/client-s3'; import { and, eq, inArray, like, ne } from 'drizzle-orm'; -import { db } from '../src/db'; -import { audiobooks, audiobookChapters, documentPreviews, documents } from '../src/db/schema'; +import { db } from '@openreader/database'; +import { audiobooks, audiobookChapters, documentPreviews, documents } from '@openreader/database/schema'; import { deleteDocumentPrefix } from '../src/lib/server/documents/blobstore'; import { deleteAudiobookPrefix } from '../src/lib/server/audiobooks/blobstore'; import { deleteTtsSegmentPrefix } from '../src/lib/server/tts/segments-blobstore'; import { getS3Client, getS3Config, isS3Configured } from '../src/lib/server/storage/s3'; -import * as authSchemaSqlite from '../src/db/schema_auth_sqlite'; -import * as authSchemaPostgres from '../src/db/schema_auth_postgres'; +import * as authSchemaSqlite from '@openreader/database/schema-auth-sqlite'; +import * as authSchemaPostgres from '@openreader/database/schema-auth-postgres'; function chunk(items: T[], size: number): T[][] { if (items.length === 0) return []; diff --git a/tests/unit/admin-providers-validation.vitest.spec.ts b/tests/unit/admin-providers-validation.vitest.spec.ts index 6511e0d..6638316 100644 --- a/tests/unit/admin-providers-validation.vitest.spec.ts +++ b/tests/unit/admin-providers-validation.vitest.spec.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; import { inArray } from 'drizzle-orm'; -import { db } from '../../src/db'; -import { adminProviders } from '../../src/db/schema'; +import { db } from '@openreader/database'; +import { adminProviders } from '@openreader/database/schema'; import { AdminProviderError, diff --git a/tests/unit/compute-worker-client-contract.vitest.spec.ts b/tests/unit/compute-worker-client-contract.vitest.spec.ts index 39adac6..3eaa1e2 100644 --- a/tests/unit/compute-worker-client-contract.vitest.spec.ts +++ b/tests/unit/compute-worker-client-contract.vitest.spec.ts @@ -1,6 +1,6 @@ import { afterEach, describe, expect, test } from 'vitest'; -import { createComputeWorkerApp, type ComputeWorkerApp } from '../../compute-worker/src/api/app'; -import { FakeControlPlane } from '../../compute-worker/tests/fixtures/fake-control-plane'; +import { createComputeWorkerApp, type ComputeWorkerApp } from '../../packages/compute-worker/src/api/app'; +import { FakeControlPlane } from '../../packages/compute-worker/tests/fixtures/fake-control-plane'; import { ComputeWorkerClient } from '../../src/lib/server/compute-worker/client'; describe('ComputeWorkerClient contract', () => { diff --git a/tests/unit/document-blob-lease.vitest.spec.ts b/tests/unit/document-blob-lease.vitest.spec.ts index 684d75b..c6aede2 100644 --- a/tests/unit/document-blob-lease.vitest.spec.ts +++ b/tests/unit/document-blob-lease.vitest.spec.ts @@ -1,10 +1,10 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; import Database from 'better-sqlite3'; import { drizzle } from 'drizzle-orm/better-sqlite3'; -import * as sqliteSchema from '../../src/db/schema_sqlite'; +import * as sqliteSchema from '@openreader/database/schema-sqlite'; const holder = vi.hoisted(() => ({ db: null as unknown as ReturnType })); -vi.mock('@/db', () => ({ +vi.mock('@openreader/database', () => ({ get db() { return holder.db; }, diff --git a/tests/unit/document-delete-cleanup.vitest.spec.ts b/tests/unit/document-delete-cleanup.vitest.spec.ts index 962905e..618f1fe 100644 --- a/tests/unit/document-delete-cleanup.vitest.spec.ts +++ b/tests/unit/document-delete-cleanup.vitest.spec.ts @@ -12,7 +12,7 @@ function resultBuilder(result: unknown[]) { }; } -vi.mock('@/db', () => { +vi.mock('@openreader/database', () => { const database = { delete: vi.fn(() => ({ where: vi.fn(() => ({ diff --git a/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts b/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts index 85b49cb..a6d2822 100644 --- a/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts +++ b/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts @@ -15,7 +15,7 @@ const hoisted = vi.hoisted(() => ({ openOperationEvents: vi.fn(), })); -vi.mock('@/db', () => ({ +vi.mock('@openreader/database', () => ({ get db() { return hoisted.db; }, diff --git a/tests/unit/pdf-parsed-route-worker-flow.vitest.spec.ts b/tests/unit/pdf-parsed-route-worker-flow.vitest.spec.ts index 7160c92..055bd3c 100644 --- a/tests/unit/pdf-parsed-route-worker-flow.vitest.spec.ts +++ b/tests/unit/pdf-parsed-route-worker-flow.vitest.spec.ts @@ -20,7 +20,7 @@ const hoisted = vi.hoisted(() => ({ buildComputeRateLimitedResponse: vi.fn(), })); -vi.mock('@/db', () => ({ +vi.mock('@openreader/database', () => ({ get db() { return hoisted.db; }, diff --git a/tests/unit/reap-orphaned-blobs.vitest.spec.ts b/tests/unit/reap-orphaned-blobs.vitest.spec.ts index 667a242..7b5b8b5 100644 --- a/tests/unit/reap-orphaned-blobs.vitest.spec.ts +++ b/tests/unit/reap-orphaned-blobs.vitest.spec.ts @@ -26,7 +26,7 @@ vi.mock('@/lib/server/documents/blob-lease', () => ({ })), })); vi.mock('@/lib/server/errors/logging', () => ({ logDegraded: vi.fn() })); -vi.mock('@/db', () => ({ +vi.mock('@openreader/database', () => ({ db: { select: () => ({ from: () => ({ diff --git a/tests/unit/runtime-seed-json.vitest.spec.ts b/tests/unit/runtime-seed-json.vitest.spec.ts index fd710bf..4dfc520 100644 --- a/tests/unit/runtime-seed-json.vitest.spec.ts +++ b/tests/unit/runtime-seed-json.vitest.spec.ts @@ -1,8 +1,8 @@ import { and, eq, inArray, not } from 'drizzle-orm'; import { describe, expect, test } from 'vitest'; -import { db } from '../../src/db'; -import { adminProviders, adminSettings } from '../../src/db/schema'; +import { db } from '@openreader/database'; +import { adminProviders, adminSettings } from '@openreader/database/schema'; import { RUNTIME_KEYS, seedRuntimeConfigFromValues } from '../../src/lib/server/admin/settings'; import { __seedInternals } from '../../src/lib/server/admin/seed'; diff --git a/tests/unit/scheduled-tasks-engine.vitest.spec.ts b/tests/unit/scheduled-tasks-engine.vitest.spec.ts index 715bac4..124fd1e 100644 --- a/tests/unit/scheduled-tasks-engine.vitest.spec.ts +++ b/tests/unit/scheduled-tasks-engine.vitest.spec.ts @@ -2,12 +2,12 @@ import { beforeEach, describe, expect, test, vi } from 'vitest'; import Database from 'better-sqlite3'; import { drizzle } from 'drizzle-orm/better-sqlite3'; import { eq } from 'drizzle-orm'; -import * as sqliteSchema from '../../src/db/schema_sqlite'; +import * as sqliteSchema from '@openreader/database/schema-sqlite'; // Back the engine with a real in-memory SQLite so the CAS claim, due-detection, // and nextRunAt arithmetic are exercised against real SQL rather than mocks. const holder = vi.hoisted(() => ({ db: null as unknown as ReturnType })); -vi.mock('@/db', () => ({ +vi.mock('@openreader/database', () => ({ get db() { return holder.db; }, diff --git a/tests/unit/transfer-user-documents.vitest.spec.ts b/tests/unit/transfer-user-documents.vitest.spec.ts index 1444d1d..11c93d7 100644 --- a/tests/unit/transfer-user-documents.vitest.spec.ts +++ b/tests/unit/transfer-user-documents.vitest.spec.ts @@ -7,7 +7,7 @@ import { documents, ttsSegmentEntries, ttsSegmentVariants, -} from '../../src/db/schema_sqlite'; +} from '@openreader/database/schema-sqlite'; import { transferUserDocumentSettings, transferUserDocuments } from '../../src/lib/server/user/claim-data'; describe('transferUserDocuments', () => { diff --git a/tests/unit/tts-segments-cache.vitest.spec.ts b/tests/unit/tts-segments-cache.vitest.spec.ts index 18ce89b..70fdf0c 100644 --- a/tests/unit/tts-segments-cache.vitest.spec.ts +++ b/tests/unit/tts-segments-cache.vitest.spec.ts @@ -15,7 +15,7 @@ function resultBuilder(result: unknown[]) { }; } -vi.mock('@/db', () => ({ +vi.mock('@openreader/database', () => ({ db: { select: vi.fn(() => ({ from: vi.fn(() => resultBuilder(mocks.selectResults.shift() ?? [])), diff --git a/tests/unit/user-data-cleanup.vitest.spec.ts b/tests/unit/user-data-cleanup.vitest.spec.ts index ff8496b..af1cdad 100644 --- a/tests/unit/user-data-cleanup.vitest.spec.ts +++ b/tests/unit/user-data-cleanup.vitest.spec.ts @@ -18,7 +18,7 @@ function resultBuilder(result: unknown[]) { }; } -vi.mock('@/db', () => { +vi.mock('@openreader/database', () => { const database = { select: vi.fn(() => ({ from: vi.fn(() => ({ diff --git a/vitest.config.ts b/vitest.config.ts index 99417d3..b04ccea 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -38,8 +38,8 @@ export default defineConfig({ test: { name: 'compute-worker', environment: 'node', - include: ['compute-worker/tests/{unit,api,compute}/**/*.test.ts'], - setupFiles: ['compute-worker/tests/setup-env.ts'], + include: ['packages/compute-worker/tests/{unit,api,compute}/**/*.test.ts'], + setupFiles: ['packages/compute-worker/tests/setup-env.ts'], }, }, ], From 9520e26a0074af8a22c091f1d4795e0adc42e6a2 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 13 Jun 2026 14:03:54 -0600 Subject: [PATCH 07/11] fix(compute-worker): wrap state store methods to ensure correct invocation Fix/refactor reconciliation logic to wrap getOpStateRecord and listOpStates methods from the state store, ensuring they are invoked as functions rather than accessed as properties. This improves type safety and prevents potential runtime errors from improper method usage. --- packages/compute-worker/src/operations/reconciliation.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/compute-worker/src/operations/reconciliation.ts b/packages/compute-worker/src/operations/reconciliation.ts index df8dc9e..921587f 100644 --- a/packages/compute-worker/src/operations/reconciliation.ts +++ b/packages/compute-worker/src/operations/reconciliation.ts @@ -55,8 +55,8 @@ export function createOperationReconciler(input: { recoveryPromise = (async () => { const recoveredStates = await recoverOrphanedOperations({ operationStateStore: { - getOpStateRecord: input.stateStore.getOpStateRecord!, - listOpStates: input.stateStore.listOpStates!, + getOpStateRecord: (opId) => input.stateStore.getOpStateRecord!(opId), + listOpStates: () => input.stateStore.listOpStates!(), }, orchestrator: { markFailedIfUnchanged: async (request) => { From 6b13a3394beab493eea0cf824bad4c7938e2e814 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 13 Jun 2026 14:09:37 -0600 Subject: [PATCH 08/11] docs(deploy): add local-slim and local-full Docker Compose examples and update docs Introduce compose.local-slim.yml and compose.local-full.yml for local builds of OpenReader and compute-worker. Update deployment documentation to describe new compose variants and usage. Adjust package.json scripts to support local-slim and local-full workflows. Rename compose files and update stack names for clarity. --- docker/examples/compose.full.yml | 2 +- ...mpose.local.yml => compose.local-full.yml} | 2 +- docker/examples/compose.local-slim.yml | 52 +++++++++++++++++ docker/examples/compose.yml | 2 +- docs-site/docs/deploy/docker-compose.md | 58 +++++++++++++------ package.json | 4 +- 6 files changed, 98 insertions(+), 22 deletions(-) rename docker/examples/{compose.local.yml => compose.local-full.yml} (99%) create mode 100644 docker/examples/compose.local-slim.yml diff --git a/docker/examples/compose.full.yml b/docker/examples/compose.full.yml index bdf2270..8776f64 100644 --- a/docker/examples/compose.full.yml +++ b/docker/examples/compose.full.yml @@ -1,4 +1,4 @@ -name: openreader-example +name: openreader-full services: openreader: diff --git a/docker/examples/compose.local.yml b/docker/examples/compose.local-full.yml similarity index 99% rename from docker/examples/compose.local.yml rename to docker/examples/compose.local-full.yml index 9c0a6d6..b035247 100644 --- a/docker/examples/compose.local.yml +++ b/docker/examples/compose.local-full.yml @@ -1,4 +1,4 @@ -name: openreader-local-build +name: openreader-local-full-build services: openreader: diff --git a/docker/examples/compose.local-slim.yml b/docker/examples/compose.local-slim.yml new file mode 100644 index 0000000..d4795aa --- /dev/null +++ b/docker/examples/compose.local-slim.yml @@ -0,0 +1,52 @@ +name: openreader-local-slim-build + +services: + openreader: + image: openreader:local + build: + context: ../.. + dockerfile: Dockerfile + depends_on: + kokoro-tts: + condition: service_started + environment: + BASE_URL: ${BASE_URL:-http://localhost:3003} + AUTH_SECRET: ${AUTH_SECRET:-local-openreader-auth-secret-change-me} + RUNTIME_SEED_JSON: |- + { + "version": 1, + "runtimeConfig": { + "defaultTtsProvider": "kokoro" + }, + "providers": [ + { + "slug": "kokoro", + "displayName": "Kokoro", + "providerType": "custom-openai", + "baseUrl": "http://kokoro-tts:8880/v1", + "defaultModel": "kokoro", + "enabled": true + } + ] + } + ports: + - "3003:3003" + - "8333:8333" + volumes: + - openreader-docstore:/app/docstore + + kokoro-tts: + image: ghcr.io/remsky/kokoro-fastapi-cpu:v0.2.4 + environment: + ONNX_NUM_THREADS: 8 + ONNX_INTER_OP_THREADS: 4 + ONNX_EXECUTION_MODE: parallel + ONNX_OPTIMIZATION_LEVEL: all + ONNX_MEMORY_PATTERN: "true" + ONNX_ARENA_EXTEND_STRATEGY: kNextPowerOfTwo + API_LOG_LEVEL: DEBUG + ports: + - "8880:8880" + +volumes: + openreader-docstore: diff --git a/docker/examples/compose.yml b/docker/examples/compose.yml index 4745d0e..e83ab7a 100644 --- a/docker/examples/compose.yml +++ b/docker/examples/compose.yml @@ -1,4 +1,4 @@ -name: openreader-slim-example +name: openreader-slim services: openreader: diff --git a/docs-site/docs/deploy/docker-compose.md b/docs-site/docs/deploy/docker-compose.md index 78786d3..08964d7 100644 --- a/docs-site/docs/deploy/docker-compose.md +++ b/docs-site/docs/deploy/docker-compose.md @@ -1,6 +1,6 @@ --- title: Docker Compose -description: Run OpenReader with the slim, full, or local-build Docker Compose examples. +description: Run OpenReader with the slim, full, local-slim, or local-full Docker Compose examples. --- import Tabs from '@theme/Tabs'; @@ -8,7 +8,8 @@ import TabItem from '@theme/TabItem'; Use these examples to run OpenReader with Kokoro-FastAPI and persistent storage. Choose the slim stack for the simplest deployment, or the full stack when you want PostgreSQL, SeaweedFS, NATS, -and the compute worker as separate containers. +and the compute worker as separate containers. Local build variants are also available for both slim +and full stacks to build the application from your current checkout. ## Prerequisites @@ -54,31 +55,42 @@ For details about running the worker separately, see [Compute Worker](./compute-worker). - + -The local-build example uses the full multi-container layout, but builds the OpenReader app and -compute-worker images from the current checkout. +The local-slim example runs a slim setup (OpenReader and Kokoro-FastAPI), but builds the OpenReader app image from the current checkout. ```bash -docker compose -f docker/examples/compose.local.yml up --build +docker compose -f docker/examples/compose.local-slim.yml up --build # Repository convenience command: pnpm compose:local ``` -Compose file: [`docker/examples/compose.local.yml`](https://github.com/richardr1126/openreader/blob/main/docker/examples/compose.local.yml) +Compose file: [`docker/examples/compose.local-slim.yml`](https://github.com/richardr1126/openreader/blob/main/docker/examples/compose.local-slim.yml) + + + + +The local-full example uses the full multi-container layout, but builds the OpenReader app and compute-worker images from the current checkout. + +```bash +docker compose -f docker/examples/compose.local-full.yml up --build +# Repository convenience command: pnpm compose:local:full +``` + +Compose file: [`docker/examples/compose.local-full.yml`](https://github.com/richardr1126/openreader/blob/main/docker/examples/compose.local-full.yml) ## Included services -| Service | Slim | Full | Local Build | -| --- | --- | --- | --- | -| OpenReader | Published image | Published image | Local build | -| Kokoro-FastAPI | Container | Container | Container | -| Database | Embedded SQLite | PostgreSQL container | PostgreSQL container | -| SeaweedFS | Embedded | Container | Container | -| NATS | Embedded | Container | Container | -| Compute worker | Embedded | Published image | Local build | +| Service | Slim | Full | Local Slim | Local Full | +| --- | --- | --- | --- | --- | +| OpenReader | Published image | Published image | Local build | Local build | +| Kokoro-FastAPI | Container | Container | Container | Container | +| Database | Embedded SQLite | PostgreSQL container | Embedded SQLite | PostgreSQL container | +| SeaweedFS | Embedded | Container | Embedded | Container | +| NATS | Embedded | Container | Embedded | Container | +| Compute worker | Embedded | Published image | Embedded | Local build | On first boot, `RUNTIME_SEED_JSON` creates an enabled Kokoro shared provider and selects it as the default TTS provider. @@ -118,15 +130,25 @@ docker compose -f docker/examples/compose.full.yml up ``` - + ```bash BASE_URL=http://192.168.0.XXX:3003 \ S3_ENDPOINT=http://192.168.0.XXX:8333 \ -docker compose -f docker/examples/compose.local.yml up --build +docker compose -f docker/examples/compose.local-slim.yml up --build # Repository convenience command: pnpm compose:local ``` + + + +```bash +BASE_URL=http://192.168.0.XXX:3003 \ +S3_ENDPOINT=http://192.168.0.XXX:8333 \ +docker compose -f docker/examples/compose.local-full.yml up --build +# Repository convenience command: pnpm compose:local:full +``` + @@ -134,7 +156,7 @@ Replace `192.168.0.XXX` with your Docker host's LAN IP and allow inbound TCP por `8333` through its firewall. :::info Internal full-stack endpoint -The full and local-build compute workers continue using `http://seaweedfs:8333` internally. +The full and local-full compute workers continue using `http://seaweedfs:8333` internally. `S3_ENDPOINT` configures the app endpoint and browser-facing presigned URLs. ::: diff --git a/package.json b/package.json index 9c9b62d..55ec014 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,9 @@ "test:compute": "vitest run --project compute-worker", "compose": "docker compose -f docker/examples/compose.yml up", "compose:full": "docker compose -f docker/examples/compose.full.yml up", - "compose:local": "docker compose -f docker/examples/compose.local.yml up --build", + "compose:local": "docker compose -f docker/examples/compose.local-slim.yml up --build", + "compose:local:slim": "docker compose -f docker/examples/compose.local-slim.yml up --build", + "compose:local:full": "docker compose -f docker/examples/compose.local-full.yml up --build", "migrate": "pnpm --filter @openreader/database migrate", "migrate-fs": "pnpm --filter @openreader/bootstrap migrate-storage", "migrate-fs:dry-run": "pnpm --filter @openreader/bootstrap migrate-storage --dry-run true", From 74a96c0d159f20c1a9cd6f949b82fb2fd89a796d Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 13 Jun 2026 14:18:45 -0600 Subject: [PATCH 09/11] refactor(compute-worker): improve docstore directory resolution logic Update resolveDocstoreDir to handle multiple environments: - Prefer monorepo root for local development - Check process.env.PWD for containerized bootstrap - Fallback to standard /app/docstore path - Default to process cwd Enhances robustness of docstore path resolution across deployment scenarios. --- .../compute-worker/src/infrastructure/platform.ts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/compute-worker/src/infrastructure/platform.ts b/packages/compute-worker/src/infrastructure/platform.ts index 9f029e4..3752a17 100644 --- a/packages/compute-worker/src/infrastructure/platform.ts +++ b/packages/compute-worker/src/infrastructure/platform.ts @@ -14,8 +14,23 @@ function findMonorepoRoot(startDir: string): string | null { } function resolveDocstoreDir(): string { + // 1. Try to find the monorepo root (works in local development) const repoRoot = findMonorepoRoot(process.cwd()); if (repoRoot) return path.join(repoRoot, 'docstore'); + + // 2. In a containerized environment, the parent bootstrap process runs in /app + // and the child process inherits process.env.PWD = '/app'. + if (process.env.PWD) { + const pwdDocstore = path.join(process.env.PWD, 'docstore'); + if (fs.existsSync(pwdDocstore)) return pwdDocstore; + } + + // 3. Fallback to the standard container app docstore path if it exists + if (fs.existsSync('/app/docstore')) { + return '/app/docstore'; + } + + // 4. Fallback to the process cwd docstore return path.join(process.cwd(), 'docstore'); } From ee979205b3f3e3eaea6d8dcdca3eecff981e6510 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 13 Jun 2026 14:35:27 -0600 Subject: [PATCH 10/11] fix(storage): unify workspace root resolution for docstore and migrations Adopt a consistent findWorkspaceRoot helper across bootstrap, storage, and database modules to reliably locate the monorepo root. Update environment file loading and docstore path construction to use the workspace root, ensuring correct file resolution in multi-package setups. This addresses issues with relative paths and improves compatibility in monorepo and containerized environments. --- packages/bootstrap/src/cli.mjs | 28 +++++++++++++---- packages/bootstrap/src/storage-migration.mjs | 33 ++++++++++++++++---- packages/database/src/index.ts | 18 ++++++++++- packages/database/src/migrate.mjs | 21 +++++++++++-- 4 files changed, 85 insertions(+), 15 deletions(-) diff --git a/packages/bootstrap/src/cli.mjs b/packages/bootstrap/src/cli.mjs index 887ecd7..04442d3 100644 --- a/packages/bootstrap/src/cli.mjs +++ b/packages/bootstrap/src/cli.mjs @@ -21,10 +21,26 @@ import { } from './embedded-seaweedfs.mjs'; import { resolveEmbeddedWorkerLaunch } from './embedded-worker.mjs'; +function findWorkspaceRoot(startDir = process.cwd()) { + let dir = startDir; + while (true) { + if (fs.existsSync(path.join(dir, 'pnpm-workspace.yaml'))) { + return dir; + } + const parent = path.dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } + return startDir; +} + +const workspaceRoot = findWorkspaceRoot(process.cwd()); + function loadEnvFiles() { - const cwd = process.cwd(); - const envPath = path.join(cwd, '.env'); - const envLocalPath = path.join(cwd, '.env.local'); + const envPath = path.join(workspaceRoot, '.env'); + const envLocalPath = path.join(workspaceRoot, '.env.local'); if (fs.existsSync(envPath)) { dotenv.config({ path: envPath }); @@ -111,7 +127,7 @@ function spawnMainCommand(command, env) { async function runDbMigrations(env) { console.log('Running database migrations...'); - await runMigrations({ cwd: process.cwd(), env }); + await runMigrations({ cwd: workspaceRoot, env }); } function runStorageMigrations(env) { @@ -278,7 +294,7 @@ async function main() { } if (useEmbeddedWeed) { - runtimeEnv.WEED_MINI_DIR = withDefault(runtimeEnv.WEED_MINI_DIR, 'docstore/seaweedfs'); + runtimeEnv.WEED_MINI_DIR = withDefault(runtimeEnv.WEED_MINI_DIR, path.join(workspaceRoot, 'docstore/seaweedfs')); runtimeEnv.WEED_MINI_WAIT_SEC = withDefault(runtimeEnv.WEED_MINI_WAIT_SEC, '20'); runtimeEnv.S3_BUCKET = withDefault(runtimeEnv.S3_BUCKET, 'openreader-documents'); runtimeEnv.S3_REGION = withDefault(runtimeEnv.S3_REGION, 'us-east-1'); @@ -371,7 +387,7 @@ async function main() { runtimeEnv.COMPUTE_WORKER_HOST = withDefault(runtimeEnv.COMPUTE_WORKER_HOST, '127.0.0.1'); runtimeEnv.COMPUTE_NATS_REPLICAS = withDefault(runtimeEnv.COMPUTE_NATS_REPLICAS, '1'); - const natsStoreDir = withDefault(runtimeEnv.EMBEDDED_NATS_STORE_DIR, 'docstore/nats/jetstream'); + const natsStoreDir = withDefault(runtimeEnv.EMBEDDED_NATS_STORE_DIR, path.join(workspaceRoot, 'docstore/nats/jetstream')); fs.mkdirSync(natsStoreDir, { recursive: true }); console.log(`Starting embedded nats-server on 127.0.0.1:${embeddedNatsPort}...`); diff --git a/packages/bootstrap/src/storage-migration.mjs b/packages/bootstrap/src/storage-migration.mjs index e2f0c7d..53f159d 100644 --- a/packages/bootstrap/src/storage-migration.mjs +++ b/packages/bootstrap/src/storage-migration.mjs @@ -17,7 +17,23 @@ const { Pool } = require('pg'); const BetterSqlite3 = require('better-sqlite3'); const ffmpegStatic = require('ffmpeg-static'); -const DOCSTORE_DIR = path.join(process.cwd(), 'docstore'); +function findWorkspaceRoot(startDir = process.cwd()) { + let dir = startDir; + while (true) { + if (fs.existsSync(path.join(dir, 'pnpm-workspace.yaml'))) { + return dir; + } + const parent = path.dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } + return startDir; +} + +const workspaceRoot = findWorkspaceRoot(process.cwd()); +const DOCSTORE_DIR = path.join(workspaceRoot, 'docstore'); const DOCUMENTS_V1_DIR = path.join(DOCSTORE_DIR, 'documents_v1'); const AUDIOBOOKS_V1_DIR = path.join(DOCSTORE_DIR, 'audiobooks_v1'); const UNCLAIMED_USER_ID = 'unclaimed'; @@ -27,10 +43,15 @@ const SAFE_AUDIOBOOK_ID_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; function loadEnvFiles() { - const envPath = path.join(process.cwd(), '.env'); - const envLocalPath = path.join(process.cwd(), '.env.local'); - if (fs.existsSync(envPath)) dotenv.config({ path: envPath }); - if (fs.existsSync(envLocalPath)) dotenv.config({ path: envLocalPath, override: true }); + const envPath = path.join(workspaceRoot, '.env'); + const envLocalPath = path.join(workspaceRoot, '.env.local'); + + if (fs.existsSync(envPath)) { + dotenv.config({ path: envPath }); + } + if (fs.existsSync(envLocalPath)) { + dotenv.config({ path: envLocalPath, override: true }); + } } function parseBool(value, fallback = false) { @@ -688,7 +709,7 @@ function openDatabase() { }; } - const dbPath = path.join(process.cwd(), 'docstore', 'sqlite3.db'); + const dbPath = path.join(DOCSTORE_DIR, 'sqlite3.db'); fs.mkdirSync(path.dirname(dbPath), { recursive: true }); const sqlite = new BetterSqlite3(dbPath); return { diff --git a/packages/database/src/index.ts b/packages/database/src/index.ts index a6e43ec..f20fd79 100644 --- a/packages/database/src/index.ts +++ b/packages/database/src/index.ts @@ -59,6 +59,21 @@ export function ensureSystemUserExists(userId: string) { } } +function findWorkspaceRoot(startDir: string = process.cwd()): string { + let dir = startDir; + while (true) { + if (fs.existsSync(path.join(dir, 'pnpm-workspace.yaml'))) { + return dir; + } + const parent = path.dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } + return startDir; +} + function getDrizzleDB() { if (dbInstance) return dbInstance; @@ -79,7 +94,8 @@ function getDrizzleDB() { const { drizzle: drizzleSqlite } = require('drizzle-orm/better-sqlite3'); // eslint-disable-next-line @typescript-eslint/no-require-imports const Database = require('better-sqlite3'); - const dbPath = path.join(process.cwd(), 'docstore', 'sqlite3.db'); + const workspaceRoot = findWorkspaceRoot(process.cwd()); + const dbPath = path.join(workspaceRoot, 'docstore', 'sqlite3.db'); const dir = path.dirname(dbPath); if (!fs.existsSync(dir)) { fs.mkdirSync(dir, { recursive: true }); diff --git a/packages/database/src/migrate.mjs b/packages/database/src/migrate.mjs index 8aba886..b022aad 100644 --- a/packages/database/src/migrate.mjs +++ b/packages/database/src/migrate.mjs @@ -13,8 +13,24 @@ function loadEnvFiles(cwd) { if (fs.existsSync(envLocalPath)) dotenv.config({ path: envLocalPath, override: true }); } +function findWorkspaceRoot(startDir = process.cwd()) { + let dir = startDir; + while (true) { + if (fs.existsSync(path.join(dir, 'pnpm-workspace.yaml'))) { + return dir; + } + const parent = path.dirname(dir); + if (parent === dir) { + break; + } + dir = parent; + } + return startDir; +} + export async function runMigrations({ cwd = process.cwd(), env = process.env } = {}) { - loadEnvFiles(cwd); + const workspaceRoot = findWorkspaceRoot(cwd); + loadEnvFiles(workspaceRoot); if (env.POSTGRES_URL?.trim()) { const [{ drizzle }, { migrate }, { Pool }] = await Promise.all([ @@ -33,7 +49,7 @@ export async function runMigrations({ cwd = process.cwd(), env = process.env } = return; } - const dbPath = path.join(cwd, 'docstore', 'sqlite3.db'); + const dbPath = path.join(workspaceRoot, 'docstore', 'sqlite3.db'); fs.mkdirSync(path.dirname(dbPath), { recursive: true }); const [{ drizzle }, { migrate }, { default: BetterSqlite3 }] = await Promise.all([ import('drizzle-orm/better-sqlite3'), @@ -49,3 +65,4 @@ export async function runMigrations({ cwd = process.cwd(), env = process.env } = sqlite.close(); } } + From d4d8f0a590de853419dd91b64c36ea3f7fcd26e0 Mon Sep 17 00:00:00 2001 From: Richard R Date: Sat, 13 Jun 2026 14:38:32 -0600 Subject: [PATCH 11/11] refactor(bootstrap): set child process cwd to workspace root in CLI spawn Update CLI command spawning to explicitly set the working directory to the workspace root. This ensures subprocesses execute in a consistent environment, improving reliability in monorepo setups. --- packages/bootstrap/src/cli.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/bootstrap/src/cli.mjs b/packages/bootstrap/src/cli.mjs index 04442d3..21e5f84 100644 --- a/packages/bootstrap/src/cli.mjs +++ b/packages/bootstrap/src/cli.mjs @@ -97,6 +97,7 @@ function forwardChildStream(stream, target) { function spawnMainCommand(command, env) { const [cmd, ...args] = command; const child = spawn(cmd, args, { + cwd: workspaceRoot, env, stdio: 'inherit', shell: process.platform === 'win32',