diff --git a/.dockerignore b/.dockerignore index 6c309d7..51d3428 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ .env .env.* +**/*.creds README.md .next node_modules diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 2c50173..d7f6733 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -62,13 +62,13 @@ jobs: platform: linux/amd64 runner: ubuntu-24.04 context: . - dockerfile: ./compute/worker/Dockerfile + dockerfile: ./compute-worker/Dockerfile - image_target: compute-worker arch: arm64 platform: linux/arm64 runner: ubuntu-24.04-arm context: . - dockerfile: ./compute/worker/Dockerfile + dockerfile: ./compute-worker/Dockerfile steps: - name: Checkout repository diff --git a/.github/workflows/vitest.yml b/.github/workflows/vitest.yml index 27c2102..c659070 100644 --- a/.github/workflows/vitest.yml +++ b/.github/workflows/vitest.yml @@ -25,3 +25,5 @@ jobs: run: pnpm migrate - name: Run Vitest suites run: pnpm test:unit + - name: Verify compute worker contract and boundary + run: pnpm compute:openapi:check && pnpm check:compute-boundary diff --git a/Dockerfile b/Dockerfile index c6b5ac6..93e32ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,8 +20,7 @@ WORKDIR /app # Copy workspace manifests needed for dependency installation COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ -COPY compute/core/package.json ./compute/core/package.json -COPY compute/worker/package.json ./compute/worker/package.json +COPY compute-worker/package.json ./compute-worker/package.json COPY docker/entrypoint-migration-tools/package.json ./docker/entrypoint-migration-tools/package.json # Install dependencies @@ -88,7 +87,7 @@ COPY --from=app-builder /app/THIRD_PARTY_LICENSES /licenses # Include SeaweedFS license text for the copied weed binary. COPY --from=seaweedfs-builder /tmp/SeaweedFS-LICENSE.txt /licenses/SeaweedFS-LICENSE.txt # Include static model notices for runtime-downloaded assets. -COPY --from=app-builder /app/compute/core/src/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt +COPY --from=app-builder /app/compute-worker/src/compute/pdf/assets/LICENSE.txt /licenses/pp-doclayoutv3-LICENSE.txt # Copy seaweedfs weed binary for optional embedded local S3. COPY --from=seaweedfs-builder /tmp/weed /usr/local/bin/weed @@ -98,7 +97,7 @@ COPY --from=nats-builder /tmp/nats-server /usr/local/bin/nats-server RUN chmod +x /usr/local/bin/nats-server # Include OpenAI Whisper license text for runtime-downloaded ONNX artifacts. -COPY --from=app-builder /app/compute/core/src/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt +COPY --from=app-builder /app/compute-worker/src/compute/whisper/assets/LICENSE.txt /licenses/openai-whisper-LICENSE.txt # Match the app's historical container port now that standalone server.js # is started directly instead of `next start -p 3003`. diff --git a/compute/worker/.env.example b/compute-worker/.env.example similarity index 100% rename from compute/worker/.env.example rename to compute-worker/.env.example diff --git a/compute/worker/Dockerfile b/compute-worker/Dockerfile similarity index 72% rename from compute/worker/Dockerfile rename to compute-worker/Dockerfile index 1b0eb3a..356190f 100644 --- a/compute/worker/Dockerfile +++ b/compute-worker/Dockerfile @@ -5,11 +5,8 @@ RUN npm install -g pnpm@11.1.2 WORKDIR /workspace COPY .npmrc package.json pnpm-lock.yaml pnpm-workspace.yaml ./ -COPY compute/worker/package.json compute/worker/package.json -COPY compute/core/package.json compute/core/package.json - -COPY compute/worker compute/worker -COPY compute/core compute/core +COPY compute-worker/package.json compute-worker/package.json +COPY compute-worker compute-worker RUN pnpm --config.inject-workspace-packages=true --filter @openreader/compute-worker deploy /opt/compute-worker diff --git a/compute/worker/docker-compose.yml b/compute-worker/docker-compose.yml similarity index 75% rename from compute/worker/docker-compose.yml rename to compute-worker/docker-compose.yml index 1bbf550..115eea8 100644 --- a/compute/worker/docker-compose.yml +++ b/compute-worker/docker-compose.yml @@ -11,13 +11,14 @@ services: compute-worker: build: - context: ../.. - dockerfile: compute/worker/Dockerfile + context: .. + dockerfile: compute-worker/Dockerfile container_name: openreader-compute-worker depends_on: - nats env_file: - - ./.env + - path: ./.env + required: false environment: NATS_URL: ${NATS_URL:-nats://nats:4222} COMPUTE_WORKER_HOST: ${COMPUTE_WORKER_HOST:-0.0.0.0} @@ -32,16 +33,12 @@ services: - action: sync+restart path: . target: /workspace - - action: rebuild - path: ../../compute/core - action: rebuild path: ./package.json - action: rebuild - path: ../../compute/core/package.json + path: ../pnpm-lock.yaml - action: rebuild - path: ../../pnpm-lock.yaml - - action: rebuild - path: ../../pnpm-workspace.yaml + path: ../pnpm-workspace.yaml volumes: nats-data: diff --git a/compute-worker/openapi.json b/compute-worker/openapi.json new file mode 100644 index 0000000..c0973f1 --- /dev/null +++ b/compute-worker/openapi.json @@ -0,0 +1,1766 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "OpenReader Compute Worker API", + "version": "1.0.0" + }, + "components": { + "securitySchemes": { + "bearerAuth": { + "type": "http", + "scheme": "bearer" + } + }, + "schemas": { + "ParsedPdfDocument": { + "type": "object", + "properties": { + "schemaVersion": { + "type": "number", + "enum": [ + 1 + ] + }, + "documentId": { + "type": "string" + }, + "parserVersion": { + "type": "string" + }, + "parsedAt": { + "type": "number" + }, + "pages": { + "type": "array", + "items": { + "type": "object", + "properties": { + "pageNumber": { + "type": "number" + }, + "width": { + "type": "number" + }, + "height": { + "type": "number" + }, + "blocks": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "kind": { + "type": "string", + "enum": [ + "abstract", + "algorithm", + "aside_text", + "chart", + "content", + "formula", + "doc_title", + "figure_title", + "footer", + "footnote", + "formula_number", + "header", + "image", + "number", + "paragraph_title", + "reference", + "reference_content", + "seal", + "table", + "text", + "vision_footnote" + ] + }, + "fragments": { + "type": "array", + "items": { + "type": "object", + "properties": { + "page": { + "type": "number" + }, + "bbox": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + }, + { + "type": "number" + }, + { + "type": "number" + } + ] + }, + "text": { + "type": "string" + }, + "readingOrder": { + "type": "number" + }, + "modelConfidence": { + "type": "number" + } + }, + "required": [ + "page", + "bbox", + "text", + "readingOrder" + ], + "additionalProperties": false + } + }, + "text": { + "type": "string" + }, + "parentSectionId": { + "type": "string" + } + }, + "required": [ + "id", + "kind", + "fragments", + "text" + ], + "additionalProperties": false + } + } + }, + "required": [ + "pageNumber", + "width", + "height", + "blocks" + ], + "additionalProperties": false + } + } + }, + "required": [ + "schemaVersion", + "documentId", + "parserVersion", + "parsedAt", + "pages" + ], + "additionalProperties": false + }, + "TTSSentenceAlignment": { + "type": "object", + "properties": { + "sentence": { + "type": "string" + }, + "sentenceIndex": { + "type": "number" + }, + "words": { + "type": "array", + "items": { + "type": "object", + "properties": { + "text": { + "type": "string" + }, + "startSec": { + "type": "number" + }, + "endSec": { + "type": "number" + }, + "charStart": { + "type": "number" + }, + "charEnd": { + "type": "number" + } + }, + "required": [ + "text", + "startSec", + "endSec", + "charStart", + "charEnd" + ], + "additionalProperties": false + } + } + }, + "required": [ + "sentence", + "sentenceIndex", + "words" + ], + "additionalProperties": false + }, + "ArtifactReference": { + "type": "object", + "properties": { + "objectKey": { + "type": "string" + } + }, + "required": [ + "objectKey" + ], + "additionalProperties": false + }, + "ErrorResponse": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + }, + "OperationError": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "PdfLayoutProgress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + }, + "PublicOperation": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + }, + "PublicOperationEvent": { + "type": "object", + "properties": { + "eventId": { + "type": "number" + }, + "snapshot": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + } + }, + "required": [ + "eventId", + "snapshot" + ], + "additionalProperties": false + }, + "PdfLayoutResolution": { + "type": "object", + "properties": { + "artifact": { + "anyOf": [ + { + "type": "object", + "properties": { + "objectKey": { + "type": "string" + } + }, + "required": [ + "objectKey" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "operation": { + "anyOf": [ + { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "artifact", + "operation" + ], + "additionalProperties": false + } + } + }, + "paths": { + "/health/live": { + "get": { + "security": [], + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + } + }, + "required": [ + "ok" + ] + } + } + } + } + } + } + }, + "/health/ready": { + "get": { + "security": [], + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "natsConnected": { + "type": "boolean" + } + }, + "required": [ + "ok", + "natsConnected" + ] + } + } + } + } + } + } + }, + "/v1/whisper-align/operations": { + "post": { + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "text": { + "type": "string", + "minLength": 1 + }, + "lang": { + "type": "string", + "minLength": 1, + "maxLength": 16 + }, + "cacheKey": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "audioObjectKey": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "required": [ + "text", + "audioObjectKey" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + }, + "/v1/pdf-layout/operations": { + "post": { + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "documentId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "namespace": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-zA-Z0-9._-]{1,128}$" + }, + { + "type": "null" + } + ] + }, + "documentObjectKey": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "replaceToken": { + "type": "string", + "minLength": 1, + "maxLength": 256 + } + }, + "required": [ + "documentId", + "namespace", + "documentObjectKey" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "202": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + }, + "/v1/pdf-layout/resolve": { + "post": { + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "documentId": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "namespace": { + "anyOf": [ + { + "type": "string", + "pattern": "^[a-zA-Z0-9._-]{1,128}$" + }, + { + "type": "null" + } + ] + }, + "documentObjectKey": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + } + }, + "required": [ + "documentId", + "namespace", + "documentObjectKey" + ], + "additionalProperties": false + } + } + } + }, + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "artifact": { + "anyOf": [ + { + "type": "object", + "properties": { + "objectKey": { + "type": "string" + } + }, + "required": [ + "objectKey" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + }, + "operation": { + "anyOf": [ + { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "artifact", + "operation" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + }, + "/v1/operations/{opId}": { + "get": { + "parameters": [ + { + "schema": { + "type": "string", + "minLength": 1 + }, + "in": "path", + "name": "opId", + "required": true + } + ], + "responses": { + "200": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "opId": { + "type": "string" + }, + "subject": { + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "whisper_align" + ] + } + }, + "required": [ + "kind" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "pdf_layout" + ] + }, + "documentId": { + "type": "string" + }, + "namespace": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "kind", + "documentId", + "namespace" + ], + "additionalProperties": false + } + ] + }, + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "succeeded", + "failed" + ] + }, + "queuedAt": { + "type": "number" + }, + "updatedAt": { + "type": "number" + }, + "startedAt": { + "type": "number" + }, + "result": {}, + "error": { + "type": "object", + "properties": { + "message": { + "type": "string" + }, + "code": { + "type": "string" + } + }, + "required": [ + "message" + ], + "additionalProperties": false + }, + "timing": { + "type": "object", + "properties": { + "queueWaitMs": { + "type": "number" + }, + "s3FetchMs": { + "type": "number" + }, + "computeMs": { + "type": "number" + } + }, + "additionalProperties": false + }, + "progress": { + "type": "object", + "properties": { + "totalPages": { + "type": "number" + }, + "pagesParsed": { + "type": "number" + }, + "currentPage": { + "type": "number" + }, + "phase": { + "type": "string", + "enum": [ + "infer", + "merge" + ] + } + }, + "required": [ + "totalPages", + "pagesParsed", + "phase" + ], + "additionalProperties": false + } + }, + "required": [ + "opId", + "subject", + "status", + "queuedAt", + "updatedAt" + ], + "additionalProperties": false + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + }, + "404": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + }, + "/v1/operations/{opId}/events": { + "get": { + "parameters": [ + { + "schema": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "number" + } + ] + }, + "in": "query", + "name": "sinceEventId", + "required": false + }, + { + "schema": { + "type": "string", + "minLength": 1 + }, + "in": "path", + "name": "opId", + "required": true + } + ], + "responses": { + "200": { + "description": "Server-sent PublicOperationEvent stream", + "content": { + "application/json": { + "schema": { + "type": "string", + "description": "Server-sent PublicOperationEvent stream" + } + } + } + }, + "400": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + }, + "404": { + "description": "Default Response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + }, + "required": [ + "error" + ], + "additionalProperties": {} + } + } + } + } + } + } + } + }, + "security": [ + { + "bearerAuth": [] + } + ] +} diff --git a/compute/worker/package.json b/compute-worker/package.json similarity index 58% rename from compute/worker/package.json rename to compute-worker/package.json index e07eded..ad31e23 100644 --- a/compute/worker/package.json +++ b/compute-worker/package.json @@ -5,15 +5,22 @@ "type": "module", "scripts": { "dev": "tsx watch src/server.ts", - "start": "tsx src/server.ts" + "start": "tsx src/server.ts", + "openapi:generate": "tsx scripts/generate-openapi.ts" }, "dependencies": { "@aws-sdk/client-s3": "^3.1061.0", + "@fastify/swagger": "^9.0.0", + "@huggingface/tokenizers": "^0.1.3", + "@napi-rs/canvas": "^0.1.100", "@nats-io/jetstream": "^3.4.0", "@nats-io/kv": "^3.4.0", "@nats-io/transport-node": "^3.4.0", - "@openreader/compute-core": "workspace:*", + "ffmpeg-static": "^5.3.0", "fastify": "^5.8.5", + "jszip": "^3.10.1", + "onnxruntime-node": "^1.26.0", + "pdfjs-dist": "4.8.69", "pino": "^10.3.1", "pino-pretty": "^13.1.3", "tsx": "^4.22.4", diff --git a/compute-worker/scripts/generate-openapi.ts b/compute-worker/scripts/generate-openapi.ts new file mode 100644 index 0000000..7af98d5 --- /dev/null +++ b/compute-worker/scripts/generate-openapi.ts @@ -0,0 +1,20 @@ +import { mkdir, writeFile } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; +import { createComputeWorkerApp } from '../src/runtime'; + +process.env.COMPUTE_WORKER_TOKEN ||= 'openapi-generation-token'; +process.env.NATS_URL ||= 'nats://127.0.0.1:4222'; + +const outputPath = resolve(process.cwd(), 'openapi.json'); +const runtime = await createComputeWorkerApp({ + workerToken: process.env.COMPUTE_WORKER_TOKEN, + disableWorkers: true, +}); + +try { + await runtime.app.ready(); + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, `${JSON.stringify(runtime.app.swagger(), null, 2)}\n`, 'utf8'); +} finally { + await runtime.close(); +} diff --git a/compute-worker/src/api/operation-keys.ts b/compute-worker/src/api/operation-keys.ts new file mode 100644 index 0000000..9c990b5 --- /dev/null +++ b/compute-worker/src/api/operation-keys.ts @@ -0,0 +1,56 @@ +import { createHash } from 'node:crypto'; +import { PDF_PARSER_VERSION } from '../compute/pdf/parser-version'; + +function sha256Hex(input: string): string { + return createHash('sha256').update(input).digest('hex'); +} + +export function buildWhisperOperationKey(input: { + audioObjectKey: string; + text: string; + lang?: string; + cacheKey?: string; +}): string { + const cacheKey = input.cacheKey?.trim(); + if (cacheKey) { + return `whisper_align|v1|cache|${cacheKey}|${input.audioObjectKey}`; + } + return [ + 'whisper_align', + 'v1', + input.audioObjectKey, + input.lang ?? '', + sha256Hex(input.text), + ].join('|'); +} + +export function buildPdfOperationKey(input: { + documentId: string; + namespace: string | null; + documentObjectKey: string; + replaceToken?: string; +}, parserVersion = PDF_PARSER_VERSION): string { + return [ + 'pdf_layout', + 'v1', + parserVersion, + input.documentId, + input.namespace ?? '', + input.documentObjectKey, + input.replaceToken?.trim() || '', + ].join('|'); +} + +export function pdfSubjectFromOperationKey(opKey: string): { + kind: 'pdf_layout'; + documentId: string; + namespace: string | null; +} | null { + const [kind, version, , documentId, namespace] = opKey.split('|'); + if (kind !== 'pdf_layout' || version !== 'v1' || !documentId) return null; + return { + kind: 'pdf_layout', + documentId, + namespace: namespace || null, + }; +} diff --git a/compute-worker/src/api/public-operation.ts b/compute-worker/src/api/public-operation.ts new file mode 100644 index 0000000..9ab98a4 --- /dev/null +++ b/compute-worker/src/api/public-operation.ts @@ -0,0 +1,44 @@ +import type { WorkerOperationState } from '../compute/api-contracts'; +import { pdfSubjectFromOperationKey } from './operation-keys'; + +export type PublicOperationSubject = + | { kind: 'whisper_align' } + | { kind: 'pdf_layout'; documentId: string; namespace: string | null }; + +export interface PublicOperation { + opId: string; + subject: PublicOperationSubject; + status: WorkerOperationState['status']; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: Result; + error?: WorkerOperationState['error']; + timing?: WorkerOperationState['timing']; + progress?: WorkerOperationState['progress']; +} + +export interface PublicOperationEvent { + eventId: number; + snapshot: PublicOperation; +} + +export function toPublicOperation( + state: WorkerOperationState, +): PublicOperation { + const subject = state.kind === 'pdf_layout' + ? (pdfSubjectFromOperationKey(state.opKey) ?? { kind: 'pdf_layout', documentId: '', namespace: null }) + : { kind: 'whisper_align' as const }; + return { + opId: state.opId, + subject, + status: state.status, + queuedAt: state.queuedAt, + updatedAt: state.updatedAt, + ...(state.startedAt === undefined ? {} : { startedAt: state.startedAt }), + ...(state.result === undefined ? {} : { result: state.result }), + ...(state.error === undefined ? {} : { error: state.error }), + ...(state.timing === undefined ? {} : { timing: state.timing }), + ...(state.progress === undefined ? {} : { progress: state.progress }), + }; +} diff --git a/compute-worker/src/api/schemas.ts b/compute-worker/src/api/schemas.ts new file mode 100644 index 0000000..66df5c5 --- /dev/null +++ b/compute-worker/src/api/schemas.ts @@ -0,0 +1,134 @@ +import { z } from 'zod'; + +const bboxSchema = z.tuple([z.number(), z.number(), z.number(), z.number()]); +const parsedPdfBlockKindSchema = z.enum([ + 'abstract', 'algorithm', 'aside_text', 'chart', 'content', 'formula', + 'doc_title', 'figure_title', 'footer', 'footnote', 'formula_number', + 'header', 'image', 'number', 'paragraph_title', 'reference', + 'reference_content', 'seal', 'table', 'text', 'vision_footnote', +]); +const documentIdSchema = z.string().trim().regex(/^[a-f0-9]{64}$/i); +const namespaceSchema = z.string().trim().regex(/^[a-zA-Z0-9._-]{1,128}$/).nullable(); + +export const parsedPdfDocumentSchema = z.object({ + schemaVersion: z.literal(1), + documentId: z.string(), + parserVersion: z.string(), + parsedAt: z.number(), + pages: z.array(z.object({ + pageNumber: z.number(), + width: z.number(), + height: z.number(), + blocks: z.array(z.object({ + id: z.string(), + kind: parsedPdfBlockKindSchema, + fragments: z.array(z.object({ + page: z.number(), + bbox: bboxSchema, + text: z.string(), + readingOrder: z.number(), + modelConfidence: z.number().optional(), + })), + text: z.string(), + parentSectionId: z.string().optional(), + })), + })), +}); + +export const ttsSentenceAlignmentSchema = z.object({ + sentence: z.string(), + sentenceIndex: z.number(), + words: z.array(z.object({ + text: z.string(), + startSec: z.number(), + endSec: z.number(), + charStart: z.number(), + charEnd: z.number(), + })), +}); + +export const whisperOperationCreateSchema = z.object({ + text: z.string().trim().min(1), + lang: z.string().trim().min(1).max(16).optional(), + cacheKey: z.string().trim().min(1).max(256).optional(), + audioObjectKey: z.string().trim().min(1).max(2048), +}); + +export const pdfOperationCreateSchema = z.object({ + documentId: documentIdSchema, + namespace: namespaceSchema, + documentObjectKey: z.string().trim().min(1).max(2048), + replaceToken: z.string().trim().min(1).max(256).optional(), +}); + +export const pdfResolveSchema = z.object({ + documentId: documentIdSchema, + namespace: namespaceSchema, + documentObjectKey: z.string().trim().min(1).max(2048), +}); + +export const operationParamsSchema = z.object({ + opId: z.string().trim().min(1), +}); + +export const operationEventsQuerySchema = z.object({ + sinceEventId: z.union([z.string(), z.number()]).optional(), +}); + +export const apiErrorResponseSchema = z.object({ + error: z.string(), +}).passthrough(); + +export const operationErrorSchema = z.object({ + message: z.string(), + code: z.string().optional(), +}); + +export const artifactReferenceSchema = z.object({ + objectKey: z.string(), +}); + +export const pdfLayoutProgressSchema = z.object({ + totalPages: z.number(), + pagesParsed: z.number(), + currentPage: z.number().optional(), + phase: z.enum(['infer', 'merge']), +}); + +export const publicOperationSchema = z.object({ + opId: z.string(), + subject: z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('whisper_align') }), + z.object({ + kind: z.literal('pdf_layout'), + documentId: z.string(), + namespace: z.string().nullable(), + }), + ]), + status: z.enum(['queued', 'running', 'succeeded', 'failed']), + queuedAt: z.number(), + updatedAt: z.number(), + startedAt: z.number().optional(), + result: z.unknown().optional(), + error: operationErrorSchema.optional(), + timing: z.object({ + queueWaitMs: z.number().optional(), + s3FetchMs: z.number().optional(), + computeMs: z.number().optional(), + }).optional(), + progress: pdfLayoutProgressSchema.optional(), +}); + +export const publicOperationEventSchema = z.object({ + eventId: z.number(), + snapshot: publicOperationSchema, +}); + +export const pdfLayoutResolutionSchema = z.object({ + artifact: artifactReferenceSchema.nullable(), + operation: publicOperationSchema.nullable(), +}); + +export function jsonSchema(schema: z.ZodType): Record { + return z.toJSONSchema(schema, { target: 'draft-7' }) as Record; +} diff --git a/compute/core/src/api-contracts/index.ts b/compute-worker/src/compute/api-contracts/index.ts similarity index 100% rename from compute/core/src/api-contracts/index.ts rename to compute-worker/src/compute/api-contracts/index.ts diff --git a/compute/core/src/config/cpu-budget.ts b/compute-worker/src/compute/config/cpu-budget.ts similarity index 100% rename from compute/core/src/config/cpu-budget.ts rename to compute-worker/src/compute/config/cpu-budget.ts diff --git a/compute/core/src/config/timeout.ts b/compute-worker/src/compute/config/timeout.ts similarity index 100% rename from compute/core/src/config/timeout.ts rename to compute-worker/src/compute/config/timeout.ts diff --git a/compute/core/src/control-plane/index.ts b/compute-worker/src/compute/control-plane/index.ts similarity index 100% rename from compute/core/src/control-plane/index.ts rename to compute-worker/src/compute/control-plane/index.ts diff --git a/compute/core/src/control-plane/orchestrator.ts b/compute-worker/src/compute/control-plane/orchestrator.ts similarity index 100% rename from compute/core/src/control-plane/orchestrator.ts rename to compute-worker/src/compute/control-plane/orchestrator.ts diff --git a/compute/core/src/control-plane/sse.ts b/compute-worker/src/compute/control-plane/sse.ts similarity index 100% rename from compute/core/src/control-plane/sse.ts rename to compute-worker/src/compute/control-plane/sse.ts diff --git a/compute/core/src/control-plane/state-machine.ts b/compute-worker/src/compute/control-plane/state-machine.ts similarity index 100% rename from compute/core/src/control-plane/state-machine.ts rename to compute-worker/src/compute/control-plane/state-machine.ts diff --git a/compute/core/src/control-plane/types.ts b/compute-worker/src/compute/control-plane/types.ts similarity index 100% rename from compute/core/src/control-plane/types.ts rename to compute-worker/src/compute/control-plane/types.ts diff --git a/compute/core/src/index.ts b/compute-worker/src/compute/index.ts similarity index 100% rename from compute/core/src/index.ts rename to compute-worker/src/compute/index.ts diff --git a/compute/core/src/local-runtime.ts b/compute-worker/src/compute/local-runtime.ts similarity index 100% rename from compute/core/src/local-runtime.ts rename to compute-worker/src/compute/local-runtime.ts diff --git a/compute/core/src/pdf/assets/LICENSE.txt b/compute-worker/src/compute/pdf/assets/LICENSE.txt similarity index 100% rename from compute/core/src/pdf/assets/LICENSE.txt rename to compute-worker/src/compute/pdf/assets/LICENSE.txt diff --git a/compute/core/src/pdf/assets/manifest.json b/compute-worker/src/compute/pdf/assets/manifest.json similarity index 100% rename from compute/core/src/pdf/assets/manifest.json rename to compute-worker/src/compute/pdf/assets/manifest.json diff --git a/compute/core/src/pdf/merge.ts b/compute-worker/src/compute/pdf/merge.ts similarity index 100% rename from compute/core/src/pdf/merge.ts rename to compute-worker/src/compute/pdf/merge.ts diff --git a/compute/core/src/pdf/model.ts b/compute-worker/src/compute/pdf/model.ts similarity index 100% rename from compute/core/src/pdf/model.ts rename to compute-worker/src/compute/pdf/model.ts diff --git a/compute/core/src/pdf/normalize-text.ts b/compute-worker/src/compute/pdf/normalize-text.ts similarity index 100% rename from compute/core/src/pdf/normalize-text.ts rename to compute-worker/src/compute/pdf/normalize-text.ts diff --git a/compute/core/src/pdf/parse.ts b/compute-worker/src/compute/pdf/parse.ts similarity index 100% rename from compute/core/src/pdf/parse.ts rename to compute-worker/src/compute/pdf/parse.ts diff --git a/compute/core/src/pdf/parser-version-key.ts b/compute-worker/src/compute/pdf/parser-version-key.ts similarity index 100% rename from compute/core/src/pdf/parser-version-key.ts rename to compute-worker/src/compute/pdf/parser-version-key.ts diff --git a/compute/core/src/pdf/parser-version.ts b/compute-worker/src/compute/pdf/parser-version.ts similarity index 100% rename from compute/core/src/pdf/parser-version.ts rename to compute-worker/src/compute/pdf/parser-version.ts diff --git a/compute/core/src/pdf/pdfjs-runtime.ts b/compute-worker/src/compute/pdf/pdfjs-runtime.ts similarity index 100% rename from compute/core/src/pdf/pdfjs-runtime.ts rename to compute-worker/src/compute/pdf/pdfjs-runtime.ts diff --git a/compute/core/src/pdf/render.ts b/compute-worker/src/compute/pdf/render.ts similarity index 100% rename from compute/core/src/pdf/render.ts rename to compute-worker/src/compute/pdf/render.ts diff --git a/compute/core/src/pdf/runLayoutModel.ts b/compute-worker/src/compute/pdf/runLayoutModel.ts similarity index 100% rename from compute/core/src/pdf/runLayoutModel.ts rename to compute-worker/src/compute/pdf/runLayoutModel.ts diff --git a/compute/core/src/pdf/stitch.ts b/compute-worker/src/compute/pdf/stitch.ts similarity index 100% rename from compute/core/src/pdf/stitch.ts rename to compute-worker/src/compute/pdf/stitch.ts diff --git a/compute/core/src/pdf/types.ts b/compute-worker/src/compute/pdf/types.ts similarity index 100% rename from compute/core/src/pdf/types.ts rename to compute-worker/src/compute/pdf/types.ts diff --git a/compute/core/src/platform/docstore.ts b/compute-worker/src/compute/platform/docstore.ts similarity index 100% rename from compute/core/src/platform/docstore.ts rename to compute-worker/src/compute/platform/docstore.ts diff --git a/compute/core/src/platform/ffmpeg.ts b/compute-worker/src/compute/platform/ffmpeg.ts similarity index 100% rename from compute/core/src/platform/ffmpeg.ts rename to compute-worker/src/compute/platform/ffmpeg.ts diff --git a/compute/core/src/types/index.ts b/compute-worker/src/compute/types/index.ts similarity index 100% rename from compute/core/src/types/index.ts rename to compute-worker/src/compute/types/index.ts diff --git a/compute/core/src/types/parsed-pdf.ts b/compute-worker/src/compute/types/parsed-pdf.ts similarity index 100% rename from compute/core/src/types/parsed-pdf.ts rename to compute-worker/src/compute/types/parsed-pdf.ts diff --git a/compute/core/src/types/tts.ts b/compute-worker/src/compute/types/tts.ts similarity index 100% rename from compute/core/src/types/tts.ts rename to compute-worker/src/compute/types/tts.ts diff --git a/compute/core/src/whisper/align.ts b/compute-worker/src/compute/whisper/align.ts similarity index 100% rename from compute/core/src/whisper/align.ts rename to compute-worker/src/compute/whisper/align.ts diff --git a/compute-worker/src/compute/whisper/alignment-map.ts b/compute-worker/src/compute/whisper/alignment-map.ts new file mode 100644 index 0000000..098374c --- /dev/null +++ b/compute-worker/src/compute/whisper/alignment-map.ts @@ -0,0 +1,44 @@ +import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts'; + +export interface WhisperWord { + start: number; + end: number; + word: string; +} + +export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment { + const lowerSentence = sentence.toLowerCase(); + let cursor = 0; + + const alignedWords: TTSSentenceWord[] = words.map((w) => { + const token = w.word.trim(); + if (!token) { + return { + text: '', + startSec: w.start, + endSec: w.end, + charStart: cursor, + charEnd: cursor, + }; + } + + const idx = lowerSentence.indexOf(token.toLowerCase(), cursor); + const start = idx >= 0 ? idx : cursor; + const end = Math.min(sentence.length, start + token.length); + cursor = Math.max(cursor, end); + + return { + text: token, + startSec: w.start, + endSec: w.end, + charStart: start, + charEnd: end, + }; + }).filter((word) => word.text.length > 0); + + return { + sentence, + sentenceIndex: 0, + words: alignedWords, + }; +} diff --git a/compute/core/src/whisper/assets/LICENSE.txt b/compute-worker/src/compute/whisper/assets/LICENSE.txt similarity index 100% rename from compute/core/src/whisper/assets/LICENSE.txt rename to compute-worker/src/compute/whisper/assets/LICENSE.txt diff --git a/compute/core/src/whisper/assets/manifest.json b/compute-worker/src/compute/whisper/assets/manifest.json similarity index 100% rename from compute/core/src/whisper/assets/manifest.json rename to compute-worker/src/compute/whisper/assets/manifest.json diff --git a/compute/core/src/whisper/assets/mel_filters.npz b/compute-worker/src/compute/whisper/assets/mel_filters.npz similarity index 100% rename from compute/core/src/whisper/assets/mel_filters.npz rename to compute-worker/src/compute/whisper/assets/mel_filters.npz diff --git a/compute/core/src/whisper/model.ts b/compute-worker/src/compute/whisper/model.ts similarity index 100% rename from compute/core/src/whisper/model.ts rename to compute-worker/src/compute/whisper/model.ts diff --git a/compute/core/src/whisper/spectral.ts b/compute-worker/src/compute/whisper/spectral.ts similarity index 100% rename from compute/core/src/whisper/spectral.ts rename to compute-worker/src/compute/whisper/spectral.ts diff --git a/compute/core/src/whisper/token-timestamps.ts b/compute-worker/src/compute/whisper/token-timestamps.ts similarity index 100% rename from compute/core/src/whisper/token-timestamps.ts rename to compute-worker/src/compute/whisper/token-timestamps.ts diff --git a/compute/worker/src/control-plane/jetstream.ts b/compute-worker/src/control-plane/jetstream.ts similarity index 99% rename from compute/worker/src/control-plane/jetstream.ts rename to compute-worker/src/control-plane/jetstream.ts index 9225d96..e5367eb 100644 --- a/compute/worker/src/control-plane/jetstream.ts +++ b/compute-worker/src/control-plane/jetstream.ts @@ -8,12 +8,12 @@ import type { OperationState, OperationStateStore, QueuedOperation, -} from '@openreader/compute-core/control-plane'; +} from '../compute/control-plane'; import type { PdfLayoutJobRequest, WhisperAlignJobRequest, WorkerOperationKind, -} from '@openreader/compute-core/api-contracts'; +} from '../compute/api-contracts'; import { createJsonCodec } from './json-codec'; export interface KvEntryLike { diff --git a/compute/worker/src/control-plane/json-codec.ts b/compute-worker/src/control-plane/json-codec.ts similarity index 100% rename from compute/worker/src/control-plane/json-codec.ts rename to compute-worker/src/control-plane/json-codec.ts diff --git a/compute/worker/src/orphan-recovery.ts b/compute-worker/src/orphan-recovery.ts similarity index 98% rename from compute/worker/src/orphan-recovery.ts rename to compute-worker/src/orphan-recovery.ts index 882ea94..8257b4d 100644 --- a/compute/worker/src/orphan-recovery.ts +++ b/compute-worker/src/orphan-recovery.ts @@ -4,7 +4,7 @@ import type { WorkerJobTiming, WorkerJobState, WorkerOperationState, -} from '@openreader/compute-core/api-contracts'; +} from './compute/api-contracts'; export type StreamedOperationState = WorkerOperationState; diff --git a/compute/worker/src/pdf-artifact-persistence.ts b/compute-worker/src/pdf-artifact-persistence.ts similarity index 100% rename from compute/worker/src/pdf-artifact-persistence.ts rename to compute-worker/src/pdf-artifact-persistence.ts diff --git a/compute/worker/src/pdf-progress.ts b/compute-worker/src/pdf-progress.ts similarity index 87% rename from compute/worker/src/pdf-progress.ts rename to compute-worker/src/pdf-progress.ts index 756a256..cc5c025 100644 --- a/compute/worker/src/pdf-progress.ts +++ b/compute-worker/src/pdf-progress.ts @@ -1,4 +1,4 @@ -import type { PdfLayoutProgress } from '@openreader/compute-core/api-contracts'; +import type { PdfLayoutProgress } from './compute/api-contracts'; export function buildInferProgressForPageStart(input: { pageNumber: number; diff --git a/compute/worker/src/runtime.ts b/compute-worker/src/runtime.ts similarity index 89% rename from compute/worker/src/runtime.ts rename to compute-worker/src/runtime.ts index a5099a6..5cafa7c 100644 --- a/compute/worker/src/runtime.ts +++ b/compute-worker/src/runtime.ts @@ -1,4 +1,5 @@ import Fastify, { type FastifyInstance, type FastifyRequest } from 'fastify'; +import swagger from '@fastify/swagger'; import { z } from 'zod'; import { connect, @@ -24,18 +25,16 @@ import { ensureComputeModels, runPdfLayoutFromPdfBuffer, runWhisperAlignmentFromAudioBuffer, -} from '@openreader/compute-core/local-runtime'; +} from './compute/local-runtime'; import { getComputeTimeoutConfig, getComputeOpStaleMs, getAvailableCpuCores, getOnnxThreadsPerJob, - PDF_PARSER_VERSION, - encodeParserVersion, withIdleTimeoutAndHardCap, withTimeout, -} from '@openreader/compute-core'; -import { encodeSseFrame, OperationOrchestrator } from '@openreader/compute-core/control-plane'; +} from './compute'; +import { encodeSseFrame, OperationOrchestrator } from './compute/control-plane'; import type { PdfLayoutJobRequest, PdfLayoutJobResult, @@ -48,7 +47,7 @@ import type { WorkerOperationRequest, WorkerOperationState, PdfLayoutProgress, -} from '@openreader/compute-core/api-contracts'; +} from './compute/api-contracts'; import { DeleteObjectCommand, GetObjectCommand, @@ -71,6 +70,26 @@ import { import { buildInferProgressForPageParsed, buildInferProgressForPageStart } from './pdf-progress'; import { buildQueueWaitTiming, decideRetryAction } from './worker-loop-policy'; import { persistParsedPdfWhileSourceExists } from './pdf-artifact-persistence'; +import { parsedPdfArtifactKey } from './storage/artifact-addressing'; +import { buildPdfOperationKey, buildWhisperOperationKey } from './api/operation-keys'; +import { toPublicOperation, type PublicOperationEvent } from './api/public-operation'; +import { + apiErrorResponseSchema, + artifactReferenceSchema, + jsonSchema, + operationEventsQuerySchema, + operationParamsSchema, + operationErrorSchema, + parsedPdfDocumentSchema, + pdfLayoutProgressSchema, + pdfLayoutResolutionSchema, + pdfOperationCreateSchema, + pdfResolveSchema, + publicOperationEventSchema, + publicOperationSchema, + ttsSentenceAlignmentSchema, + whisperOperationCreateSchema, +} from './api/schemas'; const JOBS_STREAM_NAME = 'compute_jobs'; const WHISPER_JOBS_SUBJECT = 'jobs.whisper'; @@ -88,8 +107,6 @@ const OP_EVENTS_KEEPALIVE_MS = 15_000; // client from immediately reconnecting and re-waking the worker; instead it // reconnects on a slow cadence so the container stays asleep most of the time. const OP_EVENTS_RECONNECT_HINT_MS = 120_000; -const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; -const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; const WHISPER_MAX_DELIVER = 1; const NATS_API_TIMEOUT_MS = 60_000; // Disconnect from NATS after this much continuous idle so the worker stops @@ -182,6 +199,7 @@ export interface ComputeWorkerRouteDeps { orchestrator: OrchestratorLike; operationStateStore: OperationStateStoreLike; operationEventStream: OperationEventStreamLike; + artifactExists?: (key: string) => Promise; } export interface CreateComputeWorkerAppOptions { @@ -254,20 +272,6 @@ function normalizeS3Prefix(prefix: string | undefined): string { return value ? value.replace(/^\/+|\/+$/g, '') : 'openreader'; } -function sanitizeNamespace(namespace: string | null): string | null { - if (!namespace) return null; - return SAFE_NAMESPACE_REGEX.test(namespace) ? namespace : null; -} - -function documentParsedKey(id: string, namespace: string | null, prefix: string): string { - if (!DOCUMENT_ID_REGEX.test(id)) { - throw new Error(`Invalid document id: ${id}`); - } - const ns = sanitizeNamespace(namespace); - const nsSegment = ns ? `ns/${ns}/` : ''; - return `${prefix}/documents_v1/parsed_v2/${nsSegment}${id}/${encodeParserVersion(PDF_PARSER_VERSION)}.json`; -} - function buildS3Client(): S3Client { const bucket = requireEnv('S3_BUCKET'); const region = requireEnv('S3_REGION'); @@ -357,7 +361,7 @@ function extractOpId(request: FastifyRequest, path: string): string | null { if (params && typeof params.opId === 'string' && params.opId.trim()) { return params.opId.trim(); } - const match = path.match(/^\/ops\/([^/]+)/); + const match = path.match(/^\/v1\/operations\/([^/]+)/); if (!match?.[1]) return null; try { return decodeURIComponent(match[1]); @@ -428,22 +432,7 @@ const layoutSchema = z.object({ documentObjectKey: z.string().trim().min(1).max(2048), }); -const operationCreateSchema = z.discriminatedUnion('kind', [ - z.object({ - kind: z.literal('whisper_align'), - opKey: z.string().trim().min(1).max(1024), - payload: alignSchema, - }), - z.object({ - kind: z.literal('pdf_layout'), - opKey: z.string().trim().min(1).max(1024), - payload: layoutSchema, - }), -]); - -const operationLookupSchema = z.object({ - opKey: z.string().trim().min(1).max(1024), -}); +const errorResponseSchema = jsonSchema(apiErrorResponseSchema); async function ensureJetStreamResources( jsm: JetStreamManager, @@ -567,7 +556,7 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti // Lazy NATS connection lifecycle. The worker connects on demand (first request // that needs the queue/KV) and disconnects after IDLE_DISCONNECT_MS of full idle // so it stops emitting outbound traffic and Railway can sleep it. Reconnect is - // transparent: any inbound /ops request both wakes the container and re-establishes + // transparent: any inbound operation request both wakes the container and re-establishes // the session via ensureConnected(). let session: NatsSession | null = null; let connecting: Promise | null = null; @@ -785,7 +774,7 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti if (!s3) { throw new Error('S3 access is disabled for this worker app instance'); } - const key = documentParsedKey(documentId, namespace, s3Prefix); + const key = parsedPdfArtifactKey({ documentId, namespace, prefix: s3Prefix }); const body = Buffer.from(JSON.stringify(parsed)); await s3.send(new PutObjectCommand({ Bucket: s3Bucket, @@ -805,6 +794,35 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti logger: buildLoggerConfig(), disableRequestLogging: true, }); + await app.register(swagger, { + openapi: { + openapi: '3.0.3', + info: { + title: 'OpenReader Compute Worker API', + version: '1.0.0', + }, + components: { + securitySchemes: { + bearerAuth: { + type: 'http', + scheme: 'bearer', + }, + }, + schemas: { + ParsedPdfDocument: jsonSchema(parsedPdfDocumentSchema), + TTSSentenceAlignment: jsonSchema(ttsSentenceAlignmentSchema), + ArtifactReference: jsonSchema(artifactReferenceSchema), + ErrorResponse: jsonSchema(apiErrorResponseSchema), + OperationError: jsonSchema(operationErrorSchema), + PdfLayoutProgress: jsonSchema(pdfLayoutProgressSchema), + PublicOperation: jsonSchema(publicOperationSchema), + PublicOperationEvent: jsonSchema(publicOperationEventSchema), + PdfLayoutResolution: jsonSchema(pdfLayoutResolutionSchema), + }, + }, + security: [{ bearerAuth: [] }], + }, + }); app.log.info({ jobConcurrency, whisperTimeoutMs, @@ -956,16 +974,37 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti } }); - app.get('/health/live', async () => ({ ok: true })); + app.get('/health/live', { + schema: { + security: [], + response: { 200: { type: 'object', properties: { ok: { type: 'boolean' } }, required: ['ok'] } }, + }, + }, async () => ({ ok: true })); // Reports readiness without forcing a NATS round-trip. Probing NATS here would // reconnect (and keep) the connection open, defeating idle sleep, so we only // report the current connection state. The worker reconnects lazily on the next - // /ops request regardless of what this returns. - app.get('/health/ready', async () => ({ ok: true, natsConnected: session !== null })); + // operation request regardless of what this returns. + app.get('/health/ready', { + schema: { + security: [], + response: { + 200: { + type: 'object', + properties: { ok: { type: 'boolean' }, natsConnected: { type: 'boolean' } }, + required: ['ok', 'natsConnected'], + }, + }, + }, + }, async () => ({ ok: true, natsConnected: session !== null })); - app.post('/ops', async (request, reply) => { - const parsed = operationCreateSchema.safeParse(request.body); + app.post('/v1/whisper-align/operations', { + schema: { + body: jsonSchema(whisperOperationCreateSchema), + response: { 202: jsonSchema(publicOperationSchema), 400: errorResponseSchema }, + }, + }, async (request, reply) => { + const parsed = whisperOperationCreateSchema.safeParse(request.body); if (!parsed.success) { reply.code(400); return { @@ -974,7 +1013,11 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti }; } - const requestOp = parsed.data as WorkerOperationRequest; + const requestOp: WorkerOperationRequest = { + kind: 'whisper_align', + opKey: buildWhisperOperationKey(parsed.data), + payload: parsed.data, + }; await ensureOrphanedOpRecovery(); const op = await orchestrator.enqueueOrReuse(requestOp); app.log.info({ @@ -985,11 +1028,16 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti opKeyHash: hashOpKey(requestOp.opKey.trim()).slice(0, 16), }, 'op.accepted'); reply.code(202); - return op; + return toPublicOperation(op); }); - app.post('/ops/lookup', async (request, reply) => { - const parsed = operationLookupSchema.safeParse(request.body); + app.post('/v1/pdf-layout/operations', { + schema: { + body: jsonSchema(pdfOperationCreateSchema), + response: { 202: jsonSchema(publicOperationSchema), 400: errorResponseSchema }, + }, + }, async (request, reply) => { + const parsed = pdfOperationCreateSchema.safeParse(request.body); if (!parsed.success) { reply.code(400); return { @@ -998,28 +1046,58 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti }; } + const requestOp: WorkerOperationRequest = { + kind: 'pdf_layout', + opKey: buildPdfOperationKey(parsed.data), + payload: { + documentId: parsed.data.documentId, + namespace: parsed.data.namespace, + documentObjectKey: parsed.data.documentObjectKey, + }, + }; await ensureOrphanedOpRecovery(); - if (typeof operationStateStore.getOpIndex !== 'function') { - reply.code(501); - return { error: 'Operation lookup by opKey is not supported by this state store' }; - } - const indexEntry = await operationStateStore.getOpIndex(parsed.data.opKey); - if (!indexEntry?.opId) { - reply.code(404); - return { error: 'Operation not found' }; - } - - const state = await operationStateStore.getOpState(indexEntry.opId); - if (!state) { - reply.code(404); - return { error: 'Operation not found' }; - } - - return state; + const op = await orchestrator.enqueueOrReuse(requestOp); + reply.code(202); + return toPublicOperation(op); }); - app.get('/ops/:opId', async (request, reply) => { - const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); + app.post('/v1/pdf-layout/resolve', { + schema: { + body: jsonSchema(pdfResolveSchema), + response: { + 200: jsonSchema(pdfLayoutResolutionSchema), + 400: errorResponseSchema, + }, + }, + }, async (request, reply) => { + const parsed = pdfResolveSchema.safeParse(request.body); + if (!parsed.success) { + reply.code(400); + return { error: 'Invalid request body', issues: parsed.error.issues }; + } + await ensureOrphanedOpRecovery(); + const artifactKey = parsedPdfArtifactKey({ + documentId: parsed.data.documentId, + namespace: parsed.data.namespace, + prefix: s3Prefix, + }); + const hasArtifact = await (options.routeDeps?.artifactExists ?? objectExists)(artifactKey); + const opKey = buildPdfOperationKey(parsed.data); + const index = await operationStateStore.getOpIndex?.(opKey); + const operation = index?.opId ? await operationStateStore.getOpState(index.opId) : null; + return { + artifact: hasArtifact ? { objectKey: artifactKey } : null, + operation: operation ? toPublicOperation(operation) : null, + }; + }); + + app.get('/v1/operations/:opId', { + schema: { + params: jsonSchema(operationParamsSchema), + response: { 200: jsonSchema(publicOperationSchema), 400: errorResponseSchema, 404: errorResponseSchema }, + }, + }, async (request, reply) => { + const params = operationParamsSchema.safeParse(request.params); if (!params.success) { reply.code(400); return { error: 'Invalid op id' }; @@ -1031,11 +1109,21 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti return { error: 'Operation not found' }; } - return state; + return toPublicOperation(state); }); - app.get('/ops/:opId/events', async (request, reply) => { - const params = z.object({ opId: z.string().trim().min(1) }).safeParse(request.params); + app.get('/v1/operations/:opId/events', { + schema: { + params: jsonSchema(operationParamsSchema), + querystring: jsonSchema(operationEventsQuerySchema), + response: { + 200: { type: 'string', description: 'Server-sent PublicOperationEvent stream' }, + 400: errorResponseSchema, + 404: errorResponseSchema, + }, + }, + }, async (request, reply) => { + const params = operationParamsSchema.safeParse(request.params); if (!params.success) { reply.code(400); return { error: 'Invalid op id' }; @@ -1080,9 +1168,9 @@ export async function createComputeWorkerApp(options: CreateComputeWorkerAppOpti const writeSnapshot = (snapshot: StreamedOperationState, eventId: number): void => { if (closed || reply.raw.writableEnded) return; - const frameEvent: WorkerOperationEvent = { + const frameEvent: PublicOperationEvent = { eventId, - snapshot, + snapshot: toPublicOperation(snapshot), }; reply.raw.write(encodeSseFrame({ id: eventId, diff --git a/compute/worker/src/server.ts b/compute-worker/src/server.ts similarity index 100% rename from compute/worker/src/server.ts rename to compute-worker/src/server.ts diff --git a/compute-worker/src/storage/artifact-addressing.ts b/compute-worker/src/storage/artifact-addressing.ts new file mode 100644 index 0000000..a4d1ee6 --- /dev/null +++ b/compute-worker/src/storage/artifact-addressing.ts @@ -0,0 +1,21 @@ +import { PDF_PARSER_VERSION } from '../compute/pdf/parser-version'; +import { encodeParserVersion } from '../compute/pdf/parser-version-key'; + +const DOCUMENT_ID_REGEX = /^[a-f0-9]{64}$/i; +const SAFE_NAMESPACE_REGEX = /^[a-zA-Z0-9._-]{1,128}$/; + +export function parsedPdfArtifactKey(input: { + documentId: string; + namespace: string | null; + prefix: string; + parserVersion?: string; +}): string { + if (!DOCUMENT_ID_REGEX.test(input.documentId)) { + throw new Error(`Invalid document id: ${input.documentId}`); + } + const namespace = input.namespace && SAFE_NAMESPACE_REGEX.test(input.namespace) + ? input.namespace + : null; + const namespaceSegment = namespace ? `ns/${namespace}/` : ''; + return `${input.prefix}/documents_v1/parsed_v2/${namespaceSegment}${input.documentId}/${encodeParserVersion(input.parserVersion ?? PDF_PARSER_VERSION)}.json`; +} diff --git a/compute/worker/src/worker-loop-policy.ts b/compute-worker/src/worker-loop-policy.ts similarity index 89% rename from compute/worker/src/worker-loop-policy.ts rename to compute-worker/src/worker-loop-policy.ts index 7462b4c..bce8653 100644 --- a/compute/worker/src/worker-loop-policy.ts +++ b/compute-worker/src/worker-loop-policy.ts @@ -1,4 +1,4 @@ -import type { WorkerOperationKind } from '@openreader/compute-core/api-contracts'; +import type { WorkerOperationKind } from './compute/api-contracts'; export type RetryAction = 'nak_retry' | 'term_fail'; diff --git a/compute/worker/tests/api/routes.test.ts b/compute-worker/tests/api/routes.test.ts similarity index 53% rename from compute/worker/tests/api/routes.test.ts rename to compute-worker/tests/api/routes.test.ts index dad89d9..e7336ee 100644 --- a/compute/worker/tests/api/routes.test.ts +++ b/compute-worker/tests/api/routes.test.ts @@ -25,53 +25,51 @@ describe('compute worker API routes', () => { const live = await runtime.app.inject({ method: 'GET', url: '/health/live' }); expect(live.statusCode).toBe(200); - const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/ops/op-1' }); + const protectedRoute = await runtime.app.inject({ method: 'GET', url: '/v1/operations/op-1' }); expect(protectedRoute.statusCode).toBe(401); }); test('validates operation creation body and returns 400 for invalid payload', async () => { const response = await runtime.app.inject({ method: 'POST', - url: '/ops', + url: '/v1/pdf-layout/operations', headers: AUTH, payload: { - kind: 'pdf_layout', - opKey: '', - payload: { - documentId: 'd1', - namespace: null, - documentObjectKey: 's3://bucket/doc.pdf', - }, + documentId: '', + namespace: null, + documentObjectKey: 'openreader/doc.pdf', }, }); expect(response.statusCode).toBe(400); - expect(response.json()).toMatchObject({ error: 'Invalid request body' }); + expect(response.json()).toMatchObject({ error: 'Bad Request' }); }); test('creates operation and fetches state by op id', async () => { + const documentId = 'c'.repeat(64); const create = await runtime.app.inject({ method: 'POST', - url: '/ops', + url: '/v1/pdf-layout/operations', headers: AUTH, payload: { - kind: 'pdf_layout', - opKey: 'doc-1:layout', - payload: { - documentId: 'doc-1', - namespace: null, - documentObjectKey: 'openreader/doc-1.pdf', - }, + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, }, }); expect(create.statusCode).toBe(202); const created = create.json(); - expect(created).toMatchObject({ kind: 'pdf_layout', status: 'queued' }); + expect(created).toMatchObject({ + subject: { kind: 'pdf_layout', documentId, namespace: null }, + status: 'queued', + }); + expect(created).not.toHaveProperty('opKey'); + expect(created).not.toHaveProperty('jobId'); const fetch = await runtime.app.inject({ method: 'GET', - url: `/ops/${created.opId}`, + url: `/v1/operations/${created.opId}`, headers: AUTH, }); @@ -79,17 +77,120 @@ describe('compute worker API routes', () => { expect(fetch.json()).toMatchObject({ opId: created.opId, status: 'queued' }); }); + test('reuses idempotent PDF requests and replaces them only with an explicit token', async () => { + const documentId = 'e'.repeat(64); + const payload = { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }; + const create = (body: typeof payload & { replaceToken?: string }) => runtime.app.inject({ + method: 'POST', + url: '/v1/pdf-layout/operations', + headers: AUTH, + payload: body, + }); + + const initial = (await create(payload)).json(); + const reused = (await create(payload)).json(); + const replacement = (await create({ ...payload, replaceToken: 'replace-1' })).json(); + + expect(reused.opId).toBe(initial.opId); + expect(replacement.opId).not.toBe(initial.opId); + expect(replacement.subject).toEqual(initial.subject); + }); + + test('creates whisper alignment operations without exposing internal keys', async () => { + const response = await runtime.app.inject({ + method: 'POST', + url: '/v1/whisper-align/operations', + headers: AUTH, + payload: { + text: 'Canonical worker text', + audioObjectKey: 'openreader/audio.mp3', + lang: 'en', + }, + }); + + expect(response.statusCode).toBe(202); + expect(response.json()).toMatchObject({ + subject: { kind: 'whisper_align' }, + status: 'queued', + }); + expect(response.json()).not.toHaveProperty('opKey'); + }); + + test('resolves the current PDF artifact and operation without exposing parser identity', async () => { + const documentId = 'a'.repeat(64); + const create = await runtime.app.inject({ + method: 'POST', + url: '/v1/pdf-layout/operations', + headers: AUTH, + payload: { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }, + }); + expect(create.statusCode).toBe(202); + + const artifactKey = `openreader/documents_v1/parsed_v2/${documentId}/pp-doclayoutv3-onnx%40800%2Bpdfjs%404.8.69.json`; + fake.seedArtifact(artifactKey); + const resolve = await runtime.app.inject({ + method: 'POST', + url: '/v1/pdf-layout/resolve', + headers: AUTH, + payload: { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }, + }); + + expect(resolve.statusCode).toBe(200); + expect(resolve.json()).toMatchObject({ + artifact: { objectKey: artifactKey }, + operation: { + subject: { kind: 'pdf_layout', documentId, namespace: null }, + }, + }); + expect(resolve.json().operation).not.toHaveProperty('opKey'); + }); + + test('resolves a current PDF artifact after operation state has expired', async () => { + const documentId = 'd'.repeat(64); + const artifactKey = `openreader/documents_v1/parsed_v2/${documentId}/pp-doclayoutv3-onnx%40800%2Bpdfjs%404.8.69.json`; + fake.seedArtifact(artifactKey); + + const resolve = await runtime.app.inject({ + method: 'POST', + url: '/v1/pdf-layout/resolve', + headers: AUTH, + payload: { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }, + }); + + expect(resolve.statusCode).toBe(200); + expect(resolve.json()).toEqual({ + artifact: { objectKey: artifactKey }, + operation: null, + }); + }); + test('returns not found for unknown operation and event stream lookups', async () => { const opResponse = await runtime.app.inject({ method: 'GET', - url: '/ops/missing', + url: '/v1/operations/missing', headers: AUTH, }); expect(opResponse.statusCode).toBe(404); const eventsResponse = await runtime.app.inject({ method: 'GET', - url: '/ops/missing/events', + url: '/v1/operations/missing/events', headers: AUTH, }); expect(eventsResponse.statusCode).toBe(404); @@ -109,7 +210,7 @@ describe('compute worker API routes', () => { const stream = await runtime.app.inject({ method: 'GET', - url: '/ops/op-terminal/events?sinceEventId=7', + url: '/v1/operations/op-terminal/events?sinceEventId=7', headers: AUTH, }); @@ -159,11 +260,11 @@ describe('compute worker API routes', () => { updatedAt: now - 310_000, }); - // GET /ops/:opId resolves via getOpState(), which first awaits the shared + // GET /v1/operations/:opId resolves via getOpState(), which first awaits the shared // orphanRecoveryPromise path through ensureOrphanedOpRecovery(). const fetch = await runtime.app.inject({ method: 'GET', - url: '/ops/op-stale-whisper-running', + url: '/v1/operations/op-stale-whisper-running', headers: AUTH, }); diff --git a/tests/unit/pdf-merge-text-with-regions.vitest.spec.ts b/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts similarity index 96% rename from tests/unit/pdf-merge-text-with-regions.vitest.spec.ts rename to compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts index 449824e..71436d0 100644 --- a/tests/unit/pdf-merge-text-with-regions.vitest.spec.ts +++ b/compute-worker/tests/compute/algorithms/pdf-merge-text-with-regions.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { mergeTextWithRegions } from '@openreader/compute-core'; +import { mergeTextWithRegions } from '../../../src/compute'; describe('mergeTextWithRegions', () => { test('assigns text items to containing regions by centroid and joins in reading order', () => { diff --git a/tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts b/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts similarity index 97% rename from tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts rename to compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts index b61bd26..f11272d 100644 --- a/tests/unit/pdf-parse-normalize-text-items.vitest.spec.ts +++ b/compute-worker/tests/compute/algorithms/pdf-parse-normalize-text-items.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { normalizeTextItemsForLayout } from '@openreader/compute-core'; +import { normalizeTextItemsForLayout } from '../../../src/compute'; import type { TextItem } from 'pdfjs-dist/types/src/display/api'; function makeTextItem( diff --git a/tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts b/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts similarity index 96% rename from tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts rename to compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts index 0aa37e6..fb7b602 100644 --- a/tests/unit/pdf-stitch-cross-page-blocks.vitest.spec.ts +++ b/compute-worker/tests/compute/algorithms/pdf-stitch-cross-page-blocks.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { stitchCrossPageBlocks } from '@openreader/compute-core'; -import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../src/types/parsed-pdf'; +import { stitchCrossPageBlocks } from '../../../src/compute'; +import type { ParsedPdfBlock, ParsedPdfDocument, ParsedPdfBlockKind } from '../../../src/compute/types'; import { makeParsedPdfDocument, makeParsedPdfPage } from './support/document-fixtures'; function makeBlock( diff --git a/compute-worker/tests/compute/algorithms/support/document-fixtures.ts b/compute-worker/tests/compute/algorithms/support/document-fixtures.ts new file mode 100644 index 0000000..ffcde2d --- /dev/null +++ b/compute-worker/tests/compute/algorithms/support/document-fixtures.ts @@ -0,0 +1,54 @@ +import type { BaseDocument } from '../../../src/types/documents'; +import type { ParsedPdfBlock, ParsedPdfBlockKind, ParsedPdfDocument, ParsedPdfPage } from '../../../../src/compute/types'; + +export function makeBaseDocument(overrides: Partial = {}): BaseDocument { + return { + id: 'doc-1', + name: 'document.pdf', + size: 1_024, + lastModified: 1_700_000_000_000, + type: 'pdf', + ...overrides, + }; +} + +export function makeParsedPdfBlock(input: { + id: string; + kind: ParsedPdfBlockKind; + text: string; + page: number; + readingOrder: number; +}): ParsedPdfBlock { + return { + id: input.id, + kind: input.kind, + text: input.text, + fragments: [ + { + page: input.page, + bbox: [0, 0, 100, 10], + text: input.text, + readingOrder: input.readingOrder, + }, + ], + }; +} + +export function makeParsedPdfPage(pageNumber: number, blocks: ParsedPdfBlock[]): ParsedPdfPage { + return { + pageNumber, + width: 800, + height: 1_200, + blocks, + }; +} + +export function makeParsedPdfDocument(pages: ParsedPdfPage[]): ParsedPdfDocument { + return { + schemaVersion: 1, + documentId: 'doc-fixture', + parserVersion: 'test', + parsedAt: 1_700_000_000_000, + pages, + }; +} diff --git a/tests/unit/whisper-alignment-mapping.vitest.spec.ts b/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts similarity index 97% rename from tests/unit/whisper-alignment-mapping.vitest.spec.ts rename to compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts index e5c9575..2eea68f 100644 --- a/tests/unit/whisper-alignment-mapping.vitest.spec.ts +++ b/compute-worker/tests/compute/algorithms/whisper-alignment-mapping.test.ts @@ -1,7 +1,7 @@ import { describe, expect, test } from 'vitest'; import { mapWordsToSentenceOffsets, -} from '@openreader/compute-core'; +} from '../../../src/compute'; describe('whisper alignment mapping', () => { test('maps words to sentence offsets with punctuation and repeated spaces', () => { diff --git a/tests/unit/whisper-spectral.vitest.spec.ts b/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts similarity index 93% rename from tests/unit/whisper-spectral.vitest.spec.ts rename to compute-worker/tests/compute/algorithms/whisper-spectral.test.ts index 2b87a04..ac4a3d8 100644 --- a/tests/unit/whisper-spectral.vitest.spec.ts +++ b/compute-worker/tests/compute/algorithms/whisper-spectral.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { buildGoertzelCoefficients, goertzelPower } from '@openreader/compute-core'; +import { buildGoertzelCoefficients, goertzelPower } from '../../../src/compute'; function dftPower(samples: Float32Array, k: number): number { const n = samples.length; diff --git a/tests/unit/whisper-token-timestamps.vitest.spec.ts b/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts similarity index 98% rename from tests/unit/whisper-token-timestamps.vitest.spec.ts rename to compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts index c2aded0..a6ff75d 100644 --- a/tests/unit/whisper-token-timestamps.vitest.spec.ts +++ b/compute-worker/tests/compute/algorithms/whisper-token-timestamps.test.ts @@ -3,7 +3,7 @@ import * as ort from 'onnxruntime-node'; import { buildWordsFromTimestampedTokens, extractTokenStartTimestamps, -} from '@openreader/compute-core'; +} from '../../../src/compute'; describe('whisper token timestamp alignment', () => { test('extracts monotonic token timestamps from cross-attention maps', () => { diff --git a/compute/core/tests/control-plane/orchestrator.test.ts b/compute-worker/tests/compute/control-plane/orchestrator.test.ts similarity index 96% rename from compute/core/tests/control-plane/orchestrator.test.ts rename to compute-worker/tests/compute/control-plane/orchestrator.test.ts index 80bd9fb..6c9bbab 100644 --- a/compute/core/tests/control-plane/orchestrator.test.ts +++ b/compute-worker/tests/compute/control-plane/orchestrator.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import type { WorkerOperationRequest } from '../../src/api-contracts'; -import { OperationOrchestrator } from '../../src/control-plane'; +import type { WorkerOperationRequest } from '../../../src/compute/api-contracts'; +import { OperationOrchestrator } from '../../../src/compute/control-plane'; import { InMemoryOperationEventStream, InMemoryOperationQueue, diff --git a/compute/core/tests/control-plane/run-layout-model.test.ts b/compute-worker/tests/compute/control-plane/run-layout-model.test.ts similarity index 93% rename from compute/core/tests/control-plane/run-layout-model.test.ts rename to compute-worker/tests/compute/control-plane/run-layout-model.test.ts index e951bf3..27ef706 100644 --- a/compute/core/tests/control-plane/run-layout-model.test.ts +++ b/compute-worker/tests/compute/control-plane/run-layout-model.test.ts @@ -70,13 +70,13 @@ vi.mock('@napi-rs/canvas', () => { }; }); -vi.mock('../../src/pdf/model', () => ({ +vi.mock('../../../src/compute/pdf/model', () => ({ ensureModel: vi.fn(async () => '/tmp/model.onnx'), MODEL_CONFIG_PATH: '/tmp/model-config.json', MODEL_PREPROCESSOR_PATH: '/tmp/model-preprocessor.json', })); -vi.mock('../../src/config/cpu-budget', () => ({ +vi.mock('../../../src/compute/config/cpu-budget', () => ({ getOnnxThreadsPerJob: vi.fn(() => 1), })); @@ -107,7 +107,7 @@ describe('runLayoutModel', () => { }, }; - const { runLayoutModel } = await import('../../src/pdf/runLayoutModel'); + const { runLayoutModel } = await import('../../../src/compute/pdf/runLayoutModel'); const regions = await runLayoutModel({ pageWidth: 100, pageHeight: 100, @@ -154,7 +154,7 @@ describe('runLayoutModel', () => { }, }; - const { runLayoutModel } = await import('../../src/pdf/runLayoutModel'); + const { runLayoutModel } = await import('../../../src/compute/pdf/runLayoutModel'); const regions = await runLayoutModel({ pageWidth: 100, pageHeight: 100, diff --git a/compute/core/tests/control-plane/sse.test.ts b/compute-worker/tests/compute/control-plane/sse.test.ts similarity index 96% rename from compute/core/tests/control-plane/sse.test.ts rename to compute-worker/tests/compute/control-plane/sse.test.ts index 9b8e5b9..3461229 100644 --- a/compute/core/tests/control-plane/sse.test.ts +++ b/compute-worker/tests/compute/control-plane/sse.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from 'vitest'; -import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../src/control-plane/sse'; +import { encodeSseFrame, parseSseEventId, parseSsePayload } from '../../../src/compute/control-plane/sse'; describe('sse codec', () => { test('encodes event id and payload and decodes both reliably', () => { diff --git a/compute/core/tests/control-plane/state-machine.test.ts b/compute-worker/tests/compute/control-plane/state-machine.test.ts similarity index 93% rename from compute/core/tests/control-plane/state-machine.test.ts rename to compute-worker/tests/compute/control-plane/state-machine.test.ts index e9617cb..a9d5390 100644 --- a/compute/core/tests/control-plane/state-machine.test.ts +++ b/compute-worker/tests/compute/control-plane/state-machine.test.ts @@ -1,11 +1,11 @@ import { describe, expect, test } from 'vitest'; -import type { WorkerOperationState } from '../../src/api-contracts'; +import type { WorkerOperationState } from '../../../src/compute/api-contracts'; import { explainReplacementReason, isInflightStatus, isTerminalStatus, shouldReuseExistingOperation, -} from '../../src/control-plane/state-machine'; +} from '../../../src/compute/control-plane/state-machine'; function runningState(overrides: Partial = {}): WorkerOperationState { return { diff --git a/compute/core/tests/helpers/in-memory-control-plane.ts b/compute-worker/tests/compute/helpers/in-memory-control-plane.ts similarity index 97% rename from compute/core/tests/helpers/in-memory-control-plane.ts rename to compute-worker/tests/compute/helpers/in-memory-control-plane.ts index cc9b9ce..7395b2b 100644 --- a/compute/core/tests/helpers/in-memory-control-plane.ts +++ b/compute-worker/tests/compute/helpers/in-memory-control-plane.ts @@ -1,5 +1,5 @@ import { EventEmitter } from 'node:events'; -import type { WorkerOperationKind } from '../../src/api-contracts'; +import type { WorkerOperationKind } from '../../../src/compute/api-contracts'; import type { OperationEvent, OperationEventStream, @@ -8,7 +8,7 @@ import type { OperationState, OperationStateStore, QueuedOperation, -} from '../../src/control-plane/types'; +} from '../../../src/compute/control-plane/types'; function topicFor(opId: string): string { return `op.${opId}`; diff --git a/compute/worker/tests/fixtures/fake-control-plane.ts b/compute-worker/tests/fixtures/fake-control-plane.ts similarity index 93% rename from compute/worker/tests/fixtures/fake-control-plane.ts rename to compute-worker/tests/fixtures/fake-control-plane.ts index 9822523..40ab638 100644 --- a/compute/worker/tests/fixtures/fake-control-plane.ts +++ b/compute-worker/tests/fixtures/fake-control-plane.ts @@ -4,7 +4,7 @@ import type { WorkerOperationEvent, WorkerOperationRequest, WorkerOperationState, -} from '@openreader/compute-core/api-contracts'; +} from '../../src/compute/api-contracts'; import type { ComputeWorkerRouteDeps } from '../../src/runtime'; type ComputeResult = WhisperAlignJobResult | PdfLayoutJobResult; @@ -16,6 +16,7 @@ export class FakeControlPlane { private readonly revisionByOpId = new Map(); private readonly opIdByOpKey = new Map(); private readonly eventsByOpId = new Map(); + private readonly artifactKeys = new Set(); private nextOpId = 1; readonly deps: ComputeWorkerRouteDeps = { @@ -58,6 +59,10 @@ export class FakeControlPlane { }; }, listOpStates: async () => Array.from(this.stateByOpId.values()), + getOpIndex: async (opKey) => { + const opId = this.opIdByOpKey.get(opKey); + return opId ? { opId } : null; + }, }, operationEventStream: { subscribe: async ({ opId, sinceEventId, onEvent }) => { @@ -69,6 +74,7 @@ export class FakeControlPlane { return () => undefined; }, }, + artifactExists: async (key) => this.artifactKeys.has(key), }; seedState(state: ComputeState): void { @@ -83,6 +89,10 @@ export class FakeControlPlane { this.eventsByOpId.set(opId, list); } + seedArtifact(key: string): void { + this.artifactKeys.add(key); + } + getState(opId: string): ComputeState | null { return this.stateByOpId.get(opId) ?? null; } diff --git a/compute/worker/tests/setup-env.ts b/compute-worker/tests/setup-env.ts similarity index 100% rename from compute/worker/tests/setup-env.ts rename to compute-worker/tests/setup-env.ts diff --git a/compute/worker/tests/tsconfig.json b/compute-worker/tests/tsconfig.json similarity index 100% rename from compute/worker/tests/tsconfig.json rename to compute-worker/tests/tsconfig.json diff --git a/compute/worker/tests/unit/jetstream-adapters.test.ts b/compute-worker/tests/unit/jetstream-adapters.test.ts similarity index 97% rename from compute/worker/tests/unit/jetstream-adapters.test.ts rename to compute-worker/tests/unit/jetstream-adapters.test.ts index 9f30ccc..ae86c06 100644 --- a/compute/worker/tests/unit/jetstream-adapters.test.ts +++ b/compute-worker/tests/unit/jetstream-adapters.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'vitest'; -import { OperationOrchestrator } from '@openreader/compute-core/control-plane'; -import type { WorkerOperationRequest } from '@openreader/compute-core/api-contracts'; +import { OperationOrchestrator } from '../../src/compute/control-plane'; +import type { WorkerOperationRequest } from '../../src/compute/api-contracts'; import { JetStreamOperationEventStream, JetStreamOperationQueue, diff --git a/compute/worker/tests/unit/key-and-progress.test.ts b/compute-worker/tests/unit/key-and-progress.test.ts similarity index 62% rename from compute/worker/tests/unit/key-and-progress.test.ts rename to compute-worker/tests/unit/key-and-progress.test.ts index 6887cad..cbaf5ac 100644 --- a/compute/worker/tests/unit/key-and-progress.test.ts +++ b/compute-worker/tests/unit/key-and-progress.test.ts @@ -4,6 +4,8 @@ import { buildInferProgressForPageParsed, buildInferProgressForPageStart, } from '../../src/pdf-progress'; +import { buildPdfOperationKey } from '../../src/api/operation-keys'; +import { parsedPdfArtifactKey } from '../../src/storage/artifact-addressing'; describe('compute worker helpers', () => { test('hash and kv key helpers are stable and deterministic', () => { @@ -34,4 +36,17 @@ describe('compute worker helpers', () => { phase: 'infer', }); }); + + test('parser-version changes rotate worker-owned operation and artifact identities', () => { + const request = { + documentId: 'a'.repeat(64), + namespace: null, + documentObjectKey: `openreader/${'a'.repeat(64)}.pdf`, + }; + + expect(buildPdfOperationKey(request, 'parser-v1')) + .not.toBe(buildPdfOperationKey(request, 'parser-v2')); + expect(parsedPdfArtifactKey({ ...request, prefix: 'openreader', parserVersion: 'parser-v1' })) + .not.toBe(parsedPdfArtifactKey({ ...request, prefix: 'openreader', parserVersion: 'parser-v2' })); + }); }); diff --git a/compute/worker/tests/unit/orphan-recovery.test.ts b/compute-worker/tests/unit/orphan-recovery.test.ts similarity index 100% rename from compute/worker/tests/unit/orphan-recovery.test.ts rename to compute-worker/tests/unit/orphan-recovery.test.ts diff --git a/compute/worker/tests/unit/pdf-artifact-persistence.test.ts b/compute-worker/tests/unit/pdf-artifact-persistence.test.ts similarity index 100% rename from compute/worker/tests/unit/pdf-artifact-persistence.test.ts rename to compute-worker/tests/unit/pdf-artifact-persistence.test.ts diff --git a/compute/worker/tests/unit/worker-loop-policy.test.ts b/compute-worker/tests/unit/worker-loop-policy.test.ts similarity index 100% rename from compute/worker/tests/unit/worker-loop-policy.test.ts rename to compute-worker/tests/unit/worker-loop-policy.test.ts diff --git a/compute/worker/tsconfig.json b/compute-worker/tsconfig.json similarity index 73% rename from compute/worker/tsconfig.json rename to compute-worker/tsconfig.json index 428e252..820e03b 100644 --- a/compute/worker/tsconfig.json +++ b/compute-worker/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "../../tsconfig.json", + "extends": "../tsconfig.json", "compilerOptions": { "noEmit": true, }, diff --git a/compute/core/package.json b/compute/core/package.json deleted file mode 100644 index 4f27980..0000000 --- a/compute/core/package.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "@openreader/compute-core", - "version": "0.0.0", - "private": true, - "type": "module", - "dependencies": { - "@huggingface/tokenizers": "^0.1.3", - "@napi-rs/canvas": "^0.1.100", - "ffmpeg-static": "^5.3.0", - "jszip": "^3.10.1", - "onnxruntime-node": "^1.26.0", - "pdfjs-dist": "4.8.69" - }, - "exports": { - ".": "./src/index.ts", - "./local-runtime": "./src/local-runtime.ts", - "./api-contracts": "./src/api-contracts/index.ts", - "./control-plane": "./src/control-plane/index.ts", - "./types": "./src/types/index.ts" - } -} diff --git a/compute/core/src/whisper/alignment-map.ts b/compute/core/src/whisper/alignment-map.ts deleted file mode 100644 index a246cec..0000000 --- a/compute/core/src/whisper/alignment-map.ts +++ /dev/null @@ -1,62 +0,0 @@ -import type { TTSSentenceAlignment, TTSSentenceWord } from '../types/tts'; - -// Worker-side mirror of the app's canonical audio-text cleaning rules in -// `src/lib/shared/audio-text.ts`. This is a separate build target and cannot -// import from `@/lib`, so the rules are duplicated here on purpose. The word -// `charStart`/`charEnd` offsets this module emits are consumed against text -// normalized by that shared module, so any divergence shifts viewer highlights -// off-word — keep this byte-for-byte in sync with `audio-text.ts`. -const STRIPPED_GLYPHS = /[*•◦‣⁃∙▪▫■□●○◆◇★☆▶▸►▹➤➢❖]/g; - -function preprocessSentenceForAudio(text: string): string { - return text - .replace(/\S*(?:https?:\/\/|www\.)([^\/\s]+)(?:\/\S*)?/gi, '- (link to $1) -') - .replace(/([\p{L}\p{N}\p{M}]+)-\s+([\p{L}\p{N}\p{M}]+)/gu, '$1$2') - .replace(STRIPPED_GLYPHS, '') - .replace(/\s+/g, ' ') - .trim(); -} - -export interface WhisperWord { - start: number; - end: number; - word: string; -} - -export function mapWordsToSentenceOffsets(sentence: string, words: WhisperWord[]): TTSSentenceAlignment { - const normalizedSentence = preprocessSentenceForAudio(sentence); - const lowerSentence = normalizedSentence.toLowerCase(); - let cursor = 0; - - const alignedWords: TTSSentenceWord[] = words.map((w) => { - const token = w.word.trim(); - if (!token) { - return { - text: '', - startSec: w.start, - endSec: w.end, - charStart: cursor, - charEnd: cursor, - }; - } - - const idx = lowerSentence.indexOf(token.toLowerCase(), cursor); - const start = idx >= 0 ? idx : cursor; - const end = Math.min(normalizedSentence.length, start + token.length); - cursor = Math.max(cursor, end); - - return { - text: token, - startSec: w.start, - endSec: w.end, - charStart: start, - charEnd: end, - }; - }).filter((word) => word.text.length > 0); - - return { - sentence, - sentenceIndex: 0, - words: alignedWords, - }; -} diff --git a/compute/core/tsconfig.json b/compute/core/tsconfig.json deleted file mode 100644 index 3e68ae5..0000000 --- a/compute/core/tsconfig.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "extends": "../../tsconfig.json", - "compilerOptions": { - "noEmit": true - }, - "include": ["src/**/*.ts"] -} diff --git a/docs-site/docs/deploy/compute-worker.md b/docs-site/docs/deploy/compute-worker.md index cd279db..604c7e1 100644 --- a/docs-site/docs/deploy/compute-worker.md +++ b/docs-site/docs/deploy/compute-worker.md @@ -11,7 +11,8 @@ Use this guide when OpenReader runs compute as a separate service. For the defau - Runs PDF layout parsing jobs - Stores durable job state in NATS JetStream and NATS KV -The app server submits work to `POST /ops` and listens for updates on `GET /ops/:opId/events`. +The app server submits resource-specific operations under `/v1` and listens for updates on +`GET /v1/operations/:opId/events`. ## When to use it @@ -37,7 +38,7 @@ S3_SECRET_ACCESS_KEY=... ``` :::important -`compute/worker/.env*` is only for standalone worker deployments. +`compute-worker/.env*` is only for standalone worker deployments. - Embedded/local mode: configure the root `.env` only. - External worker mode: set `COMPUTE_WORKER_URL` and `COMPUTE_WORKER_TOKEN` on the app, and worker runtime values on the worker service. diff --git a/docs-site/docs/deploy/local-development.md b/docs-site/docs/deploy/local-development.md index b14a087..6098225 100644 --- a/docs-site/docs/deploy/local-development.md +++ b/docs-site/docs/deploy/local-development.md @@ -163,17 +163,17 @@ If you need mirrors or pinned artifact locations, set `WHISPER_MODEL_BASE_URL` i External compute worker dev stack (optional) Use this only when you intentionally run compute-worker as a separate service. -Default local flow does not need `compute/worker/.env`; embedded worker startup reads root `.env`. +Default local flow does not need `compute-worker/.env`; embedded worker startup reads root `.env`. Full worker deployment details are in [Compute Worker (NATS JetStream)](./compute-worker). Start only NATS + compute-worker via compose watch: ```bash -docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --watch -# or: pnpm compute:dev:watch +docker compose --env-file compute-worker/.env -f compute-worker/docker-compose.yml up --watch +# or: pnpm compute:dev:compose ``` -`compute/worker/.env.example` contains a starter config for standalone worker service deployments. +`compute-worker/.env.example` contains a starter config for standalone worker service deployments. Run the main app separately on the host: @@ -190,9 +190,9 @@ COMPUTE_WORKER_TOKEN= Ownership in external worker mode: - root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale/retry overrides such as `COMPUTE_PDF_JOB_ATTEMPTS` -- `compute/worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning) +- `compute-worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning) -For embedded worker startup (`COMPUTE_WORKER_URL` unset), worker tuning values such as `COMPUTE_PDF_JOB_ATTEMPTS` must be set in the root `.env` because `compute/worker/.env*` is ignored in that mode. +For embedded worker startup (`COMPUTE_WORKER_URL` unset), worker tuning values such as `COMPUTE_PDF_JOB_ATTEMPTS` must be set in the root `.env` because `compute-worker/.env*` is ignored in that mode. Worker mode requires worker-reachable shared object storage (S3-compatible endpoint). For external worker mode, object storage must be shared/reachable by both app and worker services. @@ -240,7 +240,7 @@ COMPUTE_WORKER_TOKEN= Use the same ownership split: - root `.env`: app routing/auth (`COMPUTE_WORKER_URL`, `COMPUTE_WORKER_TOKEN`) plus optional shared timeout/stale overrides -- `compute/worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning) +- `compute-worker/.env*` (or worker platform env): worker runtime variables (`NATS_*`, `S3_*`, model base URLs, worker tuning) Use one of these `.env` mode templates: diff --git a/docs-site/docs/reference/stack.md b/docs-site/docs/reference/stack.md index ea847e6..f260e31 100644 --- a/docs-site/docs/reference/stack.md +++ b/docs-site/docs/reference/stack.md @@ -45,21 +45,20 @@ title: Stack ## External compute worker (optional) -Monorepo packages under `compute/`: +Standalone worker package: -- **`@openreader/compute-core`** — ONNX runtime lifecycle, model management, and inference logic shared by compute worker runtime + app/worker contracts +- **`@openreader/compute-worker`** — standalone Node.js compute service containing its private inference and queue runtime - ONNX runtime: `onnxruntime-node` with `@huggingface/tokenizers` - Whisper alignment: `onnx-community/whisper-base_timestamped` (q4) for word-level timestamps - PDF layout: `Bei0001/PP-DocLayoutV3-ONNX` for document block detection and layout parsing - PDF rendering: `pdfjs-dist`, `@napi-rs/canvas` for server-side page rasterization - Utilities: `jszip`, `ffmpeg-static` -- **`@openreader/compute-worker`** — standalone Node.js worker service - - HTTP server: [Fastify](https://fastify.dev/) v5 +- HTTP server: [Fastify](https://fastify.dev/) v5 with a versioned OpenAPI contract - Job queue + state: [NATS](https://nats.io/) JetStream WorkQueue pull consumers + NATS KV (`jobs.whisper`, `jobs.layout`) - Storage: AWS SDK v3 S3 client for reading/writing blobs - Logging: [Pino](https://getpino.io/) - Validation: [Zod](https://zod.dev/) -- Heavy compute is worker-backed via `COMPUTE_WORKER_URL` + `COMPUTE_WORKER_TOKEN` (remote queue via HTTP + NATS) +- The Next.js app communicates with the worker only through the versioned HTTP API generated from OpenAPI. ## Tooling and testing diff --git a/eslint.config.mjs b/eslint.config.mjs index 2a0310d..e7af21a 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -34,16 +34,14 @@ const UI_ARCHITECTURE_FILES = [ "src/components/player/**/*.{ts,tsx}", "src/components/reader/**/*.{ts,tsx}", ]; -const COMPUTE_CORE_IMPORT_PATTERNS = [ +const COMPUTE_WORKER_IMPORT_PATTERNS = [ { group: [ - "@openreader/compute-core/*", - "!@openreader/compute-core/local-runtime", - "!@openreader/compute-core/types", - "!@openreader/compute-core/api-contracts", + "@openreader/compute-worker", + "@openreader/compute-worker/*", ], message: - "Use '@openreader/compute-core' root imports for light APIs. Allowed subpaths are '@openreader/compute-core/local-runtime', '@openreader/compute-core/types', and '@openreader/compute-core/api-contracts'.", + "The app must communicate with compute-worker through the app-owned HTTP client and generated protocol types.", }, ]; const UI_ARCHITECTURE_IMPORT_PATHS = [ @@ -124,7 +122,7 @@ const eslintConfig = [ "no-restricted-imports": [ "error", { - patterns: COMPUTE_CORE_IMPORT_PATTERNS, + patterns: COMPUTE_WORKER_IMPORT_PATTERNS, }, ], }, @@ -142,7 +140,7 @@ const eslintConfig = [ "error", { paths: UI_ARCHITECTURE_IMPORT_PATHS, - patterns: COMPUTE_CORE_IMPORT_PATTERNS, + patterns: COMPUTE_WORKER_IMPORT_PATTERNS, }, ], }, diff --git a/next.config.ts b/next.config.ts index 1cf8c6b..a5908d7 100644 --- a/next.config.ts +++ b/next.config.ts @@ -15,7 +15,6 @@ const securityHeaders = [ }, ]; -const bundleWorkerCompute = true; const pdfjsTraceFiles = [ './node_modules/pdfjs-dist/package.json', './node_modules/pdfjs-dist/legacy/build/pdf.mjs', @@ -29,7 +28,6 @@ const serverExternalPackages = [ // Keep pdfjs-dist as a real package in node_modules. Server-side preview // rendering resolves pdf.js runtime assets from the filesystem at runtime. 'pdfjs-dist', - ...(!bundleWorkerCompute ? ['onnxruntime-node', '@huggingface/tokenizers'] : []), ]; const nextConfig: NextConfig = { @@ -88,14 +86,6 @@ const nextConfig: NextConfig = { }, ]; } - if (isServer && bundleWorkerCompute) { - config.resolve.alias = { - ...(config.resolve.alias || {}), - '@openreader/compute-core/local-runtime$': false, - 'onnxruntime-node$': false, - '@huggingface/tokenizers$': false, - }; - } return config; }, }; diff --git a/package.json b/package.json index de098e0..25d915d 100644 --- a/package.json +++ b/package.json @@ -8,13 +8,16 @@ "dev:raw": "next dev --turbopack -p 3003", "build": "next build", "build:bundle-guard": "node scripts/check-next-server-bundle.mjs", + "check:compute-boundary": "node scripts/check-compute-boundary.mjs", "start": "node scripts/openreader-entrypoint.mjs -- pnpm start:raw", "start:raw": "next start -p 3003", "lint": "next lint", "test": "playwright test", "test:e2e": "playwright test", "test:unit": "vitest run", - "test:compute": "vitest run --project compute-core --project compute-worker", + "test:compute": "vitest run --project compute-worker", + "compute:openapi:generate": "pnpm --dir compute-worker openapi:generate && openapi-typescript compute-worker/openapi.json -o src/lib/server/compute-worker/generated.ts", + "compute:openapi:check": "pnpm compute:openapi:generate && git diff --exit-code -- compute-worker/openapi.json src/lib/server/compute-worker/generated.ts", "migrate": "node drizzle/scripts/migrate.mjs", "migrate-fs": "node scripts/migrate-fs-v2.mjs", "migrate-fs:dry-run": "node scripts/migrate-fs-v2.mjs --dry-run true", @@ -27,14 +30,13 @@ "docs:clear": "pnpm --dir docs-site clear", "compute:worker:dev": "pnpm --filter @openreader/compute-worker dev", "compute:worker:start": "pnpm --filter @openreader/compute-worker start", - "compute:dev:compose": "docker compose --env-file compute/worker/.env -f compute/worker/docker-compose.yml up --build", + "compute:dev:compose": "docker compose --env-file compute-worker/.env -f compute-worker/docker-compose.yml up --build", "lint:route-errors": "node scripts/check-route-error-responses.mjs" }, "dependencies": { "@aws-sdk/client-s3": "^3.1061.0", "@aws-sdk/s3-request-presigner": "^3.1061.0", "@headlessui/react": "^2.2.10", - "@huggingface/tokenizers": "^0.1.3", "@microsoft/antissrf": "^1.0.0", "@mozilla/readability": "^0.6.0", "@napi-rs/canvas": "^0.1.100", @@ -61,7 +63,6 @@ "linkedom": "^0.18.12", "lru-cache": "^11.5.1", "next": "^15.5.19", - "onnxruntime-node": "^1.26.0", "openai": "^6.41.0", "pdfjs-dist": "4.8.69", "pg": "^8.21.0", @@ -98,6 +99,7 @@ "eslint": "^9.39.4", "eslint-config-next": "^15.5.19", "postcss": "^8.5.15", + "openapi-typescript": "^7.10.1", "tailwindcss": "^3.4.19", "typescript": "^5.9.3", "vitest": "^4.1.8" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e6e0608..32a7d38 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,9 +20,6 @@ importers: '@headlessui/react': specifier: ^2.2.10 version: 2.2.10(react-dom@19.2.7(react@19.2.7))(react@19.2.7) - '@huggingface/tokenizers': - specifier: ^0.1.3 - version: 0.1.3 '@microsoft/antissrf': specifier: ^1.0.0 version: 1.0.0 @@ -49,7 +46,7 @@ importers: version: 7.0.1 better-auth: specifier: ^1.6.14 - version: 1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))) + version: 1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))) better-sqlite3: specifier: ^12.10.0 version: 12.10.0 @@ -101,9 +98,6 @@ importers: next: specifier: ^15.5.19 version: 15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7) - onnxruntime-node: - specifier: ^1.26.0 - version: 1.26.0 openai: specifier: ^6.41.0 version: 6.41.0(zod@4.4.3) @@ -170,7 +164,7 @@ importers: version: 1.60.0 '@tailwindcss/typography': specifier: ^0.5.19 - version: 0.5.19(tailwindcss@3.4.19(tsx@4.22.4)) + version: 0.5.19(tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0)) '@types/better-sqlite3': specifier: ^7.6.13 version: 7.6.13 @@ -204,27 +198,48 @@ importers: eslint-config-next: specifier: ^15.5.19 version: 15.5.19(eslint@9.39.4(jiti@1.21.7))(typescript@5.9.3) + openapi-typescript: + specifier: ^7.10.1 + version: 7.13.0(typescript@5.9.3) postcss: specifier: ^8.5.15 version: 8.5.15 tailwindcss: specifier: ^3.4.19 - version: 3.4.19(tsx@4.22.4) + version: 3.4.19(tsx@4.22.4)(yaml@2.9.0) typescript: specifier: ^5.9.3 version: 5.9.3 vitest: specifier: ^4.1.8 - version: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)) + version: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)) - compute/core: + compute-worker: dependencies: + '@aws-sdk/client-s3': + specifier: ^3.1061.0 + version: 3.1061.0 + '@fastify/swagger': + specifier: ^9.0.0 + version: 9.7.0 '@huggingface/tokenizers': specifier: ^0.1.3 version: 0.1.3 '@napi-rs/canvas': specifier: ^0.1.100 version: 0.1.100 + '@nats-io/jetstream': + specifier: ^3.4.0 + version: 3.4.0 + '@nats-io/kv': + specifier: ^3.4.0 + version: 3.4.0 + '@nats-io/transport-node': + specifier: ^3.4.0 + version: 3.4.0 + fastify: + specifier: ^5.8.5 + version: 5.8.5 ffmpeg-static: specifier: ^5.3.0 version: 5.3.0 @@ -237,27 +252,6 @@ importers: pdfjs-dist: specifier: 4.8.69 version: 4.8.69 - - compute/worker: - dependencies: - '@aws-sdk/client-s3': - specifier: ^3.1061.0 - version: 3.1061.0 - '@nats-io/jetstream': - specifier: ^3.4.0 - version: 3.4.0 - '@nats-io/kv': - specifier: ^3.4.0 - version: 3.4.0 - '@nats-io/transport-node': - specifier: ^3.4.0 - version: 3.4.0 - '@openreader/compute-core': - specifier: workspace:* - version: link:../core - fastify: - specifier: ^5.8.5 - version: 5.8.5 pino: specifier: ^10.3.1 version: 10.3.1 @@ -424,6 +418,14 @@ packages: resolution: {integrity: sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==} engines: {node: '>=18.0.0'} + '@babel/code-frame@7.29.7': + resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + '@babel/runtime@7.29.7': resolution: {integrity: sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==} engines: {node: '>=6.9.0'} @@ -1031,6 +1033,9 @@ packages: '@fastify/proxy-addr@5.1.0': resolution: {integrity: sha512-INS+6gh91cLUjB+PVHfu1UqcB76Sqtpyp7bnL+FYojhjygvOPA9ctiD/JDKsyD9Xgu4hUhCSJBPig/w7duNajw==} + '@fastify/swagger@9.7.0': + resolution: {integrity: sha512-Vp1SC1GC2Hrkd3faFILv86BzUNyFz5N4/xdExqtCgkGASOzn/x+eMe4qXIGq7cdT6wif/P/oa6r1Ruqx19paZA==} + '@floating-ui/core@1.7.5': resolution: {integrity: sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==} @@ -1508,6 +1513,16 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0-rc.1 || ^18.0.0 || ^19.0.0-rc.1 + '@redocly/ajv@8.11.2': + resolution: {integrity: sha512-io1JpnwtIcvojV7QKDUSIuMN/ikdOUd1ReEnUnMKGfDVridQZ31J0MmIuqwuRjWDZfmvr+Q0MqCcfHM2gTivOg==} + + '@redocly/config@0.22.0': + resolution: {integrity: sha512-gAy93Ddo01Z3bHuVdPWfCwzgfaYgMdaZPcfL7JZ7hWJoK9V0lXDbigTWkhiPFAaLWzbOJ+kbUQG1+XwIm0KRGQ==} + + '@redocly/openapi-core@1.34.15': + resolution: {integrity: sha512-HAwCnNyKcs5XGQqms+9t7OdAPM/5TDstmhF+0i7tdCFato2QKuYIlyWETwkXd8c5zbltr1oB+6y9NTeQLr2d6Q==} + engines: {node: '>=18.17.0', npm: '>=9.5.0'} + '@rolldown/binding-android-arm64@1.0.2': resolution: {integrity: sha512-ZS4D1JPGn/MYQN/SYDWftIE/nVsM8j/AFOYEzAoOE2O3NktQOZru+/vYXGbR/qtdLdIfGCP0lcoJiYVzsEz+iQ==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2036,6 +2051,10 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + ajv-formats@3.0.1: resolution: {integrity: sha512-8iUql50EUR+uUcdRQ3HDqa6EVyo3docL8g5WJ3FNcWmu62IbkGUue/pEyLBW8VGKKucTPgqeks4fIU1DA4yowQ==} peerDependencies: @@ -2050,6 +2069,10 @@ packages: ajv@8.20.0: resolution: {integrity: sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==} + ansi-colors@4.1.3: + resolution: {integrity: sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==} + engines: {node: '>=6'} + ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -2384,6 +2407,9 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + change-case@5.4.4: + resolution: {integrity: sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w==} + character-entities-html4@2.1.0: resolution: {integrity: sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==} @@ -2421,6 +2447,9 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + colorette@1.4.0: + resolution: {integrity: sha512-Y2oEozpomLn7Q3HFP7dpww7AtMJplbM9lGZP6RDfHqmbeRjiwRg4n6VM6j4KLmRke85uWEI7JqF17f3pqdRA0g==} + colorette@2.0.20: resolution: {integrity: sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==} @@ -3011,6 +3040,9 @@ packages: resolution: {integrity: sha512-6bIM7fsJxeo3uXv7OncQYsBAMPJ7V16Slahl/6M98C/i2q+vB1+4a0MtrvYwDFEUrwDSbAmeLDRXsOBwrL7yAg==} hasBin: true + fastify-plugin@5.1.0: + resolution: {integrity: sha512-FAIDA8eovSt5qcDgcBvDuX/v0Cjz0ohGhENZ/wpc3y+oZCY2afZ9Baqql3g/lC+OHRnciQol4ww7tuthOb9idw==} + fastify@5.8.5: resolution: {integrity: sha512-Yqptv59pQzPgQUSIm87hMqHJmdkb1+GPxdE6vW6FRyVE9G86mt7rOghitiU4JHRaTyDUk9pfeKmDeu70lAwM4Q==} @@ -3212,6 +3244,10 @@ packages: resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} engines: {node: '>= 6'} + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + ieee754@1.2.1: resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} @@ -3234,6 +3270,10 @@ packages: resolution: {integrity: sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==} engines: {node: '>=0.8.19'} + index-to-position@1.2.0: + resolution: {integrity: sha512-Yg7+ztRkqslMAS2iFaU+Oa4KTSidr63OsFGlOrJoW981kIYO3CGCS3wA95P1mUi/IVSJkn0D479KTJpVpvFNuw==} + engines: {node: '>=18'} + inherits@2.0.4: resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} @@ -3413,9 +3453,17 @@ packages: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} + js-levenshtein@1.1.6: + resolution: {integrity: sha512-X2BB11YZtrRqY4EnQcLX5Rh373zbK4alC1FW7D7MBhL2gtcC17cTnr6DmfHZeS0s2rTHjUTMMHfG7gO8SSdw+g==} + engines: {node: '>=0.10.0'} + js-tokens@4.0.0: resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + js-yaml@4.1.1: + resolution: {integrity: sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==} + hasBin: true + js-yaml@4.2.0: resolution: {integrity: sha512-ePWsvanv0DWuDRsW8dnt+R4jQ31SCRCQ7hhNcPXZPsoBZiemuZNYGf7adZdqX2D86j6rvKp3RpCxVTSb8WQlOw==} hasBin: true @@ -3426,6 +3474,10 @@ packages: json-schema-ref-resolver@3.0.0: resolution: {integrity: sha512-hOrZIVL5jyYFjzk7+y7n5JDzGlU8rfWDuYyHwGa2WA8/pcmMHezp2xsVwxrebD/Q9t8Nc5DboieySDpCp4WG4A==} + json-schema-resolver@3.0.0: + resolution: {integrity: sha512-HqMnbz0tz2DaEJ3ntsqtx3ezzZyDE7G56A/pPY/NGmrPu76UzsWquOpHFRAf5beTNXoH2LU5cQePVvRli1nchA==} + engines: {node: '>=20'} + json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} @@ -3927,6 +3979,15 @@ packages: zod: optional: true + openapi-types@12.1.3: + resolution: {integrity: sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw==} + + openapi-typescript@7.13.0: + resolution: {integrity: sha512-EFP392gcqXS7ntPvbhBzbF8TyBA+baIYEm791Hy5YkjDYKTnk/Tn5OQeKm5BIZvJihpp8Zzr4hzx0Irde1LNGQ==} + hasBin: true + peerDependencies: + typescript: ^5.x + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -3963,6 +4024,10 @@ packages: parse-entities@4.0.2: resolution: {integrity: sha512-GG2AQYWoLgL877gQIKeRPGO1xF9+eG1ujIb5soS5gPvLQ1y2o8FL90w2QWNdf9I361Mpp7726c+lj3U0qK1uGw==} + parse-json@8.3.0: + resolution: {integrity: sha512-ybiGyvspI+fAoRQbIPRddCcSTV9/LsJbf0e/S85VLowVGzRmokfneg2kwVW/KU5rOXrPSbF1qAKPMgNTqqROQQ==} + engines: {node: '>=18'} + path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -4073,6 +4138,10 @@ packages: engines: {node: '>=18'} hasBin: true + pluralize@8.0.0: + resolution: {integrity: sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA==} + engines: {node: '>=4'} + possible-typed-array-names@1.1.0: resolution: {integrity: sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg==} engines: {node: '>= 0.4'} @@ -4615,6 +4684,10 @@ packages: suffix-thumb@5.0.2: resolution: {integrity: sha512-I5PWXAFKx3FYnI9a+dQMWNqTxoRt6vdBdb0O+BJ1sxXCWtSoQCusc13E58f+9p4MYx/qCnEMkD5jac6K2j3dgA==} + supports-color@10.2.2: + resolution: {integrity: sha512-SS+jx45GF1QjgEXQx4NJZV9ImqmO2NPz5FNsIHrsDjh2YsHnawpan7SNQ1o8NuhrbHZy9AZhIoCUiCeaW/C80g==} + engines: {node: '>=18'} + supports-color@7.2.0: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} @@ -4728,6 +4801,10 @@ packages: resolution: {integrity: sha512-Ne+eE4r0/iWnpAxD852z3A+N0Bt5RN//NjJwRd2VFHEmrywxf5vsZlh4R6lixl6B+wz/8d+maTSAkN1FIkI3LQ==} engines: {node: '>=10'} + type-fest@4.41.0: + resolution: {integrity: sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==} + engines: {node: '>=16'} + type@2.7.3: resolution: {integrity: sha512-8j+1QmAbPvLZow5Qpi6NCaN8FB60p/6x8/vfNqOk/hC+HuvFZhL4+WfekuhQLiqFZXOgQdrs3B+XxEmCc6b3FQ==} @@ -4786,6 +4863,9 @@ packages: unrs-resolver@1.12.2: resolution: {integrity: sha512-dmlRxBJJayXjqTwC+JtF1HhJmgf3ftQ3YejFcZrf4+KKtJv0qDsK1pjqaaVjG7wJ5NJ6UVP1OqRMQ71Z4C3rxQ==} + uri-js-replace@1.0.1: + resolution: {integrity: sha512-W+C9NWNLFOoBI2QWDp4UT9pv65r2w5Cx+3sTYFvtMdDBxkKt1syCqsUdSFAChbEe1uK5TfS04wt/nGwmaeIQ0g==} + uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} @@ -4943,6 +5023,18 @@ packages: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} + yaml-ast-parser@0.0.43: + resolution: {integrity: sha512-2PTINUwsRqSd+s8XxKaJWQlUuEMHJQyEuh2edBbW8KNJz0SJPwUSD2zRWqezFEdN7IzAgeuYHFUCF7o8zRdZ0A==} + + yaml@2.9.0: + resolution: {integrity: sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==} + engines: {node: '>= 14.6'} + hasBin: true + + yargs-parser@21.1.1: + resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} + engines: {node: '>=12'} + yocto-queue@0.1.0: resolution: {integrity: sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==} engines: {node: '>=10'} @@ -5224,6 +5316,14 @@ snapshots: '@aws/lambda-invoke-store@0.2.4': {} + '@babel/code-frame@7.29.7': + dependencies: + '@babel/helper-validator-identifier': 7.29.7 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/helper-validator-identifier@7.29.7': {} + '@babel/runtime@7.29.7': {} '@better-auth/core@1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0)': @@ -5546,7 +5646,7 @@ snapshots: '@eslint/config-array@0.21.2': dependencies: '@eslint/object-schema': 2.1.7 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) minimatch: 3.1.5 transitivePeerDependencies: - supports-color @@ -5562,7 +5662,7 @@ snapshots: '@eslint/eslintrc@3.3.5': dependencies: ajv: 6.15.0 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) espree: 10.4.0 globals: 14.0.0 ignore: 5.3.2 @@ -5605,6 +5705,16 @@ snapshots: '@fastify/forwarded': 3.0.1 ipaddr.js: 2.4.0 + '@fastify/swagger@9.7.0': + dependencies: + fastify-plugin: 5.1.0 + json-schema-resolver: 3.0.0 + openapi-types: 12.1.3 + rfdc: 1.4.1 + yaml: 2.9.0 + transitivePeerDependencies: + - supports-color + '@floating-ui/core@1.7.5': dependencies: '@floating-ui/utils': 0.2.11 @@ -5968,6 +6078,29 @@ snapshots: dependencies: react: 19.2.7 + '@redocly/ajv@8.11.2': + dependencies: + fast-deep-equal: 3.1.3 + json-schema-traverse: 1.0.0 + require-from-string: 2.0.2 + uri-js-replace: 1.0.1 + + '@redocly/config@0.22.0': {} + + '@redocly/openapi-core@1.34.15(supports-color@10.2.2)': + dependencies: + '@redocly/ajv': 8.11.2 + '@redocly/config': 0.22.0 + colorette: 1.4.0 + https-proxy-agent: 7.0.6(supports-color@10.2.2) + js-levenshtein: 1.1.6 + js-yaml: 4.1.1 + minimatch: 5.1.9 + pluralize: 8.0.0 + yaml-ast-parser: 0.0.43 + transitivePeerDependencies: + - supports-color + '@rolldown/binding-android-arm64@1.0.2': optional: true @@ -6088,10 +6221,10 @@ snapshots: dependencies: tslib: 2.8.1 - '@tailwindcss/typography@0.5.19(tailwindcss@3.4.19(tsx@4.22.4))': + '@tailwindcss/typography@0.5.19(tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0))': dependencies: postcss-selector-parser: 6.0.10 - tailwindcss: 3.4.19(tsx@4.22.4) + tailwindcss: 3.4.19(tsx@4.22.4)(yaml@2.9.0) '@tanstack/query-core@5.101.0': {} @@ -6220,7 +6353,7 @@ snapshots: '@typescript-eslint/types': 8.60.1 '@typescript-eslint/typescript-estree': 8.60.1(typescript@5.9.3) '@typescript-eslint/visitor-keys': 8.60.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) eslint: 9.39.4(jiti@1.21.7) typescript: 5.9.3 transitivePeerDependencies: @@ -6230,7 +6363,7 @@ snapshots: dependencies: '@typescript-eslint/tsconfig-utils': 8.60.1(typescript@5.9.3) '@typescript-eslint/types': 8.60.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) typescript: 5.9.3 transitivePeerDependencies: - supports-color @@ -6249,7 +6382,7 @@ snapshots: '@typescript-eslint/types': 8.60.1 '@typescript-eslint/typescript-estree': 8.60.1(typescript@5.9.3) '@typescript-eslint/utils': 8.60.1(eslint@9.39.4(jiti@1.21.7))(typescript@5.9.3) - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) eslint: 9.39.4(jiti@1.21.7) ts-api-utils: 2.5.0(typescript@5.9.3) typescript: 5.9.3 @@ -6264,7 +6397,7 @@ snapshots: '@typescript-eslint/tsconfig-utils': 8.60.1(typescript@5.9.3) '@typescript-eslint/types': 8.60.1 '@typescript-eslint/visitor-keys': 8.60.1 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) minimatch: 10.2.5 semver: 7.8.1 tinyglobby: 0.2.17 @@ -6375,13 +6508,13 @@ snapshots: chai: 6.2.2 tinyrainbow: 3.1.0 - '@vitest/mocker@4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))': + '@vitest/mocker@4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))': dependencies: '@vitest/spy': 4.1.8 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4) + vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0) '@vitest/pretty-format@4.1.8': dependencies: @@ -6425,10 +6558,12 @@ snapshots: agent-base@6.0.2: dependencies: - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color + agent-base@7.1.4: {} + ajv-formats@3.0.1(ajv@8.20.0): optionalDependencies: ajv: 8.20.0 @@ -6447,6 +6582,8 @@ snapshots: json-schema-traverse: 1.0.0 require-from-string: 2.0.2 + ansi-colors@4.1.3: {} + ansi-regex@5.0.1: {} ansi-regex@6.2.2: {} @@ -6632,7 +6769,7 @@ snapshots: base64-js@1.5.1: {} - better-auth@1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4))): + better-auth@1.6.14(better-sqlite3@12.10.0)(drizzle-kit@0.31.10)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0))(next@15.5.19(@playwright/test@1.60.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7))(pg@8.21.0)(react-dom@19.2.7(react@19.2.7))(react@19.2.7)(vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0))): dependencies: '@better-auth/core': 1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0) '@better-auth/drizzle-adapter': 1.6.14(@better-auth/core@1.6.14(@better-auth/utils@0.4.1)(@better-fetch/fetch@1.1.21)(better-call@1.3.5(zod@4.4.3))(jose@6.2.3)(kysely@0.28.17)(nanostores@1.3.0))(@better-auth/utils@0.4.1)(drizzle-orm@0.45.2(@types/better-sqlite3@7.6.13)(@types/pg@8.20.0)(better-sqlite3@12.10.0)(kysely@0.28.17)(pg@8.21.0)) @@ -6659,7 +6796,7 @@ snapshots: pg: 8.21.0 react: 19.2.7 react-dom: 19.2.7(react@19.2.7) - vitest: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)) + vitest: 4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)) transitivePeerDependencies: - '@cloudflare/workers-types' - '@opentelemetry/api' @@ -6765,6 +6902,8 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + change-case@5.4.4: {} + character-entities-html4@2.1.0: {} character-entities-legacy@3.0.0: {} @@ -6799,6 +6938,8 @@ snapshots: color-name@1.1.4: {} + colorette@1.4.0: {} + colorette@2.0.20: {} comma-separated-tokens@2.0.3: {} @@ -6896,9 +7037,11 @@ snapshots: dependencies: ms: 2.1.3 - debug@4.4.3: + debug@4.4.3(supports-color@10.2.2): dependencies: ms: 2.1.3 + optionalDependencies: + supports-color: 10.2.2 decode-named-character-reference@1.3.0: dependencies: @@ -7265,7 +7408,7 @@ snapshots: eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0)(eslint@9.39.4(jiti@1.21.7)): dependencies: '@nolyfill/is-core-module': 1.0.39 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) eslint: 9.39.4(jiti@1.21.7) get-tsconfig: 4.14.0 is-bun-module: 2.0.0 @@ -7390,7 +7533,7 @@ snapshots: ajv: 6.15.0 chalk: 4.1.2 cross-spawn: 7.0.6 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) escape-string-regexp: 4.0.0 eslint-scope: 8.4.0 eslint-visitor-keys: 4.2.1 @@ -7535,6 +7678,8 @@ snapshots: strnum: 2.3.0 xml-naming: 0.1.0 + fastify-plugin@5.1.0: {} + fastify@5.8.5: dependencies: '@fastify/ajv-compiler': 4.0.5 @@ -7777,7 +7922,14 @@ snapshots: https-proxy-agent@5.0.1: dependencies: agent-base: 6.0.2 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) + transitivePeerDependencies: + - supports-color + + https-proxy-agent@7.0.6(supports-color@10.2.2): + dependencies: + agent-base: 7.1.4 + debug: 4.4.3(supports-color@10.2.2) transitivePeerDependencies: - supports-color @@ -7796,6 +7948,8 @@ snapshots: imurmurhash@0.1.4: {} + index-to-position@1.2.0: {} + inherits@2.0.4: {} ini@1.3.8: {} @@ -7972,8 +8126,14 @@ snapshots: joycon@3.1.1: {} + js-levenshtein@1.1.6: {} + js-tokens@4.0.0: {} + js-yaml@4.1.1: + dependencies: + argparse: 2.0.1 + js-yaml@4.2.0: dependencies: argparse: 2.0.1 @@ -7984,6 +8144,14 @@ snapshots: dependencies: dequal: 2.0.3 + json-schema-resolver@3.0.0: + dependencies: + debug: 4.4.3(supports-color@10.2.2) + fast-uri: 3.1.2 + rfdc: 1.4.1 + transitivePeerDependencies: + - supports-color + json-schema-traverse@0.4.1: {} json-schema-traverse@1.0.0: {} @@ -8482,7 +8650,7 @@ snapshots: micromark@4.0.2: dependencies: '@types/debug': 4.1.13 - debug: 4.4.3 + debug: 4.4.3(supports-color@10.2.2) decode-named-character-reference: 1.3.0 devlop: 1.1.0 micromark-core-commonmark: 2.0.3 @@ -8658,6 +8826,18 @@ snapshots: optionalDependencies: zod: 4.4.3 + openapi-types@12.1.3: {} + + openapi-typescript@7.13.0(typescript@5.9.3): + dependencies: + '@redocly/openapi-core': 1.34.15(supports-color@10.2.2) + ansi-colors: 4.1.3 + change-case: 5.4.4 + parse-json: 8.3.0 + supports-color: 10.2.2 + typescript: 5.9.3 + yargs-parser: 21.1.1 + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -8705,6 +8885,12 @@ snapshots: is-decimal: 2.0.1 is-hexadecimal: 2.0.1 + parse-json@8.3.0: + dependencies: + '@babel/code-frame': 7.29.7 + index-to-position: 1.2.0 + type-fest: 4.41.0 + path-exists@4.0.0: {} path-expression-matcher@1.5.0: {} @@ -8819,6 +9005,8 @@ snapshots: optionalDependencies: fsevents: 2.3.2 + pluralize@8.0.0: {} + possible-typed-array-names@1.1.0: {} postcss-import@15.1.0(postcss@8.5.15): @@ -8833,13 +9021,14 @@ snapshots: camelcase-css: 2.0.1 postcss: 8.5.15 - postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4): + postcss-load-config@6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4)(yaml@2.9.0): dependencies: lilconfig: 3.1.3 optionalDependencies: jiti: 1.21.7 postcss: 8.5.15 tsx: 4.22.4 + yaml: 2.9.0 postcss-nested@6.2.0(postcss@8.5.15): dependencies: @@ -9504,6 +9693,8 @@ snapshots: suffix-thumb@5.0.2: {} + supports-color@10.2.2: {} + supports-color@7.2.0: dependencies: has-flag: 4.0.0 @@ -9512,7 +9703,7 @@ snapshots: tabbable@6.4.0: {} - tailwindcss@3.4.19(tsx@4.22.4): + tailwindcss@3.4.19(tsx@4.22.4)(yaml@2.9.0): dependencies: '@alloc/quick-lru': 5.2.0 arg: 5.0.2 @@ -9531,7 +9722,7 @@ snapshots: postcss: 8.5.15 postcss-import: 15.1.0(postcss@8.5.15) postcss-js: 4.1.0(postcss@8.5.15) - postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4) + postcss-load-config: 6.0.1(jiti@1.21.7)(postcss@8.5.15)(tsx@4.22.4)(yaml@2.9.0) postcss-nested: 6.2.0(postcss@8.5.15) postcss-selector-parser: 6.1.2 resolve: 1.22.12 @@ -9651,6 +9842,8 @@ snapshots: type-fest@0.20.2: {} + type-fest@4.41.0: {} + type@2.7.3: {} typed-array-buffer@1.0.3: @@ -9761,6 +9954,8 @@ snapshots: '@unrs/resolver-binding-win32-ia32-msvc': 1.12.2 '@unrs/resolver-binding-win32-x64-msvc': 1.12.2 + uri-js-replace@1.0.1: {} + uri-js@4.4.1: dependencies: punycode: 2.3.1 @@ -9783,7 +9978,7 @@ snapshots: '@types/unist': 3.0.3 vfile-message: 4.0.3 - vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4): + vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0): dependencies: lightningcss: 1.32.0 picomatch: 4.0.4 @@ -9796,11 +9991,12 @@ snapshots: fsevents: 2.3.3 jiti: 1.21.7 tsx: 4.22.4 + yaml: 2.9.0 - vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)): + vitest@4.1.8(@types/node@20.19.41)(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)): dependencies: '@vitest/expect': 4.1.8 - '@vitest/mocker': 4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)) + '@vitest/mocker': 4.1.8(vite@8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0)) '@vitest/pretty-format': 4.1.8 '@vitest/runner': 4.1.8 '@vitest/snapshot': 4.1.8 @@ -9817,7 +10013,7 @@ snapshots: tinyexec: 1.2.4 tinyglobby: 0.2.17 tinyrainbow: 3.1.0 - vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4) + vite: 8.0.14(@types/node@20.19.41)(esbuild@0.28.0)(jiti@1.21.7)(tsx@4.22.4)(yaml@2.9.0) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 20.19.41 @@ -9898,6 +10094,12 @@ snapshots: xtend@4.0.2: {} + yaml-ast-parser@0.0.43: {} + + yaml@2.9.0: {} + + yargs-parser@21.1.1: {} + yocto-queue@0.1.0: {} zip-stream@6.0.1: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index fcc6ecc..e94fdc1 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,6 @@ packages: - . - - compute/* + - compute-worker - docker/* allowBuilds: diff --git a/scripts/check-compute-boundary.mjs b/scripts/check-compute-boundary.mjs new file mode 100644 index 0000000..8cc6a20 --- /dev/null +++ b/scripts/check-compute-boundary.mjs @@ -0,0 +1,39 @@ +import fs from 'node:fs'; +import path from 'node:path'; + +const root = process.cwd(); +const forbidden = [ + '@openreader/compute-core', + '@openreader/compute-worker', + 'compute/core', + 'compute/worker', + 'compute-worker/src', +]; +const failures = []; + +function walk(dir) { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const fullPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + walk(fullPath); + continue; + } + if (!entry.isFile() || !/\.(ts|tsx|js|mjs|cjs)$/.test(entry.name)) continue; + const text = fs.readFileSync(fullPath, 'utf8'); + for (const pattern of forbidden) { + if (text.includes(pattern)) failures.push({ file: fullPath, pattern }); + } + } +} + +walk(path.join(root, 'src')); + +if (failures.length > 0) { + console.error('[compute-boundary] Forbidden app-to-worker source coupling detected:'); + for (const failure of failures) { + console.error(`- ${failure.pattern} in ${path.relative(root, failure.file)}`); + } + process.exit(1); +} + +console.info('[compute-boundary] OK: app communicates with compute worker through protocol/client only'); diff --git a/scripts/check-next-server-bundle.mjs b/scripts/check-next-server-bundle.mjs index 9621cb4..2b3cbcb 100644 --- a/scripts/check-next-server-bundle.mjs +++ b/scripts/check-next-server-bundle.mjs @@ -12,7 +12,8 @@ if (!fs.existsSync(serverDir)) { const forbidden = [ 'onnxruntime-node', '@huggingface/tokenizers', - '@openreader/compute-core/local-runtime', + '@openreader/compute-worker', + '/compute-worker/src/compute/', ]; const includeExt = new Set(['.js', '.mjs', '.cjs']); diff --git a/scripts/openreader-entrypoint.mjs b/scripts/openreader-entrypoint.mjs index 44588e1..957ad63 100644 --- a/scripts/openreader-entrypoint.mjs +++ b/scripts/openreader-entrypoint.mjs @@ -222,7 +222,7 @@ function spawnMainCommand(command, env) { function resolveEmbeddedWorkerLaunch() { const candidateDirs = [ path.join(process.cwd(), 'embedded-compute-worker'), - path.join(process.cwd(), 'compute', 'worker'), + path.join(process.cwd(), 'compute-worker'), ]; for (const workerDir of candidateDirs) { @@ -237,7 +237,7 @@ function resolveEmbeddedWorkerLaunch() { throw new Error( 'Could not find an embedded compute worker runtime. ' - + 'Include embedded-compute-worker/src/server.ts in the runtime image or keep compute/worker available locally.', + + 'Include embedded-compute-worker/src/server.ts in the runtime image or keep compute-worker available locally.', ); } diff --git a/src/app/api/documents/[id]/parsed/events/route.ts b/src/app/api/documents/[id]/parsed/events/route.ts index a0d3dfd..fd8e08d 100644 --- a/src/app/api/documents/[id]/parsed/events/route.ts +++ b/src/app/api/documents/[id]/parsed/events/route.ts @@ -3,7 +3,7 @@ import { and, eq, inArray } from 'drizzle-orm'; import { db } from '@/db'; import { documents } from '@/db/schema'; import { requireAuthContext } from '@/lib/server/auth/auth'; -import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker'; +import { getComputeWorkerClient } from '@/lib/server/compute-worker/client'; import { isValidDocumentId } from '@/lib/server/documents/blobstore'; import { fetchPdfParseOperation, @@ -45,14 +45,6 @@ async function loadOwnedDocumentRow(input: { return rows[0] ?? null; } -function workerEventsUrl(baseUrl: string, opId: string, sinceEventId: string | null): string { - const url = new URL(`${baseUrl}/ops/${encodeURIComponent(opId)}/events`); - if (sinceEventId) { - url.searchParams.set('sinceEventId', sinceEventId); - } - return url.toString(); -} - export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string }> }) { const { logger } = createRequestLogger({ route: '/api/documents/[id]/parsed/events', @@ -92,17 +84,11 @@ export async function GET(req: NextRequest, ctx: { params: Promise<{ id: string return NextResponse.json({ error: 'Operation not found' }, { status: 404 }); } - const cfg = getWorkerClientConfigFromEnv(); const lastEventId = req.headers.get('last-event-id'); const sinceEventId = req.nextUrl.searchParams.get('sinceEventId') || lastEventId; - const upstream = await fetch(workerEventsUrl(cfg.baseUrl, opId, sinceEventId), { - method: 'GET', - headers: { - Authorization: `Bearer ${cfg.token}`, - Accept: 'text/event-stream', - ...(lastEventId ? { 'Last-Event-ID': lastEventId } : {}), - }, - cache: 'no-store', + const upstream = await getComputeWorkerClient().openOperationEvents(opId, { + sinceEventId, + lastEventId, signal: req.signal, }); diff --git a/src/lib/server/compute-worker/client.ts b/src/lib/server/compute-worker/client.ts new file mode 100644 index 0000000..2f8c0b6 --- /dev/null +++ b/src/lib/server/compute-worker/client.ts @@ -0,0 +1,249 @@ +import type { + PdfLayoutRequest, + PdfLayoutResolution, + PdfLayoutResult, + WhisperAlignRequest, + WhisperAlignResult, + WorkerOperation, + WorkerOperationEvent, +} from './protocol'; +import { isWorkerOperation } from './protocol'; + +class WorkerHttpError extends Error { + constructor( + message: string, + readonly status: number, + readonly retryAfterMs: number | null = null, + ) { + super(message); + this.name = 'WorkerHttpError'; + } +} + +function readRequiredEnv(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required for compute worker client`); + return value; +} + +function normalizeWorkerBaseUrl(raw: string): string { + const trimmed = raw.trim(); + if (!trimmed) throw new Error('COMPUTE_WORKER_URL is empty'); + const withScheme = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(trimmed) + ? trimmed + : (/^(localhost|127(?:\.\d{1,3}){3})(:\d+)?(\/|$)/.test(trimmed) + ? `http://${trimmed}` + : `https://${trimmed}`); + const parsed = new URL(withScheme); + parsed.pathname = parsed.pathname.replace(/\/+$/, ''); + return parsed.toString().replace(/\/+$/, ''); +} + +export function getComputeWorkerConfigFromEnv(): { baseUrl: string; token: string } { + return { + baseUrl: normalizeWorkerBaseUrl(readRequiredEnv('COMPUTE_WORKER_URL')), + token: readRequiredEnv('COMPUTE_WORKER_TOKEN'), + }; +} + +export function isComputeWorkerAvailable(): boolean { + try { + getComputeWorkerConfigFromEnv(); + return true; + } catch { + return false; + } +} + +function readPositiveIntEnv(name: string, fallback: number): number { + const raw = process.env[name]?.trim(); + if (!raw) return fallback; + const parsed = Number(raw); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : fallback; +} + +function alignmentWaitTimeoutMs(): number { + return Math.max(60_000, readPositiveIntEnv('COMPUTE_WHISPER_TIMEOUT_MS', 30_000) + 15_000); +} + +function parseRetryAfterMs(value: string | null): number | null { + if (!value) return null; + const seconds = Number(value); + if (Number.isFinite(seconds)) return Math.max(0, Math.floor(seconds * 1000)); + const date = Date.parse(value); + return Number.isNaN(date) ? null : Math.max(0, date - Date.now()); +} + +function parseSseFrame(frame: string): { id: number | null; data: string | null } { + let id: number | null = null; + const data: string[] = []; + for (const line of frame.split(/\r?\n/)) { + if (line.startsWith('id:')) { + const parsed = Number(line.slice(3).trim()); + if (Number.isFinite(parsed) && parsed >= 0) id = Math.floor(parsed); + } else if (line.startsWith('data:')) { + data.push(line.slice(5).trimStart()); + } + } + return { id, data: data.length > 0 ? data.join('\n') : null }; +} + +function isTerminal(operation: WorkerOperation): boolean { + return operation.status === 'succeeded' || operation.status === 'failed'; +} + +export class ComputeWorkerClient { + private readonly baseUrl: string; + private readonly token: string; + + constructor(config = getComputeWorkerConfigFromEnv()) { + this.baseUrl = config.baseUrl; + this.token = config.token; + } + + async alignWords(input: WhisperAlignRequest): Promise { + let lastError: unknown; + for (let attempt = 1; attempt <= 2; attempt += 1) { + try { + const operation = await this.requestJson>( + 'POST', + '/v1/whisper-align/operations', + input, + ); + const final = isTerminal(operation) + ? operation + : await this.waitForOperation(operation.opId, alignmentWaitTimeoutMs()); + if (final.status !== 'succeeded' || !final.result) { + throw new Error(final.error?.message || 'Whisper worker operation did not complete'); + } + return final.result; + } catch (error) { + lastError = error; + if (attempt >= 2 || !this.shouldRetry(error)) break; + const delay = error instanceof WorkerHttpError && error.retryAfterMs !== null + ? error.retryAfterMs + : attempt * 250; + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + throw lastError instanceof Error ? lastError : new Error('Unknown compute worker failure'); + } + + createPdfLayoutOperation(input: PdfLayoutRequest): Promise> { + return this.requestJson('POST', '/v1/pdf-layout/operations', input); + } + + resolvePdfLayout(input: PdfLayoutRequest): Promise { + return this.requestJson('POST', '/v1/pdf-layout/resolve', { + documentId: input.documentId, + namespace: input.namespace, + documentObjectKey: input.documentObjectKey, + }); + } + + async getOperation(opId: string): Promise | null> { + try { + return await this.requestJson('GET', `/v1/operations/${encodeURIComponent(opId)}`); + } catch (error) { + if (error instanceof WorkerHttpError && error.status === 404) return null; + throw error; + } + } + + async openOperationEvents(opId: string, input?: { + sinceEventId?: string | null; + lastEventId?: string | null; + signal?: AbortSignal; + }): Promise { + const url = new URL(`${this.baseUrl}/v1/operations/${encodeURIComponent(opId)}/events`); + if (input?.sinceEventId) url.searchParams.set('sinceEventId', input.sinceEventId); + return fetch(url, { + method: 'GET', + headers: { + Authorization: `Bearer ${this.token}`, + Accept: 'text/event-stream', + ...(input?.lastEventId ? { 'Last-Event-ID': input.lastEventId } : {}), + }, + cache: 'no-store', + signal: input?.signal, + }); + } + + private async requestJson(method: 'GET' | 'POST', path: string, body?: unknown): Promise { + const response = await fetch(`${this.baseUrl}${path}`, { + method, + headers: { + Authorization: `Bearer ${this.token}`, + Accept: 'application/json', + ...(method === 'POST' ? { 'Content-Type': 'application/json' } : {}), + }, + ...(method === 'POST' ? { body: JSON.stringify(body ?? {}) } : {}), + cache: 'no-store', + }); + if (!response.ok) { + const detail = await response.text().catch(() => ''); + throw new WorkerHttpError( + `Worker request failed (${method} ${path}): ${response.status}${detail ? ` ${detail}` : ''}`, + response.status, + parseRetryAfterMs(response.headers.get('retry-after')), + ); + } + const parsed = await response.json() as T; + return parsed; + } + + private shouldRetry(error: unknown): boolean { + if (error instanceof WorkerHttpError) { + return error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504; + } + return error instanceof Error && /network|timeout|fetch failed/i.test(error.message); + } + + private async waitForOperation(opId: string, timeoutMs: number): Promise> { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + let lastEventId: number | null = null; + try { + while (!controller.signal.aborted) { + const response = await this.openOperationEvents(opId, { + sinceEventId: lastEventId === null ? null : String(lastEventId), + lastEventId: lastEventId === null ? null : String(lastEventId), + signal: controller.signal, + }); + if (!response.ok || !response.body) { + throw new WorkerHttpError(`Worker event stream failed: ${response.status}`, response.status); + } + const reader = response.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buffer += decoder.decode(value, { stream: true }); + while (true) { + const end = buffer.indexOf('\n\n'); + if (end < 0) break; + const frame = parseSseFrame(buffer.slice(0, end)); + buffer = buffer.slice(end + 2); + if (frame.id !== null) lastEventId = frame.id; + if (!frame.data) continue; + const event = JSON.parse(frame.data) as WorkerOperationEvent; + if (!event?.snapshot || !isWorkerOperation(event.snapshot) || event.snapshot.opId !== opId) continue; + if (isTerminal(event.snapshot)) return event.snapshot; + } + } + await new Promise((resolve) => setTimeout(resolve, 250)); + } + throw new Error(`Operation stream timed out for ${opId}`); + } finally { + clearTimeout(timeout); + } + } +} + +let client: ComputeWorkerClient | null = null; + +export function getComputeWorkerClient(): ComputeWorkerClient { + client ??= new ComputeWorkerClient(); + return client; +} diff --git a/src/lib/server/compute-worker/generated.ts b/src/lib/server/compute-worker/generated.ts new file mode 100644 index 0000000..feb2060 --- /dev/null +++ b/src/lib/server/compute-worker/generated.ts @@ -0,0 +1,700 @@ +/** + * This file was auto-generated by openapi-typescript. + * Do not make direct changes to the file. + */ + +export interface paths { + "/health/live": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + ok: boolean; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/health/ready": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + ok: boolean; + natsConnected: boolean; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/whisper-align/operations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + text: string; + lang?: string; + cacheKey?: string; + audioObjectKey: string; + }; + }; + }; + responses: { + /** @description Default Response */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/pdf-layout/operations": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + documentId: string; + namespace: string | null; + documentObjectKey: string; + replaceToken?: string; + }; + }; + }; + responses: { + /** @description Default Response */ + 202: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/pdf-layout/resolve": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": { + documentId: string; + namespace: string | null; + documentObjectKey: string; + }; + }; + }; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + artifact: { + objectKey: string; + } | null; + operation: { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + } | null; + }; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/operations/{opId}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: never; + header?: never; + path: { + opId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Default Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + /** @description Default Response */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/operations/{opId}/events": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get: { + parameters: { + query?: { + sinceEventId?: string | number; + }; + header?: never; + path: { + opId: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Server-sent PublicOperationEvent stream */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": string; + }; + }; + /** @description Default Response */ + 400: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + /** @description Default Response */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": { + error: string; + } & { + [key: string]: unknown; + }; + }; + }; + }; + }; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; +} +export type webhooks = Record; +export interface components { + schemas: { + ParsedPdfDocument: { + /** @enum {number} */ + schemaVersion: 1; + documentId: string; + parserVersion: string; + parsedAt: number; + pages: { + pageNumber: number; + width: number; + height: number; + blocks: { + id: string; + /** @enum {string} */ + kind: "abstract" | "algorithm" | "aside_text" | "chart" | "content" | "formula" | "doc_title" | "figure_title" | "footer" | "footnote" | "formula_number" | "header" | "image" | "number" | "paragraph_title" | "reference" | "reference_content" | "seal" | "table" | "text" | "vision_footnote"; + fragments: { + page: number; + bbox: [ + number, + number, + number, + number + ]; + text: string; + readingOrder: number; + modelConfidence?: number; + }[]; + text: string; + parentSectionId?: string; + }[]; + }[]; + }; + TTSSentenceAlignment: { + sentence: string; + sentenceIndex: number; + words: { + text: string; + startSec: number; + endSec: number; + charStart: number; + charEnd: number; + }[]; + }; + ArtifactReference: { + objectKey: string; + }; + ErrorResponse: { + error: string; + } & { + [key: string]: unknown; + }; + OperationError: { + message: string; + code?: string; + }; + PdfLayoutProgress: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + PublicOperation: { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + PublicOperationEvent: { + eventId: number; + snapshot: { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + }; + }; + PdfLayoutResolution: { + artifact: { + objectKey: string; + } | null; + operation: { + opId: string; + subject: { + /** @enum {string} */ + kind: "whisper_align"; + } | { + /** @enum {string} */ + kind: "pdf_layout"; + documentId: string; + namespace: string | null; + }; + /** @enum {string} */ + status: "queued" | "running" | "succeeded" | "failed"; + queuedAt: number; + updatedAt: number; + startedAt?: number; + result?: unknown; + error?: { + message: string; + code?: string; + }; + timing?: { + queueWaitMs?: number; + s3FetchMs?: number; + computeMs?: number; + }; + progress?: { + totalPages: number; + pagesParsed: number; + currentPage?: number; + /** @enum {string} */ + phase: "infer" | "merge"; + }; + } | null; + }; + }; + responses: never; + parameters: never; + requestBodies: never; + headers: never; + pathItems: never; +} +export type $defs = Record; +export type operations = Record; diff --git a/src/lib/server/compute-worker/protocol.ts b/src/lib/server/compute-worker/protocol.ts new file mode 100644 index 0000000..73b196c --- /dev/null +++ b/src/lib/server/compute-worker/protocol.ts @@ -0,0 +1,57 @@ +import type { components, paths } from './generated'; + +export type ParsedPdfDocument = components['schemas']['ParsedPdfDocument']; +export type ParsedPdfPage = ParsedPdfDocument['pages'][number]; +export type ParsedPdfBlock = ParsedPdfPage['blocks'][number]; +export type ParsedPdfBlockKind = ParsedPdfBlock['kind']; +export type ParsedPdfBlockFragment = ParsedPdfBlock['fragments'][number]; + +export type TTSSentenceAlignment = components['schemas']['TTSSentenceAlignment']; +export type TTSSentenceWord = TTSSentenceAlignment['words'][number]; + +export type PdfLayoutProgress = NonNullable; +export type WorkerOperationStatus = components['schemas']['PublicOperation']['status']; +export type WorkerOperationSubject = components['schemas']['PublicOperation']['subject']; + +export type WorkerOperation = + Omit + & { result?: Result }; + +export type WorkerOperationEvent = { + eventId: number; + snapshot: WorkerOperation; +}; + +export type WhisperAlignRequest = + paths['/v1/whisper-align/operations']['post']['requestBody']['content']['application/json']; +export type PdfLayoutRequest = + paths['/v1/pdf-layout/operations']['post']['requestBody']['content']['application/json']; +export type PdfLayoutResolveRequest = + paths['/v1/pdf-layout/resolve']['post']['requestBody']['content']['application/json']; + +export type WhisperAlignResult = { + alignments: TTSSentenceAlignment[]; + timing?: components['schemas']['PublicOperation']['timing']; +}; + +export type PdfLayoutResult = { + parsedObjectKey: string; + timing?: components['schemas']['PublicOperation']['timing']; +}; + +export type PdfLayoutResolution = { + artifact: { objectKey: string } | null; + operation: WorkerOperation | null; +}; + +export function isWorkerOperation(value: unknown): value is WorkerOperation { + if (!value || typeof value !== 'object') return false; + const record = value as Record; + return typeof record.opId === 'string' + && !!record.subject + && typeof record.subject === 'object' + && typeof (record.subject as Record).kind === 'string' + && typeof record.status === 'string' + && typeof record.queuedAt === 'number' + && typeof record.updatedAt === 'number'; +} diff --git a/src/lib/server/compute/abort-like-error.ts b/src/lib/server/compute/abort-like-error.ts deleted file mode 100644 index 2ce848b..0000000 --- a/src/lib/server/compute/abort-like-error.ts +++ /dev/null @@ -1,9 +0,0 @@ -export function isAbortLikeError(error: unknown): boolean { - if (!error) return false; - if (error instanceof DOMException) return error.name === 'AbortError'; - if (error instanceof Error) return error.name === 'AbortError' || error.message === 'This operation was aborted'; - if (typeof error === 'object' && error !== null && 'name' in error) { - return (error as { name?: unknown }).name === 'AbortError'; - } - return false; -} diff --git a/src/lib/server/compute/index.ts b/src/lib/server/compute/index.ts deleted file mode 100644 index ac62a56..0000000 --- a/src/lib/server/compute/index.ts +++ /dev/null @@ -1,18 +0,0 @@ -import type { ComputeBackend } from '@/lib/server/compute/types'; -import { isWorkerClientConfigAvailable, WorkerComputeBackend } from '@/lib/server/compute/worker'; - -let backendPromise: Promise | null = null; - -export async function getCompute(): Promise { - if (!backendPromise) { - backendPromise = Promise.resolve(new WorkerComputeBackend()).catch((error) => { - backendPromise = null; - throw error; - }); - } - return backendPromise; -} - -export function isComputeAvailable(): boolean { - return isWorkerClientConfigAvailable(); -} diff --git a/src/lib/server/compute/types.ts b/src/lib/server/compute/types.ts deleted file mode 100644 index ffd0951..0000000 --- a/src/lib/server/compute/types.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { - TTSAudioBuffer, - TTSSentenceAlignment, - ParsedPdfDocument, -} from '@openreader/compute-core/types'; -import type { PdfLayoutProgress, WhisperAlignJobBase } from '@openreader/compute-core/api-contracts'; -import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; - -export interface WhisperAlignInput extends WhisperAlignJobBase { - audioBuffer?: TTSAudioBuffer; - audioObjectKey?: string; -} - -export interface WhisperAlignResult { - alignments: TTSSentenceAlignment[]; -} - -export interface PdfLayoutInput { - documentId: string; - namespace?: string | null; - documentObjectKey?: string; - pdfBytes?: ArrayBuffer; - forceToken?: string; - onProgress?: (progress: PdfLayoutProgress) => void | Promise; - onWorkerSnapshot?: (snapshot: WorkerOperationState) => void | Promise; -} - -export type PdfLayoutResult = - | { parsed: ParsedPdfDocument; parsedObjectKey?: never } - | { parsed?: never; parsedObjectKey: string }; - -export interface ComputeBackend { - mode: 'worker'; - alignWords(input: WhisperAlignInput): Promise; - parsePdfLayout(input: PdfLayoutInput): Promise; -} diff --git a/src/lib/server/compute/worker-op-create.ts b/src/lib/server/compute/worker-op-create.ts deleted file mode 100644 index 433b68e..0000000 --- a/src/lib/server/compute/worker-op-create.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { getWorkerClientConfigFromEnv, buildPdfOpKey } from '@/lib/server/compute/worker'; -import type { PdfLayoutInput } from '@/lib/server/compute/types'; -import type { - PdfLayoutJobResult, - WorkerOperationState, -} from '@openreader/compute-core/api-contracts'; - -type CreatePdfWorkerOpInput = - Pick - & { documentObjectKey: string }; - -const CREATE_OP_TIMEOUT_MS = 10_000; - -export async function createOrReusePdfWorkerOperation( - input: CreatePdfWorkerOpInput, -): Promise> { - const cfg = getWorkerClientConfigFromEnv(); - const opKey = buildPdfOpKey({ - documentId: input.documentId, - namespace: input.namespace, - documentObjectKey: input.documentObjectKey, - forceToken: input.forceToken, - }); - - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), CREATE_OP_TIMEOUT_MS); - try { - const res = await fetch(`${cfg.baseUrl}/ops`, { - method: 'POST', - headers: { - Authorization: `Bearer ${cfg.token}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - kind: 'pdf_layout', - opKey, - payload: { - documentId: input.documentId, - namespace: input.namespace, - documentObjectKey: input.documentObjectKey, - }, - }), - cache: 'no-store', - signal: controller.signal, - }); - - if (!res.ok) { - const detail = await res.text().catch(() => ''); - throw new Error(`Worker op create failed: ${res.status}${detail ? ` ${detail}` : ''}`); - } - - const parsed = await res.json() as WorkerOperationState; - if (!parsed || typeof parsed !== 'object' || typeof parsed.opId !== 'string' || !parsed.opId.trim()) { - throw new Error('Worker op create returned invalid response'); - } - return parsed; - } finally { - clearTimeout(timeout); - } -} diff --git a/src/lib/server/compute/worker-op-state.ts b/src/lib/server/compute/worker-op-state.ts deleted file mode 100644 index a6debf0..0000000 --- a/src/lib/server/compute/worker-op-state.ts +++ /dev/null @@ -1,179 +0,0 @@ -import { getWorkerClientConfigFromEnv } from '@/lib/server/compute/worker'; -import type { WorkerOperationState } from '@openreader/compute-core/api-contracts'; -import { serverLogger } from '@/lib/server/logger'; -import { logDegraded } from '@/lib/server/errors/logging'; - -const WORKER_OP_REQUEST_TIMEOUT_MS = 2_500; - -export async function fetchWorkerOperationState( - opId: string | null | undefined, -): Promise | null> { - const normalized = opId?.trim(); - if (!normalized) return null; - - let cfg: { baseUrl: string; token: string }; - try { - cfg = getWorkerClientConfigFromEnv(); - } catch (error) { - logDegraded(serverLogger, { - event: 'compute.worker_op_state.config.invalid', - msg: 'Worker client env missing/invalid', - step: 'read_worker_config', - context: { opId: normalized }, - error, - }); - return null; - } - - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), WORKER_OP_REQUEST_TIMEOUT_MS); - - try { - const res = await fetch(`${cfg.baseUrl}/ops/${encodeURIComponent(normalized)}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${cfg.token}`, - Accept: 'application/json', - }, - cache: 'no-store', - signal: controller.signal, - }); - - if (!res.ok) { - const upstreamResponseBody = await res.text().catch(() => ''); - logDegraded(serverLogger, { - event: 'compute.worker_op_state.fetch.failed', - msg: 'Worker op request failed', - step: 'fetch_worker_op', - context: { - opId: normalized, - status: res.status, - upstreamResponseBody, - }, - error: { - name: 'WorkerOpStateFetchFailed', - message: `Worker op request failed with status ${res.status}`, - }, - }); - return null; - } - const parsed = await res.json() as WorkerOperationState; - if (!parsed || typeof parsed !== 'object' || parsed.opId !== normalized) { - logDegraded(serverLogger, { - event: 'compute.worker_op_state.response.invalid', - msg: 'Worker op response invalid', - step: 'validate_worker_op_response', - context: { opId: normalized }, - }); - return null; - } - return parsed; - } catch (error) { - logDegraded(serverLogger, { - event: 'compute.worker_op_state.fetch.error', - msg: 'Worker op request threw', - step: 'fetch_worker_op', - context: { opId: normalized }, - error, - }); - return null; - } finally { - clearTimeout(timeout); - } -} - -export async function fetchWorkerOperationStateByKey( - opKey: string | null | undefined, -): Promise | null> { - const normalized = opKey?.trim(); - if (!normalized) return null; - - let cfg: { baseUrl: string; token: string }; - try { - cfg = getWorkerClientConfigFromEnv(); - } catch (error) { - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.config.invalid', - msg: 'Worker client env missing/invalid', - step: 'read_worker_config', - context: { opKey: normalized }, - error, - }); - return null; - } - - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), WORKER_OP_REQUEST_TIMEOUT_MS); - - try { - const res = await fetch(`${cfg.baseUrl}/ops/lookup`, { - method: 'POST', - headers: { - Authorization: `Bearer ${cfg.token}`, - Accept: 'application/json', - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ opKey: normalized }), - cache: 'no-store', - signal: controller.signal, - }); - - if (res.status === 404) { - return null; - } - if (!res.ok) { - const upstreamResponseBody = await res.text().catch(() => ''); - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.fetch.failed', - msg: 'Worker op lookup failed', - step: 'fetch_worker_op_by_key', - context: { - opKey: normalized, - status: res.status, - upstreamResponseBody, - }, - error: { - name: 'WorkerOpLookupFailed', - message: `Worker op lookup failed with status ${res.status}`, - }, - }); - return null; - } - - const parsed = await res.json() as WorkerOperationState; - if (!parsed || typeof parsed !== 'object' || typeof parsed.opId !== 'string' || !parsed.opId.trim()) { - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.response.invalid', - msg: 'Worker op lookup response invalid', - step: 'validate_worker_op_lookup_response', - context: { opKey: normalized }, - }); - return null; - } - if (typeof parsed.opKey !== 'string' || parsed.opKey.trim() !== normalized) { - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.response.mismatch', - msg: 'Worker op lookup response did not match requested op key', - step: 'validate_worker_op_lookup_response', - context: { - opKey: normalized, - responseOpId: parsed.opId, - responseOpKey: typeof parsed.opKey === 'string' ? parsed.opKey : null, - }, - }); - return null; - } - return parsed; - } catch (error) { - logDegraded(serverLogger, { - event: 'compute.worker_op_lookup.fetch.error', - msg: 'Worker op lookup threw', - step: 'fetch_worker_op_by_key', - context: { opKey: normalized }, - error, - }); - return null; - } finally { - clearTimeout(timeout); - } -} diff --git a/src/lib/server/compute/worker.ts b/src/lib/server/compute/worker.ts deleted file mode 100644 index d97e34e..0000000 --- a/src/lib/server/compute/worker.ts +++ /dev/null @@ -1,706 +0,0 @@ -import { createHash, randomUUID } from 'node:crypto'; -import type { ComputeBackend, PdfLayoutInput, WhisperAlignInput, WhisperAlignResult } from '@/lib/server/compute/types'; -import { parseSseEventId, parseSsePayload } from '@openreader/compute-core'; -import { getWorkerClientWaitTimeoutMs } from '@openreader/compute-core'; -import { PDF_PARSER_VERSION } from '@openreader/compute-core/api-contracts'; -import { errorToLog, serverLogger } from '@/lib/server/logger'; -import { logDegraded, logServerError } from '@/lib/server/errors/logging'; -import type { - PdfLayoutJobRequest, - PdfLayoutJobResult, - WorkerOperationEvent, - WhisperAlignJobRequest, - WhisperAlignJobResult, - WorkerOperationState, -} from '@openreader/compute-core/api-contracts'; - -class WorkerHttpError extends Error { - status: number; - retryAfterMs: number | null; - - constructor(message: string, status: number, retryAfterMs: number | null = null) { - super(message); - this.name = 'WorkerHttpError'; - this.status = status; - this.retryAfterMs = retryAfterMs; - } -} - -const DEFAULT_RETRIES = 2; -const MAX_LOG_DETAIL_CHARS = 600; -const LOG_EVENTS = new Set([ - 'align.request.failed', - 'align.request.attempt_error', - 'pdf_layout.request.failed', - 'pdf_layout.request.attempt_error', - 'http.request.failed', - 'sse.wait.http_failed', - 'sse.wait.ended_without_terminal', - 'sse.wait.failed', -]); - -type WorkerLogLevel = 'info' | 'warn' | 'error'; - -function truncateForLog(value: string, maxChars = MAX_LOG_DETAIL_CHARS): string { - if (value.length <= maxChars) return value; - return `${value.slice(0, maxChars)}...`; -} - -function logWorker(level: WorkerLogLevel, event: string, fields: Record): void { - if (!LOG_EVENTS.has(event)) return; - const eventName = `compute.worker_client.${event}`; - const msg = `Worker client ${event}`; - if (level === 'error') { - logServerError(serverLogger, { - event: eventName, - msg, - error: fields.error ?? new Error(msg), - context: { - operation: 'compute_worker_client', - ...fields, - }, - normalize: { code: 'COMPUTE_WORKER_CLIENT_EVENT_FAILED', errorClass: 'upstream' }, - }); - return; - } - if (level === 'warn') { - logDegraded(serverLogger, { - event: eventName, - msg, - step: 'worker_client', - context: { - operation: 'compute_worker_client', - ...fields, - }, - error: fields.error, - }); - return; - } - serverLogger.info({ - event: eventName, - operation: 'compute_worker_client', - ...fields, - }, `Worker client ${event}`); -} - -function opSummary(value: unknown): Record { - if (!value || typeof value !== 'object') return {}; - const record = value as Record; - const summary: Record = {}; - if (typeof record.opId === 'string') summary.opId = record.opId; - if (typeof record.status === 'string') summary.status = record.status; - if (typeof record.jobId === 'string') summary.jobId = record.jobId; - if (typeof record.updatedAt === 'string') summary.updatedAt = record.updatedAt; - return summary; -} - -function readRequiredEnv(name: string): string { - const value = process.env[name]?.trim(); - if (!value) throw new Error(`${name} is required for compute worker client`); - return value; -} - -function normalizeWorkerBaseUrl(raw: string): string { - const trimmed = raw.trim(); - if (!trimmed) throw new Error('COMPUTE_WORKER_URL is empty'); - - const withScheme = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(trimmed) - ? trimmed - : (/^(localhost|127(?:\.\d{1,3}){3})(:\d+)?(\/|$)/.test(trimmed) - ? `http://${trimmed}` - : `https://${trimmed}`); - - let parsed: URL; - try { - parsed = new URL(withScheme); - } catch { - throw new Error( - `Invalid COMPUTE_WORKER_URL="${raw}". Expected full URL like https://example.com (or http://localhost:4000).`, - ); - } - - parsed.pathname = parsed.pathname.replace(/\/+$/, ''); - return parsed.toString().replace(/\/+$/, ''); -} - -export function getWorkerClientConfigFromEnv(): { baseUrl: string; token: string } { - return { - baseUrl: normalizeWorkerBaseUrl(readRequiredEnv('COMPUTE_WORKER_URL')), - token: readRequiredEnv('COMPUTE_WORKER_TOKEN'), - }; -} - -export function isWorkerClientConfigAvailable(): boolean { - try { - getWorkerClientConfigFromEnv(); - return true; - } catch { - return false; - } -} - -function parseRetryAfterMs(value: string | null): number | null { - if (!value) return null; - const asNum = Number(value); - if (Number.isFinite(asNum)) { - return Math.max(0, Math.floor(asNum * 1000)); - } - const when = Date.parse(value); - if (Number.isNaN(when)) return null; - return Math.max(0, when - Date.now()); -} - -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function shouldRetry(error: unknown): boolean { - if (error instanceof WorkerHttpError) { - return error.status === 429 || error.status === 502 || error.status === 503 || error.status === 504; - } - if (error instanceof Error) { - const msg = error.message.toLowerCase(); - return msg.includes('network') || msg.includes('timeout') || msg.includes('fetch failed'); - } - return false; -} - -function isTerminalStatus(status: string): boolean { - return status === 'succeeded' || status === 'failed'; -} - -function sha256Hex(input: string): string { - return createHash('sha256').update(input).digest('hex'); -} - -function buildWhisperOpKey(input: WhisperAlignInput): string { - const cacheKey = input.cacheKey?.trim(); - if (cacheKey) { - return `whisper_align|v1|cache|${cacheKey}|${input.audioObjectKey}`; - } - return [ - 'whisper_align', - 'v1', - input.audioObjectKey, - input.lang ?? '', - sha256Hex(input.text), - ].join('|'); -} - -export function buildPdfOpKey(input: PdfLayoutInput): string { - return [ - 'pdf_layout', - 'v1', - PDF_PARSER_VERSION, - input.documentId, - input.namespace ?? '', - input.documentObjectKey ?? '', - input.forceToken?.trim() || '', - ].join('|'); -} - -type RetryMeta = { - attempt: number, - maxAttempts: number, - willRetry: boolean, - delayMs: number | null, - error: unknown, -}; - -async function withRetries( - attempts: number, - operation: (attempt: number) => Promise, - onAttemptError?: (meta: RetryMeta) => void, -): Promise { - let lastError: unknown = null; - for (let attemptIndex = 0; attemptIndex < attempts; attemptIndex += 1) { - const attempt = attemptIndex + 1; - try { - return await operation(attempt); - } catch (error) { - lastError = error; - const willRetry = attemptIndex < attempts - 1 && shouldRetry(error); - let delayMs: number | null = null; - if (willRetry) { - if (error instanceof WorkerHttpError && typeof error.retryAfterMs === 'number') { - delayMs = error.retryAfterMs; - } else { - delayMs = attempt * 250; - } - } - onAttemptError?.({ - attempt, - maxAttempts: attempts, - willRetry, - delayMs, - error, - }); - if (!willRetry) break; - await sleep(delayMs ?? 0); - } - } - throw lastError instanceof Error ? lastError : new Error('Unknown worker compute failure'); -} - -export class WorkerComputeBackend implements ComputeBackend { - readonly mode = 'worker' as const; - private readonly baseUrl: string; - private readonly token: string; - private readonly waitTimeoutMsByKind: Record<'whisper_align' | 'pdf_layout', number>; - private readonly retries: number; - - constructor() { - const cfg = getWorkerClientConfigFromEnv(); - this.baseUrl = cfg.baseUrl; - this.token = cfg.token; - this.waitTimeoutMsByKind = { - whisper_align: getWorkerClientWaitTimeoutMs('whisper_align'), - pdf_layout: getWorkerClientWaitTimeoutMs('pdf_layout'), - }; - this.retries = DEFAULT_RETRIES; - } - - async alignWords(input: WhisperAlignInput): Promise { - if (!input.audioObjectKey) { - throw new Error('Worker compute alignment requires audioObjectKey'); - } - const payload: WhisperAlignJobRequest = { - text: input.text, - lang: input.lang, - cacheKey: input.cacheKey, - audioObjectKey: input.audioObjectKey, - }; - const traceId = randomUUID(); - const opKey = buildWhisperOpKey(input); - const opKeyHash = sha256Hex(opKey).slice(0, 16); - const startedAt = Date.now(); - logWorker('info', 'align.request.start', { - traceId, - kind: 'whisper_align', - opKeyHash, - audioObjectKey: input.audioObjectKey, - cacheKey: input.cacheKey ?? null, - lang: input.lang ?? null, - textLength: input.text.length, - waitTimeoutMs: this.waitTimeoutMsByKind.whisper_align, - maxRetries: this.retries, - }); - - try { - const result = await withRetries(this.retries, async (attempt) => { - const op = await this.requestJson>('POST', '/ops', { - kind: 'whisper_align', - opKey, - payload, - }, { - traceId, - kind: 'whisper_align', - opKeyHash, - attempt, - }); - - const final = isTerminalStatus(op.status) - ? op - : await this.waitForOperation(op.opId, { - traceId, - kind: 'whisper_align', - opKeyHash, - attempt, - waitTimeoutMs: this.waitTimeoutMsByKind.whisper_align, - }); - - if (final.status !== 'succeeded' || !final.result) { - throw new Error(final.error?.message || 'Whisper worker operation did not complete'); - } - return { alignments: final.result.alignments }; - }, ({ attempt, maxAttempts, willRetry, delayMs, error }) => { - logWorker(willRetry ? 'warn' : 'error', 'align.request.attempt_error', { - traceId, - kind: 'whisper_align', - opKeyHash, - attempt, - maxAttempts, - willRetry, - delayMs, - error: errorToLog(error), - }); - }); - - logWorker('info', 'align.request.succeeded', { - traceId, - kind: 'whisper_align', - opKeyHash, - durationMs: Date.now() - startedAt, - }); - return result; - } catch (error) { - logWorker('error', 'align.request.failed', { - traceId, - kind: 'whisper_align', - opKeyHash, - durationMs: Date.now() - startedAt, - error: errorToLog(error), - }); - throw error; - } - } - - async parsePdfLayout(input: PdfLayoutInput) { - if (!input.documentObjectKey) { - throw new Error('Worker compute PDF layout requires documentObjectKey'); - } - const payload: PdfLayoutJobRequest = { - documentId: input.documentId, - namespace: input.namespace ?? null, - documentObjectKey: input.documentObjectKey, - }; - const traceId = randomUUID(); - const opKey = buildPdfOpKey(input); - const opKeyHash = sha256Hex(opKey).slice(0, 16); - const startedAt = Date.now(); - logWorker('info', 'pdf_layout.request.start', { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - namespace: input.namespace ?? null, - documentObjectKey: input.documentObjectKey, - waitTimeoutMs: this.waitTimeoutMsByKind.pdf_layout, - maxRetries: this.retries, - }); - - try { - const result = await withRetries(this.retries, async (attempt) => { - const op = await this.requestJson>('POST', '/ops', { - kind: 'pdf_layout', - opKey, - payload, - }, { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - attempt, - }); - await input.onWorkerSnapshot?.(op); - - const final = isTerminalStatus(op.status) - ? op - : await this.waitForOperation(op.opId, { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - attempt, - waitTimeoutMs: this.waitTimeoutMsByKind.pdf_layout, - onSnapshot: (snapshot) => { - void input.onWorkerSnapshot?.(snapshot); - if (snapshot.progress) { - void input.onProgress?.(snapshot.progress); - } - }, - }); - - if (final.status !== 'succeeded' || !final.result) { - throw new Error(final.error?.message || 'PDF layout worker operation did not complete'); - } - if (final.result.parsedObjectKey) { - return { parsedObjectKey: final.result.parsedObjectKey }; - } - if (final.result.parsed) { - return { parsed: final.result.parsed }; - } - throw new Error('PDF layout worker operation completed without parsed output'); - }, ({ attempt, maxAttempts, willRetry, delayMs, error }) => { - logWorker(willRetry ? 'warn' : 'error', 'pdf_layout.request.attempt_error', { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - attempt, - maxAttempts, - willRetry, - delayMs, - error: errorToLog(error), - }); - }); - - logWorker('info', 'pdf_layout.request.succeeded', { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - durationMs: Date.now() - startedAt, - }); - return result; - } catch (error) { - logWorker('error', 'pdf_layout.request.failed', { - traceId, - kind: 'pdf_layout', - opKeyHash, - documentId: input.documentId, - durationMs: Date.now() - startedAt, - error: errorToLog(error), - }); - throw error; - } - } - - private async requestJson( - method: 'GET' | 'POST', - path: string, - body?: unknown, - context: Record = {}, - ): Promise { - const startedAt = Date.now(); - const traceId = typeof context.traceId === 'string' ? context.traceId : randomUUID(); - logWorker('info', 'http.request.start', { - ...context, - traceId, - method, - path, - }); - const res = await fetch(`${this.baseUrl}${path}`, { - method, - headers: { - Authorization: `Bearer ${this.token}`, - 'x-openreader-trace-id': traceId, - ...(method === 'POST' ? { 'Content-Type': 'application/json' } : {}), - }, - ...(method === 'POST' ? { body: JSON.stringify(body ?? {}) } : {}), - }); - - if (!res.ok) { - const retryAfterMs = parseRetryAfterMs(res.headers.get('retry-after')); - const upstreamResponseBody = await res.text().catch(() => ''); - const requestError = new WorkerHttpError( - `Worker request failed (${method} ${path}): ${res.status}${upstreamResponseBody ? ` ${upstreamResponseBody}` : ''}`, - res.status, - retryAfterMs, - ); - logWorker(res.status >= 500 ? 'warn' : 'error', 'http.request.failed', { - ...context, - traceId, - method, - path, - status: res.status, - retryAfterMs, - durationMs: Date.now() - startedAt, - upstreamResponseBody: truncateForLog(upstreamResponseBody), - error: errorToLog(requestError), - }); - throw requestError; - } - - const parsed = await res.json() as T; - const operationSummary = opSummary(parsed); - logWorker('info', 'http.request.succeeded', { - ...context, - traceId, - method, - path, - httpStatus: res.status, - durationMs: Date.now() - startedAt, - ...operationSummary, - }); - return parsed; - } - - private async waitForOperation( - opId: string, - context: Record & { - waitTimeoutMs?: number; - onSnapshot?: (snapshot: WorkerOperationState) => void - } = {}, - ): Promise> { - const waitTimeoutMs = typeof context.waitTimeoutMs === 'number' - ? context.waitTimeoutMs - : this.waitTimeoutMsByKind.whisper_align; - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), waitTimeoutMs); - const startedAt = Date.now(); - const traceId = typeof context.traceId === 'string' ? context.traceId : randomUUID(); - logWorker('info', 'sse.wait.start', { - ...context, - traceId, - opId, - waitTimeoutMs, - }); - - try { - let latest: WorkerOperationState | null = null; - let lastEventId: number | null = null; - let eventCount = 0; - let lastStatus: string | null = null; - - while (!controller.signal.aborted) { - const query = lastEventId && lastEventId > 0 - ? `?sinceEventId=${encodeURIComponent(String(lastEventId))}` - : ''; - const res = await fetch(`${this.baseUrl}/ops/${encodeURIComponent(opId)}/events${query}`, { - method: 'GET', - headers: { - Authorization: `Bearer ${this.token}`, - Accept: 'text/event-stream', - 'x-openreader-trace-id': traceId, - ...(lastEventId && lastEventId > 0 ? { 'Last-Event-ID': String(lastEventId) } : {}), - }, - signal: controller.signal, - }); - - if (!res.ok) { - const retryAfterMs = parseRetryAfterMs(res.headers.get('retry-after')); - const upstreamResponseBody = await res.text().catch(() => ''); - const requestError = new WorkerHttpError( - `Worker request failed (GET /ops/${encodeURIComponent(opId)}/events): ${res.status}${upstreamResponseBody ? ` ${upstreamResponseBody}` : ''}`, - res.status, - retryAfterMs, - ); - logWorker(res.status >= 500 ? 'warn' : 'error', 'sse.wait.http_failed', { - ...context, - traceId, - opId, - status: res.status, - retryAfterMs, - durationMs: Date.now() - startedAt, - upstreamResponseBody: truncateForLog(upstreamResponseBody), - error: errorToLog(requestError), - }); - throw requestError; - } - - if (!res.body) { - logWorker('error', 'sse.wait.no_body', { - ...context, - traceId, - opId, - durationMs: Date.now() - startedAt, - }); - throw new Error('Worker operation stream response has no body'); - } - - logWorker('info', 'sse.wait.connected', { - ...context, - traceId, - opId, - status: res.status, - }); - const reader = res.body.getReader(); - const decoder = new TextDecoder(); - let buffer = ''; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - buffer += decoder.decode(value, { stream: true }); - - while (true) { - const frameEnd = buffer.indexOf('\n\n'); - if (frameEnd < 0) break; - const frame = buffer.slice(0, frameEnd); - buffer = buffer.slice(frameEnd + 2); - - const eventId = parseSseEventId(frame); - if (eventId && eventId > 0) { - lastEventId = eventId; - } - const payload = parseSsePayload(frame); - if (!payload) continue; - - let event: WorkerOperationEvent | WorkerOperationState; - try { - event = JSON.parse(payload) as WorkerOperationEvent | WorkerOperationState; - } catch { - logWorker('warn', 'sse.wait.json_parse_skipped', { - ...context, - traceId, - opId, - sample: truncateForLog(payload), - }); - continue; - } - - const snapshot: WorkerOperationState = ( - event && typeof event === 'object' && 'snapshot' in event - ? (event as WorkerOperationEvent).snapshot - : (event as WorkerOperationState) - ); - if (!snapshot || typeof snapshot !== 'object') continue; - if (snapshot.opId !== opId) continue; - - eventCount += 1; - latest = snapshot; - context.onSnapshot?.(snapshot); - if (snapshot.status !== lastStatus) { - lastStatus = snapshot.status; - logWorker('info', 'sse.wait.status', { - ...context, - traceId, - opId, - eventCount, - status: snapshot.status, - jobId: snapshot.jobId ?? null, - updatedAt: snapshot.updatedAt ?? null, - }); - } - if (isTerminalStatus(snapshot.status)) { - logWorker('info', 'sse.wait.terminal', { - ...context, - traceId, - opId, - eventCount, - status: snapshot.status, - durationMs: Date.now() - startedAt, - }); - return snapshot; - } - } - } - - if (latest && isTerminalStatus(latest.status)) { - logWorker('info', 'sse.wait.terminal_after_close', { - ...context, - traceId, - opId, - eventCount, - status: latest.status, - durationMs: Date.now() - startedAt, - }); - return latest; - } - - if (controller.signal.aborted) break; - await sleep(250); - } - - if (latest && isTerminalStatus(latest.status)) { - return latest; - } - - logWorker('error', 'sse.wait.ended_without_terminal', { - ...context, - traceId, - opId, - eventCount, - latestStatus: latest?.status ?? null, - durationMs: Date.now() - startedAt, - error: { - name: 'WorkerSseWaitEndedWithoutTerminal', - message: `Operation stream ended before terminal state for op ${opId}`, - }, - }); - throw new Error(`Operation stream ended before terminal state for op ${opId}`); - } catch (error) { - logWorker('error', 'sse.wait.failed', { - ...context, - traceId, - opId, - durationMs: Date.now() - startedAt, - error: errorToLog(error), - }); - throw error; - } finally { - clearTimeout(timeout); - } - } -} diff --git a/src/lib/server/documents/blobstore.ts b/src/lib/server/documents/blobstore.ts index 230bbed..42799da 100644 --- a/src/lib/server/documents/blobstore.ts +++ b/src/lib/server/documents/blobstore.ts @@ -8,7 +8,6 @@ import { PutObjectCommand, } from '@aws-sdk/client-s3'; import { getSignedUrl } from '@aws-sdk/s3-request-presigner'; -import { PDF_PARSER_VERSION, encodeParserVersion } from '@openreader/compute-core/api-contracts'; import { getS3Client, getS3Config, getS3ProxyClient } from '@/lib/server/storage/s3'; import { serverLogger } from '@/lib/server/logger'; import { logDegraded } from '@/lib/server/errors/logging'; @@ -104,24 +103,6 @@ export function documentKey(id: string, namespace: string | null): string { return `${cfg.prefix}/documents_v1/${nsSegment}${id}`; } -export function documentParsedKey(id: string, namespace: string | null): string { - return documentParsedKeyForVersion(id, namespace, PDF_PARSER_VERSION); -} - -export function documentParsedKeyForVersion( - id: string, - namespace: string | null, - parserVersion: string, -): string { - if (!isValidDocumentId(id)) { - throw new Error(`Invalid document id: ${id}`); - } - const cfg = getS3Config(); - const ns = sanitizeNamespace(namespace); - const nsSegment = ns ? `ns/${ns}/` : ''; - return `${cfg.prefix}/documents_v1/parsed_v2/${nsSegment}${id}/${encodeParserVersion(parserVersion)}.json`; -} - export function tempDocumentUploadPrefix(userId: string, namespace: string | null): string { const cfg = getS3Config(); const ns = sanitizeNamespace(namespace); @@ -342,19 +323,6 @@ export async function getTempDocumentFinalizeReceipt( } } -export async function getParsedDocumentBlob(id: string, namespace: string | null): Promise { - const cfg = getS3Config(); - const client = getS3ProxyClient(); - const key = documentParsedKey(id, namespace); - const res = await client.send( - new GetObjectCommand({ - Bucket: cfg.bucket, - Key: key, - }), - ); - return bodyToBuffer(res.Body); -} - export async function getParsedDocumentBlobByKey(key: string): Promise { const cfg = getS3Config(); const client = getS3ProxyClient(); @@ -369,31 +337,6 @@ export async function getParsedDocumentBlobByKey(key: string): Promise { return bodyToBuffer(res.Body); } -export async function putParsedDocumentBlob(id: string, body: Buffer, namespace: string | null): Promise { - return putParsedDocumentBlobForVersion(id, body, namespace, PDF_PARSER_VERSION); -} - -export async function putParsedDocumentBlobForVersion( - id: string, - body: Buffer, - namespace: string | null, - parserVersion: string, -): Promise { - const cfg = getS3Config(); - const client = getS3ProxyClient(); - const key = documentParsedKeyForVersion(id, namespace, parserVersion); - await client.send( - new PutObjectCommand({ - Bucket: cfg.bucket, - Key: key, - Body: body, - ContentType: 'application/json', - ServerSideEncryption: 'AES256', - }), - ); - return key; -} - export async function putTempDocumentFinalizeReceipt( token: string, userId: string, @@ -502,7 +445,6 @@ export async function deleteDocumentBlob(id: string, namespace: string | null): const cfg = getS3Config(); const client = getS3ProxyClient(); const key = documentKey(id, namespace); - const parsedKey = documentParsedKey(id, namespace); const legacyParsedKey = legacyDocumentParsedKey(id, namespace); const ns = sanitizeNamespace(namespace); const nsSegment = ns ? `ns/${ns}/` : ''; @@ -510,7 +452,6 @@ export async function deleteDocumentBlob(id: string, namespace: string | null): await deleteDocumentPrefix(parsedPrefix); await deleteDocumentPrefix(`${key}/`); - await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: parsedKey })); await client.send(new DeleteObjectCommand({ Bucket: cfg.bucket, Key: legacyParsedKey })); // Delete the source after the initial derived-artifact cleanup, then sweep // parsed output once more to catch a worker that finished during deletion. diff --git a/src/lib/server/documents/pdf-preview-pdfjs-runtime.ts b/src/lib/server/documents/pdf-preview-pdfjs-runtime.ts new file mode 100644 index 0000000..4f041c5 --- /dev/null +++ b/src/lib/server/documents/pdf-preview-pdfjs-runtime.ts @@ -0,0 +1,60 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; + +let cachedStandardFontDataUrl: string | null = null; +const pdfjsPackageName = 'pdfjs-dist'; + +function candidatePackageRoots(): string[] { + const roots = new Set(); + let dir = process.cwd(); + + for (let i = 0; i < 8; i += 1) { + roots.add(path.join(dir, 'node_modules', pdfjsPackageName)); + const next = path.dirname(dir); + if (next === dir) break; + dir = next; + } + + return [...roots]; +} + +function resolvePdfjsPackageFile(relativePath: string): string { + const normalizedRelativePath = relativePath.replace(/^\/+/, ''); + const candidates = candidatePackageRoots().map((root) => path.join(root, normalizedRelativePath)); + const found = candidates.find((candidate) => fs.existsSync(candidate)); + if (found) return found; + + throw new Error( + `pdfjs-dist file not found: ${normalizedRelativePath}; checked ${candidates.join(', ')}`, + ); +} + +export function resolvePdfjsStandardFontDataUrl(): string { + if (cachedStandardFontDataUrl) return cachedStandardFontDataUrl; + + const pdfjsPackageDir = path.dirname(resolvePdfjsPackageFile('package.json')); + const standardFontDir = path.join(pdfjsPackageDir, 'standard_fonts'); + + if (!fs.existsSync(standardFontDir)) { + throw new Error(`pdfjs-dist standard_fonts directory not found at ${standardFontDir}`); + } + + cachedStandardFontDataUrl = `${standardFontDir.replace(/\/?$/, '/')}`; + return cachedStandardFontDataUrl; +} + +export function resolvePdfjsWorkerSrc(): string { + return pathToFileURL(resolvePdfjsPackageFile('legacy/build/pdf.worker.mjs')).href; +} + +export function configurePdfjsNodeRuntime(pdfjs: { + GlobalWorkerOptions?: { + workerSrc?: string; + workerPort?: unknown; + }; +}): void { + if (!pdfjs.GlobalWorkerOptions) return; + pdfjs.GlobalWorkerOptions.workerSrc = resolvePdfjsWorkerSrc(); + pdfjs.GlobalWorkerOptions.workerPort = null; +} diff --git a/src/lib/server/documents/pdf-preview-renderer.ts b/src/lib/server/documents/pdf-preview-renderer.ts new file mode 100644 index 0000000..4ef78ac --- /dev/null +++ b/src/lib/server/documents/pdf-preview-renderer.ts @@ -0,0 +1,157 @@ +import type { Canvas } from '@napi-rs/canvas'; +import { configurePdfjsNodeRuntime, resolvePdfjsStandardFontDataUrl } from './pdf-preview-pdfjs-runtime'; + +type CanvasRuntime = { + DOMMatrixCtor: unknown; + Path2DCtor: unknown; + createCanvasFn: (width: number, height: number) => Canvas; +}; + +let canvasRuntimePromise: Promise | null = null; + +async function loadCanvasRuntime(): Promise { + if (!canvasRuntimePromise) { + canvasRuntimePromise = (async () => { + const mod = await import('@napi-rs/canvas'); + const namespace = mod as Record; + const fallback = (namespace.default ?? {}) as Record; + + const createCanvasFn = (namespace.createCanvas ?? fallback.createCanvas) as + | ((width: number, height: number) => Canvas) + | undefined; + const DOMMatrixCtor = namespace.DOMMatrix ?? fallback.DOMMatrix; + const Path2DCtor = namespace.Path2D ?? fallback.Path2D; + + if (typeof createCanvasFn !== 'function') { + throw new Error( + `Canvas runtime missing createCanvas export (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`, + ); + } + if (!DOMMatrixCtor || !Path2DCtor) { + throw new Error( + `Canvas runtime missing DOMMatrix/Path2D exports (keys=${Object.keys(namespace).join(',')}; defaultKeys=${Object.keys(fallback).join(',')})`, + ); + } + + return { + DOMMatrixCtor, + Path2DCtor, + createCanvasFn, + }; + })(); + } + return canvasRuntimePromise; +} + +function ensureNodeCanvasGlobals(runtime: CanvasRuntime): void { + const g = globalThis as Record; + if (typeof g.DOMMatrix === 'undefined') g.DOMMatrix = runtime.DOMMatrixCtor; + if (typeof g.Path2D === 'undefined') g.Path2D = runtime.Path2DCtor; +} + +interface RenderInput { + pdfBytes: ArrayBuffer; + pageNumber: number; + scale?: number; + targetWidth?: number; + format?: 'png' | 'jpeg'; + jpegQuality?: number; +} + +function createPdfjsCanvasFactory(runtime: CanvasRuntime) { + return class OpenReaderCanvasFactory { + create(width: number, height: number) { + const canvas = runtime.createCanvasFn(Math.max(1, Math.floor(width)), Math.max(1, Math.floor(height))); + return { + canvas, + context: canvas.getContext('2d') as unknown as CanvasRenderingContext2D, + }; + } + + reset(target: { canvas: Canvas; context: CanvasRenderingContext2D }, width: number, height: number): void { + target.canvas.width = Math.max(1, Math.floor(width)); + target.canvas.height = Math.max(1, Math.floor(height)); + } + + destroy(target: { canvas: Canvas; context: CanvasRenderingContext2D }): void { + target.canvas.width = 0; + target.canvas.height = 0; + // @ts-expect-error pdf.js expects these nulled on destroy + target.canvas = null; + // @ts-expect-error pdf.js expects these nulled on destroy + target.context = null; + } + }; +} + +export async function renderPage({ + pdfBytes, + pageNumber, + scale = 1.5, + targetWidth, + format = 'png', + jpegQuality = 82, +}: RenderInput): Promise<{ + width: number; + height: number; + image: Buffer; + contentType: 'image/png' | 'image/jpeg'; +}> { + // pdf.js may detach the provided ArrayBuffer. Work with an isolated copy so + // callers can safely reuse their original bytes across pages/calls. + const isolatedBytes = new Uint8Array(pdfBytes).slice(); + + const canvasRuntime = await loadCanvasRuntime(); + ensureNodeCanvasGlobals(canvasRuntime); + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs'); + configurePdfjsNodeRuntime(pdfjs); + + const standardFontDataUrl = resolvePdfjsStandardFontDataUrl(); + + const loadingTask = pdfjs.getDocument({ + data: isolatedBytes, + useWorkerFetch: false, + standardFontDataUrl, + isEvalSupported: false, + // Ensure pdf.js transport uses our canvas backend in Node/Next runtime. + CanvasFactory: createPdfjsCanvasFactory(canvasRuntime), + }); + + const pdf = await loadingTask.promise; + try { + const page = await pdf.getPage(pageNumber); + const baseViewport = page.getViewport({ scale: 1.0 }); + const effectiveScale = typeof targetWidth === 'number' && Number.isFinite(targetWidth) && targetWidth > 0 + ? (Math.max(1, Math.round(targetWidth)) / Math.max(1, baseViewport.width)) + : scale; + const viewport = page.getViewport({ scale: effectiveScale }); + const width = Math.max(1, Math.floor(viewport.width)); + const height = Math.max(1, Math.floor(viewport.height)); + const canvas = canvasRuntime.createCanvasFn(width, height); + const ctx = canvas.getContext('2d'); + ctx.fillStyle = '#ffffff'; + ctx.fillRect(0, 0, width, height); + + const renderTask = page.render({ + canvasContext: ctx as unknown as CanvasRenderingContext2D, + viewport, + intent: 'display', + }); + await renderTask.promise; + const contentType = format === 'jpeg' ? 'image/jpeg' : 'image/png'; + const image = format === 'jpeg' + ? canvas.toBuffer('image/jpeg', jpegQuality) + : canvas.toBuffer('image/png'); + return { + width, + height, + image, + contentType, + }; + } finally { + await pdf.destroy().catch(() => undefined); + await loadingTask.destroy().catch(() => undefined); + } +} diff --git a/src/lib/server/documents/previews-render.ts b/src/lib/server/documents/previews-render.ts index c32bbce..033e8fc 100644 --- a/src/lib/server/documents/previews-render.ts +++ b/src/lib/server/documents/previews-render.ts @@ -2,7 +2,7 @@ import path from 'path'; import { createCanvas, loadImage } from '@napi-rs/canvas'; import JSZip from 'jszip'; import { XMLParser } from 'fast-xml-parser'; -import { renderPage } from '@openreader/compute-core'; +import { renderPage } from '@/lib/server/documents/pdf-preview-renderer'; export type RenderedDocumentPreview = { bytes: Buffer; diff --git a/src/lib/server/jobs/user-whisper-align-job.ts b/src/lib/server/jobs/user-whisper-align-job.ts index afe0127..022ee92 100644 --- a/src/lib/server/jobs/user-whisper-align-job.ts +++ b/src/lib/server/jobs/user-whisper-align-job.ts @@ -1,14 +1,13 @@ -import { getCompute } from '@/lib/server/compute'; -import type { WhisperAlignJobRequest } from '@openreader/compute-core/api-contracts'; +import { getComputeWorkerClient } from '@/lib/server/compute-worker/client'; +import type { WhisperAlignRequest } from '@/lib/server/compute-worker/protocol'; import type { TTSSentenceAlignment } from '@/types/tts'; -export type UserWhisperAlignJobRequest = WhisperAlignJobRequest & { +export type UserWhisperAlignJobRequest = WhisperAlignRequest & { sentenceIndex?: number; }; export async function userWhisperAlignJob(input: UserWhisperAlignJobRequest): Promise { - const compute = await getCompute(); - const { alignments } = await compute.alignWords({ + const { alignments } = await getComputeWorkerClient().alignWords({ audioObjectKey: input.audioObjectKey, text: input.text, cacheKey: input.cacheKey, diff --git a/src/lib/server/pdf-parse/artifact.ts b/src/lib/server/pdf-parse/artifact.ts index d195480..7e49b92 100644 --- a/src/lib/server/pdf-parse/artifact.ts +++ b/src/lib/server/pdf-parse/artifact.ts @@ -1,9 +1,8 @@ -import { PDF_PARSER_VERSION } from '@openreader/compute-core/api-contracts'; import { - documentParsedKeyForVersion, getParsedDocumentBlobByKey, isMissingBlobError, } from '@/lib/server/documents/blobstore'; +import { resolveCurrentPdfParse } from '@/lib/server/pdf-parse/operation'; import type { ParsedPdfDocument } from '@/types/parsed-pdf'; export interface ParsedPdfArtifact { @@ -13,7 +12,25 @@ export interface ParsedPdfArtifact { } function parseArtifact(bytes: Buffer): ParsedPdfDocument { - return JSON.parse(bytes.toString('utf8')) as ParsedPdfDocument; + const parsed = JSON.parse(bytes.toString('utf8')) as Partial; + if ( + parsed.schemaVersion !== 1 + || typeof parsed.documentId !== 'string' + || typeof parsed.parserVersion !== 'string' + || typeof parsed.parsedAt !== 'number' + || !Array.isArray(parsed.pages) + || parsed.pages.some((page) => ( + !page + || typeof page !== 'object' + || typeof page.pageNumber !== 'number' + || typeof page.width !== 'number' + || typeof page.height !== 'number' + || !Array.isArray(page.blocks) + )) + ) { + throw new Error('Parsed PDF artifact envelope is invalid'); + } + return parsed as ParsedPdfDocument; } export async function readParsedPdfArtifactByKey(key: string): Promise { @@ -34,6 +51,11 @@ export async function readCurrentParsedPdfArtifact(input: { documentId: string; namespace: string | null; }): Promise { - const key = documentParsedKeyForVersion(input.documentId, input.namespace, PDF_PARSER_VERSION); - return readParsedPdfArtifactByKey(key); + const resolved = await resolveCurrentPdfParse(input); + if (!resolved.artifact) return null; + const artifact = await readParsedPdfArtifactByKey(resolved.artifact.objectKey); + if (artifact && artifact.parsed.documentId !== input.documentId) { + throw new Error('Parsed PDF artifact document identity mismatch'); + } + return artifact; } diff --git a/src/lib/server/pdf-parse/operation.ts b/src/lib/server/pdf-parse/operation.ts index f00aa19..60a924a 100644 --- a/src/lib/server/pdf-parse/operation.ts +++ b/src/lib/server/pdf-parse/operation.ts @@ -1,65 +1,66 @@ -import { buildPdfOpKey } from '@/lib/server/compute/worker'; -import { createOrReusePdfWorkerOperation } from '@/lib/server/compute/worker-op-create'; -import { - fetchWorkerOperationState, - fetchWorkerOperationStateByKey, -} from '@/lib/server/compute/worker-op-state'; +import { getComputeWorkerClient } from '@/lib/server/compute-worker/client'; import { documentKey } from '@/lib/server/documents/blobstore'; -import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; +import type { + PdfLayoutResolution, + PdfLayoutResult, + WorkerOperation, +} from '@/lib/server/compute-worker/protocol'; function currentPdfOperationInput(documentId: string, namespace: string | null, forceToken?: string): { documentId: string; namespace: string | null; documentObjectKey: string; - forceToken?: string; + replaceToken?: string; } { return { documentId, namespace, documentObjectKey: documentKey(documentId, namespace), - ...(forceToken ? { forceToken } : {}), + ...(forceToken ? { replaceToken: forceToken } : {}), }; } -export function buildCurrentPdfParseOpKeyPrefix(input: { +export async function resolveCurrentPdfParse(input: { documentId: string; namespace: string | null; -}): string { - return buildPdfOpKey(currentPdfOperationInput(input.documentId, input.namespace)); +}): Promise { + return getComputeWorkerClient().resolvePdfLayout(currentPdfOperationInput( + input.documentId, + input.namespace, + )); } export async function lookupCurrentPdfParseOperation(input: { documentId: string; namespace: string | null; -}): Promise | null> { - return fetchWorkerOperationStateByKey( - buildCurrentPdfParseOpKeyPrefix(input), - ); +}): Promise | null> { + return (await resolveCurrentPdfParse(input)).operation; } export async function createOrReuseCurrentPdfParseOperation(input: { documentId: string; namespace: string | null; forceToken?: string; -}): Promise> { - return createOrReusePdfWorkerOperation(currentPdfOperationInput( +}): Promise> { + return getComputeWorkerClient().createPdfLayoutOperation(currentPdfOperationInput( input.documentId, input.namespace, input.forceToken, )); } -export async function fetchPdfParseOperation(opId: string): Promise | null> { - return fetchWorkerOperationState(opId); +export async function fetchPdfParseOperation(opId: string): Promise | null> { + return getComputeWorkerClient().getOperation(opId); } export function isPdfParseOperationForDocument( - state: WorkerOperationState, + state: WorkerOperation, input: { documentId: string; namespace: string | null; }, ): boolean { - if (state.kind !== 'pdf_layout') return false; - return state.opKey.startsWith(buildCurrentPdfParseOpKeyPrefix(input)); + return state.subject.kind === 'pdf_layout' + && state.subject.documentId === input.documentId + && state.subject.namespace === input.namespace; } diff --git a/src/lib/server/pdf-parse/snapshot.ts b/src/lib/server/pdf-parse/snapshot.ts index 49c6f61..2fba7f9 100644 --- a/src/lib/server/pdf-parse/snapshot.ts +++ b/src/lib/server/pdf-parse/snapshot.ts @@ -1,8 +1,8 @@ -import type { PdfLayoutJobResult, WorkerOperationState } from '@openreader/compute-core/api-contracts'; +import type { PdfLayoutResult, WorkerOperation } from '@/lib/server/compute-worker/protocol'; import type { PdfParseStatus } from '@/types/parsed-pdf'; import type { PdfParseSnapshot } from '@/lib/server/pdf-parse/types'; -function mapWorkerStatusToParseStatus(status: WorkerOperationState['status']): PdfParseStatus { +function mapWorkerStatusToParseStatus(status: WorkerOperation['status']): PdfParseStatus { switch (status) { case 'queued': return 'pending'; @@ -19,7 +19,7 @@ function mapWorkerStatusToParseStatus(status: WorkerOperationState['status']): P } export function parsedObjectKeyFromWorkerState( - state: WorkerOperationState, + state: WorkerOperation, ): string | null { const result = state.result; if (!result || typeof result !== 'object' || !('parsedObjectKey' in result)) return null; @@ -30,7 +30,7 @@ export function parsedObjectKeyFromWorkerState( } export function pdfParseSnapshotFromWorkerState( - state: WorkerOperationState, + state: WorkerOperation, ): PdfParseSnapshot { const parseStatus = mapWorkerStatusToParseStatus(state.status); return { diff --git a/src/lib/server/runtime-config.ts b/src/lib/server/runtime-config.ts index 87889d8..61bcfb6 100644 --- a/src/lib/server/runtime-config.ts +++ b/src/lib/server/runtime-config.ts @@ -6,7 +6,7 @@ import { type RuntimeConfigKey, type RuntimeConfigSource, } from '@/lib/server/admin/settings'; -import { isComputeAvailable } from '@/lib/server/compute'; +import { isComputeWorkerAvailable } from '@/lib/server/compute-worker/client'; export type ResolvedRuntimeConfig = RuntimeConfig & { computeAvailable: boolean; @@ -29,7 +29,7 @@ export async function getResolvedRuntimeConfig(): Promise const values = await getRuntimeConfig(); return { ...values, - computeAvailable: isComputeAvailable(), + computeAvailable: isComputeWorkerAvailable(), }; } diff --git a/src/lib/shared/audio-text.ts b/src/lib/shared/audio-text.ts index 78d8797..74fdbae 100644 --- a/src/lib/shared/audio-text.ts +++ b/src/lib/shared/audio-text.ts @@ -9,9 +9,8 @@ * position-preserving variant that reuses the same patterns/glyph set so * viewer highlight offsets line up with the audio text. * - * The compute/worker package keeps its own mirror in - * `compute/core/src/whisper/alignment-map.ts` (a separate build target that - * cannot import from `@/lib`). Keep that copy in lock-step with these rules. + * Compute alignment receives this already-canonical text over the worker API, + * so its returned character offsets are measured against exactly this value. */ /** Matches a bare or explicit URL token (http(s):// or www.). Capture 1 = domain. */ diff --git a/src/types/parsed-pdf.ts b/src/types/parsed-pdf.ts index 2c7bdf7..7da5cd5 100644 --- a/src/types/parsed-pdf.ts +++ b/src/types/parsed-pdf.ts @@ -1,12 +1,12 @@ -import type { ParsedPdfBlockKind } from '@openreader/compute-core/types'; - export { type ParsedPdfBlock, type ParsedPdfBlockFragment, type ParsedPdfBlockKind, type ParsedPdfDocument, type ParsedPdfPage, -} from '@openreader/compute-core/types'; +} from '@/lib/server/compute-worker/protocol'; + +import type { ParsedPdfBlockKind } from '@/lib/server/compute-worker/protocol'; export const PARSED_PDF_BLOCK_KINDS: ParsedPdfBlockKind[] = [ 'abstract', @@ -32,8 +32,11 @@ export const PARSED_PDF_BLOCK_KINDS: ParsedPdfBlockKind[] = [ 'vision_footnote', ]; -export type { - PdfParsePhase, - PdfParseProgress, - PdfParseStatus, -} from '@openreader/compute-core/types'; +export type PdfParseStatus = 'pending' | 'running' | 'ready' | 'failed'; +export type PdfParsePhase = 'infer' | 'merge'; +export interface PdfParseProgress { + totalPages: number; + pagesParsed: number; + currentPage?: number; + phase: PdfParsePhase; +} diff --git a/src/types/tts.ts b/src/types/tts.ts index c5f42d5..cc3073c 100644 --- a/src/types/tts.ts +++ b/src/types/tts.ts @@ -1,10 +1,11 @@ import type { CanonicalTtsSegment } from '@/lib/shared/tts-segment-plan'; export type { - TTSAudioBuffer, - TTSAudioBytes, TTSSentenceAlignment, TTSSentenceWord, -} from '@openreader/compute-core/types'; +} from '@/lib/server/compute-worker/protocol'; + +export type TTSAudioBuffer = ArrayBuffer; +export type TTSAudioBytes = number[]; export type TTSLocation = string | number; diff --git a/tests/unit/compute-worker-client-contract.vitest.spec.ts b/tests/unit/compute-worker-client-contract.vitest.spec.ts new file mode 100644 index 0000000..e37f234 --- /dev/null +++ b/tests/unit/compute-worker-client-contract.vitest.spec.ts @@ -0,0 +1,58 @@ +import { afterEach, describe, expect, test } from 'vitest'; +import { createComputeWorkerApp, type ComputeWorkerApp } from '../../compute-worker/src/runtime'; +import { FakeControlPlane } from '../../compute-worker/tests/fixtures/fake-control-plane'; +import { ComputeWorkerClient } from '../../src/lib/server/compute-worker/client'; + +describe('ComputeWorkerClient contract', () => { + let runtime: ComputeWorkerApp | null = null; + + afterEach(async () => { + await runtime?.close(); + runtime = null; + }); + + test('exercises the real Fastify API through the app-owned HTTP client', async () => { + process.env.NATS_URL = 'nats://127.0.0.1:4222'; + const fake = new FakeControlPlane(); + runtime = await createComputeWorkerApp({ + host: '127.0.0.1', + port: 0, + workerToken: 'contract-token', + disableWorkers: true, + routeDeps: fake.deps, + }); + await runtime.start(); + + const address = runtime.app.server.address(); + if (!address || typeof address === 'string') { + throw new Error('Compute worker did not bind a TCP port'); + } + const client = new ComputeWorkerClient({ + baseUrl: `http://127.0.0.1:${address.port}`, + token: 'contract-token', + }); + const documentId = 'b'.repeat(64); + const request = { + documentId, + namespace: null, + documentObjectKey: `openreader/${documentId}.pdf`, + }; + + const created = await client.createPdfLayoutOperation(request); + expect(created).toMatchObject({ + subject: { kind: 'pdf_layout', documentId, namespace: null }, + status: 'queued', + }); + expect(created).not.toHaveProperty('opKey'); + expect(created).not.toHaveProperty('jobId'); + + await expect(client.getOperation(created.opId)).resolves.toMatchObject({ + opId: created.opId, + subject: { kind: 'pdf_layout', documentId, namespace: null }, + }); + await expect(client.resolvePdfLayout(request)).resolves.toMatchObject({ + artifact: null, + operation: { opId: created.opId }, + }); + }); +}); diff --git a/tests/unit/pdf-op-key.vitest.spec.ts b/tests/unit/pdf-op-key.vitest.spec.ts deleted file mode 100644 index fbe746e..0000000 --- a/tests/unit/pdf-op-key.vitest.spec.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { describe, expect, test } from 'vitest'; -import { PDF_PARSER_VERSION } from '@openreader/compute-core'; -import { buildPdfOpKey } from '../../src/lib/server/compute/worker'; - -describe('pdf worker op key', () => { - test('includes parser version in the reuse boundary', () => { - expect(buildPdfOpKey({ - documentId: 'doc-1', - namespace: null, - documentObjectKey: 'documents/doc-1.pdf', - })).toBe(`pdf_layout|v1|${PDF_PARSER_VERSION}|doc-1||documents/doc-1.pdf|`); - }); -}); diff --git a/tests/unit/pdf-parse-artifact-envelope.vitest.spec.ts b/tests/unit/pdf-parse-artifact-envelope.vitest.spec.ts new file mode 100644 index 0000000..c9276ea --- /dev/null +++ b/tests/unit/pdf-parse-artifact-envelope.vitest.spec.ts @@ -0,0 +1,72 @@ +import { beforeEach, describe, expect, test, vi } from 'vitest'; + +const hoisted = vi.hoisted(() => ({ + getParsedDocumentBlobByKey: vi.fn(), + resolveCurrentPdfParse: vi.fn(), +})); + +vi.mock('@/lib/server/documents/blobstore', () => ({ + getParsedDocumentBlobByKey: hoisted.getParsedDocumentBlobByKey, + isMissingBlobError: vi.fn(() => false), +})); + +vi.mock('@/lib/server/pdf-parse/operation', () => ({ + resolveCurrentPdfParse: hoisted.resolveCurrentPdfParse, +})); + +describe('parsed PDF artifact envelope validation', () => { + beforeEach(() => { + hoisted.resolveCurrentPdfParse.mockReset(); + hoisted.resolveCurrentPdfParse.mockResolvedValue({ + artifact: { objectKey: 'openreader/parsed.json' }, + operation: null, + }); + hoisted.getParsedDocumentBlobByKey.mockReset(); + }); + + test('accepts a valid current artifact envelope', async () => { + hoisted.getParsedDocumentBlobByKey.mockResolvedValue(Buffer.from(JSON.stringify({ + schemaVersion: 1, + documentId: 'doc-1', + parserVersion: 'parser-v1', + parsedAt: 1, + pages: [{ pageNumber: 1, width: 100, height: 200, blocks: [] }], + }))); + + const { readCurrentParsedPdfArtifact } = await import('../../src/lib/server/pdf-parse/artifact'); + await expect(readCurrentParsedPdfArtifact({ documentId: 'doc-1', namespace: null })) + .resolves.toMatchObject({ key: 'openreader/parsed.json' }); + }); + + test('rejects malformed envelopes and document identity mismatches', async () => { + const { readCurrentParsedPdfArtifact } = await import('../../src/lib/server/pdf-parse/artifact'); + + hoisted.getParsedDocumentBlobByKey.mockResolvedValueOnce(Buffer.from(JSON.stringify({ + schemaVersion: 2, + documentId: 'doc-1', + pages: [], + }))); + await expect(readCurrentParsedPdfArtifact({ documentId: 'doc-1', namespace: null })) + .rejects.toThrow('envelope is invalid'); + + hoisted.getParsedDocumentBlobByKey.mockResolvedValueOnce(Buffer.from(JSON.stringify({ + schemaVersion: 1, + documentId: 'other-doc', + parserVersion: 'parser-v1', + parsedAt: 1, + pages: [], + }))); + await expect(readCurrentParsedPdfArtifact({ documentId: 'doc-1', namespace: null })) + .rejects.toThrow('document identity mismatch'); + + hoisted.getParsedDocumentBlobByKey.mockResolvedValueOnce(Buffer.from(JSON.stringify({ + schemaVersion: 1, + documentId: 'doc-1', + parserVersion: 'parser-v1', + parsedAt: 1, + pages: [{ pageNumber: 1, width: 100, height: 200 }], + }))); + await expect(readCurrentParsedPdfArtifact({ documentId: 'doc-1', namespace: null })) + .rejects.toThrow('envelope is invalid'); + }); +}); diff --git a/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts b/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts index 98ceb4d..85b49cb 100644 --- a/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts +++ b/tests/unit/pdf-parsed-events-route-worker-proxy.vitest.spec.ts @@ -12,7 +12,7 @@ const hoisted = vi.hoisted(() => ({ requireAuthContext: vi.fn(), fetchPdfParseOperation: vi.fn(), isPdfParseOperationForDocument: vi.fn(), - getWorkerClientConfigFromEnv: vi.fn(), + openOperationEvents: vi.fn(), })); vi.mock('@/db', () => ({ @@ -30,8 +30,10 @@ vi.mock('@/lib/server/pdf-parse/operation', () => ({ isPdfParseOperationForDocument: hoisted.isPdfParseOperationForDocument, })); -vi.mock('@/lib/server/compute/worker', () => ({ - getWorkerClientConfigFromEnv: hoisted.getWorkerClientConfigFromEnv, +vi.mock('@/lib/server/compute-worker/client', () => ({ + getComputeWorkerClient: () => ({ + openOperationEvents: hoisted.openOperationEvents, + }), })); vi.mock('@/lib/server/documents/blobstore', () => ({ @@ -84,12 +86,8 @@ describe('GET /api/documents/[id]/parsed/events worker event proxy', () => { }); hoisted.isPdfParseOperationForDocument.mockReset(); hoisted.isPdfParseOperationForDocument.mockReturnValue(true); - hoisted.getWorkerClientConfigFromEnv.mockReset(); - hoisted.getWorkerClientConfigFromEnv.mockReturnValue({ - baseUrl: 'http://worker.local', - token: 'worker-token', - }); - global.fetch = vi.fn(async () => new Response( + hoisted.openOperationEvents.mockReset(); + hoisted.openOperationEvents.mockResolvedValue(new Response( 'event: snapshot\ndata: {"eventId":1,"snapshot":{"opId":"op-1","status":"running"}}\n\n', { status: 200, @@ -97,7 +95,7 @@ describe('GET /api/documents/[id]/parsed/events worker event proxy', () => { 'Content-Type': 'text/event-stream; charset=utf-8', }, }, - )) as typeof fetch; + )); }); test('requires an opId', async () => { @@ -123,10 +121,9 @@ describe('GET /api/documents/[id]/parsed/events worker event proxy', () => { expect(text).toContain('"opId":"op-1"'); expect(hoisted.fetchPdfParseOperation).toHaveBeenCalledWith('op-1'); expect(hoisted.isPdfParseOperationForDocument).toHaveBeenCalled(); - expect(global.fetch).toHaveBeenCalledWith( - 'http://worker.local/ops/op-1/events', - expect.anything(), - ); + expect(hoisted.openOperationEvents).toHaveBeenCalledWith('op-1', expect.objectContaining({ + signal: expect.any(AbortSignal), + })); }); test('denies proxying when the op does not belong to the document', async () => { @@ -140,7 +137,7 @@ describe('GET /api/documents/[id]/parsed/events worker event proxy', () => { expect(response.status).toBe(404); await expect(response.json()).resolves.toMatchObject({ error: 'Operation not found' }); expect(hoisted.fetchPdfParseOperation).toHaveBeenCalledWith('op-1'); - expect(global.fetch).not.toHaveBeenCalled(); + expect(hoisted.openOperationEvents).not.toHaveBeenCalled(); }); afterEach(() => { diff --git a/tsconfig.json b/tsconfig.json index 2c7a43b..9cd3715 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -20,11 +20,6 @@ ], "paths": { "@/*": ["./src/*"], - "@openreader/compute-core": ["./compute/core/src/index.ts"], - "@openreader/compute-core/api-contracts": ["./compute/core/src/api-contracts/index.ts"], - "@openreader/compute-core/control-plane": ["./compute/core/src/control-plane/index.ts"], - "@openreader/compute-core/local-runtime": ["./compute/core/src/local-runtime.ts"], - "@openreader/compute-core/types": ["./compute/core/src/types/index.ts"] } }, "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], @@ -32,7 +27,7 @@ "node_modules", "docs-site", "docs-site/**", - "compute/worker", - "compute/worker/**" + "compute-worker", + "compute-worker/**" ] } diff --git a/vitest.config.ts b/vitest.config.ts index 3489ba3..99417d3 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -10,17 +10,7 @@ if (!/^https?:\/\//.test(process.env.BASE_URL ?? '')) { } const srcDir = fileURLToPath(new URL('./src/', import.meta.url)); -const computeCoreIndex = fileURLToPath(new URL('./compute/core/src/index.ts', import.meta.url)); -const computeCoreApiContracts = fileURLToPath(new URL('./compute/core/src/api-contracts/index.ts', import.meta.url)); -const computeCoreControlPlane = fileURLToPath(new URL('./compute/core/src/control-plane/index.ts', import.meta.url)); -const computeCoreLocalRuntime = fileURLToPath(new URL('./compute/core/src/local-runtime.ts', import.meta.url)); -const computeCoreTypes = fileURLToPath(new URL('./compute/core/src/types/index.ts', import.meta.url)); const alias = [ - { find: /^@openreader\/compute-core$/, replacement: computeCoreIndex }, - { find: /^@openreader\/compute-core\/api-contracts$/, replacement: computeCoreApiContracts }, - { find: /^@openreader\/compute-core\/control-plane$/, replacement: computeCoreControlPlane }, - { find: /^@openreader\/compute-core\/local-runtime$/, replacement: computeCoreLocalRuntime }, - { find: /^@openreader\/compute-core\/types$/, replacement: computeCoreTypes }, { find: /^@\//, replacement: `${srcDir}` }, { find: '@', replacement: srcDir }, ]; @@ -44,19 +34,12 @@ export default defineConfig({ setupFiles: ['tests/unit/setup-env.ts'], }, }, - { - test: { - name: 'compute-core', - environment: 'node', - include: ['compute/core/tests/control-plane/**/*.test.ts'], - }, - }, { test: { name: 'compute-worker', environment: 'node', - include: ['compute/worker/tests/{unit,api}/**/*.test.ts'], - setupFiles: ['compute/worker/tests/setup-env.ts'], + include: ['compute-worker/tests/{unit,api,compute}/**/*.test.ts'], + setupFiles: ['compute-worker/tests/setup-env.ts'], }, }, ],