diff --git a/.gitignore b/.gitignore index 6d5cc68..c006683 100644 --- a/.gitignore +++ b/.gitignore @@ -34,6 +34,7 @@ yarn-error.log* # env files (can opt-in for committing if needed) .env .dockerignore +*.creds # vercel .vercel diff --git a/compute/worker/.env.example b/compute/worker/.env.example index 4a36fdc..901417a 100644 --- a/compute/worker/.env.example +++ b/compute/worker/.env.example @@ -9,6 +9,11 @@ COMPUTE_WORKER_TOKEN=local-compute-token # NATS/JetStream NATS_URL=nats://nats:4222 +# Optional: NATS authentication credentials (e.g. for Synadia Cloud / NGS) +# You can specify the file path: +# NATS_CREDS_FILE=/path/to/NGS-Default-compute-worker.creds +# Or specify the raw credentials string content (excellent for container/cloud platforms): +# NATS_CREDS="-----BEGIN NATS USER JWT-----\n...\n-----BEGIN USER NKEY SEED-----\n..." # Shared object storage (must be reachable from worker) S3_BUCKET=openreader-documents diff --git a/compute/worker/src/server.ts b/compute/worker/src/server.ts index 8b4263b..66551aa 100644 --- a/compute/worker/src/server.ts +++ b/compute/worker/src/server.ts @@ -3,6 +3,7 @@ import { z } from 'zod'; import { connect, nanos, + credsAuthenticator, type NatsConnection, } from '@nats-io/transport-node'; import { @@ -412,7 +413,21 @@ async function main(): Promise { const attempts = readIntEnv('COMPUTE_JOB_ATTEMPTS', 2); const prewarmModels = parseBoolEnv('COMPUTE_PREWARM_MODELS', true); - const nc: NatsConnection = await connect({ servers: natsUrl }); + const connectOpts: any = { servers: natsUrl }; + const natsCreds = process.env.NATS_CREDS?.trim(); + const natsCredsFile = process.env.NATS_CREDS_FILE?.trim(); + + if (natsCreds) { + console.log('[compute-worker] Connecting to NATS using credentials string from NATS_CREDS'); + connectOpts.authenticator = credsAuthenticator(new TextEncoder().encode(natsCreds)); + } else if (natsCredsFile) { + console.log(`[compute-worker] Connecting to NATS using credentials file: ${natsCredsFile}`); + const { readFileSync } = require('fs'); + const credsData = readFileSync(natsCredsFile); + connectOpts.authenticator = credsAuthenticator(credsData); + } + + const nc: NatsConnection = await connect(connectOpts); const js: JetStreamClient = jetstream(nc); const jsm: JetStreamManager = await jetstreamManager(nc); diff --git a/docs-site/docs/deploy/compute-worker.md b/docs-site/docs/deploy/compute-worker.md index f57ce9a..16c3d8d 100644 --- a/docs-site/docs/deploy/compute-worker.md +++ b/docs-site/docs/deploy/compute-worker.md @@ -28,8 +28,15 @@ Required: - `S3_ACCESS_KEY_ID` - `S3_SECRET_ACCESS_KEY` +> [!IMPORTANT] +> **S3 credentials cannot be left blank/empty** when running in worker mode. +> While the main Next.js server can generate random, dynamic S3 keys on-the-fly when `USE_EMBEDDED_WEED_MINI=true` and `S3_*` vars are blank, the compute worker runs in a separate process and cannot connect to SeaweedFS using those dynamically generated keys. +> To use the compute worker with the embedded SeaweedFS, you **must configure identical, stable S3 credentials** (e.g. `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY`) in both the root `.env` and the compute worker `.env` files. + Common optional: +- `NATS_CREDS`: raw user credentials file content (JWT + private key), ideal for cloud container environments where mounting files is difficult. +- `NATS_CREDS_FILE`: path to a `.creds` file on the server. - `S3_ENDPOINT` (for non-AWS S3-compatible storage) - `S3_FORCE_PATH_STYLE=true` (for many S3-compatible providers) - `S3_PREFIX=openreader` @@ -60,3 +67,21 @@ COMPUTE_WORKER_TOKEN= - `GET /health/live` - `GET /health/ready` + +## Authenticating with Synadia Cloud (NGS) + +If you are using a free Synadia Cloud account to back your compute queue in production: + +1. **Obtain your credentials file**: When creating a user or a service account on Synadia Cloud, download your credentials file (usually named `.creds`). +2. **Configure NATS URL**: Synadia Cloud's server address is `tls://connect.ngs.global:4222`. Set this as your `NATS_URL`. +3. **Configure Authentication**: + - **Using a local file path**: Set `NATS_CREDS_FILE` to the path of your `.creds` file: + ```env + NATS_URL=tls://connect.ngs.global:4222 + NATS_CREDS_FILE=/app/secrets/NGS-Default-compute-worker.creds + ``` + - **Using raw content (Recommended for Railway, Fly.io, etc.)**: Set `NATS_CREDS` to the exact content of your `.creds` file (including the begin/end banners for JWT and NKEY seed). Since `.creds` contains newlines, wrap the entire value in quotes or paste it directly into your cloud provider's Secrets/Environment settings: + ```env + NATS_URL=tls://connect.ngs.global:4222 + NATS_CREDS="-----BEGIN NATS USER JWT-----\neyJ0...------END USER NKEY SEED------" + ```