From 8d0dc968b3a984fcf5d40491fcb8893eb1103e3f Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 11 Sep 2026 00:41:11 +0200 Subject: [PATCH] feat: a deploy you can repeat, and prove afterwards Reproducibility means two things here: the same commit builds the same image, and the running container can be asked which commit it is. - Base images are pinned by digest, not by tag. A tag moves; two builds of one commit could otherwise differ. These are manifest-list digests, so buildx still picks the right architecture. - scripts/build-image.sh also writes ped-ai-local:, an immutable name a deploy can refer to instead of chasing :latest. Its summary goes to stderr so stdout stays the Compose invocation. - Compose takes the image from PED_AI_IMAGE, so a deploy runs a specific revision-tagged image while a local build still uses the local tag. - scripts/deploy.sh pins that image in the file Compose interpolates from, waits for health, then asks /api/build which revision is actually serving and rolls back to the previous image if it does not match. Healthy is not the same as running what you asked for. The rollback path was exercised. - The entrypoint applies migrations before the app starts, so code and schema arrive together. node-pg-migrate takes an advisory lock; losing it is not an error, it waits and looks again, so a rolling restart does not fail. A real migration failure stops the container rather than serving on a schema that does not match the build. RUN_MIGRATIONS=false opts out. - The Forgejo workflow builds through that same script, tags by full revision, and has an opt-in deploy job. It refuses to run if the deploy directory has uncommitted work rather than resetting over it. The running image was labelled revision=unknown, and /api/build said "unknown", because `docker compose up --build` never passes GIT_REVISION. That is exactly the hole this closes. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU --- .forgejo/workflows/docker-build.yml | 60 ++++++++++++++++-- Dockerfile | 7 ++- docker-compose.yml | 17 +++-- docker-entrypoint.sh | 49 +++++++++++++++ scripts/build-image.sh | 27 +++++++- scripts/deploy.sh | 98 +++++++++++++++++++++++++++++ test/build-id.test.js | 13 +++- 7 files changed, 254 insertions(+), 17 deletions(-) create mode 100755 scripts/deploy.sh diff --git a/.forgejo/workflows/docker-build.yml b/.forgejo/workflows/docker-build.yml index 566ec971..ba37dacd 100644 --- a/.forgejo/workflows/docker-build.yml +++ b/.forgejo/workflows/docker-build.yml @@ -1,12 +1,18 @@ name: Forgejo Docker Build on: + push: + branches: [main] workflow_dispatch: inputs: push_image: description: Push image to Forgejo container registry required: false default: 'true' + deploy: + description: Deploy the built image to the host after pushing + required: false + default: 'false' jobs: root-test: @@ -37,14 +43,18 @@ jobs: - name: Validate Compose config run: docker compose -f docker-compose.yml config >/tmp/ped-ai-compose.yml + # The same script a person runs locally, so a CI image and a hand-built + # one cannot drift. It validates the revision and bakes it into the image, + # which is what makes /api/build able to say what is running. - name: Build compose service run: ./scripts/build-image.sh - - name: Tag image + - name: Tag image by revision run: | IMAGE="git.danvics.com/danvics/pediatric-ai-scribe-v3" - SHORT_SHA=$(git rev-parse --short HEAD) - docker tag ped-ai-local:latest "$IMAGE:$SHORT_SHA" + REVISION=$(git rev-parse HEAD) + # The full revision is the immutable name; :latest is only a pointer. + docker tag ped-ai-local:latest "$IMAGE:$REVISION" docker tag ped-ai-local:latest "$IMAGE:latest" - name: Push image to Forgejo registry @@ -53,7 +63,47 @@ jobs: FORGEJO_TOKEN: ${{ secrets.FORGEJO_TOKEN }} run: | IMAGE="git.danvics.com/danvics/pediatric-ai-scribe-v3" - SHORT_SHA=$(git rev-parse --short HEAD) + REVISION=$(git rev-parse HEAD) echo "$FORGEJO_TOKEN" | docker login git.danvics.com -u danvics --password-stdin - docker push "$IMAGE:$SHORT_SHA" + docker push "$IMAGE:$REVISION" docker push "$IMAGE:latest" + + # ── Deploy ────────────────────────────────────────────────────────────── + # Opt-in, because the deploy directory is also a working tree: this refuses to + # run if it has uncommitted changes rather than resetting over someone's work. + # + # It moves that checkout to the built revision first, so the Compose file and + # the entrypoint that ship with the image are the ones used to run it, then + # hands over to scripts/deploy.sh, which pins the image, waits for health, + # asks /api/build what is actually running and rolls back if it disagrees. + # Schema migrations are applied by the container's own entrypoint. + deploy: + needs: build + name: Deploy to the host + runs-on: forgejo-local + if: ${{ github.event.inputs.deploy == 'true' }} + env: + DEPLOY_DIR: ${{ vars.DEPLOY_DIR || '/home/danvics/docker/ped-ai' }} + steps: + - name: Refuse to deploy over uncommitted work + run: | + if [ -n "$(git -C "$DEPLOY_DIR" status --porcelain)" ]; then + echo "$DEPLOY_DIR has uncommitted changes; commit or stash them first." >&2 + git -C "$DEPLOY_DIR" status --short >&2 + exit 1 + fi + + # Detaches HEAD at the deployed revision, which is what you want a + # deployed tree to be. If DEPLOY_DIR is also where you write code, point + # it at a checkout of its own instead — vars.DEPLOY_DIR. + - name: Move the deploy checkout to this revision + run: | + REVISION="${{ github.sha }}" + git -C "$DEPLOY_DIR" fetch --quiet --all + git -C "$DEPLOY_DIR" checkout --quiet --detach "$REVISION" + + - name: Deploy and verify + run: | + IMAGE="git.danvics.com/danvics/pediatric-ai-scribe-v3" + REVISION="${{ github.sha }}" + "$DEPLOY_DIR/scripts/deploy.sh" "$IMAGE:$REVISION" "$REVISION" diff --git a/Dockerfile b/Dockerfile index 7f04120e..9234fa58 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,12 @@ # ─── OpenBao CLI, copied from upstream image (multi-arch automatic) ─── # Update the tag here to adopt a newer OpenBao. Binary is statically linked, # safe to drop into the Node alpine image as-is. -FROM openbao/openbao:2.5.3 AS bao-src +# Pinned by digest, not by tag: a tag is a moving pointer, so two builds of the +# same commit could otherwise produce different images. These are manifest-list +# digests, so buildx still selects the right per-architecture variant. +FROM openbao/openbao:2.5.3@sha256:fdc6da21ca6963560c32336fd7feb9cf2d5e52668f1a1647205a4b41171f0806 AS bao-src -FROM node:24-alpine +FROM node:24-alpine@sha256:e67514e5d0f6c46656005e1b693b2ec9d52e80b641307de684d4a015ba7a4eaf WORKDIR /app diff --git a/docker-compose.yml b/docker-compose.yml index 7a41876c..862a26c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,11 @@ services: context: . args: GIT_REVISION: ${GIT_REVISION:-unknown} - image: ped-ai-local:latest + # A deploy sets PED_AI_IMAGE to an immutable, revision-tagged image from the + # registry; a local build leaves it unset and uses the tag build-image.sh + # writes. Either way the running container can be asked what it is: + # /api/build returns the revision baked into it at build time. + image: ${PED_AI_IMAGE:-ped-ai-local:latest} ports: - "127.0.0.1:3552:3000" env_file: @@ -61,10 +65,11 @@ services: start_period: 20s postgres: - # Tag-pinned. If a newer pg16 image ships a different ICU library, the - # startup drift check in src/db/database.js auto-REINDEXes and - # refreshes the collation version. For stricter control, pin by digest. - image: pgvector/pgvector:pg16 + # Digest-pinned, so a rebuilt environment gets this exact Postgres. If a + # newer pg16 image ships a different ICU library, the startup drift check in + # src/db/database.js auto-REINDEXes and refreshes the collation version; + # pinning means that only happens when this line is deliberately changed. + image: pgvector/pgvector:pg16@sha256:00ba258a66dac104fd5171074a0084462a64a1369d8513f3d0a634e2f24d15bc environment: POSTGRES_DB: pedscribe POSTGRES_USER: pedscribe @@ -81,7 +86,7 @@ services: start_period: 10s redis: - image: redis:8-alpine + image: redis:8-alpine@sha256:d146f83b1e0f02fc27c26a50cee39338c736674c5959db84363e6ae3cd9e02d2 command: redis-server --appendonly yes restart: unless-stopped container_name: ped-ai-redis diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index e79e58df..f081b58e 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -76,4 +76,53 @@ else echo "[entrypoint] OPENBAO_ADDR not set — using existing environment (legacy .env path)" fi +# ── Schema migrations ──────────────────────────────────────────────── +# The code and the schema it needs ship inside the same image, so they have to +# arrive together. Applying them by hand meant a deploy could put new code in +# front of an old schema and only find out at the first request. +# +# node-pg-migrate takes a Postgres advisory lock, so two containers starting at +# once cannot both apply. The one that loses the race is not an error — it +# waits for the winner and looks again — so a rolling restart does not fail. +# +# Set RUN_MIGRATIONS=false to start without touching the schema (a read-only +# replica, or recovering from a bad migration by hand). +if [ "${RUN_MIGRATIONS:-true}" = "true" ]; then + if [ -z "${DATABASE_URL:-}" ]; then + echo "[entrypoint] FATAL: RUN_MIGRATIONS is on but DATABASE_URL is not set." >&2 + exit 1 + fi + + _MIGRATE_ATTEMPT=1 + _MIGRATE_MAX=${MIGRATION_ATTEMPTS:-10} + while : ; do + echo "[entrypoint] applying migrations (attempt ${_MIGRATE_ATTEMPT}/${_MIGRATE_MAX})..." + _MIGRATE_OUT="$(node_modules/.bin/node-pg-migrate up 2>&1)" && { + printf '%s\n' "${_MIGRATE_OUT}" + echo "[entrypoint] ✅ schema is up to date" + break + } + printf '%s\n' "${_MIGRATE_OUT}" >&2 + + # Losing the advisory lock, or racing a database that is still opening its + # listening socket, are both worth another look. Anything else is a real + # migration failure and must stop the deploy rather than serve on a schema + # that does not match the code. + if printf '%s' "${_MIGRATE_OUT}" | grep -qiE "advisory lock|ECONNREFUSED|starting up|Connection terminated"; then + if [ "${_MIGRATE_ATTEMPT}" -ge "${_MIGRATE_MAX}" ]; then + echo "[entrypoint] FATAL: could not apply migrations after ${_MIGRATE_MAX} attempts." >&2 + exit 1 + fi + _MIGRATE_ATTEMPT=$((_MIGRATE_ATTEMPT + 1)) + sleep 3 + continue + fi + + echo "[entrypoint] FATAL: migration failed. Refusing to start on a schema that does not match this build." >&2 + exit 1 + done +else + echo "[entrypoint] RUN_MIGRATIONS=false — starting without checking the schema" +fi + exec "$@" diff --git a/scripts/build-image.sh b/scripts/build-image.sh index bbfd0a42..29cee869 100755 --- a/scripts/build-image.sh +++ b/scripts/build-image.sh @@ -1,5 +1,14 @@ #!/bin/sh -# Build only; this never starts services. COMPOSE_FILE selects an alternate Compose file. +# Build only; this never starts services. +# +# The revision is validated and baked into the image (BUILD_ID + the OCI +# revision label), so the result can always be traced back to a commit and +# /api/build can report it at runtime. Plain `docker compose build` does not set +# GIT_REVISION and produces an image labelled "unknown" — which is why this is +# the supported way to build. +# +# The image is tagged twice: ped-ai-local: is immutable and is what a +# deploy should name, ped-ai-local:latest is the convenience pointer. set -eu cd "$(dirname "$0")/.." if [ -e .git ]; then @@ -13,4 +22,18 @@ else echo 'Unversioned development build: revision unknown' >&2 fi export GIT_REVISION -exec docker compose build "$@" pediatric-scribe + +# A local build always writes the local tag, even when .env pins PED_AI_IMAGE to +# a deployed registry image — otherwise building here would quietly overwrite +# the tag a deploy is pinned to. +PED_AI_IMAGE=ped-ai-local:latest +export PED_AI_IMAGE + +docker compose build "$@" pediatric-scribe + +if [ "$GIT_REVISION" != unknown ]; then + docker tag ped-ai-local:latest "ped-ai-local:$GIT_REVISION" + echo "Built ped-ai-local:$GIT_REVISION (also tagged :latest)" >&2 +else + echo 'Built ped-ai-local:latest with no recorded revision' >&2 +fi diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 00000000..55088dc9 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Deploy one immutable, revision-tagged image and prove it landed. +# +# scripts/deploy.sh [expected-revision] +# scripts/deploy.sh git.danvics.com/danvics/pediatric-ai-scribe-v3:fed4bd15 fed4bd15… +# +# The point is that a deploy is repeatable and checkable: the image is named by +# digest or by revision tag, the container is asked afterwards which revision it +# is actually running, and anything that does not line up is rolled back to the +# image that was serving a moment ago. +# +# Schema migrations are applied by the container's own entrypoint before the app +# starts, so code and schema arrive together and a failed migration stops the +# container rather than serving on a schema that does not match. +set -euo pipefail + +cd "$(dirname "$0")/.." + +IMAGE="${1:-}" +EXPECTED_REVISION="${2:-}" +SERVICE=pediatric-scribe +CONTAINER=pediatric-ai-scribe +HEALTH_URL="${DEPLOY_HEALTH_URL:-http://127.0.0.1:3552}" + +if [ -z "$IMAGE" ]; then + echo "usage: scripts/deploy.sh [expected-revision]" >&2 + exit 2 +fi + +# The image has to exist before anything is torn down. +if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then + echo "==> pulling $IMAGE" + docker pull "$IMAGE" +fi + +# What is serving right now, so there is something to go back to. +PREVIOUS_IMAGE="$(docker inspect "$CONTAINER" --format '{{.Config.Image}}' 2>/dev/null || true)" +if [ -n "$PREVIOUS_IMAGE" ]; then + echo "==> currently running: $PREVIOUS_IMAGE" +fi + +# Pin the image in the file Compose interpolates from, so a later plain +# `docker compose up -d` brings up this same image rather than silently +# reverting to the local build tag. Exactly one line is touched. +pin_image() { + local ref="$1" + touch .env + if grep -q '^PED_AI_IMAGE=' .env; then + # A literal replacement: image refs contain / : @ and must not be + # re-interpreted by sed's replacement syntax. + grep -v '^PED_AI_IMAGE=' .env > .env.deploy-tmp + printf 'PED_AI_IMAGE=%s\n' "$ref" >> .env.deploy-tmp + mv .env.deploy-tmp .env + else + printf 'PED_AI_IMAGE=%s\n' "$ref" >> .env + fi +} + +rollback() { + if [ -z "$PREVIOUS_IMAGE" ]; then + echo "==> nothing to roll back to; leaving the stack as it is" >&2 + return + fi + echo "==> rolling back to $PREVIOUS_IMAGE" >&2 + pin_image "$PREVIOUS_IMAGE" + PED_AI_IMAGE="$PREVIOUS_IMAGE" docker compose up -d --wait "$SERVICE" >&2 || true +} + +echo "==> deploying $IMAGE" +pin_image "$IMAGE" +if ! PED_AI_IMAGE="$IMAGE" docker compose up -d --wait "$SERVICE"; then + echo "==> the container did not become healthy" >&2 + rollback + exit 1 +fi + +# Healthy is not the same as "running what was asked for". The revision is baked +# into the image at build time and reported by /api/build, so this catches a +# stale tag, a cached layer or a rollback that never took. +RUNNING_REVISION="$(curl -fsS --max-time 10 "$HEALTH_URL/api/build" | sed -n 's/.*"buildId":"\([^"]*\)".*/\1/p' || true)" +echo "==> running revision: ${RUNNING_REVISION:-}" + +if [ -n "$EXPECTED_REVISION" ]; then + case "$RUNNING_REVISION" in + "$EXPECTED_REVISION"*) + echo "==> ✅ $IMAGE is serving $RUNNING_REVISION" + ;; + *) + echo "==> FATAL: expected revision $EXPECTED_REVISION but the app reports '${RUNNING_REVISION:-}'" >&2 + rollback + exit 1 + ;; + esac +elif [ -z "$RUNNING_REVISION" ] || [ "$RUNNING_REVISION" = "unknown" ]; then + # Not fatal on its own — a local build has no revision — but it means this + # deploy cannot be traced back to a commit. + echo "==> WARNING: this image records no source revision, so what is running cannot be traced to a commit." >&2 +fi diff --git a/test/build-id.test.js b/test/build-id.test.js index af8d5a6a..d2fa8e28 100644 --- a/test/build-id.test.js +++ b/test/build-id.test.js @@ -67,6 +67,7 @@ test('manual build script passes the full revision to Compose without starting s encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], env: { ...process.env, PATH: bin + path.delimiter + process.env.PATH, GIT_REVISION: 'bad-override', ...env }, }).trim().split('\n'); + // No revision means no immutable tag to write, so Compose is the only call. assert.deepEqual(build(), ['unknown', 'compose', 'build', '--no-cache', 'pediatric-scribe']); const git = (...args) => execFileSync('git', ['-C', root, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], @@ -75,7 +76,12 @@ test('manual build script passes the full revision to Compose without starting s git('-c', 'user.name=Build Test', '-c', 'user.email=build@example.invalid', '-c', 'commit.gpgsign=false', 'commit', '--allow-empty', '-m', 'test'); const sha = git('rev-parse', 'HEAD'); - assert.deepEqual(build(), [sha, 'compose', 'build', '--no-cache', 'pediatric-scribe']); + // With a revision the image is also tagged immutably, so a deploy can name a + // build rather than chasing whatever :latest happens to point at. + assert.deepEqual(build(), [ + sha, 'compose', 'build', '--no-cache', 'pediatric-scribe', + sha, 'tag', 'ped-ai-local:latest', 'ped-ai-local:' + sha, + ]); const foreign = path.join(root, 'foreign'); git('init', foreign); @@ -84,7 +90,10 @@ test('manual build script passes the full revision to Compose without starting s assert.notEqual(git('-C', foreign, 'rev-parse', 'HEAD'), sha); const controls = { GIT_DIR: path.join(foreign, '.git'), GIT_WORK_TREE: foreign, GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'core.bare', GIT_CONFIG_VALUE_0: 'true' }; - assert.deepEqual(build(controls), [sha, 'compose', 'build', '--no-cache', 'pediatric-scribe']); + assert.deepEqual(build(controls), [ + sha, 'compose', 'build', '--no-cache', 'pediatric-scribe', + sha, 'tag', 'ped-ai-local:latest', 'ped-ai-local:' + sha, + ]); // A Docker-only source archive still works without Git installed. fs.rmSync(path.join(root, '.git'), { recursive: true }); fs.symlinkSync('/usr/bin/dirname', path.join(bin, 'dirname'));