From 5a666f5ca5d2e4760323469f74fc3b30586b5972 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sun, 13 Sep 2026 00:49:25 +0200 Subject: [PATCH] test: e2e runs against its own throwaway database, not production's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e stack shared production's Postgres — same server, same database, same table. Seeded robots sat in `users` beside real clinicians, and anything a test wrote, or a migration under test changed, landed on real data. Nothing about "run the tests" should be able to reach an account belonging to a person. Now it has a Postgres and a Redis of its own, both on tmpfs: created empty on every run, held in RAM, gone on teardown. scripts/e2e.sh is one command that recreates the stack, seeds it, runs the browser and leaves the app up at 127.0.0.1:3553 so it can be clicked around in, with the report served at :3554. Two bugs fell out of it immediately, both of which only a database that did not already exist could have found: The schema could not be built from nothing. The entrypoint migrated before the app created its baseline tables, so the first migration failed on saved_encounters not existing. It never showed because every database this has ever run against already had the baseline. Then, one layer down, 1777800000000_generated-images creates a table with a foreign key to learning_content — which the baseline stopped creating when Learning Hub was removed. Restoring into a brand-new database could not have booted. The entrypoint now stands aside when the database is empty and lets the app do it in the order it already gets right, and the foreign key is only created where its target is. All 20 migrations replay from empty, producing the same 23 tables production has. Configuration lives in the database, so a throwaway one starts at defaults — 14 settings against production's 49. That is why every model picker was empty: models.custom did not exist. The tests were right and the environment was incomplete, so the seed now states what the suite depends on, with fictional model ids: a test should not pass because of a setting somebody changed on the live system last week. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU --- docker-compose.e2e.yml | 104 +++++++++++++++--- docker-entrypoint.sh | 22 ++++ e2e/playwright.config.js | 7 +- e2e/seed.js | 39 +++++++ e2e/tests/learning-tab.spec.js | 57 ---------- migrations/1777800000000_generated-images.js | 21 +++- scripts/e2e.sh | 109 ++++++++++++++----- scripts/schema-state.js | 27 +++++ 8 files changed, 284 insertions(+), 102 deletions(-) delete mode 100644 e2e/tests/learning-tab.spec.js create mode 100644 scripts/schema-state.js diff --git a/docker-compose.e2e.yml b/docker-compose.e2e.yml index cfee6c22..54c7c188 100644 --- a/docker-compose.e2e.yml +++ b/docker-compose.e2e.yml @@ -1,21 +1,68 @@ -# E2E test environment — runs a second instance of the app on port 3553 with -# Turnstile disabled so Playwright can log in without the bot challenge. -# Shares the postgres + pgdata volume with production so seeded e2e test users -# (email pattern *@ped-ai.test) persist across test runs. +# E2E test environment — a whole second copy of the app, on its own throwaway +# database, with its own throwaway Redis. # -# Bring up with: # docker compose -f docker-compose.yml -f docker-compose.e2e.yml up -d pediatric-scribe-e2e +# docker compose -f docker-compose.yml -f docker-compose.e2e.yml down -v postgres-e2e redis-e2e pediatric-scribe-e2e # -# Tear down with: -# docker compose -f docker-compose.yml -f docker-compose.e2e.yml down pediatric-scribe-e2e +# Normally you want scripts/e2e.sh, which does both around a test run. +# +# It used to share production's Postgres — same server, same database, same +# table. Seeded robots sat in `users` next to real clinicians, and anything a +# test wrote, or a migration under test changed, landed on real data. Nothing +# about "run the tests" should be able to reach an account belonging to a +# person. Now the stack has a database of its own, held in a tmpfs: it exists +# in RAM, it is created empty on every `up`, and it is gone on `down`. The +# schema is rebuilt each time by the container's own migrations, which also +# means every run proves the migrations still work from nothing. services: + # ── Throwaway Postgres ──────────────────────────────────────────────── + # Same pinned image as production, so an e2e pass says something about what + # production will do. PGDATA points at a subdirectory because initdb wants a + # 0700 directory of its own and a tmpfs mountpoint is not one. + postgres-e2e: + image: pgvector/pgvector:pg16@sha256:00ba258a66dac104fd5171074a0084462a64a1369d8513f3d0a634e2f24d15bc + container_name: pedscribe-db-e2e + environment: + POSTGRES_DB: pedscribe_e2e + POSTGRES_USER: pedscribe + POSTGRES_PASSWORD: e2e-throwaway + PGDATA: /var/lib/postgresql/data/pgdata + tmpfs: + # In RAM, so there is no volume to forget to clean up and nothing to + # survive a reboot. 1G is far more than a seeded test run uses. + - /var/lib/postgresql/data:size=1g + healthcheck: + test: ["CMD-SHELL", "pg_isready -U pedscribe -d pedscribe_e2e"] + interval: 3s + timeout: 5s + retries: 20 + restart: "no" + + # ── Throwaway Redis ─────────────────────────────────────────────────── + # Sessions and rate-limit counters. Persistence off in both directions: no + # RDB snapshots, no AOF, and /data on tmpfs, so a run cannot inherit state + # from the one before it. + redis-e2e: + image: redis:8-alpine@sha256:d146f83b1e0f02fc27c26a50cee39338c736674c5959db84363e6ae3cd9e02d2 + container_name: ped-ai-redis-e2e + command: ["redis-server", "--save", "", "--appendonly", "no"] + tmpfs: + - /data:size=64m + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 5s + retries: 20 + restart: "no" + + # ── The app under test ──────────────────────────────────────────────── pediatric-scribe-e2e: build: context: . args: GIT_REVISION: ${GIT_REVISION:-unknown} - image: ped-ai-local:latest + image: ped-ai-e2e:latest ports: - "127.0.0.1:3553:3000" networks: @@ -27,6 +74,13 @@ services: env_file: - .env environment: + # These four are the isolation. The entrypoint applies OpenBao secrets + # only for keys docker has not already set, so anything named here wins + # over the vault — which is exactly what that rule was written for. + DATABASE_URL: postgresql://pedscribe:e2e-throwaway@postgres-e2e:5432/pedscribe_e2e + REDIS_URL: redis://redis-e2e:6379 + # Never mail a real person from a test run. + SMTP_HOST: "" # Disable Turnstile entirely — both server-side verification AND the # client-side widget. Without clearing the SITE_KEY the frontend tries # to initialise the Turnstile iframe against the prod domain and @@ -34,8 +88,9 @@ services: # flags as an uncaught exception. TURNSTILE_SECRET_KEY: "" TURNSTILE_SITE_KEY: "" - # Disable SMTP so register auto-verifies the user and returns a session - SMTP_HOST: "" + # A key of its own. Rows here are throwaway, and binding them to the + # production key would be the one piece of production that leaked in. + DATA_ENCRYPTION_KEY: "e2e0000000000000000000000000000000000000000000000000000000000e2e" # Raise the login rate-limit so Playwright multi-worker runs don't # trip the production 10/15min cap. Only affects this e2e container. LOGIN_RATE_LIMIT_MAX: "500" @@ -50,16 +105,33 @@ services: volumes: - scribe-logs-e2e:/app/data/logs depends_on: - postgres: + postgres-e2e: + condition: service_healthy + redis-e2e: condition: service_healthy container_name: pediatric-ai-scribe-e2e - restart: unless-stopped + # Not unless-stopped: this is a test rig, not a service. It should not come + # back on its own after a reboot, and it should not outlive a `down`. + restart: "no" healthcheck: test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"] - interval: 30s - timeout: 10s - retries: 5 - start_period: 20s + interval: 5s + timeout: 5s + retries: 12 + start_period: 15s + + # ── The last run's report ───────────────────────────────────────────── + # Playwright writes a self-contained HTML report; this serves it so there is + # a link to open rather than a directory to find. Traces and screenshots of + # failures are in there, which is the part worth looking at on a phone. + e2e-report: + image: nginx:alpine + container_name: pediatric-ai-scribe-e2e-report + ports: + - "127.0.0.1:3554:80" + volumes: + - ./e2e/playwright-report:/usr/share/nginx/html:ro + restart: "no" volumes: scribe-logs-e2e: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index f081b58e..1da79cb0 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -93,6 +93,28 @@ if [ "${RUN_MIGRATIONS:-true}" = "true" ]; then exit 1 fi + # A database with nothing in it is the one case where migrating here is + # wrong. The schema has two layers: src/db/database.js creates the baseline + # tables on first connect, and the migrations are written to layer on top — + # the earliest of them alters saved_encounters, which only the baseline + # creates. Run first against an empty database and they fail on a table that + # does not exist yet. + # + # So: empty database, stand aside and let the app do it, which it already + # does in the right order (initDatabase, then runMigrations). Existing + # database, migrate here exactly as before, so a deploy still cannot put new + # code in front of an old schema. Unreachable, carry on into the loop below, + # which is what already handles a Postgres still opening its socket. + # + # This is why restoring into a brand-new database could not boot. + if [ "$(node scripts/schema-state.js 2>/dev/null)" = "empty" ]; then + echo "[entrypoint] database is empty — the app will create the baseline and migrate on top of it" + RUN_MIGRATIONS=false + fi +fi + +if [ "${RUN_MIGRATIONS:-true}" = "true" ]; then + _MIGRATE_ATTEMPT=1 _MIGRATE_MAX=${MIGRATION_ATTEMPTS:-10} while : ; do diff --git a/e2e/playwright.config.js b/e2e/playwright.config.js index 02d16a79..6636de91 100644 --- a/e2e/playwright.config.js +++ b/e2e/playwright.config.js @@ -25,7 +25,12 @@ module.exports = defineConfig({ fullyParallel: false, retries: 0, workers: 1, - reporter: [['list']], + // list for the terminal, html for afterwards. The html report is a + // self-contained directory with the trace and screenshot of every failure in + // it; docker-compose.e2e.yml serves it at 127.0.0.1:3554 so it is a link + // rather than a path. open:'never' because this runs in a container that has + // no browser to open it with. + reporter: [['list'], ['html', { outputFolder: 'playwright-report', open: 'never' }]], use: { baseURL: process.env.BASE_URL || 'http://127.0.0.1:3553', // The app registers a service worker that answers every /api/ request with diff --git a/e2e/seed.js b/e2e/seed.js index 4a6752b9..acbbc5cb 100644 --- a/e2e/seed.js +++ b/e2e/seed.js @@ -46,6 +46,44 @@ var ACCOUNTS = [ { email: process.env.E2E_ADMIN_EMAIL || 'e2e-admin' + TEST_DOMAIN, name: 'E2E Admin', role: 'admin' } ]; +// ── Configuration ───────────────────────────────────────────────────── +// Settings live in the database, so a throwaway database starts at defaults +// rather than at whatever production happens to be configured with. That is +// the point — a test should not pass because of a setting somebody changed on +// the live system last week — but it does mean anything the suite depends on +// has to be stated here. +// +// This is what made the model pickers empty when the e2e stack stopped sharing +// production's database: models.custom did not exist, so there was nothing to +// put in the