pediatric-ai-scribe-v3/test/prompt-administration.test.js

const { test } = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const { createRequire } = require('node:module');
const express = require('express');
const jwt = require('jsonwebtoken');
const root = path.join(__dirname, '..');
const read = file => fs.readFileSync(path.join(root, file), 'utf8');
const quiet = { log() {}, warn() {}, error() {} };

function load(file, mocks = {}, env = {}) {
  const filename = path.join(root, file);
  const native = createRequire(filename);
  const module = { exports: {} };
  vm.runInNewContext(read(file), { module, exports: module.exports, console: quiet, Buffer, process: { env },
    require: name => Object.hasOwn(mocks, name) ? mocks[name] : native(name) }, { filename });
  return module.exports;
}

// Fake only storage/transport: actual catalogue, revisions, SQL caller and routes run.
function storage() {
  const state = { settings: new Map(), rows: [], log: [], locks: new Map(), nextId: 1, fail: '', missing: false };
  function select(sql, params, pending = []) {
    if (sql.includes('prompt_revisions') && state.missing) throw Object.assign(Error('synthetic missing table'), { code: '42P01' });
    if (sql.includes('unnest')) return params[0].map(key => ({ key, value: state.settings.get(key), revision: Math.max(0, ...state.rows.filter(row => row.prompt_key === key).map(row => row.id)) }));
    if (sql.includes('FROM prompt_revisions')) {
      let rows = state.rows.concat(pending).filter(row => row.prompt_key === params[0]);
      if (sql.includes('AND id = $2')) rows = rows.filter(row => row.id === params[1]);
      rows = rows.slice().sort((a, b) => b.id - a.id);
      if (sql.includes('LIMIT 1')) rows = rows.slice(0, 1);
      if (sql.includes('LIMIT $2')) rows = rows.slice(0, params[1]);
      return rows.map(row => {
        if (sql.startsWith('SELECT id FROM')) return { id: row.id };
        if (sql.startsWith('SELECT value FROM')) return { value: row.value };
        const { prompt_key, ...result } = row;
        if (!sql.includes(', value FROM')) delete result.value;
        return result;
      });
    }
    if (sql.startsWith('SELECT value FROM app_settings')) return state.settings.has(params[0]) ? [{ value: state.settings.get(params[0]) }] : [];
    if (sql.includes('FROM app_settings')) return [...state.settings].map(([key, value]) => ({ key, value }));
    throw Error('Unexpected read SQL: ' + sql);
  }
  const db = {
    all: async (sql, params) => select(sql, params),
    get: async (sql, params) => select(sql, params)[0] || null,
    getSetting: async key => state.settings.get(key) ?? null,
    async setSetting(key, value) { state.log.push(['unversioned', key]); state.settings.set(key, value); },
    pool: { async connect() {
      const pending = [];
      let change, unlock;
      return {
        async query(sql, params = []) {
          state.log.push([sql, ...params]);
          if (state.fail && sql.includes(state.fail)) throw Error('synthetic SQL failure, potentially secret text');
          if (sql === 'BEGIN') return { rows: [] };
          if (sql.includes('pg_advisory_xact_lock')) {
            const previous = state.locks.get(params[0]) || Promise.resolve();
            const waiting = new Promise(resolve => { unlock = resolve; });
            state.locks.set(params[0], previous.then(() => waiting));
            await previous;
            return { rows: [] };
          }
          if (sql === 'COMMIT') {
            if (state.beforeCommit) state.beforeCommit();
            state.rows.push(...pending);
            if (change) { if (change.remove) state.settings.delete(change.key); else state.settings.set(change.key, change.value); }
            unlock();
            if (state.afterCommit) await state.afterCommit(pending);
            return { rows: [] };
          }
          if (sql === 'ROLLBACK') { if (unlock) unlock(); return { rows: [] }; }
          if (sql.startsWith('INSERT INTO prompt_revisions')) {
            const [prompt_key, value, wasDefault, createdBy, restoredFrom] = params;
            const id = state.nextId++;
            pending.push({ id, prompt_key, value, wasDefault, createdBy, restoredFrom, createdAt: new Date().toISOString() });
            return { rows: [{ id }] };
          }
          if (sql.startsWith('INSERT INTO app_settings') || sql.startsWith('DELETE FROM app_settings')) {
            change = { key: params[0], value: params[1], remove: sql.startsWith('DELETE') };
            return { rows: [] };
          }
          return { rows: select(sql, params, pending) };
        },
        release() { state.log.push(['release']); }
      };
    } }
  };
  return { db, state };
}

function services() {
  const prompts = load('src/utils/prompts.js');
  const clinical = { ...require('../src/utils/clinicalPrompts') };
  const catalog = load('src/utils/promptCatalog.js', { './prompts': prompts, './clinicalPrompts': clinical });
  const revisions = load('src/utils/promptRevisions.js', { './prompts': prompts, './promptCatalog': catalog });
  return { prompts, clinical, catalog, revisions, ...storage() };
}

async function application(t, svc, env = {}) {
  const logs = [];
  const authDb = { async get(sql, params) {
    if (sql.includes('FROM users')) return { id: params[0], role: params[0] === 1 ? 'admin' : 'user' };
    return { id: 1, last_activity: new Date().toISOString() };
  } };
  const auth = load('src/middleware/auth.js', { '../db/database': authDb }, { JWT_SECRET: 'synthetic-only-secret' });
  const router = load('src/routes/adminConfig.js', {
    '../db/database': svc.db, '../middleware/auth': auth, '../utils/prompts': svc.prompts,
    '../utils/promptCatalog': svc.catalog, '../utils/promptRevisions': svc.revisions,
    '../utils/logger': { audit(actor, action, detail, req, meta) { logs.push({ actor, action, detail, meta }); } }, '../utils/errors': {},
    '../utils/ttsProvider': {}, '../utils/litellm': {}, '../utils/sttProvider': {}
  }, env);
  const app = express();
  app.use(express.json()); app.use('/api/admin', router);
  const server = app.listen(0, '127.0.0.1');
  await new Promise(resolve => server.on('listening', resolve));
  t.after(() => server.close());
  return { logs, async request(method, route, body, user = 1) {
    const headers = { 'Content-Type': 'application/json' };
    if (user) headers.Authorization = 'Bearer ' + jwt.sign({ userId: user }, 'synthetic-only-secret');
    const response = await fetch('http://127.0.0.1:' + server.address().port + '/api/admin' + route, { method, headers, body: body === undefined ? undefined : JSON.stringify(body) });
    return { status: response.status, body: await response.json() };
  } };
}

test('finite catalogue, actual admin gates, revision API, stale edit, cross-key denial and no secret/helper writes', async t => {
  const svc = services(); const app = await application(t, svc);
  const routes = [
    ['GET', '/config'], ['GET', '/config/prompts'], ['GET', '/config/prompts/hpiEncounter/history'],
    ['GET', '/config/prompts/hpiEncounter/revisions/1'], ['PUT', '/config/prompt.hpiEncounter', { value: 'x' }],
    ['POST', '/config/prompts/hpiEncounter/reset', {}], ['POST', '/config/prompts/hpiEncounter/restore', { revisionId: 1 }]
  ];
  for (const [method, url, body] of routes) for (const user of [0, 2]) assert.equal((await app.request(method, url, body, user)).status, user ? 403 : 401);
  assert.equal(svc.state.log.length, 0);
  const list = (await app.request('GET', '/config/prompts')).body.prompts;
  // 31, down from 32: learning_hub.image_behavior went with the Learning Hub.
  assert.equal(list.length, 31); assert.equal(list.filter(p => p.family === 'scribe').length, 29);
  assert.equal(list.filter(p => p.family === 'clinical-text').length, 1); assert.equal(list.filter(p => p.family === 'clinical-image').length, 1);
  for (const prompt of list) { assert.equal(prompt.revision, 0); assert.equal(prompt.editable, true); assert.ok(prompt.purpose && prompt.usedBy.length && prompt.value); }
  for (const key of ['prompt.unknown', 'prompt.loadFromDb', 'prompt.updatePrompt', 'prompt.getAllPrompts', 'prompt.getDefaultPrompt', 'prompt.__proto__', 'prompt.smtp.pass']) {
    assert.equal((await app.request('PUT', '/config/' + key, { value: 'never store' })).status, 400);
  }
  for (const key of ['smtp.pass', 'memories', 'clinical_assistant.unknown', '__proto__']) {
    assert.equal((await app.request('POST', '/config/prompts/' + key + '/reset', {})).status, 404);
    assert.equal((await app.request('GET', '/config/prompts/' + key + '/history')).status, 404);
  }
  assert.equal(svc.state.log.length, 0);
  const value = '<script>inert editor text</script>\n  preserve whitespace 😀';
  const saved = await app.request('PUT', '/config/prompt.hpiEncounter', { value, expectedRevision: 0 });
  assert.equal(saved.status, 200); assert.equal(saved.body.value, value); assert.equal(saved.body.revision, 2);
  assert.equal(svc.prompts.hpiEncounter, value);
  assert.equal(svc.state.rows[0].value, svc.prompts.getDefaultPrompt('hpiEncounter'));
  assert.equal(svc.state.rows[0].wasDefault, true); assert.equal(svc.state.rows[0].createdBy, null);
  assert.equal(svc.state.rows[1].createdBy, 1);
  assert.equal((await app.request('PUT', '/config/prompt.hpiEncounter', { value: 'stale', expectedRevision: 0 })).status, 409);
  assert.equal((await app.request('PUT', '/config/prompt.hpiEncounter', { value: 'bad', expectedRevision: '2' })).status, 400);
  assert.equal((await app.request('PUT', '/config/prompt.hpiEncounter', { value: {} })).status, 400);
  const history = await app.request('GET', '/config/prompts/hpiEncounter/history');
  assert.equal(history.body.revision, 2); assert.deepEqual(history.body.revisions.map(r => r.id), [2, 1]);
  assert.equal(history.body.revisions[0].value, undefined);
  const view = await app.request('GET', '/config/prompts/prompt.hpiEncounter/revisions/2');
  assert.equal(view.body.revision.value, value);
  assert.equal((await app.request('GET', '/config/prompts/prompt.refine/revisions/2')).status, 404);
  assert.equal((await app.request('POST', '/config/prompts/prompt.refine/restore', { revisionId: 2 })).status, 404);
  for (const revisionId of [[], [2], {}, true, 0, -1, '2.5', '2x']) {
    assert.equal((await app.request('POST', '/config/prompts/hpiEncounter/restore', { revisionId })).status, 400);
  }
  assert.equal((await app.request('GET', '/config/prompts/hpiEncounter/history?limit[]=20')).status, 400);
  assert.equal((await app.request('POST', '/config/prompts/prompt.hpiEncounter/reset', { expectedRevision: 2 })).body.revision, 3);
  assert.equal(svc.prompts.hpiEncounter, svc.prompts.getDefaultPrompt('hpiEncounter'));
  assert.equal(svc.state.settings.has('prompt.hpiEncounter'), false);
  const restored = await app.request('POST', '/config/prompts/hpiEncounter/restore', { revisionId: 2, expectedRevision: 3 });
  assert.equal(restored.body.value, value); assert.equal(restored.body.revision, 4);
  assert.equal(svc.state.rows.at(-1).restoredFrom, 2);
  assert.equal((await app.request('PUT', '/config/prompt.hpiEncounter', { value: 'legacy compatible' })).status, 200);
  assert.equal((await app.request('GET', '/config/prompts')).body.prompts[0].revision, 5);
  for (const key of ['clinical_assistant.system_behavior', 'clinical_assistant.image_behavior']) {
    const clinicalSave = await app.request('PUT', '/config/' + key, { value, expectedRevision: 0 });
    assert.equal(clinicalSave.status, 200);
    assert.ok(clinicalSave.body.revision > 5);
    assert.equal(svc.state.settings.get(key), value);
    const item = (await app.request('GET', '/config/prompts')).body.prompts.find(prompt => prompt.dbKey === key);
    assert.equal(item.value, value); assert.equal(item.revision, clinicalSave.body.revision);
    assert.equal((await app.request('GET', '/config/prompts/' + key + '/history')).body.revisions.length, 2);
  }
  assert.equal(svc.state.log.some(row => row[0] === 'unversioned'), false);
  assert.doesNotMatch(JSON.stringify(app.logs), /inert editor|legacy compatible/);
});

test('first legacy baseline, reset and exact restore after shipped default changes', async () => {
  const { db, state, revisions, catalog, clinical } = services();
  const key = 'clinical_assistant.system_behavior';
  state.settings.set(key, 'Existing global override');
  await revisions.mutate(db, key, { action: 'reset', actor: 7, expectedRevision: 0 });
  assert.equal(state.rows[0].value, 'Existing global override'); assert.equal(state.rows[0].wasDefault, false);
  const original = state.rows[1].value;
  clinical.DEFAULT_BEHAVIOR = 'Synthetic later shipped default';
  await revisions.mutate(db, key, { action: 'reset' });
  assert.equal(state.rows.at(-1).value, clinical.DEFAULT_BEHAVIOR);
  const restored = await revisions.mutate(db, key, { action: 'restore', revisionId: 2 });
  assert.equal(restored.value, original);
  assert.equal(state.settings.get(key), original, 'Restored historical default is pinned as an override');
  assert.equal(catalog.effective(catalog.find(key), state.settings.get(key)).value, original);
  assert.equal(state.rows.at(-1).wasDefault, false);
});

test('real transaction calls rollback every partial write, never publish before commit, and missing schema is safe', async t => {
  for (const fail of ['INSERT INTO prompt_revisions', 'INSERT INTO app_settings', 'COMMIT']) {
    const svc = services(); const original = svc.prompts.refine;
    svc.state.fail = fail;
    await assert.rejects(svc.revisions.mutate(svc.db, 'prompt.refine', { action: 'save', value: 'Not committed' }));
    assert.equal(svc.prompts.refine, original); assert.equal(svc.state.rows.length, 0); assert.equal(svc.state.settings.size, 0);
    assert.deepEqual(svc.state.log.slice(-2).map(row => row[0]), ['ROLLBACK', 'release']);
  }
  const svc = services(); const before = svc.prompts.refine;
  svc.state.beforeCommit = () => assert.equal(svc.prompts.refine, before);
  await svc.revisions.mutate(svc.db, 'refine', { action: 'save', value: 'Committed' });
  assert.equal(svc.prompts.refine, 'Committed');
  delete svc.state.beforeCommit;
  svc.state.fail = 'DELETE FROM app_settings';
  await assert.rejects(svc.revisions.mutate(svc.db, 'refine', { action: 'reset' }));
  assert.equal(svc.state.rows.length, 2); assert.equal(svc.prompts.refine, 'Committed');
  svc.state.fail = ''; svc.state.missing = true;
  const app = await application(t, svc);
  assert.equal((await app.request('PUT', '/config/prompt.refine', { value: 'No unversioned fallback' })).status, 503);
  assert.equal(svc.state.rows.length, 2); assert.equal(svc.state.settings.get('prompt.refine'), 'Committed');
  assert.equal(svc.state.log.some(row => row[0] === 'unversioned'), false);
});

test('concurrent edits serialize one baseline, stale optimistic edits fail and delayed commit cannot regress memory', async () => {
  const svc = services();
  const edit = options => svc.revisions.mutate(svc.db, 'prompt.refine', { action: 'save', ...options });
  const attempts = await Promise.allSettled([edit({ value: 'A', expectedRevision: 0 }), edit({ value: 'B', expectedRevision: 0 })]);
  assert.equal(attempts.filter(item => item.status === 'fulfilled').length, 1);
  assert.equal(attempts.find(item => item.status === 'rejected').reason.statusCode, 409);
  assert.equal(svc.state.rows.length, 2);
  let release, committed;
  const wait = new Promise(resolve => { committed = resolve; });
  svc.state.afterCommit = async rows => { if (rows.at(-1).value === 'C') { committed(); await new Promise(resolve => { release = resolve; }); } };
  const c = edit({ value: 'C' }); await wait;
  await edit({ value: 'D' }); release(); await c;
  assert.equal(svc.state.rows.length, 4); assert.equal(svc.state.settings.get('prompt.refine'), 'D'); assert.equal(svc.prompts.refine, 'D');
  assert.equal(svc.state.log.filter(row => row[0].includes('pg_advisory_xact_lock')).length, 4);
});

test('Scribe object stays shared across consumers; defaults/helpers resist overrides and a slow startup load', async () => {
  const svc = services(); const captured = [];
  const mocks = { '../utils/prompts': svc.prompts, '../utils/ai': { async callAI(messages) { captured.push(messages[0].content); return { content: 'synthetic' }; } },
    '../middleware/auth': { authMiddleware() {} }, '../utils/logger': { audit() {} } };
  const consumer1 = load('src/routes/refine.js', mocks);
  const consumer2 = load('src/routes/refine.js', mocks);
  const call = async router => {
    const route = router.stack.find(layer => layer.route.path === '/refine').route;
    await route.stack.at(-1).handle({ body: { currentDocument: 'Synthetic', instructions: 'Synthetic' }, user: { id: 1 } }, { json() {}, status() { return this; } });
  };
  await svc.revisions.mutate(svc.db, 'refine', { action: 'save', value: 'First edit' });
  await call(consumer1); await call(consumer2);
  assert.ok(captured.every(value => value.startsWith('First edit')));
  await svc.revisions.mutate(svc.db, 'refine', { action: 'reset' });
  await call(consumer1); await call(consumer2);
  assert.ok(captured.slice(2).every(value => value.startsWith(svc.prompts.getDefaultPrompt('refine'))));
  const originalHelper = svc.prompts.loadFromDb;
  for (const key of ['loadFromDb', 'getAllPrompts', 'getDefaultPrompt', '__proto__', 'missing']) assert.equal(svc.prompts.updatePrompt(key, 'poison'), false);
  let resume; const keys = [];
  const loading = svc.prompts.loadFromDb({ async getSetting(key) { keys.push(key); if (key === 'prompt.hpiEncounter') return new Promise(resolve => { resume = resolve; }); return null; } });
  svc.prompts.updatePrompt('hpiEncounter', 'Concurrent edit'); resume('Outdated DB value'); await loading;
  assert.equal(svc.prompts.hpiEncounter, 'Concurrent edit'); assert.equal(keys.length, 29); assert.equal(svc.prompts.loadFromDb, originalHelper);
  assert.notEqual(svc.prompts.getDefaultPrompt('hpiEncounter'), 'Concurrent edit');
});

test('history is newest first, bounded 20/100, and admin budget overrides ENV with validated writes', async t => {
  const svc = services();
  for (let i = 0; i < 105; i++) await svc.revisions.mutate(svc.db, 'clinical_assistant.image_behavior', { action: 'save', value: 'synthetic ' + i });
  assert.equal((await svc.revisions.history(svc.db, 'clinical_assistant.image_behavior')).revisions.length, 20);
  assert.equal((await svc.revisions.history(svc.db, 'clinical_assistant.image_behavior', 1000)).revisions.length, 100);
  assert.equal((await svc.revisions.history(svc.db, 'clinical_assistant.image_behavior', 1)).revision, 106);
  svc.state.settings.set('clinical_assistant.conversation_chars', '999999');
  const app = await application(t, svc, { CLINICAL_ASSISTANT_CONVERSATION_CHARS: '1000' });
  const config = await app.request('GET', '/config');
  assert.deepEqual(config.body.conversationBudget, { limit: 1000, unit: 'characters', measure: 'UTF-16 code units', env: 'CLINICAL_ASSISTANT_CONVERSATION_CHARS', source: 'environment' });
  assert.equal((await app.request('PUT', '/config/clinical_assistant.conversation_chars', { value: '5000' })).status, 200);
  assert.equal(svc.state.settings.get('clinical_assistant.conversation_chars'), '5000');
  assert.equal((await app.request('PUT', '/config/clinical_assistant.conversation_chars', { value: '50' })).status, 400, 'below the 1000 floor is rejected');
  assert.equal((await app.request('PUT', '/config/clinical_assistant.conversation_chars', { value: '99999999' })).status, 400, 'above the 1000000 ceiling is rejected');
  const invalid = await application(t, svc, { CLINICAL_ASSISTANT_CONVERSATION_CHARS: 'not-a-number' });
  const unavailable = await invalid.request('GET', '/config');
  assert.equal(unavailable.status, 503); assert.equal(unavailable.body.conversationBudget, undefined);
  const defaults = await application(t, svc);
  assert.equal((await defaults.request('GET', '/config')).body.conversationBudget.limit, 120000);
  assert.equal((await defaults.request('GET', '/config')).body.conversationBudget.source, 'default');
});

test('migration owns finite append-only schema and emits reversible SQL without connecting to a database', async () => {
  const migration = require('../migrations/1777700000000_add-prompt-revisions');
  const { Migration } = require('node-pg-migrate');
  async function dryRun(direction) {
    const sql = [];
    const engine = new Migration({ query() { throw Error('Dry run must not query a database'); } },
      path.join(root, 'migrations/1777700000000_add-prompt-revisions.js'), migration,
      { dryRun: true, singleTransaction: true, migrationsTable: 'pgmigrations' }, undefined,
      { ...quiet, info() {}, debug: text => sql.push(text) });
    await engine.apply(direction);
    return sql;
  }
  const up = await dryRun('up'); const down = await dryRun('down');
  const keys = [...up[0].matchAll(/'(prompt\.[^']+|clinical_assistant\.[^']+)'/g)].map(match => match[1]);
  // Every catalogue entry, with nothing filtered out: the one exception used
  // to be learning_hub.image_behavior, which no longer exists.
  assert.deepEqual(keys.sort(), Array.from(services().catalog.entries, entry => entry.dbKey).sort());
  assert.match(up[0], /BEFORE UPDATE OR DELETE/); assert.match(up[0], /FOREIGN KEY \(prompt_key, restored_from\)/);
  assert.match(down[0], /DROP TABLE prompt_revisions/);
});

test('approved inherited Scribe and clinical default bytes remain unchanged', () => {
  const hash = value => require('node:crypto').createHash('sha256').update(value).digest('hex');
  const svc = services();
  // Hashes captured from the protected input patch, before overrides or helpers.
  // DEFAULT_BEHAVIOR changed deliberately: the acknowledgement and image rules
  // moved out of route heuristics into the prompt, then were rewritten as
  // principles ("respond to the latest message", "never repeat a previous
  // answer") rather than an enumerated list of words and cases.
  assert.equal(hash(JSON.stringify(svc.prompts.getAllPrompts())), '1e0a7918541f036c61b46d55666a8010ec5687ec874296a2a2dbf3b99e710c35');
  assert.equal(hash(svc.clinical.DEFAULT_BEHAVIOR), '183a4e8faa467242938951365153cdc9055435cc48ae47afe4160ac760bb412b');
  assert.equal(hash(svc.clinical.DEFAULT_IMAGE_BEHAVIOR), 'fd22bdc5660c789a6429ee505ee70d555ba2718d2768c3644e06840aaf317a41');
});