pediatric-ai-scribe-v3/test/image-fallback-chain.test.js

// An image model that fails should hand the job to the next one — for every
// feature, not only the Clinical Assistant, and for the failures where another
// model genuinely has a chance.
const test = require('node:test');
const assert = require('node:assert/strict');
const { classifyImageFailure, isContentRefusal, MAX_IMAGE_MODELS,
        shouldRetryImageFallback } = require('../src/utils/generatedImages');

const httpError = (status, body) => Object.assign(new Error(typeof body === 'string' ? body : 'request failed'),
  { response: { status, data: body } });

test('the provider saying "not now" is always worth another model', () => {
  // 408 is Request Timeout, 429 is Too Many Requests. Both are a server saying
  // not now, never not ever.
  for (const status of [408, 429, 500, 502, 503, 504]) {
    assert.equal(classifyImageFailure(httpError(status), null, false).retry, true, 'status ' + status);
  }
  // No response at all: a socket hang-up or a DNS failure.
  assert.equal(classifyImageFailure(new Error('socket hang up'), null, false).retry, true);
});

test('bad credentials stop the chain, because every model shares them', () => {
  for (const status of [401, 403]) {
    const verdict = classifyImageFailure(httpError(status), null, false);
    assert.equal(verdict.retry, false, 'status ' + status);
    assert.match(verdict.reason, /credentials/);
  }
});

test('a model the gateway does not have moves on rather than failing the job', () => {
  // This is a configuration mistake, and the next model is exactly the thing
  // that rescues it.
  const verdict = classifyImageFailure(httpError(404), null, false);
  assert.equal(verdict.retry, true);
  assert.match(verdict.reason, /does not have that model/);
});

test('a content refusal tries the next model — policy is a vendor decision', () => {
  const refusals = [
    httpError(400, { error: { message: 'Your request was rejected by our safety system' } }),
    httpError(400, 'content_policy_violation'),
    httpError(422, { message: 'The prompt was flagged as sensitive' }),
    httpError(400, { error: 'This request was blocked by moderation' })
  ];
  for (const err of refusals) {
    assert.equal(isContentRefusal(err), true, err.message);
    assert.equal(classifyImageFailure(err, null, false).retry, true);
  }
});

test('a malformed request is not retried, because it is malformed everywhere', () => {
  const verdict = classifyImageFailure(httpError(400, { error: 'size must be one of 1024x1024' }), null, false);
  assert.equal(verdict.retry, false);
  assert.match(verdict.reason, /rejected the request/);
  assert.equal(classifyImageFailure(httpError(413), null, false).retry, false, 'too large everywhere too');
});

test('a cancelled request and a shutdown never start more paid work', () => {
  assert.equal(classifyImageFailure(new Error('x'), { aborted: true }, false).retry, false);
  assert.equal(classifyImageFailure(Object.assign(new Error('x'), { name: 'AbortError' }), null, false).retry, false);
  assert.equal(classifyImageFailure(httpError(503), null, true).retry, false, 'shutting down');
});

test('every workflow gets fallbacks now, not the Clinical Assistant alone', () => {
  // A My Resources figure that failed used to have no second chance at all,
  // which is the case where a missing picture is most visible — a slide with a
  // hole in it.
  for (const workflow of ['clinical_assistant', 'my_resources', 'learning_hub']) {
    assert.equal(shouldRetryImageFallback({
      workflow, fallback: 'model-b', jobModel: 'model-a', error: httpError(503)
    }), true, workflow);
  }
});

test('a fallback identical to the model that just failed is not a fallback', () => {
  assert.equal(shouldRetryImageFallback({
    workflow: 'my_resources', fallback: 'model-a', jobModel: 'model-a', error: httpError(503)
  }), false);
  assert.equal(shouldRetryImageFallback({
    workflow: 'my_resources', fallback: '', jobModel: 'model-a', error: httpError(503)
  }), false);
});

test('the chain is capped, because every hop is a paid request', () => {
  assert.equal(MAX_IMAGE_MODELS, 3);
  const lib = require('fs').readFileSync(require('path').join(__dirname, '..', 'src/utils/generatedImages.js'), 'utf8');
  assert.match(lib, /chain\.slice\(0, MAX_IMAGE_MODELS\)/);
  // Duplicates collapse: naming the primary again as a fallback would pay twice
  // for the same refusal.
  assert.match(lib, /if \(chain\.indexOf\(id\) === -1\) chain\.push\(id\)/);
});

test('each hop re-leases, and records which model is actually being paid for', () => {
  const lib = require('fs').readFileSync(require('path').join(__dirname, '..', 'src/utils/generatedImages.js'), 'utf8');
  const fn = lib.slice(lib.indexOf('async function generateWithFallback'));
  assert.match(fn.slice(0, 1600), /lease_until=NOW\(\)\+interval '3 minutes'/,
    'a chain must not outlive its claim and let a second worker repeat the work');
  assert.match(fn.slice(0, 1600), /UPDATE generated_image_jobs SET model=\$3/,
    'a picture made by the third model must not be attributed to the first');
});