pediatric-ai-scribe-v3/test/review-model-vision-guard.test.js

// The slide reviewer is shown rendered images. Setting a text-only model there
// produces a failure on every generation, at request time, long after the
// administrator could have chosen differently — so the choice is checked when
// it is made.
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');

const read = file => fs.readFileSync(path.join(__dirname, '..', file), 'utf8');

// Runs the real liteLLMVisionSupport out of the route file against a mocked
// gateway, so the parsing is exercised rather than read.
function visionSupport(modelInfo, opts = {}) {
  const src = read('src/routes/adminConfig.js');
  const start = src.indexOf('async function liteLLMVisionSupport');
  const end = src.indexOf('\n}\n', start) + 3;
  assert.ok(start > 0 && end > start, 'liteLLMVisionSupport not found');

  const sandbox = {
    liteLLMBaseUrl: () => 'https://gateway.invalid',
    getLiteLLMAdminHeaders: () => ({}),
    liteLLMModelId: m => (m && m.model_name) || '',
    process: { env: opts.noGateway ? {} : { LITELLM_API_BASE: 'https://gateway.invalid' } },
    require: name => {
      assert.equal(name, 'axios');
      return { get: async () => {
        if (opts.unreachable) throw new Error('ECONNREFUSED');
        return { data: { data: modelInfo } };
      } };
    },
    module: { exports: {} }
  };
  vm.runInNewContext(src.slice(start, end) + '\nmodule.exports = liteLLMVisionSupport;', sandbox);
  return sandbox.module.exports;
}

const ROSTER = [
  { model_name: 'text-only-model', model_info: { mode: 'chat', supports_vision: false } },
  { model_name: 'seeing-model', model_info: { mode: 'chat', supports_vision: true } },
  { model_name: 'unknown-model', model_info: { mode: 'chat' } }
];

test('the gateway is read for each of the three answers', async () => {
  const ask = visionSupport(ROSTER);
  assert.equal(await ask('text-only-model'), false, 'stated blind');
  assert.equal(await ask('seeing-model'), true, 'stated sighted');
  assert.equal(await ask('unknown-model'), null, 'the gateway does not say');
  assert.equal(await ask('not-on-the-roster'), null, 'a model it has never heard of');
});

test('an unreachable gateway is not evidence that a model is blind', async () => {
  // Returning false here would refuse a perfectly good model because the
  // network blipped while an administrator was clicking Save.
  assert.equal(await visionSupport(ROSTER, { unreachable: true })('seeing-model'), null);
  assert.equal(await visionSupport(ROSTER, { noGateway: true })('text-only-model'), null);
  assert.equal(await visionSupport(ROSTER)(''), null, 'and an empty model means review is off');
});

test('the vision check is asked of the gateway, not guessed from the model name', () => {
  const route = read('src/routes/adminConfig.js');
  assert.match(route, /async function liteLLMVisionSupport/);
  assert.match(route, /info\.supports_vision === 'boolean' \? info\.supports_vision : null/,
    'three answers: true, false, and "the gateway does not say"');
  assert.doesNotMatch(route, /supports_vision.*test\(|\/vision\/\.test/,
    'never inferred from the model id');
});

test('a model the gateway reports as text-only is refused for slide review', () => {
  const route = read('src/routes/adminConfig.js');
  assert.match(route, /key === 'my_resources\.review_model'/);
  assert.match(route, /if \(canSee === false\)/,
    'strictly false — not falsy, which would also catch null');
  assert.match(route, /is a text-only model, so it cannot look at/);
});

test('an unknown model is allowed, because unknown is not proof of blindness', () => {
  // Most of the roster reports no supports_vision at all. Refusing those would
  // block the working configuration this deployment already runs on.
  const route = read('src/routes/adminConfig.js');
  const guard = route.slice(route.indexOf("key === 'my_resources.review_model'"));
  assert.doesNotMatch(guard.slice(0, 600), /canSee !== true/,
    'an unknown must not be treated as a refusal');
  assert.match(route, /The gateway being unreachable is not evidence about the model/);
});

test('turning slide review off is never blocked by the check', () => {
  const route = read('src/routes/adminConfig.js');
  assert.match(route, /key === 'my_resources\.review_model' && String\(value\)\.trim\(\)/,
    'an empty value means off, and must skip the lookup entirely');
});