// The slide reviewer is shown rendered images. Setting a text-only model there // produces a failure on every generation, at request time, long after the // administrator could have chosen differently — so the choice is checked when // it is made. const test = require('node:test'); const assert = require('node:assert/strict'); const fs = require('node:fs'); const path = require('node:path'); const vm = require('node:vm'); const read = file => fs.readFileSync(path.join(__dirname, '..', file), 'utf8'); // Runs the real liteLLMVisionSupport out of the route file against a mocked // gateway, so the parsing is exercised rather than read. function visionSupport(modelInfo, opts = {}) { const src = read('src/routes/adminConfig.js'); const start = src.indexOf('async function liteLLMVisionSupport'); const end = src.indexOf('\n}\n', start) + 3; assert.ok(start > 0 && end > start, 'liteLLMVisionSupport not found'); const sandbox = { liteLLMBaseUrl: () => 'https://gateway.invalid', getLiteLLMAdminHeaders: () => ({}), liteLLMModelId: m => (m && m.model_name) || '', process: { env: opts.noGateway ? {} : { LITELLM_API_BASE: 'https://gateway.invalid' } }, require: name => { assert.equal(name, 'axios'); return { get: async () => { if (opts.unreachable) throw new Error('ECONNREFUSED'); return { data: { data: modelInfo } }; } }; }, module: { exports: {} } }; vm.runInNewContext(src.slice(start, end) + '\nmodule.exports = liteLLMVisionSupport;', sandbox); return sandbox.module.exports; } const ROSTER = [ { model_name: 'text-only-model', model_info: { mode: 'chat', supports_vision: false } }, { model_name: 'seeing-model', model_info: { mode: 'chat', supports_vision: true } }, { model_name: 'unknown-model', model_info: { mode: 'chat' } } ]; test('the gateway is read for each of the three answers', async () => { const ask = visionSupport(ROSTER); assert.equal(await ask('text-only-model'), false, 'stated blind'); assert.equal(await ask('seeing-model'), true, 'stated sighted'); assert.equal(await ask('unknown-model'), null, 'the gateway does not say'); assert.equal(await ask('not-on-the-roster'), null, 'a model it has never heard of'); }); test('an unreachable gateway is not evidence that a model is blind', async () => { // Returning false here would refuse a perfectly good model because the // network blipped while an administrator was clicking Save. assert.equal(await visionSupport(ROSTER, { unreachable: true })('seeing-model'), null); assert.equal(await visionSupport(ROSTER, { noGateway: true })('text-only-model'), null); assert.equal(await visionSupport(ROSTER)(''), null, 'and an empty model means review is off'); }); test('the vision check is asked of the gateway, not guessed from the model name', () => { const route = read('src/routes/adminConfig.js'); assert.match(route, /async function liteLLMVisionSupport/); assert.match(route, /info\.supports_vision === 'boolean' \? info\.supports_vision : null/, 'three answers: true, false, and "the gateway does not say"'); assert.doesNotMatch(route, /supports_vision.*test\(|\/vision\/\.test/, 'never inferred from the model id'); }); test('a model the gateway reports as text-only is refused for slide review', () => { const route = read('src/routes/adminConfig.js'); assert.match(route, /key === 'my_resources\.review_model'/); assert.match(route, /if \(canSee === false\)/, 'strictly false — not falsy, which would also catch null'); assert.match(route, /is a text-only model, so it cannot look at/); }); test('an unknown model is allowed, because unknown is not proof of blindness', () => { // Most of the roster reports no supports_vision at all. Refusing those would // block the working configuration this deployment already runs on. const route = read('src/routes/adminConfig.js'); const guard = route.slice(route.indexOf("key === 'my_resources.review_model'")); assert.doesNotMatch(guard.slice(0, 600), /canSee !== true/, 'an unknown must not be treated as a refusal'); assert.match(route, /The gateway being unreachable is not evidence about the model/); }); test('turning slide review off is never blocked by the check', () => { const route = read('src/routes/adminConfig.js'); assert.match(route, /key === 'my_resources\.review_model' && String\(value\)\.trim\(\)/, 'an empty value means off, and must skip the lookup entirely'); });