pediatric-ai-scribe-v3/test/web-search.test.js

const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');

const root = path.join(__dirname, '..');
const read = file => fs.readFileSync(path.join(root, file), 'utf8');

function load(settings, fetchImpl) {
  const module = { exports: {} };
  vm.runInNewContext(read('src/utils/webSearch.js'), {
    module, exports: module.exports, console: { warn() {}, info() {} },
    fetch: fetchImpl, AbortSignal: { timeout: () => null },
    require(name) {
      if (name === '../db/database') return { getSetting: async (k, d) => (k in settings ? settings[k] : d) };
      throw new Error('unexpected import: ' + name);
    }
  });
  return module.exports;
}

const ON = {
  'websearch.enabled': 'true', 'websearch.provider': 'tavily', 'websearch.api_key': 'k', 'websearch.base_url': ''
};

test('web search is off until an administrator turns it on', async () => {
  // This is the only path that sends text outside the building, so the default
  // has to be the safe one and nothing should be able to flip it implicitly.
  const off = load({}, async () => { throw new Error('must not be called'); });
  assert.equal(await off.isAvailable(), false);
  const out = await off.search('anything');
  assert.equal(out.results.length, 0);
  assert.match(out.reason, /disabled/);

  // Enabled but unconfigured is still unavailable — no silent half-state.
  const noKey = load({ 'websearch.enabled': 'true', 'websearch.provider': 'tavily' },
    async () => { throw new Error('must not be called'); });
  assert.equal(await noKey.isAvailable(), false);
  assert.match((await noKey.search('x')).reason, /not configured/);

  // SearXNG needs a URL rather than a key, and is judged on that.
  const searx = load({ 'websearch.enabled': 'true', 'websearch.provider': 'searxng', 'websearch.base_url': 'https://s.example' },
    async () => ({ ok: true, json: async () => ({ results: [] }) }));
  assert.equal(await searx.isAvailable(), true);
});

test('every provider comes back in the same shape', async () => {
  const cases = [
    ['tavily', { results: [{ title: 'T', url: 'https://a', content: 'snippet a' }] }],
    ['serper', { organic: [{ title: 'T', link: 'https://a', snippet: 'snippet a' }] }],
    ['brave', { web: { results: [{ title: 'T', url: 'https://a', description: 'snippet a' }] } }],
    ['searxng', { results: [{ title: 'T', url: 'https://a', content: 'snippet a' }] }]
  ];
  for (const [provider, payload] of cases) {
    const lib = load(
      { 'websearch.enabled': 'true', 'websearch.provider': provider, 'websearch.api_key': 'k', 'websearch.base_url': 'https://s.example' },
      async () => ({ ok: true, status: 200, json: async () => payload }));
    const out = await lib.search('bronchiolitis');
    assert.equal(out.results.length, 1, provider + ' returned a result');
    assert.deepEqual(Object.keys(out.results[0]).sort(), ['snippet', 'title', 'url'],
      provider + ' normalises to one shape');
    assert.equal(out.provider, provider);
  }
});

test('a failed search never fails the generation', async () => {
  // Same contract as corpus retrieval: the resource is written without it, and
  // the caller is told why rather than shown an error page.
  const lib = load(ON, async () => { throw new Error('provider unreachable'); });
  const out = await lib.search('bronchiolitis');
  assert.equal(out.results.length, 0);
  assert.match(out.reason, /provider unreachable/);

  const http = load(ON, async () => ({ ok: false, status: 429, json: async () => ({}) }));
  assert.match((await http.search('x')).reason, /429/);
});

test('results are bounded, and a result with no URL is dropped', async () => {
  const many = Array.from({ length: 40 }, (_, i) => ({ title: 'T' + i, url: 'https://a/' + i, content: 'x'.repeat(4000) }));
  many.push({ title: 'no url', url: '', content: 'y' });
  const lib = load(ON, async () => ({ ok: true, json: async () => ({ results: many }) }));
  const out = await lib.search('bronchiolitis');
  assert.equal(out.results.length, lib.MAX_RESULTS, 'capped');
  assert.ok(out.results.every(r => r.url), 'nothing without a URL');
  assert.ok(out.results.every(r => r.snippet.length <= 1200), 'snippets clipped');
});

test('the tool tells the model when NOT to search', () => {
  const src = read('src/utils/webSearch.js');
  // A model given a search tool will reach for it constantly unless told
  // otherwise, and settled clinical knowledge is what the corpus is for.
  assert.match(src, /Do not use it for settled clinical knowledge/);
  assert.match(src, /One search per resource at most/);
  assert.match(src, /Never send patient details or anything identifying/);

  // One round only, enforced in the route rather than trusted to the model.
  const route = read('src/routes/myResources.js');
  assert.match(route, /toolChoice: 'none'/);
  assert.match(route, /var wantsWeb = \(String\(req\.body\.withWebSearch\) === 'true'/);
  assert.match(route, /&& await webSearch\.isAvailable\(\)/, 'and the server checks, not just the UI');
});

test('the key is masked on read and preserved when left blank', () => {
  const admin = read('src/routes/adminConfig.js');
  // Same handling the OIDC client secret gets.
  assert.match(admin, /out\['websearch\.api_key'\] = '••••••••' \+ out\['websearch\.api_key'\]\.slice\(-4\)/);
  // Changing the provider must not silently wipe a working key.
  assert.match(admin, /if \(key && key\.indexOf\('•'\) === -1\) await db\.setSetting\('websearch\.api_key'/);
  assert.match(admin, /if \(providers\.indexOf\(provider\) === -1\)/, 'and the provider is validated');
});

test('both screens say plainly that a query leaves the network', () => {
  assert.match(read('public/components/admin.html'), /This sends text outside the building/);
  assert.match(read('public/components/admin.html'), /SearXNG is the only\s*\n?\s*option here that you host yourself/);
  const mine = read('public/components/my-resources.html');
  assert.match(mine, /The search query leaves this network/);
  assert.match(mine, /Do not put anything identifying in the topic/);
  // And it is hidden entirely when unavailable, so nobody ticks a box that
  // cannot work.
  assert.match(read('public/js/myResources.js'), /if \(webRow\) webRow\.hidden = !data\.webSearchAvailable;/);
});