pediatric-ai-scribe-v3/test/web-search.test.js

const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');

const root = path.join(__dirname, '..');
const read = file => fs.readFileSync(path.join(root, file), 'utf8');

function load(settings, fetchImpl) {
  const module = { exports: {} };
  vm.runInNewContext(read('src/utils/webSearch.js'), {
    module, exports: module.exports, console: { warn() {}, info() {} },
    fetch: fetchImpl, AbortSignal: { timeout: () => null },
    require(name) {
      if (name === '../db/database') return { getSetting: async (k, d) => (k in settings ? settings[k] : d) };
      throw new Error('unexpected import: ' + name);
    }
  });
  return module.exports;
}

const ON = {
  'websearch.enabled': 'true', 'websearch.provider': 'tavily', 'websearch.api_key': 'k', 'websearch.base_url': ''
};

test('web search is off until an administrator turns it on', async () => {
  // This is the only path that sends text outside the building, so the default
  // has to be the safe one and nothing should be able to flip it implicitly.
  const off = load({}, async () => { throw new Error('must not be called'); });
  assert.equal(await off.isAvailable(), false);
  const out = await off.search('anything');
  assert.equal(out.results.length, 0);
  assert.match(out.reason, /disabled/);

  // Enabled but unconfigured is still unavailable — no silent half-state.
  const noKey = load({ 'websearch.enabled': 'true', 'websearch.provider': 'tavily' },
    async () => { throw new Error('must not be called'); });
  assert.equal(await noKey.isAvailable(), false);
  assert.match((await noKey.search('x')).reason, /not configured/);

  // SearXNG needs a URL rather than a key, and is judged on that.
  const searx = load({ 'websearch.enabled': 'true', 'websearch.provider': 'searxng', 'websearch.base_url': 'https://s.example' },
    async () => ({ ok: true, json: async () => ({ results: [] }) }));
  assert.equal(await searx.isAvailable(), true);
});

test('every provider comes back in the same shape', async () => {
  const cases = [
    ['tavily', { results: [{ title: 'T', url: 'https://a', content: 'snippet a' }] }],
    ['serper', { organic: [{ title: 'T', link: 'https://a', snippet: 'snippet a' }] }],
    ['brave', { web: { results: [{ title: 'T', url: 'https://a', description: 'snippet a' }] } }],
    ['searxng', { results: [{ title: 'T', url: 'https://a', content: 'snippet a' }] }]
  ];
  for (const [provider, payload] of cases) {
    const lib = load(
      { 'websearch.enabled': 'true', 'websearch.provider': provider, 'websearch.api_key': 'k', 'websearch.base_url': 'https://s.example' },
      async () => ({ ok: true, status: 200, json: async () => payload }));
    const out = await lib.search('bronchiolitis');
    assert.equal(out.results.length, 1, provider + ' returned a result');
    assert.deepEqual(Object.keys(out.results[0]).sort(), ['snippet', 'title', 'url'],
      provider + ' normalises to one shape');
    assert.equal(out.provider, provider);
  }
});

test('a failed search never fails the generation', async () => {
  // Same contract as corpus retrieval: the resource is written without it, and
  // the caller is told why rather than shown an error page.
  const lib = load(ON, async () => { throw new Error('provider unreachable'); });
  const out = await lib.search('bronchiolitis');
  assert.equal(out.results.length, 0);
  assert.match(out.reason, /provider unreachable/);

  const http = load(ON, async () => ({ ok: false, status: 429, json: async () => ({}) }));
  assert.match((await http.search('x')).reason, /429/);
});

test('results are bounded, and a result with no URL is dropped', async () => {
  const many = Array.from({ length: 40 }, (_, i) => ({ title: 'T' + i, url: 'https://a/' + i, content: 'x'.repeat(4000) }));
  many.push({ title: 'no url', url: '', content: 'y' });
  const lib = load(ON, async () => ({ ok: true, json: async () => ({ results: many }) }));
  const out = await lib.search('bronchiolitis');
  assert.equal(out.results.length, lib.MAX_RESULTS, 'capped');
  assert.ok(out.results.every(r => r.url), 'nothing without a URL');
  assert.ok(out.results.every(r => r.snippet.length <= 1200), 'snippets clipped');
});

test('searching is the route\u2019s job, not something the model is asked to do', () => {
  // This was a tool first. Tested live against a question explicitly about
  // recent trials, the model never called it \u2014 with or without corpus
  // grounding, and no matter how the description was worded, because the prompt
  // ends "Output ONLY Pandoc markdown" and a model told to output only markdown
  // does not emit a tool call. Calling callAI with the tool directly produced a
  // correct pubmed_search call, so the plumbing was never the problem.
  //
  // The search only ever needed the topic, and the route knows the topic before
  // it calls the model. So both searches run up front and their results go into
  // the prompt as findings, the same way corpus excerpts do.
  const route = read('src/routes/myResources.js');
  assert.match(route, /var wantsWeb = \(String\(req\.body\.withWebSearch\) === 'true'/);
  assert.match(route, /&& await webSearch\.isAvailable\(\)/, 'and the server checks, not just the UI');
  assert.match(route, /var pages = await webSearch\.search\(topic\);/);
  assert.match(route, /var papers = await pubmedSearch\.search\(topic\);/);

  // Declared before they are used. They were not, once: `var` hoisting made
  // wantsPubmed undefined at the point of the test, so the block never ran and
  // said nothing about it.
  assert.ok(route.indexOf('var wantsPubmed =') < route.indexOf('if (wantsPubmed)'),
    'declared above the branch that reads it');
  assert.ok(route.indexOf('var wantsWeb =') < route.indexOf('if (wantsWeb)'));

  // Neither search may fail a generation, so what happened is reported back
  // instead: how many results, and why there were none.
  assert.match(route, /searches\.push\(\{ tool: 'pubmed_search', query: topic, count: papers\.results\.length, reason: papers\.reason \}\);/);
  assert.match(route, /searches\.push\(\{ tool: 'web_search', query: topic, count: pages\.results\.length, reason: pages\.reason \}\);/);
  assert.match(route, /searches: searches/);

  // And neither library still advertises itself as a tool.
  assert.doesNotMatch(read('src/utils/webSearch.js'), /name: 'web_search'/);
  assert.doesNotMatch(read('src/utils/pubmedSearch.js'), /name: 'pubmed_search'/);
});

test('the key is masked on read and preserved when left blank', () => {
  const admin = read('src/routes/adminConfig.js');
  // Same handling the OIDC client secret gets.
  // Both keys, one rule: never send a key back, enough tail to recognise it.
  assert.match(admin, /\['websearch\.api_key', 'pubmed\.api_key'\]\.forEach/);
  assert.match(admin, /if \(out\[k\]\) out\[k\] = '••••••••' \+ out\[k\]\.slice\(-4\);/);
  // Changing the provider must not silently wipe a working key.
  assert.match(admin, /if \(key && key\.indexOf\('•'\) === -1\) await db\.setSetting\('websearch\.api_key'/);
  assert.match(admin, /if \(pmKey && pmKey\.indexOf\('•'\) === -1\) await db\.setSetting\('pubmed\.api_key'/);
  assert.match(admin, /if \(providers\.indexOf\(provider\) === -1\)/, 'and the provider is validated');
});

test('both screens say plainly that a query leaves the network', () => {
  assert.match(read('public/components/admin.html'), /This sends text outside the building/);
  assert.match(read('public/components/admin.html'), /SearXNG is the only\s*\n?\s*option here that you host yourself/);
  const mine = read('public/components/my-resources.html');
  assert.match(mine, /The search query leaves this network/);
  assert.match(mine, /Do not put anything identifying in the topic/);
  // And it is hidden entirely when unavailable, so nobody ticks a box that
  // cannot work.
  assert.match(read('public/js/myResources.js'), /if \(webRow\) webRow\.hidden = !data\.webSearchAvailable;/);
});