pediatric-ai-scribe-v3/test/web-search.test.js

const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');

const root = path.join(__dirname, '..');
const read = file => fs.readFileSync(path.join(root, file), 'utf8');

function load(settings, fetchImpl) {
  const module = { exports: {} };
  vm.runInNewContext(read('src/utils/webSearch.js'), {
    module, exports: module.exports, console: { warn() {}, info() {} },
    fetch: fetchImpl, AbortSignal: { timeout: () => null },
    require(name) {
      if (name === '../db/database') return { getSetting: async (k, d) => (k in settings ? settings[k] : d) };
      throw new Error('unexpected import: ' + name);
    }
  });
  return module.exports;
}

const ON = {
  'websearch.enabled': 'true', 'websearch.provider': 'tavily', 'websearch.api_key': 'k', 'websearch.base_url': ''
};

test('web search is off until an administrator turns it on', async () => {
  // This is the only path that sends text outside the building, so the default
  // has to be the safe one and nothing should be able to flip it implicitly.
  const off = load({}, async () => { throw new Error('must not be called'); });
  assert.equal(await off.isAvailable(), false);
  const out = await off.search('anything');
  assert.equal(out.results.length, 0);
  assert.match(out.reason, /disabled/);

  // Enabled but unconfigured is still unavailable — no silent half-state.
  const noKey = load({ 'websearch.enabled': 'true', 'websearch.provider': 'tavily' },
    async () => { throw new Error('must not be called'); });
  assert.equal(await noKey.isAvailable(), false);
  assert.match((await noKey.search('x')).reason, /not configured/);

  // SearXNG needs a URL rather than a key, and is judged on that.
  const searx = load({ 'websearch.enabled': 'true', 'websearch.provider': 'searxng', 'websearch.base_url': 'https://s.example' },
    async () => ({ ok: true, json: async () => ({ results: [] }) }));
  assert.equal(await searx.isAvailable(), true);
});

test('every provider comes back in the same shape', async () => {
  const cases = [
    ['tavily', { results: [{ title: 'T', url: 'https://a', content: 'snippet a' }] }],
    ['serper', { organic: [{ title: 'T', link: 'https://a', snippet: 'snippet a' }] }],
    ['brave', { web: { results: [{ title: 'T', url: 'https://a', description: 'snippet a' }] } }],
    ['exa', { results: [{ title: 'T', url: 'https://a', text: 'snippet a' }] }],
    ['searxng', { results: [{ title: 'T', url: 'https://a', content: 'snippet a' }] }]
  ];
  for (const [provider, payload] of cases) {
    const lib = load(
      { 'websearch.enabled': 'true', 'websearch.provider': provider, 'websearch.api_key': 'k', 'websearch.base_url': 'https://s.example' },
      async () => ({ ok: true, status: 200, json: async () => payload }));
    const out = await lib.search('bronchiolitis');
    assert.equal(out.results.length, 1, provider + ' returned a result');
    assert.deepEqual(Object.keys(out.results[0]).sort(), ['snippet', 'title', 'url'],
      provider + ' normalises to one shape');
    assert.equal(out.provider, provider);
  }
});

test('a failed search never fails the generation', async () => {
  // Same contract as corpus retrieval: the resource is written without it, and
  // the caller is told why rather than shown an error page.
  const lib = load(ON, async () => { throw new Error('provider unreachable'); });
  const out = await lib.search('bronchiolitis');
  assert.equal(out.results.length, 0);
  assert.match(out.reason, /provider unreachable/);

  const http = load(ON, async () => ({ ok: false, status: 429, json: async () => ({}) }));
  assert.match((await http.search('x')).reason, /429/);
});

test('results are bounded, and a result with no URL is dropped', async () => {
  const many = Array.from({ length: 40 }, (_, i) => ({ title: 'T' + i, url: 'https://a/' + i, content: 'x'.repeat(4000) }));
  many.push({ title: 'no url', url: '', content: 'y' });
  const lib = load(ON, async () => ({ ok: true, json: async () => ({ results: many }) }));
  const out = await lib.search('bronchiolitis');
  assert.equal(out.results.length, lib.MAX_RESULTS, 'capped');
  assert.ok(out.results.every(r => r.url), 'nothing without a URL');
  assert.ok(out.results.every(r => r.snippet.length <= 1200), 'snippets clipped');
});

test('searching is the route\u2019s job, not something the model is asked to do', () => {
  // This was a tool first. Tested live against a question explicitly about
  // recent trials, the model never called it \u2014 with or without corpus
  // grounding, and no matter how the description was worded, because the prompt
  // ends "Output ONLY Pandoc markdown" and a model told to output only markdown
  // does not emit a tool call. Calling callAI with the tool directly produced a
  // correct pubmed_search call, so the plumbing was never the problem.
  //
  // The search only ever needed the topic, and the route knows the topic before
  // it calls the model. So both searches run up front and their results go into
  // the prompt as findings, the same way corpus excerpts do.
  const route = read('src/routes/myResources.js');
  assert.match(route, /var wantsWeb = \(String\(body\.withWebSearch\) === 'true'/);
  assert.match(route, /&& await webSearch\.isAvailable\(\)/, 'and the server checks, not just the UI');
  assert.match(route, /var pages = await webSearch\.search\(keywords\);/);
  assert.match(route, /var papers = await pubmedSearch\.search\(keywords\);/);
  // One function, so generating and modifying cannot drift into offering
  // different sources or searching them differently.
  assert.match(route, /async function gatherSources\(subject, body, keywords\)/);
  assert.match(route, /var sources = await gatherSources\(topic, req\.body\);/, 'generate');
  assert.match(route, /var sources = await gatherSources\(subject, req\.body, existing\.topic \|\| instructions\);/,
    'and modify, whose library search gets the instruction for context and whose keyword searches do not');

  // Declared before they are used. They were not, once: `var` hoisting made
  // wantsPubmed undefined at the point of the test, so the block never ran and
  // said nothing about it.
  assert.ok(route.indexOf('var wantsPubmed =') < route.indexOf('if (wantsPubmed)'),
    'declared above the branch that reads it');
  assert.ok(route.indexOf('var wantsWeb =') < route.indexOf('if (wantsWeb)'));

  // Neither search may fail a generation, so what happened is reported back
  // instead: how many results, and why there were none.
  assert.match(route, /searches\.push\(\{ tool: 'pubmed_search', query: papers\.query \|\| keywords/);
  assert.match(route, /searches\.push\(\{ tool: 'web_search', query: keywords, count: pages\.results\.length, reason: pages\.reason \}\);/);
  assert.match(route, /searches: searches/);

  // And neither library still advertises itself as a tool.
  assert.doesNotMatch(read('src/utils/webSearch.js'), /name: 'web_search'/);
  assert.doesNotMatch(read('src/utils/pubmedSearch.js'), /name: 'pubmed_search'/);
});

test('a PubMed query that finds nothing is narrowed rather than given up on', () => {
  const src = read('src/utils/pubmedSearch.js');
  // PubMed ANDs every mapped term, so one unrecognised word zeroes the query.
  // Measured against the live API: "febrile seizures" → 6 results, "febrile
  // seizures in under-fives" → 0, and "the anatomy of croup: subglottic
  // narrowing and the steeple sign" → 0 until it was narrowed to "anatomy
  // croup", which returns 6.
  assert.match(src, /function candidates\(text\)/);
  assert.match(src, /var tries = candidates\(text\);/);
  // Longest first, so the most specific query that works is the one used.
  assert.match(src, /for \(var n = Math\.min\(words\.length, 4\); n >= 2; n--\)/);
  // A subtitle is not a subject.
  assert.match(src, /split\(\/\[:\\u2014\\u2013\]\|\\s-\\s\/\)\[0\]/);

  // Three esearch calls back to back trips NCBI's three-a-second limit without
  // a key, which turned a working search into a 429 — measured on this path.
  assert.match(src, /var gap = s\.apiKey \? 120 : 380;/);
  assert.match(src, /if \(t\) await new Promise/, 'and the first attempt waits for nothing');

  // The screen reports what was searched, so it has to be the query that
  // actually found the results.
  assert.match(src, /return \{ results: results, reason: null, query: used \};/);
  assert.match(read('src/routes/myResources.js'), /query: papers\.query \|\| keywords/);
});

test('a search that comes back empty is not an invitation to invent citations', () => {
  const route = read('src/routes/myResources.js');
  // Asked to search, found nothing, still asked for PMIDs: the model supplies
  // them from memory, and a fabricated PMID looks exactly like a real one.
  assert.match(route, /searchedAndFoundNothing: searches\.length > 0 && !literature && !webFindings/);
  // Said on both paths: refining and generating.
  assert.equal((route.match(/Do not invent a citation, a PMID/g) || []).length, 2);
});

test('the key is masked on read and preserved when left blank', () => {
  const admin = read('src/routes/adminConfig.js');
  // Same handling the OIDC client secret gets.
  // Both keys, one rule: never send a key back, enough tail to recognise it.
  assert.match(admin, /\['websearch\.api_key', 'pubmed\.api_key'\]\.forEach/);
  assert.match(admin, /if \(out\[k\]\) out\[k\] = '••••••••' \+ out\[k\]\.slice\(-4\);/);
  // Changing the provider must not silently wipe a working key.
  assert.match(admin, /if \(key && key\.indexOf\('•'\) === -1\) await db\.setSetting\('websearch\.api_key'/);
  assert.match(admin, /if \(pmKey && pmKey\.indexOf\('•'\) === -1\) await db\.setSetting\('pubmed\.api_key'/);
  assert.match(admin, /if \(providers\.indexOf\(provider\) === -1\)/, 'and the provider is validated');
});

test('both screens say plainly that a query leaves the network', () => {
  assert.match(read('public/components/admin.html'), /This sends text outside the building/);
  assert.match(read('public/components/admin.html'), /SearXNG is the only\s*\n?\s*option here that you host yourself/);
  const mine = read('public/components/my-resources.html');
  assert.match(mine, /The query leaves this network/);
  assert.match(mine, /keep the topic non-identifying/);
  assert.match(mine, /The query goes to NCBI/);
  // And it is hidden entirely when unavailable, so nobody ticks a box that
  // cannot work.
  // Both option groups are hidden from the same answer, so Generate and Modify
  // cannot end up offering different things.
  const js = read('public/js/myResources.js');
  assert.match(js, /\['mr-web-row', 'web'\]/);
  assert.match(js, /\['mr-modify-web-row', 'web'\]/);
  assert.match(js, /\['mr-pubmed-row', 'pubmed'\]/);
  assert.match(js, /\['mr-modify-pubmed-row', 'pubmed'\]/);
  assert.match(js, /row\.hidden = !available\[pair\[1\]\]/);
});

// ---- Exa -------------------------------------------------------------------
// Embeddings search rather than keywords, which suits a clinical question asked
// as a question. It is the only provider that can return the page text in the
// same call, and the snippet is the part the model reads.

test('Exa asks for the text extract in the search call', async () => {
  // Without contents, every result would need a second fetch to be useful.
  let sent = null;
  const lib = load({ 'websearch.enabled': 'true', 'websearch.provider': 'exa', 'websearch.api_key': 'k' },
    async (url, options) => { sent = { url, options }; return { ok: true, status: 200, json: async () => ({ results: [] }) }; });
  await lib.search('does dexamethasone help croup');
  assert.equal(sent.url, 'https://api.exa.ai/search');
  const body = JSON.parse(sent.options.body);
  assert.equal(body.query, 'does dexamethasone help croup');
  assert.ok(body.contents && body.contents.text, 'no text extract requested');
  // 'auto', not 'neural': pinning neural makes it worse at the keyword-shaped
  // queries the other providers handle well.
  assert.equal(body.type, 'auto');
  assert.equal(sent.options.headers['x-api-key'], 'k', 'Exa authenticates with x-api-key, not a bearer token');
});

test('Exa falls back through its snippet fields rather than returning nothing', async () => {
  const lib = load({ 'websearch.enabled': 'true', 'websearch.provider': 'exa', 'websearch.api_key': 'k' },
    async () => ({ ok: true, status: 200, json: async () => ({ results: [
      { title: 'A', url: 'https://a', text: 'from text' },
      { title: 'B', url: 'https://b', summary: 'from summary' },
      { title: 'C', url: 'https://c' }
    ] }) }));
  const out = await lib.search('x');
  assert.deepEqual(out.results.map(r => r.snippet), ['from text', 'from summary', '']);
});

test('Exa needs a key, like every provider but SearXNG', async () => {
  const lib = load({ 'websearch.enabled': 'true', 'websearch.provider': 'exa' },
    async () => { throw new Error('must not be called'); });
  assert.equal(await lib.isAvailable(), false);
});

test('the admin can choose it, and the server accepts what the admin can choose', async () => {
  // The dropdown and the route validate against the same list; a provider in
  // one and not the other is a setting that saves and then does nothing, or an
  // option that cannot be saved at all.
  const lib = load(ON, async () => ({ ok: true, status: 200, json: async () => ({ results: [] }) }));
  const markup = read('public/components/admin.html');
  for (const provider of lib.PROVIDERS) {
    assert.match(markup, new RegExp('<option value="' + provider + '"'),
      provider + ' is accepted by the server but not offered in the admin');
  }
  const offered = [...markup.matchAll(/<option value="(\w+)">[^<]*(?:Tavily|Serper|Brave|Exa|SearXNG)/g)].map(m => m[1]);
  for (const provider of offered) {
    assert.ok(lib.PROVIDERS.includes(provider), provider + ' is offered in the admin but rejected by the server');
  }
});