pediatric-ai-scribe-v3/test/my-resources-refine.test.js

// Behavioural cover for modifying a resource. The rest of my-resources.test.js
// reads the source; these run the handler, because the three faults they pin
// were all invisible to source reading — the code said the right thing and did
// the wrong one.
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');

const read = file => fs.readFileSync(path.join(__dirname, '..', file), 'utf8');

// Normalised, because that is the shape the route stores. Comparing a
// hand-written deck with a parsed reply would compare normalisation, not content.
const deckBuild = require('../src/utils/deckBuild');
const DECK = Object.assign(
  { title: 'Croup', subtitle: 'Teaching deck', date: '2026-01-01' },
  deckBuild.parse(JSON.stringify({
    slides: [
      { type: 'title', title: 'Croup' },
      { type: 'bullets', title: 'Features', bullets: ['Barking cough', 'Stridor'] }
    ]
  }))
);

function router(t, overrides = {}) {
  const module = { exports: {} };
  const aiCalls = [];
  const replies = overrides.replies ? overrides.replies.slice() : null;
  const reviewCalls = [];
  const reviewOptions = [];
  const updates = [];
  const row = Object.assign({
    id: 5, user_id: 7, title: 'Croup', kind: 'presentation', topic: 'croup',
    markdown: '# Croup\n\n- Barking cough\n- Stridor\n', deck: JSON.stringify(DECK)
  }, overrides.row || {});
  const quiet = { log() {}, error() {}, warn() {}, info() {} };

  const mocks = {
    express: require('express'),
    os: require('os'),
    path: require('path'),
    'fs/promises': require('fs/promises'),
    '../db/database': {
      get: async (sql, params) => {
        if (/^UPDATE user_resources/.test(sql)) {
          updates.push({ sql, params });
          return { id: row.id, title: row.title, updated_at: '2026-01-02T00:00:00Z' };
        }
        return row;
      },
      all: async () => [],
      run: async () => ({}),
      getSetting: async key => {
        if (key === 'clinical_assistant.image_model') return 'synthetic-image';
        if (key === 'my_resources.review_model') return overrides.visionModel || '';
        return '';
      }
    },
    '../middleware/auth': { authMiddleware: (req, res, next) => next() },
    '../middleware/rateLimit': { rateLimit() { return function(req, res, next) { next(); }; } },
    '../utils/ai': {
      callAI: async (messages, options) => {
        aiCalls.push({ messages, options });
        // A scripted sequence when the test needs the model to answer
        // differently on each attempt; the last reply repeats if it runs out.
        const content = replies ? (replies.length > 1 ? replies.shift() : replies[0]) : overrides.reply;
        return { content: content, model: 'synthetic' };
      },
      discoverModels: async () => []
    },
    '../utils/deckBuild': deckBuild,
    '../utils/deckSchema': require('../src/utils/deckSchema'),
    '../utils/deckFormats': require('../src/utils/deckFormats'),
    '../utils/deckSample': require('../src/utils/deckSample'),
    '../utils/deckReview': {
      slideImages: async () => (overrides.slideImages || []),
      review: async (deck, options) => {
        reviewCalls.push(deck);
        reviewOptions.push(options || {});
        return { deck: deck, reviewed: true, reason: 'ok' };
      },
      MAX_SLIDES: 20
    },
    '../utils/documentExport': { FORMATS: { pptx: { mime: 'application/vnd.openxmlformats-officedocument.presentationml.presentation' } }, GOTENBERG: 'http://gotenberg:3000', isSupported: () => false, filename: () => 'x', mimeFor: () => '', render: async () => ({}), renderDeck: async () => Buffer.from('pptx') },
    '../utils/generatedImages': { service: () => ({ get: async () => null, asset: async () => null }), workflows: ['my_resources'] },
    '../utils/learningRetrieval': { retrieve: async () => ({ context: '', sources: [], reason: null }) },
    '../utils/logger': quiet,
    '../utils/nextcloudFiles': { send: async () => '/PediatricScribe/2026-01-01/x.pptx' },
    '../utils/previewPages': { cacheKey: () => 'key', ensure: async () => 1, page: async () => null },
    'crypto': require('crypto'),
    '../utils/metrics': { resourceRefines: { inc() {} }, resourceVocabularyGaps: { inc() {} } },
    '../utils/pubmedSearch': { isAvailable: async () => false, search: async () => ({ results: [] }), formatForPrompt: () => '' },
    '../utils/webSearch': { isAvailable: async () => false, search: async () => ({ results: [] }), formatForPrompt: () => '' },
    '../utils/resourceImages': { tools: [{ name: 'draw' }], dispatch: async x => x, guidance: () => 'GUIDANCE.', requestedCount: () => 0, MAX_IMAGES: 6 }
  };

  vm.runInNewContext(read('src/routes/myResources.js'), {
    module, exports: module.exports, console: quiet, Buffer, JSON, Date, Math,
    process: { env: {} }, setTimeout() {},
    require(name) { assert.ok(Object.hasOwn(mocks, name), 'Unexpected import: ' + name); return mocks[name]; }
  });

  async function request(method, routePath, body) {
    const layer = module.exports.stack.find(l => l.route && l.route.path === routePath && l.route.methods[method]);
    assert.ok(layer, 'route missing: ' + method + ' ' + routePath);
    const stack = layer.route.stack;
    const handler = stack[stack.length - 1].handle;
    const res = {
      statusCode: 200, body: null,
      status(code) { this.statusCode = code; return this; },
      json(payload) { this.body = payload; return this; },
      setHeader() {}, send() { return this; }
    };
    await handler({ body, params: { id: '5' }, query: {}, user: { id: 7 } }, res);
    return res;
  }
  // Generation is queued as a job and written in the background; the tests
  // that care about what the writing does call its body directly.
  async function generate(body) {
    return module.exports.generateResource(7, body);
  }
  // Modifying is queued as a job too, so the tests that care what the writing
  // does call its body directly, exactly as the generation tests do. The reply
  // is shaped like the old synchronous one so the assertions stay about
  // behaviour rather than about how the work is scheduled.
  async function refine(body, id) {
    try {
      return { statusCode: 200, body: await module.exports.refineResource({ id: 7 }, id || 5, body) };
    } catch (err) {
      return { statusCode: err.statusCode || 500, body: { error: err.message } };
    }
  }
  return { request, generate, refine, aiCalls, updates, reviewCalls, reviewOptions };
}

test('modifying a deck asks for a deck, even when illustration is on', async () => {
  // The fault: existingDeck was declared below the branch that read it, so var
  // hoisting made it undefined there. Illustration on a deck therefore appended
  // the markdown instruction — "Returning the markdown is still required" —
  // to a prompt whose body asked for deck JSON. The model was told to produce
  // two different artifacts at once and the reply parsed as neither, so every
  // modification of a presentation with illustration ticked failed with 502.
  const r = router(null, {
    reply: JSON.stringify({ slides: [DECK.slides[0], { type: 'bullets', title: 'Features', bullets: ['Barking cough', 'Stridor', 'Hoarse voice'] }] })
  });
  const res = await r.refine( {
    instructions: 'add a third feature', withImages: 'true'
  });

  assert.equal(res.statusCode, 200, 'a deck modification with images on must not 502');
  const prompt = r.aiCalls[0].messages[0].content;
  assert.match(prompt, /Revise the following slide deck/);
  assert.match(prompt, /Add "image_prompt" to the slides/, 'the deck illustration instruction');
  assert.doesNotMatch(prompt, /Returning the markdown is still required/,
    'the markdown instruction must never reach a deck prompt');
  assert.doesNotMatch(prompt, /An illustration tool is available/,
    'a deck declares its figures; it is not given the tool');
  assert.equal(r.aiCalls[0].options.tools, undefined, 'and no tool schema is attached');
});

test('a modification the model returned unchanged says so instead of claiming success', async () => {
  // Answering "Applied" for a reply identical to the original sent people off
  // to download the same file and conclude the feature was broken. It was
  // logged server-side, where the person who could reword the instruction
  // could not see it.
  const r = router(null, { reply: JSON.stringify({ slides: DECK.slides }) });
  const res = await r.refine( { instructions: 'make it better' });

  assert.equal(res.statusCode, 200);
  assert.equal(res.body.unchanged, true, 'the caller is told the deck came back identical');
});

test('a modification that did change the deck reports itself as changed', async () => {
  const r = router(null, {
    reply: JSON.stringify({ slides: [DECK.slides[0], { type: 'bullets', title: 'Features', bullets: ['Barking cough', 'Stridor', 'Hoarse voice'] }] })
  });
  const res = await r.refine( { instructions: 'add a third feature' });

  assert.equal(res.body.unchanged, false);
  assert.equal(r.updates.length, 1, 'and the row is written');
  assert.match(r.updates[0].sql, /updated_at = NOW\(\)/);
});

test('a presentation with no deck still modifies, through the markdown path', async () => {
  const r = router(null, { row: { deck: null }, reply: '# Croup\n\n- Barking cough\n- Stridor\n- Hoarse voice\n' });
  const res = await r.refine( {
    instructions: 'add a third feature', withImages: 'true'
  });

  assert.equal(res.statusCode, 200);
  assert.equal(res.body.unchanged, false);
  const prompt = r.aiCalls[0].messages[0].content;
  assert.match(prompt, /Revise the following Pandoc markdown/);
  assert.match(prompt, /An illustration tool is available/, 'markdown has nowhere to declare a figure');
});

test('the library says which presentations carry a deck, and the row shows when it was modified', () => {
  // Both halves of "modify does nothing": the list read created_at, so a
  // modification that did apply left the visible timestamp untouched, and
  // nothing distinguished a real deck from a flat one.
  const route = read('src/routes/myResources.js');
  assert.match(route, /AS has_deck/);

  const ui = read('public/js/myResources.js');
  assert.match(ui, /row\.updated_at \? new Date\(row\.updated_at\)/);
  assert.match(ui, /edited \? 'modified '/);
  assert.match(ui, /has_deck === false \? ' · plain text, no slide layout' : ''/);
  // A modification reports from the job list now that it runs in the background,
  // so "it came back unchanged" still reaches the person who asked for it.
  assert.match(ui, /if \(r\.unchanged\) \{/);
});

// ── Generating a deck ──────────────────────────────────────────────────────

const GOOD_DECK = JSON.stringify({
  slides: [
    { type: 'title', title: 'Croup' },
    { type: 'bullets', title: 'Features', bullets: ['Barking cough', 'Stridor'] }
  ]
});

test('a deck the model fumbles once is asked for a second time, not abandoned', async () => {
  // Falling straight back to markdown after one unlucky reply produced a
  // materially worse artifact: plain slides inferred from markdown instead of
  // the layouts the model chose. Measured on the stored library, this happened
  // once in eight generations.
  const r = router(null, { replies: ['Sorry, I cannot do that.', GOOD_DECK] });
  const out = await r.generate({ topic: 'croup', kind: 'presentation' });

  assert.equal(r.aiCalls.length, 2, 'the deck is asked for twice before giving up');
  assert.match(r.aiCalls[1].messages[0].content, /teaching presentation/,
    'the retry is the same deck prompt, not the weaker markdown one');
  assert.equal(out.deckFallback, null, 'and the retry succeeded, so nothing fell back');
});

test('a deck that fails twice falls back to markdown and says why', async () => {
  const r = router(null, { replies: ['Sorry, I cannot help with that.', 'I am unable to comply.', '# Croup\n\n- Barking cough\n'] });
  const out = await r.generate({ topic: 'croup', kind: 'presentation' });

  assert.equal(r.aiCalls.length, 3, 'two deck attempts, then markdown');
  assert.equal(out.deckFallback, 'the reply was not a deck',
    'the caller is told it came out plain, and why');
});

test('a truncated deck reply is named as truncated, not as the wrong shape', async () => {
  // The three causes want different fixes — a smaller deck, a different model,
  // a reworded topic — so the message distinguishes them. Truncation is only
  // claimed for a reply that began as JSON: an apology in prose does not end in
  // "}" either, and naming that "cut short" points at the wrong fix.
  const cut = GOOD_DECK.slice(0, GOOD_DECK.length - 30);
  const r = router(null, { replies: [cut, cut, '# Croup\n'] });
  const out = await r.generate({ topic: 'croup', kind: 'presentation' });

  assert.match(out.deckFallback, /cut short at \d+ characters/);
});

test('a deck that parses first time is never asked for twice', async () => {
  const r = router(null, { replies: [GOOD_DECK] });
  const out = await r.generate({ topic: 'croup', kind: 'presentation' });

  assert.equal(r.aiCalls.length, 1, 'the retry costs a call and must only happen on failure');
  assert.equal(out.deckFallback, null);
});

// ── Thinking off ───────────────────────────────────────────────────────────

test('the writing and the review of it are both asked without thinking', async () => {
  // DeepSeek thinks by default, and a deck is a writing task: one generation
  // spent a whole 16,000-token budget reasoning and wrote nothing, and the
  // 2,000-token reviews of that same deck starved four times over. The flag is
  // DeepSeek's own field, sent by the model wrapper; this pins that the review
  // inherits the writer's rule instead of falling back to the provider default.
  const r = router(null, { visionModel: 'synthetic-vision', replies: [GOOD_DECK] });
  const out = await r.generate({ topic: 'croup', kind: 'presentation' });

  assert.equal(r.aiCalls[0].options.reasoningEffort, 'none', 'the deck is written without thinking');
  assert.equal(out.review.applied, true, 'and the review ran');
  assert.equal(r.reviewOptions[0].reasoningEffort, 'none', 'the review is asked the same way');
});

test('modifying a resource is writing too, and is asked without thinking', async () => {
  // A revision restates the whole resource — 16,000 tokens for a deck — and on
  // a thirteen-slide deck it reasoned for a minute and a half before writing a
  // word, which is past the point a browser waits for the request.
  const r = router(null, { reply: JSON.stringify({ slides: DECK.slides }) });
  const res = await r.refine( { instructions: 'make it better' });

  assert.equal(res.statusCode, 200);
  assert.equal(r.aiCalls[0].options.reasoningEffort, 'none');
});

// ── Generation is a job ────────────────────────────────────────────────────

test('generate records a job and answers at once; the writing happens after the reply', async () => {
  // A deck takes minutes and a browser holding one request open that long
  // gives up on its own (Firefox at five minutes: "NetworkError when
  // attempting to fetch resource"), while the server carried on and saved the
  // deck anyway. So the request only records what was asked.
  const r = router(null, { replies: [GOOD_DECK] });
  const res = await r.request('post', '/my-resources/generate', { topic: 'croup', kind: 'presentation' });

  assert.equal(res.statusCode, 202);
  assert.equal(res.body.success, true);
  assert.ok(res.body.job && res.body.job.id, 'the caller gets the job to poll for');
  assert.equal(r.aiCalls.length, 0, 'nothing has been asked of the model by the time the reply goes out');

  const blank = await r.request('post', '/my-resources/generate', { topic: '   ' });
  assert.equal(blank.statusCode, 400, 'a blank topic is refused before a job exists');
});

test('a job records its outcome, and a failure is named without leaking internals', async () => {
  const route = fs.readFileSync(path.join(__dirname, '..', 'src/routes/myResources.js'), 'utf8');
  const job = route.slice(route.indexOf('async function runResourceJob('), route.indexOf('async function recoverResourceJobs('));
  assert.match(job, /SET status = 'running', started_at = NOW\(\)/);
  assert.match(job, /SET status = 'done', finished_at = NOW\(\), resource_id = \?, result = \?/);
  assert.match(job, /SET status = 'failed', finished_at = NOW\(\), error = \?/);
  assert.match(job, /err\.statusCode \? err\.message : 'Generation failed'/,
    'only a message written for the user reaches the job row');
  // A restart strands queued and running rows; they are marked so on boot
  // rather than spinning for ever in the list.
  assert.match(route, /The server restarted while this was being written\. Generate it again\./);
  assert.match(route, /setTimeout\(recoverResourceJobs, 15000\)/);
});

// ── Modifying with sight ───────────────────────────────────────────────────

const TWO_PNGS = [
  { mimeType: 'image/png', dataBase64: 'aW1hZ2Ux' },
  { mimeType: 'image/png', dataBase64: 'aW1hZ2Uy' }
];

const EDITED = JSON.stringify({
  slides: [
    { type: 'title', title: 'Croup' },
    { type: 'bullets', title: 'Features', bullets: ['Barking cough', 'Stridor', 'Hoarse voice'] }
  ]
});

test('modifying a deck shows the model what the deck currently looks like', async () => {
  // Most of what people ask for while modifying is about the rendered page —
  // "that slide is crowded", "the diagram is in the wrong place" — and none of
  // it is answerable from the JSON alone.
  const r = router(null, { visionModel: 'seeing-model', slideImages: TWO_PNGS, replies: [EDITED] });
  const res = await r.refine( {
    instructions: 'slide 2 looks crowded, split it'
  });

  assert.equal(res.statusCode, 200);
  assert.equal(r.aiCalls[0].options.images, TWO_PNGS, 'the rendered slides are attached');
  assert.equal(r.aiCalls[0].options.model, 'seeing-model', 'and the model that can see does the editing');
  assert.match(r.aiCalls[0].messages[0].content, /You can see the deck as it renders now: 2 images/);
  assert.match(r.aiCalls[0].messages[0].content, /first image is slide index 0/,
    'the images must be anchored to the JSON or a slide reference means nothing');
  assert.equal(res.body.saw, 2);
});

test('the edited deck is rendered again and checked', async () => {
  // The edit was made against how the deck looked *before* it. A slide that
  // gained two bullets only overflows once it is rendered again.
  const r = router(null, { visionModel: 'seeing-model', slideImages: TWO_PNGS, replies: [EDITED] });
  const res = await r.refine( { instructions: 'add a feature' });

  assert.equal(r.reviewCalls.length, 1, 'the result goes back past the reviewer');
  assert.equal(res.body.review.applied, true);
});

test('with no vision model configured, modify still works and never renders', async () => {
  // Sight is an upgrade, not a dependency. Nothing here may become a new way
  // for a modification to fail.
  const r = router(null, { replies: [EDITED] });
  const res = await r.refine( { instructions: 'add a feature' });

  assert.equal(res.statusCode, 200);
  assert.equal(res.body.saw, 0);
  assert.equal(r.aiCalls[0].options.images, undefined);
  assert.equal(r.reviewCalls.length, 0, 'and nothing is rendered or reviewed');
});

test('a render that fails falls through to editing blind rather than failing', async () => {
  // Gotenberg down, LibreOffice wedged, a deck too big: none of them may cost
  // the author their modification.
  const r = router(null, { visionModel: 'seeing-model', slideImages: [], replies: [EDITED] });
  const res = await r.refine( { instructions: 'add a feature' });

  assert.equal(res.statusCode, 200);
  assert.equal(res.body.saw, 0);
  assert.equal(r.aiCalls[0].options.images, undefined);
  assert.notEqual(r.aiCalls[0].options.model, 'seeing-model',
    'no images means no reason to pay for the vision model');
});

test('an echo is still reported as an echo, even when the reviewer moved something', async () => {
  // The reviewer may reposition, which would otherwise make an instruction that
  // achieved nothing look like it had worked.
  const r = router(null, { visionModel: 'seeing-model', slideImages: TWO_PNGS,
                           replies: [JSON.stringify({ slides: DECK.slides })] });
  const res = await r.refine( { instructions: 'make it better' });

  assert.equal(res.body.unchanged, true, 'judged on the model edit, before the reviewer ran');
});