diff --git a/public/js/myResources.js b/public/js/myResources.js index 67bca54f..5706cb7c 100644 --- a/public/js/myResources.js +++ b/public/js/myResources.js @@ -187,6 +187,15 @@ : 'Saved. Not grounded' + (g.reason ? ' — ' + g.reason : '') + '; written from the model alone.', g.used ? 'good' : null); reportSearches(data.searches); + // A deck that fell back came out as plain slides. Silently handing + // someone the plainer artifact left them comparing two decks with no + // idea why one had layouts and the other did not — and asking again + // usually gets the designed one, which is only worth knowing if the + // fallback is visible. + if (data.deckFallback) { + status('Saved, but as plain slides: the model could not produce a slide ' + + 'design twice (' + data.deckFallback + '). Generating again usually gets one.', 'bad'); + } showIllustrations(data.imageJobs || []); reportImageFailures(data.imageFailures); loadLibrary(); diff --git a/src/routes/myResources.js b/src/routes/myResources.js index ac4fe5c2..ac7c99fd 100644 --- a/src/routes/myResources.js +++ b/src/routes/myResources.js @@ -347,15 +347,47 @@ router.post('/my-resources/generate', async function (req, res) { var vocabularyGaps = []; var deck = deckMode ? deckBuild.parse(ai && ai.content, vocabularyGaps) : null; reportVocabularyGaps(vocabularyGaps, topic); + // Why a deck became plain slides, when it did. Reported to the caller as + // well as logged: the fallback produces a usable but plainer deck, and + // saying nothing left people with a worse result and no idea it had + // happened, or that asking again would probably fix it. + var deckFallback = null; + function deckFailure(reply) { + // "Not usable" covers a truncated reply, prose instead of JSON, and an + // empty deck, and they want different fixes — a smaller deck, a different + // model, a reworded topic. Truncation is only claimed for something that + // began as JSON and stopped: an apology in prose does not end in "}" + // either, and calling that "cut short" sends the reader after the wrong + // fix. + var text = String(reply || '').trim(); + if (!text) return 'the model returned nothing'; + var startedJson = text.charAt(0) === '{' || text.charAt(0) === '['; + if (startedJson && text.slice(-1) !== '}' && text.slice(-1) !== ']') { + return 'the reply was cut short at ' + text.length + ' characters'; + } + return 'the reply was not a deck'; + } + if (deckMode && !deck) { - // The model returned something that is not a deck. Falling back to - // markdown beats saving nothing, and beats saving its apology. - // Say how it failed. "Not usable" covers a truncated reply, prose instead - // of JSON, and an empty deck, and they want different fixes. - var replyLength = String((ai && ai.content) || '').length; - var looksCut = replyLength > 0 && String(ai.content).trim().slice(-1) !== '}'; - console.warn('[my-resources] deck reply was not usable (' + - (replyLength === 0 ? 'empty reply' : looksCut ? 'cut short at ' + replyLength + ' chars' : 'not a deck') + + // One more attempt at a deck before giving up on the layout. Models are + // stochastic and this is the same prompt, not a weaker one: measured on + // the stored library, a deck failed once in eight generations, and a + // plain-markdown deck is a materially worse artifact to fall back to + // after a single unlucky reply. + var firstFailure = deckFailure(ai && ai.content); + console.warn('[my-resources] deck reply was not usable (' + firstFailure + '); asking once more'); + var retryGaps = []; + ai = await callAI([{ role: 'user', content: prompt }], options); + deck = deckBuild.parse(ai && ai.content, retryGaps); + reportVocabularyGaps(retryGaps, topic); + if (deck) console.log('[my-resources] the second deck attempt parsed'); + } + + if (deckMode && !deck) { + // Twice is enough. Falling back to markdown beats saving nothing, and + // beats saving the model's apology. + deckFallback = deckFailure(ai && ai.content); + console.warn('[my-resources] deck reply was not usable twice (' + deckFallback + '); retrying as markdown'); var plain = buildPrompt({ topic: topic, kind: kind, refinement: refinement, corpusContext: corpus.context, @@ -427,6 +459,7 @@ router.post('/my-resources/generate', async function (req, res) { imageFailures: ai.imageFailures || [], searches: searches, review: { applied: reviewed.reviewed, reason: reviewed.reason }, + deckFallback: deckFallback, model: ai && ai.model }); } catch (err) { diff --git a/test/my-resources-refine.test.js b/test/my-resources-refine.test.js index 06b6108b..57145b3d 100644 --- a/test/my-resources-refine.test.js +++ b/test/my-resources-refine.test.js @@ -26,6 +26,7 @@ const DECK = Object.assign( function router(t, overrides = {}) { const module = { exports: {} }; const aiCalls = []; + const replies = overrides.replies ? overrides.replies.slice() : null; const updates = []; const row = Object.assign({ id: 5, user_id: 7, title: 'Croup', kind: 'presentation', topic: 'croup', @@ -54,7 +55,10 @@ function router(t, overrides = {}) { '../utils/ai': { callAI: async (messages, options) => { aiCalls.push({ messages, options }); - return { content: overrides.reply, model: 'synthetic' }; + // A scripted sequence when the test needs the model to answer + // differently on each attempt; the last reply repeats if it runs out. + const content = replies ? (replies.length > 1 ? replies.shift() : replies[0]) : overrides.reply; + return { content: content, model: 'synthetic' }; }, discoverModels: async () => [] }, @@ -167,3 +171,62 @@ test('the library says which presentations carry a deck, and the row shows when assert.match(ui, /has_deck === false \? ' · plain text, no slide layout' : ''/); assert.match(ui, /if \(data\.unchanged\) \{/); }); + +// ── Generating a deck ────────────────────────────────────────────────────── + +const GOOD_DECK = JSON.stringify({ + slides: [ + { type: 'title', title: 'Croup' }, + { type: 'bullets', title: 'Features', bullets: ['Barking cough', 'Stridor'] } + ] +}); + +test('a deck the model fumbles once is asked for a second time, not abandoned', async () => { + // Falling straight back to markdown after one unlucky reply produced a + // materially worse artifact: plain slides inferred from markdown instead of + // the layouts the model chose. Measured on the stored library, this happened + // once in eight generations. + const r = router(null, { replies: ['Sorry, I cannot do that.', GOOD_DECK] }); + const res = await r.request('post', '/my-resources/generate', { + topic: 'croup', kind: 'presentation' + }); + + assert.equal(res.statusCode, 200); + assert.equal(r.aiCalls.length, 2, 'the deck is asked for twice before giving up'); + assert.match(r.aiCalls[1].messages[0].content, /teaching presentation/, + 'the retry is the same deck prompt, not the weaker markdown one'); + assert.equal(res.body.deckFallback, null, 'and the retry succeeded, so nothing fell back'); +}); + +test('a deck that fails twice falls back to markdown and says why', async () => { + const r = router(null, { replies: ['Sorry, I cannot help with that.', 'I am unable to comply.', '# Croup\n\n- Barking cough\n'] }); + const res = await r.request('post', '/my-resources/generate', { + topic: 'croup', kind: 'presentation' + }); + + assert.equal(res.statusCode, 200); + assert.equal(r.aiCalls.length, 3, 'two deck attempts, then markdown'); + assert.equal(res.body.deckFallback, 'the reply was not a deck', + 'the caller is told it came out plain, and why'); +}); + +test('a truncated deck reply is named as truncated, not as the wrong shape', async () => { + // The three causes want different fixes — a smaller deck, a different model, + // a reworded topic — so the message distinguishes them. Truncation is only + // claimed for a reply that began as JSON: an apology in prose does not end in + // "}" either, and naming that "cut short" points at the wrong fix. + const cut = GOOD_DECK.slice(0, GOOD_DECK.length - 30); + const r = router(null, { replies: [cut, cut, '# Croup\n'] }); + const res = await r.request('post', '/my-resources/generate', { topic: 'croup', kind: 'presentation' }); + + assert.match(res.body.deckFallback, /cut short at \d+ characters/); +}); + +test('a deck that parses first time is never asked for twice', async () => { + const r = router(null, { replies: [GOOD_DECK] }); + const res = await r.request('post', '/my-resources/generate', { topic: 'croup', kind: 'presentation' }); + + assert.equal(res.statusCode, 200); + assert.equal(r.aiCalls.length, 1, 'the retry costs a call and must only happen on failure'); + assert.equal(res.body.deckFallback, null); +}); diff --git a/test/slide-spec.test.js b/test/slide-spec.test.js index fe690804..8b66221e 100644 --- a/test/slide-spec.test.js +++ b/test/slide-spec.test.js @@ -214,9 +214,12 @@ test('a deck is given room to be a deck', () => { // for a figure at all. That is why decks were arriving with no images. assert.match(route, /if \(deckMode\) options\.maxTokens = 16000;/); // And the fallback says how it failed, because "not usable" covers a - // truncated reply, prose instead of JSON, and an empty deck. - assert.match(route, /cut short at ' \+ replyLength \+ ' chars'/); - assert.match(route, /empty reply/); + // truncated reply, prose instead of JSON, and an empty deck. The three + // outcomes are covered behaviourally in my-resources-refine.test.js; this + // only pins that the budget and the three causes still exist here. + assert.match(route, /cut short at ' \+ text\.length \+ ' characters'/); + assert.match(route, /the model returned nothing/); + assert.match(route, /the reply was not a deck/); }); test('a figure asked for the wrong way is honoured, not dropped', () => {