From 07d1c541a040321a5c951f442d5910fb0740a56f Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 12 Sep 2026 05:34:14 +0200 Subject: [PATCH] fix: a deck the model fumbles once is asked for again, not abandoned MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A presentation is generated as a typed deck — the model chooses each slide's layout — and that deck is what scripts/render_pptx.py renders. When the reply did not parse as a deck, the route fell straight back to asking for markdown, and slideSpec.build() then inferred slides from it. Both paths go through python-pptx, but the fallback's layout is guessed from heading and list structure rather than chosen, so everything lands as title-and-bullets. Measured on the stored library: since decks landed, 7 of 8 generations produced one and 1 did not. Models are stochastic, so one unlucky reply was costing the whole layout. It now asks a second time with the same prompt before giving up. The fallback was also invisible. It warned to the console, where the person who would simply have generated again could not see it, so they kept the plainer deck without knowing a better one was one click away. The response now carries deckFallback and the UI says it came out as plain slides, and why. Fixed the reason heuristic while adding tests for it: truncation was claimed for any reply not ending in "}", which is every prose refusal. It is now only claimed for a reply that began as JSON and stopped. The four generate tests run the handler. Verified against a mutation: removing the retry fails tests 6 and 7 and nothing else. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU --- public/js/myResources.js | 9 +++++ src/routes/myResources.js | 49 ++++++++++++++++++++---- test/my-resources-refine.test.js | 65 +++++++++++++++++++++++++++++++- test/slide-spec.test.js | 9 +++-- 4 files changed, 120 insertions(+), 12 deletions(-) diff --git a/public/js/myResources.js b/public/js/myResources.js index 67bca54f..5706cb7c 100644 --- a/public/js/myResources.js +++ b/public/js/myResources.js @@ -187,6 +187,15 @@ : 'Saved. Not grounded' + (g.reason ? ' — ' + g.reason : '') + '; written from the model alone.', g.used ? 'good' : null); reportSearches(data.searches); + // A deck that fell back came out as plain slides. Silently handing + // someone the plainer artifact left them comparing two decks with no + // idea why one had layouts and the other did not — and asking again + // usually gets the designed one, which is only worth knowing if the + // fallback is visible. + if (data.deckFallback) { + status('Saved, but as plain slides: the model could not produce a slide ' + + 'design twice (' + data.deckFallback + '). Generating again usually gets one.', 'bad'); + } showIllustrations(data.imageJobs || []); reportImageFailures(data.imageFailures); loadLibrary(); diff --git a/src/routes/myResources.js b/src/routes/myResources.js index ac4fe5c2..ac7c99fd 100644 --- a/src/routes/myResources.js +++ b/src/routes/myResources.js @@ -347,15 +347,47 @@ router.post('/my-resources/generate', async function (req, res) { var vocabularyGaps = []; var deck = deckMode ? deckBuild.parse(ai && ai.content, vocabularyGaps) : null; reportVocabularyGaps(vocabularyGaps, topic); + // Why a deck became plain slides, when it did. Reported to the caller as + // well as logged: the fallback produces a usable but plainer deck, and + // saying nothing left people with a worse result and no idea it had + // happened, or that asking again would probably fix it. + var deckFallback = null; + function deckFailure(reply) { + // "Not usable" covers a truncated reply, prose instead of JSON, and an + // empty deck, and they want different fixes — a smaller deck, a different + // model, a reworded topic. Truncation is only claimed for something that + // began as JSON and stopped: an apology in prose does not end in "}" + // either, and calling that "cut short" sends the reader after the wrong + // fix. + var text = String(reply || '').trim(); + if (!text) return 'the model returned nothing'; + var startedJson = text.charAt(0) === '{' || text.charAt(0) === '['; + if (startedJson && text.slice(-1) !== '}' && text.slice(-1) !== ']') { + return 'the reply was cut short at ' + text.length + ' characters'; + } + return 'the reply was not a deck'; + } + if (deckMode && !deck) { - // The model returned something that is not a deck. Falling back to - // markdown beats saving nothing, and beats saving its apology. - // Say how it failed. "Not usable" covers a truncated reply, prose instead - // of JSON, and an empty deck, and they want different fixes. - var replyLength = String((ai && ai.content) || '').length; - var looksCut = replyLength > 0 && String(ai.content).trim().slice(-1) !== '}'; - console.warn('[my-resources] deck reply was not usable (' + - (replyLength === 0 ? 'empty reply' : looksCut ? 'cut short at ' + replyLength + ' chars' : 'not a deck') + + // One more attempt at a deck before giving up on the layout. Models are + // stochastic and this is the same prompt, not a weaker one: measured on + // the stored library, a deck failed once in eight generations, and a + // plain-markdown deck is a materially worse artifact to fall back to + // after a single unlucky reply. + var firstFailure = deckFailure(ai && ai.content); + console.warn('[my-resources] deck reply was not usable (' + firstFailure + '); asking once more'); + var retryGaps = []; + ai = await callAI([{ role: 'user', content: prompt }], options); + deck = deckBuild.parse(ai && ai.content, retryGaps); + reportVocabularyGaps(retryGaps, topic); + if (deck) console.log('[my-resources] the second deck attempt parsed'); + } + + if (deckMode && !deck) { + // Twice is enough. Falling back to markdown beats saving nothing, and + // beats saving the model's apology. + deckFallback = deckFailure(ai && ai.content); + console.warn('[my-resources] deck reply was not usable twice (' + deckFallback + '); retrying as markdown'); var plain = buildPrompt({ topic: topic, kind: kind, refinement: refinement, corpusContext: corpus.context, @@ -427,6 +459,7 @@ router.post('/my-resources/generate', async function (req, res) { imageFailures: ai.imageFailures || [], searches: searches, review: { applied: reviewed.reviewed, reason: reviewed.reason }, + deckFallback: deckFallback, model: ai && ai.model }); } catch (err) { diff --git a/test/my-resources-refine.test.js b/test/my-resources-refine.test.js index 06b6108b..57145b3d 100644 --- a/test/my-resources-refine.test.js +++ b/test/my-resources-refine.test.js @@ -26,6 +26,7 @@ const DECK = Object.assign( function router(t, overrides = {}) { const module = { exports: {} }; const aiCalls = []; + const replies = overrides.replies ? overrides.replies.slice() : null; const updates = []; const row = Object.assign({ id: 5, user_id: 7, title: 'Croup', kind: 'presentation', topic: 'croup', @@ -54,7 +55,10 @@ function router(t, overrides = {}) { '../utils/ai': { callAI: async (messages, options) => { aiCalls.push({ messages, options }); - return { content: overrides.reply, model: 'synthetic' }; + // A scripted sequence when the test needs the model to answer + // differently on each attempt; the last reply repeats if it runs out. + const content = replies ? (replies.length > 1 ? replies.shift() : replies[0]) : overrides.reply; + return { content: content, model: 'synthetic' }; }, discoverModels: async () => [] }, @@ -167,3 +171,62 @@ test('the library says which presentations carry a deck, and the row shows when assert.match(ui, /has_deck === false \? ' · plain text, no slide layout' : ''/); assert.match(ui, /if \(data\.unchanged\) \{/); }); + +// ── Generating a deck ────────────────────────────────────────────────────── + +const GOOD_DECK = JSON.stringify({ + slides: [ + { type: 'title', title: 'Croup' }, + { type: 'bullets', title: 'Features', bullets: ['Barking cough', 'Stridor'] } + ] +}); + +test('a deck the model fumbles once is asked for a second time, not abandoned', async () => { + // Falling straight back to markdown after one unlucky reply produced a + // materially worse artifact: plain slides inferred from markdown instead of + // the layouts the model chose. Measured on the stored library, this happened + // once in eight generations. + const r = router(null, { replies: ['Sorry, I cannot do that.', GOOD_DECK] }); + const res = await r.request('post', '/my-resources/generate', { + topic: 'croup', kind: 'presentation' + }); + + assert.equal(res.statusCode, 200); + assert.equal(r.aiCalls.length, 2, 'the deck is asked for twice before giving up'); + assert.match(r.aiCalls[1].messages[0].content, /teaching presentation/, + 'the retry is the same deck prompt, not the weaker markdown one'); + assert.equal(res.body.deckFallback, null, 'and the retry succeeded, so nothing fell back'); +}); + +test('a deck that fails twice falls back to markdown and says why', async () => { + const r = router(null, { replies: ['Sorry, I cannot help with that.', 'I am unable to comply.', '# Croup\n\n- Barking cough\n'] }); + const res = await r.request('post', '/my-resources/generate', { + topic: 'croup', kind: 'presentation' + }); + + assert.equal(res.statusCode, 200); + assert.equal(r.aiCalls.length, 3, 'two deck attempts, then markdown'); + assert.equal(res.body.deckFallback, 'the reply was not a deck', + 'the caller is told it came out plain, and why'); +}); + +test('a truncated deck reply is named as truncated, not as the wrong shape', async () => { + // The three causes want different fixes — a smaller deck, a different model, + // a reworded topic — so the message distinguishes them. Truncation is only + // claimed for a reply that began as JSON: an apology in prose does not end in + // "}" either, and naming that "cut short" points at the wrong fix. + const cut = GOOD_DECK.slice(0, GOOD_DECK.length - 30); + const r = router(null, { replies: [cut, cut, '# Croup\n'] }); + const res = await r.request('post', '/my-resources/generate', { topic: 'croup', kind: 'presentation' }); + + assert.match(res.body.deckFallback, /cut short at \d+ characters/); +}); + +test('a deck that parses first time is never asked for twice', async () => { + const r = router(null, { replies: [GOOD_DECK] }); + const res = await r.request('post', '/my-resources/generate', { topic: 'croup', kind: 'presentation' }); + + assert.equal(res.statusCode, 200); + assert.equal(r.aiCalls.length, 1, 'the retry costs a call and must only happen on failure'); + assert.equal(res.body.deckFallback, null); +}); diff --git a/test/slide-spec.test.js b/test/slide-spec.test.js index fe690804..8b66221e 100644 --- a/test/slide-spec.test.js +++ b/test/slide-spec.test.js @@ -214,9 +214,12 @@ test('a deck is given room to be a deck', () => { // for a figure at all. That is why decks were arriving with no images. assert.match(route, /if \(deckMode\) options\.maxTokens = 16000;/); // And the fallback says how it failed, because "not usable" covers a - // truncated reply, prose instead of JSON, and an empty deck. - assert.match(route, /cut short at ' \+ replyLength \+ ' chars'/); - assert.match(route, /empty reply/); + // truncated reply, prose instead of JSON, and an empty deck. The three + // outcomes are covered behaviourally in my-resources-refine.test.js; this + // only pins that the budget and the three causes still exist here. + assert.match(route, /cut short at ' \+ text\.length \+ ' characters'/); + assert.match(route, /the model returned nothing/); + assert.match(route, /the reply was not a deck/); }); test('a figure asked for the wrong way is honoured, not dropped', () => {