From 4f8e68690700fb1e4d4de3685b8101bd10355360 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 11 Sep 2026 22:51:15 +0200 Subject: [PATCH] feat: record what a deck wanted and could not have MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shape vocabulary is deliberately small, which leaves the question of what to add next. Rather than guess, it now records demand. Two signals, because a model asks both ways. It can say so outright — {"kind":"unsupported","need":"a SmartArt cycle of four stages"}, which draws nothing and is told about in the same file that validates it — or it can reach for a kind, chart type or slide type that does not exist, which is the more common way of asking and just as much of a signal. Both produce a log line naming what was wanted and the topic it came up on, and increment ped_ai_deck_vocabulary_gap_total{wanted}, so it can be counted over time in Grafana rather than noticed once and forgotten. Deduplicated per generation and capped at twelve: a model that asks for a hundred things it cannot have should not write a hundred log lines. It can never fail a generation — it is a note to whoever decides what to build next. This is also the answer to whether to run model-authored code in a sandbox instead. The log will say whether the gap is real. Some of it is not closeable by any sandbox, being python-pptx's own ceiling — no SmartArt, no animations or transitions, limited chart types — and a sandbox would only let a model write code against the same library and hit the same wall. Documented in docs/my-resources.md, which the in-app Docs tab serves directly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU --- docs/my-resources.md | 35 +++++++++++++++++++++++++++++++++ src/routes/myResources.js | 23 +++++++++++++++++++++- src/utils/deckBuild.js | 4 ++-- src/utils/deckSchema.js | 7 +++++-- src/utils/metrics.js | 12 ++++++++++++ src/utils/slideShapes.js | 33 ++++++++++++++++++++++++++++--- test/slide-shapes.test.js | 41 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 147 insertions(+), 8 deletions(-) diff --git a/docs/my-resources.md b/docs/my-resources.md index 45df29da..5525342f 100644 --- a/docs/my-resources.md +++ b/docs/my-resources.md @@ -128,6 +128,41 @@ coloured boxes with arrows between them", the model produced `[rect arrow rect arrow rect]`, chose green/amber/red itself, and it rendered as asked. +### When it wants something that is not there + +The vocabulary is deliberately small, so it needs a way to find out what it is +missing. The model is told to say so: + +```json +{"kind":"unsupported","need":"a SmartArt cycle of four stages"} +``` + +Nothing is drawn for that entry. It is recorded, along with the other signal — +reaching for a kind, chart type or slide type that does not exist, which is how +a model asks by trying. Both produce a log line: + +``` +[deck-vocabulary] wanted "smartart" (used as a shape kind) while generating: croup severity +``` + +and increment `ped_ai_deck_vocabulary_gap_total{wanted="smartart"}`, so it can be +counted over time rather than noticed once. Capped per generation, deduplicated, +and it can never fail anything — it is a note to whoever decides what to build +next. + +That is the answer to "should this run model-authored code in a sandbox instead". +Maybe, one day, and the log says whether the gap is real. Today the model never +emits Python: running model-authored code to lay out a slide would be an enormous +amount of trust to buy a feature, on a server holding clinical data and secrets, +and it would need its own network-isolated container with dropped capabilities, +a read-only filesystem and hard resource limits before it was even safe to try. + +Some of the delta is not closeable by any amount of sandboxing, because it is +python-pptx's own ceiling rather than this vocabulary's: **no SmartArt, no +animations or slide transitions**, and a limited set of chart types. Those are +library limits. A sandbox would let a model write code against the same library +and hit the same wall. + Markdown is still produced, serialised from the deck, so Word export and text editing keep working and the stored artifact stays readable by a person. The deck is stored alongside it because that serialisation is lossy by design: diff --git a/src/routes/myResources.js b/src/routes/myResources.js index a62957ec..9a470583 100644 --- a/src/routes/myResources.js +++ b/src/routes/myResources.js @@ -318,7 +318,11 @@ router.post('/my-resources/generate', async function (req, res) { }); } - var deck = deckMode ? deckBuild.parse(ai && ai.content) : null; + // What the model reached for and could not have. Recorded rather than + // guessed at: the shape vocabulary should grow from real demand. + var vocabularyGaps = []; + var deck = deckMode ? deckBuild.parse(ai && ai.content, vocabularyGaps) : null; + reportVocabularyGaps(vocabularyGaps, topic); if (deckMode && !deck) { // The model returned something that is not a deck. Falling back to // markdown beats saving nothing, and beats saving its apology. @@ -538,6 +542,23 @@ router.post('/my-resources/:id/refine', async function (req, res) { } }); +// One line per distinct thing a deck wanted and could not have, plus a counter +// so it can be watched over time. Never fails anything: this is a note to +// whoever decides what to build next, not part of the generation. +function reportVocabularyGaps(gaps, topic) { + if (!gaps || !gaps.length) return; + var seen = Object.create(null); + gaps.forEach(function (gap) { + if (seen[gap.wanted]) return; + seen[gap.wanted] = true; + try { + require('../utils/metrics').deckVocabularyGaps.inc({ wanted: gap.wanted }); + } catch (e) { /* metrics are never worth an error here */ } + console.warn('[deck-vocabulary] wanted "' + gap.wanted + '" (' + gap.detail + + ') while generating: ' + String(topic || '').slice(0, 80)); + }); +} + // Fetches a resource's finished figures onto disk in the order they were made. // asset() already scopes to the owner, so this cannot reach anyone else's. function figureIdList(ids) { diff --git a/src/utils/deckBuild.js b/src/utils/deckBuild.js index 713ddea6..8379301c 100644 --- a/src/utils/deckBuild.js +++ b/src/utils/deckBuild.js @@ -104,10 +104,10 @@ async function drawFigures(deck, opts) { * deck in it, so the caller can fall back to asking for markdown instead of * saving something empty. */ -function parse(content) { +function parse(content, gaps) { var raw = extractJson(content); if (!raw) return null; - var deck = deckSchema.normalise(raw); + var deck = deckSchema.normalise(raw, gaps); if (!deck.slides.length) return null; return deck; } diff --git a/src/utils/deckSchema.js b/src/utils/deckSchema.js index 061caa29..5eaf3561 100644 --- a/src/utils/deckSchema.js +++ b/src/utils/deckSchema.js @@ -82,11 +82,14 @@ function bullets(list) { * Accept only what the renderer can draw, and never throw. A model that returns * one malformed slide should cost that slide, not the deck. */ -function normalise(raw) { +function normalise(raw, gaps) { var deck = raw && typeof raw === 'object' ? raw : {}; var slides = []; (Array.isArray(deck.slides) ? deck.slides : []).slice(0, 60).forEach(function (slide) { if (!slide || typeof slide !== 'object') return; + if (slide.type && VALID.indexOf(slide.type) === -1 && Array.isArray(gaps) && gaps.length < 12) { + gaps.push({ wanted: String(slide.type).slice(0, 60), detail: 'used as a slide type' }); + } var type = VALID.indexOf(slide.type) === -1 ? 'bullets' : slide.type; var out = { type: type, heading: text(slide.heading, 200) }; if (slide.notes) out.notes = text(slide.notes, 2000); @@ -111,7 +114,7 @@ function normalise(raw) { } else if (type === 'section') { if (!out.heading) return; } else if (type === 'custom') { - out.shapes = slideShapes.normalise(slide.shapes); + out.shapes = slideShapes.normalise(slide.shapes, gaps); // A custom slide that lost every shape in validation is a heading over an // empty frame. Its bullets, if it sent any, are a better slide than that. if (!out.shapes.length) { diff --git a/src/utils/metrics.js b/src/utils/metrics.js index 392c42dd..3cc242eb 100644 --- a/src/utils/metrics.js +++ b/src/utils/metrics.js @@ -71,8 +71,20 @@ async function metricsHandler(req, res) { res.end(await register.metrics()); } +// What a model reached for and could not have when laying out a slide. The +// vocabulary of shapes is deliberately small; this is how it learns what to grow +// into, from what people actually ask for rather than from guesses. A label per +// distinct thing wanted, so it can be counted over time in Grafana. +const deckVocabularyGaps = new client.Counter({ + name: 'ped_ai_deck_vocabulary_gap_total', + help: 'Times a generated deck asked for a shape, chart or slide type that does not exist', + labelNames: ['wanted'], + registers: [register] +}); + module.exports = { metricsHandler, metricsMiddleware, + deckVocabularyGaps, register }; diff --git a/src/utils/slideShapes.js b/src/utils/slideShapes.js index ae1eb9a6..c947b493 100644 --- a/src/utils/slideShapes.js +++ b/src/utils/slideShapes.js @@ -20,6 +20,10 @@ var KINDS = ['text', 'rect', 'roundRect', 'ellipse', 'arrow', 'arrowDown', 'chevron', 'diamond', 'hexagon', 'line', 'image', 'table', 'chart']; +// Not drawn. A way for the model to say what it wanted and could not have, so +// the vocabulary grows from what people actually ask for rather than from +// guesses about what might be useful. +var UNSUPPORTED = 'unsupported'; var CHARTS = ['column', 'bar', 'line', 'pie', 'doughnut']; var ALIGN = ['left', 'center', 'right']; var VALIGN = ['top', 'middle', 'bottom']; @@ -93,11 +97,27 @@ function chart(shape) { * understood is dropped, and a slide that loses every shape falls back to being * a plain one rather than an empty frame. */ -function normalise(list) { +function normalise(list, gaps) { if (!Array.isArray(list)) return []; var out = []; + var note = function (wanted, detail) { + if (!Array.isArray(gaps)) return; + var name = String(wanted || 'unknown').slice(0, 60); + if (gaps.length < 12) gaps.push({ wanted: name, detail: String(detail || '').slice(0, 240) }); + }; list.slice(0, MAX_SHAPES).forEach(function (raw) { if (!raw || typeof raw !== 'object') return; + + // Said outright: "I wanted a SmartArt cycle here." Recorded and not drawn. + if (raw.kind === UNSUPPORTED) { + note(raw.need || raw.kind, raw.need); + return; + } + // Reached for something that does not exist. Just as much of a signal as + // saying so, and more common — a model asks by trying. + if (raw.kind && KINDS.indexOf(raw.kind) === -1) { + note(raw.kind, 'used as a shape kind'); + } var kind = KINDS.indexOf(raw.kind) === -1 ? 'text' : raw.kind; var shape = { kind: kind, @@ -123,6 +143,7 @@ function normalise(list) { if (!built) return; shape.header = built.header; shape.rows = built.rows; } else if (kind === 'chart') { + if (raw.chart && CHARTS.indexOf(raw.chart) === -1) note(raw.chart, 'asked for as a chart type'); var data = chart(raw); if (!data) return; shape.chart = data.chart; shape.categories = data.categories; shape.series = data.series; @@ -168,8 +189,14 @@ function instructions() { '', ' Colours are six hex digits with no "#". Shapes are drawn in the order given,', ' so a later one sits on top. Keep text inside its own shape: nothing is', - ' measured for you on a custom slide.' + ' measured for you on a custom slide.', + '', + ' If you need something this list cannot express, include', + ' {"kind":"unsupported","need":"a SmartArt cycle of four stages"} and lay the', + ' slide out as well as you can without it. Nothing is drawn for that entry —', + ' it is how the vocabulary learns what to add next, so say plainly what you', + ' wanted rather than working around it silently.' ].join('\n'); } -module.exports = { normalise, instructions, KINDS, CHARTS, MAX_SHAPES }; +module.exports = { normalise, instructions, KINDS, CHARTS, MAX_SHAPES, UNSUPPORTED }; diff --git a/test/slide-shapes.test.js b/test/slide-shapes.test.js index e6bb7a42..6f71286e 100644 --- a/test/slide-shapes.test.js +++ b/test/slide-shapes.test.js @@ -86,3 +86,44 @@ test('a custom slide that loses every shape becomes a plain one', () => { assert.equal(deck.slides[0].type, 'bullets'); assert.equal(deck.slides[0].bullets[0].text, 'fallback'); }); + +test('what the model wanted and could not have is recorded', () => { + // The vocabulary is deliberately small. This is how it learns what to grow + // into — from what is actually asked for, not from guesses about what might + // be useful. Both signals count: saying so, and trying anyway. + const gaps = []; + const drawn = shapes.normalise([ + { kind: 'unsupported', need: 'a SmartArt cycle of four stages' }, + { kind: 'smartart', x: 1, y: 1, w: 10, h: 10, text: 'tried anyway' }, + { kind: 'chart', chart: 'radar', x: 1, y: 1, w: 10, h: 10, categories: ['a'], series: [{ values: [1] }] }, + { kind: 'rect', x: 1, y: 1, w: 10, h: 10, fill: 'FFFFFF', text: 'fine' }, + ], gaps); + + assert.deepEqual(gaps.map(g => g.wanted), + ['a SmartArt cycle of four stages', 'smartart', 'radar']); + // "unsupported" draws nothing; the rest of the slide still stands. + assert.deepEqual(drawn.map(d => d.kind), ['text', 'chart', 'rect']); + // An unknown chart type still renders, as a column chart, rather than vanishing. + assert.equal(drawn[1].chart, 'column'); + + // A slide type that does not exist is the same signal one level up. + const deckGaps = []; + require('../src/utils/deckSchema').normalise({ slides: [{ type: 'timeline', bullets: ['x'] }] }, deckGaps); + assert.deepEqual(deckGaps, [{ wanted: 'timeline', detail: 'used as a slide type' }]); + + // Capped: a model that asks for a hundred things it cannot have should not + // write a hundred log lines. + const many = []; + shapes.normalise(new Array(30).fill(0).map((_, i) => ({ kind: 'unsupported', need: 'thing ' + i })), many); + assert.equal(many.length, 12); + + // The model is told how to say it, in the same file that records it. + assert.match(shapes.instructions(), /"kind":"unsupported","need":/); + + // And it reaches somewhere a person will see it. + const route = read('src/routes/myResources.js'); + assert.match(route, /function reportVocabularyGaps\(gaps, topic\)/); + assert.match(route, /deckVocabularyGaps\.inc\(\{ wanted: gap\.wanted \}\)/); + assert.match(route, /\[deck-vocabulary\] wanted/); + assert.match(read('src/utils/metrics.js'), /ped_ai_deck_vocabulary_gap_total/); +});