feat: record what a deck wanted and could not have

The shape vocabulary is deliberately small, which leaves the question of what to
add next. Rather than guess, it now records demand.

Two signals, because a model asks both ways. It can say so outright —
{"kind":"unsupported","need":"a SmartArt cycle of four stages"}, which draws
nothing and is told about in the same file that validates it — or it can reach
for a kind, chart type or slide type that does not exist, which is the more
common way of asking and just as much of a signal.

Both produce a log line naming what was wanted and the topic it came up on, and
increment ped_ai_deck_vocabulary_gap_total{wanted}, so it can be counted over
time in Grafana rather than noticed once and forgotten. Deduplicated per
generation and capped at twelve: a model that asks for a hundred things it cannot
have should not write a hundred log lines. It can never fail a generation — it is
a note to whoever decides what to build next.

This is also the answer to whether to run model-authored code in a sandbox
instead. The log will say whether the gap is real. Some of it is not closeable by
any sandbox, being python-pptx's own ceiling — no SmartArt, no animations or
transitions, limited chart types — and a sandbox would only let a model write
code against the same library and hit the same wall. Documented in
docs/my-resources.md, which the in-app Docs tab serves directly.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU

docs/my-resources.md

@@ -128,6 +128,41 @@ coloured boxes with arrows between them", the model produced
 `[rect arrow rect arrow rect]`, chose green/amber/red itself, and it rendered as
 asked.
 
+### When it wants something that is not there
+
+The vocabulary is deliberately small, so it needs a way to find out what it is
+missing. The model is told to say so:
+
+```json
+{"kind":"unsupported","need":"a SmartArt cycle of four stages"}
+```
+
+Nothing is drawn for that entry. It is recorded, along with the other signal —
+reaching for a kind, chart type or slide type that does not exist, which is how
+a model asks by trying. Both produce a log line:
+
+```
+[deck-vocabulary] wanted "smartart" (used as a shape kind) while generating: croup severity
+```
+
+and increment `ped_ai_deck_vocabulary_gap_total{wanted="smartart"}`, so it can be
+counted over time rather than noticed once. Capped per generation, deduplicated,
+and it can never fail anything — it is a note to whoever decides what to build
+next.
+
+That is the answer to "should this run model-authored code in a sandbox instead".
+Maybe, one day, and the log says whether the gap is real. Today the model never
+emits Python: running model-authored code to lay out a slide would be an enormous
+amount of trust to buy a feature, on a server holding clinical data and secrets,
+and it would need its own network-isolated container with dropped capabilities,
+a read-only filesystem and hard resource limits before it was even safe to try.
+
+Some of the delta is not closeable by any amount of sandboxing, because it is
+python-pptx's own ceiling rather than this vocabulary's: **no SmartArt, no
+animations or slide transitions**, and a limited set of chart types. Those are
+library limits. A sandbox would let a model write code against the same library
+and hit the same wall.
+
 Markdown is still produced, serialised from the deck, so Word export and text
 editing keep working and the stored artifact stays readable by a person. The
 deck is stored alongside it because that serialisation is lossy by design:

src/routes/myResources.js

@@ -318,7 +318,11 @@ router.post('/my-resources/generate', async function (req, res) {
       });
     }
 
-    var deck = deckMode ? deckBuild.parse(ai && ai.content) : null;
+    // What the model reached for and could not have. Recorded rather than
+    // guessed at: the shape vocabulary should grow from real demand.
+    var vocabularyGaps = [];
+    var deck = deckMode ? deckBuild.parse(ai && ai.content, vocabularyGaps) : null;
+    reportVocabularyGaps(vocabularyGaps, topic);
     if (deckMode && !deck) {
       // The model returned something that is not a deck. Falling back to
       // markdown beats saving nothing, and beats saving its apology.
@@ -538,6 +542,23 @@ router.post('/my-resources/:id/refine', async function (req, res) {
   }
 });
 
+// One line per distinct thing a deck wanted and could not have, plus a counter
+// so it can be watched over time. Never fails anything: this is a note to
+// whoever decides what to build next, not part of the generation.
+function reportVocabularyGaps(gaps, topic) {
+  if (!gaps || !gaps.length) return;
+  var seen = Object.create(null);
+  gaps.forEach(function (gap) {
+    if (seen[gap.wanted]) return;
+    seen[gap.wanted] = true;
+    try {
+      require('../utils/metrics').deckVocabularyGaps.inc({ wanted: gap.wanted });
+    } catch (e) { /* metrics are never worth an error here */ }
+    console.warn('[deck-vocabulary] wanted "' + gap.wanted + '" (' + gap.detail +
+      ') while generating: ' + String(topic || '').slice(0, 80));
+  });
+}
+
 // Fetches a resource's finished figures onto disk in the order they were made.
 // asset() already scopes to the owner, so this cannot reach anyone else's.
 function figureIdList(ids) {

src/utils/deckBuild.js

@@ -104,10 +104,10 @@ async function drawFigures(deck, opts) {
  * deck in it, so the caller can fall back to asking for markdown instead of
  * saving something empty.
  */
-function parse(content) {
+function parse(content, gaps) {
   var raw = extractJson(content);
   if (!raw) return null;
-  var deck = deckSchema.normalise(raw);
+  var deck = deckSchema.normalise(raw, gaps);
   if (!deck.slides.length) return null;
   return deck;
 }

src/utils/deckSchema.js

@@ -82,11 +82,14 @@ function bullets(list) {
  * Accept only what the renderer can draw, and never throw. A model that returns
  * one malformed slide should cost that slide, not the deck.
  */
-function normalise(raw) {
+function normalise(raw, gaps) {
   var deck = raw && typeof raw === 'object' ? raw : {};
   var slides = [];
   (Array.isArray(deck.slides) ? deck.slides : []).slice(0, 60).forEach(function (slide) {
     if (!slide || typeof slide !== 'object') return;
+    if (slide.type && VALID.indexOf(slide.type) === -1 && Array.isArray(gaps) && gaps.length < 12) {
+      gaps.push({ wanted: String(slide.type).slice(0, 60), detail: 'used as a slide type' });
+    }
     var type = VALID.indexOf(slide.type) === -1 ? 'bullets' : slide.type;
     var out = { type: type, heading: text(slide.heading, 200) };
     if (slide.notes) out.notes = text(slide.notes, 2000);
@@ -111,7 +114,7 @@ function normalise(raw) {
     } else if (type === 'section') {
       if (!out.heading) return;
     } else if (type === 'custom') {
-      out.shapes = slideShapes.normalise(slide.shapes);
+      out.shapes = slideShapes.normalise(slide.shapes, gaps);
       // A custom slide that lost every shape in validation is a heading over an
       // empty frame. Its bullets, if it sent any, are a better slide than that.
       if (!out.shapes.length) {

src/utils/metrics.js

@@ -71,8 +71,20 @@ async function metricsHandler(req, res) {
   res.end(await register.metrics());
 }
 
+// What a model reached for and could not have when laying out a slide. The
+// vocabulary of shapes is deliberately small; this is how it learns what to grow
+// into, from what people actually ask for rather than from guesses. A label per
+// distinct thing wanted, so it can be counted over time in Grafana.
+const deckVocabularyGaps = new client.Counter({
+  name: 'ped_ai_deck_vocabulary_gap_total',
+  help: 'Times a generated deck asked for a shape, chart or slide type that does not exist',
+  labelNames: ['wanted'],
+  registers: [register]
+});
+
 module.exports = {
   metricsHandler,
   metricsMiddleware,
+  deckVocabularyGaps,
   register
 };

src/utils/slideShapes.js

@@ -20,6 +20,10 @@
 
 var KINDS = ['text', 'rect', 'roundRect', 'ellipse', 'arrow', 'arrowDown', 'chevron',
              'diamond', 'hexagon', 'line', 'image', 'table', 'chart'];
+// Not drawn. A way for the model to say what it wanted and could not have, so
+// the vocabulary grows from what people actually ask for rather than from
+// guesses about what might be useful.
+var UNSUPPORTED = 'unsupported';
 var CHARTS = ['column', 'bar', 'line', 'pie', 'doughnut'];
 var ALIGN = ['left', 'center', 'right'];
 var VALIGN = ['top', 'middle', 'bottom'];
@@ -93,11 +97,27 @@ function chart(shape) {
  * understood is dropped, and a slide that loses every shape falls back to being
  * a plain one rather than an empty frame.
  */
-function normalise(list) {
+function normalise(list, gaps) {
   if (!Array.isArray(list)) return [];
   var out = [];
+  var note = function (wanted, detail) {
+    if (!Array.isArray(gaps)) return;
+    var name = String(wanted || 'unknown').slice(0, 60);
+    if (gaps.length < 12) gaps.push({ wanted: name, detail: String(detail || '').slice(0, 240) });
+  };
   list.slice(0, MAX_SHAPES).forEach(function (raw) {
     if (!raw || typeof raw !== 'object') return;
+
+    // Said outright: "I wanted a SmartArt cycle here." Recorded and not drawn.
+    if (raw.kind === UNSUPPORTED) {
+      note(raw.need || raw.kind, raw.need);
+      return;
+    }
+    // Reached for something that does not exist. Just as much of a signal as
+    // saying so, and more common — a model asks by trying.
+    if (raw.kind && KINDS.indexOf(raw.kind) === -1) {
+      note(raw.kind, 'used as a shape kind');
+    }
     var kind = KINDS.indexOf(raw.kind) === -1 ? 'text' : raw.kind;
     var shape = {
       kind: kind,
@@ -123,6 +143,7 @@ function normalise(list) {
       if (!built) return;
       shape.header = built.header; shape.rows = built.rows;
     } else if (kind === 'chart') {
+      if (raw.chart && CHARTS.indexOf(raw.chart) === -1) note(raw.chart, 'asked for as a chart type');
       var data = chart(raw);
       if (!data) return;
       shape.chart = data.chart; shape.categories = data.categories; shape.series = data.series;
@@ -168,8 +189,14 @@ function instructions() {
     '',
     '  Colours are six hex digits with no "#". Shapes are drawn in the order given,',
     '  so a later one sits on top. Keep text inside its own shape: nothing is',
-    '  measured for you on a custom slide.'
+    '  measured for you on a custom slide.',
+    '',
+    '  If you need something this list cannot express, include',
+    '  {"kind":"unsupported","need":"a SmartArt cycle of four stages"} and lay the',
+    '  slide out as well as you can without it. Nothing is drawn for that entry —',
+    '  it is how the vocabulary learns what to add next, so say plainly what you',
+    '  wanted rather than working around it silently.'
   ].join('\n');
 }
 
-module.exports = { normalise, instructions, KINDS, CHARTS, MAX_SHAPES };
+module.exports = { normalise, instructions, KINDS, CHARTS, MAX_SHAPES, UNSUPPORTED };

test/slide-shapes.test.js

@@ -86,3 +86,44 @@ test('a custom slide that loses every shape becomes a plain one', () => {
   assert.equal(deck.slides[0].type, 'bullets');
   assert.equal(deck.slides[0].bullets[0].text, 'fallback');
 });
+
+test('what the model wanted and could not have is recorded', () => {
+  // The vocabulary is deliberately small. This is how it learns what to grow
+  // into — from what is actually asked for, not from guesses about what might
+  // be useful. Both signals count: saying so, and trying anyway.
+  const gaps = [];
+  const drawn = shapes.normalise([
+    { kind: 'unsupported', need: 'a SmartArt cycle of four stages' },
+    { kind: 'smartart', x: 1, y: 1, w: 10, h: 10, text: 'tried anyway' },
+    { kind: 'chart', chart: 'radar', x: 1, y: 1, w: 10, h: 10, categories: ['a'], series: [{ values: [1] }] },
+    { kind: 'rect', x: 1, y: 1, w: 10, h: 10, fill: 'FFFFFF', text: 'fine' },
+  ], gaps);
+
+  assert.deepEqual(gaps.map(g => g.wanted),
+    ['a SmartArt cycle of four stages', 'smartart', 'radar']);
+  // "unsupported" draws nothing; the rest of the slide still stands.
+  assert.deepEqual(drawn.map(d => d.kind), ['text', 'chart', 'rect']);
+  // An unknown chart type still renders, as a column chart, rather than vanishing.
+  assert.equal(drawn[1].chart, 'column');
+
+  // A slide type that does not exist is the same signal one level up.
+  const deckGaps = [];
+  require('../src/utils/deckSchema').normalise({ slides: [{ type: 'timeline', bullets: ['x'] }] }, deckGaps);
+  assert.deepEqual(deckGaps, [{ wanted: 'timeline', detail: 'used as a slide type' }]);
+
+  // Capped: a model that asks for a hundred things it cannot have should not
+  // write a hundred log lines.
+  const many = [];
+  shapes.normalise(new Array(30).fill(0).map((_, i) => ({ kind: 'unsupported', need: 'thing ' + i })), many);
+  assert.equal(many.length, 12);
+
+  // The model is told how to say it, in the same file that records it.
+  assert.match(shapes.instructions(), /"kind":"unsupported","need":/);
+
+  // And it reaches somewhere a person will see it.
+  const route = read('src/routes/myResources.js');
+  assert.match(route, /function reportVocabularyGaps\(gaps, topic\)/);
+  assert.match(route, /deckVocabularyGaps\.inc\(\{ wanted: gap\.wanted \}\)/);
+  assert.match(route, /\[deck-vocabulary\] wanted/);
+  assert.match(read('src/utils/metrics.js'), /ped_ai_deck_vocabulary_gap_total/);
+});