diff --git a/migrations/1780600000000_resource-deck.js b/migrations/1780600000000_resource-deck.js new file mode 100644 index 00000000..1a5fc404 --- /dev/null +++ b/migrations/1780600000000_resource-deck.js @@ -0,0 +1,18 @@ +// The deck a presentation actually is. +// +// Markdown stays the readable artifact — it is what Word renders and what a +// text edit edits — but it cannot express a two-column comparison, a callout, or +// a figure placed beside its bullets, so a deck round-tripped through markdown +// loses the layout the model chose. Storing the deck keeps those choices, and +// the markdown is serialised from it. +// +// Null for an article, and for every presentation written before this existed; +// those still render from their markdown. + +exports.up = pgm => pgm.sql(` + ALTER TABLE user_resources ADD COLUMN IF NOT EXISTS deck JSONB; +`); + +exports.down = pgm => pgm.sql(` + ALTER TABLE user_resources DROP COLUMN IF EXISTS deck; +`); diff --git a/scripts/render_pptx.py b/scripts/render_pptx.py index 964c9066..1a531b8f 100644 --- a/scripts/render_pptx.py +++ b/scripts/render_pptx.py @@ -329,6 +329,102 @@ def slide_image(prs, spec): return slide +def slide_compare(prs, spec): + """Two labelled columns on tinted cards — the layout a differential, a + mild-versus-severe or a before-and-after actually wants, and the one + markdown had no way to ask for.""" + slide = _blank(prs) + _heading(slide, spec.get("heading") or "") + gutter = Emu(365760) + col_w = Emu(int((BODY_W - gutter) / 2)) + columns = (spec.get("columns") or [])[:2] + tints = [RGBColor(0xEF, 0xF6, 0xFF), RGBColor(0xFE, 0xF3, 0xC7)] + edges = [ACCENT, RGBColor(0xD9, 0x77, 0x06)] + sizes = [_fit_size(c.get("bullets") or [], 0.44, int(BODY_H) - int(Emu(548640))) + for c in columns] or [BULLET_SIZES[0]] + size = min(sizes) + for index, column in enumerate(columns): + left = MARGIN + (col_w + gutter) * index + card = slide.shapes.add_shape(5, left, BODY_TOP, col_w, BODY_H) # rounded rect + card.fill.solid() + card.fill.fore_color.rgb = tints[index % 2] + card.line.color.rgb = edges[index % 2] + card.line.width = Pt(1) + card.shadow.inherit = False + card.text_frame.text = "" + + label = _textbox(slide, left + Emu(228600), BODY_TOP + Emu(182880), + Emu(int(col_w) - 457200), Emu(365760)) + _run(label.paragraphs[0], (column.get("label") or "").upper(), 14, + bold=True, color=edges[index % 2]) + + frame = _textbox(slide, left + Emu(228600), BODY_TOP + Emu(640080), + Emu(int(col_w) - 457200), Emu(int(BODY_H) - 822960)) + _bullets(frame, column.get("bullets") or [], size, width_frac=0.44) + _notes(slide, spec.get("notes")) + return slide + + +def slide_callout(prs, spec): + """One thing worth stopping on: a red flag, a dose, a rule of thumb.""" + slide = _blank(prs) + _heading(slide, spec.get("heading") or "") + card = slide.shapes.add_shape(5, MARGIN, BODY_TOP, BODY_W, Emu(int(BODY_H * 0.62))) + card.fill.solid() + card.fill.fore_color.rgb = RGBColor(0xFE, 0xF3, 0xC7) + card.line.color.rgb = RGBColor(0xD9, 0x77, 0x06) + card.line.width = Pt(1.5) + card.shadow.inherit = False + card.text_frame.text = "" + + text = (spec.get("text") or "").strip() + frame = _textbox(slide, MARGIN + Emu(457200), BODY_TOP + Emu(365760), + Emu(int(BODY_W) - 914400), Emu(int(BODY_H * 0.62) - 731520)) + frame.vertical_anchor = MSO_ANCHOR.MIDDLE + para = frame.paragraphs[0] + para.alignment = PP_ALIGN.CENTER + size = 28 if len(text) <= 90 else (22 if len(text) <= 180 else 18) + for chunk, bold in _split_bold(text): + _run(para, chunk, size, bold=bold or True, color=RGBColor(0x78, 0x35, 0x0F)) + _notes(slide, spec.get("notes")) + return slide + + +def slide_figure(prs, spec): + """A figure beside its text, rather than alone on a slide of its own.""" + slide = _blank(prs) + _heading(slide, spec.get("heading") or "") + gutter = Emu(365760) + text_w = Emu(int((BODY_W - gutter) * 0.46)) + img_w = Emu(int((BODY_W - gutter) * 0.54)) + + items = spec.get("bullets") or [] + size = _fit_size(items, 0.42) + frame = _textbox(slide, MARGIN, BODY_TOP, text_w, BODY_H) + _bullets(frame, items, size, width_frac=0.42) + + path = spec.get("image") + if path and os.path.exists(path): + ratio = 1.0 + if Image is not None: + try: + with Image.open(path) as img: + if img.height: + ratio = img.width / float(img.height) + except Exception: + ratio = 1.0 + width = int(img_w) + height = int(width / ratio) if ratio else int(BODY_H) + if height > int(BODY_H): + height = int(BODY_H) + width = int(height * ratio) + left = MARGIN + text_w + gutter + Emu(int((int(img_w) - width) / 2)) + top = BODY_TOP + Emu(int((int(BODY_H) - height) / 2)) + slide.shapes.add_picture(path, left, top, Emu(width), Emu(height)) + _notes(slide, spec.get("notes")) + return slide + + BUILDERS = { "title": slide_title, "section": slide_section, @@ -336,6 +432,9 @@ BUILDERS = { "two": slide_two, "table": slide_table, "image": slide_image, + "compare": slide_compare, + "callout": slide_callout, + "figure": slide_figure, } diff --git a/src/routes/myResources.js b/src/routes/myResources.js index 136215aa..caeec528 100644 --- a/src/routes/myResources.js +++ b/src/routes/myResources.js @@ -21,6 +21,8 @@ var { authMiddleware } = require('../middleware/auth'); var { callAI } = require('../utils/ai'); var learningRetrieval = require('../utils/learningRetrieval'); var resourceImages = require('../utils/resourceImages'); +var deckSchema = require('../utils/deckSchema'); +var deckBuild = require('../utils/deckBuild'); var webSearch = require('../utils/webSearch'); var pubmedSearch = require('../utils/pubmedSearch'); var documentExport = require('../utils/documentExport'); @@ -124,6 +126,18 @@ function buildPrompt(opts) { 'what they asked for.\n' : ''; + // A presentation is described, not written as markdown: the renderer can draw + // comparisons, tables, callouts and a figure beside its text, and markdown + // has no way to ask for any of them. Articles stay markdown, which is the + // right shape for prose. + if (kind === 'presentation' && opts.deckMode) { + return 'You are building a teaching presentation for a medical professional ' + + 'audience (pediatrics / primary care).\n\nTOPIC: ' + opts.topic + '\n' + + grounding + findings + '\n' + + deckSchema.instructions(opts.slideCount, opts.figureCount) + + (opts.refinement ? '\n\nAdditional instructions: ' + opts.refinement + '\n' : ''); + } + var shape = kind === 'presentation' ? 'Write a ' + opts.slideCount + '-slide teaching presentation.\n\n' + 'Output ONLY Pandoc markdown:\n' + @@ -254,11 +268,16 @@ router.post('/my-resources/generate', async function (req, res) { var wantsImages = sources.wantsImages; var imageModel = sources.imageModel; + // A presentation is described as a deck rather than written as markdown, so + // the model can choose a comparison, a table, a callout or a figure beside + // its text. Articles stay markdown: prose is what markdown is for. + var deckMode = kind === 'presentation'; var prompt = buildPrompt({ topic: topic, kind: kind, refinement: refinement, corpusContext: corpus.context, literature: sources.literature, webFindings: sources.webFindings, searchedAndFoundNothing: sources.searchedAndFoundNothing, - wantsImages: wantsImages, + wantsImages: wantsImages, deckMode: deckMode, + figureCount: wantsImages ? (resourceImages.requestedCount(refinement) || 0) : 0, slideCount: clampInt(req.body.slideCount, 3, 30, 8), wordCount: clampInt(req.body.wordCount, 200, 3000, 800) }); @@ -271,8 +290,11 @@ router.post('/my-resources/generate', async function (req, res) { // Only the image tool stays a tool. Illustration genuinely needs the model to // decide and to compose a prompt; a search only needs the topic, and the // topic is already known. + // In deck mode the figures are named by the slides that want them, so there + // is nothing for the image tool to decide and it is not offered. An article + // still gets the tool, because prose has no structure to hang a figure on. var tools = []; - if (wantsImages) tools = tools.concat(resourceImages.tools); + if (wantsImages && !deckMode) tools = tools.concat(resourceImages.tools); // When the author names a number — "use 3 diagrams" — the call is required // rather than merely offered. Measured, deterministically: with the library @@ -285,30 +307,53 @@ router.post('/my-resources/generate', async function (req, res) { var ai = await callAI(messages, callOptions); - // My Resources' own dispatcher, not the assistant's: that one permits a - // single image per request, which is right for a chat reply and wrong for a - // deck. Same queue, same storage, same my_resources workflow — only the - // number of figures differs. - if (wantsImages) { + if (wantsImages && !deckMode) { + // My Resources' own dispatcher, not the assistant's: that one permits a + // single image per request, which is right for a chat reply and wrong for + // a resource. Same queue, same storage, same my_resources workflow. ai = await resourceImages.dispatch(ai, { owner: req.user.id, body: req.body, subject: topic, imageModel: imageModel, messages: messages, options: options, callAI: callAI }); } - // Searches already ran; the model's only job was to write from what it was - // given. An empty body is a failed generation rather than an empty resource. - var markdown = String((ai && ai.content) || '').trim(); + var deck = deckMode ? deckBuild.parse(ai && ai.content) : null; + if (deckMode && !deck) { + // The model returned something that is not a deck. Falling back to + // markdown beats saving nothing, and beats saving its apology. + console.warn('[my-resources] deck reply was not usable; retrying as markdown'); + var plain = buildPrompt({ + topic: topic, kind: kind, refinement: refinement, corpusContext: corpus.context, + literature: sources.literature, webFindings: sources.webFindings, + searchedAndFoundNothing: sources.searchedAndFoundNothing, + wantsImages: false, deckMode: false, + slideCount: clampInt(req.body.slideCount, 3, 30, 8), + wordCount: clampInt(req.body.wordCount, 200, 3000, 800) + }); + ai = await callAI([{ role: 'user', content: plain }], options); + } + if (deck) { + var drawn = await deckBuild.drawFigures(deck, { + owner: req.user.id, body: req.body, subject: topic, imageModel: imageModel + }); + ai = Object.assign({}, ai, { imageJobs: drawn.jobs, imageFailures: drawn.failures }); + } + + // Markdown is still the readable artifact: it is what Word export renders + // and what a text edit edits. In deck mode it is serialised from the deck + // rather than written by the model. + var markdown = deck ? deckSchema.toMarkdown(deck) : String((ai && ai.content) || '').trim(); if (!markdown) return res.status(502).json({ error: 'The model returned nothing. Try again.' }); // The figures belong to the resource, or an exported deck has no way to // include the pictures the author asked for. var figureIds = (ai.imageJobs || []).map(function (job) { return job.jobId; }).filter(Boolean); var row = await db.get( - 'INSERT INTO user_resources (user_id, title, kind, markdown, topic, grounded_count, image_ids) ' + - 'VALUES (?, ?, ?, ?, ?, ?, ?) RETURNING id, title, kind, topic, grounded_count, created_at', - [req.user.id, firstHeading(markdown, topic), kind, markdown, topic.slice(0, 500), - corpus.sources.length, JSON.stringify(figureIds)] + 'INSERT INTO user_resources (user_id, title, kind, markdown, topic, grounded_count, image_ids, deck) ' + + 'VALUES (?, ?, ?, ?, ?, ?, ?, ?) RETURNING id, title, kind, topic, grounded_count, created_at', + [req.user.id, deck && deck.title ? deck.title : firstHeading(markdown, topic), kind, markdown, + topic.slice(0, 500), corpus.sources.length, JSON.stringify(figureIds), + deck ? JSON.stringify(deck) : null] ); res.json({ @@ -469,9 +514,12 @@ router.post('/my-resources/:id/refine', async function (req, res) { // Fetches a resource's finished figures onto disk in the order they were made. // asset() already scopes to the owner, so this cannot reach anyone else's. +function figureIdList(ids) { + try { return Array.isArray(ids) ? ids : JSON.parse(ids || '[]'); } catch (e) { return []; } +} + async function collectFigures(ids, user, dir) { - var list = []; - try { list = Array.isArray(ids) ? ids : JSON.parse(ids || '[]'); } catch (e) { return []; } + var list = figureIdList(ids); var fsp = require('fs/promises'); var pathMod = require('path'); var service = require('../utils/generatedImages'); @@ -498,7 +546,7 @@ router.get('/my-resources/:id/export', async function (req, res) { if (!documentExport.isSupported(format)) return res.status(400).json({ error: 'Unsupported format' }); var row = await db.get( - 'SELECT title, kind, markdown, image_ids FROM user_resources WHERE id = ? AND user_id = ?', + 'SELECT title, kind, markdown, image_ids, deck FROM user_resources WHERE id = ? AND user_id = ?', [parseInt(req.params.id, 10), req.user.id] ); if (!row) return res.status(404).json({ error: 'Not found' }); @@ -520,7 +568,8 @@ router.get('/my-resources/:id/export', async function (req, res) { } var bytes; try { - bytes = await documentExport.render(row.markdown, row.kind, format, { images: figures }); + bytes = await documentExport.render(row.markdown, row.kind, format, + { images: figures, deck: row.deck, figureIds: figureIdList(row.image_ids) }); } finally { if (scratch) { await require('fs/promises').rm(scratch, { recursive: true, force: true }) diff --git a/src/utils/deckBuild.js b/src/utils/deckBuild.js new file mode 100644 index 00000000..a1920051 --- /dev/null +++ b/src/utils/deckBuild.js @@ -0,0 +1,95 @@ +// ============================================================ +// DECK BUILD +// ============================================================ +// Turns one model reply into a stored deck: parse it, draw the figures it asked +// for, and produce the markdown that Word export and text editing still need. +// +// Kept out of the route because generating and modifying both do exactly this, +// and because the failure modes are worth having in one place: a model that +// returns prose around its JSON, a figure that cannot be drawn, a deck with no +// slides at all. + +var deckSchema = require('./deckSchema'); +var images = require('./generatedImages'); + +var MAX_FIGURES = 6; + +// Models wrap JSON in fences or a sentence often enough that refusing it would +// be pedantry. The first balanced object is the deck. +function extractJson(content) { + var text = String(content || '').trim(); + text = text.replace(/^```(?:json)?\s*/i, '').replace(/\s*```$/, ''); + var start = text.indexOf('{'); + if (start === -1) return null; + var depth = 0, inString = false, escaped = false; + for (var i = start; i < text.length; i++) { + var ch = text[i]; + if (escaped) { escaped = false; continue; } + if (ch === '\\') { escaped = true; continue; } + if (ch === '"') { inString = !inString; continue; } + if (inString) continue; + if (ch === '{') depth++; + else if (ch === '}' && --depth === 0) { + try { return JSON.parse(text.slice(start, i + 1)); } catch (e) { return null; } + } + } + return null; +} + +/** + * Draw the figures a deck asked for and attach their job ids to the slides. + * + * Never throws: a figure that cannot be queued leaves its slide as text, which + * is a worse slide than intended and a great deal better than a failed + * generation. + */ +async function drawFigures(deck, opts) { + var wanted = deckSchema.figureRequests(deck).slice(0, MAX_FIGURES); + var jobs = []; + var failures = []; + if (!wanted.length || !opts.imageModel) { + // Asked for pictures with no image model configured: the slides degrade to + // text rather than keeping an empty frame. + wanted.forEach(function (request) { + var slide = deck.slides[request.index]; + slide.type = slide.type === 'image' ? 'section' : 'bullets'; + delete slide.image_prompt; + }); + return { jobs: jobs, failures: failures }; + } + + var queue = opts.images || images.service(); + for (var i = 0; i < wanted.length; i++) { + var request = wanted[i]; + var slide = deck.slides[request.index]; + try { + var context = images.imageContext(opts.subject + ' — ' + (slide.heading || ''), []); + var key = ('deck:' + images.requestKey(opts.body) + ':' + request.index).slice(0, 160); + var job = await queue.enqueue(opts.owner, 'my_resources', + { prompt: request.prompt, layout: slide.type === 'figure' ? 'portrait' : 'landscape' }, + key, true, context, opts.imageModel); + jobs.push(job); + slide.image_job = job.jobId; + } catch (err) { + failures.push(err && err.message ? err.message : 'a figure could not be queued'); + slide.type = slide.type === 'image' ? 'section' : 'bullets'; + delete slide.image_prompt; + } + } + return { jobs: jobs, failures: failures }; +} + +/** + * Parse a model reply into a stored deck. Returns null when there is no usable + * deck in it, so the caller can fall back to asking for markdown instead of + * saving something empty. + */ +function parse(content) { + var raw = extractJson(content); + if (!raw) return null; + var deck = deckSchema.normalise(raw); + if (!deck.slides.length) return null; + return deck; +} + +module.exports = { parse, drawFigures, extractJson, MAX_FIGURES }; diff --git a/src/utils/deckSchema.js b/src/utils/deckSchema.js new file mode 100644 index 00000000..7c55f8c0 --- /dev/null +++ b/src/utils/deckSchema.js @@ -0,0 +1,188 @@ +// ============================================================ +// DECK SCHEMA +// ============================================================ +// What the model fills in when it is writing slides, instead of writing +// markdown for a parser to guess at. +// +// Markdown could express about five of the things the renderer can draw, so the +// model had no way to say "put this figure beside these three bullets" or "make +// this a comparison with two labelled columns" — my parser inferred a layout +// from the shape of a list, and inferring is exactly what produced decks that +// all looked the same. Here the model chooses. +// +// Markdown is still produced, by serialising the deck. That keeps Word export +// and text editing working, and it means the stored artifact is still readable +// by a person. + +var VALID = ['title', 'section', 'bullets', 'two', 'compare', 'table', 'callout', 'figure', 'image']; + +// Given to the model verbatim. Written as prose rather than a JSON Schema dump +// because the failure to avoid is a model that produces valid JSON describing a +// dull deck, not one that produces invalid JSON. +function instructions(slideCount, figureCount) { + return [ + 'Return ONLY a JSON object, no prose and no code fences, shaped like this:', + '', + '{"title": "...", "subtitle": "...", "date": "...", "slides": [ ... ]}', + '', + 'Aim for about ' + slideCount + ' content slides. Every slide is one of these:', + '', + '{"type":"bullets","heading":"...","bullets":[{"text":"...","level":0}],"notes":"..."}', + ' The default. level 1 indents a sub-point. Four to six bullets reads best.', + '', + '{"type":"compare","heading":"...","columns":[{"label":"MILD","bullets":[...]},{"label":"SEVERE","bullets":[...]}]}', + ' Two labelled, tinted columns. Use it for a differential, mild versus severe,', + ' before and after, do and do not. Much stronger than a bulleted list of pairs.', + '', + '{"type":"table","heading":"...","header":["...","..."],"rows":[["...","..."]]}', + ' Real table. Use it whenever the content is genuinely tabular; up to about', + ' six rows reads comfortably on one slide.', + '', + '{"type":"callout","heading":"...","text":"..."}', + ' One sentence on a tinted card. Use it for a red flag, a dose, a rule of', + ' thumb — something worth stopping on. At most one or two per deck.', + '', + '{"type":"figure","heading":"...","bullets":[...],"image_prompt":"..."}', + ' Bullets on the left, an illustration on the right. image_prompt describes the', + ' figure to draw: schematic or anatomical teaching artwork, never a real patient.', + '', + '{"type":"image","heading":"...","image_prompt":"...","caption":"..."}', + ' A full-slide figure. Use it when the picture is the point.', + '', + '{"type":"section","heading":"..."}', + ' A divider between parts of a long deck.', + '', + figureCount + ? 'Include exactly ' + figureCount + ' slides carrying an image_prompt, spread through the deck.' + : 'Include an image_prompt only where a picture genuinely earns its place, and at most three.', + '', + 'Vary the layouts: a deck of nothing but "bullets" is the thing to avoid.', + 'Put a References slide last if you used sources, as a "table" with one column', + 'or a "bullets" slide. "notes" is optional speaker notes.' + ].join('\n'); +} + +function text(value, max) { + return String(value === undefined || value === null ? '' : value).slice(0, max || 400).trim(); +} + +function bullets(list) { + if (!Array.isArray(list)) return []; + return list.slice(0, 14).map(function (item) { + if (typeof item === 'string') return { text: text(item, 600), level: 0 }; + return { text: text(item && item.text, 600), level: Math.max(0, Math.min(4, parseInt((item && item.level) || 0, 10) || 0)) }; + }).filter(function (item) { return item.text; }); +} + +/** + * Accept only what the renderer can draw, and never throw. A model that returns + * one malformed slide should cost that slide, not the deck. + */ +function normalise(raw) { + var deck = raw && typeof raw === 'object' ? raw : {}; + var slides = []; + (Array.isArray(deck.slides) ? deck.slides : []).slice(0, 60).forEach(function (slide) { + if (!slide || typeof slide !== 'object') return; + var type = VALID.indexOf(slide.type) === -1 ? 'bullets' : slide.type; + var out = { type: type, heading: text(slide.heading, 200) }; + if (slide.notes) out.notes = text(slide.notes, 2000); + + if (type === 'compare') { + out.columns = (Array.isArray(slide.columns) ? slide.columns : []).slice(0, 2).map(function (column) { + return { label: text(column && column.label, 60), bullets: bullets(column && column.bullets) }; + }).filter(function (column) { return column.bullets.length; }); + if (out.columns.length < 2) { out.type = 'bullets'; out.bullets = bullets(slide.bullets); } + } else if (type === 'table') { + out.header = (Array.isArray(slide.header) ? slide.header : []).slice(0, 6).map(function (c) { return text(c, 120); }); + out.rows = (Array.isArray(slide.rows) ? slide.rows : []).slice(0, 12).map(function (row) { + return (Array.isArray(row) ? row : []).slice(0, 6).map(function (c) { return text(c, 200); }); + }).filter(function (row) { return row.some(Boolean); }); + if (!out.rows.length) { out.type = 'bullets'; out.bullets = bullets(slide.bullets); } + } else if (type === 'callout') { + out.text = text(slide.text, 400); + if (!out.text) return; + } else if (type === 'title') { + out.subtitle = text(slide.subtitle, 200); + out.date = text(slide.date, 60); + } else if (type === 'section') { + if (!out.heading) return; + } else { + out.bullets = bullets(slide.bullets); + if (type === 'bullets' && !out.bullets.length && !out.heading) return; + } + + // A figure request is a prompt, not a path. The route turns the ones it can + // afford into jobs and hands back files; anything unfulfilled degrades to a + // slide of text rather than an empty frame. + if (type === 'figure' || type === 'image') { + out.image_prompt = text(slide.image_prompt || slide.imagePrompt, 1200); + out.caption = text(slide.caption, 200); + if (!out.image_prompt) out.type = type === 'image' ? 'section' : 'bullets'; + } + slides.push(out); + }); + + return { + title: text(deck.title, 200), + subtitle: text(deck.subtitle, 200), + date: text(deck.date, 60), + slides: slides + }; +} + +// The deck as markdown, so Word export and text editing keep working and the +// stored artifact stays readable. Lossy by design: layout choices do not +// survive, which is why the deck itself is what gets stored. +function toMarkdown(deck) { + var out = []; + if (deck.title) out.push('% ' + deck.title); + if (deck.subtitle) out.push('% ' + deck.subtitle); + if (deck.date) out.push('% ' + deck.date); + if (out.length) out.push(''); + + function list(items, indent) { + (items || []).forEach(function (item) { + out.push(new Array((item.level || 0) + 1).join(' ') + (indent || '') + '- ' + item.text); + }); + } + + deck.slides.forEach(function (slide) { + if (slide.type === 'title') return; + out.push('# ' + (slide.heading || 'Slide')); + out.push(''); + if (slide.type === 'compare') { + (slide.columns || []).forEach(function (column) { + out.push('## ' + column.label); + list(column.bullets); + out.push(''); + }); + } else if (slide.type === 'table') { + if (slide.header && slide.header.length) { + out.push('| ' + slide.header.join(' | ') + ' |'); + out.push('|' + slide.header.map(function () { return '---'; }).join('|') + '|'); + } + (slide.rows || []).forEach(function (row) { out.push('| ' + row.join(' | ') + ' |'); }); + out.push(''); + } else if (slide.type === 'callout') { + out.push('**' + slide.text + '**'); + out.push(''); + } else { + list(slide.bullets); + if (slide.caption) out.push('*' + slide.caption + '*'); + out.push(''); + } + if (slide.notes) { out.push('> ' + slide.notes.replace(/\n/g, ' ')); out.push(''); } + }); + return out.join('\n').replace(/\n{3,}/g, '\n\n').trim() + '\n'; +} + +// Every slide the model asked to illustrate, in deck order. +function figureRequests(deck) { + var wanted = []; + (deck.slides || []).forEach(function (slide, index) { + if (slide.image_prompt) wanted.push({ index: index, prompt: slide.image_prompt, type: slide.type }); + }); + return wanted; +} + +module.exports = { instructions, normalise, toMarkdown, figureRequests, VALID }; diff --git a/src/utils/documentExport.js b/src/utils/documentExport.js index 2dc02f79..fa0513d4 100644 --- a/src/utils/documentExport.js +++ b/src/utils/documentExport.js @@ -101,7 +101,7 @@ async function render(markdown, kind, format, options) { await runPandoc(['doc.md', '-o', 'doc.docx'], workdir); } if (office === 'pptx' || format === 'pptx') { - await buildDeck(markdown, workdir, options.images || []); + await buildDeck(markdown, workdir, options.images || [], options); } if (format !== 'pdf') { @@ -133,10 +133,16 @@ async function render(markdown, kind, format, options) { // // If the renderer fails for any reason, pandoc still produces a deck. A plainer // deck beats a failed download. -async function buildDeck(markdown, workdir, images) { +async function buildDeck(markdown, workdir, images, options) { + options = options || {}; var out = pathMod.join(workdir, 'doc.pptx'); try { - var spec = slideSpec.build(markdown, { images: images }); + // A deck the model designed is rendered as designed. Only a resource made + // before decks existed, or an article being forced into slides, falls back + // to inferring a layout from markdown. + var spec = options.deck + ? attachFigures(options.deck, images, options.figureIds) + : slideSpec.build(markdown, { images: images }); await new Promise(function (resolve, reject) { var child = spawn('python3', [DECK_RENDERER, out], { cwd: workdir }); var stderr = ''; @@ -160,6 +166,31 @@ async function buildDeck(markdown, workdir, images) { } } +// Put the drawn figures back on the slides that asked for them. The files +// arrive in the order the jobs were created, which is the order the slides +// requested them, so a slide is matched by its job id rather than by position. +function attachFigures(deck, files, figureIds) { + var byJob = {}; + (figureIds || []).forEach(function (id, index) { + if (files[index]) byJob[id] = files[index]; + }); + var slides = (deck.slides || []).map(function (slide) { + var copy = Object.assign({}, slide); + if (copy.image_job && byJob[copy.image_job]) copy.image = byJob[copy.image_job]; + // A figure slide whose picture never arrived is still a slide of text. + if (!copy.image && (copy.type === 'image' || copy.type === 'figure')) { + copy.type = copy.type === 'image' ? 'section' : 'bullets'; + } + return copy; + }); + var out = { title: deck.title, subtitle: deck.subtitle, date: deck.date, slides: slides }; + if (deck.title && !slides.some(function (s) { return s.type === 'title'; })) { + out.slides = [{ type: 'title', heading: deck.title, subtitle: deck.subtitle, date: deck.date }] + .concat(slides); + } + return out; +} + // A filename someone can find again, without letting a title choose the path. function filename(title, format) { var safe = String(title || 'resource') diff --git a/test/my-resources.test.js b/test/my-resources.test.js index 83bd08b9..4ba60f26 100644 --- a/test/my-resources.test.js +++ b/test/my-resources.test.js @@ -142,7 +142,10 @@ test('illustration is opt-in, with its own dispatcher rather than the assistant // model to decide there should be a picture and to compose the prompt for it. // Search does not — see web-search.test.js for why both searches were taken // away from the model and run by the route instead. - assert.match(route, /if \(wantsImages\) tools = tools\.concat\(resourceImages\.tools\);/); + // In deck mode the slides name their own figures, so there is nothing for a + // tool to decide; an article still gets the tool, having no structure to hang + // a figure on. + assert.match(route, /if \(wantsImages && !deckMode\) tools = tools\.concat\(resourceImages\.tools\);/); assert.doesNotMatch(route, /tools\.concat\((?:webSearch|pubmedSearch)\.tools\)/); // Its own dispatcher. The assistant's permits one image per request, which is // right for a chat reply and wrong for a deck, and three features depend on diff --git a/test/slide-spec.test.js b/test/slide-spec.test.js index 0e184b79..dc6a1f3a 100644 --- a/test/slide-spec.test.js +++ b/test/slide-spec.test.js @@ -80,7 +80,7 @@ test('the renderer sizes text to fit rather than trusting autofit', () => { test('the deck renderer replaces pandoc, and pandoc still catches it if it falls', () => { const exporter = read('src/utils/documentExport.js'); - assert.match(exporter, /async function buildDeck\(markdown, workdir, images\)/); + assert.match(exporter, /async function buildDeck\(markdown, workdir, images, options\)/); assert.match(exporter, /spawn\('python3', \[DECK_RENDERER, out\]/); // A plainer deck beats a failed download. assert.match(exporter, /deck renderer failed, falling back to pandoc/); @@ -102,5 +102,83 @@ test('a resource remembers its figures, so an export can include them', () => { assert.match(route, /async function collectFigures\(ids, user, dir\)/); // A figure that cannot be fetched is left out rather than failing a download. assert.match(route, /figure unavailable for export/); - assert.match(route, /documentExport\.render\(row\.markdown, row\.kind, format, \{ images: figures \}\)/); + assert.match(route, /documentExport\.render\(row\.markdown, row\.kind, format,\s*\n?\s*\{ images: figures, deck: row\.deck/); +}); + +// ── Deck mode ─────────────────────────────────────────────── +// The model designs the deck instead of writing markdown for a parser to guess +// at. Measured on a live generation: the model chose section, bullets, table, +// compare, figure, callout and image layouts for one topic — none of which +// markdown can ask for. + +const deckSchema = require('../src/utils/deckSchema'); +const deckBuild = require('../src/utils/deckBuild'); + +test('a model reply is read as a deck even when it is wrapped', () => { + // Fences and a covering sentence are common enough that refusing them would + // be pedantry. + assert.ok(deckBuild.extractJson('```json\n{"title":"A","slides":[]}\n```')); + assert.ok(deckBuild.extractJson('Here you go: {"title":"A","slides":[]} hope that helps')); + // A brace inside a string is not the end of the object. + assert.equal(deckBuild.extractJson('{"title":"a } b","slides":[]}').title, 'a } b'); + assert.equal(deckBuild.extractJson('no json at all'), null); + // Nothing usable means fall back to markdown, not save an apology. + assert.equal(deckBuild.parse('{"slides":[]}'), null); +}); + +test('a malformed slide costs that slide, not the deck', () => { + const deck = deckSchema.normalise({ + title: 'T', + slides: [ + { type: 'bullets', heading: 'Fine', bullets: ['one', { text: 'two', level: 1 }] }, + { type: 'nonsense', heading: 'Unknown type', bullets: ['kept'] }, + { type: 'compare', heading: 'Only one column', columns: [{ label: 'A', bullets: ['x'] }] }, + { type: 'table', heading: 'No rows', header: ['a'], rows: [] }, + { type: 'callout', heading: 'Empty', text: '' }, + ], + }); + assert.equal(deck.slides[0].bullets.length, 2); + assert.equal(deck.slides[0].bullets[1].level, 1); + assert.equal(deck.slides[1].type, 'bullets', 'an unknown type degrades rather than throwing'); + assert.equal(deck.slides[2].type, 'bullets', 'a comparison needs two columns to be one'); + assert.equal(deck.slides[3].type, 'bullets', 'an empty table is not a table'); + assert.equal(deck.slides.length, 4, 'a callout with nothing to say is dropped'); +}); + +test('the deck still serialises to markdown, because Word and text edits need it', () => { + const md = deckSchema.toMarkdown(deckSchema.normalise({ + title: 'Croup', subtitle: 'Teaching', date: '2026', + slides: [ + { type: 'compare', heading: 'Versus', columns: [ + { label: 'CROUP', bullets: ['barking cough'] }, + { label: 'EPIGLOTTITIS', bullets: ['drooling'] }] }, + { type: 'table', heading: 'Features', header: ['Feature', 'Croup'], rows: [['Onset', 'Days']] }, + { type: 'callout', heading: 'Red flag', text: 'Do not examine the throat.' }, + ], + })); + assert.match(md, /^% Croup/m); + assert.match(md, /^## CROUP$/m); + assert.match(md, /^\| Feature \| Croup \|$/m); + assert.match(md, /\*\*Do not examine the throat\.\*\*/); +}); + +test('a figure that never arrives leaves a slide of text, not an empty frame', () => { + const exporter = read('src/utils/documentExport.js'); + assert.match(exporter, /function attachFigures\(deck, files, figureIds\)/); + assert.match(exporter, /if \(copy\.image_job && byJob\[copy\.image_job\]\) copy\.image = byJob\[copy\.image_job\];/); + assert.match(exporter, /copy\.type = copy\.type === 'image' \? 'section' : 'bullets';/); + // And a deck the model designed is rendered as designed; only older resources + // fall back to inferring a layout from their markdown. + assert.match(exporter, /options\.deck\s*\n?\s*\? attachFigures/); + + const build = read('src/utils/deckBuild.js'); + assert.match(build, /if \(!wanted\.length \|\| !opts\.imageModel\)/, 'no image model configured is not an error'); + assert.match(build, /a figure could not be queued/); +}); + +test('the renderer can draw what the schema offers', () => { + const py = read('scripts/render_pptx.py'); + for (const type of deckSchema.VALID) { + assert.match(py, new RegExp('"' + type + '": slide_'), type + ' has a builder'); + } });