diff --git a/.env.example b/.env.example index 709d40c1..a9e42d9c 100644 --- a/.env.example +++ b/.env.example @@ -287,3 +287,9 @@ DB_PASSWORD=pedscribe_secret_change_me # Docker network, which needs no token. Requests arriving through the reverse # proxy (they carry X-Forwarded-For) get a 404 either way. METRICS_TOKEN= + +# Gotenberg (LibreOffice behind an HTTP API), used to turn a generated deck or +# document into PDF. Defaults to http://gotenberg:3000 on the danvics_convert +# network. PDF is the one export allowed to fail: if this is unreachable the +# PowerPoint and Word downloads still work. +GOTENBERG_URL= diff --git a/docker-compose.yml b/docker-compose.yml index bbdb65df..1d9d1412 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,6 +63,7 @@ services: - danvics_monitoring - ped-ai-storage-assets - danvics_translate + - danvics_convert healthcheck: test: ["CMD", "wget", "--spider", "-q", "http://localhost:3000/api/health"] interval: 30s @@ -128,3 +129,7 @@ networks: # hostname, which coupled a clinical app to an unrelated stack's lifecycle. danvics_translate: external: true + # Gotenberg, for turning a generated deck or document into PDF. A convenience + # export: if this is unreachable the pptx and docx still download. + danvics_convert: + external: true diff --git a/migrations/1780300000000_user-resources.js b/migrations/1780300000000_user-resources.js new file mode 100644 index 00000000..9c678885 --- /dev/null +++ b/migrations/1780300000000_user-resources.js @@ -0,0 +1,39 @@ +// Resources a user generated for themselves. +// +// Learning content is moderator-owned and published into categories for +// everyone. This is the other thing people wanted: somewhere to generate a deck +// for tomorrow's teaching session without it becoming institutional content, +// and without needing to be a moderator to do it at all. +// +// Private by construction. Every query filters on user_id, and the foreign key +// cascades, so deleting an account takes its drafts with it. There is no +// category, no publish state and no sharing: this table is one person's +// workspace, and adding sharing later should be a deliberate decision rather +// than something that leaks out of a missing WHERE clause. + +exports.up = pgm => { + pgm.sql(` + CREATE TABLE IF NOT EXISTS user_resources ( + id SERIAL PRIMARY KEY, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + title TEXT NOT NULL DEFAULT 'Untitled', + -- presentation | article. Decides which prompt writes it and which + -- formats it exports to. + kind TEXT NOT NULL DEFAULT 'presentation', + -- Markdown is the artifact. Every export is rendered from it on demand, + -- so refining means editing text rather than patching a binary. + markdown TEXT NOT NULL DEFAULT '', + topic TEXT NOT NULL DEFAULT '', + -- How many corpus excerpts it was written from; 0 means the model alone. + grounded_count INTEGER NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + CREATE INDEX IF NOT EXISTS idx_user_resources_owner + ON user_resources(user_id, created_at DESC); + `); +}; + +exports.down = pgm => { + pgm.sql('DROP TABLE IF EXISTS user_resources;'); +}; diff --git a/server.js b/server.js index 351c4577..509a8e69 100644 --- a/server.js +++ b/server.js @@ -298,6 +298,9 @@ app.get('/api/health/detailed', _hcAuth, _hcAdmin, (req, res) => { // Learning Hub routes (all authenticated users can read content & take quizzes) app.use('/api/learning', require('./src/routes/learningHub')); +// A person's own generated teaching material. Separate from Learning, which is +// moderator-owned and published; this is private to whoever made it. +app.use('/api', require('./src/routes/myResources')); // Authenticated feature routes app.use('/api', require('./src/routes/transcribe')); diff --git a/src/routes/myResources.js b/src/routes/myResources.js new file mode 100644 index 00000000..b3e1825c --- /dev/null +++ b/src/routes/myResources.js @@ -0,0 +1,257 @@ +// ============================================================ +// MY RESOURCES +// A person's own generated teaching material. +// +// Deliberately a separate pathway from Learning. Learning is moderator-owned: +// content published into categories that everyone sees. This is the other +// thing — somewhere any signed-in user can generate a deck for tomorrow's +// session, keep it, refine it and export it, without it becoming institutional +// content and without needing to be a moderator to do it at all. +// +// Nothing here is shared. Every statement filters on the owner, and there is no +// route that returns another person's work. Sharing, if it is ever wanted, +// should be a deliberate feature rather than something that leaks out of a +// forgotten WHERE clause. +// ============================================================ + +var express = require('express'); +var router = express.Router(); +var db = require('../db/database'); +var { authMiddleware } = require('../middleware/auth'); +var { callAI } = require('../utils/ai'); +var learningRetrieval = require('../utils/learningRetrieval'); +var documentExport = require('../utils/documentExport'); + +// Scoped to this router's own prefix. Mounted on /api, a bare +// router.use(authMiddleware) would gate every /api path below it in server.js. +router.use('/my-resources', authMiddleware); + +var MAX_PER_USER = 100; +var MAX_TITLE = 160; + +function clampInt(value, min, max, fallback) { + var n = parseInt(value, 10); + if (!Number.isFinite(n)) return fallback; + return Math.min(max, Math.max(min, n)); +} + +// Presentation and article are the two shapes markdown renders well into, and +// the only two the exporter knows. Anything else is rejected rather than +// guessed at. +function normalizeKind(kind) { + return String(kind) === 'article' ? 'article' : 'presentation'; +} + +function buildPrompt(opts) { + var kind = opts.kind; + var grounding = opts.corpusContext + ? '\nThe following excerpts come from this institution\'s indexed clinical library. ' + + 'Prefer them over your own recall wherever they disagree, and do not contradict them. ' + + 'They are reference material, not a template: write the resource in your own words.\n\n' + + 'Do NOT cite in the body: no [1] markers, no bracketed numbers, no parenthetical ' + + '"(Nelson, p. 2604)" inside sentences.\n\n' + + 'End with a References section listing only the excerpts you actually drew on, by title ' + + 'and page. In a presentation that is the final slide, titled References. Do not invent ' + + 'references, and do not list an excerpt you did not use.\n\n' + + 'LIBRARY EXCERPTS:\n"""\n' + opts.corpusContext + '\n"""\n' + : ''; + + var shape = kind === 'presentation' + ? 'Write a ' + opts.slideCount + '-slide teaching presentation.\n\n' + + 'Output ONLY Pandoc markdown:\n' + + '- Start with three lines each beginning with %: title, author, date\n' + + '- One level-1 heading (#) per slide; the heading is the slide\'s subject, not "Slide 3:"\n' + + '- Bullets, ordered lists, bold and italics are fine; do not nest lists more than one level\n' + + '- A slide containing a table contains ONLY that table, and a table needs a blank line\n' + + ' before and after it, or it will not render as a table\n' + + '- Prefer more slides with less on each; one idea per slide\n' + : 'Write a teaching article of roughly ' + opts.wordCount + ' words.\n\n' + + 'Output ONLY Pandoc markdown: a level-1 heading for the title, then level-2 headings for ' + + 'sections. Use prose, not slide bullets.\n'; + + return 'You are writing teaching material for a medical professional audience ' + + '(pediatrics / primary care).\n\nTOPIC: ' + opts.topic + '\n' + grounding + '\n' + shape + + (opts.refinement ? '\nAdditional instructions: ' + opts.refinement + '\n' : ''); +} + +function firstHeading(markdown, fallback) { + var m = String(markdown || '').match(/^%\s*(.+)$/m) || String(markdown || '').match(/^#\s+(.+)$/m); + return (m ? m[1] : fallback || 'Untitled').trim().slice(0, MAX_TITLE); +} + +// ── Generate ──────────────────────────────────────────────── +router.post('/my-resources/generate', async function (req, res) { + try { + var topic = String(req.body.topic || '').trim(); + if (!topic) return res.status(400).json({ error: 'A topic is required' }); + + var kind = normalizeKind(req.body.kind); + var useCorpus = String(req.body.useCorpus) !== 'false'; + var refinement = String(req.body.refinement || '').slice(0, 2000); + + var count = await db.get('SELECT COUNT(*)::int AS n FROM user_resources WHERE user_id = ?', [req.user.id]); + if (count && count.n >= MAX_PER_USER) { + return res.status(409).json({ error: 'You have reached ' + MAX_PER_USER + ' saved resources. Delete one first.' }); + } + + // Retrieval never fails a generation; the resource is then written from the + // model alone, and the response says so. + var corpus = { sources: [], context: '', reason: 'not requested' }; + if (useCorpus) corpus = await learningRetrieval.retrieve(topic, db.getSetting); + + var prompt = buildPrompt({ + topic: topic, kind: kind, refinement: refinement, corpusContext: corpus.context, + slideCount: clampInt(req.body.slideCount, 3, 30, 8), + wordCount: clampInt(req.body.wordCount, 200, 3000, 800) + }); + + var ai = await callAI([{ role: 'user', content: prompt }], { + model: req.body.model || undefined, temperature: 0.3 + }); + var markdown = String((ai && ai.content) || '').trim(); + if (!markdown) return res.status(502).json({ error: 'The model returned nothing. Try again.' }); + + var row = await db.get( + 'INSERT INTO user_resources (user_id, title, kind, markdown, topic, grounded_count) ' + + 'VALUES (?, ?, ?, ?, ?, ?) RETURNING id, title, kind, topic, grounded_count, created_at', + [req.user.id, firstHeading(markdown, topic), kind, markdown, topic.slice(0, 500), corpus.sources.length] + ); + + res.json({ + success: true, + resource: row, + markdown: markdown, + grounding: { used: Boolean(corpus.context), count: corpus.sources.length, reason: corpus.reason || null }, + model: ai && ai.model + }); + } catch (err) { + console.error('[my-resources] generate:', err.message); + res.status(err.statusCode || 500).json({ error: err.statusCode ? err.message : 'Generation failed' }); + } +}); + +// ── The owner's library ───────────────────────────────────── +router.get('/my-resources', async function (req, res) { + try { + var rows = await db.all( + 'SELECT id, title, kind, topic, grounded_count, created_at, updated_at ' + + 'FROM user_resources WHERE user_id = ? ORDER BY created_at DESC LIMIT ?', + [req.user.id, MAX_PER_USER] + ); + res.json({ success: true, resources: rows }); + } catch (err) { + console.error('[my-resources] list:', err.message); + res.status(500).json({ error: 'Could not load your resources' }); + } +}); + +router.get('/my-resources/:id', async function (req, res) { + try { + var row = await db.get( + 'SELECT id, title, kind, topic, markdown, grounded_count, created_at, updated_at ' + + 'FROM user_resources WHERE id = ? AND user_id = ?', + [parseInt(req.params.id, 10), req.user.id] + ); + if (!row) return res.status(404).json({ error: 'Not found' }); + res.json({ success: true, resource: row }); + } catch (err) { + console.error('[my-resources] get:', err.message); + res.status(500).json({ error: 'Could not load that resource' }); + } +}); + +// ── Edit and refine ───────────────────────────────────────── +router.put('/my-resources/:id', async function (req, res) { + try { + var markdown = String(req.body.markdown || ''); + if (!markdown.trim()) return res.status(400).json({ error: 'markdown is required' }); + var row = await db.get( + 'UPDATE user_resources SET markdown = ?, title = ?, updated_at = NOW() ' + + 'WHERE id = ? AND user_id = ? RETURNING id, title, updated_at', + [markdown, firstHeading(markdown), parseInt(req.params.id, 10), req.user.id] + ); + if (!row) return res.status(404).json({ error: 'Not found' }); + res.json({ success: true, resource: row }); + } catch (err) { + console.error('[my-resources] update:', err.message); + res.status(500).json({ error: 'Could not save' }); + } +}); + +router.post('/my-resources/:id/refine', async function (req, res) { + try { + var instructions = String(req.body.instructions || '').trim(); + if (!instructions) return res.status(400).json({ error: 'Say what to change' }); + + var existing = await db.get( + 'SELECT id, kind, markdown FROM user_resources WHERE id = ? AND user_id = ?', + [parseInt(req.params.id, 10), req.user.id] + ); + if (!existing) return res.status(404).json({ error: 'Not found' }); + + // The markdown is the thing being edited, which is the whole reason it is + // what gets stored: "change slide 4" is a text edit, not a binary patch. + var ai = await callAI([{ role: 'user', content: + 'Revise the following Pandoc markdown according to the instruction. ' + + 'Return ONLY the complete revised markdown, no commentary, no code fences. ' + + 'Keep the same overall structure unless the instruction asks otherwise, and keep any ' + + 'References section at the end.\n\nINSTRUCTION: ' + instructions + + '\n\nMARKDOWN:\n"""\n' + existing.markdown + '\n"""' }], + { model: req.body.model || undefined, temperature: 0.2 }); + + var revised = String((ai && ai.content) || '').trim(); + if (!revised) return res.status(502).json({ error: 'The model returned nothing. Try again.' }); + + var row = await db.get( + 'UPDATE user_resources SET markdown = ?, title = ?, updated_at = NOW() ' + + 'WHERE id = ? AND user_id = ? RETURNING id, title, updated_at', + [revised, firstHeading(revised), existing.id, req.user.id] + ); + res.json({ success: true, resource: row, markdown: revised, model: ai && ai.model }); + } catch (err) { + console.error('[my-resources] refine:', err.message); + res.status(err.statusCode || 500).json({ error: err.statusCode ? err.message : 'Refinement failed' }); + } +}); + +// ── Export ────────────────────────────────────────────────── +router.get('/my-resources/:id/export', async function (req, res) { + try { + var format = String(req.query.format || 'pptx'); + if (!documentExport.isSupported(format)) return res.status(400).json({ error: 'Unsupported format' }); + + var row = await db.get( + 'SELECT title, kind, markdown FROM user_resources WHERE id = ? AND user_id = ?', + [parseInt(req.params.id, 10), req.user.id] + ); + if (!row) return res.status(404).json({ error: 'Not found' }); + + var bytes = await documentExport.render(row.markdown, row.kind, format); + res.setHeader('Content-Type', documentExport.mimeFor(format)); + res.setHeader('Content-Disposition', + 'attachment; filename="' + documentExport.filename(row.title, format) + '"'); + res.send(bytes); + } catch (err) { + console.error('[my-resources] export:', err.message); + // PDF is the one export that depends on another service. Say which failed + // rather than reporting a generic error for a download that works in two + // other formats. + res.status(502).json({ error: String(req.query.format) === 'pdf' + ? 'PDF conversion is unavailable right now. PowerPoint and Word still work.' + : 'Could not build that file' }); + } +}); + +router.delete('/my-resources/:id', async function (req, res) { + try { + var result = await db.run('DELETE FROM user_resources WHERE id = ? AND user_id = ?', + [parseInt(req.params.id, 10), req.user.id]); + if (!result.changes) return res.status(404).json({ error: 'Not found' }); + res.json({ success: true }); + } catch (err) { + console.error('[my-resources] delete:', err.message); + res.status(500).json({ error: 'Could not delete' }); + } +}); + +module.exports = router; diff --git a/src/utils/documentExport.js b/src/utils/documentExport.js new file mode 100644 index 00000000..8d766b1c --- /dev/null +++ b/src/utils/documentExport.js @@ -0,0 +1,98 @@ +// ============================================================ +// DOCUMENT EXPORT +// Markdown in, PowerPoint / Word / PDF out. +// +// Markdown is the artifact everywhere: it is what the model writes, what a +// refinement edits, and what is stored. Every format here is rendered from it +// on demand, so nothing ever has to patch a binary to change a slide. +// +// pandoc does pptx and docx in the image. PDF needs a renderer pandoc does not +// ship, so it goes to Gotenberg, which is LibreOffice behind an HTTP API. That +// is a network call, and it is the one export allowed to fail: if Gotenberg is +// down the deck and the document still download, and only PDF is unavailable. +// ============================================================ + +var fsp = require('fs/promises'); +var os = require('os'); +var pathMod = require('path'); +var { execFile } = require('child_process'); + +var REFERENCE_DECK = pathMod.join(__dirname, '..', '..', 'assets', 'learning', 'slides-reference.pptx'); +var GOTENBERG = process.env.GOTENBERG_URL || 'http://gotenberg:3000'; + +var FORMATS = { + pptx: { ext: 'pptx', mime: 'application/vnd.openxmlformats-officedocument.presentationml.presentation' }, + docx: { ext: 'docx', mime: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' }, + pdf: { ext: 'pdf', mime: 'application/pdf' } +}; + +function isSupported(format) { return Object.hasOwn(FORMATS, String(format)); } +function mimeFor(format) { return (FORMATS[format] || {}).mime; } + +function runPandoc(args, cwd) { + return new Promise(function (resolve, reject) { + execFile('pandoc', args, { cwd: cwd, timeout: 60000, maxBuffer: 1024 * 1024 }, + function (err, stdout, stderr) { + if (err) return reject(new Error(String(stderr || err.message).slice(0, 400))); + resolve(); + }); + }); +} + +/** + * Render markdown to one format. + * + * `kind` decides how a document is laid out: a presentation becomes slides with + * the reference deck's fonts and layouts, an article becomes a Word document. + * PDF is produced by converting whichever of those two applies, so a PDF of a + * presentation looks like the presentation rather than like a long page. + * + * Returns a Buffer. Throws with a readable message; the caller decides whether + * a failed PDF is fatal. + */ +async function render(markdown, kind, format) { + if (!isSupported(format)) throw new Error('Unsupported format: ' + format); + var workdir = await fsp.mkdtemp(pathMod.join(os.tmpdir(), 'export-')); + try { + await fsp.writeFile(pathMod.join(workdir, 'doc.md'), String(markdown || ''), 'utf8'); + + var office = kind === 'presentation' ? 'pptx' : 'docx'; + var officeArgs = ['doc.md', '-o', 'doc.' + office]; + if (office === 'pptx') officeArgs.splice(1, 0, '--reference-doc=' + REFERENCE_DECK); + await runPandoc(officeArgs, workdir); + + if (format !== 'pdf') { + if (format !== office) { + // Asking for docx of a presentation, or pptx of an article. Render the + // other one rather than refusing: the markdown supports both. + var otherArgs = ['doc.md', '-o', 'doc.' + format]; + if (format === 'pptx') otherArgs.splice(1, 0, '--reference-doc=' + REFERENCE_DECK); + await runPandoc(otherArgs, workdir); + } + return await fsp.readFile(pathMod.join(workdir, 'doc.' + format)); + } + + // PDF: hand the office file to Gotenberg. Its LibreOffice keeps the deck's + // layout, which is why this is not rendered from the markdown directly. + var bytes = await fsp.readFile(pathMod.join(workdir, 'doc.' + office)); + var form = new FormData(); + form.append('files', new File([bytes], 'doc.' + office, { type: FORMATS[office].mime })); + var response = await fetch(GOTENBERG + '/forms/libreoffice/convert', { + method: 'POST', body: form, signal: AbortSignal.timeout(90000) + }); + if (!response.ok) throw new Error('PDF conversion failed (' + response.status + ')'); + return Buffer.from(await response.arrayBuffer()); + } finally { + try { await fsp.rm(workdir, { recursive: true, force: true }); } + catch (e) { console.warn('[export] could not clean', workdir, e.message); } + } +} + +// A filename someone can find again, without letting a title choose the path. +function filename(title, format) { + var safe = String(title || 'resource') + .replace(/[^a-zA-Z0-9-_\s]/g, '').replace(/\s+/g, '-').toLowerCase().slice(0, 60) || 'resource'; + return safe + '.' + (FORMATS[format] || FORMATS.pdf).ext; +} + +module.exports = { render, filename, mimeFor, isSupported, FORMATS }; diff --git a/test/my-resources.test.js b/test/my-resources.test.js new file mode 100644 index 00000000..d7af6540 --- /dev/null +++ b/test/my-resources.test.js @@ -0,0 +1,80 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); + +const root = path.join(__dirname, '..'); +const read = file => fs.readFileSync(path.join(root, file), 'utf8'); + +test('a person’s own resources are a separate pathway from Learning', () => { + const route = read('src/routes/myResources.js'); + const learning = read('src/routes/learningAI.js'); + + // Learning stays moderator-owned. This exists so that not being a moderator + // no longer means not being able to generate anything at all. + assert.match(learning, /router\.use\(moderatorMiddleware\)/, 'Learning is unchanged'); + assert.doesNotMatch(route, /moderatorMiddleware/, 'and this one never mentions it'); + assert.match(route, /router\.use\('\/my-resources', authMiddleware\)/, + 'signed in is the only requirement, and the gate names its own prefix'); +}); + +test('nothing here can return another person’s work', () => { + const route = read('src/routes/myResources.js'); + // Every statement that touches the table filters on the owner. A missing + // WHERE clause here is the whole risk, so it is asserted rather than assumed. + const statements = route.match(/'(SELECT|UPDATE|DELETE|INSERT)[^']*(?:' \+\s*\n\s*'[^']*)*'/g) || []; + const touching = statements.filter(s => /user_resources/.test(s)); + assert.ok(touching.length >= 5, 'expected the table statements to be found'); + for (const s of touching) { + if (/^'INSERT/.test(s)) continue; // supplies user_id as a value instead + assert.match(s, /user_id = \?/, 'every read and write is scoped to the owner: ' + s.slice(0, 60)); + } + assert.match(route, /INSERT INTO user_resources \(user_id,/, 'and an insert records one'); +}); + +test('markdown is the artifact; every format is rendered from it', () => { + const exporter = read('src/utils/documentExport.js'); + // Refining means editing text, never patching a binary — which is what makes + // "change slide 4" possible at all. + assert.match(exporter, /async function render\(markdown, kind, format\)/); + assert.match(exporter, /var office = kind === 'presentation' \? 'pptx' : 'docx';/); + assert.match(exporter, /--reference-doc=' \+ REFERENCE_DECK/, 'decks keep the house template'); + + // PDF goes through Gotenberg because pandoc ships no PDF engine in this + // image, and converting the office file preserves the deck's layout. + assert.match(exporter, /forms\/libreoffice\/convert/); + assert.match(exporter, /AbortSignal\.timeout\(90000\)/, 'and cannot hang a request'); + + // Temporary directories are always cleaned, including on failure. + assert.match(exporter, /\} finally \{[\s\S]{0,200}rm\(workdir, \{ recursive: true, force: true \}\)/); +}); + +test('a failed PDF says so, because the other two formats still work', () => { + const route = read('src/routes/myResources.js'); + assert.match(route, /PDF conversion is unavailable right now\. PowerPoint and Word still work\./); + // Gotenberg is a different stack; PDF is the one export allowed to fail. + assert.match(read('src/utils/documentExport.js'), /GOTENBERG_URL \|\| 'http:\/\/gotenberg:3000'/); + assert.match(read('docker-compose.yml'), /danvics_convert/, 'and ped-ai is on its network'); +}); + +test('the generated markdown is told the rules pandoc enforces', () => { + const route = read('src/routes/myResources.js'); + // The same rules the Learning prompt carries, found by rendering decks and + // looking at them: a table alone on its slide, blank lines around it, no + // "Slide 3:" prefixes, no deep nesting. + assert.match(route, /A slide containing a table contains ONLY that table/); + assert.match(route, /a table needs a blank line/); + assert.match(route, /the heading is the slide\\'s subject, not "Slide 3:"/); + // And grounded resources cite only at the end. + assert.match(route, /Do NOT cite in the body/); + assert.match(route, /In a presentation that is the final slide, titled References/); +}); + +test('a library has a ceiling, and generation says when it is reached', () => { + const route = read('src/routes/myResources.js'); + assert.match(route, /var MAX_PER_USER = 100;/); + assert.match(route, /You have reached ' \+ MAX_PER_USER \+ ' saved resources/); + // The count is per owner, so one person filling their library cannot stop + // anyone else generating. + assert.match(route, /SELECT COUNT\(\*\)::int AS n FROM user_resources WHERE user_id = \?/); +});