pediatric-ai-scribe-v3/src/routes/learningAI.js
Daniel Onyejesi 1b1fe535da Increase PDF/doc context to 50k chars, raise maxTokens ceiling to 8k
Supports large PDFs (~30 pages) from upload and Nextcloud.
maxTokens is a ceiling, not a target — short topics still get short responses.
2026-03-24 19:22:51 -04:00

545 lines
21 KiB
JavaScript

// ============================================================
// LEARNING AI ROUTES — AI-assisted content generation for Learning Hub
// ============================================================
var express = require('express');
var router = express.Router();
var multer = require('multer');
var axios = require('axios');
var path = require('path');
var { callAI } = require('../utils/ai');
var { authMiddleware, moderatorMiddleware } = require('../middleware/auth');
var db = require('../db/database');
router.use(authMiddleware);
router.use(moderatorMiddleware);
var upload = multer({
storage: multer.memoryStorage(),
limits: { fileSize: 20 * 1024 * 1024 } // 20 MB
});
// ── Text extraction helpers ──────────────────────────────────
async function extractText(buffer, mimetype, filename) {
var ext = (filename || '').split('.').pop().toLowerCase();
// PDF
if (mimetype === 'application/pdf' || ext === 'pdf') {
try {
var pdfParse = require('pdf-parse');
var data = await pdfParse(buffer);
return data.text || '';
} catch (e) {
throw new Error('Could not parse PDF: ' + e.message);
}
}
// Plain text / markdown / HTML
if (mimetype.startsWith('text/') || ['txt', 'md', 'html', 'htm', 'csv'].includes(ext)) {
return buffer.toString('utf8');
}
// JSON
if (mimetype === 'application/json' || ext === 'json') {
return buffer.toString('utf8');
}
// Fallback — try utf8
return buffer.toString('utf8');
}
// ── Build AI prompt ──────────────────────────────────────────
function buildGeneratePrompt(opts) {
var { topic, docText, contentType, questionCount, refinement, wordCount, slideCount } = opts;
var source = docText
? 'Based on the following document/resource text, generate educational content.\n\nDOCUMENT:\n"""\n' + docText.substring(0, 50000) + '\n"""\n'
: 'Generate educational content on the following topic for a medical professional audience (pediatrics / primary care).\n\nTOPIC: ' + topic + '\n';
var refineInstr = refinement ? '\n\nAdditional instructions for tone/style/focus: ' + refinement : '';
// ── Presentation ──
if (contentType === 'presentation') {
var slideHint = slideCount ? slideCount + ' slides' : '8-12 slides';
var hasQuestions = parseInt(questionCount) > 0;
if (!hasQuestions) {
// No questions — return raw Marp markdown only
return source + '\n' +
'Create a professional Marp presentation (' + slideHint + ') suitable for medical education. ' +
'Each slide should be focused and readable.' + refineInstr + `
Return ONLY valid Marp markdown (no JSON, no code fences). Start with frontmatter:
---
marp: true
theme: default
---
Then each slide separated by ---. Guidelines:
- First slide: title slide with presentation name and brief subtitle
- Use # for slide titles
- Use bullet points (- ) for lists, keep them concise (max 5 bullets per slide)
- Include a summary/key takeaways slide at the end
- Do NOT include HTML tags or inline styles`;
}
// With questions — return JSON containing both Marp markdown and questions
return source + '\n' +
'Create a professional Marp presentation (' + slideHint + ') suitable for medical education, ' +
'AND ' + questionCount + ' quiz questions based on the content.' + refineInstr + `
Return ONLY a valid JSON object (no markdown, no code fences) with this exact structure:
{
"title": "string",
"marpMarkdown": "string (the complete Marp markdown starting with frontmatter ---\\nmarp: true\\ntheme: default\\n---)",
"questions": [
{
"question_text": "string",
"question_type": "mcq",
"explanation": "string",
"options": [
{ "option_text": "string", "is_correct": false, "explanation": "string" }
]
}
]
}
Marp guidelines inside marpMarkdown:
- Start with: ---\\nmarp: true\\ntheme: default\\n---
- Separate slides with \\n---\\n
- First slide: title + subtitle. Use # for titles, bullets for content.
- Each MCQ must have exactly 4 options, exactly 1 marked is_correct: true`;
}
var wordHint = wordCount ? ' Target approximately ' + wordCount + ' words for the body.' : '';
var qInstr = parseInt(questionCount) > 0
? 'then generate ' + questionCount + ' quiz questions.'
: 'Do NOT include quiz questions (questions array should be empty []).';
var typeInstr = '';
if (contentType === 'quiz') {
typeInstr = 'This is a quiz-only resource. Write a brief introductory body (1-2 paragraphs),' + wordHint + ' ' + qInstr;
} else if (contentType === 'pearl') {
typeInstr = 'This is a clinical pearl. Write a concise, high-impact body (2-4 paragraphs focusing on key takeaways).' + wordHint + ' ' + qInstr;
} else {
typeInstr = 'This is an article. Write a comprehensive, well-structured body.' + wordHint + ' ' + qInstr;
}
return source + '\n' + typeInstr + refineInstr + `
Return ONLY a valid JSON object (no markdown, no code fences) with this exact structure:
{
"title": "string",
"subject": "string (1-3 word sub-topic label)",
"body": "string (valid HTML using only: p, h2, h3, ul, ol, li, strong, em, blockquote, code — no inline styles)",
"questions": [
{
"question_text": "string",
"question_type": "mcq",
"explanation": "string (general explanation shown after answering)",
"options": [
{ "option_text": "string", "is_correct": false, "explanation": "string (shown if this wrong option chosen)" }
]
}
]
}
Rules:
- Each MCQ must have exactly 4 options, exactly 1 marked is_correct: true
- question_type must be "mcq" or "true_false" (true_false has exactly 2 options: "True" and "False")
- body must be clean HTML, no raw markdown
- Do not include any text outside the JSON object`;
}
// ── POST /api/admin/learning/ai-generate ────────────────────
// Accepts: multipart/form-data OR application/json
router.post('/ai-generate', upload.single('file'), async function(req, res) {
try {
var topic = req.body.topic || '';
var contentType = req.body.contentType || 'article';
var questionCount = Math.min(parseInt(req.body.questionCount) || 0, 20);
var model = req.body.model || null;
var refinement = req.body.refinement || '';
var webdavPath = req.body.webdavPath || '';
var wordCount = parseInt(req.body.wordCount) || 0;
var slideCount = parseInt(req.body.slideCount) || 0;
var docText = '';
// 1 — Uploaded file
if (req.file) {
docText = await extractText(req.file.buffer, req.file.mimetype, req.file.originalname);
}
// 2 — Nextcloud WebDAV path
else if (webdavPath) {
var user = await db.get(
'SELECT nextcloud_url, nextcloud_user, nextcloud_token FROM users WHERE id = ?',
[req.user.id]
);
if (!user || !user.nextcloud_url) {
return res.status(400).json({ error: 'Nextcloud not connected. Go to Settings first.' });
}
var fileUrl = user.nextcloud_url + '/remote.php/dav/files/' + user.nextcloud_user + webdavPath;
var response = await axios.get(fileUrl, {
auth: { username: user.nextcloud_user, password: user.nextcloud_token },
responseType: 'arraybuffer',
timeout: 30000
});
var mimeType = response.headers['content-type'] || 'text/plain';
var fname = path.basename(webdavPath);
docText = await extractText(Buffer.from(response.data), mimeType, fname);
}
// 3 — Topic only (no file)
else if (!topic.trim()) {
return res.status(400).json({ error: 'Provide a topic or upload a file.' });
}
var prompt = buildGeneratePrompt({ topic, docText, contentType, questionCount, refinement, wordCount, slideCount });
var result = await callAI(
[
{ role: 'system', content: 'You are a medical education content generator. Return ONLY the requested JSON or Marp markdown — no preamble, no commentary, no code fences, no thinking. Start your response with { or --- as appropriate.' },
{ role: 'user', content: prompt }
],
{ model: model, temperature: 0.4, maxTokens: 8000 }
);
var raw = result.content.trim();
// Strip any leading text before the first { or --- (models sometimes add preamble)
if (contentType !== 'presentation' || parseInt(questionCount) > 0) {
var jsonStart = raw.indexOf('{');
if (jsonStart > 0) raw = raw.substring(jsonStart);
}
// ── Presentation ──
if (contentType === 'presentation') {
if (questionCount > 0) {
// JSON response with marpMarkdown + questions
var cleanRaw = raw.replace(/^```(?:json)?\s*/i, '').replace(/\s*```\s*$/, '');
var parsedPres;
try { parsedPres = JSON.parse(cleanRaw); }
catch(e) {
var m = cleanRaw.match(/\{[\s\S]*\}/);
try { parsedPres = m ? JSON.parse(m[0]) : null; } catch(e2) { parsedPres = null; }
}
if (parsedPres && parsedPres.marpMarkdown) {
return res.json({ success: true, contentType: 'presentation', marpMarkdown: parsedPres.marpMarkdown, questions: parsedPres.questions || [], model: result.model });
}
}
// Plain Marp markdown (no questions requested, or parse failed)
var marpMd = raw.replace(/^```(?:markdown|marp)?\s*/i, '').replace(/\s*```\s*$/, '');
return res.json({ success: true, contentType: 'presentation', marpMarkdown: marpMd, questions: [], model: result.model, docLength: docText.length });
}
// Strip code fences and any trailing text after JSON
raw = raw.replace(/^```(?:json)?\s*/i, '').replace(/\s*```\s*$/, '');
// Trim trailing non-JSON text (e.g. "Here is the JSON..." after closing brace)
var lastBrace = raw.lastIndexOf('}');
if (lastBrace !== -1 && lastBrace < raw.length - 1) raw = raw.substring(0, lastBrace + 1);
// Fix unescaped control characters inside JSON string values.
// Models sometimes output literal newlines/tabs inside strings instead of \n \t.
function sanitizeJsonString(s) {
var inStr = false, escaped = false, out = '';
for (var i = 0; i < s.length; i++) {
var c = s[i];
if (escaped) { out += c; escaped = false; continue; }
if (c === '\\' && inStr) { out += c; escaped = true; continue; }
if (c === '"') { inStr = !inStr; out += c; continue; }
if (inStr && c === '\n') { out += '\\n'; continue; }
if (inStr && c === '\r') { continue; }
if (inStr && c === '\t') { out += '\\t'; continue; }
out += c;
}
return out;
}
var parsed;
try {
parsed = JSON.parse(raw);
} catch (e) {
console.error('[LearningAI] Direct parse failed:', e.message, '| Pos:', e.message.match(/position (\d+)/)?.[1], '| Char context:', raw.substring(parseInt(e.message.match(/position (\d+)/)?.[1] || 0) - 40, parseInt(e.message.match(/position (\d+)/)?.[1] || 0) + 40));
// Attempt 2: sanitize control chars and retry
try { parsed = JSON.parse(sanitizeJsonString(raw)); }
catch (e2) {
// Attempt 3: extract first { to last }
var firstBrace = raw.indexOf('{');
var extractedJson = firstBrace >= 0 ? raw.substring(firstBrace, lastBrace + 1) : null;
if (extractedJson) {
try { parsed = JSON.parse(sanitizeJsonString(extractedJson)); }
catch (e3) {
console.error('[LearningAI] All parse attempts failed:', e3.message);
return res.status(500).json({ error: 'AI returned invalid JSON. Try again.' });
}
} else {
console.error('[LearningAI] No JSON braces found');
return res.status(500).json({ error: 'AI returned invalid JSON. Try again.' });
}
}
}
res.json({
success: true,
content: parsed,
model: result.model,
docLength: docText.length
});
} catch (err) {
console.error('[LearningAI]', err.message);
res.status(500).json({ error: err.message || 'Generation failed' });
}
});
// ── POST /api/admin/learning/ai-refine ───────────────────────
// Refine an existing body or specific question
router.post('/ai-refine', async function(req, res) {
try {
var { content, instructions, model } = req.body;
if (!content || !instructions) return res.status(400).json({ error: 'content and instructions required' });
var prompt = `You are editing educational medical content. Refine the following HTML body according to the instructions below.
INSTRUCTIONS: ${instructions}
CURRENT CONTENT:
${content.substring(0, 8000)}
Return ONLY the refined HTML body (same structure, no JSON wrapper, no markdown fences). Keep all HTML tags intact.`;
var result = await callAI(
[{ role: 'user', content: prompt }],
{ model: model, temperature: 0.3, maxTokens: 4000 }
);
var refined = result.content.trim().replace(/^```(?:html)?\s*/i, '').replace(/\s*```\s*$/, '');
res.json({ success: true, refined, model: result.model });
} catch (err) {
console.error('[LearningAI]', err.message);
res.status(500).json({ error: err.message });
}
});
// ── GET /api/admin/learning/webdav-browse ────────────────────
// Browse user's Nextcloud folder
router.get('/webdav-browse', async function(req, res) {
try {
var user = await db.get(
'SELECT nextcloud_url, nextcloud_user, nextcloud_token, nextcloud_folder, webdav_learning_path FROM users WHERE id = ?',
[req.user.id]
);
if (!user || !user.nextcloud_url) {
return res.status(400).json({ error: 'Nextcloud not connected' });
}
var browsePath = req.query.path || user.webdav_learning_path || user.nextcloud_folder || '/';
// Ensure it starts with /
if (!browsePath.startsWith('/')) browsePath = '/' + browsePath;
var davUrl = user.nextcloud_url + '/remote.php/dav/files/' + user.nextcloud_user + browsePath;
var davResponse = await axios({
method: 'PROPFIND',
url: davUrl,
auth: { username: user.nextcloud_user, password: user.nextcloud_token },
headers: { Depth: '1', 'Content-Type': 'application/xml' },
data: `<?xml version="1.0"?><d:propfind xmlns:d="DAV:"><d:prop><d:displayname/><d:getcontenttype/><d:getcontentlength/><d:resourcetype/></d:prop></d:propfind>`,
timeout: 15000
});
// Parse WebDAV XML response
var xml = davResponse.data;
var items = [];
var responseRegex = /<d:response>([\s\S]*?)<\/d:response>/g;
var match;
while ((match = responseRegex.exec(xml)) !== null) {
var block = match[1];
var hrefMatch = block.match(/<d:href>([^<]+)<\/d:href>/);
var nameMatch = block.match(/<d:displayname>([^<]*)<\/d:displayname>/);
var typeMatch = block.match(/<d:getcontenttype>([^<]*)<\/d:getcontenttype>/);
var sizeMatch = block.match(/<d:getcontentlength>([^<]*)<\/d:getcontentlength>/);
var isCollMatch = block.includes('<d:collection');
if (!hrefMatch) continue;
var href = decodeURIComponent(hrefMatch[1]);
// Strip the /remote.php/dav/files/username prefix to get the relative path
var relPath = href.replace(/^.*\/remote\.php\/dav\/files\/[^/]+/, '') || '/';
items.push({
path: relPath,
name: nameMatch ? nameMatch[1] : path.basename(relPath) || relPath,
isDir: !!isCollMatch,
contentType: typeMatch ? typeMatch[1] : '',
size: sizeMatch ? parseInt(sizeMatch[1]) : 0
});
}
// Filter: skip the parent directory entry (same path as requested)
var normBrowse = browsePath.replace(/\/$/, '');
items = items.filter(function(item) {
var normPath = item.path.replace(/\/$/, '');
return normPath !== normBrowse;
});
// Sort: directories first, then files
items.sort(function(a, b) {
if (a.isDir !== b.isDir) return a.isDir ? -1 : 1;
return a.name.localeCompare(b.name);
});
// Compute parent path
var parent = browsePath.replace(/\/$/, '');
var parentPath = parent.includes('/') ? parent.substring(0, parent.lastIndexOf('/')) || '/' : '/';
res.json({ success: true, path: browsePath, parentPath, items });
} catch (err) {
console.error('[WebDAV]', err.message);
res.status(500).json({ error: 'WebDAV browse failed: ' + err.message });
}
});
// ── POST /api/admin/learning/webdav-path ─────────────────────
// Save user's preferred WebDAV learning path
router.post('/webdav-path', async function(req, res) {
try {
var { path: wPath } = req.body;
await db.run('UPDATE users SET webdav_learning_path = ? WHERE id = ?', [wPath || null, req.user.id]);
res.json({ success: true });
} catch (err) {
res.status(500).json({ error: err.message });
}
});
// ── POST /api/admin/learning/generate-pptx ──────────────────
// Convert Marp markdown to PPTX using pptxgenjs (no browser needed)
router.post('/generate-pptx', async function(req, res) {
try {
var { markdown, title } = req.body;
if (!markdown) return res.status(400).json({ error: 'markdown required' });
var PptxGenJS = require('pptxgenjs');
var pptx = new PptxGenJS();
pptx.layout = 'LAYOUT_WIDE'; // 16:9
// Strip frontmatter
var md = markdown.replace(/^---[\s\S]*?---\n?/, '').trim();
// Split into slides on ---
var rawSlides = md.split(/\n---\n/);
rawSlides.forEach(function(slideText) {
slideText = slideText.trim();
if (!slideText) return;
var slide = pptx.addSlide();
// Add subtle background
slide.background = { color: 'FFFFFF' };
// Extract title (first # heading)
var titleMatch = slideText.match(/^#{1,3}\s+(.+)$/m);
var slideTitle = titleMatch ? titleMatch[1].trim() : '';
var body = slideText.replace(/^#{1,3}\s+.+$/m, '').trim();
// Title
if (slideTitle) {
slide.addText(slideTitle, {
x: 0.5, y: 0.4, w: '90%', h: 0.8,
fontSize: 28, bold: true, color: '1e40af',
fontFace: 'Calibri'
});
}
// Parse body: bullet lines (- item) and paragraphs
if (body) {
var lines = body.split('\n').filter(function(l) { return l.trim(); });
var bullets = [];
var paragraphs = [];
lines.forEach(function(line) {
var bulletMatch = line.match(/^[-*]\s+(.+)/);
if (bulletMatch) {
bullets.push(bulletMatch[1].trim());
} else if (!line.startsWith('#')) {
// Strip inline markdown
var clean = line.replace(/\*\*(.+?)\*\*/g, '$1').replace(/\*(.+?)\*/g, '$1').replace(/`(.+?)`/g, '$1');
paragraphs.push(clean);
}
});
var contentY = slideTitle ? 1.4 : 0.8;
if (bullets.length > 0) {
var bulletObjs = bullets.map(function(b) {
return { text: b, options: { bullet: { type: 'bullet' }, fontSize: 18, color: '374151', paraSpaceBefore: 6 } };
});
slide.addText(bulletObjs, {
x: 0.6, y: contentY, w: '88%', h: (5 - contentY),
fontFace: 'Calibri', valign: 'top'
});
} else if (paragraphs.length > 0) {
slide.addText(paragraphs.join('\n'), {
x: 0.5, y: contentY, w: '90%', h: (5 - contentY),
fontSize: 18, color: '374151', fontFace: 'Calibri',
valign: 'top', wrap: true
});
}
}
// Slide number bottom right
slide.addText('', { x: '85%', y: '90%', w: '12%', h: 0.3, fontSize: 10, color: 'aaaaaa', align: 'right' });
});
// Write to buffer and send
var pptxBuffer = await pptx.write({ outputType: 'nodebuffer' });
var safeTitle = (title || 'presentation').replace(/[^a-zA-Z0-9-_\s]/g, '').replace(/\s+/g, '-').toLowerCase();
res.setHeader('Content-Type', 'application/vnd.openxmlformats-officedocument.presentationml.presentation');
res.setHeader('Content-Disposition', 'attachment; filename="' + safeTitle + '.pptx"');
res.send(pptxBuffer);
} catch (err) {
console.error('[PPTX]', err.message);
res.status(500).json({ error: err.message });
}
});
// ── POST /api/admin/learning/preview-slides ──────────────────
// Render Marp markdown to HTML for in-browser preview
router.post('/preview-slides', async function(req, res) {
try {
var { markdown } = req.body;
if (!markdown) return res.status(400).json({ error: 'markdown required' });
var { Marp } = require('@marp-team/marp-core');
var marp = new Marp({ html: false });
var { html, css } = marp.render(markdown);
// Extract individual <section> elements for page-by-page navigation
var slides = [];
var sectionReg = /<section[^>]*>[\s\S]*?<\/section>/g;
var match;
while ((match = sectionReg.exec(html)) !== null) {
slides.push(match[0]);
}
if (slides.length === 0) slides.push(html); // fallback
res.json({ success: true, css: css, slides: slides });
} catch (err) {
res.status(500).json({ error: err.message });
}
});
module.exports = router;