// ============================================================ // LEARNING AI ROUTES — AI-assisted content generation for Learning Hub // ============================================================ var express = require('express'); var router = express.Router(); var multer = require('multer'); var axios = require('axios'); var path = require('path'); var { callAI } = require('../utils/ai'); var imageTool = require('../utils/imageTool'); var { authMiddleware, moderatorMiddleware } = require('../middleware/auth'); var db = require('../db/database'); var cryptoUtil = require('../utils/crypto'); var { assertSafeHttpsUrl } = require('../utils/urlSafety'); var learningRetrieval = require('../utils/learningRetrieval'); router.use(authMiddleware); router.use(moderatorMiddleware); var upload = multer({ storage: multer.memoryStorage(), limits: { fileSize: 100 * 1024 * 1024, // 100 MB per file (large PDFs supported) files: 10 // max 10 files at once }, fileFilter: function(req, file, cb) { // Whitelist allowed file types var allowed = [ 'application/pdf', 'text/plain', 'text/markdown', 'text/html', 'text/csv', 'application/json', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', 'application/vnd.openxmlformats-officedocument.presentationml.presentation', 'application/vnd.oasis.opendocument.text', 'application/epub+zip' ]; if (allowed.includes(file.mimetype) || file.originalname.match(/\.(pdf|txt|md|html|htm|csv|json|docx|pptx|odt|epub)$/i)) { cb(null, true); } else { cb(new Error('File type not allowed. Supported: PDF, DOCX, PPTX, ODT, EPUB, TXT, MD, HTML, CSV, JSON.')); } } }); // ── Text extraction helpers ────────────────────────────────── async function extractText(buffer, mimetype, filename) { var ext = (filename || '').split('.').pop().toLowerCase(); // PDF if (mimetype === 'application/pdf' || ext === 'pdf') { try { var pdfParse = require('pdf-parse'); var data = await pdfParse(buffer); return data.text || ''; } catch (e) { throw new Error('Could not parse PDF: ' + e.message); } } // Plain text / markdown / HTML if (mimetype.startsWith('text/') || ['txt', 'md', 'html', 'htm', 'csv'].includes(ext)) { return buffer.toString('utf8'); } // JSON if (mimetype === 'application/json' || ext === 'json') { return buffer.toString('utf8'); } // DOCX if (mimetype === 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' || ext === 'docx') { try { var mammoth = require('mammoth'); var result = await mammoth.extractRawText({ buffer: buffer }); return result.value || ''; } catch (e) { throw new Error('Could not parse DOCX: ' + e.message); } } // PPTX — extract text from slide XML inside the zip if (mimetype === 'application/vnd.openxmlformats-officedocument.presentationml.presentation' || ext === 'pptx') { try { var JSZip = require('jszip'); var zip = await JSZip.loadAsync(buffer); var slides = Object.keys(zip.files).filter(function(f) { return /^ppt\/slides\/slide\d+\.xml$/.test(f); }).sort(); var texts = []; for (var i = 0; i < slides.length; i++) { var xml = await zip.files[slides[i]].async('string'); var slideText = xml.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim(); if (slideText) texts.push('--- Slide ' + (i + 1) + ' ---\n' + slideText); } return texts.join('\n\n'); } catch (e) { throw new Error('Could not parse PPTX: ' + e.message); } } // ODT — extract text from content.xml inside the zip if (mimetype === 'application/vnd.oasis.opendocument.text' || ext === 'odt') { try { var JSZip = require('jszip'); var zip = await JSZip.loadAsync(buffer); var contentXml = await zip.files['content.xml'].async('string'); return contentXml.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim(); } catch (e) { throw new Error('Could not parse ODT: ' + e.message); } } // EPUB — extract text from XHTML chapters inside the zip if (mimetype === 'application/epub+zip' || ext === 'epub') { try { var JSZip = require('jszip'); var zip = await JSZip.loadAsync(buffer); var chapters = Object.keys(zip.files).filter(function(f) { return /\.(xhtml|html|htm)$/i.test(f) && !zip.files[f].dir; }).sort(); var texts = []; for (var i = 0; i < chapters.length; i++) { var html = await zip.files[chapters[i]].async('string'); var chapterText = html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim(); if (chapterText) texts.push(chapterText); } return texts.join('\n\n'); } catch (e) { throw new Error('Could not parse EPUB: ' + e.message); } } // Fallback — try utf8 return buffer.toString('utf8'); } // ── Build AI prompt ────────────────────────────────────────── function buildGeneratePrompt(opts) { var { topic, docText, contentType, questionCount, refinement, wordCount, slideCount, existingCategories, corpusContext } = opts; var source = docText ? 'Based on the following document/resource text, generate educational content.\n\nDOCUMENT:\n"""\n' + docText.substring(0, 50000) + '\n"""\n' : 'Generate educational content on the following topic for a medical professional audience (pediatrics / primary care).\n\nTOPIC: ' + topic + '\n'; // Material from this institution's own indexed documents, when the author // asked for it. It goes before the instructions so the model reads it as the // ground to work from, and it is explicitly preferred over recall: the point // of grounding is that local guidance wins where the two disagree. if (corpusContext) { source += '\nThe following excerpts come from this institution\'s indexed clinical library. ' + 'Prefer them over your own recall wherever they disagree, and do not contradict them. ' + 'They are reference material, not a template: write the resource in your own words.\n\n' + 'LIBRARY EXCERPTS:\n"""\n' + corpusContext + '\n"""\n'; } var refineInstr = refinement ? '\n\nAdditional instructions for tone/style/focus: ' + refinement : ''; var categoryInstr = buildCategoryInstruction(existingCategories); // ── Presentation ── if (contentType === 'presentation') { var slideHint = slideCount ? slideCount + ' slides' : '8-12 slides'; var hasQuestions = parseInt(questionCount) > 0; if (!hasQuestions) { // No questions — return raw Marp markdown only return source + '\n' + 'Create a professional Marp presentation (' + slideHint + ') suitable for medical education. ' + 'Each slide should be focused and readable.' + refineInstr + ` Return ONLY valid Marp markdown (no JSON, no code fences). Start with frontmatter: --- marp: true theme: default --- Then each slide separated by ---. Guidelines: - First slide: title slide with presentation name and brief subtitle - Use # for slide titles - Use bullet points (- ) for lists, keep them concise (max 5 bullets per slide) - A slide that contains a table must contain ONLY that table and its heading. Anything after a table starts a new, untitled slide when the deck is built. - Leave a blank line before and after every table, or it is not read as a table at all and appears as literal pipe characters on the slide. - Do not nest lists more than one level deep, and do not put an ordered list inside a bullet: it overfills the slide. - Prefer more slides with less on each. A slide should hold one idea. - A slide heading is the slide's subject. Do not number it or prefix it with "Slide 3:" — the deck numbers itself. - Include a summary/key takeaways slide at the end - Do NOT include HTML tags or inline styles`; } // With questions — return JSON containing both Marp markdown and questions return source + '\n' + 'Create a professional Marp presentation (' + slideHint + ') suitable for medical education, ' + 'AND ' + questionCount + ' quiz questions based on the content.' + categoryInstr + refineInstr + ` Return ONLY a valid JSON object (no markdown, no code fences) with this exact structure: { "title": "string", "category_name": "string (best existing category name, or a concise new category if none fit)", "marpMarkdown": "string (the complete Marp markdown starting with frontmatter ---\\nmarp: true\\ntheme: default\\n---)", "questions": [ { "question_text": "string", "question_type": "mcq", "explanation": "string", "options": [ { "option_text": "string", "is_correct": false, "explanation": "string" } ] } ] } Marp guidelines inside marpMarkdown: - Start with: ---\\nmarp: true\\ntheme: default\\n--- - Separate slides with \\n---\\n - First slide: title + subtitle. Use # for titles, bullets for content. - Each MCQ must have exactly 4 options, exactly 1 marked is_correct: true`; } var wordHint = wordCount ? ' Target approximately ' + wordCount + ' words for the body.' : ''; var qInstr = parseInt(questionCount) > 0 ? 'then generate ' + questionCount + ' quiz questions.' : 'Do NOT include quiz questions (questions array should be empty []).'; var typeInstr = ''; if (contentType === 'quiz') { typeInstr = 'This is a quiz-only resource. Write a brief introductory body (1-2 paragraphs),' + wordHint + ' ' + qInstr; } else if (contentType === 'pearl') { typeInstr = 'This is a clinical pearl. Write a concise, high-impact body (2-4 paragraphs focusing on key takeaways).' + wordHint + ' ' + qInstr; } else { typeInstr = 'This is an article. Write a comprehensive, well-structured body.' + wordHint + ' ' + qInstr; } return source + '\n' + typeInstr + categoryInstr + refineInstr + ` Return ONLY a valid JSON object (no markdown, no code fences) with this exact structure: { "title": "string", "category_name": "string (best existing category name, or a concise new category if none fit)", "subject": "string (1-3 word sub-topic label)", "body": "string (valid HTML using only: p, h2, h3, ul, ol, li, strong, em, blockquote, code — no inline styles)", "questions": [ { "question_text": "string", "question_type": "mcq", "explanation": "string (general explanation shown after answering)", "options": [ { "option_text": "string", "is_correct": false, "explanation": "string (shown if this wrong option chosen)" } ] } ] } Rules: - Each MCQ must have exactly 4 options, exactly 1 marked is_correct: true - Prefer an existing category_name when it fits; create a short broad category_name only when no existing category fits - question_type must be "mcq" or "true_false" (true_false has exactly 2 options: "True" and "False") - body must be clean HTML, no raw markdown - Do not include any text outside the JSON object`; } function buildCategoryInstruction(existingCategories) { var names = (existingCategories || []).map(function(c) { return c.name; }).filter(Boolean); if (!names.length) { return '\nAssign a short, broad category_name for this content.'; } return '\nChoose the best category_name from this existing list when appropriate: ' + names.join(', ') + '. If none fit, create one short broad category_name.'; } // ── POST /api/admin/learning/ai-generate ──────────────────── // Accepts: multipart/form-data OR application/json router.post('/ai-generate', upload.array('files', 10), async function(req, res) { try { var topic = req.body.topic || ''; var contentType = req.body.contentType || 'article'; var questionCount = Math.min(parseInt(req.body.questionCount) || 0, 20); var model = req.body.model || null; var refinement = req.body.refinement || ''; var webdavPath = req.body.webdavPath || ''; var wordCount = parseInt(req.body.wordCount) || 0; var slideCount = parseInt(req.body.slideCount) || 0; // Opt in. Grounding is the right default for clinical teaching, but a // resource on something the library does not cover is better written // without it than padded with the nearest unrelated excerpts. var useCorpus = String(req.body.useCorpus) === 'true' || req.body.useCorpus === true; if (typeof topic !== 'string' || typeof refinement !== 'string') return res.status(400).json({ error: 'topic and refinement must be text' }); var docText = ''; var fileCount = 0; // 1 — Uploaded files (multiple) if (req.files && req.files.length > 0) { var allTexts = []; for (var i = 0; i < req.files.length; i++) { var file = req.files[i]; try { var text = await extractText(file.buffer, file.mimetype, file.originalname); allTexts.push('### Source File: ' + file.originalname + '\n\n' + text); fileCount++; } catch (e) { console.error('[LearningAI] Failed to extract uploaded file:', e.message); // Continue with other files even if one fails } } docText = allTexts.join('\n\n---\n\n'); } // 2 — Nextcloud WebDAV path else if (webdavPath) { if (!await require('../utils/policy').isFeatureEnabled('nextcloud')) return res.status(403).json({ error: 'Feature disabled' }); var user = await db.get( 'SELECT nextcloud_url, nextcloud_user, nextcloud_token FROM users WHERE id = ?', [req.user.id] ); if (!user || !user.nextcloud_url) { return res.status(400).json({ error: 'Nextcloud not connected. Go to Settings first.' }); } await assertSafeHttpsUrl(user.nextcloud_url, 'Nextcloud URL'); var ncPassword; try { ncPassword = cryptoUtil.decryptString(user.nextcloud_token); } catch (decErr) { return res.status(400).json({ error: 'Nextcloud credentials invalid. Please reconnect.' }); } if (!webdavPath.startsWith('/')) webdavPath = '/' + webdavPath; var fileUrl = user.nextcloud_url + '/remote.php/dav/files/' + encodeURIComponent(user.nextcloud_user) + webdavPath; var response = await axios.get(fileUrl, { auth: { username: user.nextcloud_user, password: ncPassword }, responseType: 'arraybuffer', timeout: 30000, maxRedirects: 0 }); var mimeType = response.headers['content-type'] || 'text/plain'; var fname = path.basename(webdavPath); docText = await extractText(Buffer.from(response.data), mimeType, fname); } // 3 — Topic only (no file) else if (!topic.trim()) { return res.status(400).json({ error: 'Provide a topic or upload a file.' }); } var existingCategories = await db.all('SELECT name FROM learning_categories ORDER BY sort_order ASC, name ASC', []); // Retrieval never fails a generation: without it the resource is written // from the model alone, which is exactly what happened before this // existed. The reason is surfaced so the author is told rather than // quietly handed ungrounded material. var corpus = { sources: [], context: '', reason: 'not requested' }; if (useCorpus) { corpus = await learningRetrieval.retrieve(topic || docText || '', db.getSetting); if (corpus.reason) console.warn('[learning] corpus not used:', corpus.reason); else console.info('[learning] grounded on', corpus.sources.length, 'excerpts'); } var prompt = buildGeneratePrompt({ topic, docText, contentType, questionCount, refinement, wordCount, slideCount, existingCategories, corpusContext: corpus.context }); var aiMessages = [ { role: 'system', content: 'You are a medical education content generator. Return ONLY the requested JSON or Marp markdown — no preamble, no commentary, no code fences, no thinking. Start your response with { or --- as appropriate.' }, { role: 'user', content: prompt } ]; var aiOptions = { model: model, temperature: 0.4, maxTokens: 8000, tools: imageTool.tools }; var result = await callAI(aiMessages, aiOptions); result = await imageTool.dispatch(result, { owner: req.user.id, workflow: 'learning_hub', body: { ...req.body, docText }, imageContext: require('../utils/generatedImages').imageContext( [topic, refinement].filter(Boolean).join('\n\n') || 'Generate educational content from the supplied document.', docText ? [{ role: 'user', content: docText }] : []), messages: aiMessages, options: aiOptions, callAI }); var raw = result.content.trim(); // Strip any leading text before the first { or --- (models sometimes add preamble) if (contentType !== 'presentation' || parseInt(questionCount) > 0) { var jsonStart = raw.indexOf('{'); if (jsonStart > 0) raw = raw.substring(jsonStart); } // ── Presentation ── if (contentType === 'presentation') { if (questionCount > 0) { // JSON response with marpMarkdown + questions var cleanRaw = raw.replace(/^```(?:json)?\s*/i, '').replace(/\s*```\s*$/, ''); var parsedPres; try { parsedPres = JSON.parse(cleanRaw); } catch(e) { var m = cleanRaw.match(/\{[\s\S]*\}/); try { parsedPres = m ? JSON.parse(m[0]) : null; } catch(e2) { parsedPres = null; } } if (parsedPres && parsedPres.marpMarkdown) { return res.json({ success: true, grounding: { used: Boolean(corpus.context), count: corpus.sources.length, reason: corpus.reason || null }, contentType: 'presentation', marpMarkdown: parsedPres.marpMarkdown, category_name: parsedPres.category_name || '', questions: parsedPres.questions || [], imageJobs: result.imageJobs || [], model: result.model }); } } // Plain Marp markdown (no questions requested, or parse failed) var marpMd = raw.replace(/^```(?:markdown|marp)?\s*/i, '').replace(/\s*```\s*$/, ''); return res.json({ success: true, grounding: { used: Boolean(corpus.context), count: corpus.sources.length, reason: corpus.reason || null }, contentType: 'presentation', marpMarkdown: marpMd, questions: [], imageJobs: result.imageJobs || [], model: result.model, docLength: docText.length }); } // Strip code fences and any trailing text after JSON raw = raw.replace(/^```(?:json)?\s*/i, '').replace(/\s*```\s*$/, ''); // Trim trailing non-JSON text (e.g. "Here is the JSON..." after closing brace) var lastBrace = raw.lastIndexOf('}'); if (lastBrace !== -1 && lastBrace < raw.length - 1) raw = raw.substring(0, lastBrace + 1); // Fix unescaped control characters inside JSON string values. // Models sometimes output literal newlines/tabs inside strings instead of \n \t. function sanitizeJsonString(s) { var inStr = false, escaped = false, out = ''; for (var i = 0; i < s.length; i++) { var c = s[i]; if (escaped) { out += c; escaped = false; continue; } if (c === '\\' && inStr) { out += c; escaped = true; continue; } if (c === '"') { inStr = !inStr; out += c; continue; } if (inStr && c === '\n') { out += '\\n'; continue; } if (inStr && c === '\r') { continue; } if (inStr && c === '\t') { out += '\\t'; continue; } out += c; } return out; } var parsed; try { parsed = JSON.parse(raw); } catch (e) { console.error('[LearningAI] Direct parse failed:', e.message, '| Pos:', e.message.match(/position (\d+)/)?.[1]); // Attempt 2: sanitize control chars and retry try { parsed = JSON.parse(sanitizeJsonString(raw)); } catch (e2) { // Attempt 3: extract first { to last } var firstBrace = raw.indexOf('{'); var extractedJson = firstBrace >= 0 ? raw.substring(firstBrace, lastBrace + 1) : null; if (extractedJson) { try { parsed = JSON.parse(sanitizeJsonString(extractedJson)); } catch (e3) { console.error('[LearningAI] All parse attempts failed:', e3.message); return res.status(500).json({ error: 'AI returned invalid JSON. Try again.' }); } } else { console.error('[LearningAI] No JSON braces found'); return res.status(500).json({ error: 'AI returned invalid JSON. Try again.' }); } } } res.json({ success: true, grounding: { used: Boolean(corpus.context), count: corpus.sources.length, reason: corpus.reason || null }, content: parsed, imageJobs: result.imageJobs || [], model: result.model, docLength: docText.length, fileCount: fileCount || (webdavPath ? 1 : 0) }); } catch (err) { console.error('[LearningAI]', err.message); res.status(err.statusCode || 500).json({ error: err.statusCode ? err.message : 'Generation failed' }); } }); // ── POST /api/admin/learning/ai-refine ─────────────────────── // Refine an existing body or specific question router.post('/ai-refine', async function(req, res) { try { var { content, instructions, model } = req.body; if (typeof content !== 'string' || typeof instructions !== 'string' || !content || !instructions.trim()) return res.status(400).json({ error: 'content and instructions required' }); var prompt = `You are editing educational medical content. Refine the following HTML body according to the instructions below. INSTRUCTIONS: ${instructions} CURRENT CONTENT: ${content} For a text-only refinement, return ONLY the refined HTML body (same structure, no JSON wrapper, no markdown fences). Keep all HTML tags intact. If the instruction calls for an image, invoke generate_image instead; the existing body will be preserved regardless of any accompanying text. Do not combine image insertion with a body rewrite.`; var aiMessages = [{ role: 'user', content: prompt }]; var aiOptions = { model: model, temperature: 0.3, maxTokens: 4000, tools: imageTool.tools }; var result = await callAI(aiMessages, aiOptions); // A tool call never authorizes a text rewrite, even if the model also emits HTML. var imageOnly = Boolean(result.toolCalls?.length); if (imageOnly) result = { ...result, content }; result = await imageTool.dispatch(result, { owner: req.user.id, workflow: 'learning_hub', body: req.body, imageContext: require('../utils/generatedImages').imageContext(instructions, [{ role: 'user', content }]), messages: aiMessages, options: aiOptions, callAI }); var refined = imageOnly ? content : result.content.trim().replace(/^```(?:html)?\s*/i, '').replace(/\s*```\s*$/, ''); res.json({ success: true, refined, bodyPreserved: imageOnly, imageJobs: result.imageJobs || [], model: result.model }); } catch (err) { console.error('[LearningAI]', err.message); res.status(err.statusCode || 500).json({ error: err.statusCode ? err.message : 'Request failed' }); } }); // ── GET /api/admin/learning/webdav-browse ──────────────────── // Browse user's Nextcloud folder router.get('/webdav-browse', require('../utils/policy').requireFeature('nextcloud'), async function(req, res) { try { var user = await db.get( 'SELECT nextcloud_url, nextcloud_user, nextcloud_token, nextcloud_folder, webdav_learning_path FROM users WHERE id = ?', [req.user.id] ); if (!user || !user.nextcloud_url) { return res.status(400).json({ error: 'Nextcloud not connected' }); } await assertSafeHttpsUrl(user.nextcloud_url, 'Nextcloud URL'); var ncPassword; try { ncPassword = cryptoUtil.decryptString(user.nextcloud_token); } catch (decErr) { return res.status(400).json({ error: 'Nextcloud credentials invalid. Please reconnect.' }); } var browsePath = req.query.path || user.webdav_learning_path || user.nextcloud_folder || '/'; // Ensure it starts with / if (!browsePath.startsWith('/')) browsePath = '/' + browsePath; var davUrl = user.nextcloud_url + '/remote.php/dav/files/' + encodeURIComponent(user.nextcloud_user) + browsePath; var davResponse = await axios({ method: 'PROPFIND', url: davUrl, auth: { username: user.nextcloud_user, password: ncPassword }, headers: { Depth: '1', 'Content-Type': 'application/xml' }, data: ``, timeout: 15000, maxRedirects: 0 }); // Parse WebDAV XML response var xml = davResponse.data; var items = []; var responseRegex = /([\s\S]*?)<\/d:response>/g; var match; while ((match = responseRegex.exec(xml)) !== null) { var block = match[1]; var hrefMatch = block.match(/([^<]+)<\/d:href>/); var nameMatch = block.match(/([^<]*)<\/d:displayname>/); var typeMatch = block.match(/([^<]*)<\/d:getcontenttype>/); var sizeMatch = block.match(/([^<]*)<\/d:getcontentlength>/); var isCollMatch = block.includes(' elements for page-by-page navigation var slides = []; var sectionReg = /]*>[\s\S]*?<\/section>/g; var match; while ((match = sectionReg.exec(html)) !== null) { slides.push(match[0]); } if (slides.length === 0) slides.push(html); // fallback res.json({ success: true, css: css, slides: slides }); } catch (err) { res.status(500).json({ error: 'Request failed' }); } }); module.exports = router;