fix: article uploads are 10 MB, type-checked both ways, and sniffed

The ceiling was 100 MB per file with ten files allowed at once, and every file
is held whole in memory to be parsed — so the old limit let a single request ask
for a gigabyte of heap. A source article that size is not a thing anyone
uploads here. Now 10 MB, defined once and used by both the multer limit and the
post-upload check.

The filter accepted `allowed mime OR allowed extension`, so naming a file .pdf
was enough on its own, whatever it declared — and the extension is chosen by
whoever uploads. Both are required now.

Neither of those sees any bytes: multer filters on the headers, before the file
has arrived. verifySources() runs once the buffer exists and refuses a file
whose contents are not what its type claims, using the same helper as documents,
S3 uploads and assistant attachments. It runs before extraction, because an
extractor handed a malformed file is where the damage would happen.

The CMS screen said 100 MB and listed four of the ten accepted formats; it now
says what the server actually does.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Dv6sqaY6Vq3ChZHMem3cnU

docs/learning-hub.md

@@ -43,7 +43,7 @@ content and can optionally be generated by AI from uploaded source material.
 | Input | Notes |
 |---|---|
 | `topic` | Plain-text description of the topic |
-| Uploaded files | PDF / TXT / MD / HTML / CSV / JSON, ≤ 100 MB each, max 10 files |
+| Uploaded files | PDF / DOCX / PPTX / ODT / EPUB / TXT / MD / HTML / CSV / JSON, ≤ 10 MB each, max 10 files. The declared type must be in the allowlist *and* match the extension, and the bytes are sniffed before anything parses them. |
 | WebDAV path | Pulled from the user's connected Nextcloud instance |
 
 Parameters: `model` (from the provider whitelist), `slideCount` for

public/components/cms.html

@@ -157,7 +157,7 @@
                   <label class="lh-ai-dropzone" id="lh-ai-dropzone">
                     <i class="fas fa-cloud-upload-alt" style="font-size:28px;color:var(--blue);margin-bottom:8px;display:block;"></i>
                     <span id="lh-ai-file-label">Drop files here or click to browse</span>
-                    <small style="color:var(--g400);display:block;margin-top:4px;">PDF, TXT, MD, HTML — max 100 MB each, up to 10 files</small>
+                    <small style="color:var(--g400);display:block;margin-top:4px;">PDF, DOCX, PPTX, ODT, EPUB, TXT, MD, HTML, CSV, JSON — max 10 MB each, up to 10 files</small>
                     <input type="file" id="lh-ai-file" accept=".pdf,.txt,.md,.html,.htm,.csv,.json" multiple style="display:none;">
                   </label>
                   <div id="lh-ai-files-list" style="margin-top:8px;display:none;"></div>

src/routes/learningAI.js

@@ -18,27 +18,37 @@ var learningRetrieval = require('../utils/learningRetrieval');
 router.use(authMiddleware);
 router.use(moderatorMiddleware);
 
+// 10 MiB, down from 100. The whole file is held in memory to be parsed, so the
+// old ceiling meant ten concurrent uploads could ask for a gigabyte of heap —
+// and a source article that large is not a thing anyone uploads here.
+var MAX_SOURCE_BYTES = 10 * 1024 * 1024;
+
+var ALLOWED_SOURCE_TYPES = [
+  'application/pdf',
+  'text/plain',
+  'text/markdown',
+  'text/html',
+  'text/csv',
+  'application/json',
+  'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+  'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+  'application/vnd.oasis.opendocument.text',
+  'application/epub+zip'
+];
+var ALLOWED_SOURCE_EXTENSIONS = /\.(pdf|txt|md|html|htm|csv|json|docx|pptx|odt|epub)$/i;
+
 var upload = multer({
   storage: multer.memoryStorage(),
   limits: {
-    fileSize: 100 * 1024 * 1024, // 100 MB per file (large PDFs supported)
+    fileSize: MAX_SOURCE_BYTES,
     files: 10 // max 10 files at once
   },
   fileFilter: function(req, file, cb) {
-    // Whitelist allowed file types
-    var allowed = [
-      'application/pdf',
-      'text/plain',
-      'text/markdown',
-      'text/html',
-      'text/csv',
-      'application/json',
-      'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
-      'application/vnd.openxmlformats-officedocument.presentationml.presentation',
-      'application/vnd.oasis.opendocument.text',
-      'application/epub+zip'
-    ];
-    if (allowed.includes(file.mimetype) || file.originalname.match(/\.(pdf|txt|md|html|htm|csv|json|docx|pptx|odt|epub)$/i)) {
+    // Both, not either. This was `mime OR extension`, so naming a file .pdf was
+    // enough on its own whatever it actually declared — and the extension is
+    // chosen by whoever uploads. The bytes are checked after the buffer exists,
+    // in verifySources() below; a filter only sees the headers.
+    if (ALLOWED_SOURCE_TYPES.includes(file.mimetype) && ALLOWED_SOURCE_EXTENSIONS.test(file.originalname)) {
       cb(null, true);
     } else {
       cb(new Error('File type not allowed. Supported: PDF, DOCX, PPTX, ODT, EPUB, TXT, MD, HTML, CSV, JSON.'));
@@ -46,6 +56,23 @@ var upload = multer({
   }
 });
 
+// What multer cannot do: it filters on the declared type, before any bytes have
+// arrived. This runs once the buffer exists and refuses a file whose contents
+// are not what its type claims — the same check documents, S3 uploads and
+// assistant attachments already make, through the same helper.
+function verifySources(files) {
+  var fileType = require('../utils/fileType');
+  (files || []).forEach(function (file) {
+    if (!file || !Buffer.isBuffer(file.buffer)) return;
+    if (file.size > MAX_SOURCE_BYTES) {
+      throw new Error('"' + file.originalname + '" is larger than 10 MB.');
+    }
+    if (!fileType.matches(file.mimetype, file.buffer)) {
+      throw new Error('"' + file.originalname + '" is not the file type it claims to be.');
+    }
+  });
+}
+
 // ── Text extraction helpers ──────────────────────────────────
 
 async function extractText(buffer, mimetype, filename) {
@@ -302,6 +329,10 @@ router.post('/ai-generate', upload.array('files', 10), async function(req, res)
 
     // 1 — Uploaded files (multiple)
     if (req.files && req.files.length > 0) {
+      // Before a single byte is parsed. An extractor handed a file that is not
+      // what it claims is the place a malformed input does its damage.
+      try { verifySources(req.files); }
+      catch (e) { return res.status(400).json({ error: e.message }); }
       var allTexts = [];
       for (var i = 0; i < req.files.length; i++) {
         var file = req.files[i];

src/utils/embeddings.js

@@ -27,7 +27,7 @@ async function generateEmbedding(text, opts) {
   var dimensions = opts.dimensions || (dbDims ? parseInt(dbDims) : 0) || parseInt(process.env.EMBEDDING_DIMENSIONS) || DEFAULT_DIMS;
 
   // Truncate text to ~2000 tokens (~8000 chars) to avoid API errors
-  // NOTE: Large PDFs (e.g., 100MB) will be truncated to first ~8000 chars for embedding.
+  // NOTE: A large PDF is truncated to the first ~8000 chars for embedding.
   // The full PDF content is still extracted and stored in the database body field.
   // This is expected behavior - embeddings are semantic representations, not full-text storage.
   var truncated = text.substring(0, 8000);

test/learning-upload-limits.test.js

@@ -0,0 +1,42 @@
+// Source articles for AI generation are held whole in memory to be parsed, so
+// the ceiling matters, and the declared type is chosen by whoever uploads.
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const route = fs.readFileSync(path.join(__dirname, '..', 'src/routes/learningAI.js'), 'utf8');
+
+test('the per-file ceiling is 10 MB, in one place', () => {
+  assert.match(route, /var MAX_SOURCE_BYTES = 10 \* 1024 \* 1024;/);
+  assert.match(route, /fileSize: MAX_SOURCE_BYTES/);
+  assert.doesNotMatch(route, /100 \* 1024 \* 1024/, 'the old 100 MB ceiling is gone');
+});
+
+test('the type and the extension must both be allowed, not either', () => {
+  // It was `mime OR extension`, so naming a file .pdf was enough on its own.
+  assert.match(route, /ALLOWED_SOURCE_TYPES\.includes\(file\.mimetype\) && ALLOWED_SOURCE_EXTENSIONS\.test\(file\.originalname\)/);
+  assert.doesNotMatch(route, /allowed\.includes\(file\.mimetype\) \|\| file\.originalname\.match/);
+});
+
+test('the bytes are sniffed once the buffer exists, before anything parses them', () => {
+  // multer filters on headers alone, before any byte has arrived.
+  assert.match(route, /function verifySources\(files\)/);
+  assert.match(route, /fileType\.matches\(file\.mimetype, file\.buffer\)/);
+  assert.match(route, /is not the file type it claims to be/);
+  // And it runs ahead of extraction, not after.
+  const call = route.indexOf('verifySources(req.files)');
+  const extract = route.indexOf('await extractText(');
+  assert.ok(call > -1 && call < extract, 'verification must precede extraction');
+});
+
+test('the size is re-checked on the buffer, not trusted from the header', () => {
+  assert.match(route, /file\.size > MAX_SOURCE_BYTES/);
+  assert.match(route, /is larger than 10 MB/);
+});
+
+test('what the screen promises matches what the server accepts', () => {
+  const cms = fs.readFileSync(path.join(__dirname, '..', 'public/components/cms.html'), 'utf8');
+  assert.match(cms, /max 10 MB each, up to 10 files/);
+  assert.doesNotMatch(cms, /100 MB/);
+});