diff --git a/docs/learning-hub.md b/docs/learning-hub.md
index e0e5e12f..c9b84d03 100644
--- a/docs/learning-hub.md
+++ b/docs/learning-hub.md
@@ -43,7 +43,7 @@ content and can optionally be generated by AI from uploaded source material.
| Input | Notes |
|---|---|
| `topic` | Plain-text description of the topic |
-| Uploaded files | PDF / TXT / MD / HTML / CSV / JSON, ≤ 100 MB each, max 10 files |
+| Uploaded files | PDF / DOCX / PPTX / ODT / EPUB / TXT / MD / HTML / CSV / JSON, ≤ 10 MB each, max 10 files. The declared type must be in the allowlist *and* match the extension, and the bytes are sniffed before anything parses them. |
| WebDAV path | Pulled from the user's connected Nextcloud instance |
Parameters: `model` (from the provider whitelist), `slideCount` for
diff --git a/public/components/cms.html b/public/components/cms.html
index 59b300b4..6b4fb536 100644
--- a/public/components/cms.html
+++ b/public/components/cms.html
@@ -157,7 +157,7 @@
diff --git a/src/routes/learningAI.js b/src/routes/learningAI.js
index 6184eb62..eaa24b25 100644
--- a/src/routes/learningAI.js
+++ b/src/routes/learningAI.js
@@ -18,27 +18,37 @@ var learningRetrieval = require('../utils/learningRetrieval');
router.use(authMiddleware);
router.use(moderatorMiddleware);
+// 10 MiB, down from 100. The whole file is held in memory to be parsed, so the
+// old ceiling meant ten concurrent uploads could ask for a gigabyte of heap —
+// and a source article that large is not a thing anyone uploads here.
+var MAX_SOURCE_BYTES = 10 * 1024 * 1024;
+
+var ALLOWED_SOURCE_TYPES = [
+ 'application/pdf',
+ 'text/plain',
+ 'text/markdown',
+ 'text/html',
+ 'text/csv',
+ 'application/json',
+ 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+ 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+ 'application/vnd.oasis.opendocument.text',
+ 'application/epub+zip'
+];
+var ALLOWED_SOURCE_EXTENSIONS = /\.(pdf|txt|md|html|htm|csv|json|docx|pptx|odt|epub)$/i;
+
var upload = multer({
storage: multer.memoryStorage(),
limits: {
- fileSize: 100 * 1024 * 1024, // 100 MB per file (large PDFs supported)
+ fileSize: MAX_SOURCE_BYTES,
files: 10 // max 10 files at once
},
fileFilter: function(req, file, cb) {
- // Whitelist allowed file types
- var allowed = [
- 'application/pdf',
- 'text/plain',
- 'text/markdown',
- 'text/html',
- 'text/csv',
- 'application/json',
- 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
- 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
- 'application/vnd.oasis.opendocument.text',
- 'application/epub+zip'
- ];
- if (allowed.includes(file.mimetype) || file.originalname.match(/\.(pdf|txt|md|html|htm|csv|json|docx|pptx|odt|epub)$/i)) {
+ // Both, not either. This was `mime OR extension`, so naming a file .pdf was
+ // enough on its own whatever it actually declared — and the extension is
+ // chosen by whoever uploads. The bytes are checked after the buffer exists,
+ // in verifySources() below; a filter only sees the headers.
+ if (ALLOWED_SOURCE_TYPES.includes(file.mimetype) && ALLOWED_SOURCE_EXTENSIONS.test(file.originalname)) {
cb(null, true);
} else {
cb(new Error('File type not allowed. Supported: PDF, DOCX, PPTX, ODT, EPUB, TXT, MD, HTML, CSV, JSON.'));
@@ -46,6 +56,23 @@ var upload = multer({
}
});
+// What multer cannot do: it filters on the declared type, before any bytes have
+// arrived. This runs once the buffer exists and refuses a file whose contents
+// are not what its type claims — the same check documents, S3 uploads and
+// assistant attachments already make, through the same helper.
+function verifySources(files) {
+ var fileType = require('../utils/fileType');
+ (files || []).forEach(function (file) {
+ if (!file || !Buffer.isBuffer(file.buffer)) return;
+ if (file.size > MAX_SOURCE_BYTES) {
+ throw new Error('"' + file.originalname + '" is larger than 10 MB.');
+ }
+ if (!fileType.matches(file.mimetype, file.buffer)) {
+ throw new Error('"' + file.originalname + '" is not the file type it claims to be.');
+ }
+ });
+}
+
// ── Text extraction helpers ──────────────────────────────────
async function extractText(buffer, mimetype, filename) {
@@ -302,6 +329,10 @@ router.post('/ai-generate', upload.array('files', 10), async function(req, res)
// 1 — Uploaded files (multiple)
if (req.files && req.files.length > 0) {
+ // Before a single byte is parsed. An extractor handed a file that is not
+ // what it claims is the place a malformed input does its damage.
+ try { verifySources(req.files); }
+ catch (e) { return res.status(400).json({ error: e.message }); }
var allTexts = [];
for (var i = 0; i < req.files.length; i++) {
var file = req.files[i];
diff --git a/src/utils/embeddings.js b/src/utils/embeddings.js
index c994b166..d2d7ff83 100644
--- a/src/utils/embeddings.js
+++ b/src/utils/embeddings.js
@@ -27,7 +27,7 @@ async function generateEmbedding(text, opts) {
var dimensions = opts.dimensions || (dbDims ? parseInt(dbDims) : 0) || parseInt(process.env.EMBEDDING_DIMENSIONS) || DEFAULT_DIMS;
// Truncate text to ~2000 tokens (~8000 chars) to avoid API errors
- // NOTE: Large PDFs (e.g., 100MB) will be truncated to first ~8000 chars for embedding.
+ // NOTE: A large PDF is truncated to the first ~8000 chars for embedding.
// The full PDF content is still extracted and stored in the database body field.
// This is expected behavior - embeddings are semantic representations, not full-text storage.
var truncated = text.substring(0, 8000);
diff --git a/test/learning-upload-limits.test.js b/test/learning-upload-limits.test.js
new file mode 100644
index 00000000..d53a8709
--- /dev/null
+++ b/test/learning-upload-limits.test.js
@@ -0,0 +1,42 @@
+// Source articles for AI generation are held whole in memory to be parsed, so
+// the ceiling matters, and the declared type is chosen by whoever uploads.
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const route = fs.readFileSync(path.join(__dirname, '..', 'src/routes/learningAI.js'), 'utf8');
+
+test('the per-file ceiling is 10 MB, in one place', () => {
+ assert.match(route, /var MAX_SOURCE_BYTES = 10 \* 1024 \* 1024;/);
+ assert.match(route, /fileSize: MAX_SOURCE_BYTES/);
+ assert.doesNotMatch(route, /100 \* 1024 \* 1024/, 'the old 100 MB ceiling is gone');
+});
+
+test('the type and the extension must both be allowed, not either', () => {
+ // It was `mime OR extension`, so naming a file .pdf was enough on its own.
+ assert.match(route, /ALLOWED_SOURCE_TYPES\.includes\(file\.mimetype\) && ALLOWED_SOURCE_EXTENSIONS\.test\(file\.originalname\)/);
+ assert.doesNotMatch(route, /allowed\.includes\(file\.mimetype\) \|\| file\.originalname\.match/);
+});
+
+test('the bytes are sniffed once the buffer exists, before anything parses them', () => {
+ // multer filters on headers alone, before any byte has arrived.
+ assert.match(route, /function verifySources\(files\)/);
+ assert.match(route, /fileType\.matches\(file\.mimetype, file\.buffer\)/);
+ assert.match(route, /is not the file type it claims to be/);
+ // And it runs ahead of extraction, not after.
+ const call = route.indexOf('verifySources(req.files)');
+ const extract = route.indexOf('await extractText(');
+ assert.ok(call > -1 && call < extract, 'verification must precede extraction');
+});
+
+test('the size is re-checked on the buffer, not trusted from the header', () => {
+ assert.match(route, /file\.size > MAX_SOURCE_BYTES/);
+ assert.match(route, /is larger than 10 MB/);
+});
+
+test('what the screen promises matches what the server accepts', () => {
+ const cms = fs.readFileSync(path.join(__dirname, '..', 'public/components/cms.html'), 'utf8');
+ assert.match(cms, /max 10 MB each, up to 10 files/);
+ assert.doesNotMatch(cms, /100 MB/);
+});