danvics/openreader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(documents): improve SSRF protection and filename sanitization

Browse source 
Enhance SSRF mitigation in web-loader by expanding private IP detection for both
IPv4 and IPv6, including checks for multicast and broadcast addresses. Update
filename sanitization logic in UploadMenuDialog to collapse consecutive
disallowed characters and trim underscores, ensuring cleaner filenames. Remove
redundant userId check in import-url API route for streamlined authentication.
This commit is contained in:
Richard R 2026-06-10 12:59:14 -06:00
parent c2ff4320bf
commit 56305d5c1b
3 changed files with 218 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
src/app/api/documents/import-url/route.ts
View file

@ -10,9 +10,6 @@ export async function POST(req: NextRequest) {
// 1. Authenticate user request
const ctxOrRes = await requireAuthContext(req);
if (ctxOrRes instanceof Response) return ctxOrRes;
if (!ctxOrRes.userId) {
return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
}
// 2. Validate URL body parameter
let body;

8
src/components/documents/UploadMenuDialog.tsx
View file

@ -121,15 +121,13 @@ export function UploadMenuDialog({
const scrapeResult = await importUrl(cleanUrl);
setImportStep('converting');
// Simulated processing time for micro-interaction polish
await new Promise((resolve) => setTimeout(resolve, 800));
setImportStep('uploading');
// Create virtual file from markdown content
const displayTitle = webTitle.trim() || scrapeResult.title || 'Imported Web Page';
const safeTitle = displayTitle
.replace(/[/\\?%*:|"<>\s]/g, '_')
.replace(/[/\\?%*:|"<>\s]+/g, '_') // collapse runs of disallowed chars
.replace(/^_+|_+$/g, '') // trim leading/trailing underscores
.substring(0, 80);
const filename = `${safeTitle}.md`;
const file = new File([scrapeResult.content], filename, {

331
src/lib/server/documents/web-loader.ts
View file

@ -1,4 +1,6 @@
import dns from 'dns';
import http from 'http';
import https from 'https';
import { promisify } from 'util';
import { parseHTML } from 'linkedom';
import { Readability } from '@mozilla/readability';
@ -6,46 +8,80 @@ import TurndownService from 'turndown';
const dnsLookup = promisify(dns.lookup);
const MAX_BYTES = 2 * 1024 * 1024;
const TIMEOUT_MS = 6000;
const MAX_REDIRECTS = 5;
const USER_AGENT =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
/**
* Checks if an IP address is a private, loopback, or link-local address (SSRF mitigation).
* Checks if an IPv4 address falls in a private, loopback, link-local, multicast,
* or broadcast range. Anything that is not a clean dotted-quad is treated as
* unsafe (fail closed).
*/
function isPrivateIp(ip: string): boolean {
// IPv4 checks
if (ip === '127.0.0.1' || ip === '0.0.0.0') return true;
const ipv4Match = ip.match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/);
if (ipv4Match) {
const [, o1, o2] = ipv4Match.map(Number);
if (o1 === 10) return true; // 10.0.0.0/8
if (o1 === 172 && o2 >= 16 && o2 <= 31) return true; // 172.16.0.0/12
if (o1 === 192 && o2 === 168) return true; // 192.168.0.0/16
if (o1 === 169 && o2 === 254) return true; // 169.254.0.0/16
}
// IPv6 checks
const ipLower = ip.toLowerCase();
if (
ipLower === '::1' ||
ipLower === '::' ||
ipLower === '0:0:0:0:0:0:0:1'
) {
return true;
}
if (
ipLower.startsWith('fe80:') ||
ipLower.startsWith('fc00:') ||
ipLower.startsWith('fd00:')
) {
return true;
}
function isPrivateIpv4(ip: string): boolean {
const m = ip.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
if (!m) return true;
const octets = m.slice(1).map(Number);
if (octets.some((o) => o > 255)) return true; // malformed → unsafe
const [o1, o2] = octets;
if (o1 === 0) return true; // 0.0.0.0/8 ("this host")
if (o1 === 10) return true; // 10.0.0.0/8
if (o1 === 127) return true; // 127.0.0.0/8 loopback
if (o1 === 169 && o2 === 254) return true; // 169.254.0.0/16 link-local
if (o1 === 172 && o2 >= 16 && o2 <= 31) return true; // 172.16.0.0/12
if (o1 === 192 && o2 === 168) return true; // 192.168.0.0/16
if (o1 >= 224 && o1 <= 239) return true; // 224.0.0.0/4 multicast
if (ip === '255.255.255.255') return true; // limited broadcast
return false;
}
/**
* Resolves the URL domain and validates that it does not point to a local/private network.
* Checks if an IP address (v4 or v6) is a private, loopback, link-local,
* multicast, or otherwise non-routable address (SSRF mitigation).
*/
async function validateUrlForSsrf(urlString: string): Promise<string> {
function isPrivateIp(ip: string): boolean {
let addr = ip.trim().toLowerCase();
// Strip IPv6 brackets and zone id (e.g. [fe80::1%eth0]).
if (addr.startsWith('[') && addr.endsWith(']')) addr = addr.slice(1, -1);
const zoneIdx = addr.indexOf('%');
if (zoneIdx !== -1) addr = addr.slice(0, zoneIdx);
// Plain IPv4.
if (!addr.includes(':')) return isPrivateIpv4(addr);
// IPv6 from here.
if (addr === '::1' || addr === '::') return true; // loopback / unspecified
// IPv4-mapped (::ffff:a.b.c.d) / -compatible (::a.b.c.d) IPv6: evaluate the
// embedded IPv4 against the v4 ranges. Any address in the mapped range we
// cannot cleanly read as a dotted-quad (e.g. the hex form ::ffff:7f00:1) is
// blocked outright — fail closed. (cf. CVE-2026-47684)
if (addr.startsWith('::ffff:') || /^::\d+\.\d+\.\d+\.\d+$/.test(addr)) {
const m = addr.match(/^::(?:ffff:)?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
return m ? isPrivateIpv4(m[1]) : true;
}
if (/^fe[89ab]/.test(addr)) return true; // fe80::/10 link-local
if (/^fec/.test(addr)) return true; // fec0::/10 (deprecated site-local)
if (/^f[cd]/.test(addr)) return true; // fc00::/7 unique-local
if (/^ff/.test(addr)) return true; // ff00::/8 multicast
return false;
}
interface ValidatedTarget {
url: URL;
/** The DNS-resolved IP address the connection is pinned to. */
address: string;
}
/**
* Resolves the URL domain and validates that it does not point to a local/private
* network. Returns the parsed URL together with the resolved IP address so the
* caller can pin the connection to it, closing the DNS-rebinding window between
* validation and connect.
*/
async function validateUrlForSsrf(urlString: string): Promise<ValidatedTarget> {
let url: URL;
try {
url = new URL(urlString);
@ -63,100 +99,165 @@ async function validateUrlForSsrf(urlString: string): Promise<string> {
throw new Error('Access to localhost is forbidden');
}
let results: { address: string; family: number }[];
try {
const lookupResult = await dnsLookup(hostname);
if (isPrivateIp(lookupResult.address)) {
throw new Error(`Access to private network address is forbidden`);
}
} catch (err) {
if (err instanceof Error && err.message.includes('forbidden')) {
throw err;
}
// If lookup fails entirely, let the fetch attempt fail or resolve naturally
results = await dnsLookup(hostname, { all: true });
} catch {
// Fail closed: do not silently continue and let a later re-resolution decide.
throw new Error('Could not resolve host address');
}
return urlString;
if (results.length === 0) {
throw new Error('Could not resolve host address');
}
// Validate every resolved address; reject if any maps to a private network.
for (const r of results) {
if (isPrivateIp(r.address)) {
throw new Error('Access to private network address is forbidden');
}
}
return { url, address: results[0].address };
}
/**
* Fetches a URL with a size limit (default 2MB) and connection timeout (default 6s).
* Performs a single GET request pinned to the already-validated IP address.
* Resolves with either the response body or a redirect location to follow.
*/
function requestPinned(
target: ValidatedTarget,
maxBytes: number,
timeoutMs: number
): Promise<{ body?: string; redirectLocation?: string }> {
const { url, address } = target;
const isHttps = url.protocol === 'https:';
const lib = isHttps ? https : http;
const port = url.port ? Number(url.port) : isHttps ? 443 : 80;
const options: https.RequestOptions = {
host: address, // connect to the validated IP, not a freshly-resolved one
port,
path: `${url.pathname}${url.search}`,
method: 'GET',
headers: {
// Preserve virtual hosting (host:port) so the origin serves the right vhost.
Host: url.host,
'User-Agent': USER_AGENT,
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
},
timeout: timeoutMs,
};
if (isHttps) {
// Validate the TLS certificate against the original hostname, not the IP.
options.servername = url.hostname;
}
return new Promise((resolve, reject) => {
let settled = false;
const ok = (v: { body?: string; redirectLocation?: string }) => {
if (settled) return;
settled = true;
clearTimeout(hardTimer);
resolve(v);
};
const fail = (e: Error) => {
if (settled) return;
settled = true;
clearTimeout(hardTimer);
reject(e);
};
const req = lib.request(options, (res) => {
const status = res.statusCode ?? 0;
if (status >= 300 && status < 400 && res.headers.location) {
res.resume(); // drain so the socket can be reused/closed
ok({ redirectLocation: res.headers.location });
return;
}
if (status < 200 || status >= 300) {
res.resume();
fail(new Error(`Failed to fetch webpage (status ${status})`));
return;
}
const contentType = (res.headers['content-type'] || '').toString();
const isHtml =
contentType.includes('text/html') ||
contentType.includes('application/xhtml+xml');
const isText = contentType.includes('text/plain');
if (!isHtml && !isText) {
res.resume();
fail(
new Error('Unsupported content type. Only web pages or text files are supported.')
);
return;
}
const contentLength = res.headers['content-length'];
if (contentLength && parseInt(contentLength.toString(), 10) > maxBytes) {
res.resume();
fail(new Error('Web page size exceeds the maximum allowed limit of 2MB.'));
return;
}
// Stream and enforce the size limit dynamically (header may be absent/spoofed).
let received = 0;
const chunks: Buffer[] = [];
res.on('data', (chunk: Buffer) => {
received += chunk.length;
if (received > maxBytes) {
req.destroy();
fail(new Error('Web page size exceeds the maximum allowed limit of 2MB.'));
return;
}
chunks.push(chunk);
});
res.on('end', () => ok({ body: Buffer.concat(chunks).toString('utf-8') }));
res.on('error', fail);
});
// Hard deadline covering the whole request (connect + transfer), in addition
// to the socket idle timeout, to bound slow-read/slowloris responses.
const hardTimer = setTimeout(() => {
req.destroy(new Error('Request timed out'));
}, timeoutMs);
req.on('timeout', () => {
req.destroy(new Error('Request timed out'));
});
req.on('error', fail);
req.end();
});
}
/**
* Validates and fetches a URL with a size limit (default 2MB) and connection
* timeout (default 6s). Each redirect hop is independently re-validated and
* pinned to its resolved IP to prevent SSRF via redirects or DNS rebinding.
*/
async function fetchWithLimit(
urlStr: string,
maxBytes: number = 2 * 1024 * 1024,
timeoutMs: number = 6000
maxBytes: number = MAX_BYTES,
timeoutMs: number = TIMEOUT_MS
): Promise<string> {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
let currentUrl = urlStr;
try {
const res = await fetch(urlStr, {
signal: controller.signal,
headers: {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
},
});
for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
const target = await validateUrlForSsrf(currentUrl);
const result = await requestPinned(target, maxBytes, timeoutMs);
if (!res.ok) {
throw new Error(`Failed to fetch webpage (status ${res.status})`);
if (result.redirectLocation) {
currentUrl = new URL(result.redirectLocation, target.url).href;
continue;
}
const contentType = res.headers.get('content-type') || '';
const isHtml =
contentType.includes('text/html') ||
contentType.includes('application/xhtml+xml');
const isText = contentType.includes('text/plain');
if (!isHtml && !isText) {
throw new Error('Unsupported content type. Only web pages or text files are supported.');
}
const contentLength = res.headers.get('content-length');
if (contentLength && parseInt(contentLength, 10) > maxBytes) {
throw new Error('Web page size exceeds the maximum allowed limit of 2MB.');
}
// Read response body chunks to verify size limit dynamically in case Content-Length header is missing/spoofed
const reader = res.body?.getReader();
if (!reader) {
// Fallback for environments lacking full stream reader support
const text = await res.text();
if (Buffer.byteLength(text) > maxBytes) {
throw new Error('Web page size exceeds the maximum allowed limit of 2MB.');
}
return text;
}
let bytesReceived = 0;
const chunks: Uint8Array[] = [];
while (true) {
const { done, value } = await reader.read();
if (done) break;
if (value) {
bytesReceived += value.length;
if (bytesReceived > maxBytes) {
controller.abort();
throw new Error('Web page size exceeds the maximum allowed limit of 2MB.');
}
chunks.push(value);
}
}
const merged = new Uint8Array(bytesReceived);
let offset = 0;
for (const chunk of chunks) {
merged.set(chunk, offset);
offset += chunk.length;
}
return new TextDecoder().decode(merged);
} finally {
clearTimeout(timeoutId);
return result.body ?? '';
}
throw new Error('Too many redirects while fetching the webpage.');
}
/**
@ -165,10 +266,8 @@ async function fetchWithLimit(
export async function fetchAndParseUrl(
urlStr: string
): Promise<{ title: string; content: string }> {
// 1. SSRF prevention check
await validateUrlForSsrf(urlStr);
// 2. Fetch HTML content with limit
// 1. Fetch HTML content with limit (SSRF validation + IP pinning happens
// inside fetchWithLimit for the initial URL and every redirect hop).
const html = await fetchWithLimit(urlStr);
// 3. Parse HTML string to a virtual DOM document