diff --git a/src/app/api/documents/import-url/route.ts b/src/app/api/documents/import-url/route.ts index 62e4df1..059c9f1 100644 --- a/src/app/api/documents/import-url/route.ts +++ b/src/app/api/documents/import-url/route.ts @@ -10,9 +10,6 @@ export async function POST(req: NextRequest) { // 1. Authenticate user request const ctxOrRes = await requireAuthContext(req); if (ctxOrRes instanceof Response) return ctxOrRes; - if (!ctxOrRes.userId) { - return NextResponse.json({ error: 'Unauthorized' }, { status: 401 }); - } // 2. Validate URL body parameter let body; diff --git a/src/components/documents/UploadMenuDialog.tsx b/src/components/documents/UploadMenuDialog.tsx index 47062b0..1768cc0 100644 --- a/src/components/documents/UploadMenuDialog.tsx +++ b/src/components/documents/UploadMenuDialog.tsx @@ -121,15 +121,13 @@ export function UploadMenuDialog({ const scrapeResult = await importUrl(cleanUrl); setImportStep('converting'); - // Simulated processing time for micro-interaction polish - await new Promise((resolve) => setTimeout(resolve, 800)); - setImportStep('uploading'); - + // Create virtual file from markdown content const displayTitle = webTitle.trim() || scrapeResult.title || 'Imported Web Page'; const safeTitle = displayTitle - .replace(/[/\\?%*:|"<>\s]/g, '_') + .replace(/[/\\?%*:|"<>\s]+/g, '_') // collapse runs of disallowed chars + .replace(/^_+|_+$/g, '') // trim leading/trailing underscores .substring(0, 80); const filename = `${safeTitle}.md`; const file = new File([scrapeResult.content], filename, { diff --git a/src/lib/server/documents/web-loader.ts b/src/lib/server/documents/web-loader.ts index e247c25..c031aef 100644 --- a/src/lib/server/documents/web-loader.ts +++ b/src/lib/server/documents/web-loader.ts @@ -1,4 +1,6 @@ import dns from 'dns'; +import http from 'http'; +import https from 'https'; import { promisify } from 'util'; import { parseHTML } from 'linkedom'; import { Readability } from '@mozilla/readability'; @@ -6,46 +8,80 @@ import TurndownService from 'turndown'; const dnsLookup = promisify(dns.lookup); +const MAX_BYTES = 2 * 1024 * 1024; +const TIMEOUT_MS = 6000; +const MAX_REDIRECTS = 5; +const USER_AGENT = + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; + /** - * Checks if an IP address is a private, loopback, or link-local address (SSRF mitigation). + * Checks if an IPv4 address falls in a private, loopback, link-local, multicast, + * or broadcast range. Anything that is not a clean dotted-quad is treated as + * unsafe (fail closed). */ -function isPrivateIp(ip: string): boolean { - // IPv4 checks - if (ip === '127.0.0.1' || ip === '0.0.0.0') return true; - - const ipv4Match = ip.match(/^(\d+)\.(\d+)\.(\d+)\.(\d+)$/); - if (ipv4Match) { - const [, o1, o2] = ipv4Match.map(Number); - if (o1 === 10) return true; // 10.0.0.0/8 - if (o1 === 172 && o2 >= 16 && o2 <= 31) return true; // 172.16.0.0/12 - if (o1 === 192 && o2 === 168) return true; // 192.168.0.0/16 - if (o1 === 169 && o2 === 254) return true; // 169.254.0.0/16 - } - - // IPv6 checks - const ipLower = ip.toLowerCase(); - if ( - ipLower === '::1' || - ipLower === '::' || - ipLower === '0:0:0:0:0:0:0:1' - ) { - return true; - } - if ( - ipLower.startsWith('fe80:') || - ipLower.startsWith('fc00:') || - ipLower.startsWith('fd00:') - ) { - return true; - } - +function isPrivateIpv4(ip: string): boolean { + const m = ip.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/); + if (!m) return true; + const octets = m.slice(1).map(Number); + if (octets.some((o) => o > 255)) return true; // malformed → unsafe + const [o1, o2] = octets; + if (o1 === 0) return true; // 0.0.0.0/8 ("this host") + if (o1 === 10) return true; // 10.0.0.0/8 + if (o1 === 127) return true; // 127.0.0.0/8 loopback + if (o1 === 169 && o2 === 254) return true; // 169.254.0.0/16 link-local + if (o1 === 172 && o2 >= 16 && o2 <= 31) return true; // 172.16.0.0/12 + if (o1 === 192 && o2 === 168) return true; // 192.168.0.0/16 + if (o1 >= 224 && o1 <= 239) return true; // 224.0.0.0/4 multicast + if (ip === '255.255.255.255') return true; // limited broadcast return false; } /** - * Resolves the URL domain and validates that it does not point to a local/private network. + * Checks if an IP address (v4 or v6) is a private, loopback, link-local, + * multicast, or otherwise non-routable address (SSRF mitigation). */ -async function validateUrlForSsrf(urlString: string): Promise { +function isPrivateIp(ip: string): boolean { + let addr = ip.trim().toLowerCase(); + // Strip IPv6 brackets and zone id (e.g. [fe80::1%eth0]). + if (addr.startsWith('[') && addr.endsWith(']')) addr = addr.slice(1, -1); + const zoneIdx = addr.indexOf('%'); + if (zoneIdx !== -1) addr = addr.slice(0, zoneIdx); + + // Plain IPv4. + if (!addr.includes(':')) return isPrivateIpv4(addr); + + // IPv6 from here. + if (addr === '::1' || addr === '::') return true; // loopback / unspecified + + // IPv4-mapped (::ffff:a.b.c.d) / -compatible (::a.b.c.d) IPv6: evaluate the + // embedded IPv4 against the v4 ranges. Any address in the mapped range we + // cannot cleanly read as a dotted-quad (e.g. the hex form ::ffff:7f00:1) is + // blocked outright — fail closed. (cf. CVE-2026-47684) + if (addr.startsWith('::ffff:') || /^::\d+\.\d+\.\d+\.\d+$/.test(addr)) { + const m = addr.match(/^::(?:ffff:)?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/); + return m ? isPrivateIpv4(m[1]) : true; + } + + if (/^fe[89ab]/.test(addr)) return true; // fe80::/10 link-local + if (/^fec/.test(addr)) return true; // fec0::/10 (deprecated site-local) + if (/^f[cd]/.test(addr)) return true; // fc00::/7 unique-local + if (/^ff/.test(addr)) return true; // ff00::/8 multicast + return false; +} + +interface ValidatedTarget { + url: URL; + /** The DNS-resolved IP address the connection is pinned to. */ + address: string; +} + +/** + * Resolves the URL domain and validates that it does not point to a local/private + * network. Returns the parsed URL together with the resolved IP address so the + * caller can pin the connection to it, closing the DNS-rebinding window between + * validation and connect. + */ +async function validateUrlForSsrf(urlString: string): Promise { let url: URL; try { url = new URL(urlString); @@ -63,100 +99,165 @@ async function validateUrlForSsrf(urlString: string): Promise { throw new Error('Access to localhost is forbidden'); } + let results: { address: string; family: number }[]; try { - const lookupResult = await dnsLookup(hostname); - if (isPrivateIp(lookupResult.address)) { - throw new Error(`Access to private network address is forbidden`); - } - } catch (err) { - if (err instanceof Error && err.message.includes('forbidden')) { - throw err; - } - // If lookup fails entirely, let the fetch attempt fail or resolve naturally + results = await dnsLookup(hostname, { all: true }); + } catch { + // Fail closed: do not silently continue and let a later re-resolution decide. + throw new Error('Could not resolve host address'); } - return urlString; + if (results.length === 0) { + throw new Error('Could not resolve host address'); + } + + // Validate every resolved address; reject if any maps to a private network. + for (const r of results) { + if (isPrivateIp(r.address)) { + throw new Error('Access to private network address is forbidden'); + } + } + + return { url, address: results[0].address }; } /** - * Fetches a URL with a size limit (default 2MB) and connection timeout (default 6s). + * Performs a single GET request pinned to the already-validated IP address. + * Resolves with either the response body or a redirect location to follow. + */ +function requestPinned( + target: ValidatedTarget, + maxBytes: number, + timeoutMs: number +): Promise<{ body?: string; redirectLocation?: string }> { + const { url, address } = target; + const isHttps = url.protocol === 'https:'; + const lib = isHttps ? https : http; + const port = url.port ? Number(url.port) : isHttps ? 443 : 80; + + const options: https.RequestOptions = { + host: address, // connect to the validated IP, not a freshly-resolved one + port, + path: `${url.pathname}${url.search}`, + method: 'GET', + headers: { + // Preserve virtual hosting (host:port) so the origin serves the right vhost. + Host: url.host, + 'User-Agent': USER_AGENT, + Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + }, + timeout: timeoutMs, + }; + if (isHttps) { + // Validate the TLS certificate against the original hostname, not the IP. + options.servername = url.hostname; + } + + return new Promise((resolve, reject) => { + let settled = false; + const ok = (v: { body?: string; redirectLocation?: string }) => { + if (settled) return; + settled = true; + clearTimeout(hardTimer); + resolve(v); + }; + const fail = (e: Error) => { + if (settled) return; + settled = true; + clearTimeout(hardTimer); + reject(e); + }; + + const req = lib.request(options, (res) => { + const status = res.statusCode ?? 0; + + if (status >= 300 && status < 400 && res.headers.location) { + res.resume(); // drain so the socket can be reused/closed + ok({ redirectLocation: res.headers.location }); + return; + } + + if (status < 200 || status >= 300) { + res.resume(); + fail(new Error(`Failed to fetch webpage (status ${status})`)); + return; + } + + const contentType = (res.headers['content-type'] || '').toString(); + const isHtml = + contentType.includes('text/html') || + contentType.includes('application/xhtml+xml'); + const isText = contentType.includes('text/plain'); + if (!isHtml && !isText) { + res.resume(); + fail( + new Error('Unsupported content type. Only web pages or text files are supported.') + ); + return; + } + + const contentLength = res.headers['content-length']; + if (contentLength && parseInt(contentLength.toString(), 10) > maxBytes) { + res.resume(); + fail(new Error('Web page size exceeds the maximum allowed limit of 2MB.')); + return; + } + + // Stream and enforce the size limit dynamically (header may be absent/spoofed). + let received = 0; + const chunks: Buffer[] = []; + res.on('data', (chunk: Buffer) => { + received += chunk.length; + if (received > maxBytes) { + req.destroy(); + fail(new Error('Web page size exceeds the maximum allowed limit of 2MB.')); + return; + } + chunks.push(chunk); + }); + res.on('end', () => ok({ body: Buffer.concat(chunks).toString('utf-8') })); + res.on('error', fail); + }); + + // Hard deadline covering the whole request (connect + transfer), in addition + // to the socket idle timeout, to bound slow-read/slowloris responses. + const hardTimer = setTimeout(() => { + req.destroy(new Error('Request timed out')); + }, timeoutMs); + + req.on('timeout', () => { + req.destroy(new Error('Request timed out')); + }); + req.on('error', fail); + req.end(); + }); +} + +/** + * Validates and fetches a URL with a size limit (default 2MB) and connection + * timeout (default 6s). Each redirect hop is independently re-validated and + * pinned to its resolved IP to prevent SSRF via redirects or DNS rebinding. */ async function fetchWithLimit( urlStr: string, - maxBytes: number = 2 * 1024 * 1024, - timeoutMs: number = 6000 + maxBytes: number = MAX_BYTES, + timeoutMs: number = TIMEOUT_MS ): Promise { - const controller = new AbortController(); - const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + let currentUrl = urlStr; - try { - const res = await fetch(urlStr, { - signal: controller.signal, - headers: { - 'User-Agent': - 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - }, - }); + for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) { + const target = await validateUrlForSsrf(currentUrl); + const result = await requestPinned(target, maxBytes, timeoutMs); - if (!res.ok) { - throw new Error(`Failed to fetch webpage (status ${res.status})`); + if (result.redirectLocation) { + currentUrl = new URL(result.redirectLocation, target.url).href; + continue; } - const contentType = res.headers.get('content-type') || ''; - const isHtml = - contentType.includes('text/html') || - contentType.includes('application/xhtml+xml'); - const isText = contentType.includes('text/plain'); - - if (!isHtml && !isText) { - throw new Error('Unsupported content type. Only web pages or text files are supported.'); - } - - const contentLength = res.headers.get('content-length'); - if (contentLength && parseInt(contentLength, 10) > maxBytes) { - throw new Error('Web page size exceeds the maximum allowed limit of 2MB.'); - } - - // Read response body chunks to verify size limit dynamically in case Content-Length header is missing/spoofed - const reader = res.body?.getReader(); - if (!reader) { - // Fallback for environments lacking full stream reader support - const text = await res.text(); - if (Buffer.byteLength(text) > maxBytes) { - throw new Error('Web page size exceeds the maximum allowed limit of 2MB.'); - } - return text; - } - - let bytesReceived = 0; - const chunks: Uint8Array[] = []; - - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - if (value) { - bytesReceived += value.length; - if (bytesReceived > maxBytes) { - controller.abort(); - throw new Error('Web page size exceeds the maximum allowed limit of 2MB.'); - } - chunks.push(value); - } - } - - const merged = new Uint8Array(bytesReceived); - let offset = 0; - for (const chunk of chunks) { - merged.set(chunk, offset); - offset += chunk.length; - } - - return new TextDecoder().decode(merged); - } finally { - clearTimeout(timeoutId); + return result.body ?? ''; } + + throw new Error('Too many redirects while fetching the webpage.'); } /** @@ -165,10 +266,8 @@ async function fetchWithLimit( export async function fetchAndParseUrl( urlStr: string ): Promise<{ title: string; content: string }> { - // 1. SSRF prevention check - await validateUrlForSsrf(urlStr); - - // 2. Fetch HTML content with limit + // 1. Fetch HTML content with limit (SSRF validation + IP pinning happens + // inside fetchWithLimit for the initial URL and every redirect hop). const html = await fetchWithLimit(urlStr); // 3. Parse HTML string to a virtual DOM document