diff --git a/package.json b/package.json index 5c1e710..a8a7950 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "@aws-sdk/s3-request-presigner": "^3.1061.0", "@headlessui/react": "^2.2.10", "@huggingface/tokenizers": "^0.1.3", + "@microsoft/antissrf": "^1.0.0", "@mozilla/readability": "^0.6.0", "@napi-rs/canvas": "^0.1.100", "@tanstack/react-query": "^5.101.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 67d9390..1749ccb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -23,6 +23,9 @@ importers: '@huggingface/tokenizers': specifier: ^0.1.3 version: 0.1.3 + '@microsoft/antissrf': + specifier: ^1.0.0 + version: 1.0.0 '@mozilla/readability': specifier: ^0.6.0 version: 0.6.0 @@ -1255,6 +1258,9 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@microsoft/antissrf@1.0.0': + resolution: {integrity: sha512-AmO1ykSha52Ef/7UXvHgu8ElsGdgcLkF5nTl7YZGyR1AkbmlQYtiVeWp+CsjkFaJzKAH5ofXLZveMmOHwX/7Jw==} + '@mixmark-io/domino@2.2.0': resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==} @@ -5756,6 +5762,8 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@microsoft/antissrf@1.0.0': {} + '@mixmark-io/domino@2.2.0': {} '@mozilla/readability@0.6.0': {} diff --git a/src/app/api/documents/import-url/route.ts b/src/app/api/documents/import-url/route.ts index 059c9f1..674b93a 100644 --- a/src/app/api/documents/import-url/route.ts +++ b/src/app/api/documents/import-url/route.ts @@ -43,6 +43,7 @@ export async function POST(req: NextRequest) { // Check if it's a validation error or network error to set suitable status codes const isValidationError = errorMessage.includes('forbidden') || + errorMessage.includes('disallowed by policy') || errorMessage.includes('Invalid URL') || errorMessage.includes('Only HTTP') || errorMessage.includes('exceeds the maximum') || diff --git a/src/lib/server/documents/web-loader.ts b/src/lib/server/documents/web-loader.ts index b4482ee..cb626ef 100644 --- a/src/lib/server/documents/web-loader.ts +++ b/src/lib/server/documents/web-loader.ts @@ -1,12 +1,9 @@ -import dns from 'dns'; import http from 'http'; import https from 'https'; -import { promisify } from 'util'; import { parseHTML } from 'linkedom'; import { Readability } from '@mozilla/readability'; import TurndownService from 'turndown'; - -const dnsLookup = promisify(dns.lookup); +import { AntiSSRFPolicy, PolicyConfigOptions } from '@microsoft/antissrf'; const MAX_BYTES = 2 * 1024 * 1024; const TIMEOUT_MS = 6000; @@ -15,145 +12,51 @@ const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; /** - * Checks if an IPv4 address falls in a private, loopback, link-local, multicast, - * or broadcast range. Anything that is not a clean dotted-quad is treated as - * unsafe (fail closed). + * SSRF protection is delegated to Microsoft's AntiSSRF library. The policy's + * agents inject a connection-time DNS lookup that resolves the host and rejects + * any request whose resolved address falls in an internal/special-purpose range + * (loopback, private, link-local, CGNAT, cloud metadata, multicast, …). Because + * the check happens on the address the socket actually connects to, it is + * inherently safe against DNS rebinding, and it re-runs on every redirect hop + * (each hop is a fresh request through the agent). + * + * `ExternalOnlyLatest` blocks the `recommendedLatest` range set and stays + * up to date without code changes. We allow plain HTTP because article URLs are + * not always HTTPS; remove `allowPlainTextHttp` to require HTTPS. */ -function isPrivateIpv4(ip: string): boolean { - const m = ip.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/); - if (!m) return true; - const octets = m.slice(1).map(Number); - if (octets.some((o) => o > 255)) return true; // malformed → unsafe - const [o1, o2] = octets; - if (o1 === 0) return true; // 0.0.0.0/8 ("this host") - if (o1 === 10) return true; // 10.0.0.0/8 - if (o1 === 127) return true; // 127.0.0.0/8 loopback - if (o1 === 100 && o2 >= 64 && o2 <= 127) return true; // 100.64.0.0/10 CGNAT (cloud-internal; Alibaba metadata 100.100.100.200) - if (o1 === 169 && o2 === 254) return true; // 169.254.0.0/16 link-local - if (o1 === 172 && o2 >= 16 && o2 <= 31) return true; // 172.16.0.0/12 - if (o1 === 192 && o2 === 168) return true; // 192.168.0.0/16 - if (o1 === 198 && (o2 === 18 || o2 === 19)) return true; // 198.18.0.0/15 benchmarking - if (o1 >= 224 && o1 <= 239) return true; // 224.0.0.0/4 multicast - if (ip === '255.255.255.255') return true; // limited broadcast - return false; -} +const ssrfPolicy = new AntiSSRFPolicy(PolicyConfigOptions.ExternalOnlyLatest); +ssrfPolicy.allowPlainTextHttp = true; +const httpAgent = ssrfPolicy.getHttpAgent(); +const httpsAgent = ssrfPolicy.getHttpsAgent(); /** - * Checks if an IP address (v4 or v6) is a private, loopback, link-local, - * multicast, or otherwise non-routable address (SSRF mitigation). + * Performs a single GET request through the AntiSSRF agent. Resolves with either + * the response body or a redirect location to follow. Enforces a size limit and + * a hard timeout (resource limits, independent of the SSRF policy). */ -function isPrivateIp(ip: string): boolean { - let addr = ip.trim().toLowerCase(); - // Strip IPv6 brackets and zone id (e.g. [fe80::1%eth0]). - if (addr.startsWith('[') && addr.endsWith(']')) addr = addr.slice(1, -1); - const zoneIdx = addr.indexOf('%'); - if (zoneIdx !== -1) addr = addr.slice(0, zoneIdx); - - // Plain IPv4. - if (!addr.includes(':')) return isPrivateIpv4(addr); - - // IPv6 from here. - if (addr === '::1' || addr === '::') return true; // loopback / unspecified - - // IPv4-mapped (::ffff:a.b.c.d) / -compatible (::a.b.c.d) IPv6: evaluate the - // embedded IPv4 against the v4 ranges. Any address in the mapped range we - // cannot cleanly read as a dotted-quad (e.g. the hex form ::ffff:7f00:1) is - // blocked outright — fail closed. (cf. CVE-2026-47684) - if (addr.startsWith('::ffff:') || /^::\d+\.\d+\.\d+\.\d+$/.test(addr)) { - const m = addr.match(/^::(?:ffff:)?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/); - return m ? isPrivateIpv4(m[1]) : true; - } - - if (/^fe[89ab]/.test(addr)) return true; // fe80::/10 link-local - if (/^fe[c-f]/.test(addr)) return true; // fec0::/10 (deprecated site-local) - if (/^f[cd]/.test(addr)) return true; // fc00::/7 unique-local - if (/^ff/.test(addr)) return true; // ff00::/8 multicast - return false; -} - -interface ValidatedTarget { - url: URL; - /** The DNS-resolved IP address the connection is pinned to. */ - address: string; -} - -/** - * Resolves the URL domain and validates that it does not point to a local/private - * network. Returns the parsed URL together with the resolved IP address so the - * caller can pin the connection to it, closing the DNS-rebinding window between - * validation and connect. - */ -async function validateUrlForSsrf(urlString: string): Promise { - let url: URL; - try { - url = new URL(urlString); - } catch { - throw new Error('Invalid URL format'); - } - - if (url.protocol !== 'http:' && url.protocol !== 'https:') { - throw new Error('Only HTTP and HTTPS protocols are supported'); - } - - const hostname = url.hostname.toLowerCase(); - - if (hostname === 'localhost') { - throw new Error('Access to localhost is forbidden'); - } - - let results: { address: string; family: number }[]; - try { - results = await dnsLookup(hostname, { all: true }); - } catch { - // Fail closed: do not silently continue and let a later re-resolution decide. - throw new Error('Could not resolve host address'); - } - - if (results.length === 0) { - throw new Error('Could not resolve host address'); - } - - // Validate every resolved address; reject if any maps to a private network. - for (const r of results) { - if (isPrivateIp(r.address)) { - throw new Error('Access to private network address is forbidden'); - } - } - - return { url, address: results[0].address }; -} - -/** - * Performs a single GET request pinned to the already-validated IP address. - * Resolves with either the response body or a redirect location to follow. - */ -function requestPinned( - target: ValidatedTarget, +function requestOnce( + url: URL, maxBytes: number, timeoutMs: number ): Promise<{ body?: string; redirectLocation?: string }> { - const { url, address } = target; const isHttps = url.protocol === 'https:'; const lib = isHttps ? https : http; + const agent = isHttps ? httpsAgent : httpAgent; const port = url.port ? Number(url.port) : isHttps ? 443 : 80; const options: https.RequestOptions = { - host: address, // connect to the validated IP, not a freshly-resolved one + protocol: url.protocol, + hostname: url.hostname, // connect by hostname so the agent's safe lookup runs port, path: `${url.pathname}${url.search}`, method: 'GET', + agent, // AntiSSRF agent enforces the SSRF policy at connection time headers: { - // Preserve virtual hosting (host:port) so the origin serves the right vhost. - Host: url.host, 'User-Agent': USER_AGENT, Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', }, timeout: timeoutMs, }; - if (isHttps) { - // Validate the TLS certificate against the original hostname, not the IP. - options.servername = url.hostname; - } return new Promise((resolve, reject) => { let settled = false; @@ -237,8 +140,9 @@ function requestPinned( /** * Validates and fetches a URL with a size limit (default 2MB) and connection - * timeout (default 6s). Each redirect hop is independently re-validated and - * pinned to its resolved IP to prevent SSRF via redirects or DNS rebinding. + * timeout (default 6s). Redirects are followed manually so each hop is re-checked + * by the AntiSSRF agent and counted against the redirect cap. Returns the body + * and the final (post-redirect) URL. */ async function fetchWithLimit( urlStr: string, @@ -248,11 +152,21 @@ async function fetchWithLimit( let currentUrl = urlStr; for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) { - const target = await validateUrlForSsrf(currentUrl); - const result = await requestPinned(target, maxBytes, timeoutMs); + let url: URL; + try { + url = new URL(currentUrl); + } catch { + throw new Error('Invalid URL format'); + } + + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + throw new Error('Only HTTP and HTTPS protocols are supported'); + } + + const result = await requestOnce(url, maxBytes, timeoutMs); if (result.redirectLocation) { - currentUrl = new URL(result.redirectLocation, target.url).href; + currentUrl = new URL(result.redirectLocation, url).href; continue; } @@ -268,11 +182,11 @@ async function fetchWithLimit( export async function fetchAndParseUrl( urlStr: string ): Promise<{ title: string; content: string }> { - // 1. Fetch HTML content with limit (SSRF validation + IP pinning happens - // inside fetchWithLimit for the initial URL and every redirect hop). + // 1. Fetch HTML content with limit. The AntiSSRF agent validates the + // destination (and every redirect hop) at connection time. const { html, finalUrl } = await fetchWithLimit(urlStr); - // 3. Parse HTML string to a virtual DOM document + // 2. Parse HTML string to a virtual DOM document const { document } = parseHTML(html); // Resolve relative URLs against the final (post-redirect) location so links @@ -301,7 +215,7 @@ export async function fetchAndParseUrl( } } - // 4. Run Readability to extract core article text + // 3. Run Readability to extract core article text const reader = new Readability(document as unknown as Document); const article = reader.parse(); @@ -309,7 +223,7 @@ export async function fetchAndParseUrl( throw new Error('Could not extract meaningful article content from this webpage.'); } - // 5. Convert clean HTML content to Markdown + // 4. Convert clean HTML content to Markdown const turndownService = new TurndownService({ headingStyle: 'atx', hr: '---',