refactor(documents): replace custom SSRF checks with AntiSSRF library

Switch SSRF protection in web-loader to use @microsoft/antissrf, removing
custom IP range logic. This centralizes and future-proofs SSRF mitigation by
leveraging maintained policy sets and DNS-aware enforcement. Error handling in
the import-url API route is updated to recognize new policy error messages.
Dependency added to package.json.

package.json

@@ -35,6 +35,7 @@
     "@aws-sdk/s3-request-presigner": "^3.1061.0",
     "@headlessui/react": "^2.2.10",
     "@huggingface/tokenizers": "^0.1.3",
+    "@microsoft/antissrf": "^1.0.0",
     "@mozilla/readability": "^0.6.0",
     "@napi-rs/canvas": "^0.1.100",
     "@tanstack/react-query": "^5.101.0",

pnpm-lock.yaml

@@ -23,6 +23,9 @@ importers:
       '@huggingface/tokenizers':
         specifier: ^0.1.3
         version: 0.1.3
+      '@microsoft/antissrf':
+        specifier: ^1.0.0
+        version: 1.0.0
       '@mozilla/readability':
         specifier: ^0.6.0
         version: 0.6.0
@@ -1255,6 +1258,9 @@ packages:
   '@jridgewell/trace-mapping@0.3.31':
     resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==}
 
+  '@microsoft/antissrf@1.0.0':
+    resolution: {integrity: sha512-AmO1ykSha52Ef/7UXvHgu8ElsGdgcLkF5nTl7YZGyR1AkbmlQYtiVeWp+CsjkFaJzKAH5ofXLZveMmOHwX/7Jw==}
+
   '@mixmark-io/domino@2.2.0':
     resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==}
 
@@ -5756,6 +5762,8 @@ snapshots:
       '@jridgewell/resolve-uri': 3.1.2
       '@jridgewell/sourcemap-codec': 1.5.5
 
+  '@microsoft/antissrf@1.0.0': {}
+
   '@mixmark-io/domino@2.2.0': {}
 
   '@mozilla/readability@0.6.0': {}

src/app/api/documents/import-url/route.ts

@@ -43,6 +43,7 @@ export async function POST(req: NextRequest) {
     // Check if it's a validation error or network error to set suitable status codes
     const isValidationError =
       errorMessage.includes('forbidden') ||
+      errorMessage.includes('disallowed by policy') ||
       errorMessage.includes('Invalid URL') ||
       errorMessage.includes('Only HTTP') ||
       errorMessage.includes('exceeds the maximum') ||

src/lib/server/documents/web-loader.ts

@@ -1,12 +1,9 @@
-import dns from 'dns';
 import http from 'http';
 import https from 'https';
-import { promisify } from 'util';
 import { parseHTML } from 'linkedom';
 import { Readability } from '@mozilla/readability';
 import TurndownService from 'turndown';
-
+import { AntiSSRFPolicy, PolicyConfigOptions } from '@microsoft/antissrf';
-const dnsLookup = promisify(dns.lookup);
 
 const MAX_BYTES = 2 * 1024 * 1024;
 const TIMEOUT_MS = 6000;
@@ -15,145 +12,51 @@ const USER_AGENT =
   'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
 
 /**
- * Checks if an IPv4 address falls in a private, loopback, link-local, multicast,
+ * SSRF protection is delegated to Microsoft's AntiSSRF library. The policy's
- * or broadcast range. Anything that is not a clean dotted-quad is treated as
+ * agents inject a connection-time DNS lookup that resolves the host and rejects
- * unsafe (fail closed).
+ * any request whose resolved address falls in an internal/special-purpose range
+ * (loopback, private, link-local, CGNAT, cloud metadata, multicast, …). Because
+ * the check happens on the address the socket actually connects to, it is
+ * inherently safe against DNS rebinding, and it re-runs on every redirect hop
+ * (each hop is a fresh request through the agent).
+ *
+ * `ExternalOnlyLatest` blocks the `recommendedLatest` range set and stays
+ * up to date without code changes. We allow plain HTTP because article URLs are
+ * not always HTTPS; remove `allowPlainTextHttp` to require HTTPS.
  */
-function isPrivateIpv4(ip: string): boolean {
+const ssrfPolicy = new AntiSSRFPolicy(PolicyConfigOptions.ExternalOnlyLatest);
-  const m = ip.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
+ssrfPolicy.allowPlainTextHttp = true;
-  if (!m) return true;
+const httpAgent = ssrfPolicy.getHttpAgent();
-  const octets = m.slice(1).map(Number);
+const httpsAgent = ssrfPolicy.getHttpsAgent();
-  if (octets.some((o) => o > 255)) return true; // malformed → unsafe
-  const [o1, o2] = octets;
-  if (o1 === 0) return true; // 0.0.0.0/8 ("this host")
-  if (o1 === 10) return true; // 10.0.0.0/8
-  if (o1 === 127) return true; // 127.0.0.0/8 loopback
-  if (o1 === 100 && o2 >= 64 && o2 <= 127) return true; // 100.64.0.0/10 CGNAT (cloud-internal; Alibaba metadata 100.100.100.200)
-  if (o1 === 169 && o2 === 254) return true; // 169.254.0.0/16 link-local
-  if (o1 === 172 && o2 >= 16 && o2 <= 31) return true; // 172.16.0.0/12
-  if (o1 === 192 && o2 === 168) return true; // 192.168.0.0/16
-  if (o1 === 198 && (o2 === 18 || o2 === 19)) return true; // 198.18.0.0/15 benchmarking
-  if (o1 >= 224 && o1 <= 239) return true; // 224.0.0.0/4 multicast
-  if (ip === '255.255.255.255') return true; // limited broadcast
-  return false;
-}
 
 /**
- * Checks if an IP address (v4 or v6) is a private, loopback, link-local,
+ * Performs a single GET request through the AntiSSRF agent. Resolves with either
- * multicast, or otherwise non-routable address (SSRF mitigation).
+ * the response body or a redirect location to follow. Enforces a size limit and
+ * a hard timeout (resource limits, independent of the SSRF policy).
  */
-function isPrivateIp(ip: string): boolean {
+function requestOnce(
-  let addr = ip.trim().toLowerCase();
+  url: URL,
-  // Strip IPv6 brackets and zone id (e.g. [fe80::1%eth0]).
-  if (addr.startsWith('[') && addr.endsWith(']')) addr = addr.slice(1, -1);
-  const zoneIdx = addr.indexOf('%');
-  if (zoneIdx !== -1) addr = addr.slice(0, zoneIdx);
-
-  // Plain IPv4.
-  if (!addr.includes(':')) return isPrivateIpv4(addr);
-
-  // IPv6 from here.
-  if (addr === '::1' || addr === '::') return true; // loopback / unspecified
-
-  // IPv4-mapped (::ffff:a.b.c.d) / -compatible (::a.b.c.d) IPv6: evaluate the
-  // embedded IPv4 against the v4 ranges. Any address in the mapped range we
-  // cannot cleanly read as a dotted-quad (e.g. the hex form ::ffff:7f00:1) is
-  // blocked outright — fail closed. (cf. CVE-2026-47684)
-  if (addr.startsWith('::ffff:') || /^::\d+\.\d+\.\d+\.\d+$/.test(addr)) {
-    const m = addr.match(/^::(?:ffff:)?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
-    return m ? isPrivateIpv4(m[1]) : true;
-  }
-
-  if (/^fe[89ab]/.test(addr)) return true; // fe80::/10 link-local
-  if (/^fe[c-f]/.test(addr)) return true; // fec0::/10 (deprecated site-local)
-  if (/^f[cd]/.test(addr)) return true; // fc00::/7 unique-local
-  if (/^ff/.test(addr)) return true; // ff00::/8 multicast
-  return false;
-}
-
-interface ValidatedTarget {
-  url: URL;
-  /** The DNS-resolved IP address the connection is pinned to. */
-  address: string;
-}
-
-/**
- * Resolves the URL domain and validates that it does not point to a local/private
- * network. Returns the parsed URL together with the resolved IP address so the
- * caller can pin the connection to it, closing the DNS-rebinding window between
- * validation and connect.
- */
-async function validateUrlForSsrf(urlString: string): Promise<ValidatedTarget> {
-  let url: URL;
-  try {
-    url = new URL(urlString);
-  } catch {
-    throw new Error('Invalid URL format');
-  }
-
-  if (url.protocol !== 'http:' && url.protocol !== 'https:') {
-    throw new Error('Only HTTP and HTTPS protocols are supported');
-  }
-
-  const hostname = url.hostname.toLowerCase();
-
-  if (hostname === 'localhost') {
-    throw new Error('Access to localhost is forbidden');
-  }
-
-  let results: { address: string; family: number }[];
-  try {
-    results = await dnsLookup(hostname, { all: true });
-  } catch {
-    // Fail closed: do not silently continue and let a later re-resolution decide.
-    throw new Error('Could not resolve host address');
-  }
-
-  if (results.length === 0) {
-    throw new Error('Could not resolve host address');
-  }
-
-  // Validate every resolved address; reject if any maps to a private network.
-  for (const r of results) {
-    if (isPrivateIp(r.address)) {
-      throw new Error('Access to private network address is forbidden');
-    }
-  }
-
-  return { url, address: results[0].address };
-}
-
-/**
- * Performs a single GET request pinned to the already-validated IP address.
- * Resolves with either the response body or a redirect location to follow.
- */
-function requestPinned(
-  target: ValidatedTarget,
   maxBytes: number,
   timeoutMs: number
 ): Promise<{ body?: string; redirectLocation?: string }> {
-  const { url, address } = target;
   const isHttps = url.protocol === 'https:';
   const lib = isHttps ? https : http;
+  const agent = isHttps ? httpsAgent : httpAgent;
   const port = url.port ? Number(url.port) : isHttps ? 443 : 80;
 
   const options: https.RequestOptions = {
-    host: address, // connect to the validated IP, not a freshly-resolved one
+    protocol: url.protocol,
+    hostname: url.hostname, // connect by hostname so the agent's safe lookup runs
     port,
     path: `${url.pathname}${url.search}`,
     method: 'GET',
+    agent, // AntiSSRF agent enforces the SSRF policy at connection time
     headers: {
-      // Preserve virtual hosting (host:port) so the origin serves the right vhost.
-      Host: url.host,
       'User-Agent': USER_AGENT,
       Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
     },
     timeout: timeoutMs,
   };
-  if (isHttps) {
-    // Validate the TLS certificate against the original hostname, not the IP.
-    options.servername = url.hostname;
-  }
 
   return new Promise((resolve, reject) => {
     let settled = false;
@@ -237,8 +140,9 @@ function requestPinned(
 
 /**
  * Validates and fetches a URL with a size limit (default 2MB) and connection
- * timeout (default 6s). Each redirect hop is independently re-validated and
+ * timeout (default 6s). Redirects are followed manually so each hop is re-checked
- * pinned to its resolved IP to prevent SSRF via redirects or DNS rebinding.
+ * by the AntiSSRF agent and counted against the redirect cap. Returns the body
+ * and the final (post-redirect) URL.
  */
 async function fetchWithLimit(
   urlStr: string,
@@ -248,11 +152,21 @@ async function fetchWithLimit(
   let currentUrl = urlStr;
 
   for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
-    const target = await validateUrlForSsrf(currentUrl);
+    let url: URL;
-    const result = await requestPinned(target, maxBytes, timeoutMs);
+    try {
+      url = new URL(currentUrl);
+    } catch {
+      throw new Error('Invalid URL format');
+    }
+
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+      throw new Error('Only HTTP and HTTPS protocols are supported');
+    }
+
+    const result = await requestOnce(url, maxBytes, timeoutMs);
 
     if (result.redirectLocation) {
-      currentUrl = new URL(result.redirectLocation, target.url).href;
+      currentUrl = new URL(result.redirectLocation, url).href;
       continue;
     }
 
@@ -268,11 +182,11 @@ async function fetchWithLimit(
 export async function fetchAndParseUrl(
   urlStr: string
 ): Promise<{ title: string; content: string }> {
-  // 1. Fetch HTML content with limit (SSRF validation + IP pinning happens
+  // 1. Fetch HTML content with limit. The AntiSSRF agent validates the
-  //    inside fetchWithLimit for the initial URL and every redirect hop).
+  //    destination (and every redirect hop) at connection time.
   const { html, finalUrl } = await fetchWithLimit(urlStr);
 
-  // 3. Parse HTML string to a virtual DOM document
+  // 2. Parse HTML string to a virtual DOM document
   const { document } = parseHTML(html);
 
   // Resolve relative URLs against the final (post-redirect) location so links
@@ -301,7 +215,7 @@ export async function fetchAndParseUrl(
     }
   }
 
-  // 4. Run Readability to extract core article text
+  // 3. Run Readability to extract core article text
   const reader = new Readability(document as unknown as Document);
   const article = reader.parse();
 
@@ -309,7 +223,7 @@ export async function fetchAndParseUrl(
     throw new Error('Could not extract meaningful article content from this webpage.');
   }
 
-  // 5. Convert clean HTML content to Markdown
+  // 4. Convert clean HTML content to Markdown
   const turndownService = new TurndownService({
     headingStyle: 'atx',
     hr: '---',