danvics/openreader
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

refactor(documents): replace custom SSRF checks with AntiSSRF library

Browse source 
Switch SSRF protection in web-loader to use @microsoft/antissrf, removing
custom IP range logic. This centralizes and future-proofs SSRF mitigation by
leveraging maintained policy sets and DNS-aware enforcement. Error handling in
the import-url API route is updated to recognize new policy error messages.
Dependency added to package.json.
This commit is contained in:
Richard R 2026-06-10 13:29:18 -06:00
parent 11b2f18691
commit 08b8b06f27
4 changed files with 56 additions and 132 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -35,6 +35,7 @@
"@aws-sdk/s3-request-presigner": "^3.1061.0",
"@headlessui/react": "^2.2.10",
"@huggingface/tokenizers": "^0.1.3",
"@microsoft/antissrf": "^1.0.0",
"@mozilla/readability": "^0.6.0",
"@napi-rs/canvas": "^0.1.100",
"@tanstack/react-query": "^5.101.0",

8
pnpm-lock.yaml
View file

@ -23,6 +23,9 @@ importers:
'@huggingface/tokenizers':
specifier: ^0.1.3
version: 0.1.3
'@microsoft/antissrf':
specifier: ^1.0.0
version: 1.0.0
'@mozilla/readability':
specifier: ^0.6.0
version: 0.6.0
@ -1255,6 +1258,9 @@ packages:
'@jridgewell/trace-mapping@0.3.31':
resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==}
'@microsoft/antissrf@1.0.0':
resolution: {integrity: sha512-AmO1ykSha52Ef/7UXvHgu8ElsGdgcLkF5nTl7YZGyR1AkbmlQYtiVeWp+CsjkFaJzKAH5ofXLZveMmOHwX/7Jw==}
'@mixmark-io/domino@2.2.0':
resolution: {integrity: sha512-Y28PR25bHXUg88kCV7nivXrP2Nj2RueZ3/l/jdx6J9f8J4nsEGcgX0Qe6lt7Pa+J79+kPiJU3LguR6O/6zrLOw==}
@ -5756,6 +5762,8 @@ snapshots:
'@jridgewell/resolve-uri': 3.1.2
'@jridgewell/sourcemap-codec': 1.5.5
'@microsoft/antissrf@1.0.0': {}
'@mixmark-io/domino@2.2.0': {}
'@mozilla/readability@0.6.0': {}

1
src/app/api/documents/import-url/route.ts
View file

@ -43,6 +43,7 @@ export async function POST(req: NextRequest) {
// Check if it's a validation error or network error to set suitable status codes
const isValidationError =
errorMessage.includes('forbidden') ||
errorMessage.includes('disallowed by policy') ||
errorMessage.includes('Invalid URL') ||
errorMessage.includes('Only HTTP') ||
errorMessage.includes('exceeds the maximum') ||

178
src/lib/server/documents/web-loader.ts
View file

@ -1,12 +1,9 @@
import dns from 'dns';
import http from 'http';
import https from 'https';
import { promisify } from 'util';
import { parseHTML } from 'linkedom';
import { Readability } from '@mozilla/readability';
import TurndownService from 'turndown';
const dnsLookup = promisify(dns.lookup);
import { AntiSSRFPolicy, PolicyConfigOptions } from '@microsoft/antissrf';
const MAX_BYTES = 2 * 1024 * 1024;
const TIMEOUT_MS = 6000;
@ -15,145 +12,51 @@ const USER_AGENT =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
/**
* Checks if an IPv4 address falls in a private, loopback, link-local, multicast,
* or broadcast range. Anything that is not a clean dotted-quad is treated as
* unsafe (fail closed).
* SSRF protection is delegated to Microsoft's AntiSSRF library. The policy's
* agents inject a connection-time DNS lookup that resolves the host and rejects
* any request whose resolved address falls in an internal/special-purpose range
* (loopback, private, link-local, CGNAT, cloud metadata, multicast, …). Because
* the check happens on the address the socket actually connects to, it is
* inherently safe against DNS rebinding, and it re-runs on every redirect hop
* (each hop is a fresh request through the agent).
*
* `ExternalOnlyLatest` blocks the `recommendedLatest` range set and stays
* up to date without code changes. We allow plain HTTP because article URLs are
* not always HTTPS; remove `allowPlainTextHttp` to require HTTPS.
*/
function isPrivateIpv4(ip: string): boolean {
const m = ip.match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})$/);
if (!m) return true;
const octets = m.slice(1).map(Number);
if (octets.some((o) => o > 255)) return true; // malformed → unsafe
const [o1, o2] = octets;
if (o1 === 0) return true; // 0.0.0.0/8 ("this host")
if (o1 === 10) return true; // 10.0.0.0/8
if (o1 === 127) return true; // 127.0.0.0/8 loopback
if (o1 === 100 && o2 >= 64 && o2 <= 127) return true; // 100.64.0.0/10 CGNAT (cloud-internal; Alibaba metadata 100.100.100.200)
if (o1 === 169 && o2 === 254) return true; // 169.254.0.0/16 link-local
if (o1 === 172 && o2 >= 16 && o2 <= 31) return true; // 172.16.0.0/12
if (o1 === 192 && o2 === 168) return true; // 192.168.0.0/16
if (o1 === 198 && (o2 === 18 || o2 === 19)) return true; // 198.18.0.0/15 benchmarking
if (o1 >= 224 && o1 <= 239) return true; // 224.0.0.0/4 multicast
if (ip === '255.255.255.255') return true; // limited broadcast
return false;
}
const ssrfPolicy = new AntiSSRFPolicy(PolicyConfigOptions.ExternalOnlyLatest);
ssrfPolicy.allowPlainTextHttp = true;
const httpAgent = ssrfPolicy.getHttpAgent();
const httpsAgent = ssrfPolicy.getHttpsAgent();
/**
* Checks if an IP address (v4 or v6) is a private, loopback, link-local,
* multicast, or otherwise non-routable address (SSRF mitigation).
* Performs a single GET request through the AntiSSRF agent. Resolves with either
* the response body or a redirect location to follow. Enforces a size limit and
* a hard timeout (resource limits, independent of the SSRF policy).
*/
function isPrivateIp(ip: string): boolean {
let addr = ip.trim().toLowerCase();
// Strip IPv6 brackets and zone id (e.g. [fe80::1%eth0]).
if (addr.startsWith('[') && addr.endsWith(']')) addr = addr.slice(1, -1);
const zoneIdx = addr.indexOf('%');
if (zoneIdx !== -1) addr = addr.slice(0, zoneIdx);
// Plain IPv4.
if (!addr.includes(':')) return isPrivateIpv4(addr);
// IPv6 from here.
if (addr === '::1' || addr === '::') return true; // loopback / unspecified
// IPv4-mapped (::ffff:a.b.c.d) / -compatible (::a.b.c.d) IPv6: evaluate the
// embedded IPv4 against the v4 ranges. Any address in the mapped range we
// cannot cleanly read as a dotted-quad (e.g. the hex form ::ffff:7f00:1) is
// blocked outright — fail closed. (cf. CVE-2026-47684)
if (addr.startsWith('::ffff:') || /^::\d+\.\d+\.\d+\.\d+$/.test(addr)) {
const m = addr.match(/^::(?:ffff:)?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/);
return m ? isPrivateIpv4(m[1]) : true;
}
if (/^fe[89ab]/.test(addr)) return true; // fe80::/10 link-local
if (/^fe[c-f]/.test(addr)) return true; // fec0::/10 (deprecated site-local)
if (/^f[cd]/.test(addr)) return true; // fc00::/7 unique-local
if (/^ff/.test(addr)) return true; // ff00::/8 multicast
return false;
}
interface ValidatedTarget {
url: URL;
/** The DNS-resolved IP address the connection is pinned to. */
address: string;
}
/**
* Resolves the URL domain and validates that it does not point to a local/private
* network. Returns the parsed URL together with the resolved IP address so the
* caller can pin the connection to it, closing the DNS-rebinding window between
* validation and connect.
*/
async function validateUrlForSsrf(urlString: string): Promise<ValidatedTarget> {
let url: URL;
try {
url = new URL(urlString);
} catch {
throw new Error('Invalid URL format');
}
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
throw new Error('Only HTTP and HTTPS protocols are supported');
}
const hostname = url.hostname.toLowerCase();
if (hostname === 'localhost') {
throw new Error('Access to localhost is forbidden');
}
let results: { address: string; family: number }[];
try {
results = await dnsLookup(hostname, { all: true });
} catch {
// Fail closed: do not silently continue and let a later re-resolution decide.
throw new Error('Could not resolve host address');
}
if (results.length === 0) {
throw new Error('Could not resolve host address');
}
// Validate every resolved address; reject if any maps to a private network.
for (const r of results) {
if (isPrivateIp(r.address)) {
throw new Error('Access to private network address is forbidden');
}
}
return { url, address: results[0].address };
}
/**
* Performs a single GET request pinned to the already-validated IP address.
* Resolves with either the response body or a redirect location to follow.
*/
function requestPinned(
target: ValidatedTarget,
function requestOnce(
url: URL,
maxBytes: number,
timeoutMs: number
): Promise<{ body?: string; redirectLocation?: string }> {
const { url, address } = target;
const isHttps = url.protocol === 'https:';
const lib = isHttps ? https : http;
const agent = isHttps ? httpsAgent : httpAgent;
const port = url.port ? Number(url.port) : isHttps ? 443 : 80;
const options: https.RequestOptions = {
host: address, // connect to the validated IP, not a freshly-resolved one
protocol: url.protocol,
hostname: url.hostname, // connect by hostname so the agent's safe lookup runs
port,
path: `${url.pathname}${url.search}`,
method: 'GET',
agent, // AntiSSRF agent enforces the SSRF policy at connection time
headers: {
// Preserve virtual hosting (host:port) so the origin serves the right vhost.
Host: url.host,
'User-Agent': USER_AGENT,
Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
},
timeout: timeoutMs,
};
if (isHttps) {
// Validate the TLS certificate against the original hostname, not the IP.
options.servername = url.hostname;
}
return new Promise((resolve, reject) => {
let settled = false;
@ -237,8 +140,9 @@ function requestPinned(
/**
* Validates and fetches a URL with a size limit (default 2MB) and connection
* timeout (default 6s). Each redirect hop is independently re-validated and
* pinned to its resolved IP to prevent SSRF via redirects or DNS rebinding.
* timeout (default 6s). Redirects are followed manually so each hop is re-checked
* by the AntiSSRF agent and counted against the redirect cap. Returns the body
* and the final (post-redirect) URL.
*/
async function fetchWithLimit(
urlStr: string,
@ -248,11 +152,21 @@ async function fetchWithLimit(
let currentUrl = urlStr;
for (let redirects = 0; redirects <= MAX_REDIRECTS; redirects++) {
const target = await validateUrlForSsrf(currentUrl);
const result = await requestPinned(target, maxBytes, timeoutMs);
let url: URL;
try {
url = new URL(currentUrl);
} catch {
throw new Error('Invalid URL format');
}
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
throw new Error('Only HTTP and HTTPS protocols are supported');
}
const result = await requestOnce(url, maxBytes, timeoutMs);
if (result.redirectLocation) {
currentUrl = new URL(result.redirectLocation, target.url).href;
currentUrl = new URL(result.redirectLocation, url).href;
continue;
}
@ -268,11 +182,11 @@ async function fetchWithLimit(
export async function fetchAndParseUrl(
urlStr: string
): Promise<{ title: string; content: string }> {
// 1. Fetch HTML content with limit (SSRF validation + IP pinning happens
// inside fetchWithLimit for the initial URL and every redirect hop).
// 1. Fetch HTML content with limit. The AntiSSRF agent validates the
// destination (and every redirect hop) at connection time.
const { html, finalUrl } = await fetchWithLimit(urlStr);
// 3. Parse HTML string to a virtual DOM document
// 2. Parse HTML string to a virtual DOM document
const { document } = parseHTML(html);
// Resolve relative URLs against the final (post-redirect) location so links
@ -301,7 +215,7 @@ export async function fetchAndParseUrl(
}
}
// 4. Run Readability to extract core article text
// 3. Run Readability to extract core article text
const reader = new Readability(document as unknown as Document);
const article = reader.parse();
@ -309,7 +223,7 @@ export async function fetchAndParseUrl(
throw new Error('Could not extract meaningful article content from this webpage.');
}
// 5. Convert clean HTML content to Markdown
// 4. Convert clean HTML content to Markdown
const turndownService = new TurndownService({
headingStyle: 'atx',
hr: '---',