import { Buffer } from "node:buffer"; import { isIP } from "node:net"; import { defaultTreeAdapter, parse, serialize } from "parse5"; export const REQUEST_TIMEOUT_MS = 15_000; export const MAX_RESPONSE_BYTES = 1024 * 1024; const MAX_REDIRECTS = 5; const ACCEPT_HEADER = "text/markdown, text/html;q=0.9, text/plain;q=0.8"; const SUPPORTED_CONTENT_TYPES = new Set(["text/markdown", "text/html", "text/plain"]); const OMITTED_HTML_TAGS = new Set(["script", "style", "template"]); const PAGE_CHROME_TAGS = new Set(["nav", "header", "footer"]); const PAGE_CHROME_ROLES = new Set(["navigation", "banner", "contentinfo"]); const CONTENT_TAGS = new Set(["main", "article", "section", "aside"]); export interface WebFetchResult { body?: string; truncated: boolean; finalUrl: string; } export function validateFetchUrl(raw: string): URL { let url: URL; try { url = new URL(raw.trim()); } catch { throw new Error("Invalid URL: expected an absolute http:// or https:// URL."); } if (url.protocol !== "http:" && url.protocol !== "https:") { throw new Error(`URL scheme "${url.protocol}" is not allowed; use http:// or https://.`); } if (url.username || url.password) { throw new Error("URLs containing credentials are not allowed."); } const hostname = url.hostname .toLowerCase() .replace(/^\[|\]$/g, "") .replace(/\.$/, ""); if (hostname === "localhost" || isIP(hostname) !== 0) { throw new Error("web_fetch only accepts hostnames, not IP addresses or localhost."); } return url; } function isSupportedContentType(contentType: string): boolean { const type = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? ""; return SUPPORTED_CONTENT_TYPES.has(type); } function extractHtmlBody(text: string): string { const html = parse(text) .childNodes.filter((node) => defaultTreeAdapter.isElementNode(node)) .find((node) => node.tagName === "html"); const body = html?.childNodes .filter((node) => defaultTreeAdapter.isElementNode(node)) .find((node) => node.tagName === "body"); if (!body) { return ""; } const pending = [{ node: body, inContent: false }]; for (let current = pending.pop(); current; current = pending.pop()) { const { node, inContent } = current; node.childNodes = node.childNodes.filter((child) => { if (defaultTreeAdapter.isCommentNode(child)) { return false; } if (!defaultTreeAdapter.isElementNode(child)) { return true; } const role = child.attrs .find((attr) => attr.name === "role") ?.value.trim() .toLowerCase() .split(/\s+/, 1)[0]; const content = inContent || CONTENT_TAGS.has(child.tagName) || role === "main" || role === "article"; if ( OMITTED_HTML_TAGS.has(child.tagName) || (!content && (PAGE_CHROME_TAGS.has(child.tagName) || PAGE_CHROME_ROLES.has(role ?? ""))) ) { return false; } pending.push({ node: child, inContent: content }); return true; }); } return serialize(body); } function decodeText(bytes: Buffer, contentType: string): string { const charset = contentType.match(/;\s*charset\s*=\s*(?:"([^"]+)"|'([^']+)'|([^;\s]+))/i); const label = charset?.[1] ?? charset?.[2] ?? charset?.[3] ?? "utf-8"; try { return new TextDecoder(label).decode(bytes); } catch { throw new Error("The response declared an unsupported text charset."); } } function requestError( cause: unknown, url: URL, timeoutMs: number, timeoutSignal: AbortSignal, signal?: AbortSignal, ): Error { if (timeoutSignal.aborted) { return new Error(`Request to ${url.href} timed out after ${timeoutMs}ms.`); } if (signal?.aborted) { return new Error(`Request to ${url.href} was cancelled.`); } const detail = cause instanceof Error && cause.cause instanceof Error ? cause.cause : cause; return new Error( `Failed to fetch ${url.href}: ${detail instanceof Error ? detail.message : String(detail)}`, ); } export async function fetchUrl( url: URL, { signal, timeoutMs = REQUEST_TIMEOUT_MS, maxBytes = MAX_RESPONSE_BYTES, fetchImpl = globalThis.fetch, }: { signal?: AbortSignal; timeoutMs?: number; maxBytes?: number; fetchImpl?: typeof fetch; } = {}, ): Promise { const timeoutSignal = AbortSignal.timeout(timeoutMs); const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal; let currentUrl = validateFetchUrl(url.href); let response: Response; let redirects = 0; while (true) { try { response = await fetchImpl(currentUrl, { method: "GET", headers: { Accept: ACCEPT_HEADER }, redirect: "manual", signal: requestSignal, }); } catch (error) { throw requestError(error, currentUrl, timeoutMs, timeoutSignal, signal); } const location = response.headers.get("location"); if (response.status < 300 || response.status >= 400 || !location) { break; } await response.body?.cancel().catch(() => undefined); if (redirects >= MAX_REDIRECTS) { throw new Error(`Too many redirects fetching ${currentUrl.href}.`); } let nextUrl: URL; try { nextUrl = new URL(location, currentUrl); } catch { throw new Error(`Invalid redirect from ${currentUrl.href}.`); } currentUrl = validateFetchUrl(nextUrl.href); redirects += 1; } if (response.status >= 400) { await response.body?.cancel().catch(() => undefined); throw new Error( `Request to ${currentUrl.href} failed with HTTP ${response.status}${response.statusText ? ` ${response.statusText}` : ""}.`, ); } if (!response.body) { return { truncated: false, finalUrl: currentUrl.href, }; } const contentType = response.headers.get("content-type") ?? ""; if (!isSupportedContentType(contentType)) { await response.body.cancel().catch(() => undefined); return { body: `[Body omitted: expected text/markdown, text/html, or text/plain, but response content type is ${contentType || "missing"}.]`, truncated: false, finalUrl: currentUrl.href, }; } const reader = response.body.getReader(); const chunks: Buffer[] = []; let bytesRead = 0; let truncated = false; try { while (true) { const { done, value } = await reader.read(); if (done) { break; } const chunk = Buffer.from(value); const remaining = maxBytes - bytesRead; if (chunk.length > remaining) { chunks.push(chunk.subarray(0, remaining)); bytesRead = maxBytes; truncated = true; await reader.cancel().catch(() => undefined); break; } chunks.push(chunk); bytesRead += chunk.length; } } catch (error) { throw requestError(error, currentUrl, timeoutMs, timeoutSignal, signal); } finally { reader.releaseLock(); } const text = decodeText(Buffer.concat(chunks, bytesRead), contentType); return { body: contentType.split(";", 1)[0]?.trim().toLowerCase() === "text/html" ? extractHtmlBody(text) : text, truncated, finalUrl: currentUrl.href, }; }