Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239import { Buffer } from "node:buffer";import { isIP } from "node:net";import { defaultTreeAdapter, parse, serialize } from "parse5";
export const REQUEST_TIMEOUT_MS = 15_000;export const MAX_RESPONSE_BYTES = 1024 * 1024;const MAX_REDIRECTS = 5;const ACCEPT_HEADER = "text/markdown, text/html;q=0.9, text/plain;q=0.8";const SUPPORTED_CONTENT_TYPES = new Set(["text/markdown", "text/html", "text/plain"]);const OMITTED_HTML_TAGS = new Set(["script", "style", "template"]);const PAGE_CHROME_TAGS = new Set(["nav", "header", "footer"]);const PAGE_CHROME_ROLES = new Set(["navigation", "banner", "contentinfo"]);const CONTENT_TAGS = new Set(["main", "article", "section", "aside"]);
export interface WebFetchResult { body?: string; truncated: boolean; finalUrl: string;}
export function validateFetchUrl(raw: string): URL { let url: URL; try { url = new URL(raw.trim()); } catch { throw new Error("Invalid URL: expected an absolute http:// or https:// URL."); }
if (url.protocol !== "http:" && url.protocol !== "https:") { throw new Error(`URL scheme "${url.protocol}" is not allowed; use http:// or https://.`); } if (url.username || url.password) { throw new Error("URLs containing credentials are not allowed."); }
const hostname = url.hostname .toLowerCase() .replace(/^\[|\]$/g, "") .replace(/\.$/, ""); if (hostname === "localhost" || isIP(hostname) !== 0) { throw new Error("web_fetch only accepts hostnames, not IP addresses or localhost."); }
return url;}
function isSupportedContentType(contentType: string): boolean { const type = contentType.split(";", 1)[0]?.trim().toLowerCase() ?? ""; return SUPPORTED_CONTENT_TYPES.has(type);}
function extractHtmlBody(text: string): string { const html = parse(text) .childNodes.filter((node) => defaultTreeAdapter.isElementNode(node)) .find((node) => node.tagName === "html"); const body = html?.childNodes .filter((node) => defaultTreeAdapter.isElementNode(node)) .find((node) => node.tagName === "body"); if (!body) { return ""; } const pending = [{ node: body, inContent: false }]; for (let current = pending.pop(); current; current = pending.pop()) { const { node, inContent } = current; node.childNodes = node.childNodes.filter((child) => { if (defaultTreeAdapter.isCommentNode(child)) { return false; } if (!defaultTreeAdapter.isElementNode(child)) { return true; } const role = child.attrs .find((attr) => attr.name === "role") ?.value.trim() .toLowerCase() .split(/\s+/, 1)[0]; const content = inContent || CONTENT_TAGS.has(child.tagName) || role === "main" || role === "article"; if ( OMITTED_HTML_TAGS.has(child.tagName) || (!content && (PAGE_CHROME_TAGS.has(child.tagName) || PAGE_CHROME_ROLES.has(role ?? ""))) ) { return false; } pending.push({ node: child, inContent: content }); return true; }); } return serialize(body);}
function decodeText(bytes: Buffer, contentType: string): string { const charset = contentType.match(/;\s*charset\s*=\s*(?:"([^"]+)"|'([^']+)'|([^;\s]+))/i); const label = charset?.[1] ?? charset?.[2] ?? charset?.[3] ?? "utf-8"; try { return new TextDecoder(label).decode(bytes); } catch { throw new Error("The response declared an unsupported text charset."); }}
function requestError( cause: unknown, url: URL, timeoutMs: number, timeoutSignal: AbortSignal, signal?: AbortSignal,): Error { if (timeoutSignal.aborted) { return new Error(`Request to ${url.href} timed out after ${timeoutMs}ms.`); } if (signal?.aborted) { return new Error(`Request to ${url.href} was cancelled.`); } const detail = cause instanceof Error && cause.cause instanceof Error ? cause.cause : cause; return new Error( `Failed to fetch ${url.href}: ${detail instanceof Error ? detail.message : String(detail)}`, );}
export async function fetchUrl( url: URL, { signal, timeoutMs = REQUEST_TIMEOUT_MS, maxBytes = MAX_RESPONSE_BYTES, fetchImpl = globalThis.fetch, }: { signal?: AbortSignal; timeoutMs?: number; maxBytes?: number; fetchImpl?: typeof fetch; } = {},): Promise<WebFetchResult> { const timeoutSignal = AbortSignal.timeout(timeoutMs); const requestSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal; let currentUrl = validateFetchUrl(url.href); let response: Response; let redirects = 0;
while (true) { try { response = await fetchImpl(currentUrl, { method: "GET", headers: { Accept: ACCEPT_HEADER }, redirect: "manual", signal: requestSignal, }); } catch (error) { throw requestError(error, currentUrl, timeoutMs, timeoutSignal, signal); }
const location = response.headers.get("location"); if (response.status < 300 || response.status >= 400 || !location) { break; }
await response.body?.cancel().catch(() => undefined); if (redirects >= MAX_REDIRECTS) { throw new Error(`Too many redirects fetching ${currentUrl.href}.`); }
let nextUrl: URL; try { nextUrl = new URL(location, currentUrl); } catch { throw new Error(`Invalid redirect from ${currentUrl.href}.`); } currentUrl = validateFetchUrl(nextUrl.href); redirects += 1; }
if (response.status >= 400) { await response.body?.cancel().catch(() => undefined); throw new Error( `Request to ${currentUrl.href} failed with HTTP ${response.status}${response.statusText ? ` ${response.statusText}` : ""}.`, ); }
if (!response.body) { return { truncated: false, finalUrl: currentUrl.href, }; }
const contentType = response.headers.get("content-type") ?? ""; if (!isSupportedContentType(contentType)) { await response.body.cancel().catch(() => undefined); return { body: `[Body omitted: expected text/markdown, text/html, or text/plain, but response content type is ${contentType || "missing"}.]`, truncated: false, finalUrl: currentUrl.href, }; }
const reader = response.body.getReader(); const chunks: Buffer[] = []; let bytesRead = 0; let truncated = false; try { while (true) { const { done, value } = await reader.read(); if (done) { break; } const chunk = Buffer.from(value); const remaining = maxBytes - bytesRead; if (chunk.length > remaining) { chunks.push(chunk.subarray(0, remaining)); bytesRead = maxBytes; truncated = true; await reader.cancel().catch(() => undefined); break; } chunks.push(chunk); bytesRead += chunk.length; } } catch (error) { throw requestError(error, currentUrl, timeoutMs, timeoutSignal, signal); } finally { reader.releaseLock(); }
const text = decodeText(Buffer.concat(chunks, bytesRead), contentType); return { body: contentType.split(";", 1)[0]?.trim().toLowerCase() === "text/html" ? extractHtmlBody(text) : text, truncated, finalUrl: currentUrl.href, };}