import { execFile } from 'child_process'; import { pinnedFetch } from '../utils/pinned-fetch'; import { stripHtmlToText } from '../utils/html-strip'; interface ScrapeResult { success: boolean; url: string; title: string; description: string; content: string; contentLength: number; contentType: string; } // Hard ceiling on fetched HTML. Anything larger is a strong signal this is not // a page we care about; avoids DoS via multi-GB downloads. const SCRAPE_MAX_BYTES = 2 * 1024 * 1024; // 2 MiB const SCRAPE_TIMEOUT_MS = 15_000; const SCRAPE_USER_AGENT = 'Mozilla/5.0 (compatible; AetherOS/1.0; +https://aether.os)'; export async function scrapeUrl(url: string, _selector?: string): Promise { console.log(`[Web] Scraping: ${url}`); const response = await pinnedFetch(url, { method: 'GET', headers: { 'user-agent': SCRAPE_USER_AGENT, accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'accept-language': 'en-US,en;q=0.5', }, timeoutMs: SCRAPE_TIMEOUT_MS, maxBytes: SCRAPE_MAX_BYTES, allowedContentTypes: ['text/html', 'application/xhtml+xml', 'application/xml', 'text/plain'], }); if (response.status >= 400) { throw new Error(`HTTP ${response.status}`); } const html = response.body.toString('utf8'); const extracted = stripHtmlToText(html, { maxOutputChars: 10_000 }); return { success: true, url, title: extracted.title, description: extracted.description, content: extracted.text, contentLength: extracted.text.length, contentType: response.contentType, }; } export async function searchDuckDuckGo(q: string, limit: number = 5) { console.log(`[Web] Searching: ${q}`); const searchUrl = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(q)}`; const response = await pinnedFetch(searchUrl, { headers: { 'user-agent': SCRAPE_USER_AGENT, accept: 'text/html', }, timeoutMs: 10_000, maxBytes: SCRAPE_MAX_BYTES, allowedContentTypes: ['text/html'], }); if (response.status >= 400) { throw new Error(`Search failed: HTTP ${response.status}`); } const html = response.body.toString('utf8'); // Parse search results from DuckDuckGo HTML const results = []; const resultRegex = /]*class="result__a"[^>]*href="([^"]*)"[^>]*>([^<]*)<\/a>[\s\S]*?]*class="result__snippet"[^>]*>([^<]*)/gi; let match; while ((match = resultRegex.exec(html)) !== null && results.length < limit) { // DuckDuckGo wraps URLs, need to extract actual URL let url = match[1]; if (url.includes('uddg=')) { const urlMatch = url.match(/uddg=([^&]*)/); if (urlMatch) { url = decodeURIComponent(urlMatch[1]); } } results.push({ title: match[2].trim(), url, snippet: match[3].trim().replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>'), }); } // Fallback: simpler parsing if regex didn't match if (results.length === 0) { const linkRegex = /]*class="[^"]*result__a[^"]*"[^>]*>([^<]+)<\/a>/gi; const snippetRegex = /]*class="[^"]*result__snippet[^"]*"[^>]*>([^<]+)<\/a>/gi; const titles = []; const snippets = []; while ((match = linkRegex.exec(html)) !== null) { titles.push(match[1].trim()); } while ((match = snippetRegex.exec(html)) !== null) { snippets.push(match[1].trim()); } for (let i = 0; i < Math.min(titles.length, limit); i++) { results.push({ title: titles[i], snippet: snippets[i] || '', url: '', }); } } return { success: true, query: q, results, resultCount: results.length, }; } // Headless-browser scrape via the agent-browser CLI — gated off by default because it // executes page JS. Falls back to the simple fetch-based scraper on error or when disabled. export async function agentBrowse(url: string): Promise { // Pre-check URLs before the agent-browser binary performs its own fetch. const parsedUrl = new URL(url); if (!['http:', 'https:'].includes(parsedUrl.protocol)) { throw new Error('Invalid URL protocol'); } const { isPrivateUrlResolved } = await import('../utils/ssrf'); if (await isPrivateUrlResolved(parsedUrl)) { throw new Error('Requests to private/internal addresses are not allowed'); } if (process.env.SUBSTRATE_ENABLE_AGENT_BROWSER !== 'true') { const result = await scrapeUrl(url); return { ...result, method: 'fallback', note: 'Agent browser disabled by default for security hardening', }; } console.log(`[Web] Agent browsing: ${url}`); // Resolve the binary independent of cwd. let browserBin: string; try { browserBin = require.resolve('agent-browser/package.json').replace(/package\.json$/, 'bin/agent-browser'); } catch { browserBin = './node_modules/.bin/agent-browser'; } return new Promise((resolve, reject) => { // Use execFile with argument array to prevent shell injection execFile(browserBin, [url, '--json'], { timeout: 30000, maxBuffer: 1024 * 1024 * 5 // 5MB buffer }, (error, stdout, stderr) => { // agent-browser may log to stderr on success. if (error) { console.error('[Web] Agent browser error:', error.message); console.error('[Web] Agent browser stderr:', stderr); // Fallback to standard scraper console.log('[Web] Falling back to standard scraper...'); scrapeUrl(url) .then(res => resolve({ ...res, method: 'fallback', note: 'Agent browser failed, used standard scraper' })) .catch(err => reject(err)); return; } try { // agent-browser output might contain logs before the JSON // Find the first '{' and last '}' const jsonStart = stdout.indexOf('{'); const jsonEnd = stdout.lastIndexOf('}'); if (jsonStart === -1 || jsonEnd === -1) { throw new Error("No JSON found in output"); } const jsonStr = stdout.substring(jsonStart, jsonEnd + 1); const data = JSON.parse(jsonStr); resolve({ success: true, url, method: 'agent-browser', data }); } catch (e) { console.error('[Web] Agent browser JSON parse error:', e); // Fallback scrapeUrl(url) .then(res => resolve({ ...res, method: 'fallback', note: 'Agent browser output parse failed' })) .catch(err => reject(err)); } }); }); }