// Linear-time HTML -> text extraction without regex backtracking. // Skips non-content tags and keeps bounded work per byte. const SKIP_ELEMENTS = new Set(['script', 'style', 'head', 'nav', 'footer', 'noscript', 'template', 'svg']); const CONTENT_ELEMENTS: Array<'article' | 'main'> = ['article', 'main']; interface StripOptions { maxOutputChars?: number; } interface StripResult { title: string; description: string; text: string; } // Very small entity decoder for the common named entities plus numeric refs. // Full HTML entity tables are large; these cover 99% of scraped content. const NAMED_ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', copy: '©', reg: '®', hellip: '…', mdash: '—', ndash: '–', lsquo: '‘', rsquo: '’', ldquo: '“', rdquo: '”', }; function decodeEntities(s: string): string { return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (match, body: string) => { if (body[0] === '#') { const isHex = body[1] === 'x' || body[1] === 'X'; const num = parseInt(isHex ? body.slice(2) : body.slice(1), isHex ? 16 : 10); if (!Number.isFinite(num) || num <= 0 || num > 0x10ffff) return match; try { return String.fromCodePoint(num); } catch { return match; } } const lower = body.toLowerCase(); return NAMED_ENTITIES[lower] ?? match; }); } // Extract a single attribute value from a start-tag text chunk. Accepts // single, double or unquoted values. `tagInner` is the text between '<' and // '>', excluding those characters. function getAttr(tagInner: string, name: string): string | undefined { const re = new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i'); const m = tagInner.match(re); if (!m) return undefined; return decodeEntities(m[1] ?? m[2] ?? m[3] ?? ''); } // Strip HTML from `input`, preferring the content of
/
when // present. Returns extracted title, meta description, and plain text. export function stripHtmlToText(input: string, opts: StripOptions = {}): StripResult { const maxChars = opts.maxOutputChars ?? 10_000; const src = input; const len = src.length; let title = ''; let description = ''; // Scan for and <meta name=description>. Linear, bail early if both found. { let i = 0; while (i < len) { const lt = src.indexOf('<', i); if (lt === -1) break; const gt = src.indexOf('>', lt + 1); if (gt === -1) break; const tagInner = src.slice(lt + 1, gt); const nameMatch = tagInner.match(/^\s*\/?\s*([a-zA-Z][a-zA-Z0-9]*)/); if (!nameMatch) { i = gt + 1; continue; } const tagName = nameMatch[1].toLowerCase(); if (tagName === 'title' && tagInner[0] !== '/' && !tagInner.startsWith('/')) { const endIdx = src.toLowerCase().indexOf('</title', gt + 1); if (endIdx !== -1) { title = decodeEntities(src.slice(gt + 1, endIdx)).replace(/\s+/g, ' ').trim(); } i = gt + 1; continue; } if (tagName === 'meta' && /\bname\s*=\s*["']?description/i.test(tagInner)) { description = getAttr(tagInner, 'content') || ''; description = description.replace(/\s+/g, ' ').trim(); } if (title && description) break; i = gt + 1; } } // Prefer the first <article> or <main> when present. let body = src; for (const tag of CONTENT_ELEMENTS) { const openIdx = findOpeningTag(src, tag); if (openIdx !== -1) { const openEnd = src.indexOf('>', openIdx); if (openEnd !== -1) { const closeIdx = findClosingTag(src, tag, openEnd + 1); if (closeIdx !== -1) { body = src.slice(openEnd + 1, closeIdx); break; } } } } // Main pass: walk `body`, skipping tags and the contents of SKIP_ELEMENTS. const out: string[] = []; let charCount = 0; let i = 0; const bodyLen = body.length; const pushText = (text: string): boolean => { const decoded = decodeEntities(text).replace(/\s+/g, ' '); if (!decoded.trim() && out.length === 0) return false; out.push(decoded); charCount += decoded.length; return charCount >= maxChars; }; while (i < bodyLen) { const lt = body.indexOf('<', i); if (lt === -1) { if (pushText(body.slice(i))) break; break; } if (lt > i) { if (pushText(body.slice(i, lt))) break; } const gt = body.indexOf('>', lt + 1); if (gt === -1) break; // malformed: no more tags, stop const tagInner = body.slice(lt + 1, gt); const nameMatch = tagInner.match(/^\s*\/?\s*([a-zA-Z][a-zA-Z0-9]*)/); if (!nameMatch) { i = gt + 1; continue; } const tagName = nameMatch[1].toLowerCase(); const isClosing = tagInner.trimStart().startsWith('/'); if (!isClosing && SKIP_ELEMENTS.has(tagName)) { // Skip to matching close tag (or EOF) const closeIdx = findClosingTag(body, tagName, gt + 1); if (closeIdx === -1) break; const closeEnd = body.indexOf('>', closeIdx); i = closeEnd === -1 ? bodyLen : closeEnd + 1; continue; } i = gt + 1; } let text = out.join('').replace(/\s+/g, ' ').trim(); if (text.length > maxChars) text = text.slice(0, maxChars) + '...'; return { title, description, text }; } function findOpeningTag(src: string, tag: string): number { const re = new RegExp(`<${tag}\\b`, 'i'); const m = src.match(re); return m ? m.index ?? -1 : -1; } function findClosingTag(src: string, tag: string, from: number): number { const needle = `</${tag}`; const lower = src.toLowerCase(); return lower.indexOf(needle, from); }