Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182// Linear-time HTML -> text extraction without regex backtracking.// Skips non-content tags and keeps bounded work per byte.
const SKIP_ELEMENTS = new Set(['script', 'style', 'head', 'nav', 'footer', 'noscript', 'template', 'svg']);const CONTENT_ELEMENTS: Array<'article' | 'main'> = ['article', 'main'];
interface StripOptions { maxOutputChars?: number;}
interface StripResult { title: string; description: string; text: string;}
// Very small entity decoder for the common named entities plus numeric refs.// Full HTML entity tables are large; these cover 99% of scraped content.const NAMED_ENTITIES: Record<string, string> = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ', copy: '©', reg: '®', hellip: '…', mdash: '—', ndash: '–', lsquo: '‘', rsquo: '’', ldquo: '“', rdquo: '”',};
function decodeEntities(s: string): string { return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (match, body: string) => { if (body[0] === '#') { const isHex = body[1] === 'x' || body[1] === 'X'; const num = parseInt(isHex ? body.slice(2) : body.slice(1), isHex ? 16 : 10); if (!Number.isFinite(num) || num <= 0 || num > 0x10ffff) return match; try { return String.fromCodePoint(num); } catch { return match; } } const lower = body.toLowerCase(); return NAMED_ENTITIES[lower] ?? match; });}
// Extract a single attribute value from a start-tag text chunk. Accepts// single, double or unquoted values. `tagInner` is the text between '<' and// '>', excluding those characters.function getAttr(tagInner: string, name: string): string | undefined { const re = new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i'); const m = tagInner.match(re); if (!m) return undefined; return decodeEntities(m[1] ?? m[2] ?? m[3] ?? '');}
// Strip HTML from `input`, preferring the content of <article>/<main> when// present. Returns extracted title, meta description, and plain text.export function stripHtmlToText(input: string, opts: StripOptions = {}): StripResult { const maxChars = opts.maxOutputChars ?? 10_000; const src = input; const len = src.length;
let title = ''; let description = '';
// Scan for <title> and <meta name=description>. Linear, bail early if both found. { let i = 0; while (i < len) { const lt = src.indexOf('<', i); if (lt === -1) break; const gt = src.indexOf('>', lt + 1); if (gt === -1) break; const tagInner = src.slice(lt + 1, gt); const nameMatch = tagInner.match(/^\s*\/?\s*([a-zA-Z][a-zA-Z0-9]*)/); if (!nameMatch) { i = gt + 1; continue; } const tagName = nameMatch[1].toLowerCase(); if (tagName === 'title' && tagInner[0] !== '/' && !tagInner.startsWith('/')) { const endIdx = src.toLowerCase().indexOf('</title', gt + 1); if (endIdx !== -1) { title = decodeEntities(src.slice(gt + 1, endIdx)).replace(/\s+/g, ' ').trim(); } i = gt + 1; continue; } if (tagName === 'meta' && /\bname\s*=\s*["']?description/i.test(tagInner)) { description = getAttr(tagInner, 'content') || ''; description = description.replace(/\s+/g, ' ').trim(); } if (title && description) break; i = gt + 1; } }
// Prefer the first <article> or <main> when present. let body = src; for (const tag of CONTENT_ELEMENTS) { const openIdx = findOpeningTag(src, tag); if (openIdx !== -1) { const openEnd = src.indexOf('>', openIdx); if (openEnd !== -1) { const closeIdx = findClosingTag(src, tag, openEnd + 1); if (closeIdx !== -1) { body = src.slice(openEnd + 1, closeIdx); break; } } } }
// Main pass: walk `body`, skipping tags and the contents of SKIP_ELEMENTS. const out: string[] = []; let charCount = 0; let i = 0; const bodyLen = body.length;
const pushText = (text: string): boolean => { const decoded = decodeEntities(text).replace(/\s+/g, ' '); if (!decoded.trim() && out.length === 0) return false; out.push(decoded); charCount += decoded.length; return charCount >= maxChars; };
while (i < bodyLen) { const lt = body.indexOf('<', i); if (lt === -1) { if (pushText(body.slice(i))) break; break; } if (lt > i) { if (pushText(body.slice(i, lt))) break; } const gt = body.indexOf('>', lt + 1); if (gt === -1) break; // malformed: no more tags, stop const tagInner = body.slice(lt + 1, gt); const nameMatch = tagInner.match(/^\s*\/?\s*([a-zA-Z][a-zA-Z0-9]*)/); if (!nameMatch) { i = gt + 1; continue; } const tagName = nameMatch[1].toLowerCase(); const isClosing = tagInner.trimStart().startsWith('/');
if (!isClosing && SKIP_ELEMENTS.has(tagName)) { // Skip to matching close tag (or EOF) const closeIdx = findClosingTag(body, tagName, gt + 1); if (closeIdx === -1) break; const closeEnd = body.indexOf('>', closeIdx); i = closeEnd === -1 ? bodyLen : closeEnd + 1; continue; }
i = gt + 1; }
let text = out.join('').replace(/\s+/g, ' ').trim(); if (text.length > maxChars) text = text.slice(0, maxChars) + '...';
return { title, description, text };}
function findOpeningTag(src: string, tag: string): number { const re = new RegExp(`<${tag}\\b`, 'i'); const m = src.match(re); return m ? m.index ?? -1 : -1;}
function findClosingTag(src: string, tag: string, from: number): number { const needle = `</${tag}`; const lower = src.toLowerCase(); return lower.indexOf(needle, from);}