// Linear-time HTML -> text extraction without regex backtracking.
// Skips non-content tags and keeps bounded work per byte.
const SKIP_ELEMENTS = new Set(['script', 'style', 'head', 'nav', 'footer', 'noscript', 'template', 'svg']);
const CONTENT_ELEMENTS: Array<'article' | 'main'> = ['article', 'main'];
interface StripOptions {
maxOutputChars?: number;
}
interface StripResult {
title: string;
description: string;
text: string;
}
// Very small entity decoder for the common named entities plus numeric refs.
// Full HTML entity tables are large; these cover 99% of scraped content.
const NAMED_ENTITIES: Record = {
amp: '&',
lt: '<',
gt: '>',
quot: '"',
apos: "'",
nbsp: ' ',
copy: '©',
reg: '®',
hellip: '…',
mdash: '—',
ndash: '–',
lsquo: '‘',
rsquo: '’',
ldquo: '“',
rdquo: '”',
};
function decodeEntities(s: string): string {
return s.replace(/&(#x?[0-9a-fA-F]+|[a-zA-Z]+);/g, (match, body: string) => {
if (body[0] === '#') {
const isHex = body[1] === 'x' || body[1] === 'X';
const num = parseInt(isHex ? body.slice(2) : body.slice(1), isHex ? 16 : 10);
if (!Number.isFinite(num) || num <= 0 || num > 0x10ffff) return match;
try {
return String.fromCodePoint(num);
} catch {
return match;
}
}
const lower = body.toLowerCase();
return NAMED_ENTITIES[lower] ?? match;
});
}
// Extract a single attribute value from a start-tag text chunk. Accepts
// single, double or unquoted values. `tagInner` is the text between '<' and
// '>', excluding those characters.
function getAttr(tagInner: string, name: string): string | undefined {
const re = new RegExp(`\\b${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i');
const m = tagInner.match(re);
if (!m) return undefined;
return decodeEntities(m[1] ?? m[2] ?? m[3] ?? '');
}
// Strip HTML from `input`, preferring the content of / when
// present. Returns extracted title, meta description, and plain text.
export function stripHtmlToText(input: string, opts: StripOptions = {}): StripResult {
const maxChars = opts.maxOutputChars ?? 10_000;
const src = input;
const len = src.length;
let title = '';
let description = '';
// Scan for and . Linear, bail early if both found.
{
let i = 0;
while (i < len) {
const lt = src.indexOf('<', i);
if (lt === -1) break;
const gt = src.indexOf('>', lt + 1);
if (gt === -1) break;
const tagInner = src.slice(lt + 1, gt);
const nameMatch = tagInner.match(/^\s*\/?\s*([a-zA-Z][a-zA-Z0-9]*)/);
if (!nameMatch) { i = gt + 1; continue; }
const tagName = nameMatch[1].toLowerCase();
if (tagName === 'title' && tagInner[0] !== '/' && !tagInner.startsWith('/')) {
const endIdx = src.toLowerCase().indexOf(' or when present.
let body = src;
for (const tag of CONTENT_ELEMENTS) {
const openIdx = findOpeningTag(src, tag);
if (openIdx !== -1) {
const openEnd = src.indexOf('>', openIdx);
if (openEnd !== -1) {
const closeIdx = findClosingTag(src, tag, openEnd + 1);
if (closeIdx !== -1) {
body = src.slice(openEnd + 1, closeIdx);
break;
}
}
}
}
// Main pass: walk `body`, skipping tags and the contents of SKIP_ELEMENTS.
const out: string[] = [];
let charCount = 0;
let i = 0;
const bodyLen = body.length;
const pushText = (text: string): boolean => {
const decoded = decodeEntities(text).replace(/\s+/g, ' ');
if (!decoded.trim() && out.length === 0) return false;
out.push(decoded);
charCount += decoded.length;
return charCount >= maxChars;
};
while (i < bodyLen) {
const lt = body.indexOf('<', i);
if (lt === -1) {
if (pushText(body.slice(i))) break;
break;
}
if (lt > i) {
if (pushText(body.slice(i, lt))) break;
}
const gt = body.indexOf('>', lt + 1);
if (gt === -1) break; // malformed: no more tags, stop
const tagInner = body.slice(lt + 1, gt);
const nameMatch = tagInner.match(/^\s*\/?\s*([a-zA-Z][a-zA-Z0-9]*)/);
if (!nameMatch) {
i = gt + 1;
continue;
}
const tagName = nameMatch[1].toLowerCase();
const isClosing = tagInner.trimStart().startsWith('/');
if (!isClosing && SKIP_ELEMENTS.has(tagName)) {
// Skip to matching close tag (or EOF)
const closeIdx = findClosingTag(body, tagName, gt + 1);
if (closeIdx === -1) break;
const closeEnd = body.indexOf('>', closeIdx);
i = closeEnd === -1 ? bodyLen : closeEnd + 1;
continue;
}
i = gt + 1;
}
let text = out.join('').replace(/\s+/g, ' ').trim();
if (text.length > maxChars) text = text.slice(0, maxChars) + '...';
return { title, description, text };
}
function findOpeningTag(src: string, tag: string): number {
const re = new RegExp(`<${tag}\\b`, 'i');
const m = src.match(re);
return m ? m.index ?? -1 : -1;
}
function findClosingTag(src: string, tag: string, from: number): number {
const needle = `${tag}`;
const lower = src.toLowerCase();
return lower.indexOf(needle, from);
}