Something went wrong. Try again.
ATProto-based AI Assistant Chat interface.
Something went wrong. Try again.
TypeScript
12345678910111213141516171819202122232425262728293031323334import { Readability } from '@mozilla/readability'import { parseHTML } from 'linkedom'import TurndownService from 'turndown'
const CLUTTER = 'script, style, nav, header, footer, noscript, template, iframe'
/** Point links and images at absolute URLs, so the model can fetch the content. */function absolutize(document: ReturnType<typeof parseHTML>['document'], base: string) { for (const [selector, attribute] of [ ['a[href]', 'href'], ['img[src]', 'src'], ] as const) { for (const element of document.querySelectorAll(selector)) { const value = element.getAttribute(attribute) ?? '' if (URL.canParse(value, base)) element.setAttribute(attribute, new URL(value, base).href) } }}
/** The page's title and main text as Markdown, without navigation, scripts, and other clutter. */export function htmlToMarkdown(html: string, url: string): { title?: string; markdown: string } { const { document } = parseHTML(html) const title = document.title?.trim() || undefined for (const element of document.querySelectorAll(CLUTTER)) element.remove() absolutize(document, url) const article = new Readability(parseHTML(document.toString()).document as never).parse() if (!article?.content) return { title, markdown: '' } const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' }) return { title: article.title?.trim() || title, markdown: turndown.turndown(article.content).trim(), }}