Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299//! What parcel reads of HTML without showing it: the text of a message,//! to quote it, and the character references in Gmail's snippets; and a//! plain text body made HTML to show.//! baskingspot/html's tokenizer takes the place of this once it builds on//! Zig 0.17.
const std = @import("std");const ascii = std.ascii;const fmt = std.fmt;const mem = std.mem;const testing = std.testing;const unicode = std.unicode;const Allocator = std.mem.Allocator;const ArenaAllocator = std.heap.ArenaAllocator;const ArrayList = std.ArrayList;const Io = std.Io;const Writer = Io.Writer;
/// A plain text body as HTML: escaped, its line breaks kept, and the web/// and mailto: addresses in it links, as a mail client shows them. Lines/// quoted with ">" go in a blockquote for each ">", without the marks,/// as an HTML reply quotes, so they show and fold the same.pub fn fromText(w: *Writer, text: []const u8) !void { try w.writeAll("<div style=\"white-space: pre-wrap\">"); var depth: usize = 0; var lines = mem.splitScalar(u8, text, '\n'); var first = true; while (lines.next()) |whole| { var line = whole; var quotes: usize = 0; // "> > " and ">> " both quote twice. while (true) { const rest = mem.trimStart(u8, line, " "); if (rest.len == 0 or rest[0] != '>' or (quotes == 0 and rest.len != line.len)) break; line = rest[1..]; quotes += 1; } if (quotes > 0 and mem.startsWith(u8, line, " ")) line = line[1..]; // A quote's edges break the line, as blocks do. if (quotes == depth and !first) try w.writeByte('\n'); while (depth < quotes) : (depth += 1) try w.writeAll("<blockquote type=\"cite\">"); while (depth > quotes) : (depth -= 1) try w.writeAll("</blockquote>"); try linked(w, line); first = false; } while (depth > 0) : (depth -= 1) try w.writeAll("</blockquote>"); try w.writeAll("</div>");}
/// `text` escaped, its web and mailto: addresses links.fn linked(w: *Writer, text: []const u8) !void { var i: usize = 0; while (i < text.len) { if (urlAt(text, i)) |url| { const www = ascii.startsWithIgnoreCase(url, "www."); try w.writeAll("<a href=\""); if (www) try w.writeAll("http://"); try escape(w, url); try w.writeAll("\">"); try escape(w, url); try w.writeAll("</a>"); i += url.len; } else { try escape(w, text[i .. i + 1]); i += 1; } }}
fn escape(w: *Writer, text: []const u8) !void { for (text) |b| try switch (b) { '&' => w.writeAll("&"), '<' => w.writeAll("<"), '>' => w.writeAll(">"), '"' => w.writeAll("""), else => w.writeByte(b), };}
/// The address starting at `i`, if one does: up to a space or an angle/// bracket, less the punctuation of the sentence around it, and a closing/// parenthesis it did not open.fn urlAt(text: []const u8, i: usize) ?[]const u8 { if (i > 0 and (ascii.isAlphanumeric(text[i - 1]) or text[i - 1] == '/' or text[i - 1] == '.')) return null; const rest = text[i..]; const prefix = for ([_][]const u8{ "https://", "http://", "www.", "mailto:" }) |p| { if (ascii.startsWithIgnoreCase(rest, p)) break p; } else return null; var end: usize = 0; while (end < rest.len and rest[end] > ' ' and rest[end] != 0x7f and mem.findScalar(u8, "<>\"", rest[end]) == null) end += 1; while (end > prefix.len) : (end -= 1) { const last = rest[end - 1]; if (mem.findScalar(u8, ".,;:!?'*", last) != null) continue; if (last == ')' and mem.countScalar(u8, rest[0..end], '(') < mem.countScalar(u8, rest[0..end], ')')) continue; break; } return if (end > prefix.len) rest[0..end] else null;}
/// The text of an HTML body, roughly as it reads: tags dropped, block/// ends as line breaks, list items as bullets, scripts, styles, and the/// head left out, runs of blank lines and spaces made one.pub fn toText(arena: Allocator, source: []const u8) ![]const u8 { var out: ArrayList(u8) = try .initCapacity(arena, source.len / 2); var i: usize = 0; while (i < source.len) { if (source[i] != '<') { const end = mem.findScalarPos(u8, source, i, '<') orelse source.len; const text = try decodeEntities(arena, source[i..end]); for (text) |c| try addChar(arena, &out, if (c == '\n' or c == '\r' or c == '\t') ' ' else c); i = end; continue; } const close = mem.findScalarPos(u8, source, i, '>') orelse break; const tag = source[i + 1 .. close]; i = close + 1; const ending = tag.len > 0 and tag[0] == '/'; const name = tagName(if (ending) tag[1..] else tag); if (!ending and isOneOf(name, &.{ "script", "style", "head", "title" })) { // Skipped to its end tag. var at = i; while (ascii.findIgnoreCasePos(source, at, "</")) |e| : (at = e + 2) { if (ascii.startsWithIgnoreCase(source[e + 2 ..], name)) { i = (mem.findScalarPos(u8, source, e, '>') orelse source.len - 1) + 1; break; } } else i = source.len; } else if (isOneOf(name, &blocks)) { try newline(arena, &out); } else if (!ending and ascii.eqlIgnoreCase(name, "li")) { try newline(arena, &out); try out.appendSlice(arena, "• "); } else if (!ending and isOneOf(name, &.{ "td", "th" })) { try addChar(arena, &out, ' '); } } return mem.trim(u8, out.items, " \n");}
/// Tags whose start or end breaks the line.const blocks = [_][]const u8{ "br", "p", "div", "tr", "h1", "h2", "h3", "h4", "h5", "h6", "ul", "blockquote", "ol", "table",};
fn tagName(tag: []const u8) []const u8 { const end = mem.findAny(u8, tag, " \t\r\n/") orelse tag.len; return tag[0..end];}
fn isOneOf(name: []const u8, names: []const []const u8) bool { for (names) |n| { if (ascii.eqlIgnoreCase(name, n)) return true; } return false;}
/// Adds a character, but no space at a line's start or after another.fn addChar(arena: Allocator, out: *ArrayList(u8), c: u8) !void { if (c == ' ' and (out.items.len == 0 or out.items[out.items.len - 1] == ' ' or out.items[out.items.len - 1] == '\n')) return; try out.append(arena, c);}
/// Ends a line, leaving no space before it and one blank line at most.fn newline(arena: Allocator, out: *ArrayList(u8)) !void { while (out.items.len > 0 and out.items[out.items.len - 1] == ' ') out.items.len -= 1; if (out.items.len >= 2 and mem.eql(u8, out.items[out.items.len - 2 ..], "\n\n")) return; try out.append(arena, '\n');}
/// Decodes the character references HTML text uses, such as Gmail's/// snippets: `'` for an apostrophe.pub fn decodeEntities(arena: Allocator, text: []const u8) ![]const u8 { if (mem.findScalar(u8, text, '&') == null) return text; const named = std.StaticStringMap(u21).initComptime(.{ .{ "amp", '&' }, .{ "lt", '<' }, .{ "gt", '>' }, .{ "quot", '"' }, .{ "apos", '\'' }, .{ "nbsp", 0xa0 }, }); var out: ArrayList(u8) = try .initCapacity(arena, text.len); var i: u32 = 0; while (i < text.len) { const semi = if (text[i] == '&') mem.findScalarPos(u8, text, i, ';') else null; const ref = if (semi) |s| text[i + 1 .. s] else ""; const cp: ?u21 = if (ref.len < 2 or ref.len > 8) null else if (ref[0] == '#' and (ref[1] == 'x' or ref[1] == 'X')) fmt.parseInt(u21, ref[2..], 16) catch null else if (ref[0] == '#') fmt.parseInt(u21, ref[1..], 10) catch null else named.get(ref); if (cp != null and unicode.utf8ValidCodepoint(cp.?)) { var buf: [4]u8 = undefined; const len = unicode.utf8Encode(cp.?, &buf) catch unreachable; try out.appendSlice(arena, buf[0..len]); i = @intCast(semi.? + 1); } else { try out.append(arena, text[i]); i += 1; } } return out.items;}
/// `bytes` in UTF-8: Latin-1 and Windows-1252 turned into it, anything/// else invalid in it replaced, so a quote of it can be sent as UTF-8.pub fn toUtf8(arena: Allocator, bytes: []const u8, charset: []const u8) ![]const u8 { const latin = isOneOf(charset, &.{ "iso-8859-1", "latin1", "windows-1252", "cp1252", "us-ascii" }); if (!latin and unicode.utf8ValidateSlice(bytes)) return bytes; var out: ArrayList(u8) = try .initCapacity(arena, bytes.len); var i: usize = 0; while (i < bytes.len) { var buf: [4]u8 = undefined; if (latin) { const n = unicode.utf8Encode(bytes[i], &buf) catch unreachable; try out.appendSlice(arena, buf[0..n]); i += 1; continue; } const n = unicode.utf8ByteSequenceLength(bytes[i]) catch 0; if (n > 0 and i + n <= bytes.len and unicode.utf8ValidateSlice(bytes[i .. i + n])) { try out.appendSlice(arena, bytes[i .. i + n]); i += n; } else { try out.appendSlice(arena, "\u{fffd}"); i += 1; } } return out.items;}
test "an HTML body reads as text: blocks on lines, items as bullets, no styles" { var arena_state: ArenaAllocator = .init(testing.allocator); defer arena_state.deinit(); const text = try toText(arena_state.allocator(), \\<html><head><style>p { color: red }</style></head><body> \\<p>Hi Ana,</p><p>Fish & chips<br>at 8?</p> \\<ul><li>bread</li><LI>wine</li></ul><script>x()</script><div>Bo</div></body></html> ); try testing.expectEqualStrings("Hi Ana,\n\nFish & chips\nat 8?\n\n• bread\n• wine\n\nBo", text);}
test "Latin-1 becomes UTF-8, and bad UTF-8 is replaced" { var arena_state: ArenaAllocator = .init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); try testing.expectEqualStrings("café", try toUtf8(arena, "caf\xe9", "ISO-8859-1")); try testing.expectEqualStrings("caf\u{fffd}!", try toUtf8(arena, "caf\xe9!", "utf-8"));}
test "a reference that is not one stays as text" { var arena_state: ArenaAllocator = .init(testing.allocator); defer arena_state.deinit(); const arena = arena_state.allocator(); const decoded = try decodeEntities(arena, "AT&T; 5 < 6 &bogus; 😀"); try testing.expectEqualStrings("AT&T; 5 < 6 &bogus; 😀", decoded);}
test "plain text made HTML: escaped, and its addresses links without the punctuation around them" { var out: Writer.Allocating = .init(testing.allocator); defer out.deinit(); try fromText(&out.writer, \\See https://x.org/a?b=1&c=2. \\(www.x.org/y) \\<https://x.org/Foo_(bar)>, \\mailto:ana@x.org! \\nothttps://x.org a <b> & "c" http:// ); try testing.expectEqualStrings( \\<div style="white-space: pre-wrap">See <a href="https://x.org/a?b=1&c=2">https://x.org/a?b=1&c=2</a>. \\(<a href="http://www.x.org/y">www.x.org/y</a>) \\<<a href="https://x.org/Foo_(bar)">https://x.org/Foo_(bar)</a>>, \\<a href="mailto:ana@x.org">mailto:ana@x.org</a>! \\nothttps://x.org a <b> & "c" http://</div> , out.written());}
test "plain text made HTML: a blockquote for each > of a quoted line, without the marks" { var out: Writer.Allocating = .init(testing.allocator); defer out.deinit(); try fromText(&out.writer, \\Sure. \\ \\On Mon, Ana wrote: \\> Lunch? \\> \\>> Bo: \\> > >Deep \\> Back \\ > not one, nor 1 > 2 ); try testing.expectEqualStrings("<div style=\"white-space: pre-wrap\">Sure.\n\nOn Mon, Ana wrote:" ++ "<blockquote type=\"cite\">Lunch?\n" ++ "<blockquote type=\"cite\">Bo:<blockquote type=\"cite\">Deep</blockquote></blockquote>" ++ "Back</blockquote> > not one, nor 1 > 2</div>", out.written());}