// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-License-Identifier: MIT //! The encodings the Encoding Standard defines, and the labels that name //! them. const std = @import("std"); const labels = @import("tables/labels.zig"); /// Every encoding in the Encoding Standard, and no others: the standard is a /// closed list, and says in as many words that a user agent must not support /// anything outside it. /// /// The spelling of a member is the standard's name lowercased with the /// punctuation turned into underscores, except that `UTF-8`, `UTF-16BE` and /// `UTF-16LE` become `utf8`, `utf16be` and `utf16le` rather than the uglier /// mechanical answer. `name` gives the standard's spelling back. pub const Encoding = enum { /// The one encoding anything new should use, and the only one with an /// encoder worth reaching for. utf8, // Legacy single-byte encodings. Each is a table of 128 entries for the // bytes 0x80 through 0xFF; the lower half is always ASCII. ibm866, iso_8859_2, iso_8859_3, iso_8859_4, iso_8859_5, iso_8859_6, iso_8859_7, iso_8859_8, /// ISO-8859-8 under another name. The two are the same table and differ /// only in what a consumer should assume about layout direction, which is /// why the standard keeps them apart. iso_8859_8_i, iso_8859_10, iso_8859_13, iso_8859_14, iso_8859_15, iso_8859_16, koi8_r, koi8_u, macintosh, windows_874, windows_1250, windows_1251, /// Also reached by the labels `iso-8859-1`, `latin1`, `ascii` and /// `us-ascii`, none of which name what they say they do. Deployed content /// labelled ISO-8859-1 is windows-1252 often enough that the standard /// gives up and decodes it as such. windows_1252, windows_1253, windows_1254, windows_1255, windows_1256, windows_1257, windows_1258, x_mac_cyrillic, // Legacy multi-byte Chinese (simplified). /// Decodes as gb18030 does. Encodes as gb18030 does except that it will /// not reach for the four-byte form, which keeps its output inside what a /// GBK consumer understands. gbk, gb18030, // Legacy multi-byte Chinese (traditional). big5, // Legacy multi-byte Japanese. euc_jp, iso_2022_jp, shift_jis, // Legacy multi-byte Korean. euc_kr, /// Not an encoding so much as a refusal to be one. Several labels for /// encodings that are dangerous to support -- ISO-2022-KR, ISO-2022-CN, /// HZ-GB-2312 -- map here, and its decoder emits a single U+FFFD for the /// whole input, whatever the input was. That closes an attack where a /// server and a client disagree about which encoding a label names. replacement, utf16be, utf16le, /// Every byte above 0x7F mapped into the private use area, so that bytes /// survive a round trip through something that insists on text. x_user_defined, /// The standard's name for this encoding, in the standard's own casing. /// This is what to write into a `Content-Type` header or a `` /// element; ASCII-lowercasing it always yields one of its labels. pub fn name(self: Encoding) []const u8 { return switch (self) { .utf8 => "UTF-8", .ibm866 => "IBM866", .iso_8859_2 => "ISO-8859-2", .iso_8859_3 => "ISO-8859-3", .iso_8859_4 => "ISO-8859-4", .iso_8859_5 => "ISO-8859-5", .iso_8859_6 => "ISO-8859-6", .iso_8859_7 => "ISO-8859-7", .iso_8859_8 => "ISO-8859-8", .iso_8859_8_i => "ISO-8859-8-I", .iso_8859_10 => "ISO-8859-10", .iso_8859_13 => "ISO-8859-13", .iso_8859_14 => "ISO-8859-14", .iso_8859_15 => "ISO-8859-15", .iso_8859_16 => "ISO-8859-16", .koi8_r => "KOI8-R", .koi8_u => "KOI8-U", .macintosh => "macintosh", .windows_874 => "windows-874", .windows_1250 => "windows-1250", .windows_1251 => "windows-1251", .windows_1252 => "windows-1252", .windows_1253 => "windows-1253", .windows_1254 => "windows-1254", .windows_1255 => "windows-1255", .windows_1256 => "windows-1256", .windows_1257 => "windows-1257", .windows_1258 => "windows-1258", .x_mac_cyrillic => "x-mac-cyrillic", .gbk => "GBK", .gb18030 => "gb18030", .big5 => "Big5", .euc_jp => "EUC-JP", .iso_2022_jp => "ISO-2022-JP", .shift_jis => "Shift_JIS", .euc_kr => "EUC-KR", .replacement => "replacement", .utf16be => "UTF-16BE", .utf16le => "UTF-16LE", .x_user_defined => "x-user-defined", }; } /// The standard's "get an encoding from a string": leading and trailing /// ASCII whitespace is removed, the rest is matched ASCII /// case-insensitively against the label table, and anything else is a /// failure. /// /// This is deliberately narrower than UTS #22 prescribes. A label with a /// stray character in it -- `utf8!`, `windows 1252` -- is not an /// encoding, and guessing is how a document ends up decoded one way by /// the server and another by the browser. pub fn fromLabel(label: []const u8) ?Encoding { // ASCII whitespace, which for this purpose is TAB, LF, FF, CR and // SPACE -- and not, for instance, a vertical tab. const trimmed = std.mem.trim(u8, label, "\t\n\x0c\r "); // Every label in the standard is shorter than this; anything longer // cannot be one, and saying so keeps the buffer off the heap. var buffer: [64]u8 = undefined; if (trimmed.len > buffer.len) return null; const lowered = buffer[0..trimmed.len]; for (trimmed, lowered) |c, *out| out.* = std.ascii.toLower(c); const index = std.sort.binarySearch( labels.Entry, &labels.entries, lowered, compareLabel, ) orelse return null; return labels.entries[index].encoding; } fn compareLabel(needle: []const u8, entry: labels.Entry) std.math.Order { return std.mem.order(u8, needle, entry.label); } /// The standard's "get an output encoding": what to actually encode with /// when a document asks for this one. /// /// `replacement`, `UTF-16BE` and `UTF-16LE` have no encoder -- the first /// because it exists to refuse, the other two because emitting UTF-16 /// from a form submission is a security problem rather than a feature -- /// so all three answer UTF-8. URL parsing and HTML form submission both /// need exactly this. pub fn outputEncoding(self: Encoding) Encoding { return switch (self) { .replacement, .utf16be, .utf16le => .utf8, else => self, }; } /// Whether this encoding has an encoder at all. The three that do not are /// the three `outputEncoding` redirects. pub fn canEncode(self: Encoding) bool { return self.outputEncoding() == self; } }; test "fromLabel matches the standard's own examples" { try std.testing.expectEqual(Encoding.utf8, Encoding.fromLabel("utf-8").?); try std.testing.expectEqual(Encoding.utf8, Encoding.fromLabel("UTF-8").?); try std.testing.expectEqual(Encoding.shift_jis, Encoding.fromLabel("Shift_JIS").?); try std.testing.expectEqual(Encoding.shift_jis, Encoding.fromLabel("sjis").?); try std.testing.expectEqual(Encoding.windows_1252, Encoding.fromLabel("windows-1252").?); try std.testing.expectEqual(Encoding.big5, Encoding.fromLabel("big5-hkscs").?); } test "fromLabel trims ASCII whitespace and only ASCII whitespace" { try std.testing.expectEqual(Encoding.utf8, Encoding.fromLabel(" \t\r\nutf-8\x0c ").?); try std.testing.expect(Encoding.fromLabel("\x0butf-8") == null); try std.testing.expect(Encoding.fromLabel("utf-8\x00") == null); } test "fromLabel refuses what is not a label" { try std.testing.expect(Encoding.fromLabel("") == null); try std.testing.expect(Encoding.fromLabel("utf8!") == null); try std.testing.expect(Encoding.fromLabel("windows 1252") == null); try std.testing.expect(Encoding.fromLabel("utf-9") == null); try std.testing.expect(Encoding.fromLabel("x" ** 100) == null); } test "ISO-8859-1 and friends are windows-1252, because deployed content says so" { for ([_][]const u8{ "iso-8859-1", "latin1", "ascii", "us-ascii", "cp819" }) |label| { try std.testing.expectEqual(Encoding.windows_1252, Encoding.fromLabel(label).?); } } test "the labels that name an attack map to replacement" { for ([_][]const u8{ "iso-2022-kr", "iso-2022-cn", "hz-gb-2312", "csiso2022kr" }) |label| { try std.testing.expectEqual(Encoding.replacement, Encoding.fromLabel(label).?); } } test "lowercasing a name gives a label for the same encoding" { var buffer: [64]u8 = undefined; for (std.enums.values(Encoding)) |encoding| { const lowered = std.ascii.lowerString(&buffer, encoding.name()); try std.testing.expectEqual(encoding, Encoding.fromLabel(lowered).?); } } test "the label table is sorted, which is what makes the search work" { for (labels.entries[1..], labels.entries[0 .. labels.entries.len - 1]) |after, before| { try std.testing.expect(std.mem.lessThan(u8, before.label, after.label)); } } test "outputEncoding sends the three encoderless encodings to UTF-8" { try std.testing.expectEqual(Encoding.utf8, Encoding.replacement.outputEncoding()); try std.testing.expectEqual(Encoding.utf8, Encoding.utf16le.outputEncoding()); try std.testing.expectEqual(Encoding.utf8, Encoding.utf16be.outputEncoding()); try std.testing.expectEqual(Encoding.shift_jis, Encoding.shift_jis.outputEncoding()); try std.testing.expect(!Encoding.replacement.canEncode()); try std.testing.expect(Encoding.big5.canEncode()); }