Something went wrong. Try again.
Converting between UTF-8 and legacy character encodings, as the WHATWG Encoding Standard and glibc's iconv define them.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! The encodings the Encoding Standard defines, and the labels that name//! them.
const std = @import("std");
const labels = @import("tables/labels.zig");
/// Every encoding in the Encoding Standard, and no others: the standard is a/// closed list, and says in as many words that a user agent must not support/// anything outside it.////// The spelling of a member is the standard's name lowercased with the/// punctuation turned into underscores, except that `UTF-8`, `UTF-16BE` and/// `UTF-16LE` become `utf8`, `utf16be` and `utf16le` rather than the uglier/// mechanical answer. `name` gives the standard's spelling back.pub const Encoding = enum { /// The one encoding anything new should use, and the only one with an /// encoder worth reaching for. utf8,
// Legacy single-byte encodings. Each is a table of 128 entries for the // bytes 0x80 through 0xFF; the lower half is always ASCII. ibm866, iso_8859_2, iso_8859_3, iso_8859_4, iso_8859_5, iso_8859_6, iso_8859_7, iso_8859_8, /// ISO-8859-8 under another name. The two are the same table and differ /// only in what a consumer should assume about layout direction, which is /// why the standard keeps them apart. iso_8859_8_i, iso_8859_10, iso_8859_13, iso_8859_14, iso_8859_15, iso_8859_16, koi8_r, koi8_u, macintosh, windows_874, windows_1250, windows_1251, /// Also reached by the labels `iso-8859-1`, `latin1`, `ascii` and /// `us-ascii`, none of which name what they say they do. Deployed content /// labelled ISO-8859-1 is windows-1252 often enough that the standard /// gives up and decodes it as such. windows_1252, windows_1253, windows_1254, windows_1255, windows_1256, windows_1257, windows_1258, x_mac_cyrillic,
// Legacy multi-byte Chinese (simplified). /// Decodes as gb18030 does. Encodes as gb18030 does except that it will /// not reach for the four-byte form, which keeps its output inside what a /// GBK consumer understands. gbk, gb18030,
// Legacy multi-byte Chinese (traditional). big5,
// Legacy multi-byte Japanese. euc_jp, iso_2022_jp, shift_jis,
// Legacy multi-byte Korean. euc_kr,
/// Not an encoding so much as a refusal to be one. Several labels for /// encodings that are dangerous to support -- ISO-2022-KR, ISO-2022-CN, /// HZ-GB-2312 -- map here, and its decoder emits a single U+FFFD for the /// whole input, whatever the input was. That closes an attack where a /// server and a client disagree about which encoding a label names. replacement, utf16be, utf16le, /// Every byte above 0x7F mapped into the private use area, so that bytes /// survive a round trip through something that insists on text. x_user_defined,
/// The standard's name for this encoding, in the standard's own casing. /// This is what to write into a `Content-Type` header or a `<meta>` /// element; ASCII-lowercasing it always yields one of its labels. pub fn name(self: Encoding) []const u8 { return switch (self) { .utf8 => "UTF-8", .ibm866 => "IBM866", .iso_8859_2 => "ISO-8859-2", .iso_8859_3 => "ISO-8859-3", .iso_8859_4 => "ISO-8859-4", .iso_8859_5 => "ISO-8859-5", .iso_8859_6 => "ISO-8859-6", .iso_8859_7 => "ISO-8859-7", .iso_8859_8 => "ISO-8859-8", .iso_8859_8_i => "ISO-8859-8-I", .iso_8859_10 => "ISO-8859-10", .iso_8859_13 => "ISO-8859-13", .iso_8859_14 => "ISO-8859-14", .iso_8859_15 => "ISO-8859-15", .iso_8859_16 => "ISO-8859-16", .koi8_r => "KOI8-R", .koi8_u => "KOI8-U", .macintosh => "macintosh", .windows_874 => "windows-874", .windows_1250 => "windows-1250", .windows_1251 => "windows-1251", .windows_1252 => "windows-1252", .windows_1253 => "windows-1253", .windows_1254 => "windows-1254", .windows_1255 => "windows-1255", .windows_1256 => "windows-1256", .windows_1257 => "windows-1257", .windows_1258 => "windows-1258", .x_mac_cyrillic => "x-mac-cyrillic", .gbk => "GBK", .gb18030 => "gb18030", .big5 => "Big5", .euc_jp => "EUC-JP", .iso_2022_jp => "ISO-2022-JP", .shift_jis => "Shift_JIS", .euc_kr => "EUC-KR", .replacement => "replacement", .utf16be => "UTF-16BE", .utf16le => "UTF-16LE", .x_user_defined => "x-user-defined", }; }
/// The standard's "get an encoding from a string": leading and trailing /// ASCII whitespace is removed, the rest is matched ASCII /// case-insensitively against the label table, and anything else is a /// failure. /// /// This is deliberately narrower than UTS #22 prescribes. A label with a /// stray character in it -- `utf8!`, `windows 1252` -- is not an /// encoding, and guessing is how a document ends up decoded one way by /// the server and another by the browser. pub fn fromLabel(label: []const u8) ?Encoding { // ASCII whitespace, which for this purpose is TAB, LF, FF, CR and // SPACE -- and not, for instance, a vertical tab. const trimmed = std.mem.trim(u8, label, "\t\n\x0c\r ");
// Every label in the standard is shorter than this; anything longer // cannot be one, and saying so keeps the buffer off the heap. var buffer: [64]u8 = undefined; if (trimmed.len > buffer.len) return null; const lowered = buffer[0..trimmed.len]; for (trimmed, lowered) |c, *out| out.* = std.ascii.toLower(c);
const index = std.sort.binarySearch( labels.Entry, &labels.entries, lowered, compareLabel, ) orelse return null; return labels.entries[index].encoding; }
fn compareLabel(needle: []const u8, entry: labels.Entry) std.math.Order { return std.mem.order(u8, needle, entry.label); }
/// The standard's "get an output encoding": what to actually encode with /// when a document asks for this one. /// /// `replacement`, `UTF-16BE` and `UTF-16LE` have no encoder -- the first /// because it exists to refuse, the other two because emitting UTF-16 /// from a form submission is a security problem rather than a feature -- /// so all three answer UTF-8. URL parsing and HTML form submission both /// need exactly this. pub fn outputEncoding(self: Encoding) Encoding { return switch (self) { .replacement, .utf16be, .utf16le => .utf8, else => self, }; }
/// Whether this encoding has an encoder at all. The three that do not are /// the three `outputEncoding` redirects. pub fn canEncode(self: Encoding) bool { return self.outputEncoding() == self; }};
test "fromLabel matches the standard's own examples" { try std.testing.expectEqual(Encoding.utf8, Encoding.fromLabel("utf-8").?); try std.testing.expectEqual(Encoding.utf8, Encoding.fromLabel("UTF-8").?); try std.testing.expectEqual(Encoding.shift_jis, Encoding.fromLabel("Shift_JIS").?); try std.testing.expectEqual(Encoding.shift_jis, Encoding.fromLabel("sjis").?); try std.testing.expectEqual(Encoding.windows_1252, Encoding.fromLabel("windows-1252").?); try std.testing.expectEqual(Encoding.big5, Encoding.fromLabel("big5-hkscs").?);}
test "fromLabel trims ASCII whitespace and only ASCII whitespace" { try std.testing.expectEqual(Encoding.utf8, Encoding.fromLabel(" \t\r\nutf-8\x0c ").?); try std.testing.expect(Encoding.fromLabel("\x0butf-8") == null); try std.testing.expect(Encoding.fromLabel("utf-8\x00") == null);}
test "fromLabel refuses what is not a label" { try std.testing.expect(Encoding.fromLabel("") == null); try std.testing.expect(Encoding.fromLabel("utf8!") == null); try std.testing.expect(Encoding.fromLabel("windows 1252") == null); try std.testing.expect(Encoding.fromLabel("utf-9") == null); try std.testing.expect(Encoding.fromLabel(&@as([100]u8, @splat('x'))) == null);}
test "ISO-8859-1 and friends are windows-1252, because deployed content says so" { for ([_][]const u8{ "iso-8859-1", "latin1", "ascii", "us-ascii", "cp819" }) |label| { try std.testing.expectEqual(Encoding.windows_1252, Encoding.fromLabel(label).?); }}
test "the labels that name an attack map to replacement" { for ([_][]const u8{ "iso-2022-kr", "iso-2022-cn", "hz-gb-2312", "csiso2022kr" }) |label| { try std.testing.expectEqual(Encoding.replacement, Encoding.fromLabel(label).?); }}
test "lowercasing a name gives a label for the same encoding" { var buffer: [64]u8 = undefined; for (std.enums.values(Encoding)) |encoding| { const lowered = std.ascii.lowerString(&buffer, encoding.name()); try std.testing.expectEqual(encoding, Encoding.fromLabel(lowered).?); }}
test "the label table is sorted, which is what makes the search work" { for (labels.entries[1..], labels.entries[0 .. labels.entries.len - 1]) |after, before| { try std.testing.expect(std.mem.lessThan(u8, before.label, after.label)); }}
test "outputEncoding sends the three encoderless encodings to UTF-8" { try std.testing.expectEqual(Encoding.utf8, Encoding.replacement.outputEncoding()); try std.testing.expectEqual(Encoding.utf8, Encoding.utf16le.outputEncoding()); try std.testing.expectEqual(Encoding.utf8, Encoding.utf16be.outputEncoding()); try std.testing.expectEqual(Encoding.shift_jis, Encoding.shift_jis.outputEncoding()); try std.testing.expect(!Encoding.replacement.canEncode()); try std.testing.expect(Encoding.big5.canEncode());}