Something went wrong. Try again.
Converting between UTF-8 and legacy character encodings, as the WHATWG Encoding Standard and glibc's iconv define them.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! One-shot conversion, for text that is already in memory in one piece.//!//! Each of these is `Decoder` or `Encoder` with the loop written out and the//! result collected into an allocation. When the text arrives in chunks --//! off a socket, out of a file -- drive those directly instead, or a//! multi-byte character landing on a chunk boundary turns into a replacement//! character.
const std = @import("std");
const bom = @import("bom.zig");const glibc_unicode = @import("codecs/glibc_unicode.zig");
const Allocator = std.mem.Allocator;const CodePage = @import("ebcdic.zig").CodePage;const Decoder = @import("Decoder.zig");const Encoder = @import("Encoder.zig");const Encoding = @import("encoding.zig").Encoding;const Glibc = @import("glibc.zig").Glibc;const Korean = @import("korean.zig").Korean;const Writer = std.Io.Writer;
pub const DecodeError = Allocator.Error || error{ /// Only with `error_mode` set to `fatal`. InvalidByteSequence,};
pub const EncodeError = Allocator.Error || Encoder.NoEncoderError || EncodeEbcdicError;
/// What encoding to EBCDIC can fail with. `error.NoEncoder` is not in it:/// every EBCDIC code page has an encoder, and only three of the standard's/// encodings do not.pub const EncodeEbcdicError = Allocator.Error || error{ /// Only with `error_mode` set to `fatal`. UnmappableCodePoint,};
/// Decode `bytes` as `encoding`, returning UTF-8. Caller owns the result.////// A byte order mark is not looked for and not removed: this decodes what it/// is told to. `decodeBomAlloc` is the one that sniffs.pub fn decodeAlloc( gpa: Allocator, encoding: Encoding, bytes: []const u8, options: Decoder.Options,) DecodeError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit();
var decoder: Decoder = .init(encoding, options); try lift(decoder.decode(bytes, &allocating.writer)); try lift(decoder.finish(&allocating.writer));
return allocating.toOwnedSlice();}
/// The standard's "decode" hook: a leading byte order mark chooses the/// encoding if there is one, and `fallback` is used if there is not. The mark/// itself is not part of the result.pub fn decodeBomAlloc( gpa: Allocator, fallback: Encoding, bytes: []const u8, options: Decoder.Options,) DecodeError![]u8 { const encoding, const rest = bom.strip(bytes, fallback); return decodeAlloc(gpa, encoding, rest, options);}
/// Encode the UTF-8 in `utf8` as `encoding`. Caller owns the result.////// `error.NoEncoder` comes back for `replacement`, `UTF-16BE` and/// `UTF-16LE`, none of which the standard gives an encoder;/// `Encoding.outputEncoding` is how to avoid asking.pub fn encodeAlloc( gpa: Allocator, encoding: Encoding, utf8: []const u8, options: Encoder.Options,) EncodeError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit();
var encoder: Encoder = try .init(encoding, options); try lift(encoder.encode(utf8, &allocating.writer)); try lift(encoder.finish(&allocating.writer));
return allocating.toOwnedSlice();}
/// Decode `bytes` as an EBCDIC code page, returning UTF-8. Caller owns the/// result.////// EBCDIC is not in the Encoding Standard, so it has a function of its own/// rather than a member of `Encoding`; see `ebcdic.zig` for why that matters.pub fn decodeEbcdicAlloc( gpa: Allocator, code_page: CodePage, bytes: []const u8, options: Decoder.Options,) DecodeError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit();
var decoder: Decoder = .initEbcdic(code_page, options); try lift(decoder.decode(bytes, &allocating.writer)); try lift(decoder.finish(&allocating.writer));
return allocating.toOwnedSlice();}
/// Encode the UTF-8 in `utf8` as an EBCDIC code page. Caller owns the result.////// Every code page has an encoder, so unlike `encodeAlloc` this cannot return/// `error.NoEncoder`.pub fn encodeEbcdicAlloc( gpa: Allocator, code_page: CodePage, utf8: []const u8, options: Encoder.Options,) EncodeEbcdicError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit();
var encoder: Encoder = .initEbcdic(code_page, options); try lift(encoder.encode(utf8, &allocating.writer)); try lift(encoder.finish(&allocating.writer));
return allocating.toOwnedSlice();}
/// What encoding to one of the Korean encodings can fail with, which is the/// same as for EBCDIC: both have an encoder, so `error.NoEncoder` is not in/// it.pub const EncodeKoreanError = EncodeEbcdicError;
/// Decode `bytes` as one of the Korean encodings outside the standard,/// returning UTF-8. Caller owns the result.////// For the EUC-KR the web sends, which is code page 949, use `decodeAlloc`/// with `.euc_kr`; see `korean.zig` for how the three differ.pub fn decodeKoreanAlloc( gpa: Allocator, korean: Korean, bytes: []const u8, options: Decoder.Options,) DecodeError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit();
var decoder: Decoder = .initKorean(korean, options); try lift(decoder.decode(bytes, &allocating.writer)); try lift(decoder.finish(&allocating.writer));
return allocating.toOwnedSlice();}
/// Encode the UTF-8 in `utf8` as one of the Korean encodings outside the/// standard. Caller owns the result.////// Both can write every Hangul syllable: Johab has a code for each of the/// 11,172, and Wansung spells the 8,822 it has no code for in eight bytes.pub fn encodeKoreanAlloc( gpa: Allocator, korean: Korean, utf8: []const u8, options: Encoder.Options,) EncodeKoreanError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit();
var encoder: Encoder = .initKorean(korean, options); try lift(encoder.encode(utf8, &allocating.writer)); try lift(encoder.finish(&allocating.writer));
return allocating.toOwnedSlice();}
/// What encoding to one of glibc's charsets can fail with, which is the same/// as for EBCDIC: every one of them has an encoder.pub const EncodeGlibcError = EncodeEbcdicError;
/// What converting between two of glibc's charsets can fail with.pub const ConvertGlibcError = Allocator.Error || error{ /// The input is not valid in the charset it is said to be in, or ends /// part way through a character. glibc fails on exactly these inputs too. InvalidByteSequence, /// The target charset cannot represent a character of the input. glibc /// fails on exactly these too. UnmappableCodePoint, /// The input is text that glibc reads and this library cannot: UTF-8 /// with a value above U+10FFFF, written the way the original UTF-8 of /// RFC 2279 allows, or UCS-4 with a value above U+10FFFF or a surrogate. /// glibc converts values above U+10FFFF to UTF-8 or UCS-4, and into /// anything else it fails, as this does. A UCS-4 surrogate glibc converts /// only to UCS-4: into anything else, UTF-8 included, it is /// `InvalidByteSequence` here as it is an error in glibc. OutsideUnicode,};
/// Decode `bytes` as a glibc charset, returning UTF-8. Caller owns the/// result.////// With `.error_mode = .fatal` this fails exactly where glibc would. See/// `convertGlibcAlloc` for converting the way glibc's `iconv` does.pub fn decodeGlibcAlloc( gpa: Allocator, charset: Glibc, bytes: []const u8, options: Decoder.Options,) DecodeError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit(); var decoder: Decoder = .initGlibc(charset, options); try lift(decoder.decode(bytes, &allocating.writer)); try lift(decoder.finish(&allocating.writer)); return allocating.toOwnedSlice();}
/// Encode the UTF-8 in `utf8` as a glibc charset. Caller owns the result.////// Invalid UTF-8 in the input becomes U+FFFD, as it does for every encoder/// here, where glibc would fail: use `convertGlibcAlloc` from `.utf_8` for/// that.pub fn encodeGlibcAlloc( gpa: Allocator, charset: Glibc, utf8: []const u8, options: Encoder.Options,) EncodeGlibcError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit(); var encoder: Encoder = .initGlibc(charset, options); try lift(encoder.encode(utf8, &allocating.writer)); try lift(encoder.finish(&allocating.writer)); return allocating.toOwnedSlice();}
/// Convert `bytes` from one glibc charset to another, as glibc 2.44's/// `iconv` would. Caller owns the result.////// The whole input is converted or none of it is: either this returns the/// bytes glibc would have written, or it fails where glibc would have/// failed. Running out of memory is a failure of its own, never mistaken for/// either. The conversion goes through UTF-8, as glibc's mostly goes through/// UCS-4, and gives the same answer.////// It can disagree with glibc in two places. One is `error.OutsideUnicode`,/// which only comes back from `.utf_8`, `.iso_10646_ucs4` and `.ucs_4le`:/// there, glibc might have completed the conversion. The other is three/// pairs of charsets that glibc converts directly rather than through UCS-4,/// and for which it gives answers of its own:////// - `.euc_cn` to `.big5` turns simplified characters into traditional ones./// - `.euc_cn` to `.gbk` fails when the text ends in a two-byte character./// - `.gbk` to `.euc_cn` copies 0xA1A4 and 0xA1AA through unchanged, and/// fails on 0xA844.////// This function converts those pairs through Unicode like any other, so/// a caller that needs glibc's answer for non-ASCII text should not use it/// for them.////// A byte order mark is handled as glibc handles it: `UTF-16`, `UTF-32` and/// `UNICODE` take their byte order from a leading one and write one ahead of/// a non-empty output, and every other charset treats U+FEFF as a/// character.pub fn convertGlibcAlloc( gpa: Allocator, from: Glibc, to: Glibc, bytes: []const u8,) ConvertGlibcError![]u8 { const utf8 = decodeGlibcAlloc(gpa, from, bytes, .{ .error_mode = .fatal }) catch |err| switch (err) { error.InvalidByteSequence => { if (from == .utf_8 and isOriginalUtf8(bytes)) return error.OutsideUnicode; if (glibc_unicode.variant(from)) |v| { if (v.form == .ucs4 and glibc_unicode.isGlibcUcs4(bytes, v.order)) { // glibc writes a surrogate only as UCS-4, and fails // converting it to anything else. const to_ucs4 = if (glibc_unicode.variant(to)) |t| t.form == .ucs4 else false; if (!to_ucs4 and glibc_unicode.hasUcs4Surrogate(bytes, v.order)) return error.InvalidByteSequence; return error.OutsideUnicode; } } return error.InvalidByteSequence; }, else => |other| return other, }; defer gpa.free(utf8); return encodeGlibcAlloc(gpa, to, utf8, .{ .error_mode = .fatal });}
/// Whether `bytes` is entirely valid in the original UTF-8 of RFC 2279,/// which glibc's decoder still reads: sequences of up to six bytes, values/// up to 0x7FFFFFFF, each in its shortest form, and no surrogates. Anything/// the strict decoder refused and this accepts is a value above U+10FFFF.fn isOriginalUtf8(bytes: []const u8) bool { var i: usize = 0; while (i < bytes.len) { const lead = bytes[i]; const len: usize, const lowest: u32 = switch (lead) { 0x00...0x7f => { i += 1; continue; }, 0xc0...0xdf => .{ 2, 0x80 }, 0xe0...0xef => .{ 3, 0x800 }, 0xf0...0xf7 => .{ 4, 0x10000 }, 0xf8...0xfb => .{ 5, 0x200000 }, 0xfc...0xfd => .{ 6, 0x4000000 }, else => return false, }; if (bytes.len - i < len) return false; var value: u32 = lead & (@as(u8, 0x7f) >> @intCast(len)); for (bytes[i + 1 .. i + len]) |continuation| { if (continuation & 0xc0 != 0x80) return false; value = value << 6 | (continuation & 0x3f); } if (value < lowest) return false; if (value >= 0xd800 and value <= 0xdfff) return false; i += len; } return true;}
test "the original UTF-8 is what glibc reads" { try std.testing.expect(isOriginalUtf8("caf\xc3\xa9")); try std.testing.expect(isOriginalUtf8("\xf4\x90\x80\x80")); try std.testing.expect(isOriginalUtf8("\xf7\xbf\xbf\xbf")); try std.testing.expect(isOriginalUtf8("\xf8\x88\x80\x80\x80")); try std.testing.expect(isOriginalUtf8("\xfd\xbf\xbf\xbf\xbf\xbf")); try std.testing.expect(!isOriginalUtf8("\xfe")); try std.testing.expect(!isOriginalUtf8("\xc0\x80")); try std.testing.expect(!isOriginalUtf8("\xf8\x87\xbf\xbf\xbf")); try std.testing.expect(!isOriginalUtf8("\xed\xa0\x80")); try std.testing.expect(!isOriginalUtf8("\xf4\x90\x80"));}
test "convertGlibcAlloc tells the two UTF-8 failures apart" { const gpa = std.testing.allocator; try std.testing.expectError(error.OutsideUnicode, convertGlibcAlloc(gpa, .utf_8, .utf_8, "a\xf4\x90\x80\x80")); try std.testing.expectError(error.OutsideUnicode, convertGlibcAlloc(gpa, .utf_8, .iso_8859_1, "\xf4\x90\x80\x80")); try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .utf_8, .utf_8, "\xf4\x90\x80\x80\xff")); try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .utf_8, .utf_8, "\xed\xa0\x80"));}
test "convertGlibcAlloc converts between two legacy charsets" { const gpa = std.testing.allocator; // The pound sign is in code page 932 only as its fullwidth form, which // glibc writes for it. const out = try convertGlibcAlloc(gpa, .iso_8859_1, .cp932, "caf \xa3"); defer gpa.free(out); try std.testing.expectEqualSlices(u8, "caf \x81\x92", out); // é is not in code page 932 at all. try std.testing.expectError(error.UnmappableCodePoint, convertGlibcAlloc(gpa, .iso_8859_1, .cp932, "caf\xe9")); // 0x81 is not a character in glibc's CP1252. try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .cp1252, .utf_8, "\x81"));}
/// An `Allocating` writer fails for exactly one reason, so `WriteFailed` from/// one means the allocator said no.fn lift(result: anytype) !void { return result catch |err| switch (err) { error.WriteFailed => error.OutOfMemory, else => |other| other, };}
test "convertGlibcAlloc handles byte order marks as glibc does" { const gpa = std.testing.allocator; const native_le = @import("builtin").cpu.arch.endian() == .little;
const decoded = try convertGlibcAlloc(gpa, .utf_16, .utf_8, "\xfe\xff\x00A\xfe\xff"); defer gpa.free(decoded); try std.testing.expectEqualSlices(u8, "A\u{feff}", decoded);
const kept = try convertGlibcAlloc(gpa, .utf_16le, .utf_8, "\xff\xfeA\x00"); defer gpa.free(kept); try std.testing.expectEqualSlices(u8, "\u{feff}A", kept);
const encoded = try convertGlibcAlloc(gpa, .utf_8, .utf_16, "A"); defer gpa.free(encoded); try std.testing.expectEqualSlices(u8, if (native_le) "\xff\xfeA\x00" else "\xfe\xff\x00A", encoded);
const empty = try convertGlibcAlloc(gpa, .utf_8, .utf_32, ""); defer gpa.free(empty); try std.testing.expectEqual(0, empty.len);}
test "convertGlibcAlloc tells UCS-4 outside Unicode from bad UCS-4" { const gpa = std.testing.allocator; try std.testing.expectError(error.OutsideUnicode, convertGlibcAlloc(gpa, .iso_10646_ucs4, .utf_8, "\x00\x11\x00\x00")); try std.testing.expectError(error.OutsideUnicode, convertGlibcAlloc(gpa, .ucs_4le, .iso_10646_ucs4, "\x00\xd8\x00\x00")); // glibc reads a surrogate, but will not write one as anything but UCS-4. try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .ucs_4le, .utf_8, "\x00\xd8\x00\x00")); try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .iso_10646_ucs4, .iso_8859_1, "\x00\x11\x00\x00\x00\x00\xdc\x00")); try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .iso_10646_ucs4, .utf_8, "\x80\x00\x00\x00")); try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .utf_32be, .utf_8, "\x00\x11\x00\x00")); try std.testing.expectError(error.UnmappableCodePoint, convertGlibcAlloc(gpa, .utf_8, .iso_10646_ucs2, "\u{1f600}"));}