// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-License-Identifier: MIT //! One-shot conversion, for text that is already in memory in one piece. //! //! Each of these is `Decoder` or `Encoder` with the loop written out and the //! result collected into an allocation. When the text arrives in chunks -- //! off a socket, out of a file -- drive those directly instead, or a //! multi-byte character landing on a chunk boundary turns into a replacement //! character. const std = @import("std"); const bom = @import("bom.zig"); const glibc_unicode = @import("codecs/glibc_unicode.zig"); const Allocator = std.mem.Allocator; const CodePage = @import("ebcdic.zig").CodePage; const Decoder = @import("Decoder.zig"); const Encoder = @import("Encoder.zig"); const Encoding = @import("encoding.zig").Encoding; const Glibc = @import("glibc.zig").Glibc; const Korean = @import("korean.zig").Korean; const Writer = std.Io.Writer; pub const DecodeError = Allocator.Error || error{ /// Only with `error_mode` set to `fatal`. InvalidByteSequence, }; pub const EncodeError = Allocator.Error || Encoder.NoEncoderError || EncodeEbcdicError; /// What encoding to EBCDIC can fail with. `error.NoEncoder` is not in it: /// every EBCDIC code page has an encoder, and only three of the standard's /// encodings do not. pub const EncodeEbcdicError = Allocator.Error || error{ /// Only with `error_mode` set to `fatal`. UnmappableCodePoint, }; /// Decode `bytes` as `encoding`, returning UTF-8. Caller owns the result. /// /// A byte order mark is not looked for and not removed: this decodes what it /// is told to. `decodeBomAlloc` is the one that sniffs. pub fn decodeAlloc( gpa: Allocator, encoding: Encoding, bytes: []const u8, options: Decoder.Options, ) DecodeError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit(); var decoder: Decoder = .init(encoding, options); try lift(decoder.decode(bytes, &allocating.writer)); try lift(decoder.finish(&allocating.writer)); return allocating.toOwnedSlice(); } /// The standard's "decode" hook: a leading byte order mark chooses the /// encoding if there is one, and `fallback` is used if there is not. The mark /// itself is not part of the result. pub fn decodeBomAlloc( gpa: Allocator, fallback: Encoding, bytes: []const u8, options: Decoder.Options, ) DecodeError![]u8 { const encoding, const rest = bom.strip(bytes, fallback); return decodeAlloc(gpa, encoding, rest, options); } /// Encode the UTF-8 in `utf8` as `encoding`. Caller owns the result. /// /// `error.NoEncoder` comes back for `replacement`, `UTF-16BE` and /// `UTF-16LE`, none of which the standard gives an encoder; /// `Encoding.outputEncoding` is how to avoid asking. pub fn encodeAlloc( gpa: Allocator, encoding: Encoding, utf8: []const u8, options: Encoder.Options, ) EncodeError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit(); var encoder: Encoder = try .init(encoding, options); try lift(encoder.encode(utf8, &allocating.writer)); try lift(encoder.finish(&allocating.writer)); return allocating.toOwnedSlice(); } /// Decode `bytes` as an EBCDIC code page, returning UTF-8. Caller owns the /// result. /// /// EBCDIC is not in the Encoding Standard, so it has a function of its own /// rather than a member of `Encoding`; see `ebcdic.zig` for why that matters. pub fn decodeEbcdicAlloc( gpa: Allocator, code_page: CodePage, bytes: []const u8, options: Decoder.Options, ) DecodeError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit(); var decoder: Decoder = .initEbcdic(code_page, options); try lift(decoder.decode(bytes, &allocating.writer)); try lift(decoder.finish(&allocating.writer)); return allocating.toOwnedSlice(); } /// Encode the UTF-8 in `utf8` as an EBCDIC code page. Caller owns the result. /// /// Every code page has an encoder, so unlike `encodeAlloc` this cannot return /// `error.NoEncoder`. pub fn encodeEbcdicAlloc( gpa: Allocator, code_page: CodePage, utf8: []const u8, options: Encoder.Options, ) EncodeEbcdicError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit(); var encoder: Encoder = .initEbcdic(code_page, options); try lift(encoder.encode(utf8, &allocating.writer)); try lift(encoder.finish(&allocating.writer)); return allocating.toOwnedSlice(); } /// What encoding to one of the Korean encodings can fail with, which is the /// same as for EBCDIC: both have an encoder, so `error.NoEncoder` is not in /// it. pub const EncodeKoreanError = EncodeEbcdicError; /// Decode `bytes` as one of the Korean encodings outside the standard, /// returning UTF-8. Caller owns the result. /// /// For the EUC-KR the web sends, which is code page 949, use `decodeAlloc` /// with `.euc_kr`; see `korean.zig` for how the three differ. pub fn decodeKoreanAlloc( gpa: Allocator, korean: Korean, bytes: []const u8, options: Decoder.Options, ) DecodeError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit(); var decoder: Decoder = .initKorean(korean, options); try lift(decoder.decode(bytes, &allocating.writer)); try lift(decoder.finish(&allocating.writer)); return allocating.toOwnedSlice(); } /// Encode the UTF-8 in `utf8` as one of the Korean encodings outside the /// standard. Caller owns the result. /// /// Both can write every Hangul syllable: Johab has a code for each of the /// 11,172, and Wansung spells the 8,822 it has no code for in eight bytes. pub fn encodeKoreanAlloc( gpa: Allocator, korean: Korean, utf8: []const u8, options: Encoder.Options, ) EncodeKoreanError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit(); var encoder: Encoder = .initKorean(korean, options); try lift(encoder.encode(utf8, &allocating.writer)); try lift(encoder.finish(&allocating.writer)); return allocating.toOwnedSlice(); } /// What encoding to one of glibc's charsets can fail with, which is the same /// as for EBCDIC: every one of them has an encoder. pub const EncodeGlibcError = EncodeEbcdicError; /// What converting between two of glibc's charsets can fail with. pub const ConvertGlibcError = Allocator.Error || error{ /// The input is not valid in the charset it is said to be in, or ends /// part way through a character. glibc fails on exactly these inputs too. InvalidByteSequence, /// The target charset cannot represent a character of the input. glibc /// fails on exactly these too. UnmappableCodePoint, /// The input is text that glibc reads and this library cannot: UTF-8 /// with a value above U+10FFFF, written the way the original UTF-8 of /// RFC 2279 allows, or UCS-4 with a value above U+10FFFF or a surrogate. /// glibc converts values above U+10FFFF to UTF-8 or UCS-4, and into /// anything else it fails, as this does. A UCS-4 surrogate glibc converts /// only to UCS-4: into anything else, UTF-8 included, it is /// `InvalidByteSequence` here as it is an error in glibc. OutsideUnicode, }; /// Decode `bytes` as a glibc charset, returning UTF-8. Caller owns the /// result. /// /// With `.error_mode = .fatal` this fails exactly where glibc would. See /// `convertGlibcAlloc` for converting the way glibc's `iconv` does. pub fn decodeGlibcAlloc( gpa: Allocator, charset: Glibc, bytes: []const u8, options: Decoder.Options, ) DecodeError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit(); var decoder: Decoder = .initGlibc(charset, options); try lift(decoder.decode(bytes, &allocating.writer)); try lift(decoder.finish(&allocating.writer)); return allocating.toOwnedSlice(); } /// Encode the UTF-8 in `utf8` as a glibc charset. Caller owns the result. /// /// Invalid UTF-8 in the input becomes U+FFFD, as it does for every encoder /// here, where glibc would fail: use `convertGlibcAlloc` from `.utf_8` for /// that. pub fn encodeGlibcAlloc( gpa: Allocator, charset: Glibc, utf8: []const u8, options: Encoder.Options, ) EncodeGlibcError![]u8 { var allocating: Writer.Allocating = .init(gpa); errdefer allocating.deinit(); var encoder: Encoder = .initGlibc(charset, options); try lift(encoder.encode(utf8, &allocating.writer)); try lift(encoder.finish(&allocating.writer)); return allocating.toOwnedSlice(); } /// Convert `bytes` from one glibc charset to another, as glibc 2.44's /// `iconv` would. Caller owns the result. /// /// The whole input is converted or none of it is: either this returns the /// bytes glibc would have written, or it fails where glibc would have /// failed. Running out of memory is a failure of its own, never mistaken for /// either. The conversion goes through UTF-8, as glibc's mostly goes through /// UCS-4, and gives the same answer. /// /// It can disagree with glibc in two places. One is `error.OutsideUnicode`, /// which only comes back from `.utf_8`, `.iso_10646_ucs4` and `.ucs_4le`: /// there, glibc might have completed the conversion. The other is three /// pairs of charsets that glibc converts directly rather than through UCS-4, /// and for which it gives answers of its own: /// /// - `.euc_cn` to `.big5` turns simplified characters into traditional ones. /// - `.euc_cn` to `.gbk` fails when the text ends in a two-byte character. /// - `.gbk` to `.euc_cn` copies 0xA1A4 and 0xA1AA through unchanged, and /// fails on 0xA844. /// /// This function converts those pairs through Unicode like any other, so /// a caller that needs glibc's answer for non-ASCII text should not use it /// for them. /// /// A byte order mark is handled as glibc handles it: `UTF-16`, `UTF-32` and /// `UNICODE` take their byte order from a leading one and write one ahead of /// a non-empty output, and every other charset treats U+FEFF as a /// character. pub fn convertGlibcAlloc( gpa: Allocator, from: Glibc, to: Glibc, bytes: []const u8, ) ConvertGlibcError![]u8 { const utf8 = decodeGlibcAlloc(gpa, from, bytes, .{ .error_mode = .fatal }) catch |err| switch (err) { error.InvalidByteSequence => { if (from == .utf_8 and isOriginalUtf8(bytes)) return error.OutsideUnicode; if (glibc_unicode.variant(from)) |v| { if (v.form == .ucs4 and glibc_unicode.isGlibcUcs4(bytes, v.order)) { // glibc writes a surrogate only as UCS-4, and fails // converting it to anything else. const to_ucs4 = if (glibc_unicode.variant(to)) |t| t.form == .ucs4 else false; if (!to_ucs4 and glibc_unicode.hasUcs4Surrogate(bytes, v.order)) return error.InvalidByteSequence; return error.OutsideUnicode; } } return error.InvalidByteSequence; }, else => |other| return other, }; defer gpa.free(utf8); return encodeGlibcAlloc(gpa, to, utf8, .{ .error_mode = .fatal }); } /// Whether `bytes` is entirely valid in the original UTF-8 of RFC 2279, /// which glibc's decoder still reads: sequences of up to six bytes, values /// up to 0x7FFFFFFF, each in its shortest form, and no surrogates. Anything /// the strict decoder refused and this accepts is a value above U+10FFFF. fn isOriginalUtf8(bytes: []const u8) bool { var i: usize = 0; while (i < bytes.len) { const lead = bytes[i]; const len: usize, const lowest: u32 = switch (lead) { 0x00...0x7f => { i += 1; continue; }, 0xc0...0xdf => .{ 2, 0x80 }, 0xe0...0xef => .{ 3, 0x800 }, 0xf0...0xf7 => .{ 4, 0x10000 }, 0xf8...0xfb => .{ 5, 0x200000 }, 0xfc...0xfd => .{ 6, 0x4000000 }, else => return false, }; if (bytes.len - i < len) return false; var value: u32 = lead & (@as(u8, 0x7f) >> @intCast(len)); for (bytes[i + 1 .. i + len]) |continuation| { if (continuation & 0xc0 != 0x80) return false; value = value << 6 | (continuation & 0x3f); } if (value < lowest) return false; if (value >= 0xd800 and value <= 0xdfff) return false; i += len; } return true; } test "the original UTF-8 is what glibc reads" { try std.testing.expect(isOriginalUtf8("caf\xc3\xa9")); try std.testing.expect(isOriginalUtf8("\xf4\x90\x80\x80")); try std.testing.expect(isOriginalUtf8("\xf7\xbf\xbf\xbf")); try std.testing.expect(isOriginalUtf8("\xf8\x88\x80\x80\x80")); try std.testing.expect(isOriginalUtf8("\xfd\xbf\xbf\xbf\xbf\xbf")); try std.testing.expect(!isOriginalUtf8("\xfe")); try std.testing.expect(!isOriginalUtf8("\xc0\x80")); try std.testing.expect(!isOriginalUtf8("\xf8\x87\xbf\xbf\xbf")); try std.testing.expect(!isOriginalUtf8("\xed\xa0\x80")); try std.testing.expect(!isOriginalUtf8("\xf4\x90\x80")); } test "convertGlibcAlloc tells the two UTF-8 failures apart" { const gpa = std.testing.allocator; try std.testing.expectError(error.OutsideUnicode, convertGlibcAlloc(gpa, .utf_8, .utf_8, "a\xf4\x90\x80\x80")); try std.testing.expectError(error.OutsideUnicode, convertGlibcAlloc(gpa, .utf_8, .iso_8859_1, "\xf4\x90\x80\x80")); try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .utf_8, .utf_8, "\xf4\x90\x80\x80\xff")); try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .utf_8, .utf_8, "\xed\xa0\x80")); } test "convertGlibcAlloc converts between two legacy charsets" { const gpa = std.testing.allocator; // The pound sign is in code page 932 only as its fullwidth form, which // glibc writes for it. const out = try convertGlibcAlloc(gpa, .iso_8859_1, .cp932, "caf \xa3"); defer gpa.free(out); try std.testing.expectEqualSlices(u8, "caf \x81\x92", out); // é is not in code page 932 at all. try std.testing.expectError(error.UnmappableCodePoint, convertGlibcAlloc(gpa, .iso_8859_1, .cp932, "caf\xe9")); // 0x81 is not a character in glibc's CP1252. try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .cp1252, .utf_8, "\x81")); } /// An `Allocating` writer fails for exactly one reason, so `WriteFailed` from /// one means the allocator said no. fn lift(result: anytype) !void { return result catch |err| switch (err) { error.WriteFailed => error.OutOfMemory, else => |other| other, }; } test "convertGlibcAlloc handles byte order marks as glibc does" { const gpa = std.testing.allocator; const native_le = @import("builtin").cpu.arch.endian() == .little; const decoded = try convertGlibcAlloc(gpa, .utf_16, .utf_8, "\xfe\xff\x00A\xfe\xff"); defer gpa.free(decoded); try std.testing.expectEqualSlices(u8, "A\u{feff}", decoded); const kept = try convertGlibcAlloc(gpa, .utf_16le, .utf_8, "\xff\xfeA\x00"); defer gpa.free(kept); try std.testing.expectEqualSlices(u8, "\u{feff}A", kept); const encoded = try convertGlibcAlloc(gpa, .utf_8, .utf_16, "A"); defer gpa.free(encoded); try std.testing.expectEqualSlices(u8, if (native_le) "\xff\xfeA\x00" else "\xfe\xff\x00A", encoded); const empty = try convertGlibcAlloc(gpa, .utf_8, .utf_32, ""); defer gpa.free(empty); try std.testing.expectEqual(0, empty.len); } test "convertGlibcAlloc tells UCS-4 outside Unicode from bad UCS-4" { const gpa = std.testing.allocator; try std.testing.expectError(error.OutsideUnicode, convertGlibcAlloc(gpa, .iso_10646_ucs4, .utf_8, "\x00\x11\x00\x00")); try std.testing.expectError(error.OutsideUnicode, convertGlibcAlloc(gpa, .ucs_4le, .iso_10646_ucs4, "\x00\xd8\x00\x00")); // glibc reads a surrogate, but will not write one as anything but UCS-4. try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .ucs_4le, .utf_8, "\x00\xd8\x00\x00")); try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .iso_10646_ucs4, .iso_8859_1, "\x00\x11\x00\x00\x00\x00\xdc\x00")); try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .iso_10646_ucs4, .utf_8, "\x80\x00\x00\x00")); try std.testing.expectError(error.InvalidByteSequence, convertGlibcAlloc(gpa, .utf_32be, .utf_8, "\x00\x11\x00\x00")); try std.testing.expectError(error.UnmappableCodePoint, convertGlibcAlloc(gpa, .utf_8, .iso_10646_ucs2, "\u{1f600}")); }