// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-License-Identifier: MIT //! UTF-8 in, bytes in some other encoding out. //! //! Three encodings have no encoder: `replacement` exists to refuse, and //! neither UTF-16 variant gets one because emitting UTF-16 where a consumer //! expects bytes is a way of smuggling markup past a filter. `init` says so //! rather than guessing; `initOutput` applies `Encoding.outputEncoding` first //! and so cannot fail, which is what URL parsing and HTML form submission //! both want. //! //! The input is UTF-8. Anything in it that is not -- a truncated sequence, an //! overlong one, a lone surrogate written as three bytes -- becomes U+FFFD //! before the encoder proper sees it, which is the same thing that happens to //! a lone surrogate in a JavaScript string on its way to a byte sequence. //! ISO-2022-JP aside, a sequence split across two calls is handled. //! //! ``` //! var encoder = try charset.Encoder.init(.shift_jis, .{}); //! try encoder.encode("日本語", writer); //! try encoder.finish(writer); //! ``` const std = @import("std"); const codec = @import("codec.zig"); const codecs = @import("codecs.zig"); const ByteQueue = codec.ByteQueue; const Encoded = codec.Encoded; const Encoding = @import("encoding.zig").Encoding; const Writer = std.Io.Writer; const Encoder = @This(); /// What an encoder writes: an `Encoding` from the standard, or an EBCDIC code /// page or Korean encoding, which are not in it. See `ebcdic.zig` for why they /// are kept apart. pub const Charset = Decoder.Charset; const CodePage = @import("ebcdic.zig").CodePage; const Glibc = @import("glibc.zig").Glibc; const Korean = @import("korean.zig").Korean; const Decoder = @import("Decoder.zig"); /// What to do with a scalar value this encoding cannot represent. pub const ErrorMode = enum { /// Stop, with `error.UnmappableCodePoint`, and leave the offending scalar /// value in `unmappable`. fatal, /// Emit a decimal HTML character reference -- `国` -- and carry on. /// /// This exists because HTML form submission requires an encoder that does /// not stop, and it is worth knowing what it costs: the reference cannot /// be told apart from the same characters typed literally, so a form /// submitted this way loses information silently. Use UTF-8 and the /// question does not arise. html, }; pub const Options = struct { error_mode: ErrorMode = .fatal, }; pub const Error = error{ /// This encoding cannot represent a scalar value in the input, and /// `error_mode` is `fatal`. Which scalar value is in `unmappable`. UnmappableCodePoint, } || Writer.Error; /// `init` was given one of the three encodings with no encoder. pub const NoEncoderError = error{NoEncoder}; charset: Charset, error_mode: ErrorMode, /// The scalar value that `error.UnmappableCodePoint` was about. Meaningless /// until that error has been returned. unmappable: u21 = 0, /// Whether to write nothing for the tag characters, U+E0000 to U+E007F. Only /// glibc's charsets do, because glibc does. drops_tag_characters: bool = false, /// The input side: UTF-8 bytes on their way to becoming scalar values. input: ByteQueue = .{}, input_decoder: codecs.utf8.Decoder = .{}, state: State, const State = union(enum) { utf8: codecs.utf8.Encoder, single_byte: codecs.single_byte.Encoder, gb18030: codecs.gb18030.Encoder, big5: codecs.big5.Encoder, euc_jp: codecs.euc_jp.Encoder, iso_2022_jp: codecs.iso_2022_jp.Encoder, shift_jis: codecs.shift_jis.Encoder, euc_kr: codecs.euc_kr.Encoder, x_user_defined: codecs.x_user_defined.Encoder, ebcdic: codecs.ebcdic.Encoder, wansung: codecs.wansung.Encoder, johab: codecs.johab.Encoder, glibc_big5: codecs.glibc_big5.Encoder, glibc_combining: codecs.glibc_combining.Encoder, glibc_euc_jp: codecs.glibc_euc_jp.Encoder, glibc_sjis: codecs.glibc_sjis.Encoder, glibc_euc_kr: codecs.glibc_euc_kr.Encoder, glibc_gbk: codecs.glibc_gbk.Encoder, glibc_iso_2022_jp: codecs.glibc_iso_2022_jp.Encoder, glibc_unicode: codecs.glibc_unicode.Encoder, }; pub fn init(encoding: Encoding, options: Options) NoEncoderError!Encoder { return .{ .charset = .{ .standard = encoding }, .error_mode = options.error_mode, .state = switch (encoding) { .replacement, .utf16be, .utf16le => return error.NoEncoder, .utf8 => .{ .utf8 = .{} }, .gbk => .{ .gb18030 = .{ .is_gbk = true } }, .gb18030 => .{ .gb18030 = .{} }, .big5 => .{ .big5 = .{} }, .euc_jp => .{ .euc_jp = .{} }, .iso_2022_jp => .{ .iso_2022_jp = .{} }, .shift_jis => .{ .shift_jis = .{} }, .euc_kr => .{ .euc_kr = .{} }, .x_user_defined => .{ .x_user_defined = .{} }, else => .{ .single_byte = .{ .index = codecs.single_byte.reversedIndex(encoding).?, } }, }, }; } /// `init` on `encoding.outputEncoding()`, which is never one of the three /// without an encoder and so cannot fail. pub fn initOutput(encoding: Encoding, options: Options) Encoder { return init(encoding.outputEncoding(), options) catch unreachable; } /// An encoder for an EBCDIC code page. Every one of them has an encoder, so /// unlike `init` this cannot fail. pub fn initEbcdic(code_page: CodePage, options: Options) Encoder { return .{ .charset = .{ .ebcdic = code_page }, .error_mode = options.error_mode, .state = .{ .ebcdic = .{ .index = codecs.ebcdic.reversedIndex(code_page) } }, }; } /// An encoder for one of the Korean encodings that are not in the standard. /// Both have one, so this cannot fail. pub fn initKorean(korean: Korean, options: Options) Encoder { return .{ .charset = .{ .korean = korean }, .error_mode = options.error_mode, .state = switch (korean) { .wansung => .{ .wansung = .{} }, .johab => .{ .johab = .{} }, }, }; } /// An encoder for a charset as glibc 2.44's iconv defines it. See /// `glibc.zig`. Every one of them has an encoder, so this cannot fail. /// /// With `error_mode` set to `fatal`, which is what glibc does, this fails on /// exactly the scalar values glibc's would and otherwise writes the same /// bytes -- including writing nothing at all for the tag characters U+E0000 /// to U+E007F where glibc does (see `Glibc.dropsTagCharacters`), and the /// byte order mark that glibc's `UTF-16`, `UTF-32` and `UNICODE` write /// before the first character. `html` is this library's, not glibc's. Invalid UTF-8 in the input /// becomes U+FFFD before the encoder sees it, as it does for every encoder /// here; `convertGlibcAlloc` is the one that fails on it, as glibc does. pub fn initGlibc(charset: Glibc, options: Options) Encoder { const glibc_single_byte = codecs.glibc_single_byte; const glibc_unicode = codecs.glibc_unicode; const unicode = glibc_unicode.variant(charset); return .{ .charset = .{ .glibc = charset }, .error_mode = options.error_mode, // UCS-2 drops the tag characters too, but in its own encoder, // because glibc still writes the byte order mark of `UNICODE` ahead // of one that is dropped. .drops_tag_characters = charset.dropsTagCharacters() and unicode == null, .state = if (unicode) |v| .{ .glibc_unicode = .init(v) } else switch (charset) { .utf_8 => .{ .utf8 = .{} }, .uhc => .{ .euc_kr = .{} }, .euc_kr => .{ .glibc_euc_kr = .{} }, .johab => .{ .johab = .{ .glibc = true } }, .cp932 => .{ .shift_jis = .{ .glibc = true } }, .iso_2022_jp => .{ .glibc_iso_2022_jp = .{} }, .euc_jp => .{ .glibc_euc_jp = .{} }, .sjis => .{ .glibc_sjis = .{} }, .gbk => .{ .glibc_gbk = .{} }, .euc_cn => .{ .glibc_gbk = .{ .variant = .gb2312 } }, .big5 => .{ .glibc_big5 = .{} }, .gb18030 => .{ .gb18030 = .{ .glibc = true } }, .cp1255, .cp1258 => .{ .glibc_combining = .{ .composer = codecs.glibc_combining.composer(charset).? } }, .ibm037, .ibm500, .ibm875, .ibm1026 => .{ .ebcdic = .{ .index = glibc_single_byte.reversedEbcdicIndex(charset).? }, }, else => .{ .single_byte = .{ .index = glibc_single_byte.reversedIndex(charset).? } }, }, }; } /// Encode one chunk of UTF-8. Returns when the chunk is used up, which may /// leave a UTF-8 sequence half-read; the rest of it is expected next. pub fn encode(self: *Encoder, utf8: []const u8, writer: *Writer) Error!void { self.input.input = utf8; self.input.pos = 0; defer self.input.input = &.{}; return self.run(writer, false); } /// Tell the encoder there is no more input. /// /// This is not optional for ISO-2022-JP, which has to return to ASCII mode /// before the text ends, and it is what turns a UTF-8 sequence truncated at /// the end of the input into U+FFFD. Calling `encode` after it starts a new /// text. pub fn finish(self: *Encoder, writer: *Writer) Error!void { defer self.* = switch (self.charset) { .standard => |encoding| init(encoding, .{ .error_mode = self.error_mode }) catch unreachable, .ebcdic => |code_page| initEbcdic(code_page, .{ .error_mode = self.error_mode }), .korean => |korean| initKorean(korean, .{ .error_mode = self.error_mode }), .glibc => |charset| initGlibc(charset, .{ .error_mode = self.error_mode }), }; return self.run(writer, true); } fn run(self: *Encoder, writer: *Writer, final: bool) Error!void { while (true) { // The end of the input is passed on as a null of its own rather than // simply returning, because ISO-2022-JP answers it with the escape // sequence that closes its last mode -- and then has to be asked // again before it says it is finished. const code_point = self.nextCodePoint(final); if (code_point == null and !final) return; switch (try self.emit(code_point, writer)) { .stop => return, .again => {}, } } } const Step = enum { stop, again }; /// Runs one scalar value through the handler, including the retries an /// encoder asks for when it has to change mode first. Only ISO-2022-JP ever /// retries, and it changes state each time, so the loop is bounded. fn emit(self: *Encoder, code_point: ?u21, writer: *Writer) Error!Step { if (self.drops_tag_characters) { if (code_point) |c| if (c >= 0xe0000 and c <= 0xe007f) return .again; } var remaining: usize = 4; while (true) : (remaining -= 1) { std.debug.assert(remaining > 0); switch (self.handle(code_point)) { .finished => return .stop, .cont => return .again, .bytes => |*bytes| { try writer.writeAll(bytes.slice()); return .again; }, .bytes_and_retry => |*bytes| try writer.writeAll(bytes.slice()), .failure => |unmappable| switch (self.error_mode) { .fatal => { self.unmappable = unmappable; return error.UnmappableCodePoint; }, .html => { try self.writeCharacterReference(unmappable, writer); return .again; }, }, } } } /// Writes `中` for a scalar value this encoding cannot hold. /// /// The standard says to push the bytes 0x26, 0x23, the decimal digits and /// 0x3B, which it can, because every encoding it defines spells ASCII as /// ASCII. EBCDIC does not, so what is written here is those *characters* put /// through the encoder rather than those byte values put through nothing. For /// the standard's own encodings the two are the same sequence, ISO-2022-JP /// included -- its encoder only ever reports an error while in ASCII or Roman /// mode, where an ASCII character is its own byte. /// /// A character of the reference that the encoding cannot hold either would /// leave nothing sensible to write, so the byte the standard names is used /// instead. No encoding here is missing a digit, but a table can always be /// added. fn writeCharacterReference(self: *Encoder, code_point: u21, writer: *Writer) Writer.Error!void { var digits: [8]u8 = undefined; const decimal = std.fmt.bufPrint(&digits, "{d}", .{code_point}) catch unreachable; try self.writeAscii('&', writer); try self.writeAscii('#', writer); for (decimal) |digit| try self.writeAscii(digit, writer); try self.writeAscii(';', writer); } fn writeAscii(self: *Encoder, character: u8, writer: *Writer) Writer.Error!void { var remaining: usize = 4; while (true) : (remaining -= 1) { std.debug.assert(remaining > 0); switch (self.handle(character)) { .bytes => |*bytes| return writer.writeAll(bytes.slice()), .bytes_and_retry => |*bytes| try writer.writeAll(bytes.slice()), // Neither can happen for a single ASCII character, and if one // ever did, the byte the standard names is the honest fallback. .cont, .finished, .failure => return writer.writeAll(&.{character}), } } } fn handle(self: *Encoder, code_point: ?u21) Encoded { return switch (self.state) { inline else => |*state| state.handle(code_point), }; } /// The next scalar value of the input, or null when this chunk is used up. /// /// This is the UTF-8 decoder doing the reading, which is how a sequence /// spanning two calls to `encode` survives and how invalid input becomes /// U+FFFD rather than an error: the standard's encoders are defined over /// scalar values and have nothing to say about malformed ones. fn nextCodePoint(self: *Encoder, final: bool) ?u21 { while (true) { const byte = self.input.next(); if (byte == null and !final) return null; switch (self.input_decoder.handle(&self.input, byte)) { .finished => return null, .cont => {}, .code_point => |code_point| return code_point, .code_points => unreachable, .failure => return 0xfffd, } } }