// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-License-Identifier: MIT //! UTF-8 in, bytes in some other encoding out. //! //! Three encodings have no encoder: `replacement` exists to refuse, and //! neither UTF-16 variant gets one because emitting UTF-16 where a consumer //! expects bytes is a way of smuggling markup past a filter. `init` says so //! rather than guessing; `initOutput` applies `Encoding.outputEncoding` first //! and so cannot fail, which is what URL parsing and HTML form submission //! both want. //! //! The input is UTF-8. Anything in it that is not -- a truncated sequence, an //! overlong one, a lone surrogate written as three bytes -- becomes U+FFFD //! before the encoder proper sees it, which is the same thing that happens to //! a lone surrogate in a JavaScript string on its way to a byte sequence. //! ISO-2022-JP aside, a sequence split across two calls is handled. //! //! ``` //! var encoder = try charset.Encoder.init(.shift_jis, .{}); //! try encoder.encode("日本語", writer); //! try encoder.finish(writer); //! ``` const std = @import("std"); const codec = @import("codec.zig"); const codecs = @import("codecs.zig"); const ByteQueue = codec.ByteQueue; const Encoded = codec.Encoded; const Encoding = @import("encoding.zig").Encoding; const Writer = std.Io.Writer; const Encoder = @This(); /// What an encoder writes: an `Encoding` from the standard, or an EBCDIC code /// page, which is not in it. See `ebcdic.zig` for why the two are kept apart. pub const Charset = Decoder.Charset; const CodePage = @import("ebcdic.zig").CodePage; const Decoder = @import("Decoder.zig"); /// What to do with a scalar value this encoding cannot represent. pub const ErrorMode = enum { /// Stop, with `error.UnmappableCodePoint`, and leave the offending scalar /// value in `unmappable`. fatal, /// Emit a decimal HTML character reference -- `国` -- and carry on. /// /// This exists because HTML form submission requires an encoder that does /// not stop, and it is worth knowing what it costs: the reference cannot /// be told apart from the same characters typed literally, so a form /// submitted this way loses information silently. Use UTF-8 and the /// question does not arise. html, }; pub const Options = struct { error_mode: ErrorMode = .fatal, }; pub const Error = error{ /// This encoding cannot represent a scalar value in the input, and /// `error_mode` is `fatal`. Which scalar value is in `unmappable`. UnmappableCodePoint, } || Writer.Error; /// `init` was given one of the three encodings with no encoder. pub const NoEncoderError = error{NoEncoder}; charset: Charset, error_mode: ErrorMode, /// The scalar value that `error.UnmappableCodePoint` was about. Meaningless /// until that error has been returned. unmappable: u21 = 0, /// The input side: UTF-8 bytes on their way to becoming scalar values. input: ByteQueue = .{}, input_decoder: codecs.utf8.Decoder = .{}, state: State, const State = union(enum) { utf8: codecs.utf8.Encoder, single_byte: codecs.single_byte.Encoder, gb18030: codecs.gb18030.Encoder, big5: codecs.big5.Encoder, euc_jp: codecs.euc_jp.Encoder, iso_2022_jp: codecs.iso_2022_jp.Encoder, shift_jis: codecs.shift_jis.Encoder, euc_kr: codecs.euc_kr.Encoder, x_user_defined: codecs.x_user_defined.Encoder, ebcdic: codecs.ebcdic.Encoder, }; pub fn init(encoding: Encoding, options: Options) NoEncoderError!Encoder { return .{ .charset = .{ .standard = encoding }, .error_mode = options.error_mode, .state = switch (encoding) { .replacement, .utf16be, .utf16le => return error.NoEncoder, .utf8 => .{ .utf8 = .{} }, .gbk => .{ .gb18030 = .{ .is_gbk = true } }, .gb18030 => .{ .gb18030 = .{} }, .big5 => .{ .big5 = .{} }, .euc_jp => .{ .euc_jp = .{} }, .iso_2022_jp => .{ .iso_2022_jp = .{} }, .shift_jis => .{ .shift_jis = .{} }, .euc_kr => .{ .euc_kr = .{} }, .x_user_defined => .{ .x_user_defined = .{} }, else => .{ .single_byte = .{ .index = codecs.single_byte.reversedIndex(encoding).?, } }, }, }; } /// `init` on `encoding.outputEncoding()`, which is never one of the three /// without an encoder and so cannot fail. pub fn initOutput(encoding: Encoding, options: Options) Encoder { return init(encoding.outputEncoding(), options) catch unreachable; } /// An encoder for an EBCDIC code page. Every one of them has an encoder, so /// unlike `init` this cannot fail. pub fn initEbcdic(code_page: CodePage, options: Options) Encoder { return .{ .charset = .{ .ebcdic = code_page }, .error_mode = options.error_mode, .state = .{ .ebcdic = .{ .index = codecs.ebcdic.reversedIndex(code_page) } }, }; } /// Encode one chunk of UTF-8. Returns when the chunk is used up, which may /// leave a UTF-8 sequence half-read; the rest of it is expected next. pub fn encode(self: *Encoder, utf8: []const u8, writer: *Writer) Error!void { self.input.input = utf8; self.input.pos = 0; defer self.input.input = &.{}; return self.run(writer, false); } /// Tell the encoder there is no more input. /// /// This is not optional for ISO-2022-JP, which has to return to ASCII mode /// before the text ends, and it is what turns a UTF-8 sequence truncated at /// the end of the input into U+FFFD. Calling `encode` after it starts a new /// text. pub fn finish(self: *Encoder, writer: *Writer) Error!void { defer self.* = switch (self.charset) { .standard => |encoding| init(encoding, .{ .error_mode = self.error_mode }) catch unreachable, .ebcdic => |code_page| initEbcdic(code_page, .{ .error_mode = self.error_mode }), }; return self.run(writer, true); } fn run(self: *Encoder, writer: *Writer, final: bool) Error!void { while (true) { // The end of the input is passed on as a null of its own rather than // simply returning, because ISO-2022-JP answers it with the escape // sequence that closes its last mode -- and then has to be asked // again before it says it is finished. const code_point = self.nextCodePoint(final); if (code_point == null and !final) return; switch (try self.emit(code_point, writer)) { .stop => return, .again => {}, } } } const Step = enum { stop, again }; /// Runs one scalar value through the handler, including the retries an /// encoder asks for when it has to change mode first. Only ISO-2022-JP ever /// retries, and it changes state each time, so the loop is bounded. fn emit(self: *Encoder, code_point: ?u21, writer: *Writer) Error!Step { var remaining: usize = 4; while (true) : (remaining -= 1) { std.debug.assert(remaining > 0); switch (self.handle(code_point)) { .finished => return .stop, .cont => return .again, .bytes => |*bytes| { try writer.writeAll(bytes.slice()); return .again; }, .bytes_and_retry => |*bytes| try writer.writeAll(bytes.slice()), .failure => |unmappable| switch (self.error_mode) { .fatal => { self.unmappable = unmappable; return error.UnmappableCodePoint; }, .html => { try self.writeCharacterReference(unmappable, writer); return .again; }, }, } } } /// Writes `中` for a scalar value this encoding cannot hold. /// /// The standard says to push the bytes 0x26, 0x23, the decimal digits and /// 0x3B, which it can, because every encoding it defines spells ASCII as /// ASCII. EBCDIC does not, so what is written here is those *characters* put /// through the encoder rather than those byte values put through nothing. For /// the standard's own encodings the two are the same sequence, ISO-2022-JP /// included -- its encoder only ever reports an error while in ASCII or Roman /// mode, where an ASCII character is its own byte. /// /// A character of the reference that the encoding cannot hold either would /// leave nothing sensible to write, so the byte the standard names is used /// instead. No encoding here is missing a digit, but a table can always be /// added. fn writeCharacterReference(self: *Encoder, code_point: u21, writer: *Writer) Writer.Error!void { var digits: [8]u8 = undefined; const decimal = std.fmt.bufPrint(&digits, "{d}", .{code_point}) catch unreachable; try self.writeAscii('&', writer); try self.writeAscii('#', writer); for (decimal) |digit| try self.writeAscii(digit, writer); try self.writeAscii(';', writer); } fn writeAscii(self: *Encoder, character: u8, writer: *Writer) Writer.Error!void { var remaining: usize = 4; while (true) : (remaining -= 1) { std.debug.assert(remaining > 0); switch (self.handle(character)) { .bytes => |*bytes| return writer.writeAll(bytes.slice()), .bytes_and_retry => |*bytes| try writer.writeAll(bytes.slice()), // Neither can happen for a single ASCII character, and if one // ever did, the byte the standard names is the honest fallback. .cont, .finished, .failure => return writer.writeAll(&.{character}), } } } fn handle(self: *Encoder, code_point: ?u21) Encoded { return switch (self.state) { inline else => |*state| state.handle(code_point), }; } /// The next scalar value of the input, or null when this chunk is used up. /// /// This is the UTF-8 decoder doing the reading, which is how a sequence /// spanning two calls to `encode` survives and how invalid input becomes /// U+FFFD rather than an error: the standard's encoders are defined over /// scalar values and have nothing to say about malformed ones. fn nextCodePoint(self: *Encoder, final: bool) ?u21 { while (true) { const byte = self.input.next(); if (byte == null and !final) return null; switch (self.input_decoder.handle(&self.input, byte)) { .finished => return null, .cont => {}, .code_point => |code_point| return code_point, .code_points => unreachable, .failure => return 0xfffd, } } }