Something went wrong. Try again.
Converting between UTF-8 and legacy character encodings, as the WHATWG Encoding Standard and glibc's iconv define them.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! UTF-8 in, bytes in some other encoding out.//!//! Three encodings have no encoder: `replacement` exists to refuse, and//! neither UTF-16 variant gets one because emitting UTF-16 where a consumer//! expects bytes is a way of smuggling markup past a filter. `init` says so//! rather than guessing; `initOutput` applies `Encoding.outputEncoding` first//! and so cannot fail, which is what URL parsing and HTML form submission//! both want.//!//! The input is UTF-8. Anything in it that is not -- a truncated sequence, an//! overlong one, a lone surrogate written as three bytes -- becomes U+FFFD//! before the encoder proper sees it, which is the same thing that happens to//! a lone surrogate in a JavaScript string on its way to a byte sequence.//! ISO-2022-JP aside, a sequence split across two calls is handled.//!//! ```//! var encoder = try charset.Encoder.init(.shift_jis, .{});//! try encoder.encode("日本語", writer);//! try encoder.finish(writer);//! ```
const std = @import("std");
const codec = @import("codec.zig");const codecs = @import("codecs.zig");
const ByteQueue = codec.ByteQueue;const Encoded = codec.Encoded;const Encoding = @import("encoding.zig").Encoding;const Writer = std.Io.Writer;
const Encoder = @This();
/// What an encoder writes: an `Encoding` from the standard, or an EBCDIC code/// page or Korean encoding, which are not in it. See `ebcdic.zig` for why they/// are kept apart.pub const Charset = Decoder.Charset;
const CodePage = @import("ebcdic.zig").CodePage;const Glibc = @import("glibc.zig").Glibc;const Korean = @import("korean.zig").Korean;const Decoder = @import("Decoder.zig");
/// What to do with a scalar value this encoding cannot represent.pub const ErrorMode = enum { /// Stop, with `error.UnmappableCodePoint`, and leave the offending scalar /// value in `unmappable`. fatal, /// Emit a decimal HTML character reference -- `国` -- and carry on. /// /// This exists because HTML form submission requires an encoder that does /// not stop, and it is worth knowing what it costs: the reference cannot /// be told apart from the same characters typed literally, so a form /// submitted this way loses information silently. Use UTF-8 and the /// question does not arise. html,};
pub const Options = struct { error_mode: ErrorMode = .fatal,};
pub const Error = error{ /// This encoding cannot represent a scalar value in the input, and /// `error_mode` is `fatal`. Which scalar value is in `unmappable`. UnmappableCodePoint,} || Writer.Error;
/// `init` was given one of the three encodings with no encoder.pub const NoEncoderError = error{NoEncoder};
charset: Charset,error_mode: ErrorMode,/// The scalar value that `error.UnmappableCodePoint` was about. Meaningless/// until that error has been returned.unmappable: u21 = 0,/// Whether to write nothing for the tag characters, U+E0000 to U+E007F. Only/// glibc's charsets do, because glibc does.drops_tag_characters: bool = false,/// The input side: UTF-8 bytes on their way to becoming scalar values.input: ByteQueue = .{},input_decoder: codecs.utf8.Decoder = .{},state: State,
const State = union(enum) { utf8: codecs.utf8.Encoder, single_byte: codecs.single_byte.Encoder, gb18030: codecs.gb18030.Encoder, big5: codecs.big5.Encoder, euc_jp: codecs.euc_jp.Encoder, iso_2022_jp: codecs.iso_2022_jp.Encoder, shift_jis: codecs.shift_jis.Encoder, euc_kr: codecs.euc_kr.Encoder, x_user_defined: codecs.x_user_defined.Encoder, ebcdic: codecs.ebcdic.Encoder, wansung: codecs.wansung.Encoder, johab: codecs.johab.Encoder, glibc_big5: codecs.glibc_big5.Encoder, glibc_combining: codecs.glibc_combining.Encoder, glibc_euc_jp: codecs.glibc_euc_jp.Encoder, glibc_sjis: codecs.glibc_sjis.Encoder, glibc_euc_kr: codecs.glibc_euc_kr.Encoder, glibc_gbk: codecs.glibc_gbk.Encoder, glibc_iso_2022_jp: codecs.glibc_iso_2022_jp.Encoder, glibc_unicode: codecs.glibc_unicode.Encoder,};
pub fn init(encoding: Encoding, options: Options) NoEncoderError!Encoder { return .{ .charset = .{ .standard = encoding }, .error_mode = options.error_mode, .state = switch (encoding) { .replacement, .utf16be, .utf16le => return error.NoEncoder, .utf8 => .{ .utf8 = .{} }, .gbk => .{ .gb18030 = .{ .is_gbk = true } }, .gb18030 => .{ .gb18030 = .{} }, .big5 => .{ .big5 = .{} }, .euc_jp => .{ .euc_jp = .{} }, .iso_2022_jp => .{ .iso_2022_jp = .{} }, .shift_jis => .{ .shift_jis = .{} }, .euc_kr => .{ .euc_kr = .{} }, .x_user_defined => .{ .x_user_defined = .{} }, else => .{ .single_byte = .{ .index = codecs.single_byte.reversedIndex(encoding).?, } }, }, };}
/// `init` on `encoding.outputEncoding()`, which is never one of the three/// without an encoder and so cannot fail.pub fn initOutput(encoding: Encoding, options: Options) Encoder { return init(encoding.outputEncoding(), options) catch unreachable;}
/// An encoder for an EBCDIC code page. Every one of them has an encoder, so/// unlike `init` this cannot fail.pub fn initEbcdic(code_page: CodePage, options: Options) Encoder { return .{ .charset = .{ .ebcdic = code_page }, .error_mode = options.error_mode, .state = .{ .ebcdic = .{ .index = codecs.ebcdic.reversedIndex(code_page) } }, };}
/// An encoder for one of the Korean encodings that are not in the standard./// Both have one, so this cannot fail.pub fn initKorean(korean: Korean, options: Options) Encoder { return .{ .charset = .{ .korean = korean }, .error_mode = options.error_mode, .state = switch (korean) { .wansung => .{ .wansung = .{} }, .johab => .{ .johab = .{} }, }, };}
/// An encoder for a charset as glibc 2.44's iconv defines it. See/// `glibc.zig`. Every one of them has an encoder, so this cannot fail.////// With `error_mode` set to `fatal`, which is what glibc does, this fails on/// exactly the scalar values glibc's would and otherwise writes the same/// bytes -- including writing nothing at all for the tag characters U+E0000/// to U+E007F where glibc does (see `Glibc.dropsTagCharacters`), and the/// byte order mark that glibc's `UTF-16`, `UTF-32` and `UNICODE` write/// before the first character. `html` is this library's, not glibc's. Invalid UTF-8 in the input/// becomes U+FFFD before the encoder sees it, as it does for every encoder/// here; `convertGlibcAlloc` is the one that fails on it, as glibc does.pub fn initGlibc(charset: Glibc, options: Options) Encoder { const glibc_single_byte = codecs.glibc_single_byte; const glibc_unicode = codecs.glibc_unicode; const unicode = glibc_unicode.variant(charset); return .{ .charset = .{ .glibc = charset }, .error_mode = options.error_mode, // UCS-2 drops the tag characters too, but in its own encoder, // because glibc still writes the byte order mark of `UNICODE` ahead // of one that is dropped. .drops_tag_characters = charset.dropsTagCharacters() and unicode == null, .state = if (unicode) |v| .{ .glibc_unicode = .init(v) } else switch (charset) { .utf_8 => .{ .utf8 = .{} }, .uhc => .{ .euc_kr = .{} }, .euc_kr => .{ .glibc_euc_kr = .{} }, .johab => .{ .johab = .{ .glibc = true } }, .cp932 => .{ .shift_jis = .{ .glibc = true } }, .iso_2022_jp => .{ .glibc_iso_2022_jp = .{} }, .euc_jp => .{ .glibc_euc_jp = .{} }, .sjis => .{ .glibc_sjis = .{} }, .gbk => .{ .glibc_gbk = .{} }, .euc_cn => .{ .glibc_gbk = .{ .variant = .gb2312 } }, .big5 => .{ .glibc_big5 = .{} }, .gb18030 => .{ .gb18030 = .{ .glibc = true } }, .cp1255, .cp1258 => .{ .glibc_combining = .{ .composer = codecs.glibc_combining.composer(charset).? } }, .ibm037, .ibm500, .ibm875, .ibm1026 => .{ .ebcdic = .{ .index = glibc_single_byte.reversedEbcdicIndex(charset).? }, }, else => .{ .single_byte = .{ .index = glibc_single_byte.reversedIndex(charset).? } }, }, };}
/// Encode one chunk of UTF-8. Returns when the chunk is used up, which may/// leave a UTF-8 sequence half-read; the rest of it is expected next.pub fn encode(self: *Encoder, utf8: []const u8, writer: *Writer) Error!void { self.input.input = utf8; self.input.pos = 0; defer self.input.input = &.{}; return self.run(writer, false);}
/// Tell the encoder there is no more input.////// This is not optional for ISO-2022-JP, which has to return to ASCII mode/// before the text ends, and it is what turns a UTF-8 sequence truncated at/// the end of the input into U+FFFD. Calling `encode` after it starts a new/// text.pub fn finish(self: *Encoder, writer: *Writer) Error!void { defer self.* = switch (self.charset) { .standard => |encoding| init(encoding, .{ .error_mode = self.error_mode }) catch unreachable, .ebcdic => |code_page| initEbcdic(code_page, .{ .error_mode = self.error_mode }), .korean => |korean| initKorean(korean, .{ .error_mode = self.error_mode }), .glibc => |charset| initGlibc(charset, .{ .error_mode = self.error_mode }), }; return self.run(writer, true);}
fn run(self: *Encoder, writer: *Writer, final: bool) Error!void { while (true) { // The end of the input is passed on as a null of its own rather than // simply returning, because ISO-2022-JP answers it with the escape // sequence that closes its last mode -- and then has to be asked // again before it says it is finished. const code_point = self.nextCodePoint(final); if (code_point == null and !final) return;
switch (try self.emit(code_point, writer)) { .stop => return, .again => {}, } }}
const Step = enum { stop, again };
/// Runs one scalar value through the handler, including the retries an/// encoder asks for when it has to change mode first. Only ISO-2022-JP ever/// retries, and it changes state each time, so the loop is bounded.fn emit(self: *Encoder, code_point: ?u21, writer: *Writer) Error!Step { if (self.drops_tag_characters) { if (code_point) |c| if (c >= 0xe0000 and c <= 0xe007f) return .again; } var remaining: usize = 4; while (true) : (remaining -= 1) { std.debug.assert(remaining > 0); switch (self.handle(code_point)) { .finished => return .stop, .cont => return .again, .bytes => |*bytes| { try writer.writeAll(bytes.slice()); return .again; }, .bytes_and_retry => |*bytes| try writer.writeAll(bytes.slice()), .failure => |unmappable| switch (self.error_mode) { .fatal => { self.unmappable = unmappable; return error.UnmappableCodePoint; }, .html => { try self.writeCharacterReference(unmappable, writer); return .again; }, }, } }}
/// Writes `中` for a scalar value this encoding cannot hold.////// The standard says to push the bytes 0x26, 0x23, the decimal digits and/// 0x3B, which it can, because every encoding it defines spells ASCII as/// ASCII. EBCDIC does not, so what is written here is those *characters* put/// through the encoder rather than those byte values put through nothing. For/// the standard's own encodings the two are the same sequence, ISO-2022-JP/// included -- its encoder only ever reports an error while in ASCII or Roman/// mode, where an ASCII character is its own byte.////// A character of the reference that the encoding cannot hold either would/// leave nothing sensible to write, so the byte the standard names is used/// instead. No encoding here is missing a digit, but a table can always be/// added.fn writeCharacterReference(self: *Encoder, code_point: u21, writer: *Writer) Writer.Error!void { var digits: [8]u8 = undefined; const decimal = std.fmt.bufPrint(&digits, "{d}", .{code_point}) catch unreachable;
try self.writeAscii('&', writer); try self.writeAscii('#', writer); for (decimal) |digit| try self.writeAscii(digit, writer); try self.writeAscii(';', writer);}
fn writeAscii(self: *Encoder, character: u8, writer: *Writer) Writer.Error!void { var remaining: usize = 4; while (true) : (remaining -= 1) { std.debug.assert(remaining > 0); switch (self.handle(character)) { .bytes => |*bytes| return writer.writeAll(bytes.slice()), .bytes_and_retry => |*bytes| try writer.writeAll(bytes.slice()), // Neither can happen for a single ASCII character, and if one // ever did, the byte the standard names is the honest fallback. .cont, .finished, .failure => return writer.writeAll(&.{character}), } }}
fn handle(self: *Encoder, code_point: ?u21) Encoded { return switch (self.state) { inline else => |*state| state.handle(code_point), };}
/// The next scalar value of the input, or null when this chunk is used up.////// This is the UTF-8 decoder doing the reading, which is how a sequence/// spanning two calls to `encode` survives and how invalid input becomes/// U+FFFD rather than an error: the standard's encoders are defined over/// scalar values and have nothing to say about malformed ones.fn nextCodePoint(self: *Encoder, final: bool) ?u21 { while (true) { const byte = self.input.next(); if (byte == null and !final) return null;
switch (self.input_decoder.handle(&self.input, byte)) { .finished => return null, .cont => {}, .code_point => |code_point| return code_point, .code_points => unreachable, .failure => return 0xfffd, } }}