Something went wrong. Try again.
Converting between UTF-8 and legacy character encodings, as the WHATWG Encoding Standard and glibc's iconv define them.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! UTF-8 in, bytes in some other encoding out.//!//! Three encodings have no encoder: `replacement` exists to refuse, and//! neither UTF-16 variant gets one because emitting UTF-16 where a consumer//! expects bytes is a way of smuggling markup past a filter. `init` says so//! rather than guessing; `initOutput` applies `Encoding.outputEncoding` first//! and so cannot fail, which is what URL parsing and HTML form submission//! both want.//!//! The input is UTF-8. Anything in it that is not -- a truncated sequence, an//! overlong one, a lone surrogate written as three bytes -- becomes U+FFFD//! before the encoder proper sees it, which is the same thing that happens to//! a lone surrogate in a JavaScript string on its way to a byte sequence.//! ISO-2022-JP aside, a sequence split across two calls is handled.//!//! ```//! var encoder = try charset.Encoder.init(.shift_jis, .{});//! try encoder.encode("日本語", writer);//! try encoder.finish(writer);//! ```
const std = @import("std");
const codec = @import("codec.zig");const codecs = @import("codecs.zig");
const ByteQueue = codec.ByteQueue;const Encoded = codec.Encoded;const Encoding = @import("encoding.zig").Encoding;const Writer = std.Io.Writer;
const Encoder = @This();
/// What an encoder writes: an `Encoding` from the standard, or an EBCDIC code/// page, which is not in it. See `ebcdic.zig` for why the two are kept apart.pub const Charset = Decoder.Charset;
const CodePage = @import("ebcdic.zig").CodePage;const Decoder = @import("Decoder.zig");
/// What to do with a scalar value this encoding cannot represent.pub const ErrorMode = enum { /// Stop, with `error.UnmappableCodePoint`, and leave the offending scalar /// value in `unmappable`. fatal, /// Emit a decimal HTML character reference -- `国` -- and carry on. /// /// This exists because HTML form submission requires an encoder that does /// not stop, and it is worth knowing what it costs: the reference cannot /// be told apart from the same characters typed literally, so a form /// submitted this way loses information silently. Use UTF-8 and the /// question does not arise. html,};
pub const Options = struct { error_mode: ErrorMode = .fatal,};
pub const Error = error{ /// This encoding cannot represent a scalar value in the input, and /// `error_mode` is `fatal`. Which scalar value is in `unmappable`. UnmappableCodePoint,} || Writer.Error;
/// `init` was given one of the three encodings with no encoder.pub const NoEncoderError = error{NoEncoder};
charset: Charset,error_mode: ErrorMode,/// The scalar value that `error.UnmappableCodePoint` was about. Meaningless/// until that error has been returned.unmappable: u21 = 0,/// The input side: UTF-8 bytes on their way to becoming scalar values.input: ByteQueue = .{},input_decoder: codecs.utf8.Decoder = .{},state: State,
const State = union(enum) { utf8: codecs.utf8.Encoder, single_byte: codecs.single_byte.Encoder, gb18030: codecs.gb18030.Encoder, big5: codecs.big5.Encoder, euc_jp: codecs.euc_jp.Encoder, iso_2022_jp: codecs.iso_2022_jp.Encoder, shift_jis: codecs.shift_jis.Encoder, euc_kr: codecs.euc_kr.Encoder, x_user_defined: codecs.x_user_defined.Encoder, ebcdic: codecs.ebcdic.Encoder,};
pub fn init(encoding: Encoding, options: Options) NoEncoderError!Encoder { return .{ .charset = .{ .standard = encoding }, .error_mode = options.error_mode, .state = switch (encoding) { .replacement, .utf16be, .utf16le => return error.NoEncoder, .utf8 => .{ .utf8 = .{} }, .gbk => .{ .gb18030 = .{ .is_gbk = true } }, .gb18030 => .{ .gb18030 = .{} }, .big5 => .{ .big5 = .{} }, .euc_jp => .{ .euc_jp = .{} }, .iso_2022_jp => .{ .iso_2022_jp = .{} }, .shift_jis => .{ .shift_jis = .{} }, .euc_kr => .{ .euc_kr = .{} }, .x_user_defined => .{ .x_user_defined = .{} }, else => .{ .single_byte = .{ .index = codecs.single_byte.reversedIndex(encoding).?, } }, }, };}
/// `init` on `encoding.outputEncoding()`, which is never one of the three/// without an encoder and so cannot fail.pub fn initOutput(encoding: Encoding, options: Options) Encoder { return init(encoding.outputEncoding(), options) catch unreachable;}
/// An encoder for an EBCDIC code page. Every one of them has an encoder, so/// unlike `init` this cannot fail.pub fn initEbcdic(code_page: CodePage, options: Options) Encoder { return .{ .charset = .{ .ebcdic = code_page }, .error_mode = options.error_mode, .state = .{ .ebcdic = .{ .index = codecs.ebcdic.reversedIndex(code_page) } }, };}
/// Encode one chunk of UTF-8. Returns when the chunk is used up, which may/// leave a UTF-8 sequence half-read; the rest of it is expected next.pub fn encode(self: *Encoder, utf8: []const u8, writer: *Writer) Error!void { self.input.input = utf8; self.input.pos = 0; defer self.input.input = &.{}; return self.run(writer, false);}
/// Tell the encoder there is no more input.////// This is not optional for ISO-2022-JP, which has to return to ASCII mode/// before the text ends, and it is what turns a UTF-8 sequence truncated at/// the end of the input into U+FFFD. Calling `encode` after it starts a new/// text.pub fn finish(self: *Encoder, writer: *Writer) Error!void { defer self.* = switch (self.charset) { .standard => |encoding| init(encoding, .{ .error_mode = self.error_mode }) catch unreachable, .ebcdic => |code_page| initEbcdic(code_page, .{ .error_mode = self.error_mode }), }; return self.run(writer, true);}
fn run(self: *Encoder, writer: *Writer, final: bool) Error!void { while (true) { // The end of the input is passed on as a null of its own rather than // simply returning, because ISO-2022-JP answers it with the escape // sequence that closes its last mode -- and then has to be asked // again before it says it is finished. const code_point = self.nextCodePoint(final); if (code_point == null and !final) return;
switch (try self.emit(code_point, writer)) { .stop => return, .again => {}, } }}
const Step = enum { stop, again };
/// Runs one scalar value through the handler, including the retries an/// encoder asks for when it has to change mode first. Only ISO-2022-JP ever/// retries, and it changes state each time, so the loop is bounded.fn emit(self: *Encoder, code_point: ?u21, writer: *Writer) Error!Step { var remaining: usize = 4; while (true) : (remaining -= 1) { std.debug.assert(remaining > 0); switch (self.handle(code_point)) { .finished => return .stop, .cont => return .again, .bytes => |*bytes| { try writer.writeAll(bytes.slice()); return .again; }, .bytes_and_retry => |*bytes| try writer.writeAll(bytes.slice()), .failure => |unmappable| switch (self.error_mode) { .fatal => { self.unmappable = unmappable; return error.UnmappableCodePoint; }, .html => { try self.writeCharacterReference(unmappable, writer); return .again; }, }, } }}
/// Writes `中` for a scalar value this encoding cannot hold.////// The standard says to push the bytes 0x26, 0x23, the decimal digits and/// 0x3B, which it can, because every encoding it defines spells ASCII as/// ASCII. EBCDIC does not, so what is written here is those *characters* put/// through the encoder rather than those byte values put through nothing. For/// the standard's own encodings the two are the same sequence, ISO-2022-JP/// included -- its encoder only ever reports an error while in ASCII or Roman/// mode, where an ASCII character is its own byte.////// A character of the reference that the encoding cannot hold either would/// leave nothing sensible to write, so the byte the standard names is used/// instead. No encoding here is missing a digit, but a table can always be/// added.fn writeCharacterReference(self: *Encoder, code_point: u21, writer: *Writer) Writer.Error!void { var digits: [8]u8 = undefined; const decimal = std.fmt.bufPrint(&digits, "{d}", .{code_point}) catch unreachable;
try self.writeAscii('&', writer); try self.writeAscii('#', writer); for (decimal) |digit| try self.writeAscii(digit, writer); try self.writeAscii(';', writer);}
fn writeAscii(self: *Encoder, character: u8, writer: *Writer) Writer.Error!void { var remaining: usize = 4; while (true) : (remaining -= 1) { std.debug.assert(remaining > 0); switch (self.handle(character)) { .bytes => |*bytes| return writer.writeAll(bytes.slice()), .bytes_and_retry => |*bytes| try writer.writeAll(bytes.slice()), // Neither can happen for a single ASCII character, and if one // ever did, the byte the standard names is the honest fallback. .cont, .finished, .failure => return writer.writeAll(&.{character}), } }}
fn handle(self: *Encoder, code_point: ?u21) Encoded { return switch (self.state) { inline else => |*state| state.handle(code_point), };}
/// The next scalar value of the input, or null when this chunk is used up.////// This is the UTF-8 decoder doing the reading, which is how a sequence/// spanning two calls to `encode` survives and how invalid input becomes/// U+FFFD rather than an error: the standard's encoders are defined over/// scalar values and have nothing to say about malformed ones.fn nextCodePoint(self: *Encoder, final: bool) ?u21 { while (true) { const byte = self.input.next(); if (byte == null and !final) return null;
switch (self.input_decoder.handle(&self.input, byte)) { .finished => return null, .cont => {}, .code_point => |code_point| return code_point, .code_points => unreachable, .failure => return 0xfffd, } }}