// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-License-Identifier: MIT //! Bytes in some encoding, out as UTF-8. //! //! The decoder holds whatever state its encoding needs -- a leading byte, a //! half-read escape sequence, a leading surrogate -- so a multi-byte //! character split across two calls is decoded correctly rather than turning //! into two replacement characters. That is the whole reason this is a struct //! and not a function. When the input has no more chunks coming, call //! `finish`: a sequence left half-read at the end of the text is an error, //! and nothing else can tell the difference between that and a sequence whose //! rest has not arrived yet. //! //! ``` //! var decoder: charset.Decoder = .init(.shift_jis, .{}); //! while (readChunk()) |chunk| try decoder.decode(chunk, writer); //! try decoder.finish(writer); //! ``` //! //! For text that is already in memory in one piece, `charset.decodeAlloc` is //! this with the loop written out. const std = @import("std"); const codec = @import("codec.zig"); const codecs = @import("codecs.zig"); const ByteQueue = codec.ByteQueue; const Decoded = codec.Decoded; const Encoding = @import("encoding.zig").Encoding; const Writer = std.Io.Writer; const Decoder = @This(); /// What a decoder reads. EBCDIC is not in the Encoding Standard and so cannot /// be an `Encoding`; this is the two of them together, and is only ever /// consulted to build the same decoder again after `finish`. pub const Charset = union(enum) { standard: Encoding, ebcdic: CodePage, }; const CodePage = @import("ebcdic.zig").CodePage; /// What to do with bytes that mean nothing in this encoding. pub const ErrorMode = enum { /// Emit U+FFFD and carry on. This is what a browser does, and what the /// standard defaults to, because a document with one bad byte in it is /// still mostly a document. replacement, /// Stop, with `error.InvalidByteSequence`. An XML processor is required /// to do this, and so is anything treating the input as a checked format /// rather than as prose. fatal, }; pub const Options = struct { error_mode: ErrorMode = .replacement, }; pub const Error = error{ /// The input is not valid in this encoding, and `error_mode` is `fatal`. InvalidByteSequence, } || Writer.Error; charset: Charset, error_mode: ErrorMode, queue: ByteQueue, state: State, const State = union(enum) { utf8: codecs.utf8.Decoder, single_byte: codecs.single_byte.Decoder, gb18030: codecs.gb18030.Decoder, big5: codecs.big5.Decoder, euc_jp: codecs.euc_jp.Decoder, iso_2022_jp: codecs.iso_2022_jp.Decoder, shift_jis: codecs.shift_jis.Decoder, euc_kr: codecs.euc_kr.Decoder, utf16: codecs.utf16.Decoder, replacement: codecs.replacement.Decoder, x_user_defined: codecs.x_user_defined.Decoder, ebcdic: codecs.ebcdic.Decoder, }; pub fn init(encoding: Encoding, options: Options) Decoder { return .{ .charset = .{ .standard = encoding }, .error_mode = options.error_mode, .queue = .{}, .state = switch (encoding) { .utf8 => .{ .utf8 = .{} }, // GBK's decoder *is* gb18030's; only the encoders differ. .gbk, .gb18030 => .{ .gb18030 = .{} }, .big5 => .{ .big5 = .{} }, .euc_jp => .{ .euc_jp = .{} }, .iso_2022_jp => .{ .iso_2022_jp = .{} }, .shift_jis => .{ .shift_jis = .{} }, .euc_kr => .{ .euc_kr = .{} }, .replacement => .{ .replacement = .{} }, .utf16be => .{ .utf16 = .{ .is_big_endian = true } }, .utf16le => .{ .utf16 = .{} }, .x_user_defined => .{ .x_user_defined = .{} }, else => .{ .single_byte = .{ // Every remaining encoding is a single-byte one, so this // cannot fail; if a member is ever added to `Encoding` // without a decoder, it fails here rather than quietly // decoding as something else. .index = codecs.single_byte.index(encoding).?, }, }, }, }; } /// A decoder for an EBCDIC code page, which is not in the standard. See /// `ebcdic.zig` for why it is reached this way rather than through a member /// of `Encoding`. pub fn initEbcdic(code_page: CodePage, options: Options) Decoder { return .{ .charset = .{ .ebcdic = code_page }, .error_mode = options.error_mode, .queue = .{}, .state = .{ .ebcdic = .{ .index = codecs.ebcdic.index(code_page) } }, }; } /// Decode one chunk, writing UTF-8. Returns when the chunk is used up, which /// may leave a multi-byte character half-read; the rest of it is expected in /// the next chunk. pub fn decode(self: *Decoder, input: []const u8, writer: *Writer) Error!void { self.queue.input = input; self.queue.pos = 0; defer self.queue.input = &.{}; return self.run(writer, false); } /// Tell the decoder there are no more chunks. A character left half-read is /// an error here, handled according to `error_mode`. /// /// Calling this twice is harmless. Calling `decode` after it starts a new /// text with the encoding's initial state, which for ISO-2022-JP means ASCII /// mode rather than whatever mode the last text ended in. pub fn finish(self: *Decoder, writer: *Writer) Error!void { defer self.* = switch (self.charset) { .standard => |encoding| .init(encoding, .{ .error_mode = self.error_mode }), .ebcdic => |code_page| .initEbcdic(code_page, .{ .error_mode = self.error_mode }), }; return self.run(writer, true); } fn run(self: *Decoder, writer: *Writer, final: bool) Error!void { while (true) { const byte = self.queue.next(); if (byte == null and !final) return; switch (self.handle(byte)) { .finished => return, .cont => {}, .code_point => |code_point| try writeCodePoint(writer, code_point), .code_points => |code_points| for (code_points) |code_point| { try writeCodePoint(writer, code_point); }, .failure => switch (self.error_mode) { .replacement => try writeCodePoint(writer, 0xfffd), .fatal => return error.InvalidByteSequence, }, } } } fn handle(self: *Decoder, byte: ?u8) Decoded { return switch (self.state) { inline else => |*state| state.handle(&self.queue, byte), }; } /// Decoders return scalar values -- the standard asserts that none of them /// can produce a surrogate, and the UTF-16 decoder is the only one that could /// try, which is why it rejects an unpaired one rather than passing it on. fn writeCodePoint(writer: *Writer, code_point: u21) Writer.Error!void { var buffer: [4]u8 = undefined; const len = std.unicode.utf8Encode(code_point, &buffer) catch unreachable; return writer.writeAll(buffer[0..len]); }