Something went wrong. Try again.
Converting between UTF-8 and legacy character encodings, as the WHATWG Encoding Standard and glibc's iconv define them.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! Bytes in some encoding, out as UTF-8.//!//! The decoder holds whatever state its encoding needs -- a leading byte, a//! half-read escape sequence, a leading surrogate -- so a multi-byte//! character split across two calls is decoded correctly rather than turning//! into two replacement characters. That is the whole reason this is a struct//! and not a function. When the input has no more chunks coming, call//! `finish`: a sequence left half-read at the end of the text is an error,//! and nothing else can tell the difference between that and a sequence whose//! rest has not arrived yet.//!//! ```//! var decoder: charset.Decoder = .init(.shift_jis, .{});//! while (readChunk()) |chunk| try decoder.decode(chunk, writer);//! try decoder.finish(writer);//! ```//!//! For text that is already in memory in one piece, `charset.decodeAlloc` is//! this with the loop written out.
const std = @import("std");
const codec = @import("codec.zig");const codecs = @import("codecs.zig");
const ByteQueue = codec.ByteQueue;const Decoded = codec.Decoded;const Encoding = @import("encoding.zig").Encoding;const Writer = std.Io.Writer;
const Decoder = @This();
/// What a decoder reads. EBCDIC is not in the Encoding Standard and so cannot/// be an `Encoding`; this is the two of them together, and is only ever/// consulted to build the same decoder again after `finish`.pub const Charset = union(enum) { standard: Encoding, ebcdic: CodePage,};
const CodePage = @import("ebcdic.zig").CodePage;
/// What to do with bytes that mean nothing in this encoding.pub const ErrorMode = enum { /// Emit U+FFFD and carry on. This is what a browser does, and what the /// standard defaults to, because a document with one bad byte in it is /// still mostly a document. replacement, /// Stop, with `error.InvalidByteSequence`. An XML processor is required /// to do this, and so is anything treating the input as a checked format /// rather than as prose. fatal,};
pub const Options = struct { error_mode: ErrorMode = .replacement,};
pub const Error = error{ /// The input is not valid in this encoding, and `error_mode` is `fatal`. InvalidByteSequence,} || Writer.Error;
charset: Charset,error_mode: ErrorMode,queue: ByteQueue,state: State,
const State = union(enum) { utf8: codecs.utf8.Decoder, single_byte: codecs.single_byte.Decoder, gb18030: codecs.gb18030.Decoder, big5: codecs.big5.Decoder, euc_jp: codecs.euc_jp.Decoder, iso_2022_jp: codecs.iso_2022_jp.Decoder, shift_jis: codecs.shift_jis.Decoder, euc_kr: codecs.euc_kr.Decoder, utf16: codecs.utf16.Decoder, replacement: codecs.replacement.Decoder, x_user_defined: codecs.x_user_defined.Decoder, ebcdic: codecs.ebcdic.Decoder,};
pub fn init(encoding: Encoding, options: Options) Decoder { return .{ .charset = .{ .standard = encoding }, .error_mode = options.error_mode, .queue = .{}, .state = switch (encoding) { .utf8 => .{ .utf8 = .{} }, // GBK's decoder *is* gb18030's; only the encoders differ. .gbk, .gb18030 => .{ .gb18030 = .{} }, .big5 => .{ .big5 = .{} }, .euc_jp => .{ .euc_jp = .{} }, .iso_2022_jp => .{ .iso_2022_jp = .{} }, .shift_jis => .{ .shift_jis = .{} }, .euc_kr => .{ .euc_kr = .{} }, .replacement => .{ .replacement = .{} }, .utf16be => .{ .utf16 = .{ .is_big_endian = true } }, .utf16le => .{ .utf16 = .{} }, .x_user_defined => .{ .x_user_defined = .{} }, else => .{ .single_byte = .{ // Every remaining encoding is a single-byte one, so this // cannot fail; if a member is ever added to `Encoding` // without a decoder, it fails here rather than quietly // decoding as something else. .index = codecs.single_byte.index(encoding).?, }, }, }, };}
/// A decoder for an EBCDIC code page, which is not in the standard. See/// `ebcdic.zig` for why it is reached this way rather than through a member/// of `Encoding`.pub fn initEbcdic(code_page: CodePage, options: Options) Decoder { return .{ .charset = .{ .ebcdic = code_page }, .error_mode = options.error_mode, .queue = .{}, .state = .{ .ebcdic = .{ .index = codecs.ebcdic.index(code_page) } }, };}
/// Decode one chunk, writing UTF-8. Returns when the chunk is used up, which/// may leave a multi-byte character half-read; the rest of it is expected in/// the next chunk.pub fn decode(self: *Decoder, input: []const u8, writer: *Writer) Error!void { self.queue.input = input; self.queue.pos = 0; defer self.queue.input = &.{}; return self.run(writer, false);}
/// Tell the decoder there are no more chunks. A character left half-read is/// an error here, handled according to `error_mode`.////// Calling this twice is harmless. Calling `decode` after it starts a new/// text with the encoding's initial state, which for ISO-2022-JP means ASCII/// mode rather than whatever mode the last text ended in.pub fn finish(self: *Decoder, writer: *Writer) Error!void { defer self.* = switch (self.charset) { .standard => |encoding| .init(encoding, .{ .error_mode = self.error_mode }), .ebcdic => |code_page| .initEbcdic(code_page, .{ .error_mode = self.error_mode }), }; return self.run(writer, true);}
fn run(self: *Decoder, writer: *Writer, final: bool) Error!void { while (true) { const byte = self.queue.next(); if (byte == null and !final) return;
switch (self.handle(byte)) { .finished => return, .cont => {}, .code_point => |code_point| try writeCodePoint(writer, code_point), .code_points => |code_points| for (code_points) |code_point| { try writeCodePoint(writer, code_point); }, .failure => switch (self.error_mode) { .replacement => try writeCodePoint(writer, 0xfffd), .fatal => return error.InvalidByteSequence, }, } }}
fn handle(self: *Decoder, byte: ?u8) Decoded { return switch (self.state) { inline else => |*state| state.handle(&self.queue, byte), };}
/// Decoders return scalar values -- the standard asserts that none of them/// can produce a surrogate, and the UTF-16 decoder is the only one that could/// try, which is why it rejects an unpaired one rather than passing it on.fn writeCodePoint(writer: *Writer, code_point: u21) Writer.Error!void { var buffer: [4]u8 = undefined; const len = std.unicode.utf8Encode(code_point, &buffer) catch unreachable; return writer.writeAll(buffer[0..len]);}