// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-License-Identifier: MIT //! The vocabulary every codec in `src/codecs/` is written in. //! //! The Encoding Standard defines each encoding as a *handler*: a function //! given one item at a time that returns "finished", "continue", one or more //! items, or "error". Everything about buffering, about what an error turns //! into, and about where the output goes is outside the handler, which is why //! eleven decoders and nine encoders here need one loop between them rather //! than one each. `Decoder` and `Encoder` are that loop. const std = @import("std"); const assert = std.debug.assert; /// U+FFFF as it appears in a generated table: a noncharacter, absent from /// every index, standing for "this pointer maps to nothing". pub const unmapped = 0xffff; /// What a decoder's handler returns. pub const Decoded = union(enum) { /// The input ended cleanly, with no sequence half-read. finished, /// The standard's "continue": the byte was taken and contributed to a /// sequence that is not complete yet, so there is nothing to emit. cont, /// The standard's "one item". code_point: u21, /// The standard's "more than one item", which only Big5 ever returns, /// for the four pointers whose Unicode spelling needs a base and a /// combining mark. code_points: [2]u21, /// The standard's "error". A decoder error never carries a code point: /// there is nothing to carry, the bytes meant nothing. failure, }; /// What an encoder's handler returns. pub const Encoded = union(enum) { /// The input ended with the encoder in a state it can stop in. finished, /// Nothing to emit for this code point yet. cont, /// Bytes to emit. bytes: Bytes, /// Bytes to emit, after which the same code point is handed to the /// handler again. The standard writes this as restoring the code point to /// the input queue and returning the bytes; every encoder that does it /// restores the code point it was just given, so saying so here saves the /// encoders a queue to push back onto. ISO-2022-JP is the only one that /// needs it, to emit an escape sequence before the character that made /// the mode change necessary. bytes_and_retry: Bytes, /// The standard's "error with a code point": this encoding cannot /// represent that scalar value. The code point comes back because the /// "html" error mode has to spell it out as a character reference. failure: u21, }; /// Up to eight bytes without an allocation. Four is the most any encoder in /// the standard returns at once -- gb18030's four-byte form, and the /// three-byte escape sequences of ISO-2022-JP -- and eight is Wansung's /// spelling of a syllable KS X 1001 has no code for. pub const Bytes = struct { buffer: [8]u8 = undefined, len: u4 = 0, pub fn one(a: u8) Bytes { return .of(&.{a}); } pub fn two(a: u8, b: u8) Bytes { return .of(&.{ a, b }); } pub fn three(a: u8, b: u8, c: u8) Bytes { return .of(&.{ a, b, c }); } pub fn four(a: u8, b: u8, c: u8, d: u8) Bytes { return .of(&.{ a, b, c, d }); } pub fn of(bytes: []const u8) Bytes { var result: Bytes = .{ .len = @intCast(bytes.len) }; @memcpy(result.buffer[0..bytes.len], bytes); return result; } pub fn slice(self: *const Bytes) []const u8 { return self.buffer[0..self.len]; } }; /// The standard's I/O queue of bytes, as much of it as a decoder needs: one /// item at a time, and the ability to put items back. /// /// The putting back is not a convenience. Several decoders read a byte that /// turns out not to belong to the sequence they were assembling, and the /// standard has them return it to the queue so that it is read again from a /// clean state -- which is how `0x81 0x41` in Shift_JIS decodes as U+FFFD /// followed by `A` rather than swallowing the `A`. gb18030 can hand back /// three bytes at once, and Wansung six, after an eight-byte syllable that /// turns out not to be one; with what was already waiting, that is never more /// than eight. pub const ByteQueue = struct { /// The chunk currently being read. Between calls this is empty, and /// anything not yet consumed lives in `restored`. input: []const u8 = &.{}, pos: usize = 0, restored: [8]u8 = undefined, restored_len: usize = 0, /// The next byte, or null when this chunk is used up. Null does not mean /// the end of the text: only the caller knows whether another chunk is /// coming. pub fn next(self: *ByteQueue) ?u8 { if (self.restored_len > 0) { const byte = self.restored[0]; self.restored_len -= 1; std.mem.copyForwards(u8, self.restored[0..self.restored_len], self.restored[1 .. self.restored_len + 1]); return byte; } if (self.pos < self.input.len) { defer self.pos += 1; return self.input[self.pos]; } return null; } /// Put bytes back, to be read again in the order given and before /// anything else. pub fn restore(self: *ByteQueue, bytes: []const u8) void { assert(self.restored_len + bytes.len <= self.restored.len); std.mem.copyBackwards( u8, self.restored[bytes.len .. bytes.len + self.restored_len], self.restored[0..self.restored_len], ); @memcpy(self.restored[0..bytes.len], bytes); self.restored_len += bytes.len; } /// Whether anything restored is still waiting. A decoder that has been /// handed no more input can still have work to do. pub fn hasRestored(self: *const ByteQueue) bool { return self.restored_len > 0; } }; test "ByteQueue reads a chunk in order" { var queue: ByteQueue = .{ .input = "abc" }; try std.testing.expectEqual(@as(?u8, 'a'), queue.next()); try std.testing.expectEqual(@as(?u8, 'b'), queue.next()); try std.testing.expectEqual(@as(?u8, 'c'), queue.next()); try std.testing.expectEqual(@as(?u8, null), queue.next()); } test "ByteQueue reads restored bytes first and in order" { var queue: ByteQueue = .{ .input = "xy" }; queue.restore(&.{ 'a', 'b' }); try std.testing.expectEqual(@as(?u8, 'a'), queue.next()); queue.restore(&.{'c'}); try std.testing.expectEqual(@as(?u8, 'c'), queue.next()); try std.testing.expectEqual(@as(?u8, 'b'), queue.next()); try std.testing.expectEqual(@as(?u8, 'x'), queue.next()); try std.testing.expectEqual(@as(?u8, 'y'), queue.next()); try std.testing.expectEqual(@as(?u8, null), queue.next()); } test "Bytes carries what it was given" { const bytes: Bytes = .three(0x1b, 0x28, 0x42); try std.testing.expectEqualSlices(u8, &.{ 0x1b, 0x28, 0x42 }, bytes.slice()); }