// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-License-Identifier: MIT //! UTF-8, § 8.1. //! //! The decoder's constraints are not the obvious ones. They come from "Best //! Practices for Using U+FFFD" in the Unicode standard, and the point of them //! is that a truncated or overlong sequence produces one U+FFFD for the //! sequence rather than one per byte, and that the byte which ended the //! sequence early is put back and read again rather than swallowed. The //! moving boundaries below are how: `0xE0` cannot be followed by `0x80`, //! because that would be an overlong encoding, and `0xED` cannot be followed //! by anything from `0xA0` up, because that would be a surrogate. const std = @import("std"); const codec = @import("../codec.zig"); const ByteQueue = codec.ByteQueue; const Decoded = codec.Decoded; const Encoded = codec.Encoded; pub const Decoder = struct { code_point: u21 = 0, bytes_seen: u2 = 0, bytes_needed: u2 = 0, lower_boundary: u8 = 0x80, upper_boundary: u8 = 0xbf, pub fn handle(self: *Decoder, queue: *ByteQueue, byte: ?u8) Decoded { const b = byte orelse { if (self.bytes_needed == 0) return .finished; self.* = .{}; return .failure; }; if (self.bytes_needed == 0) { switch (b) { 0x00...0x7f => return .{ .code_point = b }, 0xc2...0xdf => { self.bytes_needed = 1; self.code_point = b & 0x1f; }, 0xe0...0xef => { if (b == 0xe0) self.lower_boundary = 0xa0; if (b == 0xed) self.upper_boundary = 0x9f; self.bytes_needed = 2; self.code_point = b & 0x0f; }, 0xf0...0xf4 => { if (b == 0xf0) self.lower_boundary = 0x90; if (b == 0xf4) self.upper_boundary = 0x8f; self.bytes_needed = 3; self.code_point = b & 0x07; }, // 0xC0 and 0xC1 could only ever begin an overlong sequence, // and 0xF5 upwards could only ever exceed U+10FFFF. else => return .failure, } return .cont; } if (b < self.lower_boundary or b > self.upper_boundary) { self.* = .{}; queue.restore(&.{b}); return .failure; } self.lower_boundary = 0x80; self.upper_boundary = 0xbf; self.code_point = (self.code_point << 6) | (b & 0x3f); self.bytes_seen += 1; if (self.bytes_seen != self.bytes_needed) return .cont; const code_point = self.code_point; self.* = .{}; return .{ .code_point = code_point }; } }; pub const Encoder = struct { pub fn handle(_: *Encoder, code_point: ?u21) Encoded { const c = code_point orelse return .finished; if (c < 0x80) return .{ .bytes = .one(@intCast(c)) }; var buffer: [4]u8 = undefined; const len = std.unicode.utf8Encode(c, &buffer) catch unreachable; return .{ .bytes = .of(buffer[0..len]) }; } };