Something went wrong. Try again.
Converting between UTF-8 and legacy character encodings, as the WHATWG Encoding Standard and glibc's iconv define them.
Something went wrong. Try again.
12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! UTF-8, § 8.1.//!//! The decoder's constraints are not the obvious ones. They come from "Best//! Practices for Using U+FFFD" in the Unicode standard, and the point of them//! is that a truncated or overlong sequence produces one U+FFFD for the//! sequence rather than one per byte, and that the byte which ended the//! sequence early is put back and read again rather than swallowed. The//! moving boundaries below are how: `0xE0` cannot be followed by `0x80`,//! because that would be an overlong encoding, and `0xED` cannot be followed//! by anything from `0xA0` up, because that would be a surrogate.
const std = @import("std");
const codec = @import("../codec.zig");
const ByteQueue = codec.ByteQueue;const Decoded = codec.Decoded;const Encoded = codec.Encoded;
pub const Decoder = struct { code_point: u21 = 0, bytes_seen: u2 = 0, bytes_needed: u2 = 0, lower_boundary: u8 = 0x80, upper_boundary: u8 = 0xbf,
pub fn handle(self: *Decoder, queue: *ByteQueue, byte: ?u8) Decoded { const b = byte orelse { if (self.bytes_needed == 0) return .finished; self.* = .{}; return .failure; };
if (self.bytes_needed == 0) { switch (b) { 0x00...0x7f => return .{ .code_point = b }, 0xc2...0xdf => { self.bytes_needed = 1; self.code_point = b & 0x1f; }, 0xe0...0xef => { if (b == 0xe0) self.lower_boundary = 0xa0; if (b == 0xed) self.upper_boundary = 0x9f; self.bytes_needed = 2; self.code_point = b & 0x0f; }, 0xf0...0xf4 => { if (b == 0xf0) self.lower_boundary = 0x90; if (b == 0xf4) self.upper_boundary = 0x8f; self.bytes_needed = 3; self.code_point = b & 0x07; }, // 0xC0 and 0xC1 could only ever begin an overlong sequence, // and 0xF5 upwards could only ever exceed U+10FFFF. else => return .failure, } return .cont; }
if (b < self.lower_boundary or b > self.upper_boundary) { self.* = .{}; queue.restore(&.{b}); return .failure; }
self.lower_boundary = 0x80; self.upper_boundary = 0xbf; self.code_point = (self.code_point << 6) | (b & 0x3f); self.bytes_seen += 1; if (self.bytes_seen != self.bytes_needed) return .cont;
const code_point = self.code_point; self.* = .{}; return .{ .code_point = code_point }; }};
pub const Encoder = struct { pub fn handle(_: *Encoder, code_point: ?u21) Encoded { const c = code_point orelse return .finished; if (c < 0x80) return .{ .bytes = .one(@intCast(c)) };
var buffer: [4]u8 = undefined; const len = std.unicode.utf8Encode(c, &buffer) catch unreachable; return .{ .bytes = .of(buffer[0..len]) }; }};