Something went wrong. Try again.
Converting between UTF-8 and legacy character encodings, as the WHATWG Encoding Standard and glibc's iconv define them.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! UTF-16BE and UTF-16LE, § 14.2 to § 14.4.//!//! One decoder with a flag, and no encoder at all: the standard gives neither//! variant one, and `Encoding.outputEncoding` sends both to UTF-8. Emitting//! UTF-16 from a form submission is a way of smuggling bytes past something//! that was reading ASCII, so the ability is left out rather than discouraged.//!//! These are the encodings, not the Unicode encoding *schemes* of the same//! names: a byte order mark is not part of either, and is handled by the//! wrapper in `bom.zig` instead. That is also why `utf-16` with no suffix is//! a label for UTF-16LE -- deployed content says little-endian far more often//! than it says anything.
const std = @import("std");
const codec = @import("../codec.zig");
const ByteQueue = codec.ByteQueue;const Decoded = codec.Decoded;
pub const Decoder = struct { /// UTF-16BE is this decoder with the flag set. is_big_endian: bool = false, leading_byte: ?u8 = null, leading_surrogate: ?u16 = null,
pub fn handle(self: *Decoder, queue: *ByteQueue, byte: ?u8) Decoded { const b = byte orelse { if (self.leading_byte == null and self.leading_surrogate == null) return .finished; self.leading_byte = null; self.leading_surrogate = null; return .failure; };
const leading_byte = self.leading_byte orelse { self.leading_byte = b; return .cont; }; self.leading_byte = null;
const code_unit: u16 = if (self.is_big_endian) (@as(u16, leading_byte) << 8) + b else (@as(u16, b) << 8) + leading_byte;
if (self.leading_surrogate) |leading_surrogate| { self.leading_surrogate = null;
if (isTrailingSurrogate(code_unit)) { const high = @as(u21, leading_surrogate - 0xd800) * 0x400; return .{ .code_point = 0x10000 + high + (code_unit - 0xdc00) }; }
// A leading surrogate with no trailing one is an error for the // *pair*, and this code unit gets another chance on its own -- // which is why it goes back to the queue rather than being // dropped with it. const byte1: u8 = @intCast(code_unit >> 8); const byte2: u8 = @intCast(code_unit & 0x00ff); if (self.is_big_endian) { queue.restore(&.{ byte1, byte2 }); } else { queue.restore(&.{ byte2, byte1 }); } return .failure; }
if (isLeadingSurrogate(code_unit)) { self.leading_surrogate = code_unit; return .cont; } if (isTrailingSurrogate(code_unit)) return .failure; return .{ .code_point = code_unit }; }};
fn isLeadingSurrogate(code_unit: u16) bool { return code_unit >= 0xd800 and code_unit <= 0xdbff;}
fn isTrailingSurrogate(code_unit: u16) bool { return code_unit >= 0xdc00 and code_unit <= 0xdfff;}