Something went wrong. Try again.
Converting between UTF-8 and legacy character encodings, as the WHATWG Encoding Standard and glibc's iconv define them.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! Looking a code point up in one of the multi-byte indexes.//!//! The standard defines "the index pointer for codePoint in index" as the//! *first* pointer corresponding to that code point, pointers being unique//! within an index while code points may repeat. Three encoders want//! something narrower than that://!//! * Shift_JIS reads index jis0208 with the entries at pointers 8272 through//! 8835 taken out, which pushes several code points onto their later//! pointer.//! * Big5 reads index Big5 with everything below pointer 5024 taken out, so//! that the Hong Kong Supplementary Character Set extensions are not//! emitted literally, and for six code points wants the *last* pointer//! rather than the first.//! * Everything else wants the plain first pointer.//!//! Rather than generate a table per encoder, the generated tables hold each//! index once, sorted by code point with ties broken by pointer. A code point//! that repeats is then a contiguous run, and each of those readings is a//! choice within the run.
const std = @import("std");
/// One index, in the two-array form `tools/gen_tables.zig` writes: code/// points sorted ascending with ties broken by pointer, and the pointer for/// each beside it.////// `CodePoint` is `u16` for every index but Big5, which reaches U+2F9D4.pub fn Index(comptime CodePoint: type) type { return struct { code_points: []const CodePoint, pointers: []const u16,
const Self = @This();
/// Every pointer this code point has, in increasing order. Empty when /// the index does not contain it. pub fn run(self: Self, code_point: u21) []const u16 { if (code_point > std.math.maxInt(CodePoint)) return &.{}; const needle: CodePoint = @intCast(code_point);
const start = std.sort.lowerBound(CodePoint, self.code_points, needle, order); var end = start; while (end < self.code_points.len and self.code_points[end] == needle) end += 1; return self.pointers[start..end]; }
/// The standard's "index pointer for codePoint in index". pub fn pointer(self: Self, code_point: u21) ?u16 { const pointers = self.run(code_point); return if (pointers.len == 0) null else pointers[0]; }
fn order(context: CodePoint, item: CodePoint) std.math.Order { return std.math.order(context, item); } };}
/// The standard's "index code point for pointer in index", over the dense/// decode array a generated table gives: the array is indexed by pointer and/// holds `0xffff` where the index has no entry.pub fn codePoint(decode: []const u16, ptr: usize) ?u21 { if (ptr >= decode.len) return null; const value = decode[ptr]; return if (value == 0xffff) null else value;}
/// The same, for Big5, whose code points do not all fit in sixteen bits.pub fn codePointWide(decode: []const u32, ptr: usize) ?u21 { if (ptr >= decode.len) return null; const value = decode[ptr]; return if (value == 0xffff) null else @intCast(value);}
test "run finds every pointer for a repeated code point" { // Pointers 2 and 5 both spell U+0041, which is the shape the exclusions // in Shift_JIS and Big5 exist to choose between. const index: Index(u16) = .{ .code_points = &.{ 0x0040, 0x0041, 0x0041, 0x0042 }, .pointers = &.{ 1, 2, 5, 9 }, }; try std.testing.expectEqualSlices(u16, &.{ 2, 5 }, index.run(0x0041)); try std.testing.expectEqualSlices(u16, &.{1}, index.run(0x0040)); try std.testing.expectEqualSlices(u16, &.{}, index.run(0x0043)); try std.testing.expectEqual(@as(?u16, 2), index.pointer(0x0041)); try std.testing.expectEqual(@as(?u16, null), index.pointer(0x0043));}
test "a code point too wide for the index is simply absent" { const index: Index(u16) = .{ .code_points = &.{0x0041}, .pointers = &.{1} }; try std.testing.expectEqual(@as(?u16, null), index.pointer(0x10000));}
test "codePoint reads the dense decode array" { const decode = [_]u16{ 0x0041, 0xffff, 0x0042 }; try std.testing.expectEqual(@as(?u21, 0x0041), codePoint(&decode, 0)); try std.testing.expectEqual(@as(?u21, null), codePoint(&decode, 1)); try std.testing.expectEqual(@as(?u21, 0x0042), codePoint(&decode, 2)); try std.testing.expectEqual(@as(?u21, null), codePoint(&decode, 3));}