// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-License-Identifier: MIT //! Looking a code point up in one of the multi-byte indexes. //! //! The standard defines "the index pointer for codePoint in index" as the //! *first* pointer corresponding to that code point, pointers being unique //! within an index while code points may repeat. Three encoders want //! something narrower than that: //! //! * Shift_JIS reads index jis0208 with the entries at pointers 8272 through //! 8835 taken out, which pushes several code points onto their later //! pointer. //! * Big5 reads index Big5 with everything below pointer 5024 taken out, so //! that the Hong Kong Supplementary Character Set extensions are not //! emitted literally, and for six code points wants the *last* pointer //! rather than the first. //! * Everything else wants the plain first pointer. //! //! Rather than generate a table per encoder, the generated tables hold each //! index once, sorted by code point with ties broken by pointer. A code point //! that repeats is then a contiguous run, and each of those readings is a //! choice within the run. const std = @import("std"); /// One index, in the two-array form `tools/gen_tables.zig` writes: code /// points sorted ascending with ties broken by pointer, and the pointer for /// each beside it. /// /// `CodePoint` is `u16` for every index but Big5, which reaches U+2F9D4. pub fn Index(comptime CodePoint: type) type { return struct { code_points: []const CodePoint, pointers: []const u16, const Self = @This(); /// Every pointer this code point has, in increasing order. Empty when /// the index does not contain it. pub fn run(self: Self, code_point: u21) []const u16 { if (code_point > std.math.maxInt(CodePoint)) return &.{}; const needle: CodePoint = @intCast(code_point); const start = std.sort.lowerBound(CodePoint, self.code_points, needle, order); var end = start; while (end < self.code_points.len and self.code_points[end] == needle) end += 1; return self.pointers[start..end]; } /// The standard's "index pointer for codePoint in index". pub fn pointer(self: Self, code_point: u21) ?u16 { const pointers = self.run(code_point); return if (pointers.len == 0) null else pointers[0]; } fn order(context: CodePoint, item: CodePoint) std.math.Order { return std.math.order(context, item); } }; } /// The standard's "index code point for pointer in index", over the dense /// decode array a generated table gives: the array is indexed by pointer and /// holds `0xffff` where the index has no entry. pub fn codePoint(decode: []const u16, ptr: usize) ?u21 { if (ptr >= decode.len) return null; const value = decode[ptr]; return if (value == 0xffff) null else value; } /// The same, for Big5, whose code points do not all fit in sixteen bits. pub fn codePointWide(decode: []const u32, ptr: usize) ?u21 { if (ptr >= decode.len) return null; const value = decode[ptr]; return if (value == 0xffff) null else @intCast(value); } test "run finds every pointer for a repeated code point" { // Pointers 2 and 5 both spell U+0041, which is the shape the exclusions // in Shift_JIS and Big5 exist to choose between. const index: Index(u16) = .{ .code_points = &.{ 0x0040, 0x0041, 0x0041, 0x0042 }, .pointers = &.{ 1, 2, 5, 9 }, }; try std.testing.expectEqualSlices(u16, &.{ 2, 5 }, index.run(0x0041)); try std.testing.expectEqualSlices(u16, &.{1}, index.run(0x0040)); try std.testing.expectEqualSlices(u16, &.{}, index.run(0x0043)); try std.testing.expectEqual(@as(?u16, 2), index.pointer(0x0041)); try std.testing.expectEqual(@as(?u16, null), index.pointer(0x0043)); } test "a code point too wide for the index is simply absent" { const index: Index(u16) = .{ .code_points = &.{0x0041}, .pointers = &.{1} }; try std.testing.expectEqual(@as(?u16, null), index.pointer(0x10000)); } test "codePoint reads the dense decode array" { const decode = [_]u16{ 0x0041, 0xffff, 0x0042 }; try std.testing.expectEqual(@as(?u21, 0x0041), codePoint(&decode, 0)); try std.testing.expectEqual(@as(?u21, null), codePoint(&decode, 1)); try std.testing.expectEqual(@as(?u21, 0x0042), codePoint(&decode, 2)); try std.testing.expectEqual(@as(?u21, null), codePoint(&decode, 3)); }