Something went wrong. Try again.
fork of gitlab.com/devnw/zig/json for zig 0.17
Something went wrong. Try again.
7.1 kB · 207 lines
Zig
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208// SPDX-License-Identifier: Apache-2.0// Copyright (c) 2024-2025 CodePros
//! UTF-8 Encoding/Decoding Utilities//!//! Provides strict UTF-8 validation and encoding/decoding functions.
const std = @import("std");
/// Value of one ASCII hex digit (0-9, a-f, A-F), or null for any other byte.pub fn hexDigitValue(c: u8) ?u4 { return switch (c) { '0'...'9' => @intCast(c - '0'), 'a'...'f' => @intCast(c - 'a' + 10), 'A'...'F' => @intCast(c - 'A' + 10), else => null, };}
/// Decodes the four hex digits of a `\uXXXX` escape. Every byte must be an/// ASCII hex digit, as RFC 8259 requires. Unlike std.fmt.parseInt this rejects/// a leading sign and `_` digit separators, so "+041", "0_41" and "-000" are/// not accepted (SONNY-2768).pub fn parseHex4(hex: *const [4]u8) ?u16 { var v: u16 = 0; for (hex) |c| v = (v << 4) | (hexDigitValue(c) orelse return null); return v;}
/// Decodes exactly two ASCII hex digits (JSON5 `\xNN`, `%NN`). Same strictness/// as parseHex4.pub fn parseHex2(hex: *const [2]u8) ?u8 { var v: u8 = 0; for (hex) |c| v = (v << 4) | (hexDigitValue(c) orelse return null); return v;}
/// Get the expected length of a UTF-8 sequence from its first bytepub fn utf8SeqLen(first: u8) !usize { if (first < 0x80) return 1; if (first >= 0xC2 and first <= 0xDF) return 2; if (first >= 0xE0 and first <= 0xEF) return 3; if (first >= 0xF0 and first <= 0xF4) return 4; return error.InvalidUtf8;}
/// Validate a UTF-8 byte sequencepub fn isValidUtf8Seq(seq: []const u8) bool { if (seq.len == 0) return false;
const len = utf8SeqLen(seq[0]) catch return false; if (seq.len != len) return false;
// NPERF-001: single decode pass. decodeUtf8 already range-checks every // continuation byte (0x80..0xBF), rejects a truncated sequence, and // computes the codepoint, so the previous separate continuation-byte // pre-scan was redundant. Apply the remaining UTF-8 validity rules // (surrogate, max, overlong) to the decoded codepoint -- decodeUtf8 does // NOT detect those, so they must stay here for correctness. const cp = decodeUtf8(seq) catch return false;
// Reject surrogates if (cp >= 0xD800 and cp <= 0xDFFF) return false; if (cp > 0x10FFFF) return false;
// Check for overlong encoding (minimum codepoint per sequence length). const min_cp: u21 = switch (len) { 1 => 0, 2 => 0x80, 3 => 0x800, 4 => 0x10000, else => return false, }; if (cp < min_cp) return false;
return true;}
/// Decode a UTF-8 sequence to a codepointpub fn decodeUtf8(seq: []const u8) !u21 { if (seq.len == 0) return error.InvalidUtf8;
const len = try utf8SeqLen(seq[0]); if (seq.len != len) return error.InvalidUtf8;
var cp: u21 = switch (len) { 1 => seq[0], 2 => @as(u21, seq[0] & 0x1F) << 6, 3 => @as(u21, seq[0] & 0x0F) << 12, 4 => @as(u21, seq[0] & 0x07) << 18, else => return error.InvalidUtf8, };
for (seq[1..], 0..) |b, i| { if (b < 0x80 or b > 0xBF) return error.InvalidUtf8; cp |= @as(u21, b & 0x3F) << @intCast(6 * (len - 2 - i)); }
return cp;}
/// Encode a codepoint to UTF-8pub fn encodeUtf8(cp: u21, buf: *[4]u8) !usize { if (cp < 0x80) { buf[0] = @intCast(cp); return 1; } if (cp < 0x800) { buf[0] = @intCast(0xC0 | (cp >> 6)); buf[1] = @intCast(0x80 | (cp & 0x3F)); return 2; } if (cp < 0x10000) { buf[0] = @intCast(0xE0 | (cp >> 12)); buf[1] = @intCast(0x80 | ((cp >> 6) & 0x3F)); buf[2] = @intCast(0x80 | (cp & 0x3F)); return 3; } if (cp <= 0x10FFFF) { buf[0] = @intCast(0xF0 | (cp >> 18)); buf[1] = @intCast(0x80 | ((cp >> 12) & 0x3F)); buf[2] = @intCast(0x80 | ((cp >> 6) & 0x3F)); buf[3] = @intCast(0x80 | (cp & 0x3F)); return 4; } return error.InvalidCodepoint;}
/// Count UTF-8 codepoints in a stringpub fn countCodepoints(s: []const u8) !usize { var count: usize = 0; var i: usize = 0; while (i < s.len) { const len = try utf8SeqLen(s[i]); if (i + len > s.len) return error.InvalidUtf8; count += 1; i += len; } return count;}
test "utf8 sequence length" { try std.testing.expectEqual(@as(usize, 1), try utf8SeqLen('a')); try std.testing.expectEqual(@as(usize, 2), try utf8SeqLen(0xC2)); try std.testing.expectEqual(@as(usize, 3), try utf8SeqLen(0xE0)); try std.testing.expectEqual(@as(usize, 4), try utf8SeqLen(0xF0));}
test "valid utf8" { try std.testing.expect(isValidUtf8Seq("a")); try std.testing.expect(isValidUtf8Seq("\xC2\x80")); try std.testing.expect(isValidUtf8Seq("\xE0\xA0\x80")); try std.testing.expect(isValidUtf8Seq("\xF0\x90\x80\x80"));}
test "invalid utf8 - overlong" { try std.testing.expect(!isValidUtf8Seq("\xC0\x80")); // Overlong NUL}
test "invalid utf8 - surrogate" { try std.testing.expect(!isValidUtf8Seq("\xED\xA0\x80")); // U+D800}
test "decode utf8" { try std.testing.expectEqual(@as(u21, 'A'), try decodeUtf8("A")); try std.testing.expectEqual(@as(u21, 0x80), try decodeUtf8("\xC2\x80")); try std.testing.expectEqual(@as(u21, 0x800), try decodeUtf8("\xE0\xA0\x80")); try std.testing.expectEqual(@as(u21, 0x10000), try decodeUtf8("\xF0\x90\x80\x80"));}
test "encode utf8" { var buf: [4]u8 = undefined;
const len1 = try encodeUtf8('A', &buf); try std.testing.expectEqual(@as(usize, 1), len1); try std.testing.expectEqualStrings("A", buf[0..len1]);
const len2 = try encodeUtf8(0x80, &buf); try std.testing.expectEqual(@as(usize, 2), len2); try std.testing.expectEqualStrings("\xC2\x80", buf[0..len2]);}
test "count codepoints" { try std.testing.expectEqual(@as(usize, 5), try countCodepoints("hello")); try std.testing.expectEqual(@as(usize, 1), try countCodepoints("\xF0\x9F\x98\x80")); // emoji}
test "parseHex4 accepts exactly four hex digits (SONNY-2768)" { try std.testing.expectEqual(@as(?u16, 0x0041), parseHex4("0041")); try std.testing.expectEqual(@as(?u16, 0xABCD), parseHex4("ABCD")); try std.testing.expectEqual(@as(?u16, 0xABCD), parseHex4("abcd")); try std.testing.expectEqual(@as(?u16, 0xABCD), parseHex4("aBcD")); try std.testing.expectEqual(@as(?u16, 0xFFFF), parseHex4("FFFF")); try std.testing.expectEqual(@as(?u16, 0), parseHex4("0000")); const bad = [_]*const [4]u8{ "+041", "-000", "-041", "_041", "0_41", "00_1", "004_", "0+41", "004-", "004g", "G041", "0x41", " 041", "004 ", }; for (bad) |h| try std.testing.expectEqual(@as(?u16, null), parseHex4(h));}
test "parseHex2 accepts exactly two hex digits (SONNY-2768)" { try std.testing.expectEqual(@as(?u8, 0x4a), parseHex2("4a")); try std.testing.expectEqual(@as(?u8, 0x4a), parseHex2("4A")); const bad = [_]*const [2]u8{ "+1", "-0", "_1", "1_", "1g", "zz" }; for (bad) |h| try std.testing.expectEqual(@as(?u8, null), parseHex2(h));}