Something went wrong. Try again.
A Zig library that reads git repositories without running git, with log, show and diff output byte-identical to git's.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! Writes `src/width_tables.zig`, the tables of code points `width.zig`//! counts as taking no columns or two, from two files of the Unicode//! Character Database://!//! curl -O https://www.unicode.org/Public/17.0.0/ucd/extracted/DerivedEastAsianWidth.txt//! curl -O https://www.unicode.org/Public/17.0.0/ucd/extracted/DerivedGeneralCategory.txt//! zig run tools/gen_width.zig -- DerivedEastAsianWidth.txt \//! DerivedGeneralCategory.txt src/width_tables.zig//!//! The files are not kept in the repository; the version is pinned here and//! the run refuses files of any other version, so the same command always//! writes the same tables.//!//! Two columns: East_Asian_Width W (Wide) or F (Fullwidth). The derived//! file is read rather than `EastAsianWidth.txt` because it states, as//! `@missing` lines, the ranges whose unassigned code points default to W//! (the CJK ideograph blocks and planes 2 and 3), which `EastAsianWidth.txt`//! gives only in prose; the explicit values are the same in both.//!//! No columns: General_Category Mn (nonspacing mark), Me (enclosing mark)//! and Cf (format), except U+00AD SOFT HYPHEN, and the Hangul medial vowels//! and final consonants U+1160..U+11FF besides. Those two exceptions are//! git's, found by running it: every code point from U+00A0 up was padded//! with `git log --format='%<(8)%s|'` and the columns it counted compared//! with these rules, which match git 2.56 at every code point when the//! tables are Unicode 17.0.0's.
const std = @import("std");const Io = std.Io;
/// The Unicode version the tables are made from.pub const unicode_version = "17.0.0";
/// Where the input files are published.pub const base_url = "https://www.unicode.org/Public/" ++ unicode_version ++ "/ucd/extracted/";
const code_points = 0x110000;
/// What is wrong with an input file.pub const Error = error{ BadLine, WrongVersion };
/// Reads the two files named on the command line and writes the third.pub fn main(init: std.process.Init) !void { const gpa = init.gpa; const io = init.io; const args = try init.minimal.args.toSlice(init.arena.allocator());
var stderr_buffer: [512]u8 = undefined; var stderr_writer: Io.File.Writer = .init(.stderr(), io, &stderr_buffer); const stderr = &stderr_writer.interface;
if (args.len != 4) { try stderr.writeAll("usage: gen_width <DerivedEastAsianWidth.txt> <DerivedGeneralCategory.txt> <output.zig>\n"); try stderr.flush(); std.process.exit(2); }
const cwd = Io.Dir.cwd(); const eaw = try cwd.readFileAlloc(io, args[1], gpa, .limited(16 << 20)); defer gpa.free(eaw); const gc = try cwd.readFileAlloc(io, args[2], gpa, .limited(16 << 20)); defer gpa.free(gc);
var out: Io.Writer.Allocating = .init(gpa); defer out.deinit(); generate(gpa, eaw, gc, &out.writer) catch |err| { try stderr.print("gen_width: {s}\n", .{@errorName(err)}); try stderr.flush(); std.process.exit(1); }; try cwd.writeFile(io, .{ .sub_path = args[3], .data = out.written() });}
/// Writes the generated file for the two inputs' contents to `w`.pub fn generate(gpa: std.mem.Allocator, eaw: []const u8, gc: []const u8, w: *Io.Writer) !void { try checkVersion(eaw, "DerivedEastAsianWidth"); try checkVersion(gc, "DerivedGeneralCategory");
const wide = try gpa.alloc(bool, code_points); defer gpa.free(wide); @memset(wide, false); const zero = try gpa.alloc(bool, code_points); defer gpa.free(zero); @memset(zero, false);
// Defaults first, then the explicit values over them. var lines = std.mem.splitScalar(u8, eaw, '\n'); while (lines.next()) |line| { const prefix = "# @missing: "; if (!std.mem.startsWith(u8, line, prefix)) continue; const entry = try parseEntry(line[prefix.len..]) orelse return error.BadLine; const value = entry.value; const is_wide = std.mem.eql(u8, value, "Wide") or std.mem.eql(u8, value, "Fullwidth"); @memset(wide[entry.first .. entry.last + 1], is_wide); } lines = std.mem.splitScalar(u8, eaw, '\n'); while (lines.next()) |line| { const entry = try parseEntry(stripComment(line)) orelse continue; const is_wide = std.mem.eql(u8, entry.value, "W") or std.mem.eql(u8, entry.value, "F"); @memset(wide[entry.first .. entry.last + 1], is_wide); }
lines = std.mem.splitScalar(u8, gc, '\n'); while (lines.next()) |line| { const entry = try parseEntry(stripComment(line)) orelse continue; const v = entry.value; if (std.mem.eql(u8, v, "Mn") or std.mem.eql(u8, v, "Me") or std.mem.eql(u8, v, "Cf")) @memset(zero[entry.first .. entry.last + 1], true); } // git's own exceptions; see the top of this file. zero[0x00ad] = false; @memset(zero[0x1160..0x1200], true);
try w.print( \\// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us> \\// SPDX-{s}: MIT \\ \\//! Generated by `tools/gen_width.zig` from the Unicode Character \\//! Database, version {s}: \\//! \\//! {s}DerivedEastAsianWidth.txt \\//! {s}DerivedGeneralCategory.txt \\//! \\//! Do not edit; regenerate it as `tools/gen_width.zig` describes. \\ \\/// The Unicode version the tables come from. \\pub const unicode_version = "{s}"; \\ \\/// An inclusive range of code points. \\pub const Range = struct {{ u21, u21 }}; \\ \\ , .{ "License-Identifier", unicode_version, base_url, base_url, unicode_version }); try writeTable(w, "zero_width", "Code points that take no columns: Mn, Me and Cf, but\n/// U+00AD, and U+1160..U+11FF.", zero); try w.writeAll("\n"); try writeTable(w, "double_width", "Code points that take two columns: East_Asian_Width W and F,\n/// unassigned ones included where the default is W.", wide);}
fn checkVersion(text: []const u8, name: []const u8) Error!void { const first = std.mem.sliceTo(text, '\n'); var buf: [64]u8 = undefined; const expected = std.fmt.bufPrint(&buf, "# {s}-{s}.txt", .{ name, unicode_version }) catch unreachable; if (!std.mem.eql(u8, std.mem.trimEnd(u8, first, " \r"), expected)) return error.WrongVersion;}
fn stripComment(line: []const u8) []const u8 { const end = std.mem.findScalar(u8, line, '#') orelse line.len; return line[0..end];}
const Entry = struct { first: u21, last: u21, value: []const u8 };
/// `XXXX; value` or `XXXX..YYYY; value`, or null for a blank line.fn parseEntry(text: []const u8) Error!?Entry { const trimmed = std.mem.trim(u8, text, " \t\r"); if (trimmed.len == 0) return null; const semi = std.mem.findScalar(u8, trimmed, ';') orelse return error.BadLine; const range = std.mem.trim(u8, trimmed[0..semi], " \t"); const value = std.mem.trim(u8, trimmed[semi + 1 ..], " \t"); if (value.len == 0) return error.BadLine; var first_text = range; var last_text = range; if (std.mem.find(u8, range, "..")) |dots| { first_text = range[0..dots]; last_text = range[dots + 2 ..]; } const first = try parseCodePoint(first_text); const last = try parseCodePoint(last_text); if (last < first) return error.BadLine; return .{ .first = first, .last = last, .value = value };}
fn parseCodePoint(text: []const u8) Error!u21 { const v = std.fmt.parseInt(u32, text, 16) catch return error.BadLine; if (v >= code_points) return error.BadLine; return @intCast(v);}
fn writeTable(w: *Io.Writer, name: []const u8, doc: []const u8, set: []const bool) !void { try w.print("/// {s}\npub const {s} = [_]Range{{\n", .{ doc, name }); var cp: usize = 0; while (cp < set.len) { if (!set[cp]) { cp += 1; continue; } const first = cp; while (cp < set.len and set[cp]) cp += 1; try w.print(" .{{ 0x{x:0>4}, 0x{x:0>4} }},\n", .{ first, cp - 1 }); } try w.writeAll("};\n");}
test "generate" { const gpa = std.testing.allocator; const eaw = \\# DerivedEastAsianWidth-17.0.0.txt \\# @missing: 0000..10FFFF; Neutral \\# @missing: 4E00..9FFF; Wide \\ \\0041..005A ; Na # Lu [26] LATIN CAPITAL LETTER A..LATIN CAPITAL LETTER Z \\3000 ; F # Zs IDEOGRAPHIC SPACE \\4E00..9FFE ; W # Lo \\9FFF ; N # made up, to show explicit values win \\ ; const gc = \\# DerivedGeneralCategory-17.0.0.txt \\00AD ; Cf # SOFT HYPHEN \\0300..036F ; Mn # [112] COMBINING GRAVE ACCENT..COMBINING LATIN SMALL LETTER X \\ ; var out: Io.Writer.Allocating = .init(gpa); defer out.deinit(); try generate(gpa, eaw, gc, &out.writer); const text = out.written(); try std.testing.expect(std.mem.find(u8, text, "pub const unicode_version = \"17.0.0\";") != null); try std.testing.expect(std.mem.find(u8, text, "pub const zero_width = [_]Range{\n" ++ " .{ 0x0300, 0x036f },\n .{ 0x1160, 0x11ff },\n};\n") != null); try std.testing.expect(std.mem.find(u8, text, "pub const double_width = [_]Range{\n" ++ " .{ 0x3000, 0x3000 },\n .{ 0x4e00, 0x9ffe },\n};\n") != null);
try std.testing.expectError(error.WrongVersion, generate(gpa, "# DerivedEastAsianWidth-16.0.0.txt\n", gc, &out.writer)); try std.testing.expectError(error.BadLine, generate(gpa, "# DerivedEastAsianWidth-17.0.0.txt\n0041 N\n", gc, &out.writer));}