Something went wrong. Try again.
A Zig library that reads git repositories without running git, with log, show and diff output byte-identical to git's.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! Loose objects: one file per object in an objects directory, named//! `xx/yyyy…` after the object ID's hexadecimal digits, holding the//! object's header and contents compressed as one zlib stream//! (gitrepository-layout(5), RFC 1950).//!//! Reading one is checked as git checks it, and refused for each//! corruption git reports, in the same circumstances; the rules were found//! by writing corrupt loose objects byte by byte and asking `git cat-file`//! about them. What git checks depends on how much of the object it reads://!//! - **The header** (`readHeader`): git inflates no more than the header's//! 32 bytes. A corrupt stream counts only if the corruption shows within//! them, which includes the stream's checksum if the whole object is that//! short; a stream that simply stops is fine if the header is complete.//! - **The whole object** (`read`): the stream must be whole, and must hold//! exactly as many bytes as the header says, with nothing after it in the//! file. One quirk is git's own: if the whole object fitted within those//! first 32 bytes, bytes past the header's size are ignored.//! - **Streaming** (`stream`): the stream must be whole, but its length is//! not compared with the header's and anything after it is ignored.//!//! The object ID is never recomputed here: `git cat-file` does not do it//! either, except for `-p` of a tree.//!//! The functions taking an `Io.Reader` do no other I/O, so they can be//! fuzzed; `Store` opens the files.
const std = @import("std");const Allocator = std.mem.Allocator;const Io = std.Io;const flate = std.compress.flate;
const object = @import("object.zig");const Format = object.Format;const Id = object.Id;const Header = object.Header;const Diagnostics = @import("Diagnostics.zig");const zlib = @import("zlib.zig");const Inflater = zlib.Inflater;const WindowWriter = zlib.WindowWriter;const ObjectDatabase = @import("ObjectDatabase.zig");const Object = ObjectDatabase.Object;const ReadError = ObjectDatabase.ReadError;const StreamError = ObjectDatabase.StreamError;
/// The loose objects of one objects directory.pub const Store = struct { /// The objects directory, open; the caller closes it. dir: Io.Dir, /// How to name the objects directory in diagnostics: git names a loose /// object's file by joining this with its `xx/yyyy…`. path: []const u8, /// The repository's object format. format: Format,
/// The file a loose object is kept in, relative to the objects /// directory: `xx/yyyy…`. pub fn objectPath(id: Id, buffer: *[object.max_hex_length + 1]u8) []const u8 { var hex: [object.max_hex_length]u8 = undefined; const digits = id.toHex(&hex); buffer[0] = digits[0]; buffer[1] = digits[1]; buffer[2] = '/'; @memcpy(buffer[3..][0 .. digits.len - 2], digits[2..]); return buffer[0 .. digits.len + 1]; }
/// Whether there is a file for the object. Like `git cat-file -e`, this /// does not read it, so a corrupt object is still there. pub fn contains(s: *const Store, io: Io, id: Id) ObjectDatabase.ContainsError!bool { var buffer: [object.max_hex_length + 1]u8 = undefined; s.dir.access(io, objectPath(id, &buffer), .{}) catch |err| switch (err) { error.FileNotFound => return false, else => return error.ObjectUnreadable, }; return true; }
/// The object's header; see `ObjectDatabase.readHeader`. pub fn readHeader(s: *const Store, io: Io, id: Id, diag: ?*Diagnostics) ReadError!Header { const file = try s.open(io, id, diag); defer file.close(io); var buffer: [4096]u8 = undefined; var reader: Io.File.Reader = .init(file, io, &buffer); return readHeaderFrom(&reader.interface, diag) catch |err| return ioError(err, &reader, diag); }
/// The whole object; see `ObjectDatabase.read`. pub fn read(s: *const Store, io: Io, gpa: Allocator, id: Id, diag: ?*Diagnostics) ReadError!Object { const file = try s.open(io, id, diag); defer file.close(io); var buffer: [64 * 1024]u8 = undefined; var reader: Io.File.Reader = .init(file, io, &buffer); return readFrom(gpa, &reader.interface, diag) catch |err| return ioError(err, &reader, diag); }
/// The object's contents, written to `out` as they are inflated; see /// `ObjectDatabase.stream`. pub fn stream(s: *const Store, io: Io, gpa: Allocator, id: Id, out: *Io.Writer, diag: ?*Diagnostics) StreamError!Header { const file = try s.open(io, id, diag); defer file.close(io); var buffer: [64 * 1024]u8 = undefined; var reader: Io.File.Reader = .init(file, io, &buffer); return streamFrom(gpa, &reader.interface, out, diag) catch |err| return ioError(err, &reader, diag); }
/// The store as an object database. pub fn database(s: *Store) ObjectDatabase { return .{ .ptr = s, .vtable = &vtable }; }
const vtable: ObjectDatabase.VTable = .{ .contains = vtableContains, .readHeader = vtableReadHeader, .read = vtableRead, .stream = vtableStream, };
fn vtableContains(ptr: *anyopaque, io: Io, id: Id, diag: ?*Diagnostics) ObjectDatabase.ContainsError!bool { _ = diag; const s: *Store = @ptrCast(@alignCast(ptr)); return s.contains(io, id); }
fn vtableReadHeader(ptr: *anyopaque, io: Io, id: Id, diag: ?*Diagnostics) ReadError!Header { const s: *Store = @ptrCast(@alignCast(ptr)); return s.readHeader(io, id, diag); }
fn vtableRead(ptr: *anyopaque, io: Io, gpa: Allocator, id: Id, diag: ?*Diagnostics) ReadError!Object { const s: *Store = @ptrCast(@alignCast(ptr)); return s.read(io, gpa, id, diag); }
fn vtableStream(ptr: *anyopaque, io: Io, gpa: Allocator, id: Id, out: *Io.Writer, diag: ?*Diagnostics) StreamError!Header { const s: *Store = @ptrCast(@alignCast(ptr)); return s.stream(io, gpa, id, out, diag); }
fn open(s: *const Store, io: Io, id: Id, diag: ?*Diagnostics) ReadError!Io.File { var buffer: [object.max_hex_length + 1]u8 = undefined; const rel = objectPath(id, &buffer); if (diag) |d| { d.object_id = id; d.setPath(&.{ s.path, rel }); } return s.dir.openFile(io, rel, .{}) catch |err| switch (err) { error.FileNotFound => return error.ObjectNotFound, else => { if (diag) |d| d.setDetail("{t}", .{err}); return error.ObjectUnreadable; }, }; }};
/// Turns a failure to read the file, which the byte-level functions see/// only as `ObjectUnreadable`, into one naming the system's error.fn ioError(err: anytype, reader: *const Io.File.Reader, diag: ?*Diagnostics) @TypeOf(err) { if (err == error.ObjectUnreadable) if (diag) |d| if (reader.err) |e| d.setDetail("{t}", .{e}); return err;}
/// Reads the header of the loose object whose stored bytes `in` reads,/// reading no more of it than that takes.pub fn readHeaderFrom(in: *Io.Reader, diag: ?*Diagnostics) ReadError!Header { try checkNotEmpty(in, diag); var buffer: [2 * Header.max_len]u8 = undefined; var w: Io.Writer = .fixed(&buffer); var inflater: Inflater = .init(in); const phase = try headerPhase(&inflater, &w, diag); return phase.header;}
/// Reads the whole of the loose object whose stored bytes `in` reads,/// checked as `ObjectDatabase.read` describes. The caller owns the result.pub fn readFrom(gpa: Allocator, in: *Io.Reader, diag: ?*Diagnostics) ReadError!Object { try checkNotEmpty(in, diag); var aw: Io.Writer.Allocating = .init(gpa); defer aw.deinit(); const w = &aw.writer; var inflater: Inflater = .init(in); const phase = try headerPhase(&inflater, w, diag); const header = phase.header; const have = phase.produced - header.len; if (phase.ended) { if (have < header.size) return error.ContentTooShort; } else { if (have > header.size) return error.ContentTooLong; var ended = false; while (!ended and w.end - header.len < header.size) { const want = header.size - (w.end - header.len); switch (inflater.pump(w, std.math.cast(usize, want) orelse std.math.maxInt(usize))) { .more => {}, .end => ended = true, else => |step| return contentFailure(step, &inflater, diag), } } if (w.end - header.len < header.size) return error.ContentTooShort; // Then the stream must end, exactly there. if (!ended) switch (inflater.pump(w, 1)) { .more => return error.ContentTooLong, .end => if (w.end - header.len > header.size) return error.ContentTooLong, else => |step| return contentFailure(step, &inflater, diag), }; } _ = in.peek(1) catch |err| switch (err) { error.EndOfStream => { const all = try aw.toOwnedSlice(); return .{ .kind = header.kind, .data = all[header.len..][0..@intCast(header.size)], .allocation = all, }; }, error.ReadFailed => return error.ObjectUnreadable, }; return error.TrailingGarbage;}
/// Writes the contents of the loose object whose stored bytes `in` reads/// to `out` as they are inflated, checked as `ObjectDatabase.stream`/// describes, and returns its header. `gpa` provides the window the/// inflater looks back into, which is all the memory it takes. If the/// stream fails part way, all that was inflated before the failure has/// been written when the error is returned.pub fn streamFrom(gpa: Allocator, in: *Io.Reader, out: *Io.Writer, diag: ?*Diagnostics) StreamError!Header { try checkNotEmpty(in, diag); const window = try gpa.alloc(u8, flate.max_window_len); defer gpa.free(window); var ww: WindowWriter = .init(window, out); var inflater: Inflater = .init(in); const phase = try headerPhase(&inflater, &ww.writer, diag); ww.skip = phase.header.len; if (!phase.ended) while (true) switch (inflater.pump(&ww.writer, std.math.maxInt(usize))) { .more => {}, .end => break, .write_failed => return error.WriteFailed, else => |step| { // Everything inflated before the failure goes out first. ww.finish() catch return error.WriteFailed; return contentFailure(step, &inflater, diag); }, }; try ww.finish(); return phase.header;}
fn checkNotEmpty(in: *Io.Reader, diag: ?*Diagnostics) ReadError!void { _ = in.peek(1) catch |err| switch (err) { error.EndOfStream => { if (diag) |d| d.detail_len = 0; return error.EmptyObjectFile; }, error.ReadFailed => return error.ObjectUnreadable, };}
fn contentFailure(step: Inflater.Step, inflater: *const Inflater, diag: ?*Diagnostics) ReadError { return switch (step) { .more, .end => unreachable, .input_ended => error.TruncatedObject, .data_error => { if (diag) |d| d.setDetail("data stream error ({s})", .{zlib.message(inflater.decompress.err.?)}); return error.ContentInflateFailed; }, .read_failed => error.ObjectUnreadable, .write_failed => error.OutOfMemory, };}
const Phase = struct { header: Header, /// How many bytes, header included, were inflated. produced: usize, /// Whether the zlib stream ended within them. ended: bool,};
/// Inflates the start of an object, into `w` from its beginning, as git/// does before it knows how big the object is: up to `Header.max_len`/// bytes, and on to the end of the stream if that is where it ends.fn headerPhase(inflater: *Inflater, w: *Io.Writer, diag: ?*Diagnostics) ReadError!Phase { std.debug.assert(w.end == 0); if (zlib.headerProblem(inflater.decompress.input) catch return error.ObjectUnreadable) |problem| { if (diag) |d| d.setDetail("{s}", .{problem}); return error.HeaderInflateFailed; } var ended = false; while (w.end < Header.max_len) { switch (inflater.pump(w, Header.max_len - w.end)) { .more => {}, .end => { ended = true; break; }, .input_ended => break, else => |step| return headerFailure(step, inflater, diag), } } if (!ended and w.end == Header.max_len) { // zlib goes on past the bytes asked for as far as it can without // writing any more, which reaches the checksum of a stream that // ends here; one more byte finds out whether this one does. switch (inflater.pump(w, 1)) { .more, .input_ended => {}, .end => ended = true, else => |step| return headerFailure(step, inflater, diag), } } const header = try Header.parse(w.buffer[0..w.end], diag); return .{ .header = header, .produced = w.end, .ended = ended };}
fn headerFailure(step: Inflater.Step, inflater: *const Inflater, diag: ?*Diagnostics) ReadError { return switch (step) { .more, .end, .input_ended => unreachable, .data_error => { if (diag) |d| d.setDetail("data stream error ({s})", .{zlib.message(inflater.decompress.err.?)}); return error.HeaderInflateFailed; }, .read_failed => error.ObjectUnreadable, .write_failed => error.OutOfMemory, };}
const testing = std.testing;const fixture = @import("testing/fixture.zig");const fuzz_input = @import("testing/fuzz.zig");
fn openStore(repo: *fixture.Repo) !Store { return .{ .dir = try repo.tmp.dir.openDir(repo.io, ".git/objects", .{}), .path = ".git/objects", .format = repo.object_format, };}
test "loose objects git wrote read back as git reads them" { const gpa = testing.allocator; var repo = try fixture.Repo.init(gpa, testing.io); defer repo.deinit(); // A blob big enough to take several windows, and compress well. var big: std.ArrayList(u8) = .empty; defer big.deinit(gpa); for (0..20000) |i| try big.print(gpa, "line {d} of a big file\n", .{i % 977}); try repo.commitFile("big", big.items, "big"); try repo.commitFile("small", "hello\n", "small"); try repo.git(&.{ "tag", "-a", "-m", "a tag", "v1" });
var store = try openStore(&repo); defer store.dir.close(testing.io); const db = store.database();
for ([_][]const u8{ "HEAD:big", "HEAD:small", "HEAD^{tree}", "HEAD", "v1" }) |rev| { const hex = try repo.output(&.{ "rev-parse", rev }); defer gpa.free(hex); const id = try Id.parseHex(repo.object_format, std.mem.trimEnd(u8, hex, "\n")); errdefer std.debug.print("reading {s}\n", .{rev}); const kind_name = try repo.output(&.{ "cat-file", "-t", rev }); defer gpa.free(kind_name); const want = try repo.output(&.{ "cat-file", std.mem.trimEnd(u8, kind_name, "\n"), rev }); defer gpa.free(want);
try testing.expect(try db.contains(testing.io, id, null)); const header = try db.readHeader(testing.io, id, null); try testing.expectEqualStrings(std.mem.trimEnd(u8, kind_name, "\n"), header.kind.name()); try testing.expectEqual(want.len, header.size);
var obj = try db.read(testing.io, gpa, id, null); defer obj.deinit(gpa); try testing.expectEqual(header.kind, obj.kind); try testing.expectEqualSlices(u8, want, obj.data); try testing.expect(Id.eql(id, object.hash(repo.object_format, obj.kind, obj.data)));
var aw: Io.Writer.Allocating = .init(gpa); defer aw.deinit(); _ = try db.stream(testing.io, gpa, id, &aw.writer, null); try testing.expectEqualSlices(u8, want, aw.written()); }
var missing: Id = .zero(repo.object_format); missing.bytes[0] = 0x11; try testing.expect(!try db.contains(testing.io, missing, null)); var diag: Diagnostics = .{}; try testing.expectError(error.ObjectNotFound, db.readHeader(testing.io, missing, &diag)); try testing.expect(Id.eql(missing, diag.object_id.?));}
/// Inflates the loose object file `file` all three ways.fn readAllWays(file: []const u8) struct { header: ReadError!Header, read: ReadError!Object, stream: StreamError![]u8 } { const gpa = testing.allocator; var in1: Io.Reader = .fixed(file); var in2: Io.Reader = .fixed(file); var in3: Io.Reader = .fixed(file); var aw: Io.Writer.Allocating = .init(gpa); const streamed: StreamError![]u8 = if (streamFrom(gpa, &in3, &aw.writer, null)) |_| aw.toOwnedSlice() else |err| err; aw.deinit(); return .{ .header = readHeaderFrom(&in1, null), .read = readFrom(gpa, &in2, null), .stream = streamed };}
test "a corrupt loose object is refused as git refuses it" { const gpa = testing.allocator; // The differential tests in tools/zig_git.zig ask git about each of // these; here, the error each read gives. const good = try fixture.zlibStored(gpa, "blob 3\x00abc"); defer gpa.free(good);
{ const r = readAllWays(good); try testing.expectEqual(3, (try r.header).size); var obj = try r.read; defer obj.deinit(gpa); try testing.expectEqualStrings("abc", obj.data); const s = try r.stream; defer gpa.free(s); try testing.expectEqualStrings("abc", s); } { // Garbage after the stream: only a whole read minds. const file = try std.mem.concat(gpa, u8, &.{ good, "garbage" }); defer gpa.free(file); const r = readAllWays(file); _ = try r.header; try testing.expectError(error.TrailingGarbage, r.read); gpa.free(try r.stream); } { // A stream cut short: the header is still readable. const r = readAllWays(good[0 .. good.len - 2]); _ = try r.header; try testing.expectError(error.TruncatedObject, r.read); try testing.expectError(error.TruncatedObject, r.stream); } { // A bad checksum on an object shorter than a header's 32 bytes is // found reading the header. const file = try gpa.dupe(u8, good); defer gpa.free(file); file[file.len - 1] ^= 1; const r = readAllWays(file); try testing.expectError(error.HeaderInflateFailed, r.header); try testing.expectError(error.HeaderInflateFailed, r.read); } { // More than the header says, within the first 32 bytes: ignored. const file = try fixture.zlibStored(gpa, "commit 1\x00ab"); defer gpa.free(file); const r = readAllWays(file); var obj = try r.read; defer obj.deinit(gpa); try testing.expectEqualStrings("a", obj.data); const s = try r.stream; defer gpa.free(s); try testing.expectEqualStrings("ab", s); } { // More than the header says, beyond them: corrupt. const file = try fixture.zlibStored(gpa, "blob 30\x00yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"); defer gpa.free(file); const r = readAllWays(file); try testing.expectError(error.ContentTooLong, r.read); gpa.free(try r.stream); } { const file = try fixture.zlibStored(gpa, "blob 50\x00yyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyyy"); defer gpa.free(file); const r = readAllWays(file); try testing.expectError(error.ContentTooShort, r.read); gpa.free(try r.stream); } try testing.expectError(error.EmptyObjectFile, readAllWays("").header); try testing.expectError(error.HeaderTooLong, readAllWays("\x78").header); try testing.expectError(error.HeaderInflateFailed, readAllWays("this is not zlib at all").header);}
fn fuzzOne(_: void, smith: *testing.Smith) anyerror!void { var buffer: [4096]u8 = undefined; const file = fuzz_input.bytes(smith, &buffer); const r = readAllWays(file); if (r.read) |obj| { var o = obj; const header = try r.header; try testing.expectEqual(header.kind, o.kind); try testing.expectEqual(header.size, o.data.len); o.deinit(testing.allocator); } else |_| {} if (r.stream) |s| testing.allocator.free(s) else |_| {}}
test "fuzz: loose objects" { try testing.fuzz({}, fuzzOne, .{ .corpus = &.{ @embedFile("fuzz/loose-blob"), @embedFile("fuzz/loose-commit"), @embedFile("fuzz/loose-tree-sha256"), } });}