Something went wrong. Try again.
A Zig library that reads git repositories without running git, with log, show and diff output byte-identical to git's.
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie <jeff@ocjtech.us>// SPDX-License-Identifier: MIT
//! A repository's object database on disk: its own objects directory and//! those of its alternates, each with its loose objects, its packs and its//! multi-pack-index, behind the one `ObjectDatabase` interface the rest of//! the library reads objects through.//!//! What is found where, and in what order, follows what git can be seen//! to do, found by putting different copies of one object in different//! places and asking `git cat-file` which it read://!//! - **Objects directories** are asked in order: the repository's own,//! then those `GIT_ALTERNATE_OBJECT_DIRECTORIES` names, then those its//! `objects/info/alternates` names, each alternate's own alternates//! following it. A directory named twice, by whatever path, is asked//! once. An alternate's alternates are followed to a depth of five;//! past that git ignores them, and says so.//! - **Within a directory**, packs come before loose objects: those the//! multi-pack-index names through it, then the rest, the most recently//! modified pack first, ties in directory order.//! - **Streaming** is the exception: git looks for a loose object in//! every directory before any pack, and so does `stream`.//! - **A copy that cannot be read** is passed over for the next one, as//! git passes over it, and the error is returned only if no copy can//! be read. A pack that cannot be opened is passed over the same way,//! and its error returned in place of `ObjectNotFound` if the object is//! found nowhere else, since it might have been in that pack.//!//! Everything is found and opened, packs and indexes memory-mapped, when//! the database is opened; afterwards nothing about it changes, so any//! number of readers may share it at once. A pack written after opening//! is not seen until it is opened again.//!//! **A REF_DELTA whose base is not in its own pack** is not resolved by//! git: such an object reads as missing, without a word, even when the//! base is in another pack or loose. A pack git wrote never has one, since//! `git index-pack --fix-thin` appends the bases a thin pack leaves out.//! By default this agrees, and such a copy fails with `DeltaBaseMissing`;//! `Options.resolve_external_bases` looks for the base in the whole//! database instead.//!//! **Objects are limited in size** when read whole: by default to a//! gigabyte (`Options.max_object_size`), which git is not. See there.//!//! The library never reads the environment, so the value of//! `GIT_ALTERNATE_OBJECT_DIRECTORIES` is an option, and `core.multiPackIndex`//! another.
const std = @import("std");const Allocator = std.mem.Allocator;const Io = std.Io;
const object = @import("object.zig");const Format = object.Format;const Id = object.Id;const Diagnostics = @import("Diagnostics.zig");const ObjectDatabase = @import("ObjectDatabase.zig");const Object = ObjectDatabase.Object;const ReadError = ObjectDatabase.ReadError;const StreamError = ObjectDatabase.StreamError;const ContainsError = ObjectDatabase.ContainsError;const PackOpenError = ObjectDatabase.PackOpenError;const loose = @import("loose.zig");const pack = @import("pack.zig");const PackIndex = @import("PackIndex.zig");const MultiPackIndex = @import("MultiPackIndex.zig");const alternates = @import("alternates.zig");
const DiskDatabase = @This();
gpa: Allocator,format: Format,/// Whether a REF_DELTA's base is looked for outside its pack.resolve_external_bases: bool,/// The most bytes an object read whole may claim, or null for no limit.max_object_size: ?u64,/// The objects directories, in the order they are asked.directories: []*Directory,/// What was wrong but worked around while opening, in the order met.problems: []Problem,
/// The default for `Options.max_object_size`: a gigabyte.pub const default_max_object_size: u64 = 1 << 30;
/// How deep alternates of alternates are followed: a directory this many/// steps from the repository's own does not have its alternates read.pub const max_alternate_depth = 5;
/// How to open a repository's object database.pub const Options = struct { /// The repository's objects directory, relative to `base` or /// absolute. Diagnostics name its files by joining this path with /// theirs, as git names them. objects_path: []const u8, /// What relative paths are relative to. base: Io.Dir, /// The repository's object format. format: Format, /// The value of `GIT_ALTERNATE_OBJECT_DIRECTORIES`, if the caller has /// one: paths separated by `alternate_separator`. alternate_list: ?[]const u8 = null, /// What relative paths in `alternate_list` are relative to: the top of /// the working tree, where git runs; `base` if null. alternate_list_base: ?Io.Dir = null, /// What separates paths in `alternate_list`. alternate_separator: u8 = alternates.list_separator, /// Whether to read multi-pack-indexes, as `core.multiPackIndex` says. use_multi_pack_index: bool = true, /// Whether a REF_DELTA whose base is not in its own pack has the base /// looked for in the rest of the database, as git does not. resolve_external_bases: bool = false, /// The most bytes an object may claim to be when it is built in /// memory, or null for no limit. Over it, a read is `ObjectTooLarge`, /// found before anything of that size is allocated: the size a loose /// object's header claims, a pack entry's size, and the result size /// each delta of a chain declares, at every link. (A delta's own /// instructions may take up to eight times the limit, since a copy /// instruction of eight bytes may make only one.) /// /// The limit exists because those sizes are claims, and believing one /// is what lets a few bytes take all the memory there is: a delta of a /// few kilobytes copying 64 KiB at a time really does make gigabytes, /// and a chain of such deltas multiplies them, while still being a /// valid delta. git has no limit and will try to read such an object; /// the default of a gigabyte, `default_max_object_size`, is far above /// anything a source repository holds and still bounds what one read /// costs. A server reading repositories it does not trust should set it /// lower, to what it is willing to spend on one object. /// /// Only reads that build the object whole are limited. `readHeader` /// reports any size without refusing it, as `git cat-file -s` does, /// since it allocates nothing of it; `stream` writes an object stored /// whole however big it is, in bounded memory, and is limited only for /// a deltified object, which has to be built in memory first. max_object_size: ?u64 = default_max_object_size,};
/// Something wrong with the object database that git says and then works/// around, and so does this: an alternate that is not there, or nested/// too deep, or a multi-pack-index that cannot be read and so is not used.pub const Problem = struct { err: ProblemError, /// The path the problem is about. path: []u8, /// More about it, or empty. detail: []u8,};
/// The problems `Problem` records, each in git's words:////// - `UnnormalizableAlternate`: "unable to normalize alternate object/// path: …": a directory on the way to it does not exist./// - `MissingAlternate`: "object directory … does not exist; check/// .git/objects/info/alternates"./// - `AlternatesTooDeep`: "…: ignoring alternate object stores, nesting/// too deep"./// - The errors of `MultiPackIndex.Error` that git goes on from, and/// `MultiPackIndexChainMismatch`: a layer of an incremental chain whose/// checksum is not the one its name says.pub const ProblemError = error{ UnnormalizableAlternate, MissingAlternate, AlternatesTooDeep, MultiPackIndexChainMismatch,} || MultiPackIndex.Error;
/// Why the database could not be opened: the objects directory could not/// be, or one of its multi-pack-indexes is broken in a way git stops at.pub const OpenError = Allocator.Error || error{ ObjectsDirectoryUnreadable, Canceled } || MultiPackIndex.Error;
/// One objects directory: its loose objects, its packs and its/// multi-pack-index.pub const Directory = struct { /// How to name it in diagnostics. path: []u8, /// Its real path, for telling whether another names the same one. real_path: []u8, loose: loose.Store, /// Most recently modified first. packs: []PackFile, /// The layers of its multi-pack-index: one, or an incremental chain. midx_layers: []MidxLayer,};
const MidxLayer = struct { map: Mapped, midx: MultiPackIndex, /// For each of the multi-pack-index's pack numbers, the pack in /// `Directory.packs`, or null if there is no such pack. packs: []?usize,};
/// A pack and its index, as far as they could be opened.pub const PackFile = struct { /// `pack-<hex>`. name: []u8, /// How to name the `.pack` and the `.idx` in diagnostics. pack_path: []u8, idx_path: []u8, mtime: i64, /// Whether a multi-pack-index finds its objects, in which case only it /// is asked. in_midx: bool = false, pack_map: Mapped = .empty, idx_map: Mapped = .empty, state: State,
/// How far the pack could be opened. pub const State = union(enum) { ok: pack.Pack, /// The index could not be read, so nothing is known of the pack. bad_index: Failure, /// The index could be read but the pack could not be used. bad_pack: struct { index: PackIndex, failure: Failure }, };
/// Why, in the words a `Diagnostics` would have. pub const Failure = struct { err: PackOpenError, detail: [64]u8 = undefined, detail_len: usize = 0, };};
/// A file memory-mapped read-only, or an empty one, which cannot be.const Mapped = struct { map: ?Io.File.MemoryMap, bytes: []const u8,
const empty: Mapped = .{ .map = null, .bytes = &.{} };
fn open(io: Io, dir: Io.Dir, sub_path: []const u8) !Mapped { const file = try dir.openFile(io, sub_path, .{}); defer file.close(io); const size = try file.length(io); if (size == 0) return empty; const len = std.math.cast(usize, size) orelse return error.FileTooBig; const map = try Io.File.MemoryMap.create(io, file, .{ .len = len, .protection = .{ .read = true }, .populate = false, }); return .{ .map = map, .bytes = map.memory[0..len] }; }
fn close(m: *Mapped, io: Io) void { if (m.map) |*map| map.destroy(io); m.* = empty; }};
/// Opens the object database whose own objects directory `options`/// names, with its alternates, packs and multi-pack-indexes. The result/// must stay where it is while `database()` is in use, and is closed with/// `close`.pub fn open(gpa: Allocator, io: Io, options: Options, diag: ?*Diagnostics) OpenError!DiskDatabase { var opener: Opener = .{ .gpa = gpa, .io = io, .options = options, .diag = diag }; errdefer opener.deinit(); const real = options.base.realPathFileAlloc(io, options.objects_path, gpa) catch |err| { if (diag) |d| { d.setPath(&.{options.objects_path}); d.setDetail("{t}", .{err}); } return error.ObjectsDirectoryUnreadable; }; defer gpa.free(real); const own = options.base.openDir(io, options.objects_path, .{}) catch |err| { if (diag) |d| { d.setPath(&.{options.objects_path}); d.setDetail("{t}", .{err}); } return error.ObjectsDirectoryUnreadable; }; // `addDirectory` takes the directory, and closes it if it fails. try opener.addDirectory(own, options.objects_path, real, 0, true); return .{ .gpa = gpa, .format = options.format, .resolve_external_bases = options.resolve_external_bases, .max_object_size = options.max_object_size, .directories = try opener.directories.toOwnedSlice(gpa), .problems = try opener.problems.toOwnedSlice(gpa), };}
/// Closes every file and frees everything.pub fn close(db: *DiskDatabase, io: Io) void { for (db.directories) |d| closeDirectory(db.gpa, io, d); db.gpa.free(db.directories); for (db.problems) |p| { db.gpa.free(p.path); db.gpa.free(p.detail); } db.gpa.free(db.problems); db.* = undefined;}
fn closeDirectory(gpa: Allocator, io: Io, d: *Directory) void { for (d.midx_layers) |*layer| { layer.map.close(io); gpa.free(layer.packs); } gpa.free(d.midx_layers); for (d.packs) |*p| { p.pack_map.close(io); p.idx_map.close(io); gpa.free(p.name); gpa.free(p.pack_path); gpa.free(p.idx_path); } gpa.free(d.packs); d.loose.dir.close(io); gpa.free(d.path); gpa.free(d.real_path); gpa.destroy(d);}
/// The state of opening a database, while it is being opened.const Opener = struct { gpa: Allocator, io: Io, options: Options, diag: ?*Diagnostics, directories: std.ArrayList(*Directory) = .empty, problems: std.ArrayList(Problem) = .empty,
fn deinit(o: *Opener) void { for (o.directories.items) |d| closeDirectory(o.gpa, o.io, d); o.directories.deinit(o.gpa); for (o.problems.items) |p| { o.gpa.free(p.path); o.gpa.free(p.detail); } o.problems.deinit(o.gpa); }
fn problem(o: *Opener, err: ProblemError, path: []const u8, detail: []const u8) Allocator.Error!void { const p = try o.gpa.dupe(u8, path); errdefer o.gpa.free(p); const d = try o.gpa.dupe(u8, detail); errdefer o.gpa.free(d); try o.problems.append(o.gpa, .{ .err = err, .path = p, .detail = d }); }
/// Adds the objects directory `dir`, which it takes, then its /// alternates. fn addDirectory(o: *Opener, dir: Io.Dir, path: []const u8, real_path: []const u8, depth: u32, is_own: bool) OpenError!void { const gpa = o.gpa; var d = gpa.create(Directory) catch |err| { var dd = dir; dd.close(o.io); return err; }; d.* = .{ .path = &.{}, .real_path = &.{}, .loose = .{ .dir = dir, .path = &.{}, .format = o.options.format, .max_object_size = o.options.max_object_size }, .packs = &.{}, .midx_layers = &.{}, }; o.directories.append(gpa, d) catch |err| { d.loose.dir.close(o.io); gpa.destroy(d); return err; }; d.path = try gpa.dupe(u8, path); d.loose.path = d.path; d.real_path = try gpa.dupe(u8, real_path); try o.loadPacks(d);
// Alternates named by the caller come before those the // repository names. if (is_own) if (o.options.alternate_list) |list| { const base = o.options.alternate_list_base orelse o.options.base; var it = alternates.iterateList(list, o.options.alternate_separator); defer it.deinit(gpa); while (try it.next(gpa)) |entry| try o.addAlternate(base, entry, entry, 1); }; var buffer: [64 * 1024]u8 = undefined; const contents = d.loose.dir.readFile(o.io, "info/alternates", &buffer) catch return; if (depth >= max_alternate_depth + 1) { try o.problem(error.AlternatesTooDeep, d.path, ""); return; } // A relative alternate is relative to this objects directory. var it = alternates.iterateFile(contents); defer it.deinit(gpa); while (try it.next(gpa)) |entry| { if (std.fs.path.isAbsolute(entry)) { try o.addAlternate(o.options.base, entry, entry, depth + 1); } else { const shown = try std.mem.concat(gpa, u8, &.{ d.real_path, "/", entry }); defer gpa.free(shown); try o.addAlternate(o.options.base, shown, shown, depth + 1); } } }
/// Adds the alternate `path`, relative to `base` if it is not /// absolute, unless it is a directory already added. fn addAlternate(o: *Opener, base: Io.Dir, path: []const u8, shown: []const u8, depth: u32) OpenError!void { const gpa = o.gpa; var trimmed = path; while (trimmed.len > 1 and trimmed[trimmed.len - 1] == '/') trimmed = trimmed[0 .. trimmed.len - 1]; const parent = std.fs.path.dirname(trimmed); const name = std.fs.path.basename(trimmed); // The real path of the directory holding it, and then its own // name: a missing directory on the way is one thing, a missing // directory at the end another. const resolved: []u8 = if (parent == null or std.mem.eql(u8, name, ".") or std.mem.eql(u8, name, "..")) whole: { break :whole realPath(o.io, gpa, base, trimmed) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => { try o.problem(error.UnnormalizableAlternate, shown, ""); return; }, }; } else r: { const real_parent = realPath(o.io, gpa, base, parent.?) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => { try o.problem(error.UnnormalizableAlternate, shown, ""); return; }, }; defer gpa.free(real_parent); break :r try std.fs.path.join(gpa, &.{ real_parent, name }); }; defer gpa.free(resolved); var dir = Io.Dir.openDirAbsolute(o.io, resolved, .{}) catch { try o.problem(error.MissingAlternate, resolved, ""); return; }; var owned = true; defer if (owned) dir.close(o.io); const real = realPath(o.io, gpa, base, resolved) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => try gpa.dupe(u8, resolved), }; defer gpa.free(real); for (o.directories.items) |d| if (std.mem.eql(u8, d.real_path, real)) return; owned = false; try o.addDirectory(dir, resolved, real, depth, false); }
/// The real path of `path`, relative to `base` unless it is absolute, /// as a plain slice the caller frees. fn realPath(io: Io, gpa: Allocator, base: Io.Dir, path: []const u8) ![]u8 { const z = try base.realPathFileAlloc(io, path, gpa); defer gpa.free(z); return gpa.dupe(u8, z); }
/// Finds and opens the directory's packs and its multi-pack-index. fn loadPacks(o: *Opener, d: *Directory) OpenError!void { const gpa = o.gpa; const io = o.io; var pack_dir = d.loose.dir.openDir(io, "pack", .{ .iterate = true }) catch return; defer pack_dir.close(io);
var names: std.ArrayList([]u8) = .empty; defer { for (names.items) |n| gpa.free(n); names.deinit(gpa); } var have_idx: std.StringHashMapUnmanaged(void) = .empty; defer { var keys = have_idx.keyIterator(); while (keys.next()) |k| gpa.free(k.*); have_idx.deinit(gpa); } var it = pack_dir.iterate(); while (it.next(io) catch |err| switch (err) { error.Canceled => return error.Canceled, else => null, }) |entry| { if (std.mem.endsWith(u8, entry.name, ".pack") and std.mem.startsWith(u8, entry.name, "pack-")) { try names.append(gpa, try gpa.dupe(u8, entry.name[0 .. entry.name.len - 5])); } else if (std.mem.endsWith(u8, entry.name, ".idx")) { const key = try gpa.dupe(u8, entry.name[0 .. entry.name.len - 4]); const gop = have_idx.getOrPut(gpa, key) catch |err| { gpa.free(key); return err; }; if (gop.found_existing) gpa.free(key); } }
var list: std.ArrayList(PackFile) = .empty; defer list.deinit(gpa); errdefer for (list.items) |*p| { p.pack_map.close(io); p.idx_map.close(io); gpa.free(p.name); gpa.free(p.pack_path); gpa.free(p.idx_path); }; // The multi-pack-index first, since it says which packs need an // index and which do not. var midx_names: std.StringHashMapUnmanaged(void) = .empty; defer midx_names.deinit(gpa); var layers: std.ArrayList(MidxLayer) = .empty; defer layers.deinit(gpa); errdefer for (layers.items) |*l| { l.map.close(io); gpa.free(l.packs); }; if (o.options.use_multi_pack_index) try o.loadMidx(d, pack_dir, &layers); for (layers.items) |*l| { var names_it = l.midx.packNames(); while (names_it.next()) |n| { const base_name = if (std.mem.endsWith(u8, n, ".idx")) n[0 .. n.len - 4] else n; try midx_names.put(gpa, base_name, {}); } }
for (names.items) |name| { const named_by_midx = midx_names.contains(name); if (!have_idx.contains(name) and !named_by_midx) continue; try list.ensureUnusedCapacity(gpa, 1); const pack_path = try std.fmt.allocPrint(gpa, "{s}/pack/{s}.pack", .{ d.path, name }); errdefer gpa.free(pack_path); const idx_path = try std.fmt.allocPrint(gpa, "{s}/pack/{s}.idx", .{ d.path, name }); errdefer gpa.free(idx_path); const owned_name = try gpa.dupe(u8, name); errdefer gpa.free(owned_name); var file_name_buf: [256]u8 = undefined; const mtime: i64 = if (pack_dir.statFile(io, std.fmt.bufPrint(&file_name_buf, "{s}.pack", .{name}) catch continue, .{})) |st| @intCast(@divFloor(st.mtime.nanoseconds, std.time.ns_per_s)) else |_| 0; var pf: PackFile = .{ .name = owned_name, .pack_path = pack_path, .idx_path = idx_path, .mtime = mtime, .in_midx = named_by_midx, .state = undefined, }; pf.state = o.openPack(&pf, pack_dir, have_idx.contains(name)); list.appendAssumeCapacity(pf); } // Most recently modified first; a stable sort keeps ties in // directory order, as git does. std.mem.sort(PackFile, list.items, {}, struct { fn newer(_: void, a: PackFile, b: PackFile) bool { return a.mtime > b.mtime; } }.newer); for (layers.items) |*l| { var names_it = l.midx.packNames(); var k: usize = 0; while (names_it.next()) |n| : (k += 1) { const base_name = if (std.mem.endsWith(u8, n, ".idx")) n[0 .. n.len - 4] else n; l.packs[k] = for (list.items, 0..) |p, i| { if (std.mem.eql(u8, p.name, base_name)) break i; } else null; } } d.packs = try list.toOwnedSlice(gpa); d.midx_layers = try layers.toOwnedSlice(gpa); }
fn openPack(o: *Opener, pf: *PackFile, pack_dir: Io.Dir, has_idx: bool) PackFile.State { const io = o.io; var name_buf: [256]u8 = undefined; if (!has_idx) return .{ .bad_index = .{ .err = error.PackIndexMissing } }; pf.idx_map = Mapped.open(io, pack_dir, std.fmt.bufPrint(&name_buf, "{s}.idx", .{pf.name}) catch unreachable) catch return .{ .bad_index = .{ .err = error.PackIndexMissing } }; var diag: Diagnostics = .{}; const index = PackIndex.parse(o.options.format, pf.idx_map.bytes, &diag) catch |err| return .{ .bad_index = failure(err, &diag) }; pf.pack_map = Mapped.open(io, pack_dir, std.fmt.bufPrint(&name_buf, "{s}.pack", .{pf.name}) catch unreachable) catch return .{ .bad_pack = .{ .index = index, .failure = .{ .err = error.PackTooSmall } } }; var p = pack.Pack.init(o.options.format, pf.pack_map.bytes, index, pf.pack_path, &diag) catch |err| return .{ .bad_pack = .{ .index = index, .failure = failure(err, &diag) } }; p.max_object_size = o.options.max_object_size; return .{ .ok = p }; }
fn failure(err: PackOpenError, diag: *const Diagnostics) PackFile.Failure { var f: PackFile.Failure = .{ .err = err }; const n = @min(diag.detail().len, f.detail.len); @memcpy(f.detail[0..n], diag.detail()[0..n]); f.detail_len = n; return f; }
/// Reads the directory's multi-pack-index, or its incremental chain, /// into `layers`; a broken one is a problem, or fatal where git stops. fn loadMidx(o: *Opener, d: *Directory, pack_dir: Io.Dir, layers: *std.ArrayList(MidxLayer)) OpenError!void { const gpa = o.gpa; const io = o.io; const path = try std.fmt.allocPrint(gpa, "{s}/pack/multi-pack-index", .{d.path}); defer gpa.free(path); if (Mapped.open(io, pack_dir, "multi-pack-index")) |map| { var m = map; if (try o.parseLayer(m, path, null)) |midx| { try o.appendLayer(layers, m, midx); } else m.close(io); return; } else |_| {}
// An incremental chain: one layer per line of the chain file, // oldest first. var buffer: [16 * 1024]u8 = undefined; const chain = pack_dir.readFile(io, "multi-pack-index.d/multi-pack-index-chain", &buffer) catch return; var lines = std.mem.tokenizeScalar(u8, chain, '\n'); var loaded: std.ArrayList(MidxLayer) = .empty; defer loaded.deinit(gpa); errdefer for (loaded.items) |*l| { l.map.close(io); gpa.free(l.packs); }; while (lines.next()) |line| { var name_buf: [128]u8 = undefined; const file = std.fmt.bufPrint(&name_buf, "multi-pack-index.d/multi-pack-index-{s}.midx", .{line}) catch break; const layer_path = try std.fmt.allocPrint(gpa, "{s}/pack/{s}", .{ d.path, file }); defer gpa.free(layer_path); var m = Mapped.open(io, pack_dir, file) catch { try o.problem(error.MultiPackIndexTooSmall, layer_path, ""); break; }; const midx = try o.parseLayer(m, layer_path, line) orelse { m.close(io); break; }; try o.appendLayer(&loaded, m, midx); } else { try layers.appendSlice(gpa, loaded.items); loaded.clearRetainingCapacity(); return; } // A broken layer: the chain is not used. for (loaded.items) |*l| { l.map.close(io); gpa.free(l.packs); } loaded.clearRetainingCapacity(); }
fn appendLayer(o: *Opener, layers: *std.ArrayList(MidxLayer), map: Mapped, midx: MultiPackIndex) Allocator.Error!void { const numbers = try o.gpa.alloc(?usize, midx.pack_count); errdefer o.gpa.free(numbers); @memset(numbers, null); try layers.append(o.gpa, .{ .map = map, .midx = midx, .packs = numbers }); }
/// Parses one multi-pack-index, returning null for a problem git goes /// on from and failing for one it stops at. fn parseLayer(o: *Opener, map: Mapped, path: []const u8, chain_name: ?[]const u8) OpenError!?MultiPackIndex { var diag: Diagnostics = .{}; const midx = MultiPackIndex.parse(o.options.format, map.bytes, &diag) catch |err| { if (MultiPackIndex.isFatal(err)) { if (o.diag) |dg| { dg.setPath(&.{path}); dg.setDetail("{s}", .{diag.detail()}); } return err; } try o.problem(err, path, diag.detail()); return null; }; if (chain_name) |name| { var hex: [object.max_hex_length]u8 = undefined; const sum: Id = .fromBytes(o.options.format, midx.checksum()); if (!std.ascii.eqlIgnoreCase(sum.toHex(&hex), name)) { try o.problem(error.MultiPackIndexChainMismatch, path, ""); return null; } } return midx; }};
/// The database as an `ObjectDatabase`.pub fn database(db: *DiskDatabase) ObjectDatabase { return .{ .ptr = db, .vtable = &vtable };}
const vtable: ObjectDatabase.VTable = .{ .contains = vtableContains, .readHeader = vtableReadHeader, .read = vtableRead, .stream = vtableStream,};
fn vtableContains(ptr: *anyopaque, io: Io, id: Id, diag: ?*Diagnostics) ContainsError!bool { const db: *DiskDatabase = @ptrCast(@alignCast(ptr)); return db.contains(io, id, diag);}
fn vtableReadHeader(ptr: *anyopaque, io: Io, id: Id, diag: ?*Diagnostics) ReadError!object.Header { const db: *DiskDatabase = @ptrCast(@alignCast(ptr)); return db.readHeaderAt(io, id, .{}, diag);}
fn vtableRead(ptr: *anyopaque, io: Io, gpa: Allocator, id: Id, diag: ?*Diagnostics) ReadError!Object { const db: *DiskDatabase = @ptrCast(@alignCast(ptr)); return db.readAt(io, gpa, id, .{}, diag);}
fn vtableStream(ptr: *anyopaque, io: Io, gpa: Allocator, id: Id, out: *Io.Writer, diag: ?*Diagnostics) StreamError!object.Header { const db: *DiskDatabase = @ptrCast(@alignCast(ptr)); return db.stream(io, gpa, id, out, diag);}
fn bases(db: *DiskDatabase) ?pack.Bases { if (!db.resolve_external_bases) return null; return .{ .ptr = db, .vtable = &bases_vtable };}
const bases_vtable: pack.Bases.VTable = .{ .readHeader = basesReadHeader, .read = basesRead,};
fn basesReadHeader(ptr: *anyopaque, io: Io, id: Id, depth: pack.Depth, diag: ?*Diagnostics) ReadError!object.Header { const db: *DiskDatabase = @ptrCast(@alignCast(ptr)); return db.readHeaderAt(io, id, depth, diag);}
fn basesRead(ptr: *anyopaque, io: Io, gpa: Allocator, id: Id, depth: pack.Depth, diag: ?*Diagnostics) ReadError!Object { const db: *DiskDatabase = @ptrCast(@alignCast(ptr)); return db.readAt(io, gpa, id, depth, diag);}
/// Where a packed object was found.const Found = struct { file: *const PackFile, pack: *const pack.Pack, offset: u64 };
/// The first error met while looking, which is the one returned if no/// copy of the object turns up, with what the diagnostics said of it.const FirstError = struct { err: ?ReadError = null, diag: Diagnostics = .{},
fn note(f: *FirstError, err: ReadError, d: *const Diagnostics) void { if (f.err != null) return; f.err = err; f.diag = d.*; }
fn noteOpen(f: *FirstError, err: PackOpenError, path: []const u8, failure: *const PackFile.Failure, id: Id) void { if (f.err != null) return; f.err = err; f.diag = .{}; f.diag.object_id = id; f.diag.setPath(&.{path}); f.diag.setDetail("{s}", .{failure.detail[0..failure.detail_len]}); }
fn finish(f: *const FirstError, diag: ?*Diagnostics, id: Id) ReadError { if (f.err) |err| { if (diag) |d| d.* = f.diag; return err; } if (diag) |d| d.object_id = id; return error.ObjectNotFound; }};
/// Steps through where a directory's packs hold an object, in the order/// git asks them, noting each pack that could not be asked.const PackSearch = struct { dir: *const Directory, id: Id, layer: usize = 0, in_layers: bool = true, next_pack: usize = 0,
fn next(s: *PackSearch, first: *FirstError, scratch: *Diagnostics) ?Found { const d = s.dir; if (s.in_layers) { while (s.layer < d.midx_layers.len) { const layer = &d.midx_layers[s.layer]; s.layer += 1; const pos = layer.midx.find(s.id) orelse continue; scratch.* = .{}; scratch.object_id = s.id; const loc = layer.midx.locationAt(pos, scratch) catch |err| { scratch.setPath(&.{d.path}); first.note(err, scratch); continue; }; const i = layer.packs[loc.pack] orelse continue; const pf = &d.packs[i]; switch (pf.state) { .ok => |*p| return .{ .file = pf, .pack = p, .offset = loc.offset }, .bad_index => |*f| first.noteOpen(f.err, if (f.err == error.PackIndexMissing) pf.pack_path else pf.idx_path, f, s.id), .bad_pack => |*b| first.noteOpen(b.failure.err, pf.pack_path, &b.failure, s.id), } } s.in_layers = false; } while (s.next_pack < d.packs.len) { const pf = &d.packs[s.next_pack]; s.next_pack += 1; if (pf.in_midx and d.midx_layers.len > 0) continue; switch (pf.state) { .ok => |*p| { scratch.* = .{}; scratch.object_id = s.id; scratch.setPath(&.{pf.idx_path}); const offset = p.find(s.id, scratch) catch |err| { first.note(err, scratch); continue; } orelse continue; return .{ .file = pf, .pack = p, .offset = offset }; }, .bad_index => |*f| if (f.err != error.PackIndexMissing) first.noteOpen(f.err, pf.idx_path, f, s.id), .bad_pack => |*b| if (b.index.find(s.id) != null) first.noteOpen(b.failure.err, pf.pack_path, &b.failure, s.id), } } return null; }};
/// Whether the database holds the object, without reading it, as/// `ObjectDatabase.contains` describes.pub fn contains(db: *DiskDatabase, io: Io, id: Id, diag: ?*Diagnostics) ContainsError!bool { var first: FirstError = .{}; var scratch: Diagnostics = .{}; for (db.directories) |d| { var search: PackSearch = .{ .dir = d, .id = id }; if (search.next(&first, &scratch)) |_| return true; if (try d.loose.contains(io, id)) return true; } if (first.err) |err| { if (diag) |dg| dg.* = first.diag; return switch (err) { error.PackIndexTooSmall, error.UnsupportedPackIndexVersion, error.NonMonotonicPackIndex, error.WrongPackIndexSize, error.PackIndexMissing, error.PackTooSmall, error.NotAPackfile, error.UnsupportedPackVersion, error.PackDoesNotMatchIndex, error.PackObjectCountMismatch, => |e| e, // An entry that could not be located is there all the same. else => true, }; } return false;}
/// The object's kind and size; see `ObjectDatabase.readHeader`. `depth`/// is how far resolving a delta chain had gone to get here, for a base.pub fn readHeaderAt(db: *DiskDatabase, io: Io, id: Id, depth: pack.Depth, diag: ?*Diagnostics) ReadError!object.Header { var first: FirstError = .{}; var scratch: Diagnostics = .{}; for (db.directories) |d| { var search: PackSearch = .{ .dir = d, .id = id }; while (search.next(&first, &scratch)) |found| { scratch = .{}; scratch.object_id = id; scratch.setPath(&.{found.file.pack_path}); if (found.pack.readHeader(io, found.offset, db.bases(), depth, &scratch)) |h| return h else |err| { if (err == error.OutOfMemory) return err; first.note(err, &scratch); } } scratch = .{}; if (d.loose.readHeader(io, id, &scratch)) |h| return h else |err| switch (err) { error.ObjectNotFound => {}, error.OutOfMemory => return err, else => first.note(err, &scratch), } } return first.finish(diag, id);}
/// The whole object; see `ObjectDatabase.read`. `depth` is how far/// resolving a delta chain had gone to get here, for a base.pub fn readAt(db: *DiskDatabase, io: Io, gpa: Allocator, id: Id, depth: pack.Depth, diag: ?*Diagnostics) ReadError!Object { var first: FirstError = .{}; var scratch: Diagnostics = .{}; for (db.directories) |d| { var search: PackSearch = .{ .dir = d, .id = id }; while (search.next(&first, &scratch)) |found| { scratch = .{}; scratch.object_id = id; scratch.setPath(&.{found.file.pack_path}); if (found.pack.read(io, gpa, found.offset, db.bases(), depth, &scratch)) |o| return o else |err| { if (err == error.OutOfMemory) return err; first.note(err, &scratch); } } scratch = .{}; if (d.loose.read(io, gpa, id, &scratch)) |o| return o else |err| switch (err) { error.ObjectNotFound => {}, error.OutOfMemory => return err, else => first.note(err, &scratch), } } return first.finish(diag, id);}
/// Writes the object's contents to `out`; see `ObjectDatabase.stream`. As/// git does, it looks for a loose copy in every directory before any/// packed one, and does not go on to another copy once it has begun.pub fn stream(db: *DiskDatabase, io: Io, gpa: Allocator, id: Id, out: *Io.Writer, diag: ?*Diagnostics) StreamError!object.Header { for (db.directories) |d| { if (d.loose.stream(io, gpa, id, out, diag)) |h| return h else |err| switch (err) { error.ObjectNotFound => {}, else => return err, } } var first: FirstError = .{}; var scratch: Diagnostics = .{}; for (db.directories) |d| { var search: PackSearch = .{ .dir = d, .id = id }; if (search.next(&first, &scratch)) |found| { if (diag) |dg| { dg.object_id = id; dg.setPath(&.{found.file.pack_path}); } return found.pack.stream(io, gpa, found.offset, out, db.bases(), .{}, diag); } } return first.finish(diag, id);}
const testing = std.testing;const fixture = @import("testing/fixture.zig");const packs = @import("testing/packs.zig");
fn openRepo(repo: *fixture.Repo, options: struct { alternate_list: ?[]const u8 = null }) !DiskDatabase { return open(repo.gpa, repo.io, .{ .objects_path = ".git/objects", .base = repo.tmp.dir, .format = repo.object_format, .alternate_list = options.alternate_list, }, null);}
fn revParse(repo: *fixture.Repo, rev: []const u8) !Id { const out = try repo.output(&.{ "rev-parse", rev }); defer repo.gpa.free(out); return Id.parseHex(repo.object_format, std.mem.trimEnd(u8, out, "\n"));}
/// Checks that every object `git rev-list --objects --all` lists reads/// through `db` as `git cat-file` reads it.fn expectAllObjectsAsGit(repo: *fixture.Repo, db: *DiskDatabase) !void { const gpa = repo.gpa; const listing = try repo.output(&.{ "rev-list", "--objects", "--all" }); defer gpa.free(listing); var lines = std.mem.tokenizeScalar(u8, listing, '\n'); var n: usize = 0; while (lines.next()) |line| : (n += 1) { const hex = line[0..repo.object_format.hexLength()]; const id = try Id.parseHex(repo.object_format, hex); const kind_name = try repo.output(&.{ "cat-file", "-t", hex }); defer gpa.free(kind_name); const want = try repo.output(&.{ "cat-file", std.mem.trimEnd(u8, kind_name, "\n"), hex }); defer gpa.free(want); errdefer std.debug.print("object {s}\n", .{hex}); const odb = db.database(); try testing.expect(try odb.contains(repo.io, id, null)); const header = try odb.readHeader(repo.io, id, null); try testing.expectEqualStrings(std.mem.trimEnd(u8, kind_name, "\n"), header.kind.name()); try testing.expectEqual(want.len, header.size); var obj = try odb.read(repo.io, gpa, id, null); defer obj.deinit(gpa); try testing.expectEqualSlices(u8, want, obj.data); var aw: Io.Writer.Allocating = .init(gpa); defer aw.deinit(); _ = try odb.stream(repo.io, gpa, id, &aw.writer, null); try testing.expectEqualSlices(u8, want, aw.written()); } try testing.expect(n > 0);}
fn makeHistory(repo: *fixture.Repo, commits: usize) !void { const gpa = repo.gpa; var text: std.ArrayList(u8) = .empty; defer text.deinit(gpa); for (0..300) |i| try text.print(gpa, "line {d}\n", .{i}); for (0..commits) |i| { const at = (i * 53) % (text.items.len - 10); text.items[at] = 'a' + @as(u8, @intCast(i % 26)); try repo.commitFile("f", text.items, "c"); var buf: [32]u8 = undefined; try repo.commitFile("g", try std.fmt.bufPrint(&buf, "{d}\n", .{i}), "g"); }}
test "loose objects, packs and a multi-pack-index all read as git reads them" { const gpa = testing.allocator; var repo = try fixture.Repo.init(gpa, testing.io); defer repo.deinit(); try makeHistory(&repo, 6); try repo.git(&.{ "repack", "-dq" }); try makeHistory(&repo, 4); try repo.git(&.{ "repack", "-dq" }); try makeHistory(&repo, 3); // and some left loose { var db = try openRepo(&repo, .{}); defer db.close(testing.io); try testing.expectEqual(2, db.directories[0].packs.len); try expectAllObjectsAsGit(&repo, &db); } try repo.git(&.{ "multi-pack-index", "write" }); try makeHistory(&repo, 2); try repo.git(&.{ "repack", "-dq" }); // a pack the multi-pack-index does not name { var db = try openRepo(&repo, .{}); defer db.close(testing.io); try testing.expectEqual(1, db.directories[0].midx_layers.len); try testing.expectEqual(0, db.problems.len); try expectAllObjectsAsGit(&repo, &db); } try repo.git(&.{ "repack", "-adfq", "--depth=20" }); { var db = try openRepo(&repo, .{}); defer db.close(testing.io); try expectAllObjectsAsGit(&repo, &db); }}
test "an incremental multi-pack-index chain is read layer by layer" { const gpa = testing.allocator; var repo = try fixture.Repo.init(gpa, testing.io); defer repo.deinit(); try makeHistory(&repo, 3); try repo.git(&.{ "repack", "-dq" }); try repo.git(&.{ "multi-pack-index", "write", "--incremental" }); try makeHistory(&repo, 3); try repo.git(&.{ "repack", "-dq" }); try repo.git(&.{ "multi-pack-index", "write", "--incremental" }); var db = try openRepo(&repo, .{}); defer db.close(testing.io); try testing.expectEqual(2, db.directories[0].midx_layers.len); try testing.expectEqual(0, db.problems.len); try expectAllObjectsAsGit(&repo, &db);}
test "alternates are followed as git follows them" { const gpa = testing.allocator; var repo = try fixture.Repo.init(gpa, testing.io); defer repo.deinit(); try makeHistory(&repo, 3); try repo.git(&.{ "repack", "-adq" }); try makeHistory(&repo, 1); // A clone borrowing everything through objects/info/alternates. try repo.git(&.{ "clone", "-q", "--shared", ".", "shared" }); // The clone's own directory, with the original as its alternate. var db = try open(gpa, testing.io, .{ .objects_path = "shared/.git/objects", .base = repo.tmp.dir, .format = repo.object_format, }, null); defer db.close(testing.io); try testing.expectEqual(2, db.directories.len); try testing.expectEqual(0, db.problems.len); try expectAllObjectsAsGit(&repo, &db);}
test "a REF_DELTA's base outside its pack is found only when asked for" { const gpa = testing.allocator; var repo = try fixture.Repo.init(gpa, testing.io); defer repo.deinit(); const f = repo.object_format; const base = "the base\n"; const loose_base = "a loose base\n"; const loose_hex = try repo.hashObject("blob", loose_base); defer gpa.free(loose_hex); const in_other = base ++ "one\n"; const Case = struct { base_id: Id, base_text: []const u8, suffix: []const u8 }; const cases = [_]Case{ .{ .base_id = object.hash(f, .blob, base), .base_text = base, .suffix = "one\n" }, .{ .base_id = try Id.parseHex(f, loose_hex), .base_text = loose_base, .suffix = "two\n" }, .{ .base_id = object.hash(f, .blob, in_other), .base_text = in_other, .suffix = "three\n" }, .{ .base_id = object.hash(f, .blob, "nowhere"), .base_text = "nowhere", .suffix = "four\n" }, }; { var w: packs.PackWriter = try .init(gpa, f); defer w.deinit(); _ = try w.whole(.blob, base); const out = try w.finish(); defer gpa.free(out.pack); defer gpa.free(out.idx); gpa.free(try packs.install(&repo, out.pack, out.idx)); } var ids: [cases.len]Id = undefined; for (cases, &ids) |case, *id| { var w: packs.PackWriter = try .init(gpa, f); defer w.deinit(); const d = try packs.deltaAppend(gpa, @intCast(case.base_text.len), case.suffix); defer gpa.free(d); const result = try std.mem.concat(gpa, u8, &.{ case.base_text, case.suffix }); defer gpa.free(result); id.* = object.hash(f, .blob, result); _ = try w.refDelta(case.base_id, d, id.*); const out = try w.finish(); defer gpa.free(out.pack); defer gpa.free(out.idx); gpa.free(try packs.install(&repo, out.pack, out.idx)); } // Two REF_DELTAs in packs of their own, each the other's base. const x = object.hash(f, .blob, "x"); const y = object.hash(f, .blob, "y"); for ([_][2]Id{ .{ x, y }, .{ y, x } }) |pair| { var w: packs.PackWriter = try .init(gpa, f); defer w.deinit(); const d = try packs.deltaInsert(gpa, 1, "x"); defer gpa.free(d); _ = try w.refDelta(pair[1], d, pair[0]); const out = try w.finish(); defer gpa.free(out.pack); defer gpa.free(out.idx); gpa.free(try packs.install(&repo, out.pack, out.idx)); }
{ // As git reads them: none is found. var db = try openRepo(&repo, .{}); defer db.close(testing.io); const odb = db.database(); for (ids) |id| { try testing.expect(try odb.contains(testing.io, id, null)); var diag: Diagnostics = .{}; try testing.expectError(error.DeltaBaseMissing, odb.readHeader(testing.io, id, &diag)); try testing.expectError(error.DeltaBaseMissing, odb.read(testing.io, gpa, id, null)); } } var db = try open(gpa, testing.io, .{ .objects_path = ".git/objects", .base = repo.tmp.dir, .format = f, .resolve_external_bases = true, }, null); defer db.close(testing.io); const odb = db.database(); for (cases[0..3], ids[0..3]) |case, id| { var obj = try odb.read(testing.io, gpa, id, null); defer obj.deinit(gpa); const want = try std.mem.concat(gpa, u8, &.{ case.base_text, case.suffix }); defer gpa.free(want); try testing.expectEqualStrings(want, obj.data); try testing.expectEqual(want.len, (try odb.readHeader(testing.io, id, null)).size); } var diag: Diagnostics = .{}; try testing.expectError(error.DeltaBaseMissing, odb.read(testing.io, gpa, ids[3], &diag)); var hex: [object.max_hex_length]u8 = undefined; try testing.expectEqualStrings(cases[3].base_id.toHex(&hex), diag.detail()); try testing.expectError(error.DeltaChainTooDeep, odb.read(testing.io, gpa, x, null)); try testing.expectError(error.DeltaChainTooDeep, odb.readHeader(testing.io, x, null));}
test "an object over max_object_size is refused, one at it is read, and null is no limit" { const gpa = testing.allocator; var repo = try fixture.Repo.init(gpa, testing.io); defer repo.deinit(); const contents = "exactly thirty-two bytes long!!\n"; // Committed, so that repacking packs it. try repo.commitFile("f", contents, "c"); const hex = try repo.hashObject("blob", contents); defer gpa.free(hex); const id = try Id.parseHex(repo.object_format, hex); // Loose, and then packed. for (0..2) |round| { if (round == 1) try repo.git(&.{ "repack", "-adq" }); for ([_]?u64{ contents.len, contents.len - 1, null }) |max| { var db = try open(gpa, testing.io, .{ .objects_path = ".git/objects", .base = repo.tmp.dir, .format = repo.object_format, .max_object_size = max, }, null); defer db.close(testing.io); const odb = db.database(); errdefer std.debug.print("round {d}, limit {?d}\n", .{ round, max }); // The header reports the size whatever the limit. try testing.expectEqual(contents.len, (try odb.readHeader(testing.io, id, null)).size); if (max != null and max.? < contents.len) { var diag: Diagnostics = .{}; try testing.expectError(error.ObjectTooLarge, odb.read(testing.io, gpa, id, &diag)); try testing.expect(Id.eql(id, diag.object_id.?)); } else { var obj = try odb.read(testing.io, gpa, id, null); defer obj.deinit(gpa); try testing.expectEqualStrings(contents, obj.data); } // Streaming an object stored whole is never limited. var aw: Io.Writer.Allocating = .init(gpa); defer aw.deinit(); _ = try odb.stream(testing.io, gpa, id, &aw.writer, null); try testing.expectEqualStrings(contents, aw.written()); } } // The default is a gigabyte. var db = try openRepo(&repo, .{}); defer db.close(testing.io); try testing.expectEqual(default_max_object_size, db.max_object_size.?);}