// SPDX-FileCopyrightText: © 2026 Jeffrey C. Ollie // SPDX-License-Identifier: MIT //! A repository's object database on disk: its own objects directory and //! those of its alternates, each with its loose objects, its packs and its //! multi-pack-index, behind the one `ObjectDatabase` interface the rest of //! the library reads objects through. //! //! What is found where, and in what order, follows what git can be seen //! to do, found by putting different copies of one object in different //! places and asking `git cat-file` which it read: //! //! - **Objects directories** are asked in order: the repository's own, //! then those `GIT_ALTERNATE_OBJECT_DIRECTORIES` names, then those its //! `objects/info/alternates` names, each alternate's own alternates //! following it. A directory named twice, by whatever path, is asked //! once. An alternate's alternates are followed to a depth of five; //! past that git ignores them, and says so. //! - **Within a directory**, packs come before loose objects: those the //! multi-pack-index names through it, then the rest, the most recently //! modified pack first, ties in directory order. //! - **Streaming** is the exception: git looks for a loose object in //! every directory before any pack, and so does `stream`. //! - **A copy that cannot be read** is passed over for the next one, as //! git passes over it, and the error is returned only if no copy can //! be read. A pack that cannot be opened is passed over the same way, //! and its error returned in place of `ObjectNotFound` if the object is //! found nowhere else, since it might have been in that pack. //! //! Everything is found and opened, packs and indexes memory-mapped, when //! the database is opened; afterwards nothing about it changes, so any //! number of readers may share it at once. A pack written after opening //! is not seen until it is opened again. //! //! **A REF_DELTA whose base is not in its own pack** is not resolved by //! git: such an object reads as missing, without a word, even when the //! base is in another pack or loose. A pack git wrote never has one, since //! `git index-pack --fix-thin` appends the bases a thin pack leaves out. //! By default this agrees, and such a copy fails with `DeltaBaseMissing`; //! `Options.resolve_external_bases` looks for the base in the whole //! database instead. //! //! **Objects are limited in size** when read whole: by default to a //! gigabyte (`Options.max_object_size`), which git is not. See there. //! //! The library never reads the environment, so the value of //! `GIT_ALTERNATE_OBJECT_DIRECTORIES` is an option, and `core.multiPackIndex` //! another. const std = @import("std"); const Allocator = std.mem.Allocator; const Io = std.Io; const object = @import("object.zig"); const Format = object.Format; const Id = object.Id; const Diagnostics = @import("Diagnostics.zig"); const ObjectDatabase = @import("ObjectDatabase.zig"); const Object = ObjectDatabase.Object; const ReadError = ObjectDatabase.ReadError; const StreamError = ObjectDatabase.StreamError; const ContainsError = ObjectDatabase.ContainsError; const PackOpenError = ObjectDatabase.PackOpenError; const loose = @import("loose.zig"); const pack = @import("pack.zig"); const PackIndex = @import("PackIndex.zig"); const MultiPackIndex = @import("MultiPackIndex.zig"); const alternates = @import("alternates.zig"); const DiskDatabase = @This(); gpa: Allocator, format: Format, /// Whether a REF_DELTA's base is looked for outside its pack. resolve_external_bases: bool, /// The most bytes an object read whole may claim, or null for no limit. max_object_size: ?u64, /// The objects directories, in the order they are asked. directories: []*Directory, /// What was wrong but worked around while opening, in the order met. problems: []Problem, /// The default for `Options.max_object_size`: a gigabyte. pub const default_max_object_size: u64 = 1 << 30; /// How deep alternates of alternates are followed: a directory this many /// steps from the repository's own does not have its alternates read. pub const max_alternate_depth = 5; /// How to open a repository's object database. pub const Options = struct { /// The repository's objects directory, relative to `base` or /// absolute. Diagnostics name its files by joining this path with /// theirs, as git names them. objects_path: []const u8, /// What relative paths are relative to. base: Io.Dir, /// The repository's object format. format: Format, /// The value of `GIT_ALTERNATE_OBJECT_DIRECTORIES`, if the caller has /// one: paths separated by `alternate_separator`. alternate_list: ?[]const u8 = null, /// What relative paths in `alternate_list` are relative to: the top of /// the working tree, where git runs; `base` if null. alternate_list_base: ?Io.Dir = null, /// What separates paths in `alternate_list`. alternate_separator: u8 = alternates.list_separator, /// Whether to read multi-pack-indexes, as `core.multiPackIndex` says. use_multi_pack_index: bool = true, /// Whether a REF_DELTA whose base is not in its own pack has the base /// looked for in the rest of the database, as git does not. resolve_external_bases: bool = false, /// The most bytes an object may claim to be when it is built in /// memory, or null for no limit. Over it, a read is `ObjectTooLarge`, /// found before anything of that size is allocated: the size a loose /// object's header claims, a pack entry's size, and the result size /// each delta of a chain declares, at every link. (A delta's own /// instructions may take up to eight times the limit, since a copy /// instruction of eight bytes may make only one.) /// /// The limit exists because those sizes are claims, and believing one /// is what lets a few bytes take all the memory there is: a delta of a /// few kilobytes copying 64 KiB at a time really does make gigabytes, /// and a chain of such deltas multiplies them, while still being a /// valid delta. git has no limit and will try to read such an object; /// the default of a gigabyte, `default_max_object_size`, is far above /// anything a source repository holds and still bounds what one read /// costs. A server reading repositories it does not trust should set it /// lower, to what it is willing to spend on one object. /// /// Only reads that build the object whole are limited. `readHeader` /// reports any size without refusing it, as `git cat-file -s` does, /// since it allocates nothing of it; `stream` writes an object stored /// whole however big it is, in bounded memory, and is limited only for /// a deltified object, which has to be built in memory first. max_object_size: ?u64 = default_max_object_size, }; /// Something wrong with the object database that git says and then works /// around, and so does this: an alternate that is not there, or nested /// too deep, or a multi-pack-index that cannot be read and so is not used. pub const Problem = struct { err: ProblemError, /// The path the problem is about. path: []u8, /// More about it, or empty. detail: []u8, }; /// The problems `Problem` records, each in git's words: /// /// - `UnnormalizableAlternate`: "unable to normalize alternate object /// path: …": a directory on the way to it does not exist. /// - `MissingAlternate`: "object directory … does not exist; check /// .git/objects/info/alternates". /// - `AlternatesTooDeep`: "…: ignoring alternate object stores, nesting /// too deep". /// - The errors of `MultiPackIndex.Error` that git goes on from, and /// `MultiPackIndexChainMismatch`: a layer of an incremental chain whose /// checksum is not the one its name says. pub const ProblemError = error{ UnnormalizableAlternate, MissingAlternate, AlternatesTooDeep, MultiPackIndexChainMismatch, } || MultiPackIndex.Error; /// Why the database could not be opened: the objects directory could not /// be, or one of its multi-pack-indexes is broken in a way git stops at. pub const OpenError = Allocator.Error || error{ ObjectsDirectoryUnreadable, Canceled } || MultiPackIndex.Error; /// One objects directory: its loose objects, its packs and its /// multi-pack-index. pub const Directory = struct { /// How to name it in diagnostics. path: []u8, /// Its real path, for telling whether another names the same one. real_path: []u8, loose: loose.Store, /// Most recently modified first. packs: []PackFile, /// The layers of its multi-pack-index: one, or an incremental chain. midx_layers: []MidxLayer, }; const MidxLayer = struct { map: Mapped, midx: MultiPackIndex, /// For each of the multi-pack-index's pack numbers, the pack in /// `Directory.packs`, or null if there is no such pack. packs: []?usize, }; /// A pack and its index, as far as they could be opened. pub const PackFile = struct { /// `pack-`. name: []u8, /// How to name the `.pack` and the `.idx` in diagnostics. pack_path: []u8, idx_path: []u8, mtime: i64, /// Whether a multi-pack-index finds its objects, in which case only it /// is asked. in_midx: bool = false, pack_map: Mapped = .empty, idx_map: Mapped = .empty, state: State, /// How far the pack could be opened. pub const State = union(enum) { ok: pack.Pack, /// The index could not be read, so nothing is known of the pack. bad_index: Failure, /// The index could be read but the pack could not be used. bad_pack: struct { index: PackIndex, failure: Failure }, }; /// Why, in the words a `Diagnostics` would have. pub const Failure = struct { err: PackOpenError, detail: [64]u8 = undefined, detail_len: usize = 0, }; }; /// A file memory-mapped read-only, or an empty one, which cannot be. const Mapped = struct { map: ?Io.File.MemoryMap, bytes: []const u8, const empty: Mapped = .{ .map = null, .bytes = &.{} }; fn open(io: Io, dir: Io.Dir, sub_path: []const u8) !Mapped { const file = try dir.openFile(io, sub_path, .{}); defer file.close(io); const size = try file.length(io); if (size == 0) return empty; const len = std.math.cast(usize, size) orelse return error.FileTooBig; const map = try Io.File.MemoryMap.create(io, file, .{ .len = len, .protection = .{ .read = true }, .populate = false, }); return .{ .map = map, .bytes = map.memory[0..len] }; } fn close(m: *Mapped, io: Io) void { if (m.map) |*map| map.destroy(io); m.* = empty; } }; /// Opens the object database whose own objects directory `options` /// names, with its alternates, packs and multi-pack-indexes. The result /// must stay where it is while `database()` is in use, and is closed with /// `close`. pub fn open(gpa: Allocator, io: Io, options: Options, diag: ?*Diagnostics) OpenError!DiskDatabase { var opener: Opener = .{ .gpa = gpa, .io = io, .options = options, .diag = diag }; errdefer opener.deinit(); const real = options.base.realPathFileAlloc(io, options.objects_path, gpa) catch |err| { if (diag) |d| { d.setPath(&.{options.objects_path}); d.setDetail("{t}", .{err}); } return error.ObjectsDirectoryUnreadable; }; defer gpa.free(real); const own = options.base.openDir(io, options.objects_path, .{}) catch |err| { if (diag) |d| { d.setPath(&.{options.objects_path}); d.setDetail("{t}", .{err}); } return error.ObjectsDirectoryUnreadable; }; // `addDirectory` takes the directory, and closes it if it fails. try opener.addDirectory(own, options.objects_path, real, 0, true); return .{ .gpa = gpa, .format = options.format, .resolve_external_bases = options.resolve_external_bases, .max_object_size = options.max_object_size, .directories = try opener.directories.toOwnedSlice(gpa), .problems = try opener.problems.toOwnedSlice(gpa), }; } /// Closes every file and frees everything. pub fn close(db: *DiskDatabase, io: Io) void { for (db.directories) |d| closeDirectory(db.gpa, io, d); db.gpa.free(db.directories); for (db.problems) |p| { db.gpa.free(p.path); db.gpa.free(p.detail); } db.gpa.free(db.problems); db.* = undefined; } fn closeDirectory(gpa: Allocator, io: Io, d: *Directory) void { for (d.midx_layers) |*layer| { layer.map.close(io); gpa.free(layer.packs); } gpa.free(d.midx_layers); for (d.packs) |*p| { p.pack_map.close(io); p.idx_map.close(io); gpa.free(p.name); gpa.free(p.pack_path); gpa.free(p.idx_path); } gpa.free(d.packs); d.loose.dir.close(io); gpa.free(d.path); gpa.free(d.real_path); gpa.destroy(d); } /// The state of opening a database, while it is being opened. const Opener = struct { gpa: Allocator, io: Io, options: Options, diag: ?*Diagnostics, directories: std.ArrayList(*Directory) = .empty, problems: std.ArrayList(Problem) = .empty, fn deinit(o: *Opener) void { for (o.directories.items) |d| closeDirectory(o.gpa, o.io, d); o.directories.deinit(o.gpa); for (o.problems.items) |p| { o.gpa.free(p.path); o.gpa.free(p.detail); } o.problems.deinit(o.gpa); } fn problem(o: *Opener, err: ProblemError, path: []const u8, detail: []const u8) Allocator.Error!void { const p = try o.gpa.dupe(u8, path); errdefer o.gpa.free(p); const d = try o.gpa.dupe(u8, detail); errdefer o.gpa.free(d); try o.problems.append(o.gpa, .{ .err = err, .path = p, .detail = d }); } /// Adds the objects directory `dir`, which it takes, then its /// alternates. fn addDirectory(o: *Opener, dir: Io.Dir, path: []const u8, real_path: []const u8, depth: u32, is_own: bool) OpenError!void { const gpa = o.gpa; var d = gpa.create(Directory) catch |err| { var dd = dir; dd.close(o.io); return err; }; d.* = .{ .path = &.{}, .real_path = &.{}, .loose = .{ .dir = dir, .path = &.{}, .format = o.options.format, .max_object_size = o.options.max_object_size }, .packs = &.{}, .midx_layers = &.{}, }; o.directories.append(gpa, d) catch |err| { d.loose.dir.close(o.io); gpa.destroy(d); return err; }; d.path = try gpa.dupe(u8, path); d.loose.path = d.path; d.real_path = try gpa.dupe(u8, real_path); try o.loadPacks(d); // Alternates named by the caller come before those the // repository names. if (is_own) if (o.options.alternate_list) |list| { const base = o.options.alternate_list_base orelse o.options.base; var it = alternates.iterateList(list, o.options.alternate_separator); defer it.deinit(gpa); while (try it.next(gpa)) |entry| try o.addAlternate(base, entry, entry, 1); }; var buffer: [64 * 1024]u8 = undefined; const contents = d.loose.dir.readFile(o.io, "info/alternates", &buffer) catch return; if (depth >= max_alternate_depth + 1) { try o.problem(error.AlternatesTooDeep, d.path, ""); return; } // A relative alternate is relative to this objects directory. var it = alternates.iterateFile(contents); defer it.deinit(gpa); while (try it.next(gpa)) |entry| { if (std.fs.path.isAbsolute(entry)) { try o.addAlternate(o.options.base, entry, entry, depth + 1); } else { const shown = try std.mem.concat(gpa, u8, &.{ d.real_path, "/", entry }); defer gpa.free(shown); try o.addAlternate(o.options.base, shown, shown, depth + 1); } } } /// Adds the alternate `path`, relative to `base` if it is not /// absolute, unless it is a directory already added. fn addAlternate(o: *Opener, base: Io.Dir, path: []const u8, shown: []const u8, depth: u32) OpenError!void { const gpa = o.gpa; var trimmed = path; while (trimmed.len > 1 and trimmed[trimmed.len - 1] == '/') trimmed = trimmed[0 .. trimmed.len - 1]; const parent = std.fs.path.dirname(trimmed); const name = std.fs.path.basename(trimmed); // The real path of the directory holding it, and then its own // name: a missing directory on the way is one thing, a missing // directory at the end another. const resolved: []u8 = if (parent == null or std.mem.eql(u8, name, ".") or std.mem.eql(u8, name, "..")) whole: { break :whole realPath(o.io, gpa, base, trimmed) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => { try o.problem(error.UnnormalizableAlternate, shown, ""); return; }, }; } else r: { const real_parent = realPath(o.io, gpa, base, parent.?) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => { try o.problem(error.UnnormalizableAlternate, shown, ""); return; }, }; defer gpa.free(real_parent); break :r try std.fs.path.join(gpa, &.{ real_parent, name }); }; defer gpa.free(resolved); var dir = Io.Dir.openDirAbsolute(o.io, resolved, .{}) catch { try o.problem(error.MissingAlternate, resolved, ""); return; }; var owned = true; defer if (owned) dir.close(o.io); const real = realPath(o.io, gpa, base, resolved) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => try gpa.dupe(u8, resolved), }; defer gpa.free(real); for (o.directories.items) |d| if (std.mem.eql(u8, d.real_path, real)) return; owned = false; try o.addDirectory(dir, resolved, real, depth, false); } /// The real path of `path`, relative to `base` unless it is absolute, /// as a plain slice the caller frees. fn realPath(io: Io, gpa: Allocator, base: Io.Dir, path: []const u8) ![]u8 { const z = try base.realPathFileAlloc(io, path, gpa); defer gpa.free(z); return gpa.dupe(u8, z); } /// Finds and opens the directory's packs and its multi-pack-index. fn loadPacks(o: *Opener, d: *Directory) OpenError!void { const gpa = o.gpa; const io = o.io; var pack_dir = d.loose.dir.openDir(io, "pack", .{ .iterate = true }) catch return; defer pack_dir.close(io); var names: std.ArrayList([]u8) = .empty; defer { for (names.items) |n| gpa.free(n); names.deinit(gpa); } var have_idx: std.StringHashMapUnmanaged(void) = .empty; defer { var keys = have_idx.keyIterator(); while (keys.next()) |k| gpa.free(k.*); have_idx.deinit(gpa); } var it = pack_dir.iterate(); while (it.next(io) catch |err| switch (err) { error.Canceled => return error.Canceled, else => null, }) |entry| { if (std.mem.endsWith(u8, entry.name, ".pack") and std.mem.startsWith(u8, entry.name, "pack-")) { try names.append(gpa, try gpa.dupe(u8, entry.name[0 .. entry.name.len - 5])); } else if (std.mem.endsWith(u8, entry.name, ".idx")) { const key = try gpa.dupe(u8, entry.name[0 .. entry.name.len - 4]); const gop = have_idx.getOrPut(gpa, key) catch |err| { gpa.free(key); return err; }; if (gop.found_existing) gpa.free(key); } } var list: std.ArrayList(PackFile) = .empty; defer list.deinit(gpa); errdefer for (list.items) |*p| { p.pack_map.close(io); p.idx_map.close(io); gpa.free(p.name); gpa.free(p.pack_path); gpa.free(p.idx_path); }; // The multi-pack-index first, since it says which packs need an // index and which do not. var midx_names: std.StringHashMapUnmanaged(void) = .empty; defer midx_names.deinit(gpa); var layers: std.ArrayList(MidxLayer) = .empty; defer layers.deinit(gpa); errdefer for (layers.items) |*l| { l.map.close(io); gpa.free(l.packs); }; if (o.options.use_multi_pack_index) try o.loadMidx(d, pack_dir, &layers); for (layers.items) |*l| { var names_it = l.midx.packNames(); while (names_it.next()) |n| { const base_name = if (std.mem.endsWith(u8, n, ".idx")) n[0 .. n.len - 4] else n; try midx_names.put(gpa, base_name, {}); } } for (names.items) |name| { const named_by_midx = midx_names.contains(name); if (!have_idx.contains(name) and !named_by_midx) continue; try list.ensureUnusedCapacity(gpa, 1); const pack_path = try std.fmt.allocPrint(gpa, "{s}/pack/{s}.pack", .{ d.path, name }); errdefer gpa.free(pack_path); const idx_path = try std.fmt.allocPrint(gpa, "{s}/pack/{s}.idx", .{ d.path, name }); errdefer gpa.free(idx_path); const owned_name = try gpa.dupe(u8, name); errdefer gpa.free(owned_name); var file_name_buf: [256]u8 = undefined; const mtime: i64 = if (pack_dir.statFile(io, std.fmt.bufPrint(&file_name_buf, "{s}.pack", .{name}) catch continue, .{})) |st| @intCast(@divFloor(st.mtime.nanoseconds, std.time.ns_per_s)) else |_| 0; var pf: PackFile = .{ .name = owned_name, .pack_path = pack_path, .idx_path = idx_path, .mtime = mtime, .in_midx = named_by_midx, .state = undefined, }; pf.state = o.openPack(&pf, pack_dir, have_idx.contains(name)); list.appendAssumeCapacity(pf); } // Most recently modified first; a stable sort keeps ties in // directory order, as git does. std.mem.sort(PackFile, list.items, {}, struct { fn newer(_: void, a: PackFile, b: PackFile) bool { return a.mtime > b.mtime; } }.newer); for (layers.items) |*l| { var names_it = l.midx.packNames(); var k: usize = 0; while (names_it.next()) |n| : (k += 1) { const base_name = if (std.mem.endsWith(u8, n, ".idx")) n[0 .. n.len - 4] else n; l.packs[k] = for (list.items, 0..) |p, i| { if (std.mem.eql(u8, p.name, base_name)) break i; } else null; } } d.packs = try list.toOwnedSlice(gpa); d.midx_layers = try layers.toOwnedSlice(gpa); } fn openPack(o: *Opener, pf: *PackFile, pack_dir: Io.Dir, has_idx: bool) PackFile.State { const io = o.io; var name_buf: [256]u8 = undefined; if (!has_idx) return .{ .bad_index = .{ .err = error.PackIndexMissing } }; pf.idx_map = Mapped.open(io, pack_dir, std.fmt.bufPrint(&name_buf, "{s}.idx", .{pf.name}) catch unreachable) catch return .{ .bad_index = .{ .err = error.PackIndexMissing } }; var diag: Diagnostics = .{}; const index = PackIndex.parse(o.options.format, pf.idx_map.bytes, &diag) catch |err| return .{ .bad_index = failure(err, &diag) }; pf.pack_map = Mapped.open(io, pack_dir, std.fmt.bufPrint(&name_buf, "{s}.pack", .{pf.name}) catch unreachable) catch return .{ .bad_pack = .{ .index = index, .failure = .{ .err = error.PackTooSmall } } }; var p = pack.Pack.init(o.options.format, pf.pack_map.bytes, index, pf.pack_path, &diag) catch |err| return .{ .bad_pack = .{ .index = index, .failure = failure(err, &diag) } }; p.max_object_size = o.options.max_object_size; return .{ .ok = p }; } fn failure(err: PackOpenError, diag: *const Diagnostics) PackFile.Failure { var f: PackFile.Failure = .{ .err = err }; const n = @min(diag.detail().len, f.detail.len); @memcpy(f.detail[0..n], diag.detail()[0..n]); f.detail_len = n; return f; } /// Reads the directory's multi-pack-index, or its incremental chain, /// into `layers`; a broken one is a problem, or fatal where git stops. fn loadMidx(o: *Opener, d: *Directory, pack_dir: Io.Dir, layers: *std.ArrayList(MidxLayer)) OpenError!void { const gpa = o.gpa; const io = o.io; const path = try std.fmt.allocPrint(gpa, "{s}/pack/multi-pack-index", .{d.path}); defer gpa.free(path); if (Mapped.open(io, pack_dir, "multi-pack-index")) |map| { var m = map; if (try o.parseLayer(m, path, null)) |midx| { try o.appendLayer(layers, m, midx); } else m.close(io); return; } else |_| {} // An incremental chain: one layer per line of the chain file, // oldest first. var buffer: [16 * 1024]u8 = undefined; const chain = pack_dir.readFile(io, "multi-pack-index.d/multi-pack-index-chain", &buffer) catch return; var lines = std.mem.tokenizeScalar(u8, chain, '\n'); var loaded: std.ArrayList(MidxLayer) = .empty; defer loaded.deinit(gpa); errdefer for (loaded.items) |*l| { l.map.close(io); gpa.free(l.packs); }; while (lines.next()) |line| { var name_buf: [128]u8 = undefined; const file = std.fmt.bufPrint(&name_buf, "multi-pack-index.d/multi-pack-index-{s}.midx", .{line}) catch break; const layer_path = try std.fmt.allocPrint(gpa, "{s}/pack/{s}", .{ d.path, file }); defer gpa.free(layer_path); var m = Mapped.open(io, pack_dir, file) catch { try o.problem(error.MultiPackIndexTooSmall, layer_path, ""); break; }; const midx = try o.parseLayer(m, layer_path, line) orelse { m.close(io); break; }; try o.appendLayer(&loaded, m, midx); } else { try layers.appendSlice(gpa, loaded.items); loaded.clearRetainingCapacity(); return; } // A broken layer: the chain is not used. for (loaded.items) |*l| { l.map.close(io); gpa.free(l.packs); } loaded.clearRetainingCapacity(); } fn appendLayer(o: *Opener, layers: *std.ArrayList(MidxLayer), map: Mapped, midx: MultiPackIndex) Allocator.Error!void { const numbers = try o.gpa.alloc(?usize, midx.pack_count); errdefer o.gpa.free(numbers); @memset(numbers, null); try layers.append(o.gpa, .{ .map = map, .midx = midx, .packs = numbers }); } /// Parses one multi-pack-index, returning null for a problem git goes /// on from and failing for one it stops at. fn parseLayer(o: *Opener, map: Mapped, path: []const u8, chain_name: ?[]const u8) OpenError!?MultiPackIndex { var diag: Diagnostics = .{}; const midx = MultiPackIndex.parse(o.options.format, map.bytes, &diag) catch |err| { if (MultiPackIndex.isFatal(err)) { if (o.diag) |dg| { dg.setPath(&.{path}); dg.setDetail("{s}", .{diag.detail()}); } return err; } try o.problem(err, path, diag.detail()); return null; }; if (chain_name) |name| { var hex: [object.max_hex_length]u8 = undefined; const sum: Id = .fromBytes(o.options.format, midx.checksum()); if (!std.ascii.eqlIgnoreCase(sum.toHex(&hex), name)) { try o.problem(error.MultiPackIndexChainMismatch, path, ""); return null; } } return midx; } }; /// The database as an `ObjectDatabase`. pub fn database(db: *DiskDatabase) ObjectDatabase { return .{ .ptr = db, .vtable = &vtable }; } const vtable: ObjectDatabase.VTable = .{ .contains = vtableContains, .readHeader = vtableReadHeader, .read = vtableRead, .stream = vtableStream, }; fn vtableContains(ptr: *anyopaque, io: Io, id: Id, diag: ?*Diagnostics) ContainsError!bool { const db: *DiskDatabase = @ptrCast(@alignCast(ptr)); return db.contains(io, id, diag); } fn vtableReadHeader(ptr: *anyopaque, io: Io, id: Id, diag: ?*Diagnostics) ReadError!object.Header { const db: *DiskDatabase = @ptrCast(@alignCast(ptr)); return db.readHeaderAt(io, id, .{}, diag); } fn vtableRead(ptr: *anyopaque, io: Io, gpa: Allocator, id: Id, diag: ?*Diagnostics) ReadError!Object { const db: *DiskDatabase = @ptrCast(@alignCast(ptr)); return db.readAt(io, gpa, id, .{}, diag); } fn vtableStream(ptr: *anyopaque, io: Io, gpa: Allocator, id: Id, out: *Io.Writer, diag: ?*Diagnostics) StreamError!object.Header { const db: *DiskDatabase = @ptrCast(@alignCast(ptr)); return db.stream(io, gpa, id, out, diag); } fn bases(db: *DiskDatabase) ?pack.Bases { if (!db.resolve_external_bases) return null; return .{ .ptr = db, .vtable = &bases_vtable }; } const bases_vtable: pack.Bases.VTable = .{ .readHeader = basesReadHeader, .read = basesRead, }; fn basesReadHeader(ptr: *anyopaque, io: Io, id: Id, depth: pack.Depth, diag: ?*Diagnostics) ReadError!object.Header { const db: *DiskDatabase = @ptrCast(@alignCast(ptr)); return db.readHeaderAt(io, id, depth, diag); } fn basesRead(ptr: *anyopaque, io: Io, gpa: Allocator, id: Id, depth: pack.Depth, diag: ?*Diagnostics) ReadError!Object { const db: *DiskDatabase = @ptrCast(@alignCast(ptr)); return db.readAt(io, gpa, id, depth, diag); } /// Where a packed object was found. const Found = struct { file: *const PackFile, pack: *const pack.Pack, offset: u64 }; /// The first error met while looking, which is the one returned if no /// copy of the object turns up, with what the diagnostics said of it. const FirstError = struct { err: ?ReadError = null, diag: Diagnostics = .{}, fn note(f: *FirstError, err: ReadError, d: *const Diagnostics) void { if (f.err != null) return; f.err = err; f.diag = d.*; } fn noteOpen(f: *FirstError, err: PackOpenError, path: []const u8, failure: *const PackFile.Failure, id: Id) void { if (f.err != null) return; f.err = err; f.diag = .{}; f.diag.object_id = id; f.diag.setPath(&.{path}); f.diag.setDetail("{s}", .{failure.detail[0..failure.detail_len]}); } fn finish(f: *const FirstError, diag: ?*Diagnostics, id: Id) ReadError { if (f.err) |err| { if (diag) |d| d.* = f.diag; return err; } if (diag) |d| d.object_id = id; return error.ObjectNotFound; } }; /// Steps through where a directory's packs hold an object, in the order /// git asks them, noting each pack that could not be asked. const PackSearch = struct { dir: *const Directory, id: Id, layer: usize = 0, in_layers: bool = true, next_pack: usize = 0, fn next(s: *PackSearch, first: *FirstError, scratch: *Diagnostics) ?Found { const d = s.dir; if (s.in_layers) { while (s.layer < d.midx_layers.len) { const layer = &d.midx_layers[s.layer]; s.layer += 1; const pos = layer.midx.find(s.id) orelse continue; scratch.* = .{}; scratch.object_id = s.id; const loc = layer.midx.locationAt(pos, scratch) catch |err| { scratch.setPath(&.{d.path}); first.note(err, scratch); continue; }; const i = layer.packs[loc.pack] orelse continue; const pf = &d.packs[i]; switch (pf.state) { .ok => |*p| return .{ .file = pf, .pack = p, .offset = loc.offset }, .bad_index => |*f| first.noteOpen(f.err, if (f.err == error.PackIndexMissing) pf.pack_path else pf.idx_path, f, s.id), .bad_pack => |*b| first.noteOpen(b.failure.err, pf.pack_path, &b.failure, s.id), } } s.in_layers = false; } while (s.next_pack < d.packs.len) { const pf = &d.packs[s.next_pack]; s.next_pack += 1; if (pf.in_midx and d.midx_layers.len > 0) continue; switch (pf.state) { .ok => |*p| { scratch.* = .{}; scratch.object_id = s.id; scratch.setPath(&.{pf.idx_path}); const offset = p.find(s.id, scratch) catch |err| { first.note(err, scratch); continue; } orelse continue; return .{ .file = pf, .pack = p, .offset = offset }; }, .bad_index => |*f| if (f.err != error.PackIndexMissing) first.noteOpen(f.err, pf.idx_path, f, s.id), .bad_pack => |*b| if (b.index.find(s.id) != null) first.noteOpen(b.failure.err, pf.pack_path, &b.failure, s.id), } } return null; } }; /// Whether the database holds the object, without reading it, as /// `ObjectDatabase.contains` describes. pub fn contains(db: *DiskDatabase, io: Io, id: Id, diag: ?*Diagnostics) ContainsError!bool { var first: FirstError = .{}; var scratch: Diagnostics = .{}; for (db.directories) |d| { var search: PackSearch = .{ .dir = d, .id = id }; if (search.next(&first, &scratch)) |_| return true; if (try d.loose.contains(io, id)) return true; } if (first.err) |err| { if (diag) |dg| dg.* = first.diag; return switch (err) { error.PackIndexTooSmall, error.UnsupportedPackIndexVersion, error.NonMonotonicPackIndex, error.WrongPackIndexSize, error.PackIndexMissing, error.PackTooSmall, error.NotAPackfile, error.UnsupportedPackVersion, error.PackDoesNotMatchIndex, error.PackObjectCountMismatch, => |e| e, // An entry that could not be located is there all the same. else => true, }; } return false; } /// The object's kind and size; see `ObjectDatabase.readHeader`. `depth` /// is how far resolving a delta chain had gone to get here, for a base. pub fn readHeaderAt(db: *DiskDatabase, io: Io, id: Id, depth: pack.Depth, diag: ?*Diagnostics) ReadError!object.Header { var first: FirstError = .{}; var scratch: Diagnostics = .{}; for (db.directories) |d| { var search: PackSearch = .{ .dir = d, .id = id }; while (search.next(&first, &scratch)) |found| { scratch = .{}; scratch.object_id = id; scratch.setPath(&.{found.file.pack_path}); if (found.pack.readHeader(io, found.offset, db.bases(), depth, &scratch)) |h| return h else |err| { if (err == error.OutOfMemory) return err; first.note(err, &scratch); } } scratch = .{}; if (d.loose.readHeader(io, id, &scratch)) |h| return h else |err| switch (err) { error.ObjectNotFound => {}, error.OutOfMemory => return err, else => first.note(err, &scratch), } } return first.finish(diag, id); } /// The whole object; see `ObjectDatabase.read`. `depth` is how far /// resolving a delta chain had gone to get here, for a base. pub fn readAt(db: *DiskDatabase, io: Io, gpa: Allocator, id: Id, depth: pack.Depth, diag: ?*Diagnostics) ReadError!Object { var first: FirstError = .{}; var scratch: Diagnostics = .{}; for (db.directories) |d| { var search: PackSearch = .{ .dir = d, .id = id }; while (search.next(&first, &scratch)) |found| { scratch = .{}; scratch.object_id = id; scratch.setPath(&.{found.file.pack_path}); if (found.pack.read(io, gpa, found.offset, db.bases(), depth, &scratch)) |o| return o else |err| { if (err == error.OutOfMemory) return err; first.note(err, &scratch); } } scratch = .{}; if (d.loose.read(io, gpa, id, &scratch)) |o| return o else |err| switch (err) { error.ObjectNotFound => {}, error.OutOfMemory => return err, else => first.note(err, &scratch), } } return first.finish(diag, id); } /// Writes the object's contents to `out`; see `ObjectDatabase.stream`. As /// git does, it looks for a loose copy in every directory before any /// packed one, and does not go on to another copy once it has begun. pub fn stream(db: *DiskDatabase, io: Io, gpa: Allocator, id: Id, out: *Io.Writer, diag: ?*Diagnostics) StreamError!object.Header { for (db.directories) |d| { if (d.loose.stream(io, gpa, id, out, diag)) |h| return h else |err| switch (err) { error.ObjectNotFound => {}, else => return err, } } var first: FirstError = .{}; var scratch: Diagnostics = .{}; for (db.directories) |d| { var search: PackSearch = .{ .dir = d, .id = id }; if (search.next(&first, &scratch)) |found| { if (diag) |dg| { dg.object_id = id; dg.setPath(&.{found.file.pack_path}); } return found.pack.stream(io, gpa, found.offset, out, db.bases(), .{}, diag); } } return first.finish(diag, id); } const testing = std.testing; const fixture = @import("testing/fixture.zig"); const packs = @import("testing/packs.zig"); fn openRepo(repo: *fixture.Repo, options: struct { alternate_list: ?[]const u8 = null }) !DiskDatabase { return open(repo.gpa, repo.io, .{ .objects_path = ".git/objects", .base = repo.tmp.dir, .format = repo.object_format, .alternate_list = options.alternate_list, }, null); } fn revParse(repo: *fixture.Repo, rev: []const u8) !Id { const out = try repo.output(&.{ "rev-parse", rev }); defer repo.gpa.free(out); return Id.parseHex(repo.object_format, std.mem.trimEnd(u8, out, "\n")); } /// Checks that every object `git rev-list --objects --all` lists reads /// through `db` as `git cat-file` reads it. fn expectAllObjectsAsGit(repo: *fixture.Repo, db: *DiskDatabase) !void { const gpa = repo.gpa; const listing = try repo.output(&.{ "rev-list", "--objects", "--all" }); defer gpa.free(listing); var lines = std.mem.tokenizeScalar(u8, listing, '\n'); var n: usize = 0; while (lines.next()) |line| : (n += 1) { const hex = line[0..repo.object_format.hexLength()]; const id = try Id.parseHex(repo.object_format, hex); const kind_name = try repo.output(&.{ "cat-file", "-t", hex }); defer gpa.free(kind_name); const want = try repo.output(&.{ "cat-file", std.mem.trimEnd(u8, kind_name, "\n"), hex }); defer gpa.free(want); errdefer std.debug.print("object {s}\n", .{hex}); const odb = db.database(); try testing.expect(try odb.contains(repo.io, id, null)); const header = try odb.readHeader(repo.io, id, null); try testing.expectEqualStrings(std.mem.trimEnd(u8, kind_name, "\n"), header.kind.name()); try testing.expectEqual(want.len, header.size); var obj = try odb.read(repo.io, gpa, id, null); defer obj.deinit(gpa); try testing.expectEqualSlices(u8, want, obj.data); var aw: Io.Writer.Allocating = .init(gpa); defer aw.deinit(); _ = try odb.stream(repo.io, gpa, id, &aw.writer, null); try testing.expectEqualSlices(u8, want, aw.written()); } try testing.expect(n > 0); } fn makeHistory(repo: *fixture.Repo, commits: usize) !void { const gpa = repo.gpa; var text: std.ArrayList(u8) = .empty; defer text.deinit(gpa); for (0..300) |i| try text.print(gpa, "line {d}\n", .{i}); for (0..commits) |i| { const at = (i * 53) % (text.items.len - 10); text.items[at] = 'a' + @as(u8, @intCast(i % 26)); try repo.commitFile("f", text.items, "c"); var buf: [32]u8 = undefined; try repo.commitFile("g", try std.fmt.bufPrint(&buf, "{d}\n", .{i}), "g"); } } test "loose objects, packs and a multi-pack-index all read as git reads them" { const gpa = testing.allocator; var repo = try fixture.Repo.init(gpa, testing.io); defer repo.deinit(); try makeHistory(&repo, 6); try repo.git(&.{ "repack", "-dq" }); try makeHistory(&repo, 4); try repo.git(&.{ "repack", "-dq" }); try makeHistory(&repo, 3); // and some left loose { var db = try openRepo(&repo, .{}); defer db.close(testing.io); try testing.expectEqual(2, db.directories[0].packs.len); try expectAllObjectsAsGit(&repo, &db); } try repo.git(&.{ "multi-pack-index", "write" }); try makeHistory(&repo, 2); try repo.git(&.{ "repack", "-dq" }); // a pack the multi-pack-index does not name { var db = try openRepo(&repo, .{}); defer db.close(testing.io); try testing.expectEqual(1, db.directories[0].midx_layers.len); try testing.expectEqual(0, db.problems.len); try expectAllObjectsAsGit(&repo, &db); } try repo.git(&.{ "repack", "-adfq", "--depth=20" }); { var db = try openRepo(&repo, .{}); defer db.close(testing.io); try expectAllObjectsAsGit(&repo, &db); } } test "an incremental multi-pack-index chain is read layer by layer" { const gpa = testing.allocator; var repo = try fixture.Repo.init(gpa, testing.io); defer repo.deinit(); try makeHistory(&repo, 3); try repo.git(&.{ "repack", "-dq" }); try repo.git(&.{ "multi-pack-index", "write", "--incremental" }); try makeHistory(&repo, 3); try repo.git(&.{ "repack", "-dq" }); try repo.git(&.{ "multi-pack-index", "write", "--incremental" }); var db = try openRepo(&repo, .{}); defer db.close(testing.io); try testing.expectEqual(2, db.directories[0].midx_layers.len); try testing.expectEqual(0, db.problems.len); try expectAllObjectsAsGit(&repo, &db); } test "alternates are followed as git follows them" { const gpa = testing.allocator; var repo = try fixture.Repo.init(gpa, testing.io); defer repo.deinit(); try makeHistory(&repo, 3); try repo.git(&.{ "repack", "-adq" }); try makeHistory(&repo, 1); // A clone borrowing everything through objects/info/alternates. try repo.git(&.{ "clone", "-q", "--shared", ".", "shared" }); // The clone's own directory, with the original as its alternate. var db = try open(gpa, testing.io, .{ .objects_path = "shared/.git/objects", .base = repo.tmp.dir, .format = repo.object_format, }, null); defer db.close(testing.io); try testing.expectEqual(2, db.directories.len); try testing.expectEqual(0, db.problems.len); try expectAllObjectsAsGit(&repo, &db); } test "a REF_DELTA's base outside its pack is found only when asked for" { const gpa = testing.allocator; var repo = try fixture.Repo.init(gpa, testing.io); defer repo.deinit(); const f = repo.object_format; const base = "the base\n"; const loose_base = "a loose base\n"; const loose_hex = try repo.hashObject("blob", loose_base); defer gpa.free(loose_hex); const in_other = base ++ "one\n"; const Case = struct { base_id: Id, base_text: []const u8, suffix: []const u8 }; const cases = [_]Case{ .{ .base_id = object.hash(f, .blob, base), .base_text = base, .suffix = "one\n" }, .{ .base_id = try Id.parseHex(f, loose_hex), .base_text = loose_base, .suffix = "two\n" }, .{ .base_id = object.hash(f, .blob, in_other), .base_text = in_other, .suffix = "three\n" }, .{ .base_id = object.hash(f, .blob, "nowhere"), .base_text = "nowhere", .suffix = "four\n" }, }; { var w: packs.PackWriter = try .init(gpa, f); defer w.deinit(); _ = try w.whole(.blob, base); const out = try w.finish(); defer gpa.free(out.pack); defer gpa.free(out.idx); gpa.free(try packs.install(&repo, out.pack, out.idx)); } var ids: [cases.len]Id = undefined; for (cases, &ids) |case, *id| { var w: packs.PackWriter = try .init(gpa, f); defer w.deinit(); const d = try packs.deltaAppend(gpa, @intCast(case.base_text.len), case.suffix); defer gpa.free(d); const result = try std.mem.concat(gpa, u8, &.{ case.base_text, case.suffix }); defer gpa.free(result); id.* = object.hash(f, .blob, result); _ = try w.refDelta(case.base_id, d, id.*); const out = try w.finish(); defer gpa.free(out.pack); defer gpa.free(out.idx); gpa.free(try packs.install(&repo, out.pack, out.idx)); } // Two REF_DELTAs in packs of their own, each the other's base. const x = object.hash(f, .blob, "x"); const y = object.hash(f, .blob, "y"); for ([_][2]Id{ .{ x, y }, .{ y, x } }) |pair| { var w: packs.PackWriter = try .init(gpa, f); defer w.deinit(); const d = try packs.deltaInsert(gpa, 1, "x"); defer gpa.free(d); _ = try w.refDelta(pair[1], d, pair[0]); const out = try w.finish(); defer gpa.free(out.pack); defer gpa.free(out.idx); gpa.free(try packs.install(&repo, out.pack, out.idx)); } { // As git reads them: none is found. var db = try openRepo(&repo, .{}); defer db.close(testing.io); const odb = db.database(); for (ids) |id| { try testing.expect(try odb.contains(testing.io, id, null)); var diag: Diagnostics = .{}; try testing.expectError(error.DeltaBaseMissing, odb.readHeader(testing.io, id, &diag)); try testing.expectError(error.DeltaBaseMissing, odb.read(testing.io, gpa, id, null)); } } var db = try open(gpa, testing.io, .{ .objects_path = ".git/objects", .base = repo.tmp.dir, .format = f, .resolve_external_bases = true, }, null); defer db.close(testing.io); const odb = db.database(); for (cases[0..3], ids[0..3]) |case, id| { var obj = try odb.read(testing.io, gpa, id, null); defer obj.deinit(gpa); const want = try std.mem.concat(gpa, u8, &.{ case.base_text, case.suffix }); defer gpa.free(want); try testing.expectEqualStrings(want, obj.data); try testing.expectEqual(want.len, (try odb.readHeader(testing.io, id, null)).size); } var diag: Diagnostics = .{}; try testing.expectError(error.DeltaBaseMissing, odb.read(testing.io, gpa, ids[3], &diag)); var hex: [object.max_hex_length]u8 = undefined; try testing.expectEqualStrings(cases[3].base_id.toHex(&hex), diag.detail()); try testing.expectError(error.DeltaChainTooDeep, odb.read(testing.io, gpa, x, null)); try testing.expectError(error.DeltaChainTooDeep, odb.readHeader(testing.io, x, null)); } test "an object over max_object_size is refused, one at it is read, and null is no limit" { const gpa = testing.allocator; var repo = try fixture.Repo.init(gpa, testing.io); defer repo.deinit(); const contents = "exactly thirty-two bytes long!!\n"; // Committed, so that repacking packs it. try repo.commitFile("f", contents, "c"); const hex = try repo.hashObject("blob", contents); defer gpa.free(hex); const id = try Id.parseHex(repo.object_format, hex); // Loose, and then packed. for (0..2) |round| { if (round == 1) try repo.git(&.{ "repack", "-adq" }); for ([_]?u64{ contents.len, contents.len - 1, null }) |max| { var db = try open(gpa, testing.io, .{ .objects_path = ".git/objects", .base = repo.tmp.dir, .format = repo.object_format, .max_object_size = max, }, null); defer db.close(testing.io); const odb = db.database(); errdefer std.debug.print("round {d}, limit {?d}\n", .{ round, max }); // The header reports the size whatever the limit. try testing.expectEqual(contents.len, (try odb.readHeader(testing.io, id, null)).size); if (max != null and max.? < contents.len) { var diag: Diagnostics = .{}; try testing.expectError(error.ObjectTooLarge, odb.read(testing.io, gpa, id, &diag)); try testing.expect(Id.eql(id, diag.object_id.?)); } else { var obj = try odb.read(testing.io, gpa, id, null); defer obj.deinit(gpa); try testing.expectEqualStrings(contents, obj.data); } // Streaming an object stored whole is never limited. var aw: Io.Writer.Allocating = .init(gpa); defer aw.deinit(); _ = try odb.stream(testing.io, gpa, id, &aw.writer, null); try testing.expectEqualStrings(contents, aw.written()); } } // The default is a gigabyte. var db = try openRepo(&repo, .{}); defer db.close(testing.io); try testing.expectEqual(default_max_object_size, db.max_object_size.?); }