diff --git a/src/internal/archive.zig b/src/internal/archive.zig index 98fb4f8..bcfc348 100644 --- a/src/internal/archive.zig +++ b/src/internal/archive.zig @@ -41,6 +41,14 @@ pub const Archive = struct { /// blocks flush at 4096 events OR 30s) last_sync_us: i64 = 0, flush_interval_us: i64 = 30 * std.time.us_per_s, + /// serializes the writer against the steady compactor's force-rotate; + /// uncontended in the single-writer steady state + mu: Io.Mutex = .init, + /// upstream's OnAppend hook: runs under `mu` with the stamped event, + /// so a force-rotate returns only after every sealed row was observed + /// (the live-tombstone ordering guarantee) + on_append_ctx: ?*anyopaque = null, + on_append: ?*const fn (*anyopaque, segment.Event) void = null, pub fn init(allocator: Allocator, io: Io, data_dir: []const u8) !Archive { var buf: [256]u8 = undefined; @@ -148,23 +156,32 @@ pub const Archive = struct { /// append one event; assigns and returns its local seq. blocks flush to /// disk at the block threshold; rotation seals at the byte budget. pub fn append(self: *Archive, ev: segment.Event, now_us: i64) !u64 { + self.mu.lockUncancelable(self.io); + defer self.mu.unlock(self.io); var stamped = ev; stamped.seq = self.next_seq; try self.writer.append(stamped); // auto-flushes a block at threshold self.next_seq += 1; + if (self.on_append) |hook| hook(self.on_append_ctx.?, stamped); if (self.writer.pending.items.len > 0 and now_us - self.last_sync_us >= self.flush_interval_us) { try self.writer.flush(); // partial block: durability over block size } if (self.writer.bytes().len > self.file_written) { try self.syncToDisk(); self.last_sync_us = now_us; - if (self.writer.bytes().len >= self.max_segment_bytes) try self.rotate(); + if (self.writer.bytes().len >= self.max_segment_bytes) try self.rotateLocked(); } return stamped.seq; } /// compress + write any pending block, fsync, advance the watermark. pub fn flushBlock(self: *Archive) !void { + self.mu.lockUncancelable(self.io); + defer self.mu.unlock(self.io); + try self.flushBlockLocked(); + } + + fn flushBlockLocked(self: *Archive) !void { try self.writer.flush(); if (self.writer.bytes().len > self.file_written) try self.syncToDisk(); } @@ -178,9 +195,16 @@ pub const Archive = struct { self.committed_seq.store(self.next_seq - 1 - self.writer.pending.items.len, .release); } - /// seal the active segment (footer + finalized header) and open the next. + /// seal the active segment (footer + finalized header) and open the + /// next. safe from another thread (the compactor's force-rotate). pub fn rotate(self: *Archive) !void { - try self.flushBlock(); + self.mu.lockUncancelable(self.io); + defer self.mu.unlock(self.io); + try self.rotateLocked(); + } + + fn rotateLocked(self: *Archive) !void { + try self.flushBlockLocked(); if (self.writer.block_count == 0) return; // nothing to seal try self.writer.seal(); const all = self.writer.bytes(); diff --git a/src/internal/compact/pass.zig b/src/internal/compact/pass.zig index 70c8bda..4b2e8a9 100644 --- a/src/internal/compact/pass.zig +++ b/src/internal/compact/pass.zig @@ -147,8 +147,9 @@ fn sweep(allocator: Allocator, io: Io, seg_dir: Io.Dir, metas: *std.ArrayList(Se } /// observe every marker row with seq > low_exclusive into the snapshot; -/// block index seq bounds prune the read -fn foldSegment( +/// block index seq bounds prune the read. pub for the steady compactor's +/// startup rebuild (fold everything above the committed watermark). +pub fn foldSegment( allocator: Allocator, io: Io, seg_dir: Io.Dir, diff --git a/src/internal/compact/steady.zig b/src/internal/compact/steady.zig new file mode 100644 index 0000000..45aaf9f --- /dev/null +++ b/src/internal/compact/steady.zig @@ -0,0 +1,239 @@ +//! steady — the long-running compactor beside the live consumer +//! +//! upstream runSteadyCompactor (docs/upstream-compaction-spec.md): a +//! pass every interval (default 4h; 0 disables compaction entirely), +//! plus an early coalesced trigger when the live in-memory tombstone +//! set reaches the cap, floored by 30s spacing so a failed pass can't +//! re-fire per event. each pass force-rotates the active segment first — +//! race-free because the live set is fed by the archive's on_append hook +//! under the writer lock, so by rotate's return every sealed row's +//! tombstone is observed and later rows land above the pass target. the +//! live set is used ONLY for the cap trigger; passes fold their drop +//! windows from disk. a failed pass logs and retries next tick — it must +//! never tear down the daemon. +//! +//! no manifest refresh / cache invalidation: stream's serving paths read +//! sealed files from disk per request, and rename is atomic — an +//! in-flight read sees either generation, both valid. revisit if a +//! resident manifest or decoded-block cache is added. + +const std = @import("std"); +const archive_mod = @import("../archive.zig"); +const pass = @import("pass.zig"); +const segment = @import("../segment.zig"); +const tombstone = @import("../tombstone.zig"); +const watermark = @import("watermark.zig"); + +const Io = std.Io; +const Allocator = std.mem.Allocator; +const log = std.log.scoped(.stream); + +pub const Compactor = struct { + allocator: Allocator, + io: Io, + archive: *archive_mod.Archive, + data_dir: []const u8, + /// seconds between passes; 0 disables (run() returns immediately) + interval_s: u64 = 4 * 3600, + tombstone_cap: usize = pass.default_tombstone_cap, + min_trigger_spacing_s: u64 = 30, + /// guards `live` (observe on the writer thread, evict on ours) + mu: Io.Mutex = .init, + live: tombstone.Snapshot, + stop_flag: std.atomic.Value(bool) = .init(false), + /// coalesced cap trigger (upstream's 1-buffered channel) + trigger: std.atomic.Value(bool) = .init(false), + + pub fn init(allocator: Allocator, io: Io, archive: *archive_mod.Archive, data_dir: []const u8) Compactor { + return .{ + .allocator = allocator, + .io = io, + .archive = archive, + .data_dir = data_dir, + .live = tombstone.Snapshot.init(allocator), + }; + } + + pub fn deinit(self: *Compactor) void { + self.live.deinit(); + } + + pub fn requestStop(self: *Compactor) void { + self.stop_flag.store(true, .release); + } + + /// archive on_append hook; runs under the archive writer lock + pub fn onAppend(ctx: *anyopaque, ev: segment.Event) void { + const self: *Compactor = @ptrCast(@alignCast(ctx)); + self.mu.lockUncancelable(self.io); + defer self.mu.unlock(self.io); + self.live.observe(ev) catch |err| { + log.warn("live tombstone observe failed: {s}", .{@errorName(err)}); + return; + }; + if (self.live.len() >= self.tombstone_cap) self.trigger.store(true, .release); + } + + /// startup rebuild (upstream rebuildLiveTombstones): fold sealed rows + /// above the committed watermark into the live set. cost scales with + /// the watermark backlog — segments/blocks at or below it are pruned. + /// the recovered-then-sealed active segment is covered; the fresh + /// active is empty. + pub fn rebuild(self: *Compactor) !void { + var wm = try watermark.Store.init(self.io, self.data_dir); + defer wm.deinit(self.io); + const w = wm.load(self.io) orelse 0; + + var seg_dir = try self.openSegDir(); + defer seg_dir.close(self.io); + self.mu.lockUncancelable(self.io); + defer self.mu.unlock(self.io); + var it = seg_dir.iterate(); + while (try it.next(self.io)) |entry| { + const idx = archive_mod.parseSegmentIndex(entry.name) orelse continue; + pass.foldSegment(self.allocator, self.io, seg_dir, idx, w, &self.live) catch |err| switch (err) { + error.ActiveSegment => continue, + else => return err, + }; + } + log.info("live tombstones rebuilt above watermark {d}: {d} entries", .{ w, self.live.len() }); + } + + /// timer loop; spawn via io.concurrent, stop via requestStop + pub fn run(self: *Compactor) void { + if (self.interval_s == 0) return; + var since_pass: u64 = 0; + while (!self.stop_flag.load(.acquire)) { + self.io.sleep(Io.Duration.fromSeconds(1), .awake) catch break; + since_pass += 1; + const due = since_pass >= self.interval_s or + (self.trigger.load(.acquire) and since_pass >= self.min_trigger_spacing_s); + if (!due) continue; + self.trigger.store(false, .release); + const stats = self.runOnce() catch |err| { + // failure policy: never tear down the daemon; retry next tick + log.warn("compaction pass failed: {s} (retrying next interval)", .{@errorName(err)}); + since_pass = 0; + continue; + }; + log.info("compaction pass: watermark {d}, {d} chunks, {d}/{d} segments rewritten, {d} rows dropped", .{ + stats.target_watermark, stats.chunks, stats.segments_rewritten, stats.segments_examined, stats.rows_dropped, + }); + since_pass = 0; + } + } + + /// one force-rotate + pass + eviction; also the test seam + pub fn runOnce(self: *Compactor) !pass.Stats { + // rows deleted while their segment was active get compacted this + // pass; on_append ordering makes this race-free (module doc) + try self.archive.rotate(); + + var wm = try watermark.Store.init(self.io, self.data_dir); + defer wm.deinit(self.io); + var seg_dir = try self.openSegDir(); + defer seg_dir.close(self.io); + + // the pass gets no live-set handle: it folds from disk, and we + // evict once after the final chunk commit (observe and evict then + // never race; entries <= W only cover physically-compacted rows) + const stats = try pass.run(self.allocator, self.io, seg_dir, &wm, null, .{ + .tombstone_cap = self.tombstone_cap, + }); + if (wm.load(self.io)) |w| { + self.mu.lockUncancelable(self.io); + defer self.mu.unlock(self.io); + self.live.evict(w); + } + return stats; + } + + fn openSegDir(self: *Compactor) !Io.Dir { + var buf: [256]u8 = undefined; + const seg_path = try std.fmt.bufPrint(&buf, "{s}/segments", .{self.data_dir}); + return Io.Dir.cwd().openDir(self.io, seg_path, .{ .iterate = true }); + } +}; + +// === tests === + +const testing = std.testing; + +fn row(kind: segment.Kind, did: []const u8, rkey: []const u8, payload: []const u8) segment.Event { + return .{ + .seq = 0, // archive assigns + .witnessed_at = 1, + .indexed_at = 0, + .kind = kind, + .did = did, + .collection = "app.bsky.feed.post", + .rkey = rkey, + .rev = "3l3qo2vutsw2b", + .payload = payload, + }; +} + +test "steady: hook feeds live set, runOnce force-rotates, compacts, evicts" { + var threaded: Io.Threaded = .init(testing.allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [Io.Dir.max_path_bytes]u8 = undefined; + const n = try tmp.dir.realPath(io, &path_buf); + var data_dir_buf: [Io.Dir.max_path_bytes]u8 = undefined; + const data_dir = try std.fmt.bufPrint(&data_dir_buf, "{s}/data", .{path_buf[0..n]}); + + var a = try archive_mod.Archive.init(testing.allocator, io, data_dir); + defer a.deinit(); + + var c = Compactor.init(testing.allocator, io, &a, data_dir); + defer c.deinit(); + a.on_append_ctx = &c; + a.on_append = Compactor.onAppend; + + // create k1, then delete it — both land in the ACTIVE segment + _ = try a.append(row(.create, "did:plc:a", "k1", "x"), 1); + _ = try a.append(row(.create, "did:plc:a", "k2", "x"), 2); + _ = try a.append(row(.delete, "did:plc:a", "k1", ""), 3); + try testing.expectEqual(@as(usize, 1), c.live.len()); + + // force-rotate seals them; the pass drops the superseded create + const stats = try c.runOnce(); + try testing.expectEqual(@as(u64, 3), stats.target_watermark); + try testing.expectEqual(@as(u64, 1), stats.segments_rewritten); + try testing.expectEqual(@as(u64, 1), stats.rows_dropped); + // eviction cleared the covered entry (delete seq 3 <= watermark 3) + try testing.expectEqual(@as(usize, 0), c.live.len()); + + // rebuild from disk reproduces an empty live set above the watermark + var c2 = Compactor.init(testing.allocator, io, &a, data_dir); + defer c2.deinit(); + try c2.rebuild(); + try testing.expectEqual(@as(usize, 0), c2.live.len()); +} + +test "steady: cap sets the coalesced trigger" { + var threaded: Io.Threaded = .init(testing.allocator, .{}); + defer threaded.deinit(); + const io = threaded.io(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [Io.Dir.max_path_bytes]u8 = undefined; + const n = try tmp.dir.realPath(io, &path_buf); + var data_dir_buf: [Io.Dir.max_path_bytes]u8 = undefined; + const data_dir = try std.fmt.bufPrint(&data_dir_buf, "{s}/data", .{path_buf[0..n]}); + + var a = try archive_mod.Archive.init(testing.allocator, io, data_dir); + defer a.deinit(); + var c = Compactor.init(testing.allocator, io, &a, data_dir); + defer c.deinit(); + c.tombstone_cap = 2; + a.on_append_ctx = &c; + a.on_append = Compactor.onAppend; + + _ = try a.append(row(.delete, "did:plc:a", "k1", ""), 1); + try testing.expect(!c.trigger.load(.acquire)); + _ = try a.append(row(.delete, "did:plc:a", "k2", ""), 2); + try testing.expect(c.trigger.load(.acquire)); +} diff --git a/src/main.zig b/src/main.zig index e8fd96b..9a2c7d1 100644 --- a/src/main.zig +++ b/src/main.zig @@ -12,6 +12,7 @@ const xrpcapi = @import("internal/xrpcapi.zig"); const metrics = @import("internal/metrics.zig"); const pipeline_mod = @import("internal/pipeline.zig"); const server_mod = @import("internal/server.zig"); +const steady_mod = @import("internal/compact/steady.zig"); const tail_mod = @import("internal/tail.zig"); const Io = std.Io; @@ -44,6 +45,8 @@ pub fn main(init: std.process.Init.Minimal) !void { var backfill_max_repos: usize = 0; var do_backfill = false; var relay_http: []const u8 = "http://localhost:7777"; + // upstream CompactionInterval default: 4h; 0 disables compaction + var compaction_interval_s: u64 = 4 * 3600; var arg_it = init.args.iterate(); _ = arg_it.next(); // program name @@ -64,6 +67,8 @@ pub fn main(init: std.process.Init.Minimal) !void { backfill_max_repos = try std.fmt.parseInt(usize, arg["--backfill-max-repos=".len..], 10); } else if (std.mem.startsWith(u8, arg, "--relay-http=")) { relay_http = arg["--relay-http=".len..]; + } else if (std.mem.startsWith(u8, arg, "--compaction-interval=")) { + compaction_interval_s = try std.fmt.parseInt(u64, arg["--compaction-interval=".len..], 10); } else if (std.mem.eql(u8, arg, "--backfill")) { do_backfill = true; } else if (std.mem.eql(u8, arg, "--no-verify")) { @@ -145,6 +150,24 @@ pub fn main(init: std.process.Init.Minimal) !void { if (cursor) |c| log.info("resuming from persisted cursor {d}", .{c}); } + // steady compactor: live tombstone set fed from the archive's append + // hook, periodic + cap-triggered passes. interval 0 disables all of it + // (run() returns immediately, no hook, no rebuild). + var compactor = steady_mod.Compactor.init(allocator, io, &archive, data_dir); + defer compactor.deinit(); + compactor.interval_s = compaction_interval_s; + if (compaction_interval_s != 0) { + try compactor.rebuild(); + archive.on_append_ctx = &compactor; + archive.on_append = steady_mod.Compactor.onAppend; + log.info("steady compactor on (interval {d}s)", .{compaction_interval_s}); + } + var compactor_future = try io.concurrent(steady_mod.Compactor.run, .{&compactor}); + defer { + compactor.requestStop(); + _ = compactor_future.cancel(io); + } + var consumer: ingest.Consumer = .{ .allocator = allocator, .io = io, diff --git a/src/tests.zig b/src/tests.zig index 3c2dd48..ce16648 100644 --- a/src/tests.zig +++ b/src/tests.zig @@ -20,6 +20,7 @@ comptime { _ = @import("internal/segment_rewrite.zig"); _ = @import("internal/compact/watermark.zig"); _ = @import("internal/compact/pass.zig"); + _ = @import("internal/compact/steady.zig"); _ = @import("internal/segment_fixture_test.zig"); _ = @import("internal/verify.zig"); _ = @import("internal/wire.zig");