diff --git a/deploy/grafana/stream.json b/deploy/grafana/stream.json index fceaa8b..727455f 100644 --- a/deploy/grafana/stream.json +++ b/deploy/grafana/stream.json @@ -7381,7 +7381,7 @@ "type": "prometheus", "uid": "${datasource}" }, - "expr": "sum(jetstream_backfill_progress_completed{job=~\"$job\",instance=~\"$instance\"})", + "expr": "sum(stream_backfill_repos_durable{status=\"complete\",job=~\"$job\",instance=~\"$instance\"})", "legendFormat": "durably committed", "refId": "C", "range": true, diff --git a/docs/grafana-dashboard.md b/docs/grafana-dashboard.md index f9c1359..0e2e64a 100644 --- a/docs/grafana-dashboard.md +++ b/docs/grafana-dashboard.md @@ -11,7 +11,7 @@ The exact reviewed bytes are vendored at `deploy/grafana/jetstream-upstream.json`; deployment does not depend on the network or a moving branch. Stream provisions `deploy/grafana/stream.json` (SHA-256 -`c2600122ad4cf2acd81d013438c5b536cc46f37255ea8cf7e829e7536c9321f9`). It is +`d00582d478cdf0c8b00d348dd953c79651c4961e161592bf5eee9f389b3b51d9`). It is deterministically generated by `tools/adapt_dashboard.py` from the pinned source. The generator refuses a changed source checksum, and the tests prove every upstream non-runtime panel remains structurally identical. Stream adds diff --git a/docs/semantic-parity.md b/docs/semantic-parity.md index ec93430..3898a8b 100644 --- a/docs/semantic-parity.md +++ b/docs/semantic-parity.md @@ -82,8 +82,8 @@ The detailed bootstrap audit is in | Timestamp import | **verified** | CSV parsing, rule precedence, patch topology preservation, job persistence, restart, HTTP authentication, and write/fsync/rename power-loss cuts have concrete tests. | Public visibility after import rides the cold path, which is now verified; this row does not separately re-prove replay. | | Status: hosts and accounts | **verified** | Durable host aggregates and account lookup/verification have focused production HTTP tests, including rate limiting and restart. Together with the segments, collections and summary views this is upstream's status surface rather than a subset of it. | Rate-limit windowing is Stream's own; upstream does not limit these routes. | | Status: summary, collections, segments | **verified** | The field report at `/status` carries upstream's status information set: repository rollup with percent complete (`BackfillStats`), live cursors (`LiveStats`), retention with configured lookback and oldest retained seq (`CursorLookbackStats`), timestamp-import job state and phase (`ImportInfo`), and the durable lifecycle phase with its age and backfill duration (`PhaseInfo`). Segments and collections render from resident manifest metadata, cross-checked against `listSegments`. Absence is distinguished from zero throughout — no repository state does not render as "0 of 0 complete", and disabled cursor replay is reported rather than shown as no retention. A test pins the set so it cannot quietly shrink back to process-local counters. | The ASCII river at `GET /` is an intentional aesthetic divergence and is not a status surface; upstream has no equivalent and none is wanted. Per-tree storage totals and a `PebbleStats`-equivalent metadata-store size are not surfaced. | -| Prometheus metric exposition | **partial** | Metrics are emitted in Prometheus syntax, and focused tests show that a subset increments at their intended boundaries. | The dashboard contract checks metric-family presence, not producer semantics, label equivalence, monotonicity, or non-placeholder behavior. Several metrics described as durable progress were process-local or tied to the wrong batch boundary. | -| Grafana dashboard reuse | **partial** | The upstream dashboard is checksum-pinned; deliberate runtime-only panels are separated from upstream panels; the public experiment dashboard can be provisioned read-only. | A query finding a named family does not prove semantic equivalence. Progress panels must be derived only from genuinely durable, restart-stable counters. Dashboard admission remains blocked by producer semantics. | +| Prometheus metric exposition | **partial** | Metrics are emitted in Prometheus syntax and focused tests show families increment at their intended boundaries. Backfill progress now has a restart-stable series: `stream_backfill_repos_durable{status=...}` is read from the metadata store at scrape time, so it continues across a restart instead of resetting — measured at 22 complete before a restart and 49 after, where the process-local gauge starts again from zero. Absence is distinguished from zero: no repository state emits no series rather than zero-of-zero. | Producer semantics are proved for the progress families specifically, not for every family on the dashboard. Label equivalence with upstream and monotonicity are still unproved in general. | +| Grafana dashboard reuse | **partial** | The upstream dashboard is checksum-pinned and deliberate runtime-only panels are separated from upstream panels. The progress chart's "durably committed" series now queries the durable store-backed gauge; it previously queried `jetstream_backfill_progress_completed`, a process-local counter, so the series carrying that label showed this process's work rather than the archive's — the mislabel that made a restart read as progress in July. A contract test pins which metric backs that legend. | The remaining panels are still validated by family presence rather than producer semantics, so this row tracks the progress chart only. | | Public/debug listeners | **verified** | Production-process tests establish route isolation, disabled debug binding, public WebSocket admission, readiness, and the historical combined listener mode. | This row does not establish lifecycle readiness or data completeness. | | Graceful shutdown | **verified** | Listener admission stops, clients receive 1001 within configured budgets, blocked peers are interrupted, and a hung listRepos bootstrap exits cooperatively in the focused receipt. | Listener admission stops, clients receive 1001 within configured budgets, blocked peers are interrupted, and a hung listRepos bootstrap exits cooperatively. The exact deployed image now is admitted: `shutdown-contract` passes inside `receipts/79ecfa5.json`, bound to the published digest. Dependency reconnect failures are no longer process-fatal. | | Logging | **verified** | Real processes exercise JSON/text selection, level filtering, default values, and invalid configuration. | Semantic parity is limited to the public controls and rendered fields, not byte-identical Go logging internals. | diff --git a/src/internal/metrics.zig b/src/internal/metrics.zig index 5eb069e..592c943 100644 --- a/src/internal/metrics.zig +++ b/src/internal/metrics.zig @@ -417,6 +417,23 @@ pub const TailGauges = struct { tip: u64, }; +/// Durable repository counts, read from the metadata store at scrape time. +/// These are the only backfill progress figures that survive a restart: the +/// process-local counters reset to zero and climb again, which is what made a +/// restart look like progress on the July dashboard. Null when no repository +/// state exists, which must not render as zero. +pub const DurableRepos = struct { + complete: u64 = 0, + pending: u64 = 0, + failed: u64 = 0, + unavailable: u64 = 0, + not_started: u64 = 0, + + pub fn total(self: DurableRepos) u64 { + return self.complete + self.pending + self.failed + self.unavailable + self.not_started; + } +}; + pub fn format( buf: []u8, stats: *const Stats, @@ -425,6 +442,7 @@ pub fn format( now_s: i64, process: process_metrics.Snapshot, data_dir_free_bytes: ?u64, + durable_repos: ?DurableRepos, ) []const u8 { @setEvalBranchQuota(2_000); var w: Io.Writer = .fixed(buf); @@ -1255,6 +1273,31 @@ pub fn format( stats.import_job_duration_count.load(.monotonic), }) catch {}; + // Durable repository progress. Sourced from the metadata store rather than + // from process-local counters, so a restart continues the curve instead of + // restarting it. Emitted only when repository state exists: absent is not + // zero, and a progress panel reading zero-of-zero is worse than a gap. + if (durable_repos) |repos| { + w.print( + \\# TYPE stream_backfill_repos_durable gauge + \\stream_backfill_repos_durable{{status="complete"}} {d} + \\stream_backfill_repos_durable{{status="pending"}} {d} + \\stream_backfill_repos_durable{{status="failed"}} {d} + \\stream_backfill_repos_durable{{status="unavailable"}} {d} + \\stream_backfill_repos_durable{{status="not_started"}} {d} + \\# TYPE stream_backfill_repos_durable_total gauge + \\stream_backfill_repos_durable_total {d} + \\ + , .{ + repos.complete, + repos.pending, + repos.failed, + repos.unavailable, + repos.not_started, + repos.total(), + }) catch {}; + } + return w.buffered(); } @@ -1468,11 +1511,15 @@ test "format renders counters" { .major_page_faults = 2, .voluntary_context_switches = 41, .involuntary_context_switches = 3, - }, 123_456); + }, 123_456, .{ .complete = 90, .pending = 6, .failed = 3, .unavailable = 1, .not_started = 0 }); try std.testing.expect(std.mem.indexOf(u8, out, "stream_events_total 7") != null); try std.testing.expect(std.mem.indexOf(u8, out, "stream_verify_total{result=\"valid\"} 3") != null); try std.testing.expect(std.mem.indexOf(u8, out, "stream_dropped_events_total{reason=\"invalid_rev\"} 1") != null); try std.testing.expect(std.mem.indexOf(u8, out, "jetstream_ingest_dropped_events_total{source=\"live\",reason=\"invalid_rev\"} 1") != null); + // Durable progress is the restart-stable series: it comes from the store, + // not from this process's counters. + try std.testing.expect(std.mem.indexOf(u8, out, "stream_backfill_repos_durable{status=\"complete\"} 90") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "stream_backfill_repos_durable_total 100") != null); try std.testing.expect(std.mem.indexOf(u8, out, "jetstream_ingest_dropped_events_total{source=\"backfill\",reason=\"invalid_rkey\"} 2") != null); try std.testing.expect(std.mem.indexOf(u8, out, "stream_build_info{git_sha=") != null); try std.testing.expect(std.mem.indexOf(u8, out, "stream_subscribers_active 2") != null); diff --git a/src/internal/server.zig b/src/internal/server.zig index 2c3ab42..5e7654e 100644 --- a/src/internal/server.zig +++ b/src/internal/server.zig @@ -1100,7 +1100,7 @@ fn respondMetrics(conn: *websocket.Conn, hub: *Hub, observation: *HttpObservatio .pinned_overrun_bytes = g.pinned_overrun_bytes, .base = g.base, .tip = g.tip, - }, now_s, process_metrics.collect(hub.io), if (hub.archive) |archive| disk_space.freeBytes(archive.dir) else null); + }, now_s, process_metrics.collect(hub.io), if (hub.archive) |archive| disk_space.freeBytes(archive.dir) else null, readDurableRepos(hub)); respondPartsMaybeHead(conn, "200 OK", "text/plain; version=0.0.4", out, "", head); } @@ -1558,6 +1558,21 @@ fn readImportJob(hub: *Hub, buf: []u8) ?[]const u8 { }) catch null; } +/// Durable repository counts for the progress gauges, read at scrape time. +/// Null when no repository state exists at all — a progress panel showing +/// zero-of-zero is worse than a gap in the series. +fn readDurableRepos(hub: *Hub) ?metrics.DurableRepos { + const store = hub.repo_status orelse return null; + if (store.count() == 0) return null; + return .{ + .complete = store.countByStatus(.complete), + .pending = store.countByStatus(.pending), + .failed = store.countByStatus(.failed), + .unavailable = store.countByStatus(.unavailable), + .not_started = store.countByStatus(.not_started), + }; +} + fn respond(conn: *websocket.Conn, status: []const u8, content_type: []const u8, resp_body: []const u8) void { respondMaybeHead(conn, status, content_type, resp_body, false); } diff --git a/tests/test_dashboard_contract.py b/tests/test_dashboard_contract.py index dca5c3a..e50af89 100644 --- a/tests/test_dashboard_contract.py +++ b/tests/test_dashboard_contract.py @@ -72,7 +72,11 @@ class DashboardContractTest(unittest.TestCase): [ "jetstream_backfill_discovered_total", "jetstream_backfill_completion_queued_total", - "jetstream_backfill_progress_completed", + # The series legended "durably committed" must come from the + # metadata store, not from a process-local counter that resets + # on restart. Reading the latter under that label is what made a + # restart look like progress in July. + "stream_backfill_repos_durable", ], [ check_dashboard.METRIC_RE.findall(target["expr"])[0] diff --git a/tools/adapt_dashboard.py b/tools/adapt_dashboard.py index 2bbf939..d3c8e8c 100644 --- a/tools/adapt_dashboard.py +++ b/tools/adapt_dashboard.py @@ -81,11 +81,16 @@ def progress_panel(template: dict) -> dict: legendFormat="processed", refId="B", ) + # jetstream_backfill_progress_completed is process-local: it resets to zero + # on restart and climbs again, so a series labelled "durably committed" was + # showing this process's work rather than the archive's. That mislabel is + # what made a restart read as progress in July. The durable series is read + # from the metadata store at scrape time and continues across a restart. durable = copy.deepcopy(discovered) durable.update( expr=( - 'sum(jetstream_backfill_progress_completed' - '{job=~"$job",instance=~"$instance"})' + 'sum(stream_backfill_repos_durable{status="complete",' + 'job=~"$job",instance=~"$instance"})' ), legendFormat="durably committed", refId="C", diff --git a/tools/check_dashboard.py b/tools/check_dashboard.py index 5eba6cd..8bfd78f 100755 --- a/tools/check_dashboard.py +++ b/tools/check_dashboard.py @@ -17,7 +17,7 @@ UPSTREAM_DASHBOARD = ROOT / "deploy/grafana/jetstream-upstream.json" DEFAULT_DASHBOARD = ROOT / "deploy/grafana/stream.json" EXPECTED_SHA256 = { UPSTREAM_DASHBOARD.resolve(): "ef6033c182d8c3f9d88af7d91e61126724efbe72e6b940984c9de12e9530cd91", - DEFAULT_DASHBOARD.resolve(): "c2600122ad4cf2acd81d013438c5b536cc46f37255ea8cf7e829e7536c9321f9", + DEFAULT_DASHBOARD.resolve(): "d00582d478cdf0c8b00d348dd953c79651c4961e161592bf5eee9f389b3b51d9", } METRIC_RE = re.compile(r"\b(?:jetstream|go|process|stream)_[a-zA-Z_:][a-zA-Z0-9_:]*") TYPE_RE = re.compile(r"^# TYPE ([a-zA-Z_:][a-zA-Z0-9_:]*) (\w+)$")