From 81531310d4efb70b1812a2eca1423b809dc98dfd Mon Sep 17 00:00:00 2001 From: Tim Disney Date: Wed, 22 Jul 2026 15:22:09 -0700 Subject: [PATCH] Phase 4 (merge): harden poller per Codex review (findings 1-3) 1. Restart-safe writes: deterministic rkey merge- + adopt-on-RecordAlreadyExists (fetch existing, no duplicate), mirroring turn-socket.ts. Closes the restart-during-ingestion-lag double write the in-memory alreadyWritten set alone cannot. Export base32Encode. 2. Writer must be a trusted member: restrict actor choice to actors holding an active grant in the index's (per-space) membership view; skip + log once when none, so we never emit merges trustRecords would discard. 3. Clamp poll timing in resolveRunConfig (floor base >=1ms, raise cap to the base) and cap the poller's first backoff step against a base > cap misconfig. Co-Authored-By: Claude Fable 5 --- packages/daemon/src/cli.ts | 8 ++- packages/daemon/src/merge-poll.ts | 111 ++++++++++++++++++++++------- packages/daemon/src/turn-socket.ts | 2 +- 3 files changed, 94 insertions(+), 27 deletions(-) diff --git a/packages/daemon/src/cli.ts b/packages/daemon/src/cli.ts index 37f7c5b..35fc987 100644 --- a/packages/daemon/src/cli.ts +++ b/packages/daemon/src/cli.ts @@ -174,6 +174,7 @@ export function resolveRunConfig(run: DaemonRunConfig): ResolvedRunConfig { "Desktop, where bind-mounted unix sockets don't work).", ) } + const mergePollInterval = Math.max(1, run.mergePollIntervalMs ?? 60_000) return { spaces: run.spaces, stateDir: run.stateDir ?? defaultRunStateDir(), @@ -182,8 +183,11 @@ export function resolveRunConfig(run: DaemonRunConfig): ResolvedRunConfig { timeoutMs: run.timeoutMs ?? 900_000, cooldownMs: run.cooldownMs ?? 300_000, retryBound: run.retryBound ?? 3, - mergePollIntervalMs: run.mergePollIntervalMs ?? 60_000, - mergePollBackoffMaxMs: run.mergePollBackoffMaxMs ?? 30 * 60_000, + // Clamp poll timing: a zero/negative base interval would busy-poll the forge, so floor it at + // 1ms; the backoff cap is floored at the base interval so `backoffMaxMs < pollIntervalMs` can + // never yield an uncapped first delay (the poller also caps its first step defensively). + mergePollIntervalMs: mergePollInterval, + mergePollBackoffMaxMs: Math.max(mergePollInterval, run.mergePollBackoffMaxMs ?? 30 * 60_000), ...(run.network !== undefined ? { network: run.network } : egress diff --git a/packages/daemon/src/merge-poll.ts b/packages/daemon/src/merge-poll.ts index 0e4e150..9ffb916 100644 --- a/packages/daemon/src/merge-poll.ts +++ b/packages/daemon/src/merge-poll.ts @@ -1,3 +1,5 @@ +import { createHash } from 'node:crypto' +import { XrpcError } from '@radial/atproto' import { COLLECTIONS, type ArtifactRecord, @@ -6,6 +8,7 @@ import { type MergeRecord, } from '@radial/core' import type { ActorRegistry, LoadedActor } from './actors.js' +import { base32Encode } from './turn-socket.js' /** * The forge state source the poller reads (design §10). A minimal LOCAL structural type so this @@ -31,18 +34,33 @@ export interface MergeCandidate { const artifactKey = (record: { uri: string; cid: string }): string => `${record.uri}#${record.cid}` /** - * The writing actor is deterministic (design decision, locked): the `implementation`-producing - * actor if one is loaded, else the lowest DID among all loaded actors. Ties within the - * implementation-producing set also break by lowest DID. Returns undefined only when no actor is - * loaded at all (nothing can write). + * Deterministic rkey for a merge annotation of an exact artifact version: `merge-` plus 24 base32 + * chars of sha256(artifactUri#artifactCid). Per-actor repos make this unique per daemon-actor, so a + * restart that re-derives the same rkey collides on the PDS (RecordAlreadyExists) instead of writing + * a second record — closing the restart-during-ingestion-lag double-write that the in-memory + * `alreadyWritten` set alone cannot (it is lost on restart). Mirrors `planArtifactRkey` in + * turn-socket.ts and its RecordAlreadyExists adoption path. */ -export function selectWritingActor(actors: ActorRegistry): LoadedActor | undefined { +export function mergeRkey(artifactUri: string, artifactCid: string): string { + const digest = createHash('sha256').update(`${artifactUri}#${artifactCid}`).digest() + return `merge-${base32Encode(digest).slice(0, 24)}` +} + +/** + * The writing actor is deterministic (design decision, locked): among the ELIGIBLE actors, the + * `implementation`-producing one if present, else the lowest DID. Ties within the + * implementation-producing set also break by lowest DID. `eligible` must already be restricted to + * actors that hold an active membership grant in the target space (see `selectMergeCandidates`), so + * a merge is never written under a DID whose records the space would discard. Returns undefined when + * no eligible actor exists. + */ +export function selectWritingActor(eligible: LoadedActor[]): LoadedActor | undefined { const byDid = (a: LoadedActor, b: LoadedActor): number => a.did.localeCompare(b.did) - const implementers = actors.all + const implementers = eligible .filter((actor) => actor.artifactTypes.includes('implementation')) .sort(byDid) if (implementers[0]) return implementers[0] - return [...actors.all].sort(byDid)[0] + return [...eligible].sort(byDid)[0] } /** @@ -51,16 +69,29 @@ export function selectWritingActor(actors: ActorRegistry): LoadedActor | undefin * (the in-process guard against the sync-lag double-write race), and (c) has no trusted merge in the * index authored by one of OUR actor DIDs — the "a given daemon writes at most one merge per * artifact" rule, deduped against already-ingested writes. Equivalent merges from OTHER operators' - * DIDs are intentionally ignored here: each daemon still posts its own single annotation. Ordered by - * artifact uri so polling/scheduling is stable across ticks. + * DIDs are intentionally ignored here: each daemon still posts its own single annotation. + * + * The writing actor is restricted to actors holding an active membership grant in THIS index's space + * (materialize is per-space, so `index.members` is that space's trust view). If no loaded actor is + * an active member, every candidate is skipped and `onSkip` is invoked once — otherwise the daemon + * would emit merge records that `trustRecords` discards, producing no merged signal and endless + * ignored writes. Ordered by artifact uri so polling/scheduling is stable across ticks. */ export function selectMergeCandidates( index: MaterializedIndex, actors: ActorRegistry, alreadyWritten: Set, + onSkip?: (reason: string) => void, ): MergeCandidate[] { - const actor = selectWritingActor(actors) - if (!actor) return [] + const activeMemberDids = new Set( + index.members.filter((member) => member.active).map((member) => member.did), + ) + const eligible = actors.all.filter((actor) => activeMemberDids.has(actor.did)) + const actor = selectWritingActor(eligible) + if (!actor) { + onSkip?.('no loaded actor holds an active membership grant in this space to author a merge') + return [] + } const ourDids = new Set(actors.all.map((candidate) => candidate.did)) const candidates: MergeCandidate[] = [] @@ -85,7 +116,7 @@ export interface MergePollerDeps { adapter: ForgeStateSource /** Monotonic wall clock in epoch milliseconds; injectable so tests drive backoff without timers. */ now: () => number - /** Base per-PR poll interval; also the first backoff step. */ + /** Base per-PR poll interval; also the first backoff step (capped at `backoffMaxMs`). */ pollIntervalMs: number /** Exponential backoff cap. */ backoffMaxMs: number @@ -101,20 +132,24 @@ interface PrSchedule { * Background merge-observation poller, modeled on `TurnDispatcher`: `pump` fires off forge lookups * for eligible PRs and returns immediately — it never awaits a lookup, so it never blocks the * ingestion loop that calls it. Per-PR exponential backoff (base `pollIntervalMs`, capped at - * `backoffMaxMs`) is applied on open/closed/error; a merged PR gets exactly one merge annotation - * written (via the selected actor) and is then dropped from scheduling forever. + * `backoffMaxMs` — and the first step is likewise capped, so a misconfigured base > cap can't + * produce an uncapped first delay) is applied on open/closed/error; a merged PR gets exactly one + * merge annotation written (via the selected actor) and is then dropped from scheduling forever. * - * The in-process `#alreadyWritten` set closes the sync-lag double-write race — the same one - * `dispatch.ts` documents for its in-flight guard: between writing a merge and the next ingestion - * cycle surfacing it in `index.merges`, `selectMergeCandidates`'s index-based dedup can't see our - * own write yet, so without this set a second `pump` in that window would write a duplicate. Keyed - * by the artifact strongref (uri#cid), exactly the key the index-based dedup uses once caught up. + * Double-write defense is two-layered. Within a process, the in-memory `#alreadyWritten` set closes + * the sync-lag race — the same one `dispatch.ts` documents — between writing a merge and the next + * ingestion cycle surfacing it in `index.merges`. Across a restart (which loses that set), the + * DETERMINISTIC rkey (`mergeRkey`) makes a re-derived write collide on the PDS; the collision is + * ADOPTED as success (fetch the existing record, no duplicate), exactly like turn-socket.ts's + * artifact submission. */ export class MergePoller { readonly #deps: MergePollerDeps readonly #schedules = new Map() readonly #inFlight = new Map>() readonly #alreadyWritten = new Set() + /** Skip reasons already logged, so a stable no-eligible-writer condition is reported once. */ + readonly #loggedSkips = new Set() constructor(deps: MergePollerDeps) { this.#deps = deps @@ -124,15 +159,25 @@ export class MergePoller { return this.#inFlight.size } + /** The first backoff step, never above the cap (defends a base > cap misconfiguration). */ + #initialBackoff(): number { + return Math.min(this.#deps.pollIntervalMs, this.#deps.backoffMaxMs) + } + /** Launches due forge lookups for eligible PRs; returns immediately. */ pump(index: MaterializedIndex, actors: ActorRegistry): void { const now = this.#deps.now() - for (const candidate of selectMergeCandidates(index, actors, this.#alreadyWritten)) { + const candidates = selectMergeCandidates(index, actors, this.#alreadyWritten, (reason) => { + if (this.#loggedSkips.has(reason)) return + this.#loggedSkips.add(reason) + this.#deps.log?.(`not observing merges: ${reason}`) + }) + for (const candidate of candidates) { const { prUrl } = candidate if (this.#inFlight.has(prUrl)) continue const schedule = this.#schedules.get(prUrl) if (schedule && schedule.nextPollAt > now) continue - if (!schedule) this.#schedules.set(prUrl, { nextPollAt: 0, backoffMs: this.#deps.pollIntervalMs }) + if (!schedule) this.#schedules.set(prUrl, { nextPollAt: 0, backoffMs: this.#initialBackoff() }) const promise = this.#poll(candidate).finally(() => { this.#inFlight.delete(prUrl) @@ -169,11 +214,29 @@ export class MergePoller { ...(result.mergedAt !== undefined ? { mergedAt: result.mergedAt } : {}), createdAt: new Date(this.#deps.now()).toISOString(), } + const rkey = mergeRkey(artifact.uri, artifact.cid) try { - const ref = await actor.client.create(COLLECTIONS.merge, record) + let ref: { uri: string; cid: string } + try { + ref = await actor.client.create(COLLECTIONS.merge, record, { rkey }) + } catch (error) { + // Restart-safe adoption: a deterministic-rkey collision means we (this actor) already wrote + // this merge in a prior process; adopt the existing record as success rather than duplicate. + if ( + error instanceof XrpcError && + error.status === 400 && + error.error === 'RecordAlreadyExists' + ) { + const existing = await actor.client.getOwnRecord(COLLECTIONS.merge, rkey) + if (!existing) throw error + ref = { uri: existing.uri, cid: existing.cid } + } else { + throw error + } + } this.#alreadyWritten.add(key) this.#schedules.delete(prUrl) // merged is terminal: stop scheduling this PR forever. - this.#deps.log?.(`wrote merge for ${prUrl} (${artifact.uri}) as ${actor.did}: ${ref.uri}`) + this.#deps.log?.(`observed merge for ${prUrl} (${artifact.uri}) as ${actor.did}: ${ref.uri}`) } catch (error) { const message = error instanceof Error ? error.message : String(error) this.#deps.log?.(`failed to write merge for ${prUrl}: ${message}; backing off`) @@ -185,7 +248,7 @@ export class MergePoller { const now = this.#deps.now() const schedule = this.#schedules.get(prUrl) ?? { nextPollAt: 0, - backoffMs: this.#deps.pollIntervalMs, + backoffMs: this.#initialBackoff(), } schedule.nextPollAt = now + schedule.backoffMs schedule.backoffMs = Math.min(schedule.backoffMs * 2, this.#deps.backoffMaxMs) diff --git a/packages/daemon/src/turn-socket.ts b/packages/daemon/src/turn-socket.ts index ac340b5..aeceee5 100644 --- a/packages/daemon/src/turn-socket.ts +++ b/packages/daemon/src/turn-socket.ts @@ -54,7 +54,7 @@ export type TurnListen = { kind: 'unix'; path: string } | { kind: 'tcp'; host: s const BASE32_ALPHABET = 'abcdefghijklmnopqrstuvwxyz234567' -function base32Encode(bytes: Uint8Array): string { +export function base32Encode(bytes: Uint8Array): string { let bits = 0 let value = 0 let output = '' -- knotmirror 0.0.1