Something went wrong. Try again.
prefect server in zig
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784//! In-process background-task queue + worker tracker.//!//! Mirrors python prefect's `server/task_queue.py` + `server/models/task_workers.py`//! shape. Two pieces://!//! 1. Per-`task_key` queue (scheduled + retry). When a task_run lands in//! SCHEDULED with `state_details.deferred=true`, the API layer calls//! `enqueue(task_key, ...)` and we try to hand it to a registered//! worker immediately. If no worker is listening for that key, the//! task_run sits in the queue until one subscribes (or until the//! server restarts — same single-replica risk as the python impl,//! filed upstream as PrefectHQ/prefect#21218).//!//! 2. Worker tracker. Each `/task_runs/subscriptions/scheduled` WS//! connection registers via `addSubscriber()` with its task_keys +//! client_id. The tracker answers `POST /task_workers/filter`.//!//! Dispatch model is exclusive consumption: one task_run → one worker. We//! attempt dispatch on two events: (a) a task_run is enqueued, (b) a//! subscriber acks the previous run. Subscribers are marked `busy` from//! the moment we send them a task_run until they ack — within that window//! they're skipped by `tryDispatch`. On WS send failure or disconnect,//! any in-flight run for that subscriber is pushed back onto the retry//! sub-queue so another worker can pick it up.//!//! Today this is a memory-only singleton. Multi-replica safety (redis//! backend; see issue #21218) follows once the memory path is wire-correct.
const std = @import("std");const Allocator = std.mem.Allocator;const Io = std.Io;const ws = @import("websocket");const redis = @import("redis");
const log = @import("logging.zig");const broker = @import("broker.zig");const time_util = @import("utilities/time.zig");const sync = @import("utilities/sync.zig");const observability = @import("observability.zig");
const WsHandle = *ws.Conn;
const MAX_SUBSCRIBERS = 256;
/// Memory backend keeps queues in-process; redis backend pushes payloads to/// Redis LISTs and uses the broker for cross-replica dispatch hints. The/// subscriber registry is in-process either way — WS connections are local/// to whichever replica accepted them. Cross-replica visibility for/// `POST /task_workers/filter` lives on a TTL'd `prefect:workers:{id}`/// key in redis mode.pub const Dialect = enum { memory, redis };
/// Broker topic for cross-replica dispatch hints. Replica A enqueues to a/// redis LIST, publishes `{task_key}` here, and every other replica's/// task_workers consumes the hint and attempts to drain matching local/// subscribers.const DISPATCH_HINT_TOPIC = "task-workers-dispatch-hint";const PRESENCE_TTL_SECONDS: u32 = 30;
/// One enqueued task_run waiting to be picked up by a worker. We hold the/// fully-rendered JSON payload so dispatch is a memcpy + writeText with no/// DB read.const QueuedRun = struct { task_run_id: [36]u8 = undefined, task_run_id_len: u8 = 0, /// Owned by `task_workers.allocator`. Freed on dispatch or queue drop. payload_json: []const u8, /// True if this run came from the retry sub-queue (drained first). is_retry: bool = false,};
/// Per-task_key queue. Scheduled drains after retry.const KeyQueue = struct { scheduled: std.ArrayListUnmanaged(QueuedRun) = .empty, retry: std.ArrayListUnmanaged(QueuedRun) = .empty,};
const Subscriber = struct { /// Valid only when `active` is true. handle: WsHandle = undefined, /// Heap-owned strings; freed in removeSubscriber. client_id: []const u8 = "", /// Heap-owned slice of heap-owned strings. task_keys: []const []const u8 = &.{}, active: bool = false, /// True between "we sent this worker a task_run" and "they acked." Skipped /// by tryDispatch while busy to enforce exclusive consumption. busy: bool = false, /// Set when busy. Used on disconnect to push the in-flight run back to /// the retry queue under its task_key. nil-terminated by `inflight_key_len`. inflight_task_key_buf: [256]u8 = undefined, inflight_task_key_len: usize = 0, /// Wall-clock at last observe (re-observed on subscribe + on each ack). /// Used by /task_workers/filter to age out stale entries. last_seen_ms: i64 = 0,};
var instance: ?*TaskWorkers = null;var storage: TaskWorkers = undefined;
/// Initialize the singleton. Idempotent. Call once during server startup./// Returns an error if dialect=redis and the broker subscribe fails: that/// would put us in a state where cross-replica dispatch silently doesn't/// work, which is the entire reason for the redis backend.pub fn init(allocator: Allocator, io: Io, dialect: Dialect) !void { if (instance != null) return; storage = TaskWorkers.init(allocator, io, dialect); try storage.start(); instance = &storage;}
/// Tear down the singleton. Idempotent.pub fn deinit() void { if (instance) |s| { s.stop(); } instance = null;}
pub const TaskWorkers = struct { allocator: Allocator, io: Io, dialect: Dialect = .memory,
/// Guards queues + subscribers. Both are touched from API-handler threads /// (enqueue + WS callbacks) so a single mutex is simplest. mutex: Io.Mutex = Io.Mutex.init,
/// Memory dialect: `task_key` → KeyQueue. Lazily initialized as keys /// appear via enqueue or subscribe — both can be the first to observe a /// new key. Unused in redis dialect (queues live in redis). queues: std.StringHashMapUnmanaged(KeyQueue) = .empty,
subscribers: [MAX_SUBSCRIBERS]Subscriber = undefined, subscriber_count: usize = 0, subscribers_initialized: bool = false,
/// Redis-only: broker consumer handle for cross-replica dispatch hints. dispatch_hint_handle: ?broker.ConsumerHandle = null,
pub fn init(allocator: Allocator, io: Io, dialect: Dialect) TaskWorkers { // Eagerly initialize the subscribers array so every read path can // assume `active` is a defined bool. Lazy init was easy to get wrong // — snapshotWorkers on a replica that hasn't seen any subscribers // would otherwise iterate undefined memory. var tw: TaskWorkers = .{ .allocator = allocator, .io = io, .dialect = dialect }; for (&tw.subscribers) |*s| s.* = .{}; tw.subscribers_initialized = true; return tw; }
pub fn start(self: *TaskWorkers) !void { instance = self; if (self.dialect == .redis) { // Subscribe ephemerally so every replica receives every hint — // analogous to events broadcaster. The hint payload is the // task_key; on receipt we attempt local dispatch from redis. // Hard error if subscribe fails: without it we can't act on // work enqueued by other replicas, which is the entire reason // for the redis backend (PrefectHQ/prefect#21218). const b = broker.getBroker() orelse { log.err("task_workers", "redis dialect requested but no broker available", .{}); return error.BrokerUnavailable; }; const ts = time_util.milliTimestamp(self.io); var group_buf: [64]u8 = undefined; const group = std.fmt.bufPrint(&group_buf, "dispatch-hint-{d}", .{ts}) catch "dispatch-hint"; self.dispatch_hint_handle = try b.subscribeEphemeral( DISPATCH_HINT_TOPIC, group, onDispatchHintCallback, ); log.info("task_workers", "started (redis backend)", .{}); } else { log.info("task_workers", "started (memory backend)", .{}); } }
pub fn stop(self: *TaskWorkers) void { if (self.dispatch_hint_handle) |h| { if (broker.getBroker()) |b| b.unsubscribe(h); self.dispatch_hint_handle = null; } instance = null; }
fn initSubscribers(self: *TaskWorkers) void { if (self.subscribers_initialized) return; for (&self.subscribers) |*s| s.active = false; self.subscribers_initialized = true; }
/// Push a task_run onto the queue for `task_key` and immediately try to /// hand it to a registered worker. If no worker is currently free for /// that key, the run waits. /// /// `payload_json` is duplicated into the allocator; caller keeps ownership /// of theirs. pub fn enqueue(self: *TaskWorkers, task_key: []const u8, task_run_id: []const u8, payload_json: []const u8) !void { observability.task_workers_enqueued_total.inc(); if (self.dialect == .redis) { // Push payload to redis LIST (LPUSH/RPOP gives FIFO). try self.redisEnqueue(task_key, payload_json, false); log.debug("task_workers", "redis-enqueued task_run {s} (key={s})", .{ task_run_id, task_key });
// Fan out dispatch hint to every replica so whoever has the // subscriber drains it. Failure here is a real error — without // the hint, work enqueued on this replica is invisible to // workers connected to other replicas until something else // triggers their drain. Propagate; the caller decides whether // that's recoverable. const b = broker.getBroker() orelse return error.BrokerUnavailable; try b.publish(DISPATCH_HINT_TOPIC, task_run_id, task_key);
// Also try our own local subscribers immediately. self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io); self.initSubscribers(); self.tryDispatchRedisLocked(task_key); return; }
self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io);
self.initSubscribers();
const owned_payload = try self.allocator.dupe(u8, payload_json); errdefer self.allocator.free(owned_payload);
var run = QueuedRun{ .payload_json = owned_payload }; const id_copy_len = @min(task_run_id.len, run.task_run_id.len); @memcpy(run.task_run_id[0..id_copy_len], task_run_id[0..id_copy_len]); run.task_run_id_len = @intCast(id_copy_len);
const q = try self.getOrCreateQueue(task_key); try q.scheduled.append(self.allocator, run);
log.debug("task_workers", "enqueued task_run {s} (key={s}, depth={d})", .{ task_run_id, task_key, q.scheduled.items.len + q.retry.items.len, });
self.tryDispatchLocked(task_key); }
fn getOrCreateQueue(self: *TaskWorkers, task_key: []const u8) !*KeyQueue { const gop = try self.queues.getOrPut(self.allocator, task_key); if (!gop.found_existing) { // own the key string gop.key_ptr.* = try self.allocator.dupe(u8, task_key); gop.value_ptr.* = .{}; } return gop.value_ptr; }
/// Register a WS subscriber. `task_keys` and `client_id` are duplicated /// into the allocator; caller keeps their copies. /// /// Returns false if at MAX_SUBSCRIBERS capacity. pub fn addSubscriber(self: *TaskWorkers, handle: WsHandle, client_id: []const u8, task_keys: []const []const u8) !bool { self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io);
self.initSubscribers();
for (&self.subscribers) |*s| { if (!s.active) { const owned_client = try self.allocator.dupe(u8, client_id); errdefer self.allocator.free(owned_client);
const owned_keys = try self.allocator.alloc([]const u8, task_keys.len); errdefer self.allocator.free(owned_keys); var dup_count: usize = 0; errdefer for (owned_keys[0..dup_count]) |k| self.allocator.free(k); for (task_keys, 0..) |k, i| { owned_keys[i] = try self.allocator.dupe(u8, k); dup_count = i + 1; }
s.handle = handle; s.client_id = owned_client; s.task_keys = owned_keys; s.active = true; s.busy = false; s.inflight_task_key_len = 0; s.last_seen_ms = time_util.milliTimestamp(self.io); self.subscriber_count += 1; observability.task_workers_subscribers.inc();
log.info("task_workers", "subscriber {s} registered for keys={d} (total {d})", .{ client_id, task_keys.len, self.subscriber_count, });
if (self.dialect == .redis) { self.publishPresence(s) catch |err| { log.warn("task_workers", "presence publish for {s} failed: {} — filter API may not see this worker from other replicas", .{ client_id, err }); }; }
// try to drain any pending work this subscriber can take for (owned_keys) |k| { if (self.dialect == .redis) self.tryDispatchRedisLocked(k) else self.tryDispatchLocked(k); }
return true; } }
log.warn("task_workers", "at capacity ({d}), rejecting new subscriber", .{MAX_SUBSCRIBERS}); return false; }
pub fn removeSubscriber(self: *TaskWorkers, handle: WsHandle) void { self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io);
for (&self.subscribers) |*s| { if (s.active and s.handle == handle) { // If they were in-flight on a task_run, that run is at risk // of being lost. Today we don't track the QueuedRun payload // after dispatch, so the run is gone if the worker // disconnects mid-ack. (Python has the same issue — when // websocket fails post-send and pre-ack, it pushes to retry // queue from a stored copy.) Track this gap explicitly: if (s.busy and s.inflight_task_key_len > 0) { const key = s.inflight_task_key_buf[0..s.inflight_task_key_len]; log.warn("task_workers", "subscriber {s} disconnected mid-flight on key={s} — task_run may be stuck", .{ s.client_id, key, }); }
if (self.dialect == .redis) { self.deletePresence(s.client_id) catch |err| { log.warn("task_workers", "presence delete failed: {}", .{err}); }; }
self.freeSubscriberOwned(s); s.active = false; self.subscriber_count -= 1; observability.task_workers_subscribers.dec(); log.debug("task_workers", "subscriber removed (total {d})", .{self.subscriber_count}); return; } } }
fn freeSubscriberOwned(self: *TaskWorkers, s: *Subscriber) void { if (s.client_id.len > 0) self.allocator.free(s.client_id); for (s.task_keys) |k| self.allocator.free(k); if (s.task_keys.len > 0) self.allocator.free(s.task_keys); s.client_id = ""; s.task_keys = &.{}; }
/// Called when a subscriber sends `{"type":"ack"}` after processing a run. pub fn ack(self: *TaskWorkers, handle: WsHandle) void { self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io);
for (&self.subscribers) |*s| { if (s.active and s.handle == handle) { s.busy = false; s.inflight_task_key_len = 0; s.last_seen_ms = time_util.milliTimestamp(self.io); if (self.dialect == .redis) { self.publishPresence(s) catch {}; } // try to immediately give them the next run for (s.task_keys) |k| { if (self.dialect == .redis) self.tryDispatchRedisLocked(k) else self.tryDispatchLocked(k); } return; } } }
/// Send one queued run for `task_key` to any free subscriber that listens /// for that key. Caller must hold the mutex. fn tryDispatchLocked(self: *TaskWorkers, task_key: []const u8) void { const q = self.queues.getPtr(task_key) orelse return; if (q.scheduled.items.len == 0 and q.retry.items.len == 0) return;
// pick a free subscriber listening for this key for (&self.subscribers) |*s| { if (!s.active or s.busy) continue; var listens = false; for (s.task_keys) |k| { if (std.mem.eql(u8, k, task_key)) { listens = true; break; } } if (!listens) continue;
// pop next: retry drains first const run = if (q.retry.items.len > 0) q.retry.orderedRemove(0) else q.scheduled.orderedRemove(0); defer self.allocator.free(run.payload_json);
s.handle.writeText(run.payload_json) catch |err| { observability.task_workers_send_failed_total.inc(); observability.task_workers_requeued_retry_total.inc(); log.warn("task_workers", "writeText to subscriber {s} failed: {} — re-enqueueing to retry", .{ s.client_id, err }); self.requeueRetryLocked(task_key, run.payload_json) catch |re_err| { log.err("task_workers", "failed to re-enqueue: {}", .{re_err}); }; self.freeSubscriberOwned(s); s.active = false; s.busy = false; self.subscriber_count -|= 1; observability.task_workers_subscribers.dec(); return; };
s.busy = true; const klen = @min(task_key.len, s.inflight_task_key_buf.len); @memcpy(s.inflight_task_key_buf[0..klen], task_key[0..klen]); s.inflight_task_key_len = klen; s.last_seen_ms = time_util.milliTimestamp(self.io);
observability.task_workers_dispatched_total.inc(); log.debug("task_workers", "dispatched to {s} (key={s})", .{ s.client_id, task_key }); return; } }
/// Push `payload_json` (caller-owned) back onto the retry sub-queue under /// `task_key`. We dupe so the caller can free their copy. fn requeueRetryLocked(self: *TaskWorkers, task_key: []const u8, payload_json: []const u8) !void { const owned = try self.allocator.dupe(u8, payload_json); errdefer self.allocator.free(owned); const q = try self.getOrCreateQueue(task_key); try q.retry.append(self.allocator, .{ .payload_json = owned, .is_retry = true }); }
// ======================================================================== // Redis backend helpers // ========================================================================
/// LPUSH the payload onto either the scheduled or retry list for this /// task_key. RPOP at dispatch time gives FIFO; retry list is drained /// first. fn redisEnqueue(self: *TaskWorkers, task_key: []const u8, payload_json: []const u8, is_retry: bool) !void { var client = try connectRedis(self.allocator); defer client.close(); var keys_buf: [256]u8 = undefined; const key = try formatQueueKey(&keys_buf, task_key, is_retry); var lists = client.lists(); const values = [_][]const u8{payload_json}; _ = try lists.lpush(key, &values); }
/// Drain redis: drain retry list first, then scheduled. Hand each popped /// payload to a free local subscriber listening for this key. Caller /// must hold the mutex. fn tryDispatchRedisLocked(self: *TaskWorkers, task_key: []const u8) void { var dispatched_any = true; while (dispatched_any) { dispatched_any = false;
// pick a free local subscriber listening for this key const sub: ?*Subscriber = blk: { for (&self.subscribers) |*s| { if (!s.active or s.busy) continue; for (s.task_keys) |k| { if (std.mem.eql(u8, k, task_key)) break :blk s; } } break :blk null; }; const s = sub orelse return;
// pop from redis: retry first, then scheduled const payload = self.redisPopFor(task_key) catch |err| { log.warn("task_workers", "redis pop for key={s} failed: {}", .{ task_key, err }); return; }; const owned_payload = payload orelse return; // queue empty defer self.allocator.free(owned_payload);
s.handle.writeText(owned_payload) catch |err| { observability.task_workers_send_failed_total.inc(); observability.task_workers_requeued_retry_total.inc(); log.warn("task_workers", "writeText to {s} failed: {} — re-enqueueing to retry", .{ s.client_id, err }); self.redisEnqueue(task_key, owned_payload, true) catch |re_err| { log.err("task_workers", "retry re-enqueue failed: {}", .{re_err}); }; self.freeSubscriberOwned(s); s.active = false; s.busy = false; self.subscriber_count -|= 1; observability.task_workers_subscribers.dec(); return; };
s.busy = true; const klen = @min(task_key.len, s.inflight_task_key_buf.len); @memcpy(s.inflight_task_key_buf[0..klen], task_key[0..klen]); s.inflight_task_key_len = klen; s.last_seen_ms = time_util.milliTimestamp(self.io);
observability.task_workers_dispatched_total.inc(); log.debug("task_workers", "redis-dispatched to {s} (key={s})", .{ s.client_id, task_key }); dispatched_any = true; } }
/// Caller owns the returned slice (allocated via self.allocator). Returns /// null if both retry and scheduled lists are empty. fn redisPopFor(self: *TaskWorkers, task_key: []const u8) !?[]const u8 { var client = try connectRedis(self.allocator); defer client.close(); var keys_buf: [256]u8 = undefined; var lists = client.lists();
// retry first const retry_key = try formatQueueKey(&keys_buf, task_key, true); if (try lists.rpop(retry_key)) |val| { const owned = try self.allocator.dupe(u8, val); return owned; }
const sched_key = try formatQueueKey(&keys_buf, task_key, false); if (try lists.rpop(sched_key)) |val| { const owned = try self.allocator.dupe(u8, val); return owned; } return null; }
/// Write `prefect:workers:{client_id}` with the worker's task_keys and /// last_seen, TTL'd so a crashed worker ages out without explicit DELETE. /// `s` is read under the caller's mutex; we don't hold any extra lock. fn publishPresence(self: *TaskWorkers, s: *const Subscriber) !void { var client = try connectRedis(self.allocator); defer client.close();
// Build JSON payload: {"task_keys":[...],"last_seen_ms":N} var buf: std.Io.Writer.Allocating = .init(self.allocator); defer buf.deinit(); var jw: std.json.Stringify = .{ .writer = &buf.writer }; try jw.beginObject(); try jw.objectField("task_keys"); try jw.beginArray(); for (s.task_keys) |k| try jw.write(k); try jw.endArray(); try jw.objectField("last_seen_ms"); try jw.write(s.last_seen_ms); try jw.endObject();
var key_buf: [256]u8 = undefined; const key = try std.fmt.bufPrint(&key_buf, "prefect:workers:{s}", .{s.client_id}); var strings = client.strings(); try strings.set(key, buf.written()); var keys = client.keys(); _ = try keys.expire(key, PRESENCE_TTL_SECONDS); }
fn deletePresence(self: *TaskWorkers, client_id: []const u8) !void { var client = try connectRedis(self.allocator); defer client.close(); var key_buf: [256]u8 = undefined; const key = try std.fmt.bufPrint(&key_buf, "prefect:workers:{s}", .{client_id}); var keys = client.keys(); _ = try keys.del(&.{key}); }};
// ============================================================================// Redis helpers (file scope so dispatch callback can call them)// ============================================================================
fn connectRedis(alloc: Allocator) !redis.Client { const cfg = broker.parseRedisConfig(); return redis.Client.connectWithConfig(sync.globalIo(), alloc, .{ .host = cfg.host, .port = cfg.port, .username = cfg.username, .password = cfg.password, .db = cfg.db, });}
fn formatQueueKey(buf: *[256]u8, task_key: []const u8, is_retry: bool) ![]const u8 { const suffix = if (is_retry) "retry" else "scheduled"; return std.fmt.bufPrint(buf, "prefect:tasks:{s}:{s}", .{ task_key, suffix });}
/// Broker callback for dispatch hints (redis backend). The hint payload is/// the task_key string; we try to drain one of our local subscribers.fn onDispatchHintCallback(msg: *const broker.Message) anyerror!void { const self = instance orelse return; if (self.dialect != .redis) return; const task_key = msg.data; self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io); self.initSubscribers(); self.tryDispatchRedisLocked(task_key);}
// ============================================================================// Free-function shims for handler call sites// ============================================================================
pub fn enqueue(task_key: []const u8, task_run_id: []const u8, payload_json: []const u8) void { const self = instance orelse { log.warn("task_workers", "enqueue({s}) but task_workers not running", .{task_key}); return; }; self.enqueue(task_key, task_run_id, payload_json) catch |err| { log.err("task_workers", "enqueue failed: {}", .{err}); };}
pub fn addSubscriber(handle: WsHandle, client_id: []const u8, task_keys: []const []const u8) !bool { const self = instance orelse return false; return self.addSubscriber(handle, client_id, task_keys);}
pub fn removeSubscriber(handle: WsHandle) void { const self = instance orelse return; self.removeSubscriber(handle);}
pub fn ack(handle: WsHandle) void { const self = instance orelse return; self.ack(handle);}
/// Snapshot of currently-registered workers for /task_workers/filter.pub const WorkerSnapshot = struct { client_id: []const u8, task_keys: []const []const u8, last_seen_ms: i64,};
/// `task_keys` is the filter: if empty, return all workers; otherwise return/// only workers listening for at least one matching key. Allocations are/// duplicated into `alloc`.////// In redis mode, also walks `prefect:workers:*` to include workers/// connected to other replicas. Local subscribers and remote ones can/// collide on client_id (a redis presence key from this replica matches a/// local subscriber); the local one wins.pub fn snapshotWorkers(alloc: Allocator, task_keys_filter: []const []const u8) ![]WorkerSnapshot { const self = instance orelse return alloc.alloc(WorkerSnapshot, 0);
var list = std.ArrayListUnmanaged(WorkerSnapshot).empty; errdefer list.deinit(alloc);
var seen_client_ids = std.StringHashMapUnmanaged(void).empty; defer seen_client_ids.deinit(alloc);
{ self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io);
for (&self.subscribers) |*s| { if (!s.active) continue; if (task_keys_filter.len > 0) { var any = false; outer: for (s.task_keys) |sk| { for (task_keys_filter) |fk| { if (std.mem.eql(u8, sk, fk)) { any = true; break :outer; } } } if (!any) continue; } const client_dup = try alloc.dupe(u8, s.client_id); const keys_dup = try alloc.alloc([]const u8, s.task_keys.len); for (s.task_keys, 0..) |k, i| keys_dup[i] = try alloc.dupe(u8, k); try list.append(alloc, .{ .client_id = client_dup, .task_keys = keys_dup, .last_seen_ms = s.last_seen_ms, }); try seen_client_ids.put(alloc, client_dup, {}); } }
if (self.dialect == .redis) { // SCAN prefect:workers:* and merge in any non-local presence. appendRemoteWorkers(alloc, &list, &seen_client_ids, task_keys_filter) catch |err| { log.warn("task_workers", "remote worker SCAN failed: {} — returning only local", .{err}); }; }
return list.toOwnedSlice(alloc);}
fn appendRemoteWorkers( alloc: Allocator, list: *std.ArrayListUnmanaged(WorkerSnapshot), seen: *std.StringHashMapUnmanaged(void), task_keys_filter: []const []const u8,) !void { var client = try connectRedis(alloc); defer client.close(); var keys = client.keys(); var strings = client.strings();
// Slices returned from redis commands point into the client's read buffer // and are clobbered by the NEXT command. So: collect all "prefect:workers:*" // keys (duped) across all SCAN iterations first, then GET each in a second // pass. Within the GET pass, dupe the body before any subsequent command. var full_keys = std.ArrayListUnmanaged([]const u8).empty; defer { for (full_keys.items) |k| alloc.free(k); full_keys.deinit(alloc); }
var cursor_storage: []const u8 = try alloc.dupe(u8, "0"); defer alloc.free(cursor_storage);
while (true) { const res = try keys.scan(cursor_storage, "prefect:workers:*", 100); for (res.keys) |kv| { // Redis bulk strings come back as `.bulk` (nullable); simple // strings (`.string`) are reserved for status replies like OK. // We accept either form defensively. const s = switch (kv) { .bulk => |b| b orelse continue, .string => |s| s, else => continue, }; try full_keys.append(alloc, try alloc.dupe(u8, s)); } if (std.mem.eql(u8, res.cursor, "0")) break; const next_cursor = try alloc.dupe(u8, res.cursor); alloc.free(cursor_storage); cursor_storage = next_cursor; }
const PREFIX = "prefect:workers:"; for (full_keys.items) |full_key| { if (!std.mem.startsWith(u8, full_key, PREFIX)) continue; const client_id = full_key[PREFIX.len..]; if (seen.contains(client_id)) continue;
const ephemeral_body = (try strings.get(full_key)) orelse continue; const body = try alloc.dupe(u8, ephemeral_body); defer alloc.free(body);
const parsed = std.json.parseFromSlice(std.json.Value, alloc, body, .{}) catch continue; if (parsed.value != .object) continue; const task_keys_val = parsed.value.object.get("task_keys") orelse continue; if (task_keys_val != .array) continue; const last_seen_val = parsed.value.object.get("last_seen_ms") orelse continue; if (last_seen_val != .integer) continue;
if (task_keys_filter.len > 0) { var any = false; outer: for (task_keys_val.array.items) |sk| { if (sk != .string) continue; for (task_keys_filter) |fk| { if (std.mem.eql(u8, sk.string, fk)) { any = true; break :outer; } } } if (!any) continue; }
const client_dup = try alloc.dupe(u8, client_id); const keys_dup = try alloc.alloc([]const u8, task_keys_val.array.items.len); for (task_keys_val.array.items, 0..) |sk, i| { keys_dup[i] = if (sk == .string) try alloc.dupe(u8, sk.string) else try alloc.dupe(u8, ""); } try list.append(alloc, .{ .client_id = client_dup, .task_keys = keys_dup, .last_seen_ms = last_seen_val.integer, }); try seen.put(alloc, client_dup, {}); }}