//! In-process background-task queue + worker tracker. //! //! Mirrors python prefect's `server/task_queue.py` + `server/models/task_workers.py` //! shape. Two pieces: //! //! 1. Per-`task_key` queue (scheduled + retry). When a task_run lands in //! SCHEDULED with `state_details.deferred=true`, the API layer calls //! `enqueue(task_key, ...)` and we try to hand it to a registered //! worker immediately. If no worker is listening for that key, the //! task_run sits in the queue until one subscribes (or until the //! server restarts — same single-replica risk as the python impl, //! filed upstream as PrefectHQ/prefect#21218). //! //! 2. Worker tracker. Each `/task_runs/subscriptions/scheduled` WS //! connection registers via `addSubscriber()` with its task_keys + //! client_id. The tracker answers `POST /task_workers/filter`. //! //! Dispatch model is exclusive consumption: one task_run → one worker. We //! attempt dispatch on two events: (a) a task_run is enqueued, (b) a //! subscriber acks the previous run. Subscribers are marked `busy` from //! the moment we send them a task_run until they ack — within that window //! they're skipped by `tryDispatch`. On WS send failure or disconnect, //! any in-flight run for that subscriber is pushed back onto the retry //! sub-queue so another worker can pick it up. //! //! Today this is a memory-only singleton. Multi-replica safety (redis //! backend; see issue #21218) follows once the memory path is wire-correct. const std = @import("std"); const Allocator = std.mem.Allocator; const Io = std.Io; const ws = @import("websocket"); const redis = @import("redis"); const log = @import("logging.zig"); const broker = @import("broker.zig"); const time_util = @import("utilities/time.zig"); const sync = @import("utilities/sync.zig"); const observability = @import("observability.zig"); const WsHandle = *ws.Conn; const MAX_SUBSCRIBERS = 256; /// Memory backend keeps queues in-process; redis backend pushes payloads to /// Redis LISTs and uses the broker for cross-replica dispatch hints. The /// subscriber registry is in-process either way — WS connections are local /// to whichever replica accepted them. Cross-replica visibility for /// `POST /task_workers/filter` lives on a TTL'd `prefect:workers:{id}` /// key in redis mode. pub const Dialect = enum { memory, redis }; /// Broker topic for cross-replica dispatch hints. Replica A enqueues to a /// redis LIST, publishes `{task_key}` here, and every other replica's /// task_workers consumes the hint and attempts to drain matching local /// subscribers. const DISPATCH_HINT_TOPIC = "task-workers-dispatch-hint"; const PRESENCE_TTL_SECONDS: u32 = 30; /// One enqueued task_run waiting to be picked up by a worker. We hold the /// fully-rendered JSON payload so dispatch is a memcpy + writeText with no /// DB read. const QueuedRun = struct { task_run_id: [36]u8 = undefined, task_run_id_len: u8 = 0, /// Owned by `task_workers.allocator`. Freed on dispatch or queue drop. payload_json: []const u8, /// True if this run came from the retry sub-queue (drained first). is_retry: bool = false, }; /// Per-task_key queue. Scheduled drains after retry. const KeyQueue = struct { scheduled: std.ArrayListUnmanaged(QueuedRun) = .empty, retry: std.ArrayListUnmanaged(QueuedRun) = .empty, }; const Subscriber = struct { /// Valid only when `active` is true. handle: WsHandle = undefined, /// Heap-owned strings; freed in removeSubscriber. client_id: []const u8 = "", /// Heap-owned slice of heap-owned strings. task_keys: []const []const u8 = &.{}, active: bool = false, /// True between "we sent this worker a task_run" and "they acked." Skipped /// by tryDispatch while busy to enforce exclusive consumption. busy: bool = false, /// Set when busy. Used on disconnect to push the in-flight run back to /// the retry queue under its task_key. nil-terminated by `inflight_key_len`. inflight_task_key_buf: [256]u8 = undefined, inflight_task_key_len: usize = 0, /// Wall-clock at last observe (re-observed on subscribe + on each ack). /// Used by /task_workers/filter to age out stale entries. last_seen_ms: i64 = 0, }; var instance: ?*TaskWorkers = null; var storage: TaskWorkers = undefined; /// Initialize the singleton. Idempotent. Call once during server startup. /// Returns an error if dialect=redis and the broker subscribe fails: that /// would put us in a state where cross-replica dispatch silently doesn't /// work, which is the entire reason for the redis backend. pub fn init(allocator: Allocator, io: Io, dialect: Dialect) !void { if (instance != null) return; storage = TaskWorkers.init(allocator, io, dialect); try storage.start(); instance = &storage; } /// Tear down the singleton. Idempotent. pub fn deinit() void { if (instance) |s| { s.stop(); } instance = null; } pub const TaskWorkers = struct { allocator: Allocator, io: Io, dialect: Dialect = .memory, /// Guards queues + subscribers. Both are touched from API-handler threads /// (enqueue + WS callbacks) so a single mutex is simplest. mutex: Io.Mutex = Io.Mutex.init, /// Memory dialect: `task_key` → KeyQueue. Lazily initialized as keys /// appear via enqueue or subscribe — both can be the first to observe a /// new key. Unused in redis dialect (queues live in redis). queues: std.StringHashMapUnmanaged(KeyQueue) = .empty, subscribers: [MAX_SUBSCRIBERS]Subscriber = undefined, subscriber_count: usize = 0, subscribers_initialized: bool = false, /// Redis-only: broker consumer handle for cross-replica dispatch hints. dispatch_hint_handle: ?broker.ConsumerHandle = null, pub fn init(allocator: Allocator, io: Io, dialect: Dialect) TaskWorkers { // Eagerly initialize the subscribers array so every read path can // assume `active` is a defined bool. Lazy init was easy to get wrong // — snapshotWorkers on a replica that hasn't seen any subscribers // would otherwise iterate undefined memory. var tw: TaskWorkers = .{ .allocator = allocator, .io = io, .dialect = dialect }; for (&tw.subscribers) |*s| s.* = .{}; tw.subscribers_initialized = true; return tw; } pub fn start(self: *TaskWorkers) !void { instance = self; if (self.dialect == .redis) { // Subscribe ephemerally so every replica receives every hint — // analogous to events broadcaster. The hint payload is the // task_key; on receipt we attempt local dispatch from redis. // Hard error if subscribe fails: without it we can't act on // work enqueued by other replicas, which is the entire reason // for the redis backend (PrefectHQ/prefect#21218). const b = broker.getBroker() orelse { log.err("task_workers", "redis dialect requested but no broker available", .{}); return error.BrokerUnavailable; }; const ts = time_util.milliTimestamp(self.io); var group_buf: [64]u8 = undefined; const group = std.fmt.bufPrint(&group_buf, "dispatch-hint-{d}", .{ts}) catch "dispatch-hint"; self.dispatch_hint_handle = try b.subscribeEphemeral( DISPATCH_HINT_TOPIC, group, onDispatchHintCallback, ); log.info("task_workers", "started (redis backend)", .{}); } else { log.info("task_workers", "started (memory backend)", .{}); } } pub fn stop(self: *TaskWorkers) void { if (self.dispatch_hint_handle) |h| { if (broker.getBroker()) |b| b.unsubscribe(h); self.dispatch_hint_handle = null; } instance = null; } fn initSubscribers(self: *TaskWorkers) void { if (self.subscribers_initialized) return; for (&self.subscribers) |*s| s.active = false; self.subscribers_initialized = true; } /// Push a task_run onto the queue for `task_key` and immediately try to /// hand it to a registered worker. If no worker is currently free for /// that key, the run waits. /// /// `payload_json` is duplicated into the allocator; caller keeps ownership /// of theirs. pub fn enqueue(self: *TaskWorkers, task_key: []const u8, task_run_id: []const u8, payload_json: []const u8) !void { observability.task_workers_enqueued_total.inc(); if (self.dialect == .redis) { // Push payload to redis LIST (LPUSH/RPOP gives FIFO). try self.redisEnqueue(task_key, payload_json, false); log.debug("task_workers", "redis-enqueued task_run {s} (key={s})", .{ task_run_id, task_key }); // Fan out dispatch hint to every replica so whoever has the // subscriber drains it. Failure here is a real error — without // the hint, work enqueued on this replica is invisible to // workers connected to other replicas until something else // triggers their drain. Propagate; the caller decides whether // that's recoverable. const b = broker.getBroker() orelse return error.BrokerUnavailable; try b.publish(DISPATCH_HINT_TOPIC, task_run_id, task_key); // Also try our own local subscribers immediately. self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io); self.initSubscribers(); self.tryDispatchRedisLocked(task_key); return; } self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io); self.initSubscribers(); const owned_payload = try self.allocator.dupe(u8, payload_json); errdefer self.allocator.free(owned_payload); var run = QueuedRun{ .payload_json = owned_payload }; const id_copy_len = @min(task_run_id.len, run.task_run_id.len); @memcpy(run.task_run_id[0..id_copy_len], task_run_id[0..id_copy_len]); run.task_run_id_len = @intCast(id_copy_len); const q = try self.getOrCreateQueue(task_key); try q.scheduled.append(self.allocator, run); log.debug("task_workers", "enqueued task_run {s} (key={s}, depth={d})", .{ task_run_id, task_key, q.scheduled.items.len + q.retry.items.len, }); self.tryDispatchLocked(task_key); } fn getOrCreateQueue(self: *TaskWorkers, task_key: []const u8) !*KeyQueue { const gop = try self.queues.getOrPut(self.allocator, task_key); if (!gop.found_existing) { // own the key string gop.key_ptr.* = try self.allocator.dupe(u8, task_key); gop.value_ptr.* = .{}; } return gop.value_ptr; } /// Register a WS subscriber. `task_keys` and `client_id` are duplicated /// into the allocator; caller keeps their copies. /// /// Returns false if at MAX_SUBSCRIBERS capacity. pub fn addSubscriber(self: *TaskWorkers, handle: WsHandle, client_id: []const u8, task_keys: []const []const u8) !bool { self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io); self.initSubscribers(); for (&self.subscribers) |*s| { if (!s.active) { const owned_client = try self.allocator.dupe(u8, client_id); errdefer self.allocator.free(owned_client); const owned_keys = try self.allocator.alloc([]const u8, task_keys.len); errdefer self.allocator.free(owned_keys); var dup_count: usize = 0; errdefer for (owned_keys[0..dup_count]) |k| self.allocator.free(k); for (task_keys, 0..) |k, i| { owned_keys[i] = try self.allocator.dupe(u8, k); dup_count = i + 1; } s.handle = handle; s.client_id = owned_client; s.task_keys = owned_keys; s.active = true; s.busy = false; s.inflight_task_key_len = 0; s.last_seen_ms = time_util.milliTimestamp(self.io); self.subscriber_count += 1; observability.task_workers_subscribers.inc(); log.info("task_workers", "subscriber {s} registered for keys={d} (total {d})", .{ client_id, task_keys.len, self.subscriber_count, }); if (self.dialect == .redis) { self.publishPresence(s) catch |err| { log.warn("task_workers", "presence publish for {s} failed: {} — filter API may not see this worker from other replicas", .{ client_id, err }); }; } // try to drain any pending work this subscriber can take for (owned_keys) |k| { if (self.dialect == .redis) self.tryDispatchRedisLocked(k) else self.tryDispatchLocked(k); } return true; } } log.warn("task_workers", "at capacity ({d}), rejecting new subscriber", .{MAX_SUBSCRIBERS}); return false; } pub fn removeSubscriber(self: *TaskWorkers, handle: WsHandle) void { self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io); for (&self.subscribers) |*s| { if (s.active and s.handle == handle) { // If they were in-flight on a task_run, that run is at risk // of being lost. Today we don't track the QueuedRun payload // after dispatch, so the run is gone if the worker // disconnects mid-ack. (Python has the same issue — when // websocket fails post-send and pre-ack, it pushes to retry // queue from a stored copy.) Track this gap explicitly: if (s.busy and s.inflight_task_key_len > 0) { const key = s.inflight_task_key_buf[0..s.inflight_task_key_len]; log.warn("task_workers", "subscriber {s} disconnected mid-flight on key={s} — task_run may be stuck", .{ s.client_id, key, }); } if (self.dialect == .redis) { self.deletePresence(s.client_id) catch |err| { log.warn("task_workers", "presence delete failed: {}", .{err}); }; } self.freeSubscriberOwned(s); s.active = false; self.subscriber_count -= 1; observability.task_workers_subscribers.dec(); log.debug("task_workers", "subscriber removed (total {d})", .{self.subscriber_count}); return; } } } fn freeSubscriberOwned(self: *TaskWorkers, s: *Subscriber) void { if (s.client_id.len > 0) self.allocator.free(s.client_id); for (s.task_keys) |k| self.allocator.free(k); if (s.task_keys.len > 0) self.allocator.free(s.task_keys); s.client_id = ""; s.task_keys = &.{}; } /// Called when a subscriber sends `{"type":"ack"}` after processing a run. pub fn ack(self: *TaskWorkers, handle: WsHandle) void { self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io); for (&self.subscribers) |*s| { if (s.active and s.handle == handle) { s.busy = false; s.inflight_task_key_len = 0; s.last_seen_ms = time_util.milliTimestamp(self.io); if (self.dialect == .redis) { self.publishPresence(s) catch {}; } // try to immediately give them the next run for (s.task_keys) |k| { if (self.dialect == .redis) self.tryDispatchRedisLocked(k) else self.tryDispatchLocked(k); } return; } } } /// Send one queued run for `task_key` to any free subscriber that listens /// for that key. Caller must hold the mutex. fn tryDispatchLocked(self: *TaskWorkers, task_key: []const u8) void { const q = self.queues.getPtr(task_key) orelse return; if (q.scheduled.items.len == 0 and q.retry.items.len == 0) return; // pick a free subscriber listening for this key for (&self.subscribers) |*s| { if (!s.active or s.busy) continue; var listens = false; for (s.task_keys) |k| { if (std.mem.eql(u8, k, task_key)) { listens = true; break; } } if (!listens) continue; // pop next: retry drains first const run = if (q.retry.items.len > 0) q.retry.orderedRemove(0) else q.scheduled.orderedRemove(0); defer self.allocator.free(run.payload_json); s.handle.writeText(run.payload_json) catch |err| { observability.task_workers_send_failed_total.inc(); observability.task_workers_requeued_retry_total.inc(); log.warn("task_workers", "writeText to subscriber {s} failed: {} — re-enqueueing to retry", .{ s.client_id, err }); self.requeueRetryLocked(task_key, run.payload_json) catch |re_err| { log.err("task_workers", "failed to re-enqueue: {}", .{re_err}); }; self.freeSubscriberOwned(s); s.active = false; s.busy = false; self.subscriber_count -|= 1; observability.task_workers_subscribers.dec(); return; }; s.busy = true; const klen = @min(task_key.len, s.inflight_task_key_buf.len); @memcpy(s.inflight_task_key_buf[0..klen], task_key[0..klen]); s.inflight_task_key_len = klen; s.last_seen_ms = time_util.milliTimestamp(self.io); observability.task_workers_dispatched_total.inc(); log.debug("task_workers", "dispatched to {s} (key={s})", .{ s.client_id, task_key }); return; } } /// Push `payload_json` (caller-owned) back onto the retry sub-queue under /// `task_key`. We dupe so the caller can free their copy. fn requeueRetryLocked(self: *TaskWorkers, task_key: []const u8, payload_json: []const u8) !void { const owned = try self.allocator.dupe(u8, payload_json); errdefer self.allocator.free(owned); const q = try self.getOrCreateQueue(task_key); try q.retry.append(self.allocator, .{ .payload_json = owned, .is_retry = true }); } // ======================================================================== // Redis backend helpers // ======================================================================== /// LPUSH the payload onto either the scheduled or retry list for this /// task_key. RPOP at dispatch time gives FIFO; retry list is drained /// first. fn redisEnqueue(self: *TaskWorkers, task_key: []const u8, payload_json: []const u8, is_retry: bool) !void { var client = try connectRedis(self.allocator); defer client.close(); var keys_buf: [256]u8 = undefined; const key = try formatQueueKey(&keys_buf, task_key, is_retry); var lists = client.lists(); const values = [_][]const u8{payload_json}; _ = try lists.lpush(key, &values); } /// Drain redis: drain retry list first, then scheduled. Hand each popped /// payload to a free local subscriber listening for this key. Caller /// must hold the mutex. fn tryDispatchRedisLocked(self: *TaskWorkers, task_key: []const u8) void { var dispatched_any = true; while (dispatched_any) { dispatched_any = false; // pick a free local subscriber listening for this key const sub: ?*Subscriber = blk: { for (&self.subscribers) |*s| { if (!s.active or s.busy) continue; for (s.task_keys) |k| { if (std.mem.eql(u8, k, task_key)) break :blk s; } } break :blk null; }; const s = sub orelse return; // pop from redis: retry first, then scheduled const payload = self.redisPopFor(task_key) catch |err| { log.warn("task_workers", "redis pop for key={s} failed: {}", .{ task_key, err }); return; }; const owned_payload = payload orelse return; // queue empty defer self.allocator.free(owned_payload); s.handle.writeText(owned_payload) catch |err| { observability.task_workers_send_failed_total.inc(); observability.task_workers_requeued_retry_total.inc(); log.warn("task_workers", "writeText to {s} failed: {} — re-enqueueing to retry", .{ s.client_id, err }); self.redisEnqueue(task_key, owned_payload, true) catch |re_err| { log.err("task_workers", "retry re-enqueue failed: {}", .{re_err}); }; self.freeSubscriberOwned(s); s.active = false; s.busy = false; self.subscriber_count -|= 1; observability.task_workers_subscribers.dec(); return; }; s.busy = true; const klen = @min(task_key.len, s.inflight_task_key_buf.len); @memcpy(s.inflight_task_key_buf[0..klen], task_key[0..klen]); s.inflight_task_key_len = klen; s.last_seen_ms = time_util.milliTimestamp(self.io); observability.task_workers_dispatched_total.inc(); log.debug("task_workers", "redis-dispatched to {s} (key={s})", .{ s.client_id, task_key }); dispatched_any = true; } } /// Caller owns the returned slice (allocated via self.allocator). Returns /// null if both retry and scheduled lists are empty. fn redisPopFor(self: *TaskWorkers, task_key: []const u8) !?[]const u8 { var client = try connectRedis(self.allocator); defer client.close(); var keys_buf: [256]u8 = undefined; var lists = client.lists(); // retry first const retry_key = try formatQueueKey(&keys_buf, task_key, true); if (try lists.rpop(retry_key)) |val| { const owned = try self.allocator.dupe(u8, val); return owned; } const sched_key = try formatQueueKey(&keys_buf, task_key, false); if (try lists.rpop(sched_key)) |val| { const owned = try self.allocator.dupe(u8, val); return owned; } return null; } /// Write `prefect:workers:{client_id}` with the worker's task_keys and /// last_seen, TTL'd so a crashed worker ages out without explicit DELETE. /// `s` is read under the caller's mutex; we don't hold any extra lock. fn publishPresence(self: *TaskWorkers, s: *const Subscriber) !void { var client = try connectRedis(self.allocator); defer client.close(); // Build JSON payload: {"task_keys":[...],"last_seen_ms":N} var buf: std.Io.Writer.Allocating = .init(self.allocator); defer buf.deinit(); var jw: std.json.Stringify = .{ .writer = &buf.writer }; try jw.beginObject(); try jw.objectField("task_keys"); try jw.beginArray(); for (s.task_keys) |k| try jw.write(k); try jw.endArray(); try jw.objectField("last_seen_ms"); try jw.write(s.last_seen_ms); try jw.endObject(); var key_buf: [256]u8 = undefined; const key = try std.fmt.bufPrint(&key_buf, "prefect:workers:{s}", .{s.client_id}); var strings = client.strings(); try strings.set(key, buf.written()); var keys = client.keys(); _ = try keys.expire(key, PRESENCE_TTL_SECONDS); } fn deletePresence(self: *TaskWorkers, client_id: []const u8) !void { var client = try connectRedis(self.allocator); defer client.close(); var key_buf: [256]u8 = undefined; const key = try std.fmt.bufPrint(&key_buf, "prefect:workers:{s}", .{client_id}); var keys = client.keys(); _ = try keys.del(&.{key}); } }; // ============================================================================ // Redis helpers (file scope so dispatch callback can call them) // ============================================================================ fn connectRedis(alloc: Allocator) !redis.Client { const cfg = broker.parseRedisConfig(); return redis.Client.connectWithConfig(sync.globalIo(), alloc, .{ .host = cfg.host, .port = cfg.port, .username = cfg.username, .password = cfg.password, .db = cfg.db, }); } fn formatQueueKey(buf: *[256]u8, task_key: []const u8, is_retry: bool) ![]const u8 { const suffix = if (is_retry) "retry" else "scheduled"; return std.fmt.bufPrint(buf, "prefect:tasks:{s}:{s}", .{ task_key, suffix }); } /// Broker callback for dispatch hints (redis backend). The hint payload is /// the task_key string; we try to drain one of our local subscribers. fn onDispatchHintCallback(msg: *const broker.Message) anyerror!void { const self = instance orelse return; if (self.dialect != .redis) return; const task_key = msg.data; self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io); self.initSubscribers(); self.tryDispatchRedisLocked(task_key); } // ============================================================================ // Free-function shims for handler call sites // ============================================================================ pub fn enqueue(task_key: []const u8, task_run_id: []const u8, payload_json: []const u8) void { const self = instance orelse { log.warn("task_workers", "enqueue({s}) but task_workers not running", .{task_key}); return; }; self.enqueue(task_key, task_run_id, payload_json) catch |err| { log.err("task_workers", "enqueue failed: {}", .{err}); }; } pub fn addSubscriber(handle: WsHandle, client_id: []const u8, task_keys: []const []const u8) !bool { const self = instance orelse return false; return self.addSubscriber(handle, client_id, task_keys); } pub fn removeSubscriber(handle: WsHandle) void { const self = instance orelse return; self.removeSubscriber(handle); } pub fn ack(handle: WsHandle) void { const self = instance orelse return; self.ack(handle); } /// Snapshot of currently-registered workers for /task_workers/filter. pub const WorkerSnapshot = struct { client_id: []const u8, task_keys: []const []const u8, last_seen_ms: i64, }; /// `task_keys` is the filter: if empty, return all workers; otherwise return /// only workers listening for at least one matching key. Allocations are /// duplicated into `alloc`. /// /// In redis mode, also walks `prefect:workers:*` to include workers /// connected to other replicas. Local subscribers and remote ones can /// collide on client_id (a redis presence key from this replica matches a /// local subscriber); the local one wins. pub fn snapshotWorkers(alloc: Allocator, task_keys_filter: []const []const u8) ![]WorkerSnapshot { const self = instance orelse return alloc.alloc(WorkerSnapshot, 0); var list = std.ArrayListUnmanaged(WorkerSnapshot).empty; errdefer list.deinit(alloc); var seen_client_ids = std.StringHashMapUnmanaged(void).empty; defer seen_client_ids.deinit(alloc); { self.mutex.lockUncancelable(self.io); defer self.mutex.unlock(self.io); for (&self.subscribers) |*s| { if (!s.active) continue; if (task_keys_filter.len > 0) { var any = false; outer: for (s.task_keys) |sk| { for (task_keys_filter) |fk| { if (std.mem.eql(u8, sk, fk)) { any = true; break :outer; } } } if (!any) continue; } const client_dup = try alloc.dupe(u8, s.client_id); const keys_dup = try alloc.alloc([]const u8, s.task_keys.len); for (s.task_keys, 0..) |k, i| keys_dup[i] = try alloc.dupe(u8, k); try list.append(alloc, .{ .client_id = client_dup, .task_keys = keys_dup, .last_seen_ms = s.last_seen_ms, }); try seen_client_ids.put(alloc, client_dup, {}); } } if (self.dialect == .redis) { // SCAN prefect:workers:* and merge in any non-local presence. appendRemoteWorkers(alloc, &list, &seen_client_ids, task_keys_filter) catch |err| { log.warn("task_workers", "remote worker SCAN failed: {} — returning only local", .{err}); }; } return list.toOwnedSlice(alloc); } fn appendRemoteWorkers( alloc: Allocator, list: *std.ArrayListUnmanaged(WorkerSnapshot), seen: *std.StringHashMapUnmanaged(void), task_keys_filter: []const []const u8, ) !void { var client = try connectRedis(alloc); defer client.close(); var keys = client.keys(); var strings = client.strings(); // Slices returned from redis commands point into the client's read buffer // and are clobbered by the NEXT command. So: collect all "prefect:workers:*" // keys (duped) across all SCAN iterations first, then GET each in a second // pass. Within the GET pass, dupe the body before any subsequent command. var full_keys = std.ArrayListUnmanaged([]const u8).empty; defer { for (full_keys.items) |k| alloc.free(k); full_keys.deinit(alloc); } var cursor_storage: []const u8 = try alloc.dupe(u8, "0"); defer alloc.free(cursor_storage); while (true) { const res = try keys.scan(cursor_storage, "prefect:workers:*", 100); for (res.keys) |kv| { // Redis bulk strings come back as `.bulk` (nullable); simple // strings (`.string`) are reserved for status replies like OK. // We accept either form defensively. const s = switch (kv) { .bulk => |b| b orelse continue, .string => |s| s, else => continue, }; try full_keys.append(alloc, try alloc.dupe(u8, s)); } if (std.mem.eql(u8, res.cursor, "0")) break; const next_cursor = try alloc.dupe(u8, res.cursor); alloc.free(cursor_storage); cursor_storage = next_cursor; } const PREFIX = "prefect:workers:"; for (full_keys.items) |full_key| { if (!std.mem.startsWith(u8, full_key, PREFIX)) continue; const client_id = full_key[PREFIX.len..]; if (seen.contains(client_id)) continue; const ephemeral_body = (try strings.get(full_key)) orelse continue; const body = try alloc.dupe(u8, ephemeral_body); defer alloc.free(body); const parsed = std.json.parseFromSlice(std.json.Value, alloc, body, .{}) catch continue; if (parsed.value != .object) continue; const task_keys_val = parsed.value.object.get("task_keys") orelse continue; if (task_keys_val != .array) continue; const last_seen_val = parsed.value.object.get("last_seen_ms") orelse continue; if (last_seen_val != .integer) continue; if (task_keys_filter.len > 0) { var any = false; outer: for (task_keys_val.array.items) |sk| { if (sk != .string) continue; for (task_keys_filter) |fk| { if (std.mem.eql(u8, sk.string, fk)) { any = true; break :outer; } } } if (!any) continue; } const client_dup = try alloc.dupe(u8, client_id); const keys_dup = try alloc.alloc([]const u8, task_keys_val.array.items.len); for (task_keys_val.array.items, 0..) |sk, i| { keys_dup[i] = if (sk == .string) try alloc.dupe(u8, sk.string) else try alloc.dupe(u8, ""); } try list.append(alloc, .{ .client_id = client_dup, .task_keys = keys_dup, .last_seen_ms = last_seen_val.integer, }); try seen.put(alloc, client_dup, {}); } }