initial commit

pulled code out of horizon

.gitignore

···
+.zig-cache/
+zig-out/

LICENSE

···
+Copyright 2025 Tim Culverhouse
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the “Software”), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

README.md

···
+# Ourio
+
+Ourio (prounounced "oreo", think "Ouroboros") is an asynchronous IO runtime
+built heavily around the semantics of io_uring. The design is inspired by
+[libxev](https://github.com/mitchellh/libxev), which is in turn inspired by
+[TigerBeetle](https://github.com/tigerbeetle/tigerbeetle).
+
+Ourio has only a slightly different approach: it is designed to encourage
+message passing approach to asynchronous IO. Users of the library give each task
+a Context, which contains a pointer, a callback, *and a message*. The message is
+implemented as a u16, and generally you should use an enum for it. The idea is
+that you can minimize the number of callback functions required by tagging tasks
+with a small amount of semantic meaning in the `msg` field.
+
+Ourio has io_uring and kqueue backends. Ourio supports the `msg_ring`
+capability of io_uring to pass a completion from one ring to another. This
+allows a multithreaded application to implement message passing using io_uring
+(or kqueue, if that's your flavor). Multithreaded applications should plan to
+use one `Runtime` per thread. Submission onto the runtime is not thread safe,
+any message passing must occur using `msg_ring` rather than directly submitting
+a task to another
+
+Ourio also includes a fully mockable IO runtime to make it easy to unit test
+your async code.
+
+## Tasks
+
+### Deadlines and Cancelation
+
+Each IO operation creates a `Task`. When scheduling a task on the runtime, the
+caller receives a pointer to the `Task` at which point they may cancel it, or
+set a deadline:
+
+```zig
+// Timers are always relative time
+const task = try rt.timer(.{.sec = 3}, .{.cb = onCompletion, .msg = 0});
+
+// If the deadline expired, the task will be sent to the onCompletion callback
+// with a result of error.Canceled. Deadlines are always absolute time
+try task.setDeadline(rt, .{.sec = std.time.timestamp() + 3});
+
+// Alternatively, we can hold on to the pointer for the task while it is with
+// the runtime and cancel it. The Context we give to the cancel function let's
+// us know the result of the cancelation, but we will also receive a message
+// from the original task with error.Canceled. We can ignore the cancel result
+// by using the default context value
+try task.cancel(rt, .{});
+```
+
+### Passing tasks between threads
+
+Say we `accept` a connection in one thread, and want to send the file descriptor
+to another for handling.
+
+```zig
+const target_task = try main_rt.getTask();
+target_task.* {
+    .userdata = &foo,
+    .msg = @intFromEnum(Msg.some_message),
+    .cb = Worker.onCompletion,
+    .req = .{ .userfd = fd },
+};
+
+// Send target_task from the main_rt thread to the thread_rt Runtime. The
+// thread_rt Runtime will then // process the task as a completion, ie
+// Worker.onCompletion will be called with // this task. That thread can then
+// schedule a recv, a write, etc on the file // descriptor it just received.
+_ = try main_rt.msgRing(thread_rt, target_task, .{});
+```
+
+### Multiple Runtimes on the same thread
+
+You can have multiple Runtimes in a single thread. One could be a priority
+Runtime, or handle specific types of tasks, etc. Poll any runtime from any other
+runtime.
+
+```zig
+const fd = rt1.backend.pollableFd();
+_ = try rt2.poll(fd, .{
+	.cb = onCompletion, 
+	.msg = @intFromEnum(Msg.rt1_has_completions)}
+    );
+```
+
+## Example
+
+An example implementation of an asynchronous writer to two file descriptors:
+
+```zig
+const std = @import("std");
+const io = @import("ourio");
+const posix = std.posix;
+
+pub const MultiWriter = struct {
+    fd1: posix.fd_t,
+    fd1_written: usize = 0,
+
+    fd2: posix.fd_2,
+    fd2_written: usize = 0,
+
+    buf: std.ArrayListUnmanaged(u8),
+
+    pub const Msg = enum { fd1, fd2 };
+
+    pub fn init(fd1: posix.fd_t, fd2: posix.fd_t) MultiWriter {
+        return .{ .fd1 = fd1, .fd2 = fd2 };
+    }
+
+    pub fn write(self: *MultiWriter, gpa: Allocator, bytes: []const u8) !void {
+        try self.buf.appendSlice(gpa, bytes);
+    }
+
+    pub fn flush(self: *MultiWriter, rt: *io.Runtime) !void {
+        if (self.fd1_written < self.buf.items.len) {
+            _ = try rt.write(self.fd1, self.buf.items[self.fd1_written..], .{
+                .ptr = self,
+                .msg = @intFromEnum(Msg.fd1),
+                .cb = MultiWriter.onCompletion,
+            });
+        }
+
+        if (self.fd2_written < self.buf.items.len) {
+            _ = try rt.write(self.fd2, self.buf.items[self.fd2_written], .{
+                .ptr = self,
+                .msg = @intFromEnum(Msg.fd2),
+                .cb = MultiWriter.onCompletion,
+            });
+        }
+    }
+
+    pub fn onCompletion(rt: *io.Runtime, task: io.Task) anyerror!void {
+        const self = task.userdataCast(MultiWriter);
+        const result = task.result.?;
+
+        const n = try result.write;
+        switch (task.msgToEnum(MultiWriter.Msg)) {
+            .fd1 => self.fd1_written += n,
+            .fd2 => self.fd2_written += n,
+        }
+
+        const len = self.buf.items.len;
+
+        if (self.fd1_written < len or self.fd2_written < len) 
+	    return self.flush(rt);
+
+        self.fd1_written = 0;
+        self.fd2_written = 0;
+        self.buf.clearRetainingCapacity();
+    }
+};
+
+pub fn main() !void {
+    var gpa: std.heap.DebugAllocator(.{}) = .init;
+    var rt: io.Runtime = try .init(gpa.allocator(), 16);
+    defer rt.deinit();
+
+    // Pretend I created some files
+    const fd1: posix.fd_t = 5;
+    const fd2: posix.fd_t = 6;
+
+    var mw: MultiWriter = .init(fd1, fd2);
+    try mw.write(gpa.allocator(), "Hello, world!");
+    try mw.flush(&rt);
+
+    try rt.run(.until_done);
+}
+```

build.zig

···
+const std = @import("std");
+
+pub fn build(b: *std.Build) void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+
+    const io_mod = b.createModule(.{
+        .root_source_file = b.path("src/main.zig"),
+        .target = target,
+        .optimize = optimize,
+    });
+    const tls_dep = b.dependency("tls", .{ .target = target, .optimize = optimize });
+    io_mod.addImport("tls", tls_dep.module("tls"));
+
+    const lib_unit_tests = b.addTest(.{
+        .root_module = io_mod,
+    });
+
+    const run_lib_unit_tests = b.addRunArtifact(lib_unit_tests);
+
+    const test_step = b.step("test", "Run unit tests");
+    test_step.dependOn(&run_lib_unit_tests.step);
+}

build.zig.zon

···
+.{
+    .name = .ourio,
+    .version = "0.0.0",
+    .fingerprint = 0xbfe65736d1b3cffe, // Changing this has security and trust implications.
+
+    .minimum_zig_version = "0.14.0",
+
+    .dependencies = .{
+        .tls = .{
+            .url = "git+https://github.com/ianic/tls.zig#8250aa9184fbad99983b32411bbe1a5d2fd6f4b7",
+            .hash = "tls-0.1.0-ER2e0pU3BQB-UD2_s90uvppceH_h4KZxtHCrCct8L054",
+        },
+    },
+
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+        "src",
+    },
+}

src/Kqueue.zig

···
+const Kqueue = @This();
+
+const std = @import("std");
+const builtin = @import("builtin");
+
+const io = @import("main.zig");
+
+const Allocator = std.mem.Allocator;
+const EV = std.c.EV;
+const EVFILT = std.c.EVFILT;
+const Queue = @import("queue.zig").Intrusive;
+const assert = std.debug.assert;
+const posix = std.posix;
+
+gpa: Allocator,
+kq: posix.fd_t,
+/// Items we have prepared and waiting to be put into kqueue
+submission_queue: std.ArrayListUnmanaged(posix.Kevent) = .empty,
+
+/// Tasks that have been submitted to kqueue
+in_flight: Queue(io.Task, .in_flight) = .{},
+
+/// Tasks which were completed synchronously during submission
+synchronous_queue: Queue(io.Task, .complete) = .{},
+
+/// Queue for other kqueue instances to send "completion" tasks to this thread
+msg_ring_queue: Queue(io.Task, .complete) = .{},
+
+/// Mutex to guard access to the msg_ring_queue. We *could* merge these tasks with the
+/// synchronous_queue, however we want to limit contention and msg_ring is probably pretty rare
+/// compared to synchronous tasks
+msg_ring_mutex: std.Thread.Mutex = .{},
+
+/// List of timers, sorted descending so when we pop we get the next timer to expire
+timers: std.ArrayListUnmanaged(Timer) = .empty,
+
+events: [128]posix.Kevent = undefined,
+event_idx: usize = 0,
+
+const Timer = union(enum) {
+    /// a deadline timer cancels a task if it fires
+    deadline: struct {
+        /// the deadline task. If the parent completes before the deadline, the parent will set the
+        /// deadline task state to .free
+        task: *io.Task,
+
+        /// the task to cancel if the deadline expires
+        parent: *io.Task,
+    },
+
+    /// a regular timer
+    timeout: struct {
+        /// The task for the timer
+        task: *io.Task,
+
+        /// Absolute time in ms the timer was added
+        added_ms: i64,
+    },
+
+    /// Timer expires in the return value milliseconds from now
+    fn expiresInMs(self: Timer, now_abs: i64) i64 {
+        switch (self) {
+            .deadline => |deadline| {
+                const ts = deadline.task.req.deadline;
+                const expires_ms = ts.sec * std.time.ms_per_s + @divTrunc(ts.nsec, std.time.ns_per_ms);
+                return expires_ms - now_abs;
+            },
+
+            // timeouts are relative, so we add the time it was added to the queue
+            .timeout => |timeout| {
+                const ts = timeout.task.req.timer;
+                const expires_ms = ts.sec * std.time.ms_per_s +
+                    @divTrunc(ts.nsec, std.time.ns_per_ms) +
+                    timeout.added_ms;
+                return expires_ms - now_abs;
+            },
+        }
+    }
+
+    /// returns a timespec suitable for a kevent timeout. Relative to now
+    fn timespec(self: Timer) posix.timespec {
+        const expires = self.expiresInMs(std.time.milliTimestamp());
+        return .{ .sec = @divFloor(expires, 1000), .nsec = @mod(expires, 1000) * std.time.ns_per_ms };
+    }
+
+    fn lessThan(now_ms: i64, lhs: Timer, rhs: Timer) bool {
+        // reverse sort (we want soonest expiring last)
+        return lhs.expiresInMs(now_ms) > rhs.expiresInMs(now_ms);
+    }
+};
+
+/// Messages we are waiting on using an EVFILT.USER
+const UserMsg = enum {
+    /// A general wakeup message
+    wakeup,
+
+    fn fromInt(v: i64) UserMsg {
+        return @enumFromInt(v);
+    }
+};
+
+/// Initialize a Ring
+pub fn init(gpa: Allocator, _: u16) !Kqueue {
+    const kq = try posix.kqueue();
+
+    // Register a wakeup EVFILT.USER to wake up this kqueue
+    var kevent = evSet(
+        @intFromEnum(UserMsg.wakeup),
+        EVFILT.USER,
+        EV.ADD | EV.CLEAR,
+        null,
+    );
+    kevent.fflags = std.c.NOTE.FFNOP;
+    _ = try posix.kevent(kq, &.{kevent}, &.{}, null);
+
+    return .{ .gpa = gpa, .kq = kq };
+}
+
+pub fn deinit(self: *Kqueue, gpa: Allocator) void {
+    while (self.msg_ring_queue.pop()) |task| gpa.destroy(task);
+    while (self.in_flight.pop()) |task| gpa.destroy(task);
+
+    self.submission_queue.deinit(gpa);
+    self.timers.deinit(gpa);
+
+    posix.close(self.kq);
+    self.* = undefined;
+}
+
+/// Initializes a child Ring which can be woken up by self. This must be called from the thread
+/// which will operate the child ring. Initializes with the same queue size as the parent
+pub fn initChild(self: Kqueue, entries: u16) !Kqueue {
+    return init(self.gpa, entries);
+}
+
+pub fn submitAndWait(self: *Kqueue, queue: *io.SubmissionQueue) !void {
+    defer self.submission_queue.clearRetainingCapacity();
+    while (queue.pop()) |task| {
+        try self.prepTask(task);
+
+        // If this task is queued and has a deadline we need to schedule a timer
+        if (task.deadline) |deadline| {
+            try self.addTimer(.{ .deadline = .{ .task = deadline, .parent = task } });
+        }
+    }
+
+    // Sort our timers
+    const now = std.time.milliTimestamp();
+    std.sort.insertion(Timer, self.timers.items, now, Timer.lessThan);
+
+    if (self.synchronous_queue.empty()) {
+        // We don't have any synchronous completions, so we need to wait for some from kqueue
+        return self.wait();
+    }
+
+    // We already have completions from synchronous tasks. Submit our queued events and grab any new
+    // completions for processing. We do so with a 0 timeout so that we are only grabbing already
+    // completed items
+    const timeout: posix.timespec = .{ .sec = 0, .nsec = 0 };
+    self.event_idx = try posix.kevent(self.kq, self.submission_queue.items, &self.events, &timeout);
+}
+
+fn wait(self: *Kqueue) !void {
+    assert(self.synchronous_queue.empty());
+
+    // Go through our times until the first unexpired one
+    while (true) {
+        const t = self.timers.getLastOrNull() orelse break;
+        const timeout: posix.timespec = t.timespec();
+
+        self.event_idx = try posix.kevent(
+            self.kq,
+            self.submission_queue.items,
+            &self.events,
+            &timeout,
+        );
+        return;
+    }
+
+    // We had no timers so we wait indefinitely
+    self.event_idx = try posix.kevent(
+        self.kq,
+        self.submission_queue.items,
+        &self.events,
+        null,
+    );
+}
+
+pub fn submit(self: *Kqueue, queue: *io.SubmissionQueue) !void {
+    defer self.submission_queue.clearRetainingCapacity();
+    while (queue.pop()) |task| {
+        try self.prepTask(task);
+
+        // If this task is queued and has a deadline we need to schedule a timer
+        if (task.state == .in_flight and task.deadline != null) {
+            const deadline = task.deadline.?;
+            try self.addTimer(.{ .deadline = .{ .task = deadline, .parent = task } });
+        }
+    }
+
+    // Sort our timers
+    const now = std.time.milliTimestamp();
+    std.sort.insertion(Timer, self.timers.items, now, Timer.lessThan);
+
+    // For submit, we don't try to reap any completions. Calls to submit will likely be relying on a
+    // `poll` of the kqueue. We check in reapCompletinos if we have no reaped events and grab them
+    // there if needed
+    const timeout: posix.timespec = .{ .sec = 0, .nsec = 0 };
+    _ = try posix.kevent(self.kq, self.submission_queue.items, &.{}, &timeout);
+}
+
+/// preps a task to be submitted into the kqueue
+fn prepTask(self: *Kqueue, task: *io.Task) !void {
+    return switch (task.req) {
+        .accept => |req| {
+            self.in_flight.push(task);
+            const kevent = evSet(@intCast(req), EVFILT.READ, EV.ADD, task);
+            try self.submission_queue.append(self.gpa, kevent);
+        },
+
+        .cancel => |req| {
+            // Cancel tasks are handled like this:
+            // 0. If the task is not in flight, we don't do anything
+            // 1. We set the cancel_task state as canceled
+            // 2. If there is a kqueue event associated with it, we prep a kevent with EV.DELETE
+            // 3. We add the task to the synchronous queue. This let's us ensure we hold the
+            //    cancel state until we've submitted the kevent associated with it. If any
+            //    completions occur while the state is canceled, we will ignore them.
+            // 4. Callbacks are not called in the submit phase. We only call the callback of the
+            //    cancel_task with error.Canceled in reapCompletions
+            // 5. In reapCompletions, we will return both the task and the cancel_task to the free
+            //    list
+
+            self.synchronous_queue.push(task);
+
+            task.result = .{ .cancel = {} };
+            switch (req) {
+                .all => {
+                    while (self.in_flight.head) |t| {
+                        defer self.synchronous_queue.push(t);
+
+                        self.cancelTask(t) catch continue;
+                    }
+
+                    while (self.timers.getLastOrNull()) |tmr| {
+                        switch (tmr) {
+                            inline else => |v| {
+                                defer self.synchronous_queue.push(v.task);
+                                self.cancelTask(v.task) catch continue;
+                            },
+                        }
+                    }
+                },
+                .task => |task_to_cancel| {
+                    switch (task_to_cancel.state) {
+                        .free, .canceled, .complete => {
+                            task.result = .{ .cancel = error.NotCanceled };
+                            return;
+                        },
+                        .in_flight => {},
+                    }
+                    try self.cancelTask(task_to_cancel);
+                },
+            }
+        },
+
+        .close => |req| {
+            task.result = .{ .close = {} };
+            self.synchronous_queue.push(task);
+            posix.close(req);
+        },
+
+        .connect => |req| {
+            // Set nonblocking. Call connect. Then add it to the kqueue. This will return as
+            // writeable when the connect is complete
+            const arg: posix.O = .{ .NONBLOCK = true };
+            const arg_u32: u32 = @bitCast(arg);
+            _ = posix.fcntl(req.fd, posix.F.SETFL, arg_u32) catch {
+                task.result = .{ .connect = error.Unexpected };
+                self.synchronous_queue.push(task);
+                return;
+            };
+
+            if (posix.connect(req.fd, req.addr, req.addr_len)) {
+                // We connected immediately. No need to add to kqueue. Just push to the synchronous
+                // queue to call the callback later
+                task.result = .{ .connect = {} };
+                self.synchronous_queue.push(task);
+            } else |err| {
+                switch (err) {
+                    error.WouldBlock => {
+                        self.in_flight.push(task);
+                        // This is the error we expect. Add the event to kqueue
+                        const kevent = evSet(@intCast(req.fd), EVFILT.WRITE, EV.ADD | EV.ONESHOT, task);
+                        try self.submission_queue.append(self.gpa, kevent);
+                    },
+                    else => {
+                        task.result = .{ .connect = error.Unexpected };
+                        self.synchronous_queue.push(task);
+                    },
+                }
+            }
+        },
+
+        // deadlines are handled separately
+        .deadline => unreachable,
+
+        .msg_ring => |req| {
+            const target = req.target;
+
+            target.backend.platform.msg_ring_mutex.lock();
+            target.backend.platform.msg_ring_queue.push(req.task);
+            target.backend.platform.msg_ring_mutex.unlock();
+
+            task.result = .{ .msg_ring = {} };
+            self.synchronous_queue.push(task);
+
+            // wake up the other ring
+            var kevent = evSet(
+                @intFromEnum(UserMsg.wakeup),
+                EVFILT.USER,
+                0,
+                null,
+            );
+            kevent.fflags |= std.c.NOTE.TRIGGER;
+            // Trigger the wakeup
+            _ = try posix.kevent(target.backend.platform.kq, &.{kevent}, &.{}, null);
+        },
+
+        .noop => {
+            task.result = .noop;
+            self.synchronous_queue.push(task);
+        },
+
+        .poll => |req| {
+            self.in_flight.push(task);
+            if (req.mask & posix.POLL.IN != 0) {
+                const kevent = evSet(@intCast(req.fd), EVFILT.READ, EV.ADD | EV.ONESHOT, task);
+                try self.submission_queue.append(self.gpa, kevent);
+            }
+            if (req.mask & posix.POLL.OUT != 0) {
+                const kevent = evSet(@intCast(req.fd), EVFILT.WRITE, EV.ADD | EV.ONESHOT, task);
+                try self.submission_queue.append(self.gpa, kevent);
+            }
+        },
+
+        .recv => |req| {
+            self.in_flight.push(task);
+            const kevent = evSet(@intCast(req.fd), EVFILT.READ, EV.ADD | EV.ONESHOT, task);
+            try self.submission_queue.append(self.gpa, kevent);
+        },
+
+        .socket => |req| {
+            self.synchronous_queue.push(task);
+            if (posix.socket(req.domain, req.type, req.protocol)) |fd|
+                task.result = .{ .socket = fd }
+            else |_|
+                task.result = .{ .socket = error.Unexpected };
+        },
+
+        .timer => {
+            const now = std.time.milliTimestamp();
+            try self.addTimer(.{ .timeout = .{ .task = task, .added_ms = now } });
+        },
+
+        // user* fields are never seen by the runtime, only for internal message passing
+        .userfd, .usermsg, .userptr => unreachable,
+
+        .write => |req| {
+            self.in_flight.push(task);
+            const kevent = evSet(@intCast(req.fd), EVFILT.WRITE, EV.ADD | EV.ONESHOT, task);
+            try self.submission_queue.append(self.gpa, kevent);
+        },
+
+        .writev => |req| {
+            self.in_flight.push(task);
+            const kevent = evSet(@intCast(req.fd), EVFILT.WRITE, EV.ADD | EV.ONESHOT, task);
+            try self.submission_queue.append(self.gpa, kevent);
+        },
+    };
+}
+
+fn cancelTask(self: *Kqueue, task: *io.Task) !void {
+    task.state = .canceled;
+    if (task.deadline) |d| d.state = .canceled;
+
+    switch (task.req) {
+        // Handled synchronously. Probably we couldn't cancel it, but if we did somehow we
+        // don't need to do anything either way
+        .cancel,
+        .close,
+        .msg_ring,
+        .noop,
+        .socket,
+        .userfd,
+        .usermsg,
+        .userptr,
+        => {},
+
+        .accept => |cancel_req| {
+            self.in_flight.remove(task);
+            task.result = .{ .accept = error.Canceled };
+            const kevent = evSet(@intCast(cancel_req), EVFILT.READ, EV.DELETE, task);
+            try self.submission_queue.append(self.gpa, kevent);
+        },
+
+        .connect => |cancel_req| {
+            self.in_flight.remove(task);
+            task.result = .{ .connect = error.Canceled };
+            const kevent = evSet(
+                @intCast(cancel_req.fd),
+                EVFILT.WRITE,
+                EV.DELETE,
+                task,
+            );
+            try self.submission_queue.append(self.gpa, kevent);
+        },
+
+        .deadline => {
+            // What does it mean to cancel a deadline? We remove the deadline from
+            // the parent and the timer from our list
+            for (self.timers.items, 0..) |t, i| {
+                if (t == .deadline and t.deadline.task == task) {
+                    // Set the parent deadline to null
+                    t.deadline.parent.deadline = null;
+                    // Remove the timer
+                    _ = self.timers.orderedRemove(i);
+                    task.result = .{ .deadline = error.Canceled };
+                    return;
+                }
+            } else task.result = .{ .cancel = error.EntryNotFound };
+        },
+
+        .poll => |cancel_req| {
+            self.in_flight.remove(task);
+            task.result = .{ .poll = error.Canceled };
+            if (cancel_req.mask & posix.POLL.IN != 0) {
+                const kevent = evSet(@intCast(cancel_req.fd), EVFILT.READ, EV.DELETE, task);
+                try self.submission_queue.append(self.gpa, kevent);
+            }
+            if (cancel_req.mask & posix.POLL.OUT != 0) {
+                const kevent = evSet(@intCast(cancel_req.fd), EVFILT.WRITE, EV.DELETE, task);
+                try self.submission_queue.append(self.gpa, kevent);
+            }
+        },
+
+        .recv => |cancel_req| {
+            self.in_flight.remove(task);
+            task.result = .{ .recv = error.Canceled };
+            const kevent = evSet(
+                @intCast(cancel_req.fd),
+                EVFILT.READ,
+                EV.DELETE,
+                task,
+            );
+            try self.submission_queue.append(self.gpa, kevent);
+        },
+
+        .timer => {
+            for (self.timers.items, 0..) |t, i| {
+                if (t == .timeout and t.timeout.task == task) {
+                    // Remove the timer
+                    _ = self.timers.orderedRemove(i);
+                    task.result = .{ .timer = error.Canceled };
+                    return;
+                }
+            } else task.result = .{ .cancel = error.EntryNotFound };
+        },
+
+        .write => |cancel_req| {
+            self.in_flight.remove(task);
+            task.result = .{ .write = error.Canceled };
+            const kevent = evSet(
+                @intCast(cancel_req.fd),
+                EVFILT.WRITE,
+                EV.DELETE,
+                task,
+            );
+            try self.submission_queue.append(self.gpa, kevent);
+        },
+
+        .writev => |cancel_req| {
+            self.in_flight.remove(task);
+            task.result = .{ .writev = error.Canceled };
+            const kevent = evSet(
+                @intCast(cancel_req.fd),
+                EVFILT.WRITE,
+                EV.DELETE,
+                task,
+            );
+            try self.submission_queue.append(self.gpa, kevent);
+        },
+    }
+}
+
+fn addTimer(self: *Kqueue, t: Timer) !void {
+    try self.timers.append(self.gpa, t);
+}
+
+fn evSet(ident: usize, filter: i16, flags: u16, ptr: ?*anyopaque) posix.Kevent {
+    return switch (builtin.os.tag) {
+        .netbsd,
+        .dragonfly,
+        .openbsd,
+        .macos,
+        .ios,
+        .tvos,
+        .watchos,
+        .visionos,
+        => .{
+            .ident = ident,
+            .filter = filter,
+            .flags = flags,
+            .fflags = 0,
+            .data = 0,
+            .udata = @intFromPtr(ptr),
+        },
+
+        .freebsd => .{
+            .ident = ident,
+            .filter = filter,
+            .flags = flags,
+            .fflags = 0,
+            .data = 0,
+            .udata = @intFromPtr(ptr),
+            ._ext = &.{ 0, 0, 0, 0 },
+        },
+
+        else => @compileError("kqueue not supported"),
+    };
+}
+
+pub fn done(self: *Kqueue) bool {
+    if (self.timers.items.len == 0 and
+        self.in_flight.empty() and
+        self.submission_queue.items.len == 0)
+    {
+        self.msg_ring_mutex.lock();
+        defer self.msg_ring_mutex.unlock();
+        return self.msg_ring_queue.empty();
+    }
+
+    return false;
+}
+
+/// Return a file descriptor which can be used to poll the ring for completions
+pub fn pollableFd(self: Kqueue) !posix.fd_t {
+    return self.kq;
+}
+
+pub fn reapCompletions(self: *Kqueue, rt: *io.Runtime) anyerror!void {
+    defer self.event_idx = 0;
+
+    if (self.event_idx == 0) {
+        const timeout: posix.timespec = .{ .sec = 0, .nsec = 0 };
+        self.event_idx = try posix.kevent(
+            self.kq,
+            self.submission_queue.items,
+            &self.events,
+            &timeout,
+        );
+    }
+
+    for (self.events[0..self.event_idx]) |event| {
+        // if the event is a USER filter, we check our msg_ring_queue
+        if (event.filter == EVFILT.USER) {
+            switch (UserMsg.fromInt(event.data)) {
+                .wakeup => {
+                    // We got a message in our msg_ring_queue
+                    self.msg_ring_mutex.lock();
+                    defer self.msg_ring_mutex.unlock();
+
+                    while (self.msg_ring_queue.pop()) |task| {
+                        // For canceled msg_rings we do nothing
+                        if (task.state == .canceled) continue;
+
+                        defer self.releaseTask(rt, task);
+                        if (task.result == null) task.result = .noop;
+                        try task.callback(rt, task.*);
+                    }
+                },
+            }
+            continue;
+        }
+
+        const task: *io.Task = @ptrFromInt(event.udata);
+        if (task.state == .canceled) continue;
+        try self.handleCompletion(rt, task, event);
+    }
+
+    while (self.synchronous_queue.pop()) |task| {
+        try self.handleSynchronousCompletion(rt, task);
+    }
+
+    const now = std.time.milliTimestamp();
+    while (self.timers.getLastOrNull()) |t| {
+        if (t.expiresInMs(now) > 0) break;
+        _ = self.timers.pop();
+        try self.handleExpiredTimer(rt, t);
+    }
+}
+
+/// Handle a completion which was done synchronously. The work has already been done, we just need
+/// to call the callback and return the task(s) to the free list
+fn handleSynchronousCompletion(
+    self: *Kqueue,
+    rt: *io.Runtime,
+    task: *io.Task,
+) !void {
+    switch (task.req) {
+        // async tasks. These can be handled synchronously in a cancel all
+        .accept,
+        .poll,
+        .recv,
+        .write,
+        .writev,
+
+        // Timers can be handled synchronously in a cancel all
+        .deadline,
+        .timer,
+
+        .connect, // connect is handled both sync and async
+        .close,
+        .msg_ring,
+        .noop,
+        .socket,
+        .userfd,
+        .usermsg,
+        .userptr,
+        => {
+            assert(task.result != null);
+            defer self.releaseTask(rt, task);
+            try task.callback(rt, task.*);
+        },
+
+        .cancel => |c| {
+            assert(task.result != null);
+            defer self.releaseTask(rt, task);
+            try task.callback(rt, task.*);
+
+            switch (c) {
+                .all => {},
+
+                .task => |ct| {
+                    // If the cancel had an error, we don't need to return the task_to_cancel
+                    _ = task.result.?.cancel catch return;
+                    // On success, it is our job to call the canceled tasks' callback and return the
+                    // task to the free list
+                    defer self.releaseTask(rt, ct);
+                    const result: io.Result = switch (ct.req) {
+                        .accept => .{ .accept = error.Canceled },
+                        .cancel => .{ .cancel = error.Canceled },
+                        .close => .{ .close = error.Canceled },
+                        .connect => .{ .connect = error.Canceled },
+                        .deadline => .{ .deadline = error.Canceled },
+                        .msg_ring => .{ .msg_ring = error.Canceled },
+                        .noop => unreachable,
+                        .poll => .{ .poll = error.Canceled },
+                        .recv => .{ .recv = error.Canceled },
+                        .socket => .{ .socket = error.Canceled },
+                        .timer => .{ .timer = error.Canceled },
+                        .userfd, .usermsg, .userptr => unreachable,
+                        .write => .{ .write = error.Canceled },
+                        .writev => .{ .writev = error.Canceled },
+                    };
+                    ct.result = result;
+                    try ct.callback(rt, ct.*);
+                },
+            }
+        },
+    }
+}
+
+fn dataToE(result: i64) std.posix.E {
+    if (result > 0) {
+        return @as(std.posix.E, @enumFromInt(-result));
+    }
+    return .SUCCESS;
+}
+
+fn unexpectedError(err: posix.E) posix.UnexpectedError {
+    std.log.err("unexpected posix error: {}", .{err});
+    return error.Unexpected;
+}
+
+fn handleCompletion(
+    self: *Kqueue,
+    rt: *io.Runtime,
+    task: *io.Task,
+    event: posix.Kevent,
+) !void {
+    switch (task.req) {
+        .cancel,
+        .close,
+        .deadline,
+        .msg_ring,
+        .noop,
+        .socket,
+        .timer,
+        .userfd,
+        .usermsg,
+        .userptr,
+        => unreachable,
+
+        .accept => |req| {
+            // Accept is a multishot request, so we don't remove it from the in_flight queue
+            if (event.flags & EV.ERROR != 0) {
+                // Interpret data as an errno
+                const err = unexpectedError(dataToE(event.data));
+                task.result = .{ .accept = err };
+                return task.callback(rt, task.*);
+            }
+
+            if (posix.accept(req, null, null, 0)) |fd|
+                task.result = .{ .accept = fd }
+            else |_|
+                task.result = .{ .accept = error.Unexpected };
+            return task.callback(rt, task.*);
+        },
+
+        .connect => {
+            defer self.releaseTask(rt, task);
+            self.in_flight.remove(task);
+            if (event.flags & EV.ERROR != 0) {
+                // Interpret data as an errno
+                const err = unexpectedError(dataToE(event.data));
+                task.result = .{ .connect = err };
+            } else task.result = .{ .connect = {} };
+            return task.callback(rt, task.*);
+        },
+
+        .poll => {
+            defer self.releaseTask(rt, task);
+            self.in_flight.remove(task);
+            if (event.flags & EV.ERROR != 0) {
+                // Interpret data as an errno
+                const err = unexpectedError(dataToE(event.data));
+                task.result = .{ .poll = err };
+            } else task.result = .{ .poll = {} };
+            return task.callback(rt, task.*);
+        },
+
+        .recv => |req| {
+            defer self.releaseTask(rt, task);
+            self.in_flight.remove(task);
+            if (event.flags & EV.ERROR != 0) {
+                // Interpret data as an errno
+                const err = unexpectedError(dataToE(event.data));
+                task.result = .{ .recv = err };
+                return task.callback(rt, task.*);
+            }
+            if (posix.recv(req.fd, req.buffer, 0)) |n|
+                task.result = .{ .recv = n }
+            else |_|
+                task.result = .{ .recv = error.Unexpected };
+            return task.callback(rt, task.*);
+        },
+
+        .write => |req| {
+            defer self.releaseTask(rt, task);
+            self.in_flight.remove(task);
+            if (event.flags & EV.ERROR != 0) {
+                // Interpret data as an errno
+                const err = unexpectedError(dataToE(event.data));
+                task.result = .{ .write = err };
+                return task.callback(rt, task.*);
+            }
+            if (posix.write(req.fd, req.buffer)) |n|
+                task.result = .{ .write = n }
+            else |_|
+                task.result = .{ .write = error.Unexpected };
+            return task.callback(rt, task.*);
+        },
+
+        .writev => |req| {
+            defer self.releaseTask(rt, task);
+            self.in_flight.remove(task);
+            if (event.flags & EV.ERROR != 0) {
+                // Interpret data as an errno
+                const err = unexpectedError(dataToE(event.data));
+                task.result = .{ .writev = err };
+                return task.callback(rt, task.*);
+            }
+            if (posix.writev(req.fd, req.vecs)) |n|
+                task.result = .{ .writev = n }
+            else |_|
+                task.result = .{ .writev = error.Unexpected };
+            return task.callback(rt, task.*);
+        },
+    }
+}
+
+fn releaseTask(self: *Kqueue, rt: *io.Runtime, task: *io.Task) void {
+    rt.free_q.push(task);
+    if (task.deadline) |d| {
+        // remove the deadline
+        for (self.timers.items, 0..) |t, i| {
+            if (t == .deadline and t.deadline.task == d) {
+                // Remove the timer
+                _ = self.timers.orderedRemove(i);
+                rt.free_q.push(t.deadline.task);
+                return;
+            }
+        }
+    }
+}
+
+fn handleExpiredTimer(self: *Kqueue, rt: *io.Runtime, t: Timer) !void {
+    switch (t) {
+        .deadline => |deadline| {
+            defer self.releaseTask(rt, deadline.task);
+            if (deadline.task.state == .canceled) return;
+
+            try deadline.parent.cancel(rt, .{});
+        },
+
+        .timeout => |timeout| {
+            const task = timeout.task;
+            defer self.releaseTask(rt, task);
+            if (task.state == .canceled) return;
+            task.result = .{ .timer = {} };
+            try task.callback(rt, task.*);
+        },
+    }
+}

src/Mock.zig

···
+const Mock = @This();
+
+const std = @import("std");
+
+const io = @import("main.zig");
+
+const Allocator = std.mem.Allocator;
+const Queue = @import("queue.zig").Intrusive;
+const assert = std.debug.assert;
+const posix = std.posix;
+
+completions: Queue(io.Task, .complete) = .{},
+
+accept_cb: ?*const fn (*io.Task) io.Result = null,
+cancel_cb: ?*const fn (*io.Task) io.Result = null,
+close_cb: ?*const fn (*io.Task) io.Result = null,
+connect_cb: ?*const fn (*io.Task) io.Result = null,
+deadline_cb: ?*const fn (*io.Task) io.Result = null,
+msg_ring_cb: ?*const fn (*io.Task) io.Result = null,
+noop_cb: ?*const fn (*io.Task) io.Result = null,
+poll_cb: ?*const fn (*io.Task) io.Result = null,
+recv_cb: ?*const fn (*io.Task) io.Result = null,
+socket_cb: ?*const fn (*io.Task) io.Result = null,
+timer_cb: ?*const fn (*io.Task) io.Result = null,
+write_cb: ?*const fn (*io.Task) io.Result = null,
+writev_cb: ?*const fn (*io.Task) io.Result = null,
+
+userfd_cb: ?*const fn (*io.Task) io.Result = null,
+usermsg_cb: ?*const fn (*io.Task) io.Result = null,
+userptr_cb: ?*const fn (*io.Task) io.Result = null,
+
+/// Initialize a Ring
+pub fn init(_: u16) !Mock {
+    return .{};
+}
+
+pub fn deinit(self: *Mock, _: Allocator) void {
+    self.* = undefined;
+}
+
+pub fn done(self: *Mock) bool {
+    return self.completions.empty();
+}
+
+/// Initializes a child Ring which can be woken up by self. This must be called from the thread
+/// which will operate the child ring. Initializes with the same queue size as the parent
+pub fn initChild(_: Mock, entries: u16) !Mock {
+    return init(entries);
+}
+
+/// Return a file descriptor which can be used to poll the ring for completions
+pub fn pollableFd(_: *Mock) !posix.fd_t {
+    return -1;
+}
+
+pub fn submitAndWait(self: *Mock, queue: *Queue(io.Task, .in_flight)) !void {
+    return self.submit(queue);
+}
+
+pub fn submit(self: *Mock, queue: *Queue(io.Task, .in_flight)) !void {
+    while (queue.pop()) |task| {
+        task.result = switch (task.req) {
+            .accept => if (self.accept_cb) |cb| cb(task) else return error.NoMockCallback,
+            .cancel => if (self.cancel_cb) |cb| cb(task) else return error.NoMockCallback,
+            .close => if (self.close_cb) |cb| cb(task) else return error.NoMockCallback,
+            .connect => if (self.connect_cb) |cb| cb(task) else return error.NoMockCallback,
+            .deadline => if (self.deadline_cb) |cb| cb(task) else return error.NoMockCallback,
+            .msg_ring => if (self.msg_ring_cb) |cb| cb(task) else return error.NoMockCallback,
+            .noop => if (self.noop_cb) |cb| cb(task) else return error.NoMockCallback,
+            .poll => if (self.poll_cb) |cb| cb(task) else return error.NoMockCallback,
+            .recv => if (self.recv_cb) |cb| cb(task) else return error.NoMockCallback,
+            .socket => if (self.socket_cb) |cb| cb(task) else return error.NoMockCallback,
+            .timer => if (self.timer_cb) |cb| cb(task) else return error.NoMockCallback,
+            .userfd => if (self.userfd_cb) |cb| cb(task) else return error.NoMockCallback,
+            .usermsg => if (self.usermsg_cb) |cb| cb(task) else return error.NoMockCallback,
+            .userptr => if (self.userptr_cb) |cb| cb(task) else return error.NoMockCallback,
+            .write => if (self.write_cb) |cb| cb(task) else return error.NoMockCallback,
+            .writev => if (self.writev_cb) |cb| cb(task) else return error.NoMockCallback,
+        };
+        self.completions.push(task);
+    }
+}
+
+pub fn reapCompletions(self: *Mock, rt: *io.Runtime) anyerror!void {
+    while (self.completions.pop()) |task| {
+        try task.callback(rt, task.*);
+        rt.free_q.push(task);
+    }
+}

src/Task.zig

···
+const Task = @This();
+
+const std = @import("std");
+const io = @import("main.zig");
+
+const Allocator = std.mem.Allocator;
+const Runtime = io.Runtime;
+
+userdata: ?*anyopaque = null,
+msg: u16 = 0,
+callback: io.Callback = io.noopCallback,
+req: io.Request = .noop,
+
+result: ?io.Result = null,
+
+state: enum {
+    /// The task is free to be scheduled
+    free,
+
+    /// The task is in flight and may not be rescheduled. Some operations generate multiple
+    /// completions, so it is possible to receive a task in Callback and the task is still
+    /// considered to be in flight
+    in_flight,
+
+    /// The task was completed
+    complete,
+
+    /// The operation was canceled
+    canceled,
+} = .free,
+
+/// Deadline for the task to complete, in absolute time. If 0, there is no deadline
+deadline: ?*Task = null,
+
+next: ?*Task = null,
+prev: ?*Task = null,
+
+pub fn setDeadline(
+    self: *Task,
+    rt: *Runtime,
+    deadline: io.Timespec,
+) Allocator.Error!void {
+    std.debug.assert(!deadline.isZero());
+    const task = try rt.getTask();
+
+    task.* = .{
+        .callback = io.noopCallback,
+        .userdata = null,
+        .msg = 0,
+        .req = .{ .deadline = deadline },
+    };
+
+    self.deadline = task;
+}
+
+pub fn cancel(
+    self: *Task,
+    rt: *Runtime,
+    ctx: io.Context,
+) Allocator.Error!void {
+    const task = try rt.getTask();
+    task.* = .{
+        .callback = ctx.cb,
+        .msg = ctx.msg,
+        .userdata = ctx.ptr,
+        .req = .{ .cancel = .{ .task = self } },
+    };
+    rt.submission_q.push(task);
+}
+
+pub fn userdataCast(self: Task, comptime T: type) *T {
+    return @ptrCast(@alignCast(self.userdata));
+}
+
+pub fn msgToEnum(self: Task, comptime Enum: type) Enum {
+    return @enumFromInt(self.msg);
+}

src/Uring.zig

···
+const Uring = @This();
+
+const std = @import("std");
+const builtin = @import("builtin");
+
+const io = @import("main.zig");
+
+const Allocator = std.mem.Allocator;
+const Queue = @import("queue.zig").Intrusive;
+const assert = std.debug.assert;
+const linux = std.os.linux;
+const posix = std.posix;
+
+const common_flags: u32 =
+    linux.IORING_SETUP_SUBMIT_ALL | // Keep submitting events even if one had an error
+    linux.IORING_SETUP_CLAMP | // Clamp entries to system supported max
+    linux.IORING_SETUP_DEFER_TASKRUN | // Defer work until we submit tasks. Requires SINGLE_ISSUER
+    linux.IORING_SETUP_COOP_TASKRUN | // Don't interupt userspace when task is complete
+    linux.IORING_SETUP_SINGLE_ISSUER; // Only a single thread will issue tasks
+
+const msg_ring_received_cqe = 1 << 8;
+
+ring: linux.IoUring,
+in_flight: Queue(io.Task, .in_flight) = .{},
+eventfd: ?posix.fd_t = null,
+
+/// Initialize a Ring
+pub fn init(_: Allocator, entries: u16) !Uring {
+    var params = std.mem.zeroInit(linux.io_uring_params, .{
+        .flags = common_flags,
+        .sq_thread_idle = 1000,
+    });
+
+    return .{ .ring = try .init_params(entries, &params) };
+}
+
+pub fn deinit(self: *Uring, gpa: Allocator) void {
+    while (self.in_flight.pop()) |task| gpa.destroy(task);
+
+    if (self.ring.fd >= 0) {
+        self.ring.deinit();
+    }
+    if (self.eventfd) |fd| {
+        posix.close(fd);
+        self.eventfd = null;
+    }
+    self.* = undefined;
+}
+
+/// Initializes a child Ring which can be woken up by self. This must be called from the thread
+/// which will operate the child ring. Initializes with the same queue size as the parent
+pub fn initChild(self: Uring, entries: u16) !Uring {
+    const flags: u32 = common_flags | linux.IORING_SETUP_ATTACH_WQ;
+
+    var params = std.mem.zeroInit(linux.io_uring_params, .{
+        .flags = flags,
+        .sq_thread_idle = 1000,
+        .wq_fd = @as(u32, @bitCast(self.ring.fd)),
+    });
+
+    return .{ .ring = try .init_params(entries, &params) };
+}
+
+pub fn done(self: *Uring) bool {
+    return self.in_flight.empty();
+}
+
+/// Return a file descriptor which can be used to poll the ring for completions
+pub fn pollableFd(self: *Uring) !posix.fd_t {
+    if (self.eventfd) |fd| return fd;
+    const fd: posix.fd_t = @intCast(linux.eventfd(0, linux.EFD.CLOEXEC | linux.EFD.NONBLOCK));
+    try self.ring.register_eventfd(fd);
+    self.eventfd = fd;
+    return fd;
+}
+
+pub fn submitAndWait(self: *Uring, queue: *io.SubmissionQueue) !void {
+    var sqes_available = self.sqesAvailable();
+    while (queue.pop()) |task| {
+        const sqes_required = sqesRequired(task);
+        if (sqes_available < sqes_required) {
+            sqes_available += try self.ring.submit();
+            continue;
+        }
+        defer sqes_available -= sqes_required;
+        self.prepTask(task);
+    }
+
+    while (true) {
+        _ = self.ring.submit_and_wait(1) catch |err| {
+            switch (err) {
+                error.SignalInterrupt => continue,
+                else => return err,
+            }
+        };
+        return;
+    }
+}
+
+pub fn submit(self: *Uring, queue: *io.SubmissionQueue) !void {
+    var sqes_available = self.sqesAvailable();
+    while (queue.pop()) |task| {
+        const sqes_required = sqesRequired(task);
+        if (sqes_available < sqes_required) {
+            sqes_available += try self.ring.submit();
+            continue;
+        }
+        defer sqes_available -= sqes_required;
+        self.prepTask(task);
+    }
+    const n = try self.ring.submit();
+    _ = try self.ring.enter(n, 0, linux.IORING_ENTER_GETEVENTS);
+}
+
+fn sqesRequired(task: *const io.Task) u32 {
+    return if (task.deadline == null) 1 else 2;
+}
+
+fn sqesAvailable(self: *Uring) u32 {
+    return @intCast(self.ring.sq.sqes.len - self.ring.sq_ready());
+}
+
+fn prepTask(self: *Uring, task: *io.Task) void {
+    self.in_flight.push(task);
+    switch (task.req) {
+        .noop => {
+            const sqe = self.getSqe();
+            sqe.prep_nop();
+            sqe.user_data = @intFromPtr(task);
+            self.prepDeadline(task, sqe);
+        },
+
+        // Deadlines are always prepared from their parent task
+        .deadline => unreachable,
+
+        .timer => |*t| {
+            const sqe = self.getSqe();
+            sqe.prep_timeout(@ptrCast(t), 0, linux.IORING_TIMEOUT_REALTIME);
+            sqe.user_data = @intFromPtr(task);
+            self.prepDeadline(task, sqe);
+        },
+
+        .cancel => |c| {
+            const sqe = self.getSqe();
+            switch (c) {
+                .all => sqe.prep_cancel(0, linux.IORING_ASYNC_CANCEL_ANY),
+                .task => |t| sqe.prep_cancel(@intFromPtr(t), 0),
+            }
+            sqe.user_data = @intFromPtr(task);
+            self.prepDeadline(task, sqe);
+        },
+
+        .accept => |fd| {
+            const sqe = self.getSqe();
+            sqe.prep_multishot_accept(fd, null, null, 0);
+            sqe.user_data = @intFromPtr(task);
+            self.prepDeadline(task, sqe);
+        },
+
+        .msg_ring => |msg| {
+            const sqe = self.getSqe();
+            const fd = msg.target.backend.platform.ring.fd;
+            sqe.prep_rw(.MSG_RING, fd, 0, 0, @intFromPtr(msg.task));
+            sqe.user_data = @intFromPtr(task);
+            // Pass flags on the sent CQE. We use this to distinguish between a received message and
+            // a message freom our own loop
+            sqe.rw_flags |= linux.IORING_MSG_RING_FLAGS_PASS;
+            sqe.splice_fd_in |= msg_ring_received_cqe;
+            self.prepDeadline(task, sqe);
+        },
+
+        .recv => |req| {
+            const sqe = self.getSqe();
+            sqe.prep_recv(req.fd, req.buffer, 0);
+            sqe.user_data = @intFromPtr(task);
+            self.prepDeadline(task, sqe);
+        },
+
+        .write => |req| {
+            const sqe = self.getSqe();
+            sqe.prep_write(req.fd, req.buffer, 0);
+            sqe.user_data = @intFromPtr(task);
+            self.prepDeadline(task, sqe);
+        },
+
+        .writev => |req| {
+            const sqe = self.getSqe();
+            sqe.prep_writev(req.fd, req.vecs, 0);
+            sqe.user_data = @intFromPtr(task);
+            self.prepDeadline(task, sqe);
+        },
+
+        .close => |fd| {
+            const sqe = self.getSqe();
+            sqe.prep_close(fd);
+            sqe.user_data = @intFromPtr(task);
+            self.prepDeadline(task, sqe);
+        },
+
+        .poll => |req| {
+            const sqe = self.getSqe();
+            sqe.prep_poll_add(req.fd, req.mask);
+            sqe.user_data = @intFromPtr(task);
+            self.prepDeadline(task, sqe);
+        },
+
+        .socket => |req| {
+            const sqe = self.getSqe();
+            sqe.prep_socket(req.domain, req.type, req.protocol, 0);
+            sqe.user_data = @intFromPtr(task);
+            self.prepDeadline(task, sqe);
+        },
+
+        .connect => |req| {
+            const sqe = self.getSqe();
+            sqe.prep_connect(req.fd, req.addr, req.addr_len);
+            sqe.user_data = @intFromPtr(task);
+            self.prepDeadline(task, sqe);
+        },
+
+        // user* is only sent internally between rings and higher level wrappers
+        .userfd, .usermsg, .userptr => unreachable,
+    }
+}
+
+fn prepDeadline(self: *Uring, parent_task: *io.Task, parent_sqe: *linux.io_uring_sqe) void {
+    const task = parent_task.deadline orelse return;
+    self.in_flight.push(task);
+    assert(task.req == .deadline);
+    parent_sqe.flags |= linux.IOSQE_IO_LINK;
+
+    const sqe = self.getSqe();
+    const flags = linux.IORING_TIMEOUT_ABS | // absolute time
+        linux.IORING_TIMEOUT_REALTIME; // use the realtime clock (as opposed to boot time)
+    sqe.prep_link_timeout(@ptrCast(&task.req.deadline), flags);
+    sqe.user_data = @intFromPtr(task);
+}
+
+/// Get an sqe from the ring. Caller should only call this function if they are sure we have an SQE
+/// available. Asserts that we have one available
+fn getSqe(self: *Uring) *linux.io_uring_sqe {
+    assert(self.ring.sq.sqes.len > self.ring.sq_ready());
+    return self.ring.get_sqe() catch unreachable;
+}
+
+pub fn reapCompletions(self: *Uring, rt: *io.Runtime) anyerror!void {
+    var cqes: [64]linux.io_uring_cqe = undefined;
+    const n = self.ring.copy_cqes(&cqes, 0) catch |err| {
+        switch (err) {
+            error.SignalInterrupt => return,
+            else => return err,
+        }
+    };
+    for (cqes[0..n]) |cqe| {
+        const task: *io.Task = @ptrFromInt(cqe.user_data);
+
+        task.result = switch (task.req) {
+            .noop => .noop,
+
+            // Deadlines we don't do anything for, these are always sent to a noopCallback
+            .deadline => .{ .deadline = {} },
+
+            .timer => .{ .timer = switch (cqeToE(cqe.res)) {
+                .SUCCESS, .TIME => {},
+                .INVAL, .FAULT => io.ResultError.Invalid,
+                .CANCELED => io.ResultError.Canceled,
+                else => |e| unexpectedError(e),
+            } },
+
+            .cancel => .{ .cancel = switch (cqeToE(cqe.res)) {
+                .SUCCESS => {},
+                .INVAL => io.ResultError.Invalid,
+                .CANCELED => io.ResultError.Canceled,
+                .NOENT => io.CancelError.EntryNotFound,
+                .ALREADY => io.CancelError.NotCanceled,
+                else => |e| unexpectedError(e),
+            } },
+
+            .accept => .{ .accept = switch (cqeToE(cqe.res)) {
+                .SUCCESS => cqe.res,
+                .INVAL => io.ResultError.Invalid,
+                .CANCELED => io.ResultError.Canceled,
+                else => |e| unexpectedError(e),
+            } },
+
+            .msg_ring => .{ .msg_ring = switch (cqeToE(cqe.res)) {
+                .SUCCESS => {},
+                .INVAL => io.ResultError.Invalid,
+                .CANCELED => io.ResultError.Canceled,
+                else => |e| unexpectedError(e),
+            } },
+
+            .recv => .{ .recv = switch (cqeToE(cqe.res)) {
+                .SUCCESS => @intCast(cqe.res),
+                .INVAL => io.ResultError.Invalid,
+                .CANCELED => io.ResultError.Canceled,
+                .CONNRESET => io.RecvError.ConnectionResetByPeer,
+                else => |e| unexpectedError(e),
+            } },
+
+            .write => .{ .write = switch (cqeToE(cqe.res)) {
+                .SUCCESS => @intCast(cqe.res),
+                .INVAL => io.ResultError.Invalid,
+                .CANCELED => io.ResultError.Canceled,
+                else => |e| unexpectedError(e),
+            } },
+
+            .writev => .{ .writev = switch (cqeToE(cqe.res)) {
+                .SUCCESS => @intCast(cqe.res),
+                .INVAL => io.ResultError.Invalid,
+                .CANCELED => io.ResultError.Canceled,
+                else => |e| unexpectedError(e),
+            } },
+
+            .close => .{ .close = switch (cqeToE(cqe.res)) {
+                .SUCCESS => {},
+                .INVAL => io.ResultError.Invalid,
+                .CANCELED => io.ResultError.Canceled,
+                else => |e| unexpectedError(e),
+            } },
+
+            .poll => .{ .poll = switch (cqeToE(cqe.res)) {
+                .SUCCESS => {},
+                .INVAL => io.ResultError.Invalid,
+                .CANCELED => io.ResultError.Canceled,
+                else => |e| unexpectedError(e),
+            } },
+
+            .socket => .{ .socket = switch (cqeToE(cqe.res)) {
+                .SUCCESS => @intCast(cqe.res),
+                .INVAL => io.ResultError.Invalid,
+                .CANCELED => io.ResultError.Canceled,
+                else => |e| unexpectedError(e),
+            } },
+
+            .connect => .{ .connect = switch (cqeToE(cqe.res)) {
+                .SUCCESS => {},
+                .INVAL => io.ResultError.Invalid,
+                .CANCELED => io.ResultError.Canceled,
+                else => |e| unexpectedError(e),
+            } },
+
+            .usermsg => .{ .usermsg = @intCast(cqe.res) },
+
+            // userfd should never reach the runtime
+            .userfd, .userptr => unreachable,
+        };
+
+        try task.callback(rt, task.*);
+
+        if (cqe.flags & msg_ring_received_cqe != 0) {
+            // This message was received from another ring. We don't decrement inflight for this.
+            // But we do need to set the task as free because we will add it to our free list
+            rt.free_q.push(task);
+        } else if (cqe.flags & linux.IORING_CQE_F_MORE == 0) {
+            // If the cqe doesn't have IORING_CQE_F_MORE set, then this task is complete and free to
+            // be rescheduled
+            task.state = .complete;
+            self.in_flight.remove(task);
+            rt.free_q.push(task);
+        }
+    }
+}
+
+fn cqeToE(result: i32) std.posix.E {
+    if (result > -4096 and result < 0) {
+        return @as(std.posix.E, @enumFromInt(-result));
+    }
+    return .SUCCESS;
+}
+
+fn unexpectedError(err: posix.E) posix.UnexpectedError {
+    std.log.err("unexpected posix error: {}", .{err});
+    return error.Unexpected;
+}

src/main.zig

···
+const std = @import("std");
+const builtin = @import("builtin");
+
+const Allocator = std.mem.Allocator;
+pub const Mock = @import("Mock.zig");
+const Queue = @import("queue.zig").Intrusive;
+const io = @This();
+const posix = std.posix;
+
+pub const has_kqueue = switch (builtin.os.tag) {
+    .dragonfly,
+    .freebsd,
+    .ios,
+    .macos,
+    .netbsd,
+    .openbsd,
+    .tvos,
+    .visionos,
+    .watchos,
+    => true,
+
+    else => false,
+};
+pub const has_io_uring = builtin.os.tag == .linux;
+
+pub const Task = @import("Task.zig");
+pub const Callback = *const fn (*Runtime, Task) anyerror!void;
+pub fn noopCallback(_: *Runtime, _: Task) anyerror!void {}
+
+pub const RunCondition = enum {
+    once,
+    until_done,
+    forever,
+};
+
+pub const Context = struct {
+    ptr: ?*anyopaque = null,
+    msg: u16 = 0,
+    cb: Callback = noopCallback,
+};
+
+/// Used for timeouts and deadlines. We make this struct extern because we will ptrCast it to the
+/// linux kernel timespec struct
+pub const Timespec = extern struct {
+    sec: i64 = 0,
+    nsec: i64 = 0,
+
+    pub fn isZero(self: Timespec) bool {
+        return self.sec == 0 and self.nsec == 0;
+    }
+};
+
+pub const Backend = union(enum) {
+    mock: Mock,
+
+    platform: switch (builtin.os.tag) {
+        .dragonfly,
+        .freebsd,
+        .ios,
+        .macos,
+        .netbsd,
+        .openbsd,
+        .tvos,
+        .visionos,
+        .watchos,
+        => @import("Kqueue.zig"),
+
+        .linux => @import("Uring.zig"),
+
+        else => @compileError("unsupported os"),
+    },
+
+    pub fn initChild(self: *Backend, entries: u16) !Backend {
+        switch (self.*) {
+            .mock => return .{ .mock = .{} },
+            .platform => |*p| return .{ .platform = try p.initChild(entries) },
+        }
+    }
+
+    pub fn deinit(self: *Backend, gpa: Allocator) void {
+        switch (self.*) {
+            inline else => |*backend| backend.deinit(gpa),
+        }
+    }
+
+    pub fn pollableFd(self: *Backend) !posix.fd_t {
+        return switch (self.*) {
+            inline else => |*backend| backend.pollableFd(),
+        };
+    }
+
+    pub fn submitAndWait(self: *Backend, queue: *SubmissionQueue) !void {
+        return switch (self.*) {
+            inline else => |*backend| backend.submitAndWait(queue),
+        };
+    }
+
+    pub fn submit(self: *Backend, queue: *SubmissionQueue) !void {
+        return switch (self.*) {
+            inline else => |*backend| backend.submit(queue),
+        };
+    }
+
+    pub fn reapCompletions(
+        self: *Backend,
+        rt: *Runtime,
+    ) !void {
+        return switch (self.*) {
+            inline else => |*backend| backend.reapCompletions(rt),
+        };
+    }
+
+    pub fn done(self: *Backend) bool {
+        return switch (self.*) {
+            inline else => |*backend| backend.done(),
+        };
+    }
+};
+
+pub const CompletionQueue = Queue(Task, .complete);
+pub const FreeQueue = Queue(Task, .free);
+pub const SubmissionQueue = Queue(Task, .in_flight);
+
+pub const Runtime = struct {
+    backend: Backend,
+    gpa: Allocator,
+
+    completion_q: CompletionQueue = .{},
+    submission_q: SubmissionQueue = .{},
+    free_q: FreeQueue = .{},
+
+    pub fn init(gpa: Allocator, entries: u16) !Runtime {
+        return .{
+            .backend = .{ .platform = try .init(gpa, entries) },
+            .gpa = gpa,
+        };
+    }
+
+    pub fn initChild(self: *Runtime, entries: u16) !Runtime {
+        return .{
+            .backend = try self.backend.initChild(entries),
+            .gpa = self.gpa,
+        };
+    }
+
+    pub fn initMock(gpa: Allocator, entries: u16) !Runtime {
+        return .{
+            .backend = .{ .mock = try .init(entries) },
+            .gpa = gpa,
+            .free_q = .{},
+            .submission_q = .{},
+            .completion_q = .{},
+        };
+    }
+
+    pub fn deinit(self: *Runtime) void {
+        self.backend.deinit(self.gpa);
+        while (self.free_q.pop()) |task| self.gpa.destroy(task);
+        while (self.submission_q.pop()) |task| self.gpa.destroy(task);
+        while (self.completion_q.pop()) |task| self.gpa.destroy(task);
+    }
+
+    pub fn run(self: *Runtime, condition: RunCondition) !void {
+        while (true) {
+            try self.backend.submitAndWait(&self.submission_q);
+            try self.backend.reapCompletions(self);
+            switch (condition) {
+                .once => return,
+                .until_done => if (self.backend.done() and self.submission_q.empty()) return,
+                .forever => {},
+            }
+        }
+    }
+
+    pub fn getTask(self: *Runtime) Allocator.Error!*Task {
+        return self.free_q.pop() orelse try self.gpa.create(Task);
+    }
+
+    pub fn noop(
+        self: *Runtime,
+        ctx: Context,
+    ) Allocator.Error!*Task {
+        const task = try self.getTask();
+        task.* = .{
+            .userdata = ctx.ptr,
+            .msg = ctx.msg,
+            .callback = ctx.cb,
+            .req = .noop,
+        };
+
+        self.submission_q.push(task);
+        return task;
+    }
+
+    pub fn timer(
+        self: *Runtime,
+        duration: Timespec,
+        ctx: Context,
+    ) Allocator.Error!*Task {
+        const task = try self.getTask();
+        task.* = .{
+            .userdata = ctx.ptr,
+            .msg = ctx.msg,
+            .callback = ctx.cb,
+            .req = .{ .timer = duration },
+        };
+
+        self.submission_q.push(task);
+        return task;
+    }
+
+    pub fn cancelAll(self: *Runtime) Allocator.Error!void {
+        const task = try self.getTask();
+        task.* = .{
+            .req = .{ .cancel = .all },
+        };
+
+        self.submission_q.push(task);
+    }
+
+    pub fn accept(
+        self: *Runtime,
+        fd: posix.fd_t,
+        ctx: Context,
+    ) Allocator.Error!*Task {
+        const task = try self.getTask();
+        task.* = .{
+            .userdata = ctx.ptr,
+            .msg = ctx.msg,
+            .callback = ctx.cb,
+            .req = .{ .accept = fd },
+        };
+
+        self.submission_q.push(task);
+        return task;
+    }
+
+    pub fn msgRing(
+        self: *Runtime,
+        target: *Runtime,
+        target_task: *Task, // The task that the target ring will receive. The callbacks of
+        // this task are what will be called when the target receives the message
+
+        ctx: Context,
+    ) Allocator.Error!*Task {
+        // This is the task to send the message
+        const task = try self.getTask();
+        task.* = .{
+            .userdata = ctx.ptr,
+            .msg = ctx.msg,
+            .callback = ctx.cb,
+            .req = .{ .msg_ring = .{
+                .target = target,
+                .task = target_task,
+            } },
+        };
+        target_task.state = .in_flight;
+        self.submission_q.push(task);
+        return task;
+    }
+
+    pub fn recv(
+        self: *Runtime,
+        fd: posix.fd_t,
+        buffer: []u8,
+        ctx: Context,
+    ) Allocator.Error!*Task {
+        const task = try self.getTask();
+        task.* = .{
+            .userdata = ctx.ptr,
+            .msg = ctx.msg,
+            .callback = ctx.cb,
+            .req = .{ .recv = .{
+                .fd = fd,
+                .buffer = buffer,
+            } },
+        };
+
+        self.submission_q.push(task);
+        return task;
+    }
+
+    pub fn write(
+        self: *Runtime,
+        fd: posix.fd_t,
+        buffer: []const u8,
+        ctx: Context,
+    ) Allocator.Error!*Task {
+        const task = try self.getTask();
+        task.* = .{
+            .userdata = ctx.ptr,
+            .msg = ctx.msg,
+            .callback = ctx.cb,
+            .req = .{ .write = .{
+                .fd = fd,
+                .buffer = buffer,
+            } },
+        };
+
+        self.submission_q.push(task);
+        return task;
+    }
+
+    pub fn writev(
+        self: *Runtime,
+        fd: posix.fd_t,
+        vecs: []const posix.iovec_const,
+        ctx: Context,
+    ) Allocator.Error!*Task {
+        const task = try self.getTask();
+        task.* = .{
+            .userdata = ctx.ptr,
+            .msg = ctx.msg,
+            .callback = ctx.cb,
+            .req = .{ .writev = .{
+                .fd = fd,
+                .vecs = vecs,
+            } },
+        };
+
+        self.submission_q.push(task);
+        return task;
+    }
+
+    pub fn close(
+        self: *Runtime,
+        fd: posix.fd_t,
+        ctx: Context,
+    ) Allocator.Error!*Task {
+        const task = try self.getTask();
+        task.* = .{
+            .userdata = ctx.ptr,
+            .msg = ctx.msg,
+            .callback = ctx.cb,
+            .req = .{ .close = fd },
+        };
+
+        self.submission_q.push(task);
+        return task;
+    }
+
+    pub fn poll(
+        self: *Runtime,
+        fd: posix.fd_t,
+        mask: u32,
+        ctx: Context,
+    ) Allocator.Error!*Task {
+        const task = try self.getTask();
+        task.* = .{
+            .userdata = ctx.ptr,
+            .msg = ctx.msg,
+            .callback = ctx.cb,
+            .req = .{ .poll = .{ .fd = fd, .mask = mask } },
+        };
+
+        self.submission_q.push(task);
+        return task;
+    }
+
+    pub fn socket(
+        self: *Runtime,
+        domain: u32,
+        socket_type: u32,
+        protocol: u32,
+        ctx: Context,
+    ) Allocator.Error!*Task {
+        const task = try self.getTask();
+        task.* = .{
+            .userdata = ctx.ptr,
+            .msg = ctx.msg,
+            .callback = ctx.cb,
+            .req = .{ .socket = .{ .domain = domain, .type = socket_type, .protocol = protocol } },
+        };
+
+        self.submission_q.push(task);
+        return task;
+    }
+
+    pub fn connect(
+        self: *Runtime,
+        fd: posix.socket_t,
+        addr: *posix.sockaddr,
+        addr_len: posix.socklen_t,
+        ctx: Context,
+    ) Allocator.Error!*Task {
+        const task = try self.getTask();
+        task.* = .{
+            .userdata = ctx.ptr,
+            .msg = ctx.msg,
+            .callback = ctx.cb,
+            .req = .{ .connect = .{ .fd = fd, .addr = addr, .addr_len = addr_len } },
+        };
+
+        self.submission_q.push(task);
+        return task;
+    }
+};
+
+pub const Op = enum {
+    noop,
+    deadline,
+    timer,
+    cancel,
+    accept,
+    msg_ring,
+    recv,
+    write,
+    writev,
+    close,
+    poll,
+    socket,
+    connect,
+
+    /// userfd is meant to send file descriptors between Runtime instances (using msgRing)
+    userfd,
+    /// usermsg is meant to send a u16 between runtime instances (using msgRing)
+    usermsg,
+    /// userptr is meant to send a pointer between runtime instances (using msgRing)
+    userptr,
+};
+
+pub const Request = union(Op) {
+    noop,
+    deadline: Timespec,
+    timer: Timespec,
+    cancel: union(enum) {
+        all,
+        task: *Task,
+    },
+    accept: posix.fd_t,
+    msg_ring: struct {
+        target: *Runtime,
+        task: *Task,
+    },
+    recv: struct {
+        fd: posix.fd_t,
+        buffer: []u8,
+    },
+    write: struct {
+        fd: posix.fd_t,
+        buffer: []const u8,
+    },
+    writev: struct {
+        fd: posix.fd_t,
+        vecs: []const posix.iovec_const,
+    },
+    close: posix.fd_t,
+    poll: struct {
+        fd: posix.fd_t,
+        mask: u32,
+    },
+    socket: struct {
+        domain: u32,
+        type: u32,
+        protocol: u32,
+    },
+    connect: struct {
+        fd: posix.socket_t,
+        addr: *posix.sockaddr,
+        addr_len: posix.socklen_t,
+    },
+
+    userfd,
+    usermsg,
+    userptr,
+};
+
+pub const Result = union(Op) {
+    noop,
+    deadline: ResultError!void,
+    timer: ResultError!void,
+    cancel: CancelError!void,
+    accept: ResultError!posix.fd_t,
+    msg_ring: ResultError!void,
+    recv: RecvError!usize,
+    write: ResultError!usize,
+    writev: ResultError!usize,
+    close: ResultError!void,
+    poll: ResultError!void,
+    socket: ResultError!posix.fd_t,
+    connect: ResultError!void,
+
+    userfd: anyerror!posix.fd_t,
+    usermsg: u16,
+    userptr: anyerror!?*anyopaque,
+};
+
+pub const ResultError = error{
+    /// The request was invalid
+    Invalid,
+    /// The request was canceled
+    Canceled,
+    /// An unexpected error occured
+    Unexpected,
+};
+
+pub const CancelError = ResultError || error{
+    /// The entry to cancel couldn't be found
+    EntryNotFound,
+    /// The entry couldn't be canceled
+    NotCanceled,
+};
+
+pub const RecvError = ResultError || error{
+    /// The entry to cancel couldn't be found
+    ConnectionResetByPeer,
+};
+
+test {
+    _ = @import("net.zig");
+    _ = @import("queue.zig");
+    _ = @import("tls.zig");
+
+    _ = @import("Mock.zig");
+
+    if (has_io_uring) _ = @import("Uring.zig");
+    if (has_kqueue) _ = @import("Kqueue.zig");
+}
+
+/// Foo is only for testing
+const Foo = struct {
+    bar: usize = 0,
+
+    fn callback(_: *io.Runtime, task: io.Task) anyerror!void {
+        const self = task.userdataCast(Foo);
+        self.bar += 1;
+    }
+};
+
+test "runtime: noop" {
+    var rt: io.Runtime = try .init(std.testing.allocator, 16);
+    defer rt.deinit();
+
+    var foo: Foo = .{};
+
+    const ctx: Context = .{ .ptr = &foo, .cb = Foo.callback };
+
+    // noop is triggered synchronously with submit. If we wait, we'll be waiting forever
+    _ = try rt.noop(ctx);
+    try rt.run(.once);
+    try std.testing.expectEqual(1, foo.bar);
+    _ = try rt.noop(ctx);
+    _ = try rt.noop(ctx);
+    try rt.run(.once);
+    try std.testing.expectEqual(3, foo.bar);
+}
+
+test "runtime: timer" {
+    var rt: io.Runtime = try .init(std.testing.allocator, 16);
+    defer rt.deinit();
+
+    var foo: Foo = .{};
+
+    const ctx: Context = .{ .ptr = &foo, .cb = Foo.callback };
+
+    const start = std.time.nanoTimestamp();
+    const end = start + 100 * std.time.ns_per_ms;
+    _ = try rt.timer(.{ .nsec = 100 * std.time.ns_per_ms }, ctx);
+    try rt.run(.once);
+    try std.testing.expect(std.time.nanoTimestamp() > end);
+    try std.testing.expectEqual(1, foo.bar);
+}
+
+test "runtime: poll" {
+    var rt: io.Runtime = try .init(std.testing.allocator, 16);
+    defer rt.deinit();
+
+    var foo: Foo = .{};
+    const pipe = try posix.pipe2(.{ .CLOEXEC = true });
+
+    const ctx: Context = .{ .ptr = &foo, .cb = Foo.callback };
+
+    _ = try rt.poll(pipe[0], posix.POLL.IN, ctx);
+    try std.testing.expectEqual(1, rt.submission_q.len());
+
+    _ = try posix.write(pipe[1], "io_uring is the best");
+    try rt.run(.once);
+    try std.testing.expectEqual(1, foo.bar);
+}
+
+test "runtime: deadline doesn't call user callback" {
+    const gpa = std.testing.allocator;
+    var rt = try io.Runtime.init(gpa, 16);
+    defer rt.deinit();
+
+    var foo: Foo = .{};
+    const ctx: Context = .{ .ptr = &foo, .cb = Foo.callback };
+    const task = try rt.noop(ctx);
+    try task.setDeadline(&rt, .{ .sec = 1 });
+
+    try rt.run(.until_done);
+
+    // Callback only called once
+    try std.testing.expectEqual(1, foo.bar);
+}
+
+test "runtime: timeout" {
+    const gpa = std.testing.allocator;
+    var rt = try io.Runtime.init(gpa, 16);
+    defer rt.deinit();
+
+    var foo: Foo = .{};
+    const ctx: Context = .{ .ptr = &foo, .cb = Foo.callback };
+
+    const delay = 1 * std.time.ns_per_ms;
+    _ = try rt.timer(.{ .nsec = delay }, ctx);
+
+    const start = std.time.nanoTimestamp();
+    try rt.run(.until_done);
+    try std.testing.expect(start + delay < std.time.nanoTimestamp());
+    try std.testing.expectEqual(1, foo.bar);
+}
+
+test "runtime: cancel" {
+    const gpa = std.testing.allocator;
+    var rt = try io.Runtime.init(gpa, 16);
+    defer rt.deinit();
+
+    var foo: Foo = .{};
+    const ctx: Context = .{ .ptr = &foo, .cb = Foo.callback };
+
+    const delay = 1 * std.time.ns_per_s;
+    const task = try rt.timer(.{ .nsec = delay }, ctx);
+
+    try task.cancel(&rt, .{});
+
+    const start = std.time.nanoTimestamp();
+    try rt.run(.until_done);
+    // Expect that we didn't delay long enough
+    try std.testing.expect(start + delay > std.time.nanoTimestamp());
+    try std.testing.expectEqual(1, foo.bar);
+}
+
+test "runtime: cancel all" {
+    const gpa = std.testing.allocator;
+    var rt = try io.Runtime.init(gpa, 16);
+    defer rt.deinit();
+
+    const Foo2 = struct {
+        bar: usize = 0,
+
+        fn callback(_: *io.Runtime, task: io.Task) anyerror!void {
+            const self = task.userdataCast(@This());
+            const result = task.result.?;
+            _ = result.timer catch |err| {
+                switch (err) {
+                    error.Canceled => self.bar += 1,
+                    else => {},
+                }
+            };
+        }
+    };
+    var foo: Foo2 = .{};
+    const ctx: Context = .{ .ptr = &foo, .cb = Foo2.callback };
+
+    const delay = 1 * std.time.ns_per_s;
+    _ = try rt.timer(.{ .nsec = delay }, ctx);
+    _ = try rt.timer(.{ .nsec = delay }, ctx);
+    _ = try rt.timer(.{ .nsec = delay }, ctx);
+    _ = try rt.timer(.{ .nsec = delay }, ctx);
+
+    try rt.cancelAll();
+    const start = std.time.nanoTimestamp();
+    try rt.run(.until_done);
+    // Expect that we didn't delay long enough
+    try std.testing.expect(start + delay > std.time.nanoTimestamp());
+    try std.testing.expectEqual(4, foo.bar);
+}
+
+test "runtime: msgRing" {
+    const gpa = std.testing.allocator;
+    var rt1 = try io.Runtime.init(gpa, 16);
+    defer rt1.deinit();
+
+    var rt2 = try rt1.initChild(16);
+    defer rt2.deinit();
+
+    const Foo2 = struct {
+        rt1: bool = false,
+        rt2: bool = false,
+
+        const Msg = enum { rt1, rt2 };
+
+        fn callback(_: *io.Runtime, task: io.Task) anyerror!void {
+            const self = task.userdataCast(@This());
+            const msg = task.msgToEnum(Msg);
+            switch (msg) {
+                .rt1 => self.rt1 = true,
+                .rt2 => self.rt2 = true,
+            }
+        }
+    };
+
+    var foo: Foo2 = .{};
+
+    // The task we will send from rt1 to rt2
+    const target_task = try rt1.getTask();
+    target_task.* = .{
+        .userdata = &foo,
+        .callback = Foo2.callback,
+        .msg = @intFromEnum(Foo2.Msg.rt2),
+        .result = .{ .usermsg = 0 },
+    };
+
+    _ = try rt1.msgRing(
+        &rt2,
+        target_task,
+        .{ .cb = Foo2.callback, .msg = @intFromEnum(Foo2.Msg.rt1), .ptr = &foo },
+    );
+
+    try rt1.run(.until_done);
+    try std.testing.expect(foo.rt1);
+    try std.testing.expect(!foo.rt2);
+    try rt2.run(.until_done);
+    try std.testing.expect(foo.rt1);
+    try std.testing.expect(foo.rt2);
+}

src/net.zig

···
+const std = @import("std");
+const io = @import("main.zig");
+
+const posix = std.posix;
+const Allocator = std.mem.Allocator;
+const Uri = std.Uri;
+
+const assert = std.debug.assert;
+
+pub fn tcpConnectToHost(
+    rt: *io.Runtime,
+    host: []const u8,
+    port: u16,
+    ctx: io.Context,
+) !*ConnectTask {
+    // TODO: getAddressList could be rewritten to be async. It accesses the filesystem and could
+    // make a DNS request
+    const list = try std.net.getAddressList(rt.gpa, host, port);
+    defer list.deinit();
+
+    const addr = for (list.addrs) |addr| {
+        break addr;
+    } else return error.AddressNotFound;
+
+    return tcpConnectToAddr(rt, addr, ctx);
+}
+
+pub fn tcpConnectToAddr(
+    rt: *io.Runtime,
+    addr: std.net.Address,
+    ctx: io.Context,
+) Allocator.Error!*ConnectTask {
+    const conn = try rt.gpa.create(ConnectTask);
+    errdefer rt.gpa.destroy(conn);
+
+    conn.* = .{
+        .ctx = ctx,
+
+        .addr = addr,
+        .fd = null,
+        .task = undefined,
+    };
+
+    conn.task = try rt.socket(
+        conn.addr.any.family,
+        posix.SOCK.STREAM | posix.SOCK.CLOEXEC,
+        posix.IPPROTO.TCP,
+        .{ .ptr = conn, .msg = @intFromEnum(ConnectTask.Msg.socket), .cb = ConnectTask.handleMsg },
+    );
+
+    return conn;
+}
+
+pub const ConnectTask = struct {
+    ctx: io.Context,
+
+    addr: std.net.Address,
+    fd: ?posix.fd_t,
+
+    /// Task is the current task we are operating on. We store this to provide cancelation
+    task: *io.Task,
+
+    pub const Msg = enum {
+        socket,
+        connect,
+    };
+
+    /// Cancels the current task. Not guaranteed to actually cancel. User's callback will get an
+    /// error.Canceled if cancelation was successful, otherwise the operation will complete as
+    /// normal and this is essentially a no-op
+    pub fn cancel(self: *ConnectTask, rt: *io.Runtime) void {
+        _ = self.task.cancel(rt, null, 0, io.noopCallback) catch {};
+    }
+
+    pub fn handleMsg(rt: *io.Runtime, task: io.Task) anyerror!void {
+        const self = task.userdataCast(ConnectTask);
+        const result = task.result.?;
+        switch (task.msgToEnum(Msg)) {
+            .socket => {
+                assert(result == .socket);
+                self.fd = result.socket catch |err| {
+                    defer rt.gpa.destroy(self);
+                    try self.ctx.cb(rt, .{
+                        .userdata = self.ctx.ptr,
+                        .msg = self.ctx.msg,
+                        .result = .{ .userfd = err },
+                        .callback = self.ctx.cb,
+                        .req = .userfd,
+                    });
+                    return;
+                };
+
+                self.task = try rt.connect(
+                    self.fd.?,
+                    &self.addr.any,
+                    self.addr.getOsSockLen(),
+                    .{ .ptr = self, .msg = @intFromEnum(Msg.connect), .cb = ConnectTask.handleMsg },
+                );
+            },
+
+            .connect => {
+                assert(result == .connect);
+                defer rt.gpa.destroy(self);
+
+                _ = result.connect catch |err| {
+                    try self.ctx.cb(rt, .{
+                        .userdata = self.ctx.ptr,
+                        .msg = self.ctx.msg,
+                        .result = .{ .userfd = err },
+                        .callback = self.ctx.cb,
+                        .req = .userfd,
+                    });
+                    _ = try rt.close(self.fd.?, .{});
+                    return;
+                };
+
+                try self.ctx.cb(rt, .{
+                    .userdata = self.ctx.ptr,
+                    .msg = self.ctx.msg,
+                    .result = .{ .userfd = self.fd.? },
+                    .callback = self.ctx.cb,
+                    .req = .userfd,
+                });
+            },
+        }
+    }
+};
+
+test "tcp connect" {
+    var rt: io.Runtime = try .init(std.testing.allocator, 16);
+    defer rt.deinit();
+
+    const addr: std.net.Address = try .parseIp4("127.0.0.1", 80);
+
+    {
+        // Happy path
+        const conn = try tcpConnectToAddr(&rt, addr, .{});
+        errdefer std.testing.allocator.destroy(conn);
+
+        const task1 = rt.submission_q.pop().?;
+        defer std.testing.allocator.destroy(task1);
+        try std.testing.expect(task1.req == .socket);
+        try std.testing.expect(rt.submission_q.pop() == null);
+
+        const fd: posix.fd_t = 7;
+        try ConnectTask.handleMsg(&rt, .{
+            .userdata = conn,
+            .msg = @intFromEnum(ConnectTask.Msg.socket),
+            .result = .{ .socket = fd },
+            .req = .userfd,
+        });
+
+        const task2 = rt.submission_q.pop().?;
+        defer std.testing.allocator.destroy(task2);
+        try std.testing.expect(task2.req == .connect);
+        try std.testing.expect(rt.submission_q.pop() == null);
+
+        try ConnectTask.handleMsg(&rt, .{
+            .userdata = conn,
+            .msg = @intFromEnum(ConnectTask.Msg.connect),
+            .result = .{ .connect = {} },
+            .req = .userfd,
+        });
+        try std.testing.expect(rt.submission_q.pop() == null);
+    }
+
+    {
+        // socket error
+        const conn = try tcpConnectToAddr(&rt, addr, .{});
+        errdefer std.testing.allocator.destroy(conn);
+
+        const task1 = rt.submission_q.pop().?;
+        defer std.testing.allocator.destroy(task1);
+
+        try ConnectTask.handleMsg(&rt, .{
+            .userdata = conn,
+            .msg = @intFromEnum(ConnectTask.Msg.socket),
+            .result = .{ .socket = error.Canceled },
+            .req = .userfd,
+        });
+        try std.testing.expect(rt.submission_q.pop() == null);
+    }
+
+    {
+        // connect error
+        const conn = try tcpConnectToAddr(&rt, addr, .{});
+        errdefer std.testing.allocator.destroy(conn);
+
+        const task1 = rt.submission_q.pop().?;
+        defer std.testing.allocator.destroy(task1);
+        try std.testing.expect(task1.req == .socket);
+        try std.testing.expect(rt.submission_q.pop() == null);
+
+        const fd: posix.fd_t = 7;
+        try ConnectTask.handleMsg(&rt, .{
+            .userdata = conn,
+            .msg = @intFromEnum(ConnectTask.Msg.socket),
+            .result = .{ .socket = fd },
+            .req = .userfd,
+        });
+
+        const task2 = rt.submission_q.pop().?;
+        defer std.testing.allocator.destroy(task2);
+        try std.testing.expect(task2.req == .connect);
+        try std.testing.expect(rt.submission_q.pop() == null);
+
+        try ConnectTask.handleMsg(&rt, .{
+            .userdata = conn,
+            .msg = @intFromEnum(ConnectTask.Msg.connect),
+            .result = .{ .connect = error.Canceled },
+            .req = .noop,
+        });
+        const task3 = rt.submission_q.pop().?;
+        defer std.testing.allocator.destroy(task3);
+        try std.testing.expect(task3.req == .close);
+        try std.testing.expect(rt.submission_q.pop() == null);
+    }
+}

src/queue.zig

···
+// This code is mostly a copy of the intrusive queue code from libxev. I've modified it to be a
+// doubly linked list that also ensures a certain state is set on each node when put into the list
+//
+// MIT License
+//
+// Copyright (c) 2023 Mitchell Hashimoto
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
+// associated documentation files (the "Software"), to deal in the Software without restriction,
+// including without limitation the rights to use, copy, modify, merge, publish, distribute,
+// sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in all copies or
+// substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
+// NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
+// DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+const std = @import("std");
+const assert = std.debug.assert;
+
+/// An intrusive queue implementation. The type T must have a field
+/// "next" of type `?*T` and a field "state" which is an enum with a value matching the passed in
+/// value
+pub fn Intrusive(comptime T: type, comptime state: @Type(.enum_literal)) type {
+    return struct {
+        const Self = @This();
+
+        const set_state = state;
+
+        /// Head is the front of the queue and tail is the back of the queue.
+        head: ?*T = null,
+        tail: ?*T = null,
+
+        /// Enqueue a new element to the back of the queue.
+        pub fn push(self: *Self, v: *T) void {
+            assert(v.next == null);
+            v.state = set_state;
+
+            if (self.tail) |tail| {
+                // If we have elements in the queue, then we add a new tail.
+                tail.next = v;
+                v.prev = tail;
+                self.tail = v;
+            } else {
+                // No elements in the queue we setup the initial state.
+                self.head = v;
+                self.tail = v;
+            }
+        }
+
+        /// Dequeue the next element from the queue.
+        pub fn pop(self: *Self) ?*T {
+            // The next element is in "head".
+            const next = self.head orelse return null;
+
+            // If the head and tail are equal this is the last element
+            // so we also set tail to null so we can now be empty.
+            if (self.head == self.tail) self.tail = null;
+
+            // Head is whatever is next (if we're the last element,
+            // this will be null);
+            self.head = next.next;
+            if (self.head) |head| head.prev = null;
+
+            // We set the "next" field to null so that this element
+            // can be inserted again.
+            next.next = null;
+            next.prev = null;
+            return next;
+        }
+
+        /// Returns true if the queue is empty.
+        pub fn empty(self: Self) bool {
+            return self.head == null;
+        }
+
+        /// Removes the item from the queue. Asserts that Queue contains the item
+        pub fn remove(self: *Self, item: *T) void {
+            assert(self.hasItem(item));
+            if (item.prev) |prev| prev.next = item.next else self.head = item.next;
+
+            if (item.next) |next| next.prev = item.prev else self.tail = item.prev;
+
+            item.prev = null;
+            item.next = null;
+        }
+
+        pub fn hasItem(self: Self, item: *T) bool {
+            var maybe_node = self.head;
+            while (maybe_node) |node| {
+                if (node == item) return true;
+                maybe_node = node.next;
+            } else return false;
+        }
+
+        pub fn len(self: Self) usize {
+            var count: usize = 0;
+            var maybe_node = self.head;
+            while (maybe_node) |node| {
+                count += 1;
+                maybe_node = node.next;
+            }
+            return count;
+        }
+    };
+}

src/tls.zig

···
+const std = @import("std");
+const tls = @import("tls");
+const io = @import("main.zig");
+
+const Allocator = std.mem.Allocator;
+const CertBundle = tls.config.cert.Bundle;
+const assert = std.debug.assert;
+const mem = std.mem;
+const posix = std.posix;
+
+pub const Client = struct {
+    gpa: Allocator,
+    fd: posix.fd_t,
+    tls: tls.nonblock.Connection,
+    recv_task: ?*io.Task = null,
+
+    read_buf: [tls.max_ciphertext_record_len]u8 = undefined,
+    read_end: usize = 0,
+
+    cleartext_buf: std.ArrayListUnmanaged(u8) = .empty,
+    ciphertext_buf: std.ArrayListUnmanaged(u8) = .empty,
+    written: usize = 0,
+
+    userdata: ?*anyopaque = null,
+    callback: *const fn (*io.Runtime, io.Task) anyerror!void = io.noopCallback,
+    close_msg: u16 = 0,
+    write_msg: u16 = 0,
+    recv_msg: u16 = 0,
+
+    pub const HandshakeTask = struct {
+        userdata: ?*anyopaque,
+        callback: io.Callback,
+        msg: u16,
+
+        fd: posix.fd_t,
+        buffer: [tls.max_ciphertext_record_len]u8 = undefined,
+        read_end: usize = 0,
+        handshake: tls.nonblock.Client,
+        task: *io.Task,
+
+        pub fn handleMsg(rt: *io.Runtime, task: io.Task) anyerror!void {
+            const self = task.userdataCast(HandshakeTask);
+            const result = task.result.?;
+
+            switch (result) {
+                .write => {
+                    _ = result.write catch |err| {
+                        defer rt.gpa.destroy(self);
+                        // send the error to the callback
+                        try self.callback(rt, .{
+                            .userdata = self.userdata,
+                            .msg = self.msg,
+                            .result = .{ .userptr = err },
+                            .callback = self.callback,
+                            .req = .userptr,
+                        });
+                        return;
+                    };
+
+                    if (self.handshake.done()) {
+                        defer rt.gpa.destroy(self);
+                        // Handshake is done. Create a client and deliver it to the callback
+                        const client = try self.initClient(rt.gpa);
+                        try self.callback(rt, .{
+                            .userdata = self.userdata,
+                            .msg = self.msg,
+                            .result = .{ .userptr = client },
+                            .callback = self.callback,
+                            .req = .userptr,
+                        });
+                        return;
+                    }
+
+                    // Arm a recv task
+                    self.task = try rt.recv(self.fd, &self.buffer, .{
+                        .ptr = self,
+                        .cb = handleMsg,
+                    });
+                },
+
+                .recv => {
+                    const n = result.recv catch |err| {
+                        defer rt.gpa.destroy(self);
+                        // send the error to the callback
+                        try self.callback(rt, .{
+                            .userdata = self.userdata,
+                            .msg = self.msg,
+                            .result = .{ .userptr = err },
+                            .callback = self.callback,
+                            .req = .userptr,
+                        });
+                        return;
+                    };
+
+                    self.read_end += n;
+                    const slice = self.buffer[0..self.read_end];
+                    var scratch: [tls.max_ciphertext_record_len]u8 = undefined;
+                    const r = try self.handshake.run(slice, &scratch);
+
+                    if (r.unused_recv.len > 0) {
+                        // Arm a recv task
+                        self.task = try rt.recv(self.fd, self.buffer[self.read_end..], .{
+                            .ptr = self,
+                            .cb = handleMsg,
+                        });
+                        return;
+                    }
+
+                    if (r.send.len > 0) {
+                        // Queue another send
+                        @memcpy(self.buffer[0..r.send.len], r.send);
+                        self.task = try rt.write(
+                            self.fd,
+                            self.buffer[0..r.send.len],
+                            .{ .ptr = self, .cb = HandshakeTask.handleMsg },
+                        );
+                        return;
+                    }
+
+                    if (self.handshake.done()) {
+                        defer rt.gpa.destroy(self);
+                        // Handshake is done. Create a client and deliver it to the callback
+                        const client = try self.initClient(rt.gpa);
+                        try self.callback(rt, .{
+                            .userdata = self.userdata,
+                            .msg = self.msg,
+                            .result = .{ .userptr = client },
+                            .callback = self.callback,
+                            .req = .userptr,
+                        });
+                        return;
+                    }
+                },
+
+                else => unreachable,
+            }
+        }
+
+        fn initClient(self: *HandshakeTask, gpa: Allocator) !*Client {
+            const client = try gpa.create(Client);
+            client.* = .{
+                .gpa = gpa,
+                .fd = self.fd,
+                .tls = .{ .cipher = self.handshake.inner.cipher },
+            };
+            return client;
+        }
+
+        /// Tries to cancel the handshake. Callback will receive an error.Canceled if cancelation
+        /// was successful, otherwise handhsake will proceed
+        pub fn cancel(self: *HandshakeTask, rt: *io.Runtime) void {
+            self.task.cancel(rt, null, 0, io.noopCallback) catch {};
+        }
+    };
+
+    const Msg = enum {
+        write,
+        recv,
+        close_notify,
+    };
+
+    /// Initializes a handshake, which will ultimately deliver a Client to the callback via a
+    /// userptr result
+    pub fn init(
+        rt: *io.Runtime,
+        fd: posix.fd_t,
+        opts: tls.config.Client,
+        ctx: io.Context,
+    ) !*HandshakeTask {
+        const hs = try rt.gpa.create(HandshakeTask);
+        hs.* = .{
+            .userdata = ctx.ptr,
+            .callback = ctx.cb,
+            .msg = ctx.msg,
+
+            .fd = fd,
+            .handshake = .init(opts),
+            .task = undefined,
+        };
+
+        const result = try hs.handshake.run("", &hs.buffer);
+        const hs_ctx: io.Context = .{ .ptr = hs, .cb = HandshakeTask.handleMsg };
+        hs.task = try rt.write(hs.fd, result.send, hs_ctx);
+        return hs;
+    }
+
+    pub fn deinit(self: *Client, gpa: Allocator) void {
+        self.ciphertext_buf.deinit(gpa);
+        self.cleartext_buf.deinit(gpa);
+    }
+
+    pub fn close(self: *Client, gpa: Allocator, rt: *io.Runtime) !void {
+        // close notify is 2 bytes long
+        const len = self.tls.encryptedLength(2);
+        try self.ciphertext_buf.ensureUnusedCapacity(gpa, len);
+        const buf = self.ciphertext_buf.unusedCapacitySlice();
+        const msg = try self.tls.close(buf);
+
+        self.ciphertext_buf.items.len += msg.len;
+        _ = try rt.write(self.fd, self.ciphertext_buf.items[self.written..], .{
+            .ptr = self,
+            .cb = Client.onCompletion,
+            .msg = @intFromEnum(Client.Msg.close_notify),
+        });
+
+        if (self.recv_task) |task| {
+            try task.cancel(rt, .{});
+            self.recv_task = null;
+        }
+    }
+
+    fn onCompletion(rt: *io.Runtime, task: io.Task) anyerror!void {
+        const self = task.userdataCast(Client);
+        const result = task.result.?;
+
+        switch (task.msgToEnum(Client.Msg)) {
+            .recv => {
+                assert(result == .recv);
+                self.recv_task = null;
+                const n = result.recv catch |err| {
+                    return self.callback(rt, .{
+                        .userdata = self.userdata,
+                        .msg = self.recv_msg,
+                        .callback = self.callback,
+                        .req = .{ .recv = .{ .fd = self.fd, .buffer = &self.read_buf } },
+                        .result = .{ .recv = err },
+                    });
+                };
+                self.read_end += n;
+                const end = self.read_end;
+                const r = try self.tls.decrypt(self.read_buf[0..end], self.read_buf[0..end]);
+
+                if (r.cleartext.len > 0) {
+                    try self.callback(rt, .{
+                        .userdata = self.userdata,
+                        .msg = self.recv_msg,
+                        .callback = self.callback,
+                        .req = .{ .recv = .{ .fd = self.fd, .buffer = &self.read_buf } },
+                        .result = .{ .recv = r.cleartext.len },
+                    });
+                }
+                mem.copyForwards(u8, &self.read_buf, r.unused_ciphertext);
+                self.read_end = r.unused_ciphertext.len;
+
+                if (r.closed) {
+                    _ = try rt.close(self.fd, self.closeContext());
+                    return;
+                }
+
+                self.recv_task = try rt.recv(
+                    self.fd,
+                    self.read_buf[self.read_end..],
+                    self.recvContext(),
+                );
+            },
+
+            .write => {
+                assert(result == .write);
+                const n = result.write catch {
+                    return self.callback(rt, .{
+                        .userdata = self.userdata,
+                        .msg = self.write_msg,
+                        .callback = self.callback,
+                        .req = .{ .write = .{ .fd = self.fd, .buffer = self.ciphertext_buf.items } },
+                        .result = .{ .write = error.Unexpected },
+                    });
+                };
+                self.written += n;
+
+                if (self.written < self.ciphertext_buf.items.len) {
+                    _ = try rt.write(
+                        self.fd,
+                        self.ciphertext_buf.items[self.written..],
+                        self.writeContext(),
+                    );
+                } else {
+                    defer {
+                        self.written = 0;
+                        self.ciphertext_buf.clearRetainingCapacity();
+                    }
+                    return self.callback(rt, .{
+                        .userdata = self.userdata,
+                        .msg = self.write_msg,
+                        .callback = self.callback,
+                        .req = .{ .write = .{ .fd = self.fd, .buffer = self.ciphertext_buf.items } },
+                        .result = .{ .write = self.written },
+                    });
+                }
+            },
+
+            .close_notify => {
+                assert(result == .write);
+                const n = result.write catch {
+                    return self.callback(rt, .{
+                        .userdata = self.userdata,
+                        .msg = self.close_msg,
+                        .callback = self.callback,
+                        .req = .{ .close = self.fd },
+                        .result = .{ .close = error.Unexpected },
+                    });
+                };
+
+                self.written += n;
+
+                if (self.written < self.ciphertext_buf.items.len) {
+                    _ = try rt.write(self.fd, self.ciphertext_buf.items[self.written..], .{
+                        .ptr = self,
+                        .cb = Client.onCompletion,
+                        .msg = @intFromEnum(Client.Msg.close_notify),
+                    });
+                } else {
+                    self.written = 0;
+                    self.ciphertext_buf.clearRetainingCapacity();
+                    _ = try rt.close(self.fd, self.closeContext());
+                }
+            },
+        }
+    }
+
+    pub fn recv(self: *Client, rt: *io.Runtime) !void {
+        if (self.recv_task != null) return;
+        self.recv_task = try rt.recv(
+            self.fd,
+            self.read_buf[self.read_end..],
+            self.recvContext(),
+        );
+    }
+
+    pub fn write(self: *Client, gpa: Allocator, bytes: []const u8) Allocator.Error!void {
+        try self.cleartext_buf.appendSlice(gpa, bytes);
+    }
+
+    pub fn flush(self: *Client, gpa: Allocator, rt: *io.Runtime) !void {
+        const len = self.tls.encryptedLength(self.cleartext_buf.items.len);
+        try self.ciphertext_buf.ensureUnusedCapacity(gpa, len);
+        const slice = self.ciphertext_buf.unusedCapacitySlice();
+        const result = try self.tls.encrypt(self.cleartext_buf.items, slice);
+        self.ciphertext_buf.appendSliceAssumeCapacity(result.ciphertext);
+        self.cleartext_buf.replaceRangeAssumeCapacity(0, result.cleartext_pos, "");
+
+        _ = try rt.write(
+            self.fd,
+            self.ciphertext_buf.items.len,
+            self,
+            @intFromEnum(Client.Msg.write),
+            Client.onCompletion,
+        );
+    }
+
+    fn closeContext(self: Client) io.Context {
+        return .{ .ptr = self.userdata, .cb = self.callback, .msg = self.close_msg };
+    }
+
+    fn recvContext(self: *Client) io.Context {
+        return .{
+            .ptr = self,
+            .cb = Client.onCompletion,
+            .msg = @intFromEnum(Client.Msg.recv),
+        };
+    }
+
+    fn writeContext(self: *Client) io.Context {
+        return .{
+            .ptr = self,
+            .cb = Client.onCompletion,
+            .msg = @intFromEnum(Client.Msg.write),
+        };
+    }
+};
+
+test "tls: Client" {
+    const net = @import("net.zig");
+    const gpa = std.testing.allocator;
+
+    var rt = try io.Runtime.init(gpa, 16);
+    defer rt.deinit();
+
+    const Foo = struct {
+        const Self = @This();
+        gpa: Allocator,
+        fd: ?posix.fd_t = null,
+        tls: ?*Client = null,
+
+        const Msg = enum {
+            connect,
+            handshake,
+            close,
+            write,
+            recv,
+        };
+
+        fn callback(_: *io.Runtime, task: io.Task) anyerror!void {
+            const self = task.userdataCast(Self);
+            const result = task.result.?;
+            errdefer {
+                if (self.tls) |client| {
+                    client.deinit(self.gpa);
+                    self.gpa.destroy(client);
+                    self.tls = null;
+                }
+            }
+
+            switch (task.msgToEnum(Msg)) {
+                .connect => {
+                    self.fd = try result.userfd;
+                },
+                .handshake => {
+                    const ptr = try result.userptr;
+                    self.tls = @ptrCast(@alignCast(ptr));
+                    self.tls.?.userdata = self;
+                    self.tls.?.close_msg = @intFromEnum(@This().Msg.close);
+                    self.tls.?.write_msg = @intFromEnum(@This().Msg.write);
+                    self.tls.?.recv_msg = @intFromEnum(@This().Msg.recv);
+                    self.tls.?.callback = @This().callback;
+                },
+                .close => {
+                    self.tls.?.deinit(self.gpa);
+                    self.gpa.destroy(self.tls.?);
+                    self.tls = null;
+                    self.fd = null;
+                },
+
+                else => {},
+            }
+        }
+    };
+
+    var foo: Foo = .{ .gpa = gpa };
+    defer {
+        if (foo.tls) |client| {
+            client.deinit(gpa);
+            gpa.destroy(client);
+        }
+        if (foo.fd) |fd| posix.close(fd);
+    }
+
+    _ = try net.tcpConnectToHost(
+        &rt,
+        "google.com",
+        443,
+        .{ .ptr = &foo, .cb = Foo.callback, .msg = @intFromEnum(Foo.Msg.connect) },
+    );
+
+    try rt.run(.until_done);
+
+    try std.testing.expect(foo.fd != null);
+
+    var bundle: CertBundle = .{};
+    try bundle.rescan(gpa);
+    defer bundle.deinit(gpa);
+
+    _ = try Client.init(
+        &rt,
+        foo.fd.?,
+        .{ .root_ca = bundle, .host = "google.com" },
+        .{ .ptr = &foo, .cb = Foo.callback, .msg = @intFromEnum(Foo.Msg.handshake) },
+    );
+    try rt.run(.until_done);
+    try std.testing.expect(foo.tls != null);
+
+    try foo.tls.?.recv(&rt);
+    try foo.tls.?.close(gpa, &rt);
+    try rt.run(.until_done);
+    try std.testing.expect(foo.tls == null);
+    try std.testing.expect(foo.fd == null);
+}