Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050// SPDX-License-Identifier: GPL-2.0/* * Code related to the io_uring_register() syscall * * Copyright (C) 2023 Jens Axboe */#include <linux/kernel.h>#include <linux/errno.h>#include <linux/syscalls.h>#include <linux/refcount.h>#include <linux/bits.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/slab.h>#include <linux/uaccess.h>#include <linux/nospec.h>#include <linux/compat.h>#include <linux/io_uring.h>#include <linux/io_uring_types.h>
#include "filetable.h"#include "io_uring.h"#include "opdef.h"#include "tctx.h"#include "rsrc.h"#include "sqpoll.h"#include "register.h"#include "cancel.h"#include "kbuf.h"#include "napi.h"#include "eventfd.h"#include "msg_ring.h"#include "memmap.h"#include "zcrx.h"#include "query.h"#include "bpf_filter.h"
#define IORING_MAX_RESTRICTIONS (IORING_RESTRICTION_LAST + \ IORING_REGISTER_LAST + IORING_OP_LAST)
static __cold int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args){ struct io_uring_probe *p; size_t size; int i, ret;
if (nr_args > IORING_OP_LAST) nr_args = IORING_OP_LAST;
size = struct_size(p, ops, nr_args); p = memdup_user(arg, size); if (IS_ERR(p)) return PTR_ERR(p); ret = -EINVAL; if (memchr_inv(p, 0, size)) goto out;
p->last_op = IORING_OP_LAST - 1;
for (i = 0; i < nr_args; i++) { p->ops[i].op = i; if (io_uring_op_supported(i)) p->ops[i].flags = IO_URING_OP_SUPPORTED; } p->ops_len = i;
ret = 0; if (copy_to_user(arg, p, size)) ret = -EFAULT;out: kfree(p); return ret;}
int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id){ const struct cred *creds;
creds = xa_erase(&ctx->personalities, id); if (creds) { put_cred(creds); return 0; }
return -EINVAL;}
static int io_register_personality(struct io_ring_ctx *ctx){ const struct cred *creds; u32 id; int ret;
creds = get_current_cred();
ret = xa_alloc_cyclic(&ctx->personalities, &id, (void *)creds, XA_LIMIT(0, USHRT_MAX), &ctx->pers_next, GFP_KERNEL); if (ret < 0) { put_cred(creds); return ret; } return id;}
/* * Returns number of restrictions parsed and added on success, or < 0 for * an error. */static __cold int io_parse_restrictions(void __user *arg, unsigned int nr_args, struct io_restriction *restrictions){ struct io_uring_restriction *res; size_t size; int i, ret;
if (!arg || nr_args > IORING_MAX_RESTRICTIONS) return -EINVAL;
size = array_size(nr_args, sizeof(*res)); if (size == SIZE_MAX) return -EOVERFLOW;
res = memdup_user(arg, size); if (IS_ERR(res)) return PTR_ERR(res);
ret = -EINVAL;
for (i = 0; i < nr_args; i++) { switch (res[i].opcode) { case IORING_RESTRICTION_REGISTER_OP: if (res[i].register_op >= IORING_REGISTER_LAST) goto err; __set_bit(res[i].register_op, restrictions->register_op); restrictions->reg_registered = true; break; case IORING_RESTRICTION_SQE_OP: if (res[i].sqe_op >= IORING_OP_LAST) goto err; __set_bit(res[i].sqe_op, restrictions->sqe_op); restrictions->op_registered = true; break; case IORING_RESTRICTION_SQE_FLAGS_ALLOWED: restrictions->sqe_flags_allowed = res[i].sqe_flags; restrictions->op_registered = true; break; case IORING_RESTRICTION_SQE_FLAGS_REQUIRED: restrictions->sqe_flags_required = res[i].sqe_flags; restrictions->op_registered = true; break; default: goto err; } } ret = nr_args; if (!nr_args) { restrictions->op_registered = true; restrictions->reg_registered = true; }err: kfree(res); return ret;}
static __cold int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg, unsigned int nr_args){ int ret;
/* Restrictions allowed only if rings started disabled */ if (!(ctx->flags & IORING_SETUP_R_DISABLED)) return -EBADFD;
/* We allow only a single restrictions registration */ if (ctx->restrictions.op_registered || ctx->restrictions.reg_registered) return -EBUSY;
ret = io_parse_restrictions(arg, nr_args, &ctx->restrictions); /* * Reset all restrictions if an error happened, but retain any COW'ed * settings. */ if (ret < 0) { struct io_bpf_filters *bpf = ctx->restrictions.bpf_filters; bool cowed = ctx->restrictions.bpf_filters_cow;
memset(&ctx->restrictions, 0, sizeof(ctx->restrictions)); ctx->restrictions.bpf_filters = bpf; ctx->restrictions.bpf_filters_cow = cowed; return ret; } if (ctx->restrictions.op_registered) ctx->int_flags |= IO_RING_F_OP_RESTRICTED; if (ctx->restrictions.reg_registered) ctx->int_flags |= IO_RING_F_REG_RESTRICTED; return 0;}
static int io_register_restrictions_task(void __user *arg, unsigned int nr_args){ struct io_uring_task_restriction __user *ures = arg; struct io_uring_task_restriction tres; struct io_restriction *res; int ret;
/* Disallow if task already has registered restrictions */ if (current->io_uring_restrict) return -EPERM; /* * Similar to seccomp, disallow setting a filter if task_no_new_privs * is false and we're not CAP_SYS_ADMIN. */ if (!task_no_new_privs(current) && !ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN)) return -EACCES; if (nr_args != 1) return -EINVAL;
if (copy_from_user(&tres, arg, sizeof(tres))) return -EFAULT;
if (tres.flags) return -EINVAL; if (!mem_is_zero(tres.resv, sizeof(tres.resv))) return -EINVAL;
res = kzalloc_obj(*res, GFP_KERNEL_ACCOUNT); if (!res) return -ENOMEM;
ret = io_parse_restrictions(ures->restrictions, tres.nr_res, res); if (ret < 0) { kfree(res); return ret; } current->io_uring_restrict = res; return 0;}
static int io_register_bpf_filter_task(void __user *arg, unsigned int nr_args){ struct io_restriction *res; int ret;
/* * Similar to seccomp, disallow setting a filter if task_no_new_privs * is false and we're not CAP_SYS_ADMIN. */ if (!task_no_new_privs(current) && !ns_capable_noaudit(current_user_ns(), CAP_SYS_ADMIN)) return -EACCES;
if (nr_args != 1) return -EINVAL;
/* If no task restrictions exist, setup a new set */ res = current->io_uring_restrict; if (!res) { res = kzalloc_obj(*res, GFP_KERNEL_ACCOUNT); if (!res) return -ENOMEM; }
ret = io_register_bpf_filter(res, arg); if (ret) { if (res != current->io_uring_restrict) kfree(res); return ret; } if (!current->io_uring_restrict) current->io_uring_restrict = res; return 0;}
static int io_register_enable_rings(struct io_ring_ctx *ctx){ if (!(ctx->flags & IORING_SETUP_R_DISABLED)) return -EBADFD;
if (ctx->flags & IORING_SETUP_SINGLE_ISSUER) { ctx->submitter_task = get_task_struct(current); /* * Lazy activation attempts would fail if it was polled before * submitter_task is set. */ if (wq_has_sleeper(&ctx->poll_wq)) io_activate_pollwq(ctx); }
/* Keep submitter_task store before clearing IORING_SETUP_R_DISABLED */ smp_store_release(&ctx->flags, ctx->flags & ~IORING_SETUP_R_DISABLED); if (ctx->sq_data && wq_has_sleeper(&ctx->sq_data->wait)) wake_up(&ctx->sq_data->wait); return 0;}
static __cold int __io_register_iowq_aff(struct io_ring_ctx *ctx, cpumask_var_t new_mask){ int ret;
if (!(ctx->flags & IORING_SETUP_SQPOLL)) { ret = io_wq_cpu_affinity(current->io_uring, new_mask); } else { mutex_unlock(&ctx->uring_lock); ret = io_sqpoll_wq_cpu_affinity(ctx, new_mask); mutex_lock(&ctx->uring_lock); }
return ret;}
static __cold int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg, unsigned len){ cpumask_var_t new_mask; int ret;
if (!alloc_cpumask_var(&new_mask, GFP_KERNEL)) return -ENOMEM;
cpumask_clear(new_mask); if (len > cpumask_size()) len = cpumask_size();
#ifdef CONFIG_COMPAT if (in_compat_syscall()) ret = compat_get_bitmap(cpumask_bits(new_mask), (const compat_ulong_t __user *)arg, len * 8 /* CHAR_BIT */); else#endif ret = copy_from_user(new_mask, arg, len);
if (ret) { free_cpumask_var(new_mask); return -EFAULT; }
ret = __io_register_iowq_aff(ctx, new_mask); free_cpumask_var(new_mask); return ret;}
static __cold int io_unregister_iowq_aff(struct io_ring_ctx *ctx){ return __io_register_iowq_aff(ctx, NULL);}
static __cold int io_register_iowq_max_workers(struct io_ring_ctx *ctx, void __user *arg) __must_hold(&ctx->uring_lock){ struct io_tctx_node *node; struct io_uring_task *tctx = NULL; struct io_sq_data *sqd = NULL; __u32 new_count[2]; int i, ret;
if (copy_from_user(new_count, arg, sizeof(new_count))) return -EFAULT; for (i = 0; i < ARRAY_SIZE(new_count); i++) if (new_count[i] > INT_MAX) return -EINVAL;
if (ctx->flags & IORING_SETUP_SQPOLL) { sqd = ctx->sq_data; if (sqd) { struct task_struct *tsk;
/* * Observe the correct sqd->lock -> ctx->uring_lock * ordering. Fine to drop uring_lock here, we hold * a ref to the ctx. */ refcount_inc(&sqd->refs); mutex_unlock(&ctx->uring_lock); mutex_lock(&sqd->lock); mutex_lock(&ctx->uring_lock); tsk = sqpoll_task_locked(sqd); if (tsk) tctx = tsk->io_uring; } } else { tctx = current->io_uring; }
BUILD_BUG_ON(sizeof(new_count) != sizeof(ctx->iowq_limits));
for (i = 0; i < ARRAY_SIZE(new_count); i++) if (new_count[i]) ctx->iowq_limits[i] = new_count[i]; ctx->int_flags |= IO_RING_F_IOWQ_LIMITS_SET;
if (tctx && tctx->io_wq) { ret = io_wq_max_workers(tctx->io_wq, new_count); if (ret) goto err; } else { memset(new_count, 0, sizeof(new_count)); }
if (sqd) { mutex_unlock(&ctx->uring_lock); mutex_unlock(&sqd->lock); io_put_sq_data(sqd); mutex_lock(&ctx->uring_lock); }
if (copy_to_user(arg, new_count, sizeof(new_count))) return -EFAULT;
/* that's it for SQPOLL, only the SQPOLL task creates requests */ if (sqd) return 0;
/* now propagate the restriction to all registered users */ mutex_lock(&ctx->tctx_lock); list_for_each_entry(node, &ctx->tctx_list, ctx_node) { tctx = node->task->io_uring; if (WARN_ON_ONCE(!tctx->io_wq)) continue;
for (i = 0; i < ARRAY_SIZE(new_count); i++) new_count[i] = ctx->iowq_limits[i]; /* ignore errors, it always returns zero anyway */ (void)io_wq_max_workers(tctx->io_wq, new_count); } mutex_unlock(&ctx->tctx_lock); return 0;err: if (sqd) { mutex_unlock(&ctx->uring_lock); mutex_unlock(&sqd->lock); io_put_sq_data(sqd); mutex_lock(&ctx->uring_lock); } return ret;}
static int io_register_clock(struct io_ring_ctx *ctx, struct io_uring_clock_register __user *arg){ struct io_uring_clock_register reg;
if (copy_from_user(®, arg, sizeof(reg))) return -EFAULT; if (memchr_inv(®.__resv, 0, sizeof(reg.__resv))) return -EINVAL;
switch (reg.clockid) { case CLOCK_MONOTONIC: ctx->clock_offset = 0; break; case CLOCK_BOOTTIME: ctx->clock_offset = TK_OFFS_BOOT; break; default: return -EINVAL; }
ctx->clockid = reg.clockid; return 0;}
/* * State to maintain until we can swap. Both new and old state, used for * either mapping or freeing. */struct io_ring_ctx_rings { struct io_rings *rings; struct io_uring_sqe *sq_sqes;
struct io_mapped_region sq_region; struct io_mapped_region ring_region;};
static void io_register_free_rings(struct io_ring_ctx *ctx, struct io_ring_ctx_rings *r){ io_free_region(ctx->user, &r->sq_region); io_free_region(ctx->user, &r->ring_region);}
#define swap_old(ctx, o, n, field) \ do { \ (o).field = (ctx)->field; \ (ctx)->field = (n).field; \ } while (0)
#define RESIZE_FLAGS (IORING_SETUP_CQSIZE | IORING_SETUP_CLAMP)#define COPY_FLAGS (IORING_SETUP_NO_SQARRAY | IORING_SETUP_SQE128 | \ IORING_SETUP_CQE32 | IORING_SETUP_NO_MMAP | \ IORING_SETUP_CQE_MIXED | IORING_SETUP_SQE_MIXED)
static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg){ struct io_ctx_config config; struct io_uring_region_desc rd; struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL; unsigned i, tail, old_head; struct io_uring_params *p = &config.p; struct io_rings_layout *rl = &config.layout; u32 *o_sq_array, *n_sq_array = NULL; int ret;
memset(&config, 0, sizeof(config));
/* limited to DEFER_TASKRUN for now */ if (!(ctx->flags & IORING_SETUP_DEFER_TASKRUN)) return -EINVAL; if (copy_from_user(p, arg, sizeof(*p))) return -EFAULT; if (p->flags & ~RESIZE_FLAGS) return -EINVAL;
/* properties that are always inherited */ p->flags |= (ctx->flags & COPY_FLAGS);
ret = io_prepare_config(&config); if (unlikely(ret)) return ret;
memset(&rd, 0, sizeof(rd)); rd.size = PAGE_ALIGN(rl->rings_size); if (p->flags & IORING_SETUP_NO_MMAP) { rd.user_addr = p->cq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } ret = io_create_region(ctx, &n.ring_region, &rd, IORING_OFF_CQ_RING); if (ret) return ret;
n.rings = io_region_get_ptr(&n.ring_region);
/* * At this point n.rings is shared with userspace, just like o.rings * is as well. While we don't expect userspace to modify it while * a resize is in progress, and it's most likely that userspace will * shoot itself in the foot if it does, we can't always assume good * intent... Use read/write once helpers from here on to indicate the * shared nature of it. */ WRITE_ONCE(n.rings->sq_ring_mask, p->sq_entries - 1); WRITE_ONCE(n.rings->cq_ring_mask, p->cq_entries - 1); WRITE_ONCE(n.rings->sq_ring_entries, p->sq_entries); WRITE_ONCE(n.rings->cq_ring_entries, p->cq_entries);
if (copy_to_user(arg, p, sizeof(*p))) { io_register_free_rings(ctx, &n); return -EFAULT; }
memset(&rd, 0, sizeof(rd)); rd.size = PAGE_ALIGN(rl->sq_size); if (p->flags & IORING_SETUP_NO_MMAP) { rd.user_addr = p->sq_off.user_addr; rd.flags |= IORING_MEM_REGION_TYPE_USER; } ret = io_create_region(ctx, &n.sq_region, &rd, IORING_OFF_SQES); if (ret) { io_register_free_rings(ctx, &n); return ret; } n.sq_sqes = io_region_get_ptr(&n.sq_region);
/* * If using SQPOLL, park the thread */ if (ctx->sq_data) { mutex_unlock(&ctx->uring_lock); io_sq_thread_park(ctx->sq_data); mutex_lock(&ctx->uring_lock); }
/* * We'll do the swap. Grab the ctx->mmap_lock, which will exclude * any new mmap's on the ring fd. Clear out existing mappings to prevent * mmap from seeing them, as we'll unmap them. Any attempt to mmap * existing rings beyond this point will fail. Not that it could proceed * at this point anyway, as the io_uring mmap side needs go grab the * ctx->mmap_lock as well. Likewise, hold the completion lock over the * duration of the actual swap. */ mutex_lock(&ctx->mmap_lock); spin_lock(&ctx->completion_lock); o.rings = ctx->rings; ctx->rings = NULL; o.sq_sqes = ctx->sq_sqes; ctx->sq_sqes = NULL; o_sq_array = ctx->sq_array; if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) n_sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
/* * Now copy SQ and CQ entries, if any. If either of the destination * rings can't hold what is already there, then fail the operation. */ tail = READ_ONCE(o.rings->sq.tail); old_head = READ_ONCE(o.rings->sq.head); if (tail - old_head > p->sq_entries) goto overflow; for (i = old_head; i < tail; i++) { unsigned int dst, src; size_t sq_size;
dst = i & (p->sq_entries - 1); src = i & (ctx->sq_entries - 1); if (n_sq_array) { src = READ_ONCE(o_sq_array[src]); if (unlikely(src >= ctx->sq_entries)) { WRITE_ONCE(n_sq_array[dst], UINT_MAX); continue; } WRITE_ONCE(n_sq_array[dst], dst); }
sq_size = sizeof(struct io_uring_sqe); if (ctx->flags & IORING_SETUP_SQE128) { dst <<= 1; src <<= 1; sq_size <<= 1; } memcpy(&n.sq_sqes[dst], &o.sq_sqes[src], sq_size); } WRITE_ONCE(n.rings->sq.head, old_head); WRITE_ONCE(n.rings->sq.tail, tail);
tail = READ_ONCE(o.rings->cq.tail); old_head = READ_ONCE(o.rings->cq.head); if (tail - old_head > p->cq_entries) {overflow: /* restore old rings, and return -EOVERFLOW via cleanup path */ ctx->rings = o.rings; ctx->sq_sqes = o.sq_sqes; to_free = &n; ret = -EOVERFLOW; goto out; } for (i = old_head; i < tail; i++) { unsigned index, dst_mask, src_mask; size_t cq_size;
index = i; cq_size = sizeof(struct io_uring_cqe); src_mask = ctx->cq_entries - 1; dst_mask = p->cq_entries - 1; if (ctx->flags & IORING_SETUP_CQE32) { index <<= 1; cq_size <<= 1; src_mask = (ctx->cq_entries << 1) - 1; dst_mask = (p->cq_entries << 1) - 1; } memcpy(&n.rings->cqes[index & dst_mask], &o.rings->cqes[index & src_mask], cq_size); } WRITE_ONCE(n.rings->cq.head, old_head); WRITE_ONCE(n.rings->cq.tail, tail); /* invalidate cached cqe refill */ ctx->cqe_cached = ctx->cqe_sentinel = NULL;
WRITE_ONCE(n.rings->sq_dropped, READ_ONCE(o.rings->sq_dropped)); atomic_set(&n.rings->sq_flags, atomic_read(&o.rings->sq_flags)); WRITE_ONCE(n.rings->cq_flags, READ_ONCE(o.rings->cq_flags)); WRITE_ONCE(n.rings->cq_overflow, READ_ONCE(o.rings->cq_overflow));
/* all done, store old pointers and assign new ones */ if (n_sq_array) ctx->sq_array = n_sq_array;
ctx->sq_entries = p->sq_entries; ctx->cq_entries = p->cq_entries;
/* * Just mark any flag we may have missed and that the application * should act on unconditionally. Worst case it'll be an extra * syscall. */ atomic_or(IORING_SQ_TASKRUN | IORING_SQ_NEED_WAKEUP, &n.rings->sq_flags); ctx->rings = n.rings; rcu_assign_pointer(ctx->rings_rcu, n.rings);
ctx->sq_sqes = n.sq_sqes; swap_old(ctx, o, n, ring_region); swap_old(ctx, o, n, sq_region); to_free = &o; ret = 0;out: spin_unlock(&ctx->completion_lock); mutex_unlock(&ctx->mmap_lock); /* Wait for concurrent io_ctx_mark_taskrun() */ if (to_free == &o) synchronize_rcu_expedited(); io_register_free_rings(ctx, to_free);
if (ctx->sq_data) io_sq_thread_unpark(ctx->sq_data);
return ret;}
static int io_register_mem_region(struct io_ring_ctx *ctx, void __user *uarg){ struct io_uring_mem_region_reg __user *reg_uptr = uarg; struct io_uring_mem_region_reg reg; struct io_uring_region_desc __user *rd_uptr; struct io_uring_region_desc rd; struct io_mapped_region region = {}; int ret;
if (io_region_is_set(&ctx->param_region)) return -EBUSY; if (copy_from_user(®, reg_uptr, sizeof(reg))) return -EFAULT; rd_uptr = u64_to_user_ptr(reg.region_uptr); if (copy_from_user(&rd, rd_uptr, sizeof(rd))) return -EFAULT; if (memchr_inv(®.__resv, 0, sizeof(reg.__resv))) return -EINVAL; if (reg.flags & ~IORING_MEM_REGION_REG_WAIT_ARG) return -EINVAL;
/* * This ensures there are no waiters. Waiters are unlocked and it's * hard to synchronise with them, especially if we need to initialise * the region. */ if ((reg.flags & IORING_MEM_REGION_REG_WAIT_ARG) && !(ctx->flags & IORING_SETUP_R_DISABLED)) return -EINVAL;
ret = io_create_region(ctx, ®ion, &rd, IORING_MAP_OFF_PARAM_REGION); if (ret) return ret; if (copy_to_user(rd_uptr, &rd, sizeof(rd))) { io_free_region(ctx->user, ®ion); return -EFAULT; }
if (reg.flags & IORING_MEM_REGION_REG_WAIT_ARG) { ctx->cq_wait_arg = io_region_get_ptr(®ion); ctx->cq_wait_size = rd.size; }
io_region_publish(ctx, ®ion, &ctx->param_region); return 0;}
static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode, void __user *arg, unsigned nr_args) __releases(ctx->uring_lock) __acquires(ctx->uring_lock){ int ret;
/* * We don't quiesce the refs for register anymore and so it can't be * dying as we're holding a file ref here. */ if (WARN_ON_ONCE(percpu_ref_is_dying(&ctx->refs))) return -ENXIO;
if (ctx->submitter_task && ctx->submitter_task != current) return -EEXIST;
if ((ctx->int_flags & IO_RING_F_REG_RESTRICTED) && !(ctx->flags & IORING_SETUP_R_DISABLED)) { opcode = array_index_nospec(opcode, IORING_REGISTER_LAST); if (!test_bit(opcode, ctx->restrictions.register_op)) return -EACCES; }
switch (opcode) { case IORING_REGISTER_BUFFERS: ret = -EFAULT; if (!arg) break; ret = io_sqe_buffers_register(ctx, arg, nr_args, NULL); break; case IORING_UNREGISTER_BUFFERS: ret = -EINVAL; if (arg || nr_args) break; ret = io_sqe_buffers_unregister(ctx); break; case IORING_REGISTER_FILES: ret = -EFAULT; if (!arg) break; ret = io_sqe_files_register(ctx, arg, nr_args, NULL); break; case IORING_UNREGISTER_FILES: ret = -EINVAL; if (arg || nr_args) break; ret = io_sqe_files_unregister(ctx); break; case IORING_REGISTER_FILES_UPDATE: ret = io_register_files_update(ctx, arg, nr_args); break; case IORING_REGISTER_EVENTFD: ret = -EINVAL; if (nr_args != 1) break; ret = io_eventfd_register(ctx, arg, 0); break; case IORING_REGISTER_EVENTFD_ASYNC: ret = -EINVAL; if (nr_args != 1) break; ret = io_eventfd_register(ctx, arg, 1); break; case IORING_UNREGISTER_EVENTFD: ret = -EINVAL; if (arg || nr_args) break; ret = io_eventfd_unregister(ctx); break; case IORING_REGISTER_PROBE: ret = -EINVAL; if (!arg || nr_args > 256) break; ret = io_probe(ctx, arg, nr_args); break; case IORING_REGISTER_PERSONALITY: ret = -EINVAL; if (arg || nr_args) break; ret = io_register_personality(ctx); break; case IORING_UNREGISTER_PERSONALITY: ret = -EINVAL; if (arg) break; ret = io_unregister_personality(ctx, nr_args); break; case IORING_REGISTER_ENABLE_RINGS: ret = -EINVAL; if (arg || nr_args) break; ret = io_register_enable_rings(ctx); break; case IORING_REGISTER_RESTRICTIONS: ret = io_register_restrictions(ctx, arg, nr_args); break; case IORING_REGISTER_FILES2: ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_FILE); break; case IORING_REGISTER_FILES_UPDATE2: ret = io_register_rsrc_update(ctx, arg, nr_args, IORING_RSRC_FILE); break; case IORING_REGISTER_BUFFERS2: ret = io_register_rsrc(ctx, arg, nr_args, IORING_RSRC_BUFFER); break; case IORING_REGISTER_BUFFERS_UPDATE: ret = io_register_rsrc_update(ctx, arg, nr_args, IORING_RSRC_BUFFER); break; case IORING_REGISTER_IOWQ_AFF: ret = -EINVAL; if (!arg || !nr_args) break; ret = io_register_iowq_aff(ctx, arg, nr_args); break; case IORING_UNREGISTER_IOWQ_AFF: ret = -EINVAL; if (arg || nr_args) break; ret = io_unregister_iowq_aff(ctx); break; case IORING_REGISTER_IOWQ_MAX_WORKERS: ret = -EINVAL; if (!arg || nr_args != 2) break; ret = io_register_iowq_max_workers(ctx, arg); break; case IORING_REGISTER_RING_FDS: ret = io_ringfd_register(ctx, arg, nr_args); break; case IORING_UNREGISTER_RING_FDS: ret = io_ringfd_unregister(ctx, arg, nr_args); break; case IORING_REGISTER_PBUF_RING: ret = -EINVAL; if (!arg || nr_args != 1) break; ret = io_register_pbuf_ring(ctx, arg); break; case IORING_UNREGISTER_PBUF_RING: ret = -EINVAL; if (!arg || nr_args != 1) break; ret = io_unregister_pbuf_ring(ctx, arg); break; case IORING_REGISTER_SYNC_CANCEL: ret = -EINVAL; if (!arg || nr_args != 1) break; ret = io_sync_cancel(ctx, arg); break; case IORING_REGISTER_FILE_ALLOC_RANGE: ret = -EINVAL; if (!arg || nr_args) break; ret = io_register_file_alloc_range(ctx, arg); break; case IORING_REGISTER_PBUF_STATUS: ret = -EINVAL; if (!arg || nr_args != 1) break; ret = io_register_pbuf_status(ctx, arg); break; case IORING_REGISTER_NAPI: ret = -EINVAL; if (!arg || nr_args != 1) break; ret = io_register_napi(ctx, arg); break; case IORING_UNREGISTER_NAPI: ret = -EINVAL; if (nr_args != 1) break; ret = io_unregister_napi(ctx, arg); break; case IORING_REGISTER_CLOCK: ret = -EINVAL; if (!arg || nr_args) break; ret = io_register_clock(ctx, arg); break; case IORING_REGISTER_CLONE_BUFFERS: ret = -EINVAL; if (!arg || nr_args != 1) break; ret = io_register_clone_buffers(ctx, arg); break; case IORING_REGISTER_ZCRX_IFQ: ret = -EINVAL; if (!arg || nr_args != 1) break; ret = io_register_zcrx(ctx, arg); break; case IORING_REGISTER_RESIZE_RINGS: ret = -EINVAL; if (!arg || nr_args != 1) break; ret = io_register_resize_rings(ctx, arg); break; case IORING_REGISTER_MEM_REGION: ret = -EINVAL; if (!arg || nr_args != 1) break; ret = io_register_mem_region(ctx, arg); break; case IORING_REGISTER_QUERY: ret = io_query(arg, nr_args); break; case IORING_REGISTER_ZCRX_CTRL: ret = io_zcrx_ctrl(ctx, arg, nr_args); break; case IORING_REGISTER_BPF_FILTER: ret = -EINVAL;
if (nr_args != 1) break; ret = io_register_bpf_filter(&ctx->restrictions, arg); if (!ret) WRITE_ONCE(ctx->bpf_filters, ctx->restrictions.bpf_filters->filters); break; default: ret = -EINVAL; break; }
return ret;}
static int io_uring_register_send_msg_ring(void __user *arg, unsigned int nr_args){ struct io_uring_sqe sqe;
if (!arg || nr_args != 1) return -EINVAL; if (copy_from_user(&sqe, arg, sizeof(sqe))) return -EFAULT; /* no flags supported */ if (sqe.flags) return -EINVAL; if (sqe.opcode != IORING_OP_MSG_RING) return -EINVAL;
return io_uring_sync_msg_ring(&sqe);}
/* * "blind" registration opcodes are ones where there's no ring given, and * hence the source fd must be -1. */static int io_uring_register_blind(unsigned int opcode, void __user *arg, unsigned int nr_args){ switch (opcode) { case IORING_REGISTER_SEND_MSG_RING: return io_uring_register_send_msg_ring(arg, nr_args); case IORING_REGISTER_QUERY: return io_query(arg, nr_args); case IORING_REGISTER_RESTRICTIONS: return io_register_restrictions_task(arg, nr_args); case IORING_REGISTER_BPF_FILTER: return io_register_bpf_filter_task(arg, nr_args); } return -EINVAL;}
SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, void __user *, arg, unsigned int, nr_args){ struct io_ring_ctx *ctx; long ret = -EBADF; struct file *file; bool use_registered_ring;
use_registered_ring = !!(opcode & IORING_REGISTER_USE_REGISTERED_RING); opcode &= ~IORING_REGISTER_USE_REGISTERED_RING;
if (opcode >= IORING_REGISTER_LAST) return -EINVAL;
if (fd == -1) return io_uring_register_blind(opcode, arg, nr_args);
file = io_uring_ctx_get_file(fd, use_registered_ring); if (IS_ERR(file)) return PTR_ERR(file); ctx = file->private_data;
mutex_lock(&ctx->uring_lock); ret = __io_uring_register(ctx, opcode, arg, nr_args);
trace_io_uring_register(ctx, opcode, ctx->file_table.data.nr, ctx->buf_table.nr, ret); mutex_unlock(&ctx->uring_lock);
if (!use_registered_ring) fput(file); return ret;}