Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449// SPDX-License-Identifier: GPL-2.0#include <linux/kernel.h>#include <linux/errno.h>#include <linux/fs.h>#include <linux/file.h>#include <linux/fdtable.h>#include <linux/fsnotify.h>#include <linux/namei.h>#include <linux/pipe_fs_i.h>#include <linux/watch_queue.h>#include <linux/io_uring.h>
#include <uapi/linux/io_uring.h>
#include "../fs/internal.h"
#include "filetable.h"#include "io_uring.h"#include "rsrc.h"#include "openclose.h"
struct io_open { struct file *file; int dfd; u32 file_slot; struct delayed_filename filename; struct open_how how; unsigned long nofile;};
struct io_close { struct file *file; int fd; u32 file_slot;};
struct io_fixed_install { struct file *file; unsigned int o_flags;};
static bool io_openat_force_async(struct io_open *open){ /* * Don't bother trying for O_TRUNC, O_CREAT, or O_TMPFILE open, * it'll always -EAGAIN. Note that we test for __O_TMPFILE because * O_TMPFILE includes O_DIRECTORY, which isn't a flag we need to force * async for. */ return open->how.flags & (O_TRUNC | O_CREAT | __O_TMPFILE);}
static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe){ struct io_open *open = io_kiocb_to_cmd(req, struct io_open); const char __user *fname; int ret;
if (unlikely(sqe->buf_index)) return -EINVAL; if (unlikely(req->flags & REQ_F_FIXED_FILE)) return -EBADF;
/* open.how should be already initialised */ if (!(open->how.flags & O_PATH) && force_o_largefile()) open->how.flags |= O_LARGEFILE;
open->dfd = READ_ONCE(sqe->fd); fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); ret = delayed_getname(&open->filename, fname); if (unlikely(ret)) return ret; req->flags |= REQ_F_NEED_CLEANUP;
open->file_slot = READ_ONCE(sqe->file_index); if (open->file_slot && (open->how.flags & O_CLOEXEC)) return -EINVAL;
open->nofile = rlimit(RLIMIT_NOFILE); if (io_openat_force_async(open)) req->flags |= REQ_F_FORCE_ASYNC; return 0;}
void io_openat_bpf_populate(struct io_uring_bpf_ctx *bctx, struct io_kiocb *req){ struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
bctx->open.flags = open->how.flags; bctx->open.mode = open->how.mode; bctx->open.resolve = open->how.resolve;}
int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe){ struct io_open *open = io_kiocb_to_cmd(req, struct io_open); u64 mode = READ_ONCE(sqe->len); u64 flags = READ_ONCE(sqe->open_flags);
open->how = build_open_how(flags, mode); return __io_openat_prep(req, sqe);}
int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe){ struct io_open *open = io_kiocb_to_cmd(req, struct io_open); struct open_how __user *how; size_t len; int ret;
how = u64_to_user_ptr(READ_ONCE(sqe->addr2)); len = READ_ONCE(sqe->len); if (len < OPEN_HOW_SIZE_VER0) return -EINVAL;
ret = copy_struct_from_user(&open->how, sizeof(open->how), how, len); if (ret) return ret;
return __io_openat_prep(req, sqe);}
int io_openat2(struct io_kiocb *req, unsigned int issue_flags){ struct io_open *open = io_kiocb_to_cmd(req, struct io_open); struct open_flags op; struct file *file; bool resolve_nonblock, nonblock_set; bool fixed = !!open->file_slot; CLASS(filename_complete_delayed, name)(&open->filename); int ret;
ret = build_open_flags(&open->how, &op); if (ret) goto err; nonblock_set = op.open_flag & O_NONBLOCK; resolve_nonblock = open->how.resolve & RESOLVE_CACHED; if (issue_flags & IO_URING_F_NONBLOCK) { WARN_ON_ONCE(io_openat_force_async(open)); op.lookup_flags |= LOOKUP_CACHED; op.open_flag |= O_NONBLOCK; }
if (!fixed) { ret = __get_unused_fd_flags(open->how.flags, open->nofile); if (ret < 0) goto err; }
file = do_file_open(open->dfd, name, &op); if (IS_ERR(file)) { /* * We could hang on to this 'fd' on retrying, but seems like * marginal gain for something that is now known to be a slower * path. So just put it, and we'll get a new one when we retry. */ if (!fixed) put_unused_fd(ret);
ret = PTR_ERR(file); /* only retry if RESOLVE_CACHED wasn't already set by application */ if (ret == -EAGAIN && !resolve_nonblock && (issue_flags & IO_URING_F_NONBLOCK)) { ret = putname_to_delayed(&open->filename, no_free_ptr(name)); if (likely(!ret)) return -EAGAIN; } goto err; }
if ((issue_flags & IO_URING_F_NONBLOCK) && !nonblock_set) file->f_flags &= ~O_NONBLOCK;
if (!fixed) fd_install(ret, file); else ret = io_fixed_fd_install(req, issue_flags, file, open->file_slot);err: req->flags &= ~REQ_F_NEED_CLEANUP; if (ret < 0) req_set_fail(req); io_req_set_res(req, ret, 0); return IOU_COMPLETE;}
int io_openat(struct io_kiocb *req, unsigned int issue_flags){ return io_openat2(req, issue_flags);}
void io_open_cleanup(struct io_kiocb *req){ struct io_open *open = io_kiocb_to_cmd(req, struct io_open);
dismiss_delayed_filename(&open->filename);}
int __io_close_fixed(struct io_ring_ctx *ctx, unsigned int issue_flags, unsigned int offset){ int ret;
io_ring_submit_lock(ctx, issue_flags); ret = io_fixed_fd_remove(ctx, offset); io_ring_submit_unlock(ctx, issue_flags);
return ret;}
static inline int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags){ struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
return __io_close_fixed(req->ctx, issue_flags, close->file_slot - 1);}
int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe){ struct io_close *close = io_kiocb_to_cmd(req, struct io_close);
if (sqe->off || sqe->addr || sqe->len || sqe->rw_flags || sqe->buf_index) return -EINVAL; if (req->flags & REQ_F_FIXED_FILE) return -EBADF;
close->fd = READ_ONCE(sqe->fd); close->file_slot = READ_ONCE(sqe->file_index); if (close->file_slot && close->fd) return -EINVAL;
return 0;}
int io_close(struct io_kiocb *req, unsigned int issue_flags){ struct files_struct *files = current->files; struct io_close *close = io_kiocb_to_cmd(req, struct io_close); struct file *file; int ret = -EBADF;
if (close->file_slot) { ret = io_close_fixed(req, issue_flags); goto err; }
spin_lock(&files->file_lock); file = files_lookup_fd_locked(files, close->fd); if (!file || io_is_uring_fops(file)) { spin_unlock(&files->file_lock); goto err; }
/* if the file has a flush method, be safe and punt to async */ if (file->f_op->flush && (issue_flags & IO_URING_F_NONBLOCK)) { spin_unlock(&files->file_lock); return -EAGAIN; }
file = file_close_fd_locked(files, close->fd); spin_unlock(&files->file_lock); if (!file) goto err;
/* No ->flush() or already async, safely close from here */ ret = filp_close(file, current->files);err: if (ret < 0) req_set_fail(req); io_req_set_res(req, ret, 0); return IOU_COMPLETE;}
int io_install_fixed_fd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe){ struct io_fixed_install *ifi; unsigned int flags;
if (sqe->off || sqe->addr || sqe->len || sqe->buf_index || sqe->splice_fd_in || sqe->addr3) return -EINVAL;
/* must be a fixed file */ if (!(req->flags & REQ_F_FIXED_FILE)) return -EBADF;
flags = READ_ONCE(sqe->install_fd_flags); if (flags & ~IORING_FIXED_FD_NO_CLOEXEC) return -EINVAL;
/* ensure the task's creds are used when installing/receiving fds */ if (req->flags & REQ_F_CREDS) return -EPERM;
/* default to O_CLOEXEC, disable if IORING_FIXED_FD_NO_CLOEXEC is set */ ifi = io_kiocb_to_cmd(req, struct io_fixed_install); ifi->o_flags = O_CLOEXEC; if (flags & IORING_FIXED_FD_NO_CLOEXEC) ifi->o_flags = 0;
return 0;}
int io_install_fixed_fd(struct io_kiocb *req, unsigned int issue_flags){ struct io_fixed_install *ifi; int ret;
ifi = io_kiocb_to_cmd(req, struct io_fixed_install); ret = receive_fd(req->file, NULL, ifi->o_flags); if (ret < 0) req_set_fail(req); io_req_set_res(req, ret, 0); return IOU_COMPLETE;}
struct io_pipe { struct file *file; int __user *fds; int flags; int file_slot; unsigned long nofile;};
int io_pipe_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe){ struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe);
if (sqe->fd || sqe->off || sqe->addr3) return -EINVAL;
p->fds = u64_to_user_ptr(READ_ONCE(sqe->addr)); p->flags = READ_ONCE(sqe->pipe_flags); if (p->flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE)) return -EINVAL;
p->file_slot = READ_ONCE(sqe->file_index); p->nofile = rlimit(RLIMIT_NOFILE); return 0;}
static int io_pipe_fixed(struct io_kiocb *req, struct file **files, unsigned int issue_flags){ struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); struct io_ring_ctx *ctx = req->ctx; bool alloc_slot; int ret, fds[2] = { -1, -1 }; int slot = p->file_slot;
if (p->flags & O_CLOEXEC) return -EINVAL;
alloc_slot = slot == IORING_FILE_INDEX_ALLOC;
io_ring_submit_lock(ctx, issue_flags);
ret = __io_fixed_fd_install(ctx, files[0], slot); if (ret < 0) goto err; fds[0] = alloc_slot ? ret : slot - 1; files[0] = NULL;
/* * If a specific slot is given, next one will be used for * the write side. */ if (!alloc_slot) slot++;
ret = __io_fixed_fd_install(ctx, files[1], slot); if (ret < 0) goto err; fds[1] = alloc_slot ? ret : slot - 1; files[1] = NULL;
io_ring_submit_unlock(ctx, issue_flags);
if (!copy_to_user(p->fds, fds, sizeof(fds))) return 0;
ret = -EFAULT; io_ring_submit_lock(ctx, issue_flags);err: if (fds[0] != -1) io_fixed_fd_remove(ctx, fds[0]); if (fds[1] != -1) io_fixed_fd_remove(ctx, fds[1]); io_ring_submit_unlock(ctx, issue_flags); return ret;}
static int io_pipe_fd(struct io_kiocb *req, struct file **files){ struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); int ret, fds[2] = { -1, -1 };
ret = __get_unused_fd_flags(p->flags, p->nofile); if (ret < 0) goto err; fds[0] = ret;
ret = __get_unused_fd_flags(p->flags, p->nofile); if (ret < 0) goto err; fds[1] = ret;
if (!copy_to_user(p->fds, fds, sizeof(fds))) { fd_install(fds[0], files[0]); fd_install(fds[1], files[1]); return 0; } ret = -EFAULT;err: if (fds[0] != -1) put_unused_fd(fds[0]); if (fds[1] != -1) put_unused_fd(fds[1]); return ret;}
int io_pipe(struct io_kiocb *req, unsigned int issue_flags){ struct io_pipe *p = io_kiocb_to_cmd(req, struct io_pipe); struct file *files[2]; int ret;
ret = create_pipe_files(files, p->flags); if (ret) return ret;
if (!!p->file_slot) ret = io_pipe_fixed(req, files, issue_flags); else ret = io_pipe_fd(req, files);
io_req_set_res(req, ret, 0); if (!ret) return IOU_COMPLETE;
req_set_fail(req); if (files[0]) fput(files[0]); if (files[1]) fput(files[1]); return ret;}