Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423/* SPDX-License-Identifier: GPL-2.0-or-later *//* * Berkeley style UIO structures - Alan Cox 1994. */#ifndef __LINUX_UIO_H#define __LINUX_UIO_H
#include <linux/kernel.h>#include <linux/mm_types.h>#include <linux/ucopysize.h>#include <uapi/linux/uio.h>
struct page;struct folio_queue;
typedef unsigned int __bitwise iov_iter_extraction_t;
struct kvec { void *iov_base; /* and that should *never* hold a userland pointer */ size_t iov_len;};
enum iter_type { /* iter types */ ITER_UBUF, ITER_IOVEC, ITER_BVEC, ITER_KVEC, ITER_FOLIOQ, ITER_XARRAY, ITER_DISCARD,};
#define ITER_SOURCE 1 // == WRITE#define ITER_DEST 0 // == READ
struct iov_iter_state { size_t iov_offset; size_t count; unsigned long nr_segs;};
struct iov_iter { u8 iter_type; bool nofault; bool data_source; size_t iov_offset; /* * Hack alert: overlay ubuf_iovec with iovec + count, so * that the members resolve correctly regardless of the type * of iterator used. This means that you can use: * * &iter->__ubuf_iovec or iter->__iov * * interchangably for the user_backed cases, hence simplifying * some of the cases that need to deal with both. */ union { /* * This really should be a const, but we cannot do that without * also modifying any of the zero-filling iter init functions. * Leave it non-const for now, but it should be treated as such. */ struct iovec __ubuf_iovec; struct { union { /* use iter_iov() to get the current vec */ const struct iovec *__iov; const struct kvec *kvec; const struct bio_vec *bvec; const struct folio_queue *folioq; struct xarray *xarray; void __user *ubuf; }; size_t count; }; }; union { unsigned long nr_segs; u8 folioq_slot; loff_t xarray_start; };};
typedef __u16 uio_meta_flags_t;
struct uio_meta { uio_meta_flags_t flags; u16 app_tag; u64 seed; struct iov_iter iter;};
static inline const struct iovec *iter_iov(const struct iov_iter *iter){ if (iter->iter_type == ITER_UBUF) return (const struct iovec *) &iter->__ubuf_iovec; return iter->__iov;}
#define iter_iov_addr(iter) (iter_iov(iter)->iov_base + (iter)->iov_offset)
static inline size_t iter_iov_len(const struct iov_iter *i){ if (i->iter_type == ITER_UBUF) return i->count; return iter_iov(i)->iov_len - i->iov_offset;}
static inline enum iter_type iov_iter_type(const struct iov_iter *i){ return i->iter_type;}
static inline void iov_iter_save_state(struct iov_iter *iter, struct iov_iter_state *state){ state->iov_offset = iter->iov_offset; state->count = iter->count; state->nr_segs = iter->nr_segs;}
static inline bool iter_is_ubuf(const struct iov_iter *i){ return iov_iter_type(i) == ITER_UBUF;}
static inline bool iter_is_iovec(const struct iov_iter *i){ return iov_iter_type(i) == ITER_IOVEC;}
static inline bool iov_iter_is_kvec(const struct iov_iter *i){ return iov_iter_type(i) == ITER_KVEC;}
static inline bool iov_iter_is_bvec(const struct iov_iter *i){ return iov_iter_type(i) == ITER_BVEC;}
static inline bool iov_iter_is_discard(const struct iov_iter *i){ return iov_iter_type(i) == ITER_DISCARD;}
static inline bool iov_iter_is_folioq(const struct iov_iter *i){ return iov_iter_type(i) == ITER_FOLIOQ;}
static inline bool iov_iter_is_xarray(const struct iov_iter *i){ return iov_iter_type(i) == ITER_XARRAY;}
static inline unsigned char iov_iter_rw(const struct iov_iter *i){ return i->data_source ? WRITE : READ;}
static inline bool user_backed_iter(const struct iov_iter *i){ return iter_is_ubuf(i) || iter_is_iovec(i);}
/* * Total number of bytes covered by an iovec. * * NOTE that it is not safe to use this function until all the iovec's * segment lengths have been validated. Because the individual lengths can * overflow a size_t when added together. */static inline size_t iov_length(const struct iovec *iov, unsigned long nr_segs){ unsigned long seg; size_t ret = 0;
for (seg = 0; seg < nr_segs; seg++) ret += iov[seg].iov_len; return ret;}
void iov_iter_advance(struct iov_iter *i, size_t bytes);void iov_iter_revert(struct iov_iter *i, size_t bytes);size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t bytes);size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t bytes);size_t iov_iter_single_seg_count(const struct iov_iter *i);size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes, struct iov_iter *i);size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes, struct iov_iter *i);size_t copy_folio_from_iter_atomic(struct folio *folio, size_t offset, size_t bytes, struct iov_iter *i);
size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i);size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i);size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i);
static inline size_t copy_folio_to_iter(struct folio *folio, size_t offset, size_t bytes, struct iov_iter *i){ return copy_page_to_iter(&folio->page, offset, bytes, i);}
static inline size_t copy_folio_from_iter(struct folio *folio, size_t offset, size_t bytes, struct iov_iter *i){ return copy_page_from_iter(&folio->page, offset, bytes, i);}
size_t copy_page_to_iter_nofault(struct page *page, unsigned offset, size_t bytes, struct iov_iter *i);
static __always_inline __must_checksize_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i){ if (check_copy_size(addr, bytes, true)) return _copy_to_iter(addr, bytes, i); return 0;}
static __always_inline __must_checksize_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i){ if (check_copy_size(addr, bytes, false)) return _copy_from_iter(addr, bytes, i); return 0;}
static __always_inline __must_checkbool copy_to_iter_full(const void *addr, size_t bytes, struct iov_iter *i){ size_t copied = copy_to_iter(addr, bytes, i); if (likely(copied == bytes)) return true; iov_iter_revert(i, copied); return false;}
static __always_inline __must_checkbool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i){ size_t copied = copy_from_iter(addr, bytes, i); if (likely(copied == bytes)) return true; iov_iter_revert(i, copied); return false;}
static __always_inline __must_checksize_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i){ if (check_copy_size(addr, bytes, false)) return _copy_from_iter_nocache(addr, bytes, i); return 0;}
static __always_inline __must_checkbool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i){ size_t copied = copy_from_iter_nocache(addr, bytes, i); if (likely(copied == bytes)) return true; iov_iter_revert(i, copied); return false;}
#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE/* * Note, users like pmem that depend on the stricter semantics of * _copy_from_iter_flushcache() than _copy_from_iter_nocache() must check for * IS_ENABLED(CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE) before assuming that the * destination is flushed from the cache on return. */size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i);#else#define _copy_from_iter_flushcache _copy_from_iter_nocache#endif
#ifdef CONFIG_ARCH_HAS_COPY_MCsize_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i);#else#define _copy_mc_to_iter _copy_to_iter#endif
size_t iov_iter_zero(size_t bytes, struct iov_iter *);unsigned long iov_iter_alignment(const struct iov_iter *i);unsigned long iov_iter_gap_alignment(const struct iov_iter *i);void iov_iter_init(struct iov_iter *i, unsigned int direction, const struct iovec *iov, unsigned long nr_segs, size_t count);void iov_iter_kvec(struct iov_iter *i, unsigned int direction, const struct kvec *kvec, unsigned long nr_segs, size_t count);void iov_iter_bvec(struct iov_iter *i, unsigned int direction, const struct bio_vec *bvec, unsigned long nr_segs, size_t count);void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count);void iov_iter_folio_queue(struct iov_iter *i, unsigned int direction, const struct folio_queue *folioq, unsigned int first_slot, unsigned int offset, size_t count);void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray, loff_t start, size_t count);ssize_t iov_iter_get_pages2(struct iov_iter *i, struct page **pages, size_t maxsize, unsigned maxpages, size_t *start);ssize_t iov_iter_get_pages_alloc2(struct iov_iter *i, struct page ***pages, size_t maxsize, size_t *start);int iov_iter_npages(const struct iov_iter *i, int maxpages);void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state);
const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags);
static inline size_t iov_iter_count(const struct iov_iter *i){ return i->count;}
/* * Cap the iov_iter by given limit; note that the second argument is * *not* the new size - it's upper limit for such. Passing it a value * greater than the amount of data in iov_iter is fine - it'll just do * nothing in that case. */static inline void iov_iter_truncate(struct iov_iter *i, u64 count){ /* * count doesn't have to fit in size_t - comparison extends both * operands to u64 here and any value that would be truncated by * conversion in assignement is by definition greater than all * values of size_t, including old i->count. */ if (i->count > count) i->count = count;}
/* * reexpand a previously truncated iterator; count must be no more than how much * we had shrunk it. */static inline void iov_iter_reexpand(struct iov_iter *i, size_t count){ i->count = count;}
static inline intiov_iter_npages_cap(struct iov_iter *i, int maxpages, size_t max_bytes){ size_t shorted = 0; int npages;
if (iov_iter_count(i) > max_bytes) { shorted = iov_iter_count(i) - max_bytes; iov_iter_truncate(i, max_bytes); } npages = iov_iter_npages(i, maxpages); if (shorted) iov_iter_reexpand(i, iov_iter_count(i) + shorted);
return npages;}
struct iovec *iovec_from_user(const struct iovec __user *uvector, unsigned long nr_segs, unsigned long fast_segs, struct iovec *fast_iov, bool compat);ssize_t import_iovec(int type, const struct iovec __user *uvec, unsigned nr_segs, unsigned fast_segs, struct iovec **iovp, struct iov_iter *i);ssize_t __import_iovec(int type, const struct iovec __user *uvec, unsigned nr_segs, unsigned fast_segs, struct iovec **iovp, struct iov_iter *i, bool compat);int import_ubuf(int type, void __user *buf, size_t len, struct iov_iter *i);
static inline void iov_iter_ubuf(struct iov_iter *i, unsigned int direction, void __user *buf, size_t count){ WARN_ON(direction & ~(READ | WRITE)); *i = (struct iov_iter) { .iter_type = ITER_UBUF, .data_source = direction, .ubuf = buf, .count = count, .nr_segs = 1 };}/* Flags for iov_iter_get/extract_pages*() *//* Allow P2PDMA on the extracted pages */#define ITER_ALLOW_P2PDMA ((__force iov_iter_extraction_t)0x01)
ssize_t iov_iter_extract_pages(struct iov_iter *i, struct page ***pages, size_t maxsize, unsigned int maxpages, iov_iter_extraction_t extraction_flags, size_t *offset0);ssize_t iov_iter_extract_bvecs(struct iov_iter *iter, struct bio_vec *bv, size_t max_size, unsigned short *nr_vecs, unsigned short max_vecs, iov_iter_extraction_t extraction_flags);
/** * iov_iter_extract_will_pin - Indicate how pages from the iterator will be retained * @iter: The iterator * * Examine the iterator and indicate by returning true or false as to how, if * at all, pages extracted from the iterator will be retained by the extraction * function. * * %true indicates that the pages will have a pin placed in them that the * caller must unpin. This is must be done for DMA/async DIO to force fork() * to forcibly copy a page for the child (the parent must retain the original * page). * * %false indicates that no measures are taken and that it's up to the caller * to retain the pages. */static inline bool iov_iter_extract_will_pin(const struct iov_iter *iter){ return user_backed_iter(iter);}
struct sg_table;ssize_t extract_iter_to_sg(struct iov_iter *iter, size_t len, struct sg_table *sgtable, unsigned int sg_max, iov_iter_extraction_t extraction_flags);
#endif