Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309// SPDX-License-Identifier: GPL-2.0-or-later/* Iterator helpers. * * Copyright (C) 2022 Red Hat, Inc. All Rights Reserved. * Written by David Howells (dhowells@redhat.com) */
#include <linux/export.h>#include <linux/slab.h>#include <linux/mm.h>#include <linux/uio.h>#include <linux/scatterlist.h>#include <linux/netfs.h>#include "internal.h"
/** * netfs_extract_user_iter - Extract the pages from a user iterator into a bvec * @orig: The original iterator * @orig_len: The amount of iterator to copy * @new: The iterator to be set up * @extraction_flags: Flags to qualify the request * * Extract the page fragments from the given amount of the source iterator and * build up a second iterator that refers to all of those bits. This allows * the original iterator to be disposed of. * * @extraction_flags can have ITER_ALLOW_P2PDMA set to request peer-to-peer DMA be * allowed on the pages extracted. * * On success, the number of elements in the bvec is returned, the original * iterator will have been advanced by the amount extracted. * * The iov_iter_extract_mode() function should be used to query how cleanup * should be performed. */ssize_t netfs_extract_user_iter(struct iov_iter *orig, size_t orig_len, struct iov_iter *new, iov_iter_extraction_t extraction_flags){ struct bio_vec *bv = NULL; struct page **pages; unsigned int cur_npages; unsigned int max_pages; unsigned int npages = 0; unsigned int i; ssize_t ret = 0; size_t count = orig_len, offset, len; size_t bv_size, pg_size;
if (WARN_ON_ONCE(!iter_is_ubuf(orig) && !iter_is_iovec(orig))) return -EIO;
max_pages = iov_iter_npages(orig, INT_MAX); bv_size = array_size(max_pages, sizeof(*bv)); bv = kvmalloc(bv_size, GFP_KERNEL); if (!bv) return -ENOMEM;
/* Put the page list at the end of the bvec list storage. bvec * elements are larger than page pointers, so as long as we work * 0->last, we should be fine. */ pg_size = array_size(max_pages, sizeof(*pages)); pages = (void *)bv + bv_size - pg_size;
while (count && npages < max_pages) { ret = iov_iter_extract_pages(orig, &pages, count, max_pages - npages, extraction_flags, &offset); if (unlikely(ret <= 0)) { ret = ret ?: -EIO; break; }
if (WARN(ret > count, "%s: extract_pages overrun %zd > %zu bytes\n", __func__, ret, count)) { ret = -EIO; break; }
cur_npages = DIV_ROUND_UP(offset + ret, PAGE_SIZE); if (WARN(cur_npages > max_pages - npages, "%s: extract_pages overrun %u > %u pages\n", __func__, npages + cur_npages, max_pages)) { ret = -EIO; break; }
count -= ret; ret += offset;
for (i = 0; i < cur_npages; i++) { len = ret > PAGE_SIZE ? PAGE_SIZE : ret; bvec_set_page(bv + npages + i, *pages++, len - offset, offset); ret -= len; offset = 0; }
npages += cur_npages; }
/* Note: Don't try to clean up after EIO. Either we got no pages, so * nothing to clean up, or we got a buffer overrun, memory corruption * and can't trust the stuff in the buffer (a WARN was emitted). */
if (ret < 0 && (ret == -ENOMEM || npages == 0)) { for (i = 0; i < npages; i++) unpin_user_page(bv[i].bv_page); kvfree(bv); return ret; }
iov_iter_bvec(new, orig->data_source, bv, npages, orig_len - count); return npages;}EXPORT_SYMBOL_GPL(netfs_extract_user_iter);
/* * Select the span of a bvec iterator we're going to use. Limit it by both maximum * size and maximum number of segments. Returns the size of the span in bytes. */static size_t netfs_limit_bvec(const struct iov_iter *iter, size_t start_offset, size_t max_size, size_t max_segs){ const struct bio_vec *bvecs = iter->bvec; unsigned int nbv = iter->nr_segs, ix = 0, nsegs = 0; size_t len, span = 0, n = iter->count; size_t skip = iter->iov_offset + start_offset;
if (WARN_ON(!iov_iter_is_bvec(iter)) || WARN_ON(start_offset > n) || n == 0) return 0;
while (n && ix < nbv && skip) { len = bvecs[ix].bv_len; if (skip < len) break; skip -= len; n -= len; ix++; }
while (n && ix < nbv) { len = min3(n, bvecs[ix].bv_len - skip, max_size); span += len; nsegs++; ix++; if (span >= max_size || nsegs >= max_segs) break; skip = 0; n -= len; }
return min(span, max_size);}
/* * Select the span of a kvec iterator we're going to use. Limit it by both * maximum size and maximum number of segments. Returns the size of the span * in bytes. */static size_t netfs_limit_kvec(const struct iov_iter *iter, size_t start_offset, size_t max_size, size_t max_segs){ const struct kvec *kvecs = iter->kvec; unsigned int nkv = iter->nr_segs, ix = 0, nsegs = 0; size_t len, span = 0, n = iter->count; size_t skip = iter->iov_offset + start_offset;
if (WARN_ON(!iov_iter_is_kvec(iter)) || WARN_ON(start_offset > n) || n == 0) return 0;
while (n && ix < nkv && skip) { len = kvecs[ix].iov_len; if (skip < len) break; skip -= len; n -= len; ix++; }
while (n && ix < nkv) { len = min3(n, kvecs[ix].iov_len - skip, max_size); span += len; nsegs++; ix++; if (span >= max_size || nsegs >= max_segs) break; skip = 0; n -= len; }
return min(span, max_size);}
/* * Select the span of an xarray iterator we're going to use. Limit it by both * maximum size and maximum number of segments. It is assumed that segments * can be larger than a page in size, provided they're physically contiguous. * Returns the size of the span in bytes. */static size_t netfs_limit_xarray(const struct iov_iter *iter, size_t start_offset, size_t max_size, size_t max_segs){ struct folio *folio; unsigned int nsegs = 0; loff_t pos = iter->xarray_start + iter->iov_offset; pgoff_t index = pos / PAGE_SIZE; size_t span = 0, n = iter->count;
XA_STATE(xas, iter->xarray, index);
if (WARN_ON(!iov_iter_is_xarray(iter)) || WARN_ON(start_offset > n) || n == 0) return 0; max_size = min(max_size, n - start_offset);
rcu_read_lock(); xas_for_each(&xas, folio, ULONG_MAX) { size_t offset, flen, len; if (xas_retry(&xas, folio)) continue; if (WARN_ON(xa_is_value(folio))) break; if (WARN_ON(folio_test_hugetlb(folio))) break;
flen = folio_size(folio); offset = offset_in_folio(folio, pos); len = min(max_size, flen - offset); span += len; nsegs++; if (span >= max_size || nsegs >= max_segs) break; }
rcu_read_unlock(); return min(span, max_size);}
/* * Select the span of a folio queue iterator we're going to use. Limit it by * both maximum size and maximum number of segments. Returns the size of the * span in bytes. */static size_t netfs_limit_folioq(const struct iov_iter *iter, size_t start_offset, size_t max_size, size_t max_segs){ const struct folio_queue *folioq = iter->folioq; unsigned int nsegs = 0; unsigned int slot = iter->folioq_slot; size_t span = 0, n = iter->count;
if (WARN_ON(!iov_iter_is_folioq(iter)) || WARN_ON(start_offset > n) || n == 0) return 0; max_size = umin(max_size, n - start_offset);
if (slot >= folioq_nr_slots(folioq)) { folioq = folioq->next; slot = 0; }
start_offset += iter->iov_offset; do { size_t flen = folioq_folio_size(folioq, slot);
if (start_offset < flen) { span += flen - start_offset; nsegs++; start_offset = 0; } else { start_offset -= flen; } if (span >= max_size || nsegs >= max_segs) break;
slot++; if (slot >= folioq_nr_slots(folioq)) { folioq = folioq->next; slot = 0; } } while (folioq);
return umin(span, max_size);}
size_t netfs_limit_iter(const struct iov_iter *iter, size_t start_offset, size_t max_size, size_t max_segs){ if (iov_iter_is_folioq(iter)) return netfs_limit_folioq(iter, start_offset, max_size, max_segs); if (iov_iter_is_bvec(iter)) return netfs_limit_bvec(iter, start_offset, max_size, max_segs); if (iov_iter_is_xarray(iter)) return netfs_limit_xarray(iter, start_offset, max_size, max_segs); if (iov_iter_is_kvec(iter)) return netfs_limit_kvec(iter, start_offset, max_size, max_segs); BUG();}EXPORT_SYMBOL(netfs_limit_iter);