Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657// SPDX-License-Identifier: GPL-2.0/* FUSE: Filesystem in Userspace Copyright (C) 2001-2018 Miklos Szeredi <miklos@szeredi.hu>*/
#include "fuse_i.h"#include <linux/iversion.h>#include <linux/posix_acl.h>#include <linux/pagemap.h>#include <linux/highmem.h>#include <linux/vmalloc.h>
static bool fuse_use_readdirplus(struct inode *dir, struct dir_context *ctx){ struct fuse_conn *fc = get_fuse_conn(dir); struct fuse_inode *fi = get_fuse_inode(dir);
if (!fc->do_readdirplus) return false; if (!fc->readdirplus_auto) return true; if (test_and_clear_bit(FUSE_I_ADVISE_RDPLUS, &fi->state)) return true; if (ctx->pos == 0) return true; return false;}
static void fuse_add_dirent_to_cache(struct file *file, struct fuse_dirent *dirent, loff_t pos){ struct fuse_inode *fi = get_fuse_inode(file_inode(file)); size_t reclen = FUSE_DIRENT_SIZE(dirent); pgoff_t index; struct page *page; loff_t size; u64 version; unsigned int offset; void *addr;
/* Dirent doesn't fit in readdir cache page? Skip caching. */ if (reclen > PAGE_SIZE) return;
spin_lock(&fi->rdc.lock); /* * Is cache already completed? Or this entry does not go at the end of * cache? */ if (fi->rdc.cached || pos != fi->rdc.pos) { spin_unlock(&fi->rdc.lock); return; } version = fi->rdc.version; size = fi->rdc.size; offset = offset_in_page(size); index = size >> PAGE_SHIFT; /* Dirent doesn't fit in current page? Jump to next page. */ if (offset + reclen > PAGE_SIZE) { index++; offset = 0; } spin_unlock(&fi->rdc.lock);
if (offset) { page = find_lock_page(file->f_mapping, index); } else { page = find_or_create_page(file->f_mapping, index, mapping_gfp_mask(file->f_mapping)); } if (!page) return;
spin_lock(&fi->rdc.lock); /* Raced with another readdir */ if (fi->rdc.version != version || fi->rdc.size != size || WARN_ON(fi->rdc.pos != pos)) goto unlock;
addr = kmap_local_page(page); if (!offset) { clear_page(addr); SetPageUptodate(page); } memcpy(addr + offset, dirent, reclen); kunmap_local(addr); fi->rdc.size = (index << PAGE_SHIFT) + offset + reclen; fi->rdc.pos = dirent->off;unlock: spin_unlock(&fi->rdc.lock); unlock_page(page); put_page(page);}
static void fuse_readdir_cache_end(struct file *file, loff_t pos){ struct fuse_inode *fi = get_fuse_inode(file_inode(file)); loff_t end;
spin_lock(&fi->rdc.lock); /* does cache end position match current position? */ if (fi->rdc.pos != pos) { spin_unlock(&fi->rdc.lock); return; }
fi->rdc.cached = true; end = ALIGN(fi->rdc.size, PAGE_SIZE); spin_unlock(&fi->rdc.lock);
/* truncate unused tail of cache */ truncate_inode_pages(file->f_mapping, end);}
static bool fuse_emit(struct file *file, struct dir_context *ctx, struct fuse_dirent *dirent){ struct fuse_file *ff = file->private_data;
if (ff->open_flags & FOPEN_CACHE_DIR) fuse_add_dirent_to_cache(file, dirent, ctx->pos);
return dir_emit(ctx, dirent->name, dirent->namelen, dirent->ino, dirent->type | FILLDIR_FLAG_NOINTR);}
static int parse_dirfile(char *buf, size_t nbytes, struct file *file, struct dir_context *ctx){ while (nbytes >= FUSE_NAME_OFFSET) { struct fuse_dirent *dirent = (struct fuse_dirent *) buf; size_t reclen = FUSE_DIRENT_SIZE(dirent); if (!dirent->namelen || dirent->namelen > FUSE_NAME_MAX) return -EIO; if (reclen > nbytes) break; if (memchr(dirent->name, '/', dirent->namelen) != NULL) return -EIO;
if (!fuse_emit(file, ctx, dirent)) break;
buf += reclen; nbytes -= reclen; ctx->pos = dirent->off; }
return 0;}
static int fuse_direntplus_link(struct file *file, struct fuse_direntplus *direntplus, u64 attr_version, u64 evict_ctr){ struct fuse_entry_out *o = &direntplus->entry_out; struct fuse_dirent *dirent = &direntplus->dirent; struct dentry *parent = file->f_path.dentry; struct qstr name = QSTR_INIT(dirent->name, dirent->namelen); struct dentry *dentry; struct dentry *alias; struct inode *dir = d_inode(parent); struct fuse_conn *fc; struct inode *inode; int epoch;
if (!o->nodeid) { /* * Unlike in the case of fuse_lookup, zero nodeid does not mean * ENOENT. Instead, it only means the userspace filesystem did * not want to return attributes/handle for this entry. * * So do nothing. */ return 0; }
if (name.name[0] == '.') { /* * We could potentially refresh the attributes of the directory * and its parent? */ if (name.len == 1) return 0; if (name.name[1] == '.' && name.len == 2) return 0; }
if (invalid_nodeid(o->nodeid)) return -EIO; if (fuse_invalid_attr(&o->attr)) return -EIO;
fc = get_fuse_conn(dir); epoch = atomic_read(&fc->epoch);
name.hash = full_name_hash(parent, name.name, name.len); dentry = d_lookup(parent, &name); if (!dentry) {retry: dentry = d_alloc_parallel(parent, &name); if (IS_ERR(dentry)) return PTR_ERR(dentry); } if (!d_in_lookup(dentry)) { struct fuse_inode *fi; inode = d_inode(dentry); if (inode && get_node_id(inode) != o->nodeid) inode = NULL; if (!inode || fuse_stale_inode(inode, o->generation, &o->attr)) { if (inode) fuse_make_bad(inode); d_invalidate(dentry); dput(dentry); goto retry; } if (fuse_is_bad(inode)) { dput(dentry); return -EIO; }
fi = get_fuse_inode(inode); spin_lock(&fi->lock); fi->nlookup++; spin_unlock(&fi->lock);
forget_all_cached_acls(inode); fuse_change_attributes(inode, &o->attr, NULL, ATTR_TIMEOUT(o), attr_version); /* * The other branch comes via fuse_iget() * which bumps nlookup inside */ } else { inode = fuse_iget(dir->i_sb, o->nodeid, o->generation, &o->attr, ATTR_TIMEOUT(o), attr_version, evict_ctr); if (!inode) inode = ERR_PTR(-ENOMEM);
alias = d_splice_alias(inode, dentry); d_lookup_done(dentry); if (alias) { dput(dentry); dentry = alias; } if (IS_ERR(dentry)) { if (!IS_ERR(inode)) { struct fuse_inode *fi = get_fuse_inode(inode);
spin_lock(&fi->lock); fi->nlookup--; spin_unlock(&fi->lock); } return PTR_ERR(dentry); } } if (fc->readdirplus_auto) set_bit(FUSE_I_INIT_RDPLUS, &get_fuse_inode(inode)->state); dentry->d_time = epoch; fuse_change_entry_timeout(dentry, o);
dput(dentry); return 0;}
static void fuse_force_forget(struct file *file, u64 nodeid){ struct inode *inode = file_inode(file); struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_forget_in inarg; FUSE_ARGS(args);
memset(&inarg, 0, sizeof(inarg)); inarg.nlookup = 1; args.opcode = FUSE_FORGET; args.nodeid = nodeid; args.in_numargs = 1; args.in_args[0].size = sizeof(inarg); args.in_args[0].value = &inarg; args.force = true; args.noreply = true;
fuse_simple_request(fm, &args); /* ignore errors */}
static int parse_dirplusfile(char *buf, size_t nbytes, struct file *file, struct dir_context *ctx, u64 attr_version, u64 evict_ctr){ struct fuse_direntplus *direntplus; struct fuse_dirent *dirent; size_t reclen; int over = 0; int ret;
while (nbytes >= FUSE_NAME_OFFSET_DIRENTPLUS) { direntplus = (struct fuse_direntplus *) buf; dirent = &direntplus->dirent; reclen = FUSE_DIRENTPLUS_SIZE(direntplus);
if (!dirent->namelen || dirent->namelen > FUSE_NAME_MAX) return -EIO; if (reclen > nbytes) break; if (memchr(dirent->name, '/', dirent->namelen) != NULL) return -EIO;
if (!over) { /* We fill entries into dstbuf only as much as it can hold. But we still continue iterating over remaining entries to link them. If not, we need to send a FORGET for each of those which we did not link. */ over = !fuse_emit(file, ctx, dirent); if (!over) ctx->pos = dirent->off; }
buf += reclen; nbytes -= reclen;
ret = fuse_direntplus_link(file, direntplus, attr_version, evict_ctr); if (ret) fuse_force_forget(file, direntplus->entry_out.nodeid); }
return 0;}
static struct page **fuse_readdir_alloc_buf(struct fuse_args_pages *ap, size_t *bufsize){ unsigned int i, nr_alloc, nr_pages = DIV_ROUND_UP(*bufsize, PAGE_SIZE); struct page **pages = kcalloc(nr_pages, sizeof(*pages), GFP_KERNEL);
if (!pages) return NULL;
nr_alloc = alloc_pages_bulk(GFP_KERNEL, nr_pages, pages); if (!nr_alloc) goto free_array;
if (nr_alloc < nr_pages) { nr_pages = nr_alloc; *bufsize = (size_t) nr_pages << PAGE_SHIFT; }
ap->folios = fuse_folios_alloc(nr_pages, GFP_KERNEL, &ap->descs); if (!ap->folios) goto release_pages;
for (i = 0; i < nr_pages; i++) { ap->folios[i] = page_folio(pages[i]); ap->descs[i].length = min_t(size_t, *bufsize - (size_t)i * PAGE_SIZE, PAGE_SIZE); } ap->num_folios = nr_pages; ap->args.out_pages = true;
return pages;
release_pages: release_pages(pages, nr_pages);free_array: kfree(pages); return NULL;}
static int fuse_readdir_uncached(struct file *file, struct dir_context *ctx){ int plus; ssize_t res; struct inode *inode = file_inode(file); struct fuse_mount *fm = get_fuse_mount(inode); struct fuse_conn *fc = fm->fc; struct fuse_io_args ia = {}; struct fuse_args_pages *ap = &ia.ap; void *buf; size_t bufsize = clamp((unsigned int) ctx->count, PAGE_SIZE, fc->max_pages << PAGE_SHIFT); u64 attr_version = 0, evict_ctr = 0; bool locked; struct page **pages = fuse_readdir_alloc_buf(ap, &bufsize);
if (!pages) return -ENOMEM;
plus = fuse_use_readdirplus(inode, ctx); if (plus) { attr_version = fuse_get_attr_version(fm->fc); evict_ctr = fuse_get_evict_ctr(fm->fc); fuse_read_args_fill(&ia, file, ctx->pos, bufsize, FUSE_READDIRPLUS); } else { fuse_read_args_fill(&ia, file, ctx->pos, bufsize, FUSE_READDIR); } locked = fuse_lock_inode(inode); res = fuse_simple_request(fm, &ap->args); fuse_unlock_inode(inode, locked); if (res < 0) goto out;
if (!res) { struct fuse_file *ff = file->private_data;
if (ff->open_flags & FOPEN_CACHE_DIR) fuse_readdir_cache_end(file, ctx->pos); goto out; }
buf = vm_map_ram(pages, ap->num_folios, -1); if (!buf) { res = -ENOMEM; } else { if (plus) res = parse_dirplusfile(buf, res, file, ctx, attr_version, evict_ctr); else res = parse_dirfile(buf, res, file, ctx);
vm_unmap_ram(buf, ap->num_folios); }out: kfree(ap->folios); release_pages(pages, ap->num_folios); kfree(pages);
fuse_invalidate_atime(inode);
return res;}
enum fuse_parse_result { FOUND_ERR = -1, FOUND_NONE = 0, FOUND_SOME, FOUND_ALL,};
static enum fuse_parse_result fuse_parse_cache(struct fuse_file *ff, void *addr, unsigned int size, struct dir_context *ctx){ unsigned int offset = offset_in_page(ff->readdir.cache_off); enum fuse_parse_result res = FOUND_NONE;
WARN_ON(offset >= size);
for (;;) { struct fuse_dirent *dirent = addr + offset; unsigned int nbytes = size - offset; size_t reclen;
if (nbytes < FUSE_NAME_OFFSET || !dirent->namelen) break;
reclen = FUSE_DIRENT_SIZE(dirent); /* derefs ->namelen */
if (WARN_ON(dirent->namelen > FUSE_NAME_MAX)) return FOUND_ERR; if (WARN_ON(reclen > nbytes)) return FOUND_ERR; if (WARN_ON(memchr(dirent->name, '/', dirent->namelen) != NULL)) return FOUND_ERR;
if (ff->readdir.pos == ctx->pos) { res = FOUND_SOME; if (!dir_emit(ctx, dirent->name, dirent->namelen, dirent->ino, dirent->type | FILLDIR_FLAG_NOINTR)) return FOUND_ALL; ctx->pos = dirent->off; } ff->readdir.pos = dirent->off; ff->readdir.cache_off += reclen;
offset += reclen; }
return res;}
static void fuse_rdc_reset(struct inode *inode){ struct fuse_inode *fi = get_fuse_inode(inode);
fi->rdc.cached = false; fi->rdc.version++; fi->rdc.size = 0; fi->rdc.pos = 0; fi->rdc.epoch = 0;}
#define UNCACHED 1
static int fuse_readdir_cached(struct file *file, struct dir_context *ctx){ struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(file); struct fuse_conn *fc = get_fuse_conn(inode); struct fuse_inode *fi = get_fuse_inode(inode); enum fuse_parse_result res; pgoff_t index; unsigned int size; struct page *page; void *addr;
/* Seeked? If so, reset the cache stream */ if (ff->readdir.pos != ctx->pos) { ff->readdir.pos = 0; ff->readdir.cache_off = 0; }
/* * We're just about to start reading into the cache or reading the * cache; both cases require an up-to-date mtime value. */ if (!ctx->pos && fc->auto_inval_data) { int err = fuse_update_attributes(inode, file, STATX_MTIME);
if (err) return err; }
retry: spin_lock(&fi->rdc.lock);retry_locked: if (!fi->rdc.cached) { /* Starting cache? Set cache mtime. */ if (!ctx->pos && !fi->rdc.size) { fi->rdc.mtime = inode_get_mtime(inode); fi->rdc.iversion = inode_query_iversion(inode); fi->rdc.epoch = atomic_read(&fc->epoch); } spin_unlock(&fi->rdc.lock); return UNCACHED; } /* * When at the beginning of the directory (i.e. just after opendir(3) or * rewinddir(3)), then need to check whether directory contents have * changed, and reset the cache if so. */ if (!ctx->pos) { struct timespec64 mtime = inode_get_mtime(inode);
if (inode_peek_iversion(inode) != fi->rdc.iversion || !timespec64_equal(&fi->rdc.mtime, &mtime) || fi->rdc.epoch != atomic_read(&fc->epoch)) { fuse_rdc_reset(inode); goto retry_locked; } }
/* * If cache version changed since the last getdents() call, then reset * the cache stream. */ if (ff->readdir.version != fi->rdc.version) { ff->readdir.pos = 0; ff->readdir.cache_off = 0; } /* * If at the beginning of the cache, than reset version to * current. */ if (ff->readdir.pos == 0) ff->readdir.version = fi->rdc.version;
WARN_ON(fi->rdc.size < ff->readdir.cache_off);
index = ff->readdir.cache_off >> PAGE_SHIFT;
if (index == (fi->rdc.size >> PAGE_SHIFT)) size = offset_in_page(fi->rdc.size); else size = PAGE_SIZE; spin_unlock(&fi->rdc.lock);
/* EOF? */ if (offset_in_page(ff->readdir.cache_off) == size) return 0;
page = find_get_page_flags(file->f_mapping, index, FGP_ACCESSED | FGP_LOCK); /* Page gone missing, then re-added to cache, but not initialized? */ if (page && !PageUptodate(page)) { unlock_page(page); put_page(page); page = NULL; } spin_lock(&fi->rdc.lock); if (!page) { /* * Uh-oh: page gone missing, cache is useless */ if (fi->rdc.version == ff->readdir.version) fuse_rdc_reset(inode); goto retry_locked; }
/* Make sure it's still the same version after getting the page. */ if (ff->readdir.version != fi->rdc.version) { spin_unlock(&fi->rdc.lock); unlock_page(page); put_page(page); goto retry; } spin_unlock(&fi->rdc.lock);
/* * Contents of the page are now protected against changing by holding * the page lock. */ addr = kmap_local_page(page); res = fuse_parse_cache(ff, addr, size, ctx); kunmap_local(addr); unlock_page(page); put_page(page);
if (res == FOUND_ERR) return -EIO;
if (res == FOUND_ALL) return 0;
if (size == PAGE_SIZE) { /* We hit end of page: skip to next page. */ ff->readdir.cache_off = ALIGN(ff->readdir.cache_off, PAGE_SIZE); goto retry; }
/* * End of cache reached. If found position, then we are done, otherwise * need to fall back to uncached, since the position we were looking for * wasn't in the cache. */ return res == FOUND_SOME ? 0 : UNCACHED;}
int fuse_readdir(struct file *file, struct dir_context *ctx){ struct fuse_file *ff = file->private_data; struct inode *inode = file_inode(file); int err;
if (fuse_is_bad(inode)) return -EIO;
err = UNCACHED; if (ff->open_flags & FOPEN_CACHE_DIR) err = fuse_readdir_cached(file, ctx); if (err == UNCACHED) err = fuse_readdir_uncached(file, ctx);
return err;}