Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477// SPDX-License-Identifier: GPL-2.0/* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */#include <linux/memremap.h>#include <linux/pagemap.h>#include <linux/module.h>#include <linux/device.h>#include <linux/cdev.h>#include <linux/slab.h>#include <linux/dax.h>#include <linux/fs.h>#include <linux/mm.h>#include <linux/mman.h>#include "dax-private.h"#include "bus.h"
static int __check_vma(struct dev_dax *dev_dax, vma_flags_t flags, unsigned long start, unsigned long end, struct file *file, const char *func){ struct device *dev = &dev_dax->dev; unsigned long mask;
if (!dax_alive(dev_dax->dax_dev)) return -ENXIO;
/* prevent private mappings from being established */ if (!vma_flags_test_any(&flags, VMA_MAYSHARE_BIT)) { dev_info_ratelimited(dev, "%s: %s: fail, attempted private mapping\n", current->comm, func); return -EINVAL; }
mask = dev_dax->align - 1; if (start & mask || end & mask) { dev_info_ratelimited(dev, "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n", current->comm, func, start, end, mask); return -EINVAL; }
if (!file_is_dax(file)) { dev_info_ratelimited(dev, "%s: %s: fail, vma is not DAX capable\n", current->comm, func); return -EINVAL; }
return 0;}
static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma, const char *func){ return __check_vma(dev_dax, vma->flags, vma->vm_start, vma->vm_end, vma->vm_file, func);}
static void dax_set_mapping(struct vm_fault *vmf, unsigned long pfn, unsigned long fault_size){ unsigned long i, nr_pages = fault_size / PAGE_SIZE; struct file *filp = vmf->vma->vm_file; struct dev_dax *dev_dax = filp->private_data; pgoff_t pgoff;
/* mapping is only set on the head */ if (dev_dax->pgmap->vmemmap_shift) nr_pages = 1;
pgoff = linear_page_index(vmf->vma, ALIGN_DOWN(vmf->address, fault_size));
for (i = 0; i < nr_pages; i++) { struct folio *folio = pfn_folio(pfn + i);
if (folio->mapping) continue;
folio->mapping = filp->f_mapping; folio->index = pgoff + i; }}
static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf){ struct device *dev = &dev_dax->dev; phys_addr_t phys; unsigned long pfn; unsigned int fault_size = PAGE_SIZE;
if (check_vma(dev_dax, vmf->vma, __func__)) return VM_FAULT_SIGBUS;
if (dev_dax->align > PAGE_SIZE) { dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", dev_dax->align, fault_size); return VM_FAULT_SIGBUS; }
if (fault_size != dev_dax->align) return VM_FAULT_SIGBUS;
phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE); if (phys == -1) { dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff); return VM_FAULT_SIGBUS; }
pfn = PHYS_PFN(phys);
dax_set_mapping(vmf, pfn, fault_size);
return vmf_insert_page_mkwrite(vmf, pfn_to_page(pfn), vmf->flags & FAULT_FLAG_WRITE);}
static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf){ unsigned long pmd_addr = vmf->address & PMD_MASK; struct device *dev = &dev_dax->dev; phys_addr_t phys; pgoff_t pgoff; unsigned long pfn; unsigned int fault_size = PMD_SIZE;
if (check_vma(dev_dax, vmf->vma, __func__)) return VM_FAULT_SIGBUS;
if (dev_dax->align > PMD_SIZE) { dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", dev_dax->align, fault_size); return VM_FAULT_SIGBUS; }
if (fault_size < dev_dax->align) return VM_FAULT_SIGBUS; else if (fault_size > dev_dax->align) return VM_FAULT_FALLBACK;
/* if we are outside of the VMA */ if (pmd_addr < vmf->vma->vm_start || (pmd_addr + PMD_SIZE) > vmf->vma->vm_end) return VM_FAULT_SIGBUS;
pgoff = linear_page_index(vmf->vma, pmd_addr); phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE); if (phys == -1) { dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); return VM_FAULT_SIGBUS; }
pfn = PHYS_PFN(phys);
dax_set_mapping(vmf, pfn, fault_size);
return vmf_insert_folio_pmd(vmf, page_folio(pfn_to_page(pfn)), vmf->flags & FAULT_FLAG_WRITE);}
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUDstatic vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf){ unsigned long pud_addr = vmf->address & PUD_MASK; struct device *dev = &dev_dax->dev; phys_addr_t phys; pgoff_t pgoff; unsigned long pfn; unsigned int fault_size = PUD_SIZE;
if (check_vma(dev_dax, vmf->vma, __func__)) return VM_FAULT_SIGBUS;
if (dev_dax->align > PUD_SIZE) { dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", dev_dax->align, fault_size); return VM_FAULT_SIGBUS; }
if (fault_size < dev_dax->align) return VM_FAULT_SIGBUS; else if (fault_size > dev_dax->align) return VM_FAULT_FALLBACK;
/* if we are outside of the VMA */ if (pud_addr < vmf->vma->vm_start || (pud_addr + PUD_SIZE) > vmf->vma->vm_end) return VM_FAULT_SIGBUS;
pgoff = linear_page_index(vmf->vma, pud_addr); phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE); if (phys == -1) { dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); return VM_FAULT_SIGBUS; }
pfn = PHYS_PFN(phys);
dax_set_mapping(vmf, pfn, fault_size);
return vmf_insert_folio_pud(vmf, page_folio(pfn_to_page(pfn)), vmf->flags & FAULT_FLAG_WRITE);}#elsestatic vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf){ return VM_FAULT_FALLBACK;}#endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf, unsigned int order){ struct file *filp = vmf->vma->vm_file; vm_fault_t rc = VM_FAULT_SIGBUS; int id; struct dev_dax *dev_dax = filp->private_data;
dev_dbg(&dev_dax->dev, "%s: op=%s addr=%#lx order=%d\n", current->comm, (vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read", vmf->address & ~((1UL << (order + PAGE_SHIFT)) - 1), order);
id = dax_read_lock(); if (order == 0) rc = __dev_dax_pte_fault(dev_dax, vmf); else if (order == PMD_ORDER) rc = __dev_dax_pmd_fault(dev_dax, vmf); else if (order == PUD_ORDER) rc = __dev_dax_pud_fault(dev_dax, vmf); else rc = VM_FAULT_SIGBUS;
dax_read_unlock(id);
return rc;}
static vm_fault_t dev_dax_fault(struct vm_fault *vmf){ return dev_dax_huge_fault(vmf, 0);}
static int dev_dax_may_split(struct vm_area_struct *vma, unsigned long addr){ struct file *filp = vma->vm_file; struct dev_dax *dev_dax = filp->private_data;
if (!IS_ALIGNED(addr, dev_dax->align)) return -EINVAL; return 0;}
static unsigned long dev_dax_pagesize(struct vm_area_struct *vma){ struct file *filp = vma->vm_file; struct dev_dax *dev_dax = filp->private_data;
return dev_dax->align;}
static const struct vm_operations_struct dax_vm_ops = { .fault = dev_dax_fault, .huge_fault = dev_dax_huge_fault, .may_split = dev_dax_may_split, .pagesize = dev_dax_pagesize,};
static int dax_mmap_prepare(struct vm_area_desc *desc){ struct file *filp = desc->file; struct dev_dax *dev_dax = filp->private_data; int rc, id;
dev_dbg(&dev_dax->dev, "trace\n");
/* * We lock to check dax_dev liveness and will re-check at * fault time. */ id = dax_read_lock(); rc = __check_vma(dev_dax, desc->vma_flags, desc->start, desc->end, filp, __func__); dax_read_unlock(id); if (rc) return rc;
desc->vm_ops = &dax_vm_ops; vma_desc_set_flags(desc, VMA_HUGEPAGE_BIT); return 0;}
/* return an unmapped area aligned to the dax region specified alignment */static unsigned long dax_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags){ unsigned long off, off_end, off_align, len_align, addr_align, align; struct dev_dax *dev_dax = filp ? filp->private_data : NULL;
if (!dev_dax || addr) goto out;
align = dev_dax->align; off = pgoff << PAGE_SHIFT; off_end = off + len; off_align = round_up(off, align);
if ((off_end <= off_align) || ((off_end - off_align) < align)) goto out;
len_align = len + align; if ((off + len_align) < off) goto out;
addr_align = mm_get_unmapped_area(filp, addr, len_align, pgoff, flags); if (!IS_ERR_VALUE(addr_align)) { addr_align += (off - addr_align) & (align - 1); return addr_align; } out: return mm_get_unmapped_area(filp, addr, len, pgoff, flags);}
static const struct address_space_operations dev_dax_aops = { .dirty_folio = noop_dirty_folio,};
static int dax_open(struct inode *inode, struct file *filp){ struct dax_device *dax_dev = inode_dax(inode); struct inode *__dax_inode = dax_inode(dax_dev); struct dev_dax *dev_dax = dax_get_private(dax_dev);
dev_dbg(&dev_dax->dev, "trace\n"); inode->i_mapping = __dax_inode->i_mapping; inode->i_mapping->host = __dax_inode; inode->i_mapping->a_ops = &dev_dax_aops; filp->f_mapping = inode->i_mapping; filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping); filp->f_sb_err = file_sample_sb_err(filp); filp->private_data = dev_dax; inode->i_flags = S_DAX;
return 0;}
static int dax_release(struct inode *inode, struct file *filp){ struct dev_dax *dev_dax = filp->private_data;
dev_dbg(&dev_dax->dev, "trace\n"); return 0;}
static const struct file_operations dax_fops = { .llseek = noop_llseek, .owner = THIS_MODULE, .open = dax_open, .release = dax_release, .get_unmapped_area = dax_get_unmapped_area, .mmap_prepare = dax_mmap_prepare, .fop_flags = FOP_MMAP_SYNC,};
static void dev_dax_cdev_del(void *cdev){ cdev_del(cdev);}
static void dev_dax_kill(void *dev_dax){ kill_dev_dax(dev_dax);}
static int dev_dax_probe(struct dev_dax *dev_dax){ struct dax_device *dax_dev = dev_dax->dax_dev; struct device *dev = &dev_dax->dev; struct dev_pagemap *pgmap; struct inode *inode; struct cdev *cdev; void *addr; int rc, i;
if (static_dev_dax(dev_dax)) { if (dev_dax->nr_range > 1) { dev_warn(dev, "static pgmap / multi-range device conflict\n"); return -EINVAL; }
pgmap = dev_dax->pgmap; } else { if (dev_dax->pgmap) { dev_warn(dev, "dynamic-dax with pre-populated page map\n"); return -EINVAL; }
pgmap = devm_kzalloc(dev, struct_size(pgmap, ranges, dev_dax->nr_range - 1), GFP_KERNEL); if (!pgmap) return -ENOMEM;
pgmap->nr_range = dev_dax->nr_range; dev_dax->pgmap = pgmap;
for (i = 0; i < dev_dax->nr_range; i++) { struct range *range = &dev_dax->ranges[i].range; pgmap->ranges[i] = *range; } }
for (i = 0; i < dev_dax->nr_range; i++) { struct range *range = &dev_dax->ranges[i].range;
if (!devm_request_mem_region(dev, range->start, range_len(range), dev_name(dev))) { dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n", i, range->start, range->end); return -EBUSY; } }
pgmap->type = MEMORY_DEVICE_GENERIC; if (dev_dax->align > PAGE_SIZE) pgmap->vmemmap_shift = order_base_2(dev_dax->align >> PAGE_SHIFT); addr = devm_memremap_pages(dev, pgmap); if (IS_ERR(addr)) return PTR_ERR(addr);
inode = dax_inode(dax_dev); cdev = inode->i_cdev; cdev_init(cdev, &dax_fops); cdev->owner = dev->driver->owner; cdev_set_parent(cdev, &dev->kobj); rc = cdev_add(cdev, dev->devt, 1); if (rc) return rc;
rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev); if (rc) return rc;
run_dax(dax_dev); return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax);}
static struct dax_device_driver device_dax_driver = { .probe = dev_dax_probe, .type = DAXDRV_DEVICE_TYPE,};
static int __init dax_init(void){ return dax_driver_register(&device_dax_driver);}
static void __exit dax_exit(void){ dax_driver_unregister(&device_dax_driver);}
MODULE_AUTHOR("Intel Corporation");MODULE_DESCRIPTION("Device DAX: direct access device driver");MODULE_LICENSE("GPL v2");module_init(dax_init);module_exit(dax_exit);MODULE_ALIAS_DAX_DEVICE(0);