Something went wrong. Try again.
(I'm so sorry!) Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
C
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499/* SPDX-License-Identifier: GPL-2.0 *//* * include/linux/userfaultfd_k.h * * Copyright (C) 2015 Red Hat, Inc. * */
#ifndef _LINUX_USERFAULTFD_K_H#define _LINUX_USERFAULTFD_K_H
#ifdef CONFIG_USERFAULTFD
#include <linux/userfaultfd.h> /* linux/include/uapi/linux/userfaultfd.h */
#include <linux/fcntl.h>#include <linux/mm.h>#include <linux/swap.h>#include <linux/leafops.h>#include <asm-generic/pgtable_uffd.h>#include <linux/hugetlb_inline.h>
/* The set of all possible UFFD-related VM flags. */#define __VM_UFFD_FLAGS (VM_UFFD_MISSING | VM_UFFD_MINOR | \ VM_UFFD_WP | VM_UFFD_RWP)
#define __VMA_UFFD_FLAGS mk_vma_flags_from_masks(VMA_UFFD_MISSING, VMA_UFFD_WP, \ VMA_UFFD_MINOR, VMA_UFFD_RWP)
/* * CAREFUL: Check include/uapi/asm-generic/fcntl.h when defining * new flags, since they might collide with O_* ones. We want * to re-use O_* flags that couldn't possibly have a meaning * from userfaultfd, in order to leave a free define-space for * shared O_* flags. */#define UFFD_SHARED_FCNTL_FLAGS (O_CLOEXEC | O_NONBLOCK)
/* * Start with fault_pending_wqh and fault_wqh so they're more likely * to be in the same cacheline. * * Locking order: * fd_wqh.lock * fault_pending_wqh.lock * fault_wqh.lock * event_wqh.lock * * To avoid deadlocks, IRQs must be disabled when taking any of the above locks, * since fd_wqh.lock is taken by aio_poll() while it's holding a lock that's * also taken in IRQ context. */struct userfaultfd_ctx { /* waitqueue head for the pending (i.e. not read) userfaults */ wait_queue_head_t fault_pending_wqh; /* waitqueue head for the userfaults */ wait_queue_head_t fault_wqh; /* waitqueue head for the pseudo fd to wakeup poll/read */ wait_queue_head_t fd_wqh; /* waitqueue head for events */ wait_queue_head_t event_wqh; /* a refile sequence protected by fault_pending_wqh lock */ seqcount_spinlock_t refile_seq; /* pseudo fd refcounting */ refcount_t refcount; /* userfaultfd syscall flags */ unsigned int flags; /* features requested from the userspace */ unsigned int features; /* released */ bool released; /* * Prevents userfaultfd operations (fill/move/wp) from happening while * some non-cooperative event(s) is taking place. Increments are done * in write-mode. Whereas, userfaultfd operations, which includes * reading mmap_changing, is done under read-mode. */ struct rw_semaphore map_changing_lock; /* memory mappings are changing because of non-cooperative event */ atomic_t mmap_changing; /* mm with one ore more vmas attached to this userfaultfd_ctx */ struct mm_struct *mm;};
extern vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason);
/* VMA userfaultfd operations */struct vm_uffd_ops { /* Checks if a VMA can support userfaultfd */ bool (*can_userfault)(struct vm_area_struct *vma, vm_flags_t vm_flags); /* * Called to resolve UFFDIO_CONTINUE request. * Should return the folio found at pgoff in the VMA's pagecache if it * exists or ERR_PTR otherwise. * The returned folio is locked and with reference held. */ struct folio *(*get_folio_noalloc)(struct inode *inode, pgoff_t pgoff); /* * Called during resolution of UFFDIO_COPY request. * Should allocate and return a folio or NULL if allocation fails. */ struct folio *(*alloc_folio)(struct vm_area_struct *vma, unsigned long addr); /* * Called during resolution of UFFDIO_COPY request. * Should only be called with a folio returned by alloc_folio() above. * The folio will be set to locked. * Returns 0 on success, error code on failure. */ int (*filemap_add)(struct folio *folio, struct vm_area_struct *vma, unsigned long addr); /* * Called during resolution of UFFDIO_COPY request on the error * handling path. * Should revert the operation of ->filemap_add(). */ void (*filemap_remove)(struct folio *folio, struct vm_area_struct *vma);};
/* A combined operation mode + behavior flags. */typedef unsigned int __bitwise uffd_flags_t;
/* Mutually exclusive modes of operation. */enum mfill_atomic_mode { MFILL_ATOMIC_COPY, MFILL_ATOMIC_ZEROPAGE, MFILL_ATOMIC_CONTINUE, MFILL_ATOMIC_POISON, NR_MFILL_ATOMIC_MODES,};
#define MFILL_ATOMIC_MODE_BITS (const_ilog2(NR_MFILL_ATOMIC_MODES - 1) + 1)#define MFILL_ATOMIC_BIT(nr) BIT(MFILL_ATOMIC_MODE_BITS + (nr))#define MFILL_ATOMIC_FLAG(nr) ((__force uffd_flags_t) MFILL_ATOMIC_BIT(nr))#define MFILL_ATOMIC_MODE_MASK ((__force uffd_flags_t) (MFILL_ATOMIC_BIT(0) - 1))
static inline bool uffd_flags_mode_is(uffd_flags_t flags, enum mfill_atomic_mode expected){ return (flags & MFILL_ATOMIC_MODE_MASK) == ((__force uffd_flags_t) expected);}
static inline uffd_flags_t uffd_flags_set_mode(uffd_flags_t flags, enum mfill_atomic_mode mode){ flags &= ~MFILL_ATOMIC_MODE_MASK; return flags | ((__force uffd_flags_t) mode);}
/* Flags controlling behavior. These behavior changes are mode-independent. */#define MFILL_ATOMIC_WP MFILL_ATOMIC_FLAG(0)
extern long uffd_wp_range(struct vm_area_struct *vma, unsigned long start, unsigned long len, bool enable_wp);extern int mrwprotect_range(struct userfaultfd_ctx *ctx, unsigned long start, unsigned long len, bool enable_rwp);
/* move_pages */void double_pt_lock(spinlock_t *ptl1, spinlock_t *ptl2);void double_pt_unlock(spinlock_t *ptl1, spinlock_t *ptl2);int move_pages_huge_pmd(struct mm_struct *mm, pmd_t *dst_pmd, pmd_t *src_pmd, pmd_t dst_pmdval, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma, unsigned long dst_addr, unsigned long src_addr);
/* mm helpers */static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, struct vm_userfaultfd_ctx vm_ctx){ return vma->vm_userfaultfd_ctx.ctx == vm_ctx.ctx;}
/* * Never enable huge pmd sharing on some uffd registered vmas: * * - VM_UFFD_WP and VM_UFFD_RWP VMAs, because the write protect / access * tracking information is per pgtable entry. * * - VM_UFFD_MINOR VMAs, because otherwise we would never get minor faults for * VMAs which share huge pmds. (If you have two mappings to the same * underlying pages, and fault in the non-UFFD-registered one with a write, * with huge pmd sharing this would *also* setup the second UFFD-registered * mapping, and we'd not get minor faults.) */static inline bool uffd_disable_huge_pmd_share(struct vm_area_struct *vma){ return vma_test_any_mask(vma, mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP, VMA_UFFD_MINOR));}
/* * Don't do fault around for WP, RWP or MINOR registered uffd range. For * MINOR registered range, fault around will be a total disaster and ptes can * be installed without notifications; for WP it should mostly be fine as long * as the fault around checks for pte_none() before the installation, however * to be super safe we just forbid it; for RWP, pre-faulted neighbours would * be indistinguishable from accessed pages in PAGEMAP_SCAN (PAGE_IS_ACCESSED) * and pollute the tracked working set, so each page must be populated by its * own fault. */static inline bool uffd_disable_fault_around(struct vm_area_struct *vma){ return vma_test_any_mask(vma, mk_vma_flags_from_masks(VMA_UFFD_WP, VMA_UFFD_RWP, VMA_UFFD_MINOR));}
static inline bool userfaultfd_missing(struct vm_area_struct *vma){ return vma_test_any_mask(vma, VMA_UFFD_MISSING);}
static inline bool userfaultfd_wp(struct vm_area_struct *vma){ return vma_test_any_mask(vma, VMA_UFFD_WP);}
static inline bool userfaultfd_minor(struct vm_area_struct *vma){ return vma_test_any_mask(vma, VMA_UFFD_MINOR);}
static inline bool userfaultfd_rwp(struct vm_area_struct *vma){ /* * Callers gate PAGE_NONE usage on this; PAGE_NONE is a BUILD_BUG() * without CONFIG_ARCH_HAS_PTE_PROTNONE, so fold to false. */ if (!IS_ENABLED(CONFIG_ARCH_HAS_PTE_PROTNONE)) return false; return vma_test_single_mask(vma, VMA_UFFD_RWP);}
static inline bool userfaultfd_protected(struct vm_area_struct *vma){ return userfaultfd_wp(vma) || userfaultfd_rwp(vma);}
static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, pte_t pte){ return userfaultfd_wp(vma) && pte_uffd(pte);}
static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, pmd_t pmd){ return userfaultfd_wp(vma) && pmd_uffd(pmd);}
static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma, pte_t pte){ return userfaultfd_rwp(vma) && pte_uffd(pte);}
static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma, pmd_t pmd){ return userfaultfd_rwp(vma) && pmd_uffd(pmd);}
static inline bool userfaultfd_armed(struct vm_area_struct *vma){ return vma_test_any_mask(vma, __VMA_UFFD_FLAGS);}
static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma){ struct userfaultfd_ctx *uffd_ctx = vma->vm_userfaultfd_ctx.ctx;
return uffd_ctx && (uffd_ctx->features & UFFD_FEATURE_EVENT_REMAP) == 0;}
extern int dup_userfaultfd(struct vm_area_struct *, struct list_head *);extern void dup_userfaultfd_complete(struct list_head *);void dup_userfaultfd_fail(struct list_head *);
extern void mremap_userfaultfd_prep(struct vm_area_struct *, struct vm_userfaultfd_ctx *);extern void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *, unsigned long from, unsigned long to, unsigned long len);void mremap_userfaultfd_fail(struct vm_userfaultfd_ctx *);
extern bool userfaultfd_remove(struct vm_area_struct *vma, unsigned long start, unsigned long end);
extern int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start, unsigned long end, struct list_head *uf);extern void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf);extern bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma);extern bool userfaultfd_wp_async(struct vm_area_struct *vma);extern bool userfaultfd_rwp_async(struct vm_area_struct *vma);
static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma){ /* Only wr-protect mode uses pte markers */ if (!userfaultfd_wp(vma)) return false;
/* File-based uffd-wp always need markers */ if (!vma_is_anonymous(vma)) return true;
/* * Anonymous uffd-wp only needs the markers if WP_UNPOPULATED * enabled (to apply markers on zero pages). */ return userfaultfd_wp_unpopulated(vma);}
/* * Returns true if this swap pte carries uffd-tracked state in either * form (pte marker or a normal swap pte), false otherwise. */static inline bool pte_swp_uffd_any(pte_t pte){ if (!uffd_supports_wp_marker()) return false;
if (pte_present(pte)) return false;
if (pte_swp_uffd(pte)) return true;
if (pte_is_uffd_wp_marker(pte)) return true;
return false;}#else /* CONFIG_USERFAULTFD */
/* mm helpers */static inline vm_fault_t handle_userfault(struct vm_fault *vmf, unsigned long reason){ return VM_FAULT_SIGBUS;}
static inline long uffd_wp_range(struct vm_area_struct *vma, unsigned long start, unsigned long len, bool enable_wp){ return false;}
static inline bool is_mergeable_vm_userfaultfd_ctx(struct vm_area_struct *vma, struct vm_userfaultfd_ctx vm_ctx){ return true;}
static inline bool userfaultfd_missing(struct vm_area_struct *vma){ return false;}
static inline bool userfaultfd_wp(struct vm_area_struct *vma){ return false;}
static inline bool userfaultfd_minor(struct vm_area_struct *vma){ return false;}
static inline bool userfaultfd_rwp(struct vm_area_struct *vma){ return false;}
static inline bool userfaultfd_protected(struct vm_area_struct *vma){ return false;}
static inline bool userfaultfd_pte_wp(struct vm_area_struct *vma, pte_t pte){ return false;}
static inline bool userfaultfd_huge_pmd_wp(struct vm_area_struct *vma, pmd_t pmd){ return false;}
static inline bool userfaultfd_pte_rwp(struct vm_area_struct *vma, pte_t pte){ return false;}
static inline bool userfaultfd_huge_pmd_rwp(struct vm_area_struct *vma, pmd_t pmd){ return false;}
static inline bool userfaultfd_armed(struct vm_area_struct *vma){ return false;}
static inline int dup_userfaultfd(struct vm_area_struct *vma, struct list_head *l){ return 0;}
static inline void dup_userfaultfd_complete(struct list_head *l){}
static inline void dup_userfaultfd_fail(struct list_head *l){}
static inline void mremap_userfaultfd_prep(struct vm_area_struct *vma, struct vm_userfaultfd_ctx *ctx){}
static inline void mremap_userfaultfd_complete(struct vm_userfaultfd_ctx *ctx, unsigned long from, unsigned long to, unsigned long len){}
static inline void mremap_userfaultfd_fail(struct vm_userfaultfd_ctx *ctx){}
static inline bool userfaultfd_remove(struct vm_area_struct *vma, unsigned long start, unsigned long end){ return true;}
static inline int userfaultfd_unmap_prep(struct vm_area_struct *vma, unsigned long start, unsigned long end, struct list_head *uf){ return 0;}
static inline void userfaultfd_unmap_complete(struct mm_struct *mm, struct list_head *uf){}
static inline bool uffd_disable_fault_around(struct vm_area_struct *vma){ return false;}
static inline bool userfaultfd_wp_unpopulated(struct vm_area_struct *vma){ return false;}
static inline bool userfaultfd_wp_async(struct vm_area_struct *vma){ return false;}
static inline bool userfaultfd_rwp_async(struct vm_area_struct *vma){ return false;}
static inline bool vma_has_uffd_without_event_remap(struct vm_area_struct *vma){ return false;}
static inline bool userfaultfd_wp_use_markers(struct vm_area_struct *vma){ return false;}
/* * Returns true if this swap pte carries uffd-tracked state in either * form (pte marker or a normal swap pte), false otherwise. */static inline bool pte_swp_uffd_any(pte_t pte){ return false;}#endif /* CONFIG_USERFAULTFD */#endif /* _LINUX_USERFAULTFD_K_H */