Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834/* SPDX-License-Identifier: GPL-2.0 */#ifndef _LINUX_HUGE_MM_H#define _LINUX_HUGE_MM_H
#include <linux/mm_types.h>
#include <linux/fs.h> /* only for vma_is_dax() */#include <linux/kobject.h>
vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf);int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, struct vm_area_struct *dst_vma, struct vm_area_struct *src_vma);bool huge_pmd_set_accessed(struct vm_fault *vmf);int copy_huge_pud(struct mm_struct *dst_mm, struct mm_struct *src_mm, pud_t *dst_pud, pud_t *src_pud, unsigned long addr, struct vm_area_struct *vma);
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUDvoid huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud);#elsestatic inline void huge_pud_set_accessed(struct vm_fault *vmf, pud_t orig_pud){}#endif
vm_fault_t do_huge_pmd_wp_page(struct vm_fault *vmf);bool madvise_free_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long next);bool zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr);int zap_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pud, unsigned long addr);bool move_huge_pmd(struct vm_area_struct *vma, unsigned long old_addr, unsigned long new_addr, pmd_t *old_pmd, pmd_t *new_pmd);int change_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, pgprot_t newprot, unsigned long cp_flags);
vm_fault_t vmf_insert_pfn_pmd(struct vm_fault *vmf, unsigned long pfn, bool write);vm_fault_t vmf_insert_pfn_pud(struct vm_fault *vmf, unsigned long pfn, bool write);vm_fault_t vmf_insert_folio_pmd(struct vm_fault *vmf, struct folio *folio, bool write);vm_fault_t vmf_insert_folio_pud(struct vm_fault *vmf, struct folio *folio, bool write);
enum transparent_hugepage_flag { TRANSPARENT_HUGEPAGE_UNSUPPORTED, TRANSPARENT_HUGEPAGE_FLAG, TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_DIRECT_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_KSWAPD_OR_MADV_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG, TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG,};
struct kobject;struct kobj_attribute;
ssize_t single_hugepage_flag_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count, enum transparent_hugepage_flag flag);ssize_t single_hugepage_flag_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf, enum transparent_hugepage_flag flag);extern struct kobj_attribute shmem_enabled_attr;extern struct kobj_attribute thpsize_shmem_enabled_attr;
/* * Mask of all large folio orders supported for anonymous THP; all orders up to * and including PMD_ORDER, except order-0 (which is not "huge") and order-1 * (which is a limitation of the THP implementation). */#define THP_ORDERS_ALL_ANON ((BIT(PMD_ORDER + 1) - 1) & ~(BIT(0) | BIT(1)))
/* * Mask of all large folio orders supported for file THP. Folios in a DAX * file is never split and the MAX_PAGECACHE_ORDER limit does not apply to * it. Same to PFNMAPs where there's neither page* nor pagecache. */#define THP_ORDERS_ALL_SPECIAL_DAX \ (BIT(PMD_ORDER) | BIT(PUD_ORDER))#define THP_ORDERS_ALL_FILE_DEFAULT \ ((BIT(MAX_PAGECACHE_ORDER + 1) - 1) & ~BIT(0))
/* * Mask of all large folio orders supported for THP. */#define THP_ORDERS_ALL \ (THP_ORDERS_ALL_ANON | THP_ORDERS_ALL_SPECIAL_DAX | THP_ORDERS_ALL_FILE_DEFAULT)
enum tva_type { TVA_SMAPS, /* Exposing "THPeligible:" in smaps. */ TVA_PAGEFAULT, /* Serving a page fault. */ TVA_KHUGEPAGED, /* Khugepaged collapse. */ TVA_FORCED_COLLAPSE, /* Forced collapse (e.g. MADV_COLLAPSE). */};
#define thp_vma_allowable_order(vma, vm_flags, type, order) \ (!!thp_vma_allowable_orders(vma, vm_flags, type, BIT(order)))
#define split_folio(f) split_folio_to_list(f, NULL)
#ifdef CONFIG_PGTABLE_HAS_HUGE_LEAVES#define HPAGE_PMD_SHIFT PMD_SHIFT#define HPAGE_PUD_SHIFT PUD_SHIFT#else#define HPAGE_PMD_SHIFT ({ BUILD_BUG(); 0; })#define HPAGE_PUD_SHIFT ({ BUILD_BUG(); 0; })#endif
#define HPAGE_PMD_ORDER (HPAGE_PMD_SHIFT-PAGE_SHIFT)#define HPAGE_PMD_NR (1<<HPAGE_PMD_ORDER)#define HPAGE_PMD_MASK (~(HPAGE_PMD_SIZE - 1))#define HPAGE_PMD_SIZE ((1UL) << HPAGE_PMD_SHIFT)
#define HPAGE_PUD_ORDER (HPAGE_PUD_SHIFT-PAGE_SHIFT)#define HPAGE_PUD_NR (1<<HPAGE_PUD_ORDER)#define HPAGE_PUD_MASK (~(HPAGE_PUD_SIZE - 1))#define HPAGE_PUD_SIZE ((1UL) << HPAGE_PUD_SHIFT)
enum mthp_stat_item { MTHP_STAT_ANON_FAULT_ALLOC, MTHP_STAT_ANON_FAULT_FALLBACK, MTHP_STAT_ANON_FAULT_FALLBACK_CHARGE, MTHP_STAT_COLLAPSE_ALLOC, MTHP_STAT_COLLAPSE_ALLOC_FAILED, MTHP_STAT_ZSWPOUT, MTHP_STAT_SWPIN, MTHP_STAT_SWPIN_FALLBACK, MTHP_STAT_SWPIN_FALLBACK_CHARGE, MTHP_STAT_SWPOUT, MTHP_STAT_SWPOUT_FALLBACK, MTHP_STAT_SHMEM_ALLOC, MTHP_STAT_SHMEM_FALLBACK, MTHP_STAT_SHMEM_FALLBACK_CHARGE, MTHP_STAT_SPLIT, MTHP_STAT_SPLIT_FAILED, MTHP_STAT_SPLIT_DEFERRED, MTHP_STAT_NR_ANON, MTHP_STAT_NR_ANON_PARTIALLY_MAPPED, MTHP_STAT_COLLAPSE_EXCEED_SWAP, MTHP_STAT_COLLAPSE_EXCEED_NONE, MTHP_STAT_COLLAPSE_EXCEED_SHARED, __MTHP_STAT_COUNT};
#if defined(CONFIG_TRANSPARENT_HUGEPAGE) && defined(CONFIG_SYSFS)struct mthp_stat { unsigned long stats[ilog2(MAX_PTRS_PER_PTE) + 1][__MTHP_STAT_COUNT];};
DECLARE_PER_CPU(struct mthp_stat, mthp_stats);
static inline void mod_mthp_stat(int order, enum mthp_stat_item item, int delta){ if (order <= 0 || order > PMD_ORDER) return;
this_cpu_add(mthp_stats.stats[order][item], delta);}
static inline void count_mthp_stat(int order, enum mthp_stat_item item){ mod_mthp_stat(order, item, 1);}
#elsestatic inline void mod_mthp_stat(int order, enum mthp_stat_item item, int delta){}
static inline void count_mthp_stat(int order, enum mthp_stat_item item){}#endif
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
extern unsigned long transparent_hugepage_flags;extern unsigned long huge_anon_orders_always;extern unsigned long huge_anon_orders_madvise;extern unsigned long huge_anon_orders_inherit;
static inline bool hugepage_global_enabled(void){ return transparent_hugepage_flags & ((1<<TRANSPARENT_HUGEPAGE_FLAG) | (1<<TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG));}
static inline bool hugepage_global_always(void){ return transparent_hugepage_flags & (1<<TRANSPARENT_HUGEPAGE_FLAG);}
static inline int highest_order(unsigned long orders){ return fls_long(orders) - 1;}
static inline int next_order(unsigned long *orders, int prev){ *orders &= ~BIT(prev); return highest_order(*orders);}
/* * Do the below checks: * - For file vma, check if the linear page offset of vma is * order-aligned within the file. The hugepage is * guaranteed to be order-aligned within the file, but we must * check that the order-aligned addresses in the VMA map to * order-aligned offsets within the file, else the hugepage will * not be mappable. * - For all vmas, check if the haddr is in an aligned hugepage * area. */static inline bool thp_vma_suitable_order(struct vm_area_struct *vma, unsigned long addr, int order){ unsigned long hpage_size = PAGE_SIZE << order; unsigned long haddr;
/* Don't have to check pgoff for anonymous vma */ if (!vma_is_anonymous(vma)) { if (!IS_ALIGNED((vma->vm_start >> PAGE_SHIFT) - vma->vm_pgoff, hpage_size >> PAGE_SHIFT)) return false; }
haddr = ALIGN_DOWN(addr, hpage_size);
if (haddr < vma->vm_start || haddr + hpage_size > vma->vm_end) return false; return true;}
/* * Make sure huge_gfp is always more limited than limit_gfp. * Some shmem users want THP allocation to be done less aggressively * and only in certain zone. */static inline gfp_t thp_shmem_limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp){ gfp_t allowflags = __GFP_IO | __GFP_FS | __GFP_RECLAIM; gfp_t denyflags = __GFP_NOWARN | __GFP_NORETRY; gfp_t zoneflags = limit_gfp & GFP_ZONEMASK; gfp_t result = huge_gfp & ~(allowflags | GFP_ZONEMASK);
/* Allow allocations only from the originally specified zones. */ result |= zoneflags;
/* * Minimize the result gfp by taking the union with the deny flags, * and the intersection of the allow flags. */ result |= (limit_gfp & denyflags); result |= (huge_gfp & limit_gfp) & allowflags;
return result;}
/* * Filter the bitfield of input orders to the ones suitable for use in the vma. * See thp_vma_suitable_order(). * All orders that pass the checks are returned as a bitfield. */static inline unsigned long thp_vma_suitable_orders(struct vm_area_struct *vma, unsigned long addr, unsigned long orders){ int order;
/* * Iterate over orders, highest to lowest, removing orders that don't * meet alignment requirements from the set. Exit loop at first order * that meets requirements, since all lower orders must also meet * requirements. */
order = highest_order(orders);
while (orders) { if (thp_vma_suitable_order(vma, addr, order)) break; order = next_order(&orders, order); }
return orders;}
unsigned long __thp_vma_allowable_orders(struct vm_area_struct *vma, vm_flags_t vm_flags, enum tva_type type, unsigned long orders);
/** * thp_vma_allowable_orders - determine hugepage orders that are allowed for vma * @vma: the vm area to check * @vm_flags: use these vm_flags instead of vma->vm_flags * @type: TVA type * @orders: bitfield of all orders to consider * * Calculates the intersection of the requested hugepage orders and the allowed * hugepage orders for the provided vma. Permitted orders are encoded as a set * bit at the corresponding bit position (bit-2 corresponds to order-2, bit-3 * corresponds to order-3, etc). Order-0 is never considered a hugepage order. * * Return: bitfield of orders allowed for hugepage in the vma. 0 if no hugepage * orders are allowed. */static inlineunsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, vm_flags_t vm_flags, enum tva_type type, unsigned long orders){ /* * Optimization to check if required orders are enabled early. Only * forced collapse ignores sysfs configs. */ if (type != TVA_FORCED_COLLAPSE && vma_is_anonymous(vma)) { unsigned long mask = READ_ONCE(huge_anon_orders_always);
if (vm_flags & VM_HUGEPAGE) mask |= READ_ONCE(huge_anon_orders_madvise); if (hugepage_global_always() || ((vm_flags & VM_HUGEPAGE) && hugepage_global_enabled())) mask |= READ_ONCE(huge_anon_orders_inherit);
orders &= mask; if (!orders) return 0; }
return __thp_vma_allowable_orders(vma, vm_flags, type, orders);}
struct thpsize { struct kobject kobj; struct list_head node; int order;};
#define to_thpsize(kobj) container_of(kobj, struct thpsize, kobj)
#define transparent_hugepage_use_zero_page() \ (transparent_hugepage_flags & \ (1<<TRANSPARENT_HUGEPAGE_USE_ZERO_PAGE_FLAG))
/* * Check whether THPs are explicitly disabled for this VMA, for example, * through madvise or prctl. */static inline bool vma_thp_disabled(struct vm_area_struct *vma, vm_flags_t vm_flags, bool forced_collapse){ /* Are THPs disabled for this VMA? */ if (vm_flags & VM_NOHUGEPAGE) return true; /* Are THPs disabled for all VMAs in the whole process? */ if (mm_flags_test(MMF_DISABLE_THP_COMPLETELY, vma->vm_mm)) return true; /* * Are THPs disabled only for VMAs where we didn't get an explicit * advise to use them? */ if (vm_flags & VM_HUGEPAGE) return false; /* * Forcing a collapse (e.g., madv_collapse), is a clear advice to * use THPs. */ if (forced_collapse) return false; return mm_flags_test(MMF_DISABLE_THP_EXCEPT_ADVISED, vma->vm_mm);}
static inline bool thp_disabled_by_hw(void){ /* If the hardware/firmware marked hugepage support disabled. */ return transparent_hugepage_flags & (1 << TRANSPARENT_HUGEPAGE_UNSUPPORTED);}
unsigned long thp_get_unmapped_area(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags);unsigned long thp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags);
enum split_type { SPLIT_TYPE_UNIFORM, SPLIT_TYPE_NON_UNIFORM,};
int __split_huge_page_to_list_to_order(struct page *page, struct list_head *list, unsigned int new_order);int folio_split_unmapped(struct folio *folio, unsigned int new_order);unsigned int min_order_for_split(struct folio *folio);int split_folio_to_list(struct folio *folio, struct list_head *list);int folio_check_splittable(struct folio *folio, unsigned int new_order, enum split_type split_type);int folio_split(struct folio *folio, unsigned int new_order, struct page *page, struct list_head *list);
static inline int split_huge_page_to_list_to_order(struct page *page, struct list_head *list, unsigned int new_order){ return __split_huge_page_to_list_to_order(page, list, new_order);}static inline int split_huge_page_to_order(struct page *page, unsigned int new_order){ return split_huge_page_to_list_to_order(page, NULL, new_order);}
static inline int split_huge_page(struct page *page){ return split_huge_page_to_list_to_order(page, NULL, 0);}
int folio_memcg_alloc_deferred(struct folio *folio);
void deferred_split_folio(struct folio *folio, bool partially_mapped);
void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, unsigned long address, bool freeze);
/** * pmd_is_huge() - Is this PMD either a huge PMD entry or a software leaf entry? * @pmd: The PMD to check. * * A huge PMD entry is a non-empty entry which is present and marked huge or a * software leaf entry. This check be performed without the appropriate locks * held, in which case the condition should be rechecked after they are * acquired. * * Returns: true if this PMD is huge, false otherwise. */static inline bool pmd_is_huge(pmd_t pmd){ if (pmd_present(pmd)) { return pmd_trans_huge(pmd); } else if (!pmd_none(pmd)) { /* * Non-present PMDs must be valid huge non-present entries. We * cannot assert that here due to header dependency issues. */ return true; }
return false;}
#define split_huge_pmd(__vma, __pmd, __address) \ do { \ pmd_t *____pmd = (__pmd); \ if (pmd_is_huge(*____pmd)) \ __split_huge_pmd(__vma, __pmd, __address, \ false); \ } while (0)
void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, bool freeze);
void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, unsigned long address);
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUDint change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pudp, unsigned long addr, pgprot_t newprot, unsigned long cp_flags);#elsestatic inline intchange_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pudp, unsigned long addr, pgprot_t newprot, unsigned long cp_flags) { return 0; }#endif
#define split_huge_pud(__vma, __pud, __address) \ do { \ pud_t *____pud = (__pud); \ if (pud_trans_huge(*____pud)) \ __split_huge_pud(__vma, __pud, __address); \ } while (0)
int hugepage_madvise(struct vm_area_struct *vma, vm_flags_t *vm_flags, int advice);int madvise_collapse(struct vm_area_struct *vma, unsigned long start, unsigned long end, bool *lock_dropped);void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, unsigned long end, struct vm_area_struct *next);spinlock_t *__pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma);spinlock_t *__pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma);
/* mmap_lock must be held on entry */static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma){ if (pmd_is_huge(*pmd)) return __pmd_trans_huge_lock(pmd, vma);
return NULL;}static inline spinlock_t *pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma){ if (pud_trans_huge(*pud)) return __pud_trans_huge_lock(pud, vma); else return NULL;}
/** * folio_test_pmd_mappable - Can we map this folio with a PMD? * @folio: The folio to test * * Return: true - @folio can be mapped, false - @folio cannot be mapped. */static inline bool folio_test_pmd_mappable(struct folio *folio){ return folio_order(folio) >= HPAGE_PMD_ORDER;}
vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf);
vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf);
extern struct folio *huge_zero_folio;extern unsigned long huge_zero_pfn;
static inline bool is_huge_zero_folio(const struct folio *folio){ VM_WARN_ON_ONCE(!folio);
return READ_ONCE(huge_zero_folio) == folio;}
static inline bool is_huge_zero_pfn(unsigned long pfn){ return READ_ONCE(huge_zero_pfn) == (pfn & ~(HPAGE_PMD_NR - 1));}
static inline bool is_huge_zero_pmd(pmd_t pmd){ return pmd_present(pmd) && is_huge_zero_pfn(pmd_pfn(pmd));}
struct folio *mm_get_huge_zero_folio(struct mm_struct *mm);void mm_put_huge_zero_folio(struct mm_struct *mm);
static inline struct folio *get_persistent_huge_zero_folio(void){ if (!IS_ENABLED(CONFIG_PERSISTENT_HUGE_ZERO_FOLIO)) return NULL;
if (unlikely(!huge_zero_folio)) return NULL;
return huge_zero_folio;}
static inline bool thp_migration_supported(void){ return IS_ENABLED(CONFIG_ARCH_ENABLE_THP_MIGRATION);}
void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, pmd_t *pmd, bool freeze);bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp, struct folio *folio);void map_anon_folio_pmd_nopf(struct folio *folio, pmd_t *pmd, struct vm_area_struct *vma, unsigned long haddr);
#else /* CONFIG_TRANSPARENT_HUGEPAGE */
static inline bool folio_test_pmd_mappable(struct folio *folio){ return false;}
static inline bool thp_vma_suitable_order(struct vm_area_struct *vma, unsigned long addr, int order){ return false;}
static inline gfp_t thp_shmem_limit_gfp_mask(gfp_t huge_gfp, gfp_t limit_gfp){ return huge_gfp;}
static inline unsigned long thp_vma_suitable_orders(struct vm_area_struct *vma, unsigned long addr, unsigned long orders){ return 0;}
static inline unsigned long thp_vma_allowable_orders(struct vm_area_struct *vma, vm_flags_t vm_flags, enum tva_type type, unsigned long orders){ return 0;}
#define transparent_hugepage_flags 0UL
#define thp_get_unmapped_area NULL
static inline unsigned longthp_get_unmapped_area_vmflags(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags, vm_flags_t vm_flags){ return 0;}
static inline boolcan_split_folio(struct folio *folio, int caller_pins, int *pextra_pins){ return false;}static inline intsplit_huge_page_to_list_to_order(struct page *page, struct list_head *list, unsigned int new_order){ VM_WARN_ON_ONCE_PAGE(1, page); return -EINVAL;}static inline int split_huge_page_to_order(struct page *page, unsigned int new_order){ VM_WARN_ON_ONCE_PAGE(1, page); return -EINVAL;}static inline int split_huge_page(struct page *page){ VM_WARN_ON_ONCE_PAGE(1, page); return -EINVAL;}
static inline unsigned int min_order_for_split(struct folio *folio){ VM_WARN_ON_ONCE_FOLIO(1, folio); return 0;}
static inline int split_folio_to_list(struct folio *folio, struct list_head *list){ VM_WARN_ON_ONCE_FOLIO(1, folio); return -EINVAL;}
static inline int folio_split(struct folio *folio, unsigned int new_order, struct page *page, struct list_head *list){ VM_WARN_ON_ONCE_FOLIO(1, folio); return -EINVAL;}
static inline int folio_memcg_alloc_deferred(struct folio *folio){ return 0;}
static inline void deferred_split_folio(struct folio *folio, bool partially_mapped){}
#define split_huge_pmd(__vma, __pmd, __address) \ do { } while (0)
static inline void __split_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, unsigned long address, bool freeze) {}static inline void split_huge_pmd_address(struct vm_area_struct *vma, unsigned long address, bool freeze) {}static inline void split_huge_pmd_locked(struct vm_area_struct *vma, unsigned long address, pmd_t *pmd, bool freeze) {}
static inline bool unmap_huge_pmd_locked(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp, struct folio *folio){ return false;}
#define split_huge_pud(__vma, __pmd, __address) \ do { } while (0)
static inline int hugepage_madvise(struct vm_area_struct *vma, vm_flags_t *vm_flags, int advice){ return -EINVAL;}
static inline int madvise_collapse(struct vm_area_struct *vma, unsigned long start, unsigned long end, bool *lock_dropped){ return -EINVAL;}
static inline void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, unsigned long end, struct vm_area_struct *next){}static inline spinlock_t *pmd_trans_huge_lock(pmd_t *pmd, struct vm_area_struct *vma){ return NULL;}static inline spinlock_t *pud_trans_huge_lock(pud_t *pud, struct vm_area_struct *vma){ return NULL;}
static inline vm_fault_t do_huge_pmd_numa_page(struct vm_fault *vmf){ return 0;}
static inline vm_fault_t do_huge_pmd_device_private(struct vm_fault *vmf){ return 0;}
static inline bool is_huge_zero_folio(const struct folio *folio){ return false;}
static inline bool is_huge_zero_pfn(unsigned long pfn){ return false;}
static inline bool is_huge_zero_pmd(pmd_t pmd){ return false;}
static inline void mm_put_huge_zero_folio(struct mm_struct *mm){ return;}
static inline bool thp_migration_supported(void){ return false;}
static inline int highest_order(unsigned long orders){ return 0;}
static inline int next_order(unsigned long *orders, int prev){ return 0;}
static inline void __split_huge_pud(struct vm_area_struct *vma, pud_t *pud, unsigned long address){}
static inline int change_huge_pud(struct mmu_gather *tlb, struct vm_area_struct *vma, pud_t *pudp, unsigned long addr, pgprot_t newprot, unsigned long cp_flags){ return 0;}
static inline struct folio *get_persistent_huge_zero_folio(void){ return NULL;}
static inline bool pmd_is_huge(pmd_t pmd){ return false;}#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
static inline bool is_pmd_order(unsigned int order){ return order == HPAGE_PMD_ORDER;}
static inline int split_folio_to_list_to_order(struct folio *folio, struct list_head *list, int new_order){ return split_huge_page_to_list_to_order(&folio->page, list, new_order);}
static inline int split_folio_to_order(struct folio *folio, int new_order){ return split_folio_to_list_to_order(folio, NULL, new_order);}
/** * largest_zero_folio - Get the largest zero size folio available * * This function shall be used when mm_get_huge_zero_folio() cannot be * used as there is no appropriate mm lifetime to tie the huge zero folio * from the caller. * * Deduce the size of the folio with folio_size instead of assuming the * folio size. * * Return: pointer to PMD sized zero folio if CONFIG_PERSISTENT_HUGE_ZERO_FOLIO * is enabled or a single page sized zero folio */static inline struct folio *largest_zero_folio(void){ struct folio *folio = get_persistent_huge_zero_folio();
if (folio) return folio;
return page_folio(ZERO_PAGE(0));}#endif /* _LINUX_HUGE_MM_H */