Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
8.0 kB · 315 lines
C
at master
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316/* SPDX-License-Identifier: GPL-2.0 */#ifndef __ASM_GENERIC_PGALLOC_H#define __ASM_GENERIC_PGALLOC_H
#ifdef CONFIG_MMU
#define GFP_PGTABLE_KERNEL (GFP_KERNEL | __GFP_ZERO | __GFP_SKIP_KASAN)#define GFP_PGTABLE_USER (GFP_PGTABLE_KERNEL | __GFP_ACCOUNT)
/** * __pte_alloc_one_kernel - allocate memory for a PTE-level kernel page table * @mm: the mm_struct of the current context * * This function is intended for architectures that need * anything beyond simple page allocation. * * Return: pointer to the allocated memory or %NULL on error */static inline pte_t *__pte_alloc_one_kernel_noprof(struct mm_struct *mm){ struct ptdesc *ptdesc = pagetable_alloc_noprof(GFP_PGTABLE_KERNEL, 0);
if (!ptdesc) return NULL; if (!pagetable_pte_ctor(mm, ptdesc)) { pagetable_free(ptdesc); return NULL; }
ptdesc_set_kernel(ptdesc);
return ptdesc_address(ptdesc);}#define __pte_alloc_one_kernel(...) alloc_hooks(__pte_alloc_one_kernel_noprof(__VA_ARGS__))
#ifndef __HAVE_ARCH_PTE_ALLOC_ONE_KERNEL/** * pte_alloc_one_kernel - allocate memory for a PTE-level kernel page table * @mm: the mm_struct of the current context * * Return: pointer to the allocated memory or %NULL on error */static inline pte_t *pte_alloc_one_kernel_noprof(struct mm_struct *mm){ return __pte_alloc_one_kernel_noprof(mm);}#define pte_alloc_one_kernel(...) alloc_hooks(pte_alloc_one_kernel_noprof(__VA_ARGS__))#endif
/** * pte_free_kernel - free PTE-level kernel page table memory * @mm: the mm_struct of the current context * @pte: pointer to the memory containing the page table */static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte){ pagetable_dtor_free(virt_to_ptdesc(pte));}
/** * __pte_alloc_one - allocate memory for a PTE-level user page table * @mm: the mm_struct of the current context * @gfp: GFP flags to use for the allocation * * Allocate memory for a page table and ptdesc and runs pagetable_pte_ctor(). * * This function is intended for architectures that need * anything beyond simple page allocation or must have custom GFP flags. * * Return: `struct page` referencing the ptdesc or %NULL on error */static inline pgtable_t __pte_alloc_one_noprof(struct mm_struct *mm, gfp_t gfp){ struct ptdesc *ptdesc;
ptdesc = pagetable_alloc_noprof(gfp, 0); if (!ptdesc) return NULL; if (!pagetable_pte_ctor(mm, ptdesc)) { pagetable_free(ptdesc); return NULL; }
return ptdesc_page(ptdesc);}#define __pte_alloc_one(...) alloc_hooks(__pte_alloc_one_noprof(__VA_ARGS__))
#ifndef __HAVE_ARCH_PTE_ALLOC_ONE/** * pte_alloc_one - allocate a page for PTE-level user page table * @mm: the mm_struct of the current context * * Allocate memory for a page table and ptdesc and runs pagetable_pte_ctor(). * * Return: `struct page` referencing the ptdesc or %NULL on error */static inline pgtable_t pte_alloc_one_noprof(struct mm_struct *mm){ return __pte_alloc_one_noprof(mm, GFP_PGTABLE_USER);}#define pte_alloc_one(...) alloc_hooks(pte_alloc_one_noprof(__VA_ARGS__))#endif
/* * Should really implement gc for free page table pages. This could be * done with a reference count in struct page. */
/** * pte_free - free PTE-level user page table memory * @mm: the mm_struct of the current context * @pte_page: the `struct page` referencing the ptdesc */static inline void pte_free(struct mm_struct *mm, struct page *pte_page){ struct ptdesc *ptdesc = page_ptdesc(pte_page);
pagetable_dtor_free(ptdesc);}
#if CONFIG_PGTABLE_LEVELS > 2
#ifndef __HAVE_ARCH_PMD_ALLOC_ONE/** * pmd_alloc_one - allocate memory for a PMD-level page table * @mm: the mm_struct of the current context * * Allocate memory for a page table and ptdesc and runs pagetable_pmd_ctor(). * * Allocations use %GFP_PGTABLE_USER in user context and * %GFP_PGTABLE_KERNEL in kernel context. * * Return: pointer to the allocated memory or %NULL on error */static inline pmd_t *pmd_alloc_one_noprof(struct mm_struct *mm, unsigned long addr){ struct ptdesc *ptdesc; gfp_t gfp = GFP_PGTABLE_USER;
if (mm == &init_mm) gfp = GFP_PGTABLE_KERNEL; ptdesc = pagetable_alloc_noprof(gfp, 0); if (!ptdesc) return NULL; if (!pagetable_pmd_ctor(mm, ptdesc)) { pagetable_free(ptdesc); return NULL; }
if (mm == &init_mm) ptdesc_set_kernel(ptdesc);
return ptdesc_address(ptdesc);}#define pmd_alloc_one(...) alloc_hooks(pmd_alloc_one_noprof(__VA_ARGS__))#endif
#ifndef __HAVE_ARCH_PMD_FREEstatic inline void pmd_free(struct mm_struct *mm, pmd_t *pmd){ struct ptdesc *ptdesc = virt_to_ptdesc(pmd);
BUG_ON((unsigned long)pmd & (PAGE_SIZE-1)); pagetable_dtor_free(ptdesc);}#endif
#endif /* CONFIG_PGTABLE_LEVELS > 2 */
#if CONFIG_PGTABLE_LEVELS > 3
static inline pud_t *__pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr){ gfp_t gfp = GFP_PGTABLE_USER; struct ptdesc *ptdesc;
if (mm == &init_mm) gfp = GFP_PGTABLE_KERNEL;
ptdesc = pagetable_alloc_noprof(gfp, 0); if (!ptdesc) return NULL;
pagetable_pud_ctor(ptdesc);
if (mm == &init_mm) ptdesc_set_kernel(ptdesc);
return ptdesc_address(ptdesc);}#define __pud_alloc_one(...) alloc_hooks(__pud_alloc_one_noprof(__VA_ARGS__))
#ifndef __HAVE_ARCH_PUD_ALLOC_ONE/** * pud_alloc_one - allocate memory for a PUD-level page table * @mm: the mm_struct of the current context * * Allocate memory for a page table using %GFP_PGTABLE_USER for user context * and %GFP_PGTABLE_KERNEL for kernel context. * * Return: pointer to the allocated memory or %NULL on error */static inline pud_t *pud_alloc_one_noprof(struct mm_struct *mm, unsigned long addr){ return __pud_alloc_one_noprof(mm, addr);}#define pud_alloc_one(...) alloc_hooks(pud_alloc_one_noprof(__VA_ARGS__))#endif
static inline void __pud_free(struct mm_struct *mm, pud_t *pud){ struct ptdesc *ptdesc = virt_to_ptdesc(pud);
BUG_ON((unsigned long)pud & (PAGE_SIZE-1)); pagetable_dtor_free(ptdesc);}
#ifndef __HAVE_ARCH_PUD_FREEstatic inline void pud_free(struct mm_struct *mm, pud_t *pud){ __pud_free(mm, pud);}#endif
#endif /* CONFIG_PGTABLE_LEVELS > 3 */
#if CONFIG_PGTABLE_LEVELS > 4
static inline p4d_t *__p4d_alloc_one_noprof(struct mm_struct *mm, unsigned long addr){ gfp_t gfp = GFP_PGTABLE_USER; struct ptdesc *ptdesc;
if (mm == &init_mm) gfp = GFP_PGTABLE_KERNEL;
ptdesc = pagetable_alloc_noprof(gfp, 0); if (!ptdesc) return NULL;
pagetable_p4d_ctor(ptdesc);
if (mm == &init_mm) ptdesc_set_kernel(ptdesc);
return ptdesc_address(ptdesc);}#define __p4d_alloc_one(...) alloc_hooks(__p4d_alloc_one_noprof(__VA_ARGS__))
#ifndef __HAVE_ARCH_P4D_ALLOC_ONEstatic inline p4d_t *p4d_alloc_one_noprof(struct mm_struct *mm, unsigned long addr){ return __p4d_alloc_one_noprof(mm, addr);}#define p4d_alloc_one(...) alloc_hooks(p4d_alloc_one_noprof(__VA_ARGS__))#endif
static inline void __p4d_free(struct mm_struct *mm, p4d_t *p4d){ struct ptdesc *ptdesc = virt_to_ptdesc(p4d);
BUG_ON((unsigned long)p4d & (PAGE_SIZE-1)); pagetable_dtor_free(ptdesc);}
#ifndef __HAVE_ARCH_P4D_FREEstatic inline void p4d_free(struct mm_struct *mm, p4d_t *p4d){ if (!mm_p4d_folded(mm)) __p4d_free(mm, p4d);}#endif
#endif /* CONFIG_PGTABLE_LEVELS > 4 */
static inline pgd_t *__pgd_alloc_noprof(struct mm_struct *mm, unsigned int order){ gfp_t gfp = GFP_PGTABLE_USER; struct ptdesc *ptdesc;
if (mm == &init_mm) gfp = GFP_PGTABLE_KERNEL;
ptdesc = pagetable_alloc_noprof(gfp, order); if (!ptdesc) return NULL;
pagetable_pgd_ctor(ptdesc);
if (mm == &init_mm) ptdesc_set_kernel(ptdesc);
return ptdesc_address(ptdesc);}#define __pgd_alloc(...) alloc_hooks(__pgd_alloc_noprof(__VA_ARGS__))
static inline void __pgd_free(struct mm_struct *mm, pgd_t *pgd){ struct ptdesc *ptdesc = virt_to_ptdesc(pgd);
BUG_ON((unsigned long)pgd & (PAGE_SIZE-1)); pagetable_dtor_free(ptdesc);}
#ifndef __HAVE_ARCH_PGD_FREEstatic inline void pgd_free(struct mm_struct *mm, pgd_t *pgd){ __pgd_free(mm, pgd);}#endif
#endif /* CONFIG_MMU */
#endif /* __ASM_GENERIC_PGALLOC_H */