tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
fork atom

mm: remove quicklist page table caches

Patch series "mm: remove quicklist page table caches".

A while ago Nicholas proposed to remove quicklist page table caches [1].

I've rebased his patch on the curren upstream and switched ia64 and sh to
use generic versions of PTE allocation.

[1] https://lore.kernel.org/linux-mm/20190711030339.20892-1-npiggin@gmail.com

This patch (of 3):

Remove page table allocator "quicklists". These have been around for a
long time, but have not got much traction in the last decade and are only
used on ia64 and sh architectures.

The numbers in the initial commit look interesting but probably don't
apply anymore. If anybody wants to resurrect this it's in the git
history, but it's unhelpful to have this code and divergent allocator
behaviour for minor archs.

Also it might be better to instead make more general improvements to page
allocator if this is still so slow.

Link: http://lkml.kernel.org/r/1565250728-21721-2-git-send-email-rppt@linux.ibm.com
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Mike Rapoport <rppt@linux.ibm.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

authored by

Nicholas Piggin and committed by
Linus Torvalds 13224794 7b167b68

+25 -395
-2
arch/alpha/include/asm/pgalloc.h
··· 53 53 free_page((unsigned long)pmd); 54 54 } 55 55 56 - #define check_pgt_cache() do { } while (0) 57 - 58 56 #endif /* _ALPHA_PGALLOC_H */
-1
arch/arc/include/asm/pgalloc.h
··· 129 129 130 130 #define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) 131 131 132 - #define check_pgt_cache() do { } while (0) 133 132 #define pmd_pgtable(pmd) ((pgtable_t) pmd_page_vaddr(pmd)) 134 133 135 134 #endif /* _ASM_ARC_PGALLOC_H */
-2
arch/arm/include/asm/pgalloc.h
··· 15 15 #include <asm/cacheflush.h> 16 16 #include <asm/tlbflush.h> 17 17 18 - #define check_pgt_cache() do { } while (0) 19 - 20 18 #ifdef CONFIG_MMU 21 19 22 20 #define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_USER))
-2
arch/arm64/include/asm/pgalloc.h
··· 15 15 16 16 #include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */ 17 17 18 - #define check_pgt_cache() do { } while (0) 19 - 20 18 #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) 21 19 22 20 #if CONFIG_PGTABLE_LEVELS > 2
-2
arch/csky/include/asm/pgalloc.h
··· 75 75 tlb_remove_page(tlb, pte); \ 76 76 } while (0) 77 77 78 - #define check_pgt_cache() do {} while (0) 79 - 80 78 extern void pagetable_init(void); 81 79 extern void pre_mmu_init(void); 82 80 extern void pre_trap_init(void);
-2
arch/hexagon/include/asm/pgalloc.h
··· 13 13 14 14 #include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */ 15 15 16 - #define check_pgt_cache() do {} while (0) 17 - 18 16 extern unsigned long long kmap_generation; 19 17 20 18 /*
-4
arch/ia64/Kconfig
··· 72 72 config ZONE_DMA32 73 73 def_bool y 74 74 75 - config QUICKLIST 76 - bool 77 - default y 78 - 79 75 config MMU 80 76 bool 81 77 default y
+12 -20
arch/ia64/include/asm/pgalloc.h
··· 19 19 #include <linux/mm.h> 20 20 #include <linux/page-flags.h> 21 21 #include <linux/threads.h> 22 - #include <linux/quicklist.h> 23 22 24 23 #include <asm/mmu_context.h> 25 24 26 25 static inline pgd_t *pgd_alloc(struct mm_struct *mm) 27 26 { 28 - return quicklist_alloc(0, GFP_KERNEL, NULL); 27 + return (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); 29 28 } 30 29 31 30 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) 32 31 { 33 - quicklist_free(0, NULL, pgd); 32 + free_page((unsigned long)pgd); 34 33 } 35 34 36 35 #if CONFIG_PGTABLE_LEVELS == 4 ··· 41 42 42 43 static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) 43 44 { 44 - return quicklist_alloc(0, GFP_KERNEL, NULL); 45 + return (pud_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); 45 46 } 46 47 47 48 static inline void pud_free(struct mm_struct *mm, pud_t *pud) 48 49 { 49 - quicklist_free(0, NULL, pud); 50 + free_page((unsigned long)pud); 50 51 } 51 52 #define __pud_free_tlb(tlb, pud, address) pud_free((tlb)->mm, pud) 52 53 #endif /* CONFIG_PGTABLE_LEVELS == 4 */ ··· 59 60 60 61 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) 61 62 { 62 - return quicklist_alloc(0, GFP_KERNEL, NULL); 63 + return (pmd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); 63 64 } 64 65 65 66 static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) 66 67 { 67 - quicklist_free(0, NULL, pmd); 68 + free_page((unsigned long)pmd); 68 69 } 69 70 70 71 #define __pmd_free_tlb(tlb, pmd, address) pmd_free((tlb)->mm, pmd) ··· 85 86 static inline pgtable_t pte_alloc_one(struct mm_struct *mm) 86 87 { 87 88 struct page *page; 88 - void *pg; 89 89 90 - pg = quicklist_alloc(0, GFP_KERNEL, NULL); 91 - if (!pg) 90 + page = alloc_page(GFP_KERNEL | __GFP_ZERO); 91 + if (!page) 92 92 return NULL; 93 - page = virt_to_page(pg); 94 93 if (!pgtable_page_ctor(page)) { 95 - quicklist_free(0, NULL, pg); 94 + __free_page(page); 96 95 return NULL; 97 96 } 98 97 return page; ··· 98 101 99 102 static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) 100 103 { 101 - return quicklist_alloc(0, GFP_KERNEL, NULL); 104 + return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); 102 105 } 103 106 104 107 static inline void pte_free(struct mm_struct *mm, pgtable_t pte) 105 108 { 106 109 pgtable_page_dtor(pte); 107 - quicklist_free_page(0, NULL, pte); 110 + __free_page(pte); 108 111 } 109 112 110 113 static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) 111 114 { 112 - quicklist_free(0, NULL, pte); 113 - } 114 - 115 - static inline void check_pgt_cache(void) 116 - { 117 - quicklist_trim(0, NULL, 25, 16); 115 + free_page((unsigned long)pte); 118 116 } 119 117 120 118 #define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte)
-2
arch/m68k/include/asm/pgtable_mm.h
··· 181 181 */ 182 182 #define pgtable_cache_init() do { } while (0) 183 183 184 - #define check_pgt_cache() do { } while (0) 185 - 186 184 #endif /* _M68K_PGTABLE_H */
-2
arch/m68k/include/asm/pgtable_no.h
··· 60 60 61 61 #include <asm-generic/pgtable.h> 62 62 63 - #define check_pgt_cache() do { } while (0) 64 - 65 63 #endif /* _M68KNOMMU_PGTABLE_H */
+7 -82
arch/microblaze/include/asm/pgalloc.h
··· 21 21 #include <asm/cache.h> 22 22 #include <asm/pgtable.h> 23 23 24 - #define PGDIR_ORDER 0 25 - 26 - /* 27 - * This is handled very differently on MicroBlaze since out page tables 28 - * are all 0's and I want to be able to use these zero'd pages elsewhere 29 - * as well - it gives us quite a speedup. 30 - * -- Cort 31 - */ 32 - extern struct pgtable_cache_struct { 33 - unsigned long *pgd_cache; 34 - unsigned long *pte_cache; 35 - unsigned long pgtable_cache_sz; 36 - } quicklists; 37 - 38 - #define pgd_quicklist (quicklists.pgd_cache) 39 - #define pmd_quicklist ((unsigned long *)0) 40 - #define pte_quicklist (quicklists.pte_cache) 41 - #define pgtable_cache_size (quicklists.pgtable_cache_sz) 42 - 43 - extern unsigned long *zero_cache; /* head linked list of pre-zero'd pages */ 44 - extern atomic_t zero_sz; /* # currently pre-zero'd pages */ 45 - extern atomic_t zeropage_hits; /* # zero'd pages request that we've done */ 46 - extern atomic_t zeropage_calls; /* # zero'd pages request that've been made */ 47 - extern atomic_t zerototal; /* # pages zero'd over time */ 48 - 49 - #define zero_quicklist (zero_cache) 50 - #define zero_cache_sz (zero_sz) 51 - #define zero_cache_calls (zeropage_calls) 52 - #define zero_cache_hits (zeropage_hits) 53 - #define zero_cache_total (zerototal) 54 - 55 - /* 56 - * return a pre-zero'd page from the list, 57 - * return NULL if none available -- Cort 58 - */ 59 - extern unsigned long get_zero_page_fast(void); 60 - 61 24 extern void __bad_pte(pmd_t *pmd); 62 25 63 - static inline pgd_t *get_pgd_slow(void) 26 + static inline pgd_t *get_pgd(void) 64 27 { 65 - pgd_t *ret; 66 - 67 - ret = (pgd_t *)__get_free_pages(GFP_KERNEL, PGDIR_ORDER); 68 - if (ret != NULL) 69 - clear_page(ret); 70 - return ret; 28 + return (pgd_t *)__get_free_pages(GFP_KERNEL|__GFP_ZERO, 0); 71 29 } 72 30 73 - static inline pgd_t *get_pgd_fast(void) 74 - { 75 - unsigned long *ret; 76 - 77 - ret = pgd_quicklist; 78 - if (ret != NULL) { 79 - pgd_quicklist = (unsigned long *)(*ret); 80 - ret[0] = 0; 81 - pgtable_cache_size--; 82 - } else 83 - ret = (unsigned long *)get_pgd_slow(); 84 - return (pgd_t *)ret; 85 - } 86 - 87 - static inline void free_pgd_fast(pgd_t *pgd) 88 - { 89 - *(unsigned long **)pgd = pgd_quicklist; 90 - pgd_quicklist = (unsigned long *) pgd; 91 - pgtable_cache_size++; 92 - } 93 - 94 - static inline void free_pgd_slow(pgd_t *pgd) 31 + static inline void free_pgd(pgd_t *pgd) 95 32 { 96 33 free_page((unsigned long)pgd); 97 34 } 98 35 99 - #define pgd_free(mm, pgd) free_pgd_fast(pgd) 100 - #define pgd_alloc(mm) get_pgd_fast() 36 + #define pgd_free(mm, pgd) free_pgd(pgd) 37 + #define pgd_alloc(mm) get_pgd() 101 38 102 39 #define pmd_pgtable(pmd) pmd_page(pmd) 103 40 ··· 52 115 struct page *ptepage; 53 116 54 117 #ifdef CONFIG_HIGHPTE 55 - int flags = GFP_KERNEL | __GFP_HIGHMEM; 118 + int flags = GFP_KERNEL | __GFP_ZERO | __GFP_HIGHMEM; 56 119 #else 57 - int flags = GFP_KERNEL; 120 + int flags = GFP_KERNEL | __GFP_ZERO; 58 121 #endif 59 122 60 123 ptepage = alloc_pages(flags, 0); 61 124 if (!ptepage) 62 125 return NULL; 63 - clear_highpage(ptepage); 64 126 if (!pgtable_page_ctor(ptepage)) { 65 127 __free_page(ptepage); 66 128 return NULL; 67 129 } 68 130 return ptepage; 69 - } 70 - 71 - static inline void pte_free_fast(pte_t *pte) 72 - { 73 - *(unsigned long **)pte = pte_quicklist; 74 - pte_quicklist = (unsigned long *) pte; 75 - pgtable_cache_size++; 76 131 } 77 132 78 133 static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) ··· 100 171 #define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) 101 172 #define pgd_populate(mm, pmd, pte) BUG() 102 173 103 - extern int do_check_pgt_cache(int, int); 104 - 105 174 #endif /* CONFIG_MMU */ 106 - 107 - #define check_pgt_cache() do { } while (0) 108 175 109 176 #endif /* _ASM_MICROBLAZE_PGALLOC_H */
-4
arch/microblaze/mm/pgtable.c
··· 44 44 unsigned long ioremap_bot; 45 45 EXPORT_SYMBOL(ioremap_bot); 46 46 47 - #ifndef CONFIG_SMP 48 - struct pgtable_cache_struct quicklists; 49 - #endif 50 - 51 47 static void __iomem *__ioremap(phys_addr_t addr, unsigned long size, 52 48 unsigned long flags) 53 49 {
-2
arch/mips/include/asm/pgalloc.h
··· 105 105 106 106 #endif /* __PAGETABLE_PUD_FOLDED */ 107 107 108 - #define check_pgt_cache() do { } while (0) 109 - 110 108 extern void pagetable_init(void); 111 109 112 110 #endif /* _ASM_PGALLOC_H */
-2
arch/nds32/include/asm/pgalloc.h
··· 23 23 extern pgd_t *pgd_alloc(struct mm_struct *mm); 24 24 extern void pgd_free(struct mm_struct *mm, pgd_t * pgd); 25 25 26 - #define check_pgt_cache() do { } while (0) 27 - 28 26 static inline pgtable_t pte_alloc_one(struct mm_struct *mm) 29 27 { 30 28 pgtable_t pte;
-2
arch/nios2/include/asm/pgalloc.h
··· 45 45 tlb_remove_page((tlb), (pte)); \ 46 46 } while (0) 47 47 48 - #define check_pgt_cache() do { } while (0) 49 - 50 48 #endif /* _ASM_NIOS2_PGALLOC_H */
-2
arch/openrisc/include/asm/pgalloc.h
··· 101 101 102 102 #define pmd_pgtable(pmd) pmd_page(pmd) 103 103 104 - #define check_pgt_cache() do { } while (0) 105 - 106 104 #endif
-2
arch/parisc/include/asm/pgalloc.h
··· 124 124 pmd_populate_kernel(mm, pmd, page_address(pte_page)) 125 125 #define pmd_pgtable(pmd) pmd_page(pmd) 126 126 127 - #define check_pgt_cache() do { } while (0) 128 - 129 127 #endif
-2
arch/powerpc/include/asm/pgalloc.h
··· 64 64 extern struct kmem_cache *pgtable_cache[]; 65 65 #define PGT_CACHE(shift) pgtable_cache[shift] 66 66 67 - static inline void check_pgt_cache(void) { } 68 - 69 67 #ifdef CONFIG_PPC_BOOK3S 70 68 #include <asm/book3s/pgalloc.h> 71 69 #else
-4
arch/riscv/include/asm/pgalloc.h
··· 82 82 tlb_remove_page((tlb), pte); \ 83 83 } while (0) 84 84 85 - static inline void check_pgt_cache(void) 86 - { 87 - } 88 - 89 85 #endif /* _ASM_RISCV_PGALLOC_H */
-1
arch/s390/include/asm/pgtable.h
··· 1686 1686 * No page table caches to initialise 1687 1687 */ 1688 1688 static inline void pgtable_cache_init(void) { } 1689 - static inline void check_pgt_cache(void) { } 1690 1689 1691 1690 #include <asm-generic/pgtable.h> 1692 1691
+6 -16
arch/sh/include/asm/pgalloc.h
··· 2 2 #ifndef __ASM_SH_PGALLOC_H 3 3 #define __ASM_SH_PGALLOC_H 4 4 5 - #include <linux/quicklist.h> 6 5 #include <asm/page.h> 7 - 8 - #define QUICK_PT 0 /* Other page table pages that are zero on free */ 9 6 10 7 extern pgd_t *pgd_alloc(struct mm_struct *); 11 8 extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); ··· 31 34 */ 32 35 static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) 33 36 { 34 - return quicklist_alloc(QUICK_PT, GFP_KERNEL, NULL); 37 + return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO); 35 38 } 36 39 37 40 static inline pgtable_t pte_alloc_one(struct mm_struct *mm) 38 41 { 39 42 struct page *page; 40 - void *pg; 41 43 42 - pg = quicklist_alloc(QUICK_PT, GFP_KERNEL, NULL); 43 - if (!pg) 44 + page = alloc_page(GFP_KERNEL | __GFP_ZERO); 45 + if (!page) 44 46 return NULL; 45 - page = virt_to_page(pg); 46 47 if (!pgtable_page_ctor(page)) { 47 - quicklist_free(QUICK_PT, NULL, pg); 48 + __free_page(page); 48 49 return NULL; 49 50 } 50 51 return page; ··· 50 55 51 56 static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) 52 57 { 53 - quicklist_free(QUICK_PT, NULL, pte); 58 + free_page((unsigned long)pte); 54 59 } 55 60 56 61 static inline void pte_free(struct mm_struct *mm, pgtable_t pte) 57 62 { 58 63 pgtable_page_dtor(pte); 59 - quicklist_free_page(QUICK_PT, NULL, pte); 64 + __free_page(pte); 60 65 } 61 66 62 67 #define __pte_free_tlb(tlb,pte,addr) \ ··· 73 78 tlb_remove_page((tlb), page); \ 74 79 } while (0); 75 80 #endif 76 - 77 - static inline void check_pgt_cache(void) 78 - { 79 - quicklist_trim(QUICK_PT, NULL, 25, 16); 80 - } 81 81 82 82 #endif /* __ASM_SH_PGALLOC_H */
-3
arch/sh/mm/Kconfig
··· 1 1 # SPDX-License-Identifier: GPL-2.0 2 2 menu "Memory management options" 3 3 4 - config QUICKLIST 5 - def_bool y 6 - 7 4 config MMU 8 5 bool "Support for memory management hardware" 9 6 depends on !CPU_SH2
-2
arch/sparc/include/asm/pgalloc_32.h
··· 17 17 18 18 extern struct resource sparc_iomap; 19 19 20 - #define check_pgt_cache() do { } while (0) 21 - 22 20 pgd_t *get_pgd_fast(void); 23 21 static inline void free_pgd_fast(pgd_t *pgd) 24 22 {
-2
arch/sparc/include/asm/pgalloc_64.h
··· 69 69 #define pmd_populate(MM, PMD, PTE) pmd_set(MM, PMD, PTE) 70 70 #define pmd_pgtable(PMD) ((pte_t *)__pmd_page(PMD)) 71 71 72 - #define check_pgt_cache() do { } while (0) 73 - 74 72 void pgtable_free(void *table, bool is_page); 75 73 76 74 #ifdef CONFIG_SMP
-1
arch/sparc/mm/init_32.c
··· 31 31 #include <asm/page.h> 32 32 #include <asm/pgtable.h> 33 33 #include <asm/vaddrs.h> 34 - #include <asm/pgalloc.h> /* bug in asm-generic/tlb.h: check_pgt_cache */ 35 34 #include <asm/setup.h> 36 35 #include <asm/tlb.h> 37 36 #include <asm/prom.h>
-2
arch/um/include/asm/pgalloc.h
··· 43 43 #define __pmd_free_tlb(tlb,x, address) tlb_remove_page((tlb),virt_to_page(x)) 44 44 #endif 45 45 46 - #define check_pgt_cache() do { } while (0) 47 - 48 46 #endif 49 47
-2
arch/unicore32/include/asm/pgalloc.h
··· 18 18 #define __HAVE_ARCH_PTE_ALLOC_ONE 19 19 #include <asm-generic/pgalloc.h> 20 20 21 - #define check_pgt_cache() do { } while (0) 22 - 23 21 #define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_PRESENT) 24 22 #define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_PRESENT) 25 23
-1
arch/x86/include/asm/pgtable_32.h
··· 30 30 extern pmd_t initial_pg_pmd[]; 31 31 32 32 static inline void pgtable_cache_init(void) { } 33 - static inline void check_pgt_cache(void) { } 34 33 void paging_init(void); 35 34 void sync_initial_page_table(void); 36 35
-1
arch/x86/include/asm/pgtable_64.h
··· 242 242 #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN 243 243 244 244 #define pgtable_cache_init() do { } while (0) 245 - #define check_pgt_cache() do { } while (0) 246 245 247 246 #define PAGE_AGP PAGE_KERNEL_NOCACHE 248 247 #define HAVE_PAGE_AGP 1
-3
arch/xtensa/include/asm/tlbflush.h
··· 160 160 invalidate_dtlb_entry(tlb_entry); 161 161 } 162 162 163 - #define check_pgt_cache() do { } while (0) 164 - 165 - 166 163 /* 167 164 * DO NOT USE THESE FUNCTIONS. These instructions aren't part of the Xtensa 168 165 * ISA and exist only for test purposes..
-4
fs/proc/meminfo.c
··· 8 8 #include <linux/mmzone.h> 9 9 #include <linux/proc_fs.h> 10 10 #include <linux/percpu.h> 11 - #include <linux/quicklist.h> 12 11 #include <linux/seq_file.h> 13 12 #include <linux/swap.h> 14 13 #include <linux/vmstat.h> ··· 105 106 global_zone_page_state(NR_KERNEL_STACK_KB)); 106 107 show_val_kb(m, "PageTables: ", 107 108 global_zone_page_state(NR_PAGETABLE)); 108 - #ifdef CONFIG_QUICKLIST 109 - show_val_kb(m, "Quicklists: ", quicklist_total_size()); 110 - #endif 111 109 112 110 show_val_kb(m, "NFS_Unstable: ", 113 111 global_node_page_state(NR_UNSTABLE_NFS));
-5
include/asm-generic/pgalloc.h
··· 102 102 __free_page(pte_page); 103 103 } 104 104 105 - #else /* CONFIG_MMU */ 106 - 107 - /* This is enough for a nommu architecture */ 108 - #define check_pgt_cache() do { } while (0) 109 - 110 105 #endif /* CONFIG_MMU */ 111 106 112 107 #endif /* __ASM_GENERIC_PGALLOC_H */
-94
include/linux/quicklist.h
··· 1 - /* SPDX-License-Identifier: GPL-2.0 */ 2 - #ifndef LINUX_QUICKLIST_H 3 - #define LINUX_QUICKLIST_H 4 - /* 5 - * Fast allocations and disposal of pages. Pages must be in the condition 6 - * as needed after allocation when they are freed. Per cpu lists of pages 7 - * are kept that only contain node local pages. 8 - * 9 - * (C) 2007, SGI. Christoph Lameter <cl@linux.com> 10 - */ 11 - #include <linux/kernel.h> 12 - #include <linux/gfp.h> 13 - #include <linux/percpu.h> 14 - 15 - #ifdef CONFIG_QUICKLIST 16 - 17 - struct quicklist { 18 - void *page; 19 - int nr_pages; 20 - }; 21 - 22 - DECLARE_PER_CPU(struct quicklist, quicklist)[CONFIG_NR_QUICK]; 23 - 24 - /* 25 - * The two key functions quicklist_alloc and quicklist_free are inline so 26 - * that they may be custom compiled for the platform. 27 - * Specifying a NULL ctor can remove constructor support. Specifying 28 - * a constant quicklist allows the determination of the exact address 29 - * in the per cpu area. 30 - * 31 - * The fast patch in quicklist_alloc touched only a per cpu cacheline and 32 - * the first cacheline of the page itself. There is minmal overhead involved. 33 - */ 34 - static inline void *quicklist_alloc(int nr, gfp_t flags, void (*ctor)(void *)) 35 - { 36 - struct quicklist *q; 37 - void **p = NULL; 38 - 39 - q =&get_cpu_var(quicklist)[nr]; 40 - p = q->page; 41 - if (likely(p)) { 42 - q->page = p[0]; 43 - p[0] = NULL; 44 - q->nr_pages--; 45 - } 46 - put_cpu_var(quicklist); 47 - if (likely(p)) 48 - return p; 49 - 50 - p = (void *)__get_free_page(flags | __GFP_ZERO); 51 - if (ctor && p) 52 - ctor(p); 53 - return p; 54 - } 55 - 56 - static inline void __quicklist_free(int nr, void (*dtor)(void *), void *p, 57 - struct page *page) 58 - { 59 - struct quicklist *q; 60 - 61 - q = &get_cpu_var(quicklist)[nr]; 62 - *(void **)p = q->page; 63 - q->page = p; 64 - q->nr_pages++; 65 - put_cpu_var(quicklist); 66 - } 67 - 68 - static inline void quicklist_free(int nr, void (*dtor)(void *), void *pp) 69 - { 70 - __quicklist_free(nr, dtor, pp, virt_to_page(pp)); 71 - } 72 - 73 - static inline void quicklist_free_page(int nr, void (*dtor)(void *), 74 - struct page *page) 75 - { 76 - __quicklist_free(nr, dtor, page_address(page), page); 77 - } 78 - 79 - void quicklist_trim(int nr, void (*dtor)(void *), 80 - unsigned long min_pages, unsigned long max_free); 81 - 82 - unsigned long quicklist_total_size(void); 83 - 84 - #else 85 - 86 - static inline unsigned long quicklist_total_size(void) 87 - { 88 - return 0; 89 - } 90 - 91 - #endif 92 - 93 - #endif /* LINUX_QUICKLIST_H */ 94 -
-1
kernel/sched/idle.c
··· 238 238 tick_nohz_idle_enter(); 239 239 240 240 while (!need_resched()) { 241 - check_pgt_cache(); 242 241 rmb(); 243 242 244 243 local_irq_disable();
-5
lib/show_mem.c
··· 6 6 */ 7 7 8 8 #include <linux/mm.h> 9 - #include <linux/quicklist.h> 10 9 #include <linux/cma.h> 11 10 12 11 void show_mem(unsigned int filter, nodemask_t *nodemask) ··· 37 38 printk("%lu pages reserved\n", reserved); 38 39 #ifdef CONFIG_CMA 39 40 printk("%lu pages cma reserved\n", totalcma_pages); 40 - #endif 41 - #ifdef CONFIG_QUICKLIST 42 - printk("%lu pages in pagetable cache\n", 43 - quicklist_total_size()); 44 41 #endif 45 42 #ifdef CONFIG_MEMORY_FAILURE 46 43 printk("%lu pages hwpoisoned\n", atomic_long_read(&num_poisoned_pages));
-5
mm/Kconfig
··· 273 273 by default when ZONE_DMA or HIGHMEM is selected, but you 274 274 may say n to override this. 275 275 276 - config NR_QUICK 277 - int 278 - depends on QUICKLIST 279 - default "1" 280 - 281 276 config VIRT_TO_BUS 282 277 bool 283 278 help
-1
mm/Makefile
··· 72 72 obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o 73 73 obj-$(CONFIG_MEMTEST) += memtest.o 74 74 obj-$(CONFIG_MIGRATION) += migrate.o 75 - obj-$(CONFIG_QUICKLIST) += quicklist.o 76 75 obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o 77 76 obj-$(CONFIG_PAGE_COUNTER) += page_counter.o 78 77 obj-$(CONFIG_MEMCG) += memcontrol.o vmpressure.o
-2
mm/mmu_gather.c
··· 271 271 272 272 tlb_flush_mmu(tlb); 273 273 274 - /* keep the page table cache within bounds */ 275 - check_pgt_cache(); 276 274 #ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER 277 275 tlb_batch_list_free(tlb); 278 276 #endif
-103
mm/quicklist.c
··· 1 - // SPDX-License-Identifier: GPL-2.0 2 - /* 3 - * Quicklist support. 4 - * 5 - * Quicklists are light weight lists of pages that have a defined state 6 - * on alloc and free. Pages must be in the quicklist specific defined state 7 - * (zero by default) when the page is freed. It seems that the initial idea 8 - * for such lists first came from Dave Miller and then various other people 9 - * improved on it. 10 - * 11 - * Copyright (C) 2007 SGI, 12 - * Christoph Lameter <cl@linux.com> 13 - * Generalized, added support for multiple lists and 14 - * constructors / destructors. 15 - */ 16 - #include <linux/kernel.h> 17 - 18 - #include <linux/gfp.h> 19 - #include <linux/mm.h> 20 - #include <linux/mmzone.h> 21 - #include <linux/quicklist.h> 22 - 23 - DEFINE_PER_CPU(struct quicklist [CONFIG_NR_QUICK], quicklist); 24 - 25 - #define FRACTION_OF_NODE_MEM 16 26 - 27 - static unsigned long max_pages(unsigned long min_pages) 28 - { 29 - unsigned long node_free_pages, max; 30 - int node = numa_node_id(); 31 - struct zone *zones = NODE_DATA(node)->node_zones; 32 - int num_cpus_on_node; 33 - 34 - node_free_pages = 35 - #ifdef CONFIG_ZONE_DMA 36 - zone_page_state(&zones[ZONE_DMA], NR_FREE_PAGES) + 37 - #endif 38 - #ifdef CONFIG_ZONE_DMA32 39 - zone_page_state(&zones[ZONE_DMA32], NR_FREE_PAGES) + 40 - #endif 41 - zone_page_state(&zones[ZONE_NORMAL], NR_FREE_PAGES); 42 - 43 - max = node_free_pages / FRACTION_OF_NODE_MEM; 44 - 45 - num_cpus_on_node = cpumask_weight(cpumask_of_node(node)); 46 - max /= num_cpus_on_node; 47 - 48 - return max(max, min_pages); 49 - } 50 - 51 - static long min_pages_to_free(struct quicklist *q, 52 - unsigned long min_pages, long max_free) 53 - { 54 - long pages_to_free; 55 - 56 - pages_to_free = q->nr_pages - max_pages(min_pages); 57 - 58 - return min(pages_to_free, max_free); 59 - } 60 - 61 - /* 62 - * Trim down the number of pages in the quicklist 63 - */ 64 - void quicklist_trim(int nr, void (*dtor)(void *), 65 - unsigned long min_pages, unsigned long max_free) 66 - { 67 - long pages_to_free; 68 - struct quicklist *q; 69 - 70 - q = &get_cpu_var(quicklist)[nr]; 71 - if (q->nr_pages > min_pages) { 72 - pages_to_free = min_pages_to_free(q, min_pages, max_free); 73 - 74 - while (pages_to_free > 0) { 75 - /* 76 - * We pass a gfp_t of 0 to quicklist_alloc here 77 - * because we will never call into the page allocator. 78 - */ 79 - void *p = quicklist_alloc(nr, 0, NULL); 80 - 81 - if (dtor) 82 - dtor(p); 83 - free_page((unsigned long)p); 84 - pages_to_free--; 85 - } 86 - } 87 - put_cpu_var(quicklist); 88 - } 89 - 90 - unsigned long quicklist_total_size(void) 91 - { 92 - unsigned long count = 0; 93 - int cpu; 94 - struct quicklist *ql, *q; 95 - 96 - for_each_online_cpu(cpu) { 97 - ql = per_cpu(quicklist, cpu); 98 - for (q = ql; q < ql + CONFIG_NR_QUICK; q++) 99 - count += q->nr_pages; 100 - } 101 - return count; 102 - } 103 -