Merge branch 'pgtable-layout-rewrite' into for-v5.7

+14 -17

arch/m68k/include/asm/mcf_pgalloc.h

··· 28 28 return (pmd_t *) pgd; 29 29 } 30 30 31 - #define pmd_populate(mm, pmd, page) (pmd_val(*pmd) = \ 32 - (unsigned long)(page_address(page))) 31 + #define pmd_populate(mm, pmd, pte) (pmd_val(*pmd) = (unsigned long)(pte)) 33 32 34 - #define pmd_populate_kernel(mm, pmd, pte) (pmd_val(*pmd) = (unsigned long)(pte)) 33 + #define pmd_populate_kernel pmd_populate 35 34 36 - #define pmd_pgtable(pmd) pmd_page(pmd) 35 + #define pmd_pgtable(pmd) pfn_to_virt(pmd_val(pmd) >> PAGE_SHIFT) 37 36 38 - static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page, 37 + static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable, 39 38 unsigned long address) 40 39 { 40 + struct page *page = virt_to_page(pgtable); 41 + 41 42 pgtable_pte_page_dtor(page); 42 43 __free_page(page); 43 44 } 44 45 45 - static inline struct page *pte_alloc_one(struct mm_struct *mm) 46 + static inline pgtable_t pte_alloc_one(struct mm_struct *mm) 46 47 { 47 48 struct page *page = alloc_pages(GFP_DMA, 0); 48 49 pte_t *pte; ··· 55 54 return NULL; 56 55 } 57 56 58 - pte = kmap(page); 59 - if (pte) { 60 - clear_page(pte); 61 - __flush_page_to_ram(pte); 62 - flush_tlb_kernel_page(pte); 63 - nocache_page(pte); 64 - } 65 - kunmap(page); 57 + pte = page_address(page); 58 + clear_page(pte); 66 59 67 - return page; 60 + return pte; 68 61 } 69 62 70 - static inline void pte_free(struct mm_struct *mm, struct page *page) 63 + static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable) 71 64 { 65 + struct page *page = virt_to_page(pgtable); 66 + 72 67 pgtable_pte_page_dtor(page); 73 68 __free_page(page); 74 69 } ··· 87 90 new_pgd = (pgd_t *)__get_free_page(GFP_DMA | __GFP_NOWARN); 88 91 if (!new_pgd) 89 92 return NULL; 90 - memcpy(new_pgd, swapper_pg_dir, PAGE_SIZE); 93 + memcpy(new_pgd, swapper_pg_dir, PTRS_PER_PGD * sizeof(pgd_t)); 91 94 memset(new_pgd, 0, PAGE_OFFSET >> PGDIR_SHIFT); 92 95 return new_pgd; 93 96 }

+26 -48

arch/m68k/include/asm/motorola_pgalloc.h

··· 5 5 #include <asm/tlb.h> 6 6 #include <asm/tlbflush.h> 7 7 8 - extern pmd_t *get_pointer_table(void); 9 - extern int free_pointer_table(pmd_t *); 8 + extern void mmu_page_ctor(void *page); 9 + extern void mmu_page_dtor(void *page); 10 + 11 + enum m68k_table_types { 12 + TABLE_PGD = 0, 13 + TABLE_PMD = 0, /* same size as PGD */ 14 + TABLE_PTE = 1, 15 + }; 16 + 17 + extern void init_pointer_table(void *table, int type); 18 + extern void *get_pointer_table(int type); 19 + extern int free_pointer_table(void *table, int type); 10 20 11 21 static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) 12 22 { 13 - pte_t *pte; 14 - 15 - pte = (pte_t *)__get_free_page(GFP_KERNEL|__GFP_ZERO); 16 - if (pte) { 17 - __flush_page_to_ram(pte); 18 - flush_tlb_kernel_page(pte); 19 - nocache_page(pte); 20 - } 21 - 22 - return pte; 23 + return get_pointer_table(TABLE_PTE); 23 24 } 24 25 25 26 static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) 26 27 { 27 - cache_page(pte); 28 - free_page((unsigned long) pte); 28 + free_pointer_table(pte, TABLE_PTE); 29 29 } 30 30 31 31 static inline pgtable_t pte_alloc_one(struct mm_struct *mm) 32 32 { 33 - struct page *page; 34 - pte_t *pte; 35 - 36 - page = alloc_pages(GFP_KERNEL|__GFP_ZERO, 0); 37 - if(!page) 38 - return NULL; 39 - if (!pgtable_pte_page_ctor(page)) { 40 - __free_page(page); 41 - return NULL; 42 - } 43 - 44 - pte = kmap(page); 45 - __flush_page_to_ram(pte); 46 - flush_tlb_kernel_page(pte); 47 - nocache_page(pte); 48 - kunmap(page); 49 - return page; 33 + return get_pointer_table(TABLE_PTE); 50 34 } 51 35 52 - static inline void pte_free(struct mm_struct *mm, pgtable_t page) 36 + static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable) 53 37 { 54 - pgtable_pte_page_dtor(page); 55 - cache_page(kmap(page)); 56 - kunmap(page); 57 - __free_page(page); 38 + free_pointer_table(pgtable, TABLE_PTE); 58 39 } 59 40 60 - static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t page, 41 + static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pgtable, 61 42 unsigned long address) 62 43 { 63 - pgtable_pte_page_dtor(page); 64 - cache_page(kmap(page)); 65 - kunmap(page); 66 - __free_page(page); 44 + free_pointer_table(pgtable, TABLE_PTE); 67 45 } 68 46 69 47 70 48 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) 71 49 { 72 - return get_pointer_table(); 50 + return get_pointer_table(TABLE_PMD); 73 51 } 74 52 75 53 static inline int pmd_free(struct mm_struct *mm, pmd_t *pmd) 76 54 { 77 - return free_pointer_table(pmd); 55 + return free_pointer_table(pmd, TABLE_PMD); 78 56 } 79 57 80 58 static inline int __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd, 81 59 unsigned long address) 82 60 { 83 - return free_pointer_table(pmd); 61 + return free_pointer_table(pmd, TABLE_PMD); 84 62 } 85 63 86 64 87 65 static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) 88 66 { 89 - pmd_free(mm, (pmd_t *)pgd); 67 + free_pointer_table(pgd, TABLE_PGD); 90 68 } 91 69 92 70 static inline pgd_t *pgd_alloc(struct mm_struct *mm) 93 71 { 94 - return (pgd_t *)get_pointer_table(); 72 + return get_pointer_table(TABLE_PGD); 95 73 } 96 74 97 75 ··· 80 102 81 103 static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd, pgtable_t page) 82 104 { 83 - pmd_set(pmd, page_address(page)); 105 + pmd_set(pmd, page); 84 106 } 85 - #define pmd_pgtable(pmd) pmd_page(pmd) 107 + #define pmd_pgtable(pmd) ((pgtable_t)__pmd_page(pmd)) 86 108 87 109 static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) 88 110 {

+21 -15

arch/m68k/include/asm/motorola_pgtable.h

··· 23 23 #define _DESCTYPE_MASK 0x003 24 24 25 25 #define _CACHEMASK040 (~0x060) 26 - #define _TABLE_MASK (0xfffffe00) 26 + 27 + /* 28 + * Currently set to the minimum alignment of table pointers (256 bytes). 29 + * The hardware only uses the low 4 bits for state: 30 + * 31 + * 3 - Used 32 + * 2 - Write Protected 33 + * 0,1 - Descriptor Type 34 + * 35 + * and has the rest of the bits reserved. 36 + */ 37 + #define _TABLE_MASK (0xffffff00) 27 38 28 39 #define _PAGE_TABLE (_PAGE_SHORT) 29 40 #define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_NOCACHE) ··· 119 108 120 109 static inline void pmd_set(pmd_t *pmdp, pte_t *ptep) 121 110 { 122 - unsigned long ptbl = virt_to_phys(ptep) | _PAGE_TABLE | _PAGE_ACCESSED; 123 - unsigned long *ptr = pmdp->pmd; 124 - short i = 16; 125 - while (--i >= 0) { 126 - *ptr++ = ptbl; 127 - ptbl += (sizeof(pte_t)*PTRS_PER_PTE/16); 128 - } 111 + pmd_val(*pmdp) = virt_to_phys(ptep) | _PAGE_TABLE | _PAGE_ACCESSED; 129 112 } 130 113 131 114 static inline void pud_set(pud_t *pudp, pmd_t *pmdp) ··· 143 138 #define pmd_none(pmd) (!pmd_val(pmd)) 144 139 #define pmd_bad(pmd) ((pmd_val(pmd) & _DESCTYPE_MASK) != _PAGE_TABLE) 145 140 #define pmd_present(pmd) (pmd_val(pmd) & _PAGE_TABLE) 146 - #define pmd_clear(pmdp) ({ \ 147 - unsigned long *__ptr = pmdp->pmd; \ 148 - short __i = 16; \ 149 - while (--__i >= 0) \ 150 - *__ptr++ = 0; \ 151 - }) 152 - #define pmd_page(pmd) virt_to_page(__va(pmd_val(pmd))) 141 + #define pmd_clear(pmdp) ({ pmd_val(*pmdp) = 0; }) 142 + 143 + /* 144 + * m68k does not have huge pages (020/030 actually could), but generic code 145 + * expects pmd_page() to exists, only to then DCE it all. Provide a dummy to 146 + * make the compiler happy. 147 + */ 148 + #define pmd_page(pmd) NULL 153 149 154 150 155 151 #define pud_none(pud) (!pud_val(pud))

+13 -3

arch/m68k/include/asm/page.h

··· 22 22 * These are used to make use of C type-checking.. 23 23 */ 24 24 #if !defined(CONFIG_MMU) || CONFIG_PGTABLE_LEVELS == 3 25 - typedef struct { unsigned long pmd[16]; } pmd_t; 26 - #define pmd_val(x) ((&x)->pmd[0]) 27 - #define __pmd(x) ((pmd_t) { { (x) }, }) 25 + typedef struct { unsigned long pmd; } pmd_t; 26 + #define pmd_val(x) ((&x)->pmd) 27 + #define __pmd(x) ((pmd_t) { (x) } ) 28 28 #endif 29 29 30 30 typedef struct { unsigned long pte; } pte_t; 31 31 typedef struct { unsigned long pgd; } pgd_t; 32 32 typedef struct { unsigned long pgprot; } pgprot_t; 33 + 34 + #if defined(CONFIG_SUN3) 35 + /* 36 + * Sun3 still uses the asm-generic/pgalloc.h code and thus needs this 37 + * definition. It would be possible to unify Sun3 and ColdFire pgalloc and have 38 + * all of m68k use the same type. 39 + */ 33 40 typedef struct page *pgtable_t; 41 + #else 42 + typedef pte_t *pgtable_t; 43 + #endif 34 44 35 45 #define pte_val(x) ((x).pte) 36 46 #define pgd_val(x) ((x).pgd)

+5 -5

arch/m68k/include/asm/pgtable_mm.h

··· 36 36 37 37 /* PMD_SHIFT determines the size of the area a second-level page table can map */ 38 38 #if CONFIG_PGTABLE_LEVELS == 3 39 - #define PMD_SHIFT 22 39 + #define PMD_SHIFT 18 40 40 #endif 41 41 #define PMD_SIZE (1UL << PMD_SHIFT) 42 42 #define PMD_MASK (~(PMD_SIZE-1)) ··· 67 67 #define PTRS_PER_PMD 1 68 68 #define PTRS_PER_PGD 1024 69 69 #else 70 - #define PTRS_PER_PTE 1024 71 - #define PTRS_PER_PMD 8 70 + #define PTRS_PER_PTE 64 71 + #define PTRS_PER_PMD 128 72 72 #define PTRS_PER_PGD 128 73 73 #endif 74 74 #define USER_PTRS_PER_PGD (TASK_SIZE/PGDIR_SIZE) ··· 76 76 77 77 /* Virtual address region for use by kernel_map() */ 78 78 #ifdef CONFIG_SUN3 79 - #define KMAP_START 0x0DC00000 80 - #define KMAP_END 0x0E000000 79 + #define KMAP_START 0x0dc00000 80 + #define KMAP_END 0x0e000000 81 81 #elif defined(CONFIG_COLDFIRE) 82 82 #define KMAP_START 0xe0000000 83 83 #define KMAP_END 0xf0000000

+20 -14

arch/m68k/mm/init.c

··· 40 40 void *empty_zero_page; 41 41 EXPORT_SYMBOL(empty_zero_page); 42 42 43 - #if !defined(CONFIG_SUN3) && !defined(CONFIG_COLDFIRE) 44 - extern void init_pointer_table(unsigned long ptable); 45 - extern pmd_t *zero_pgtable; 46 - #endif 47 - 48 43 #ifdef CONFIG_MMU 49 44 50 45 pg_data_t pg_data_map[MAX_NUMNODES]; ··· 120 125 static inline void init_pointer_tables(void) 121 126 { 122 127 #if defined(CONFIG_MMU) && !defined(CONFIG_SUN3) && !defined(CONFIG_COLDFIRE) 123 - int i; 128 + int i, j; 124 129 125 130 /* insert pointer tables allocated so far into the tablelist */ 126 - init_pointer_table((unsigned long)kernel_pg_dir); 131 + init_pointer_table(kernel_pg_dir, TABLE_PGD); 127 132 for (i = 0; i < PTRS_PER_PGD; i++) { 128 - pud_t *pud = (pud_t *)(&kernel_pg_dir[i]); 133 + pud_t *pud = (pud_t *)&kernel_pg_dir[i]; 134 + pmd_t *pmd_dir; 129 135 130 - if (pud_present(*pud)) 131 - init_pointer_table(pgd_page_vaddr(kernel_pg_dir[i])); 136 + if (!pud_present(*pud)) 137 + continue; 138 + 139 + pmd_dir = (pmd_t *)pgd_page_vaddr(kernel_pg_dir[i]); 140 + init_pointer_table(pmd_dir, TABLE_PMD); 141 + 142 + for (j = 0; j < PTRS_PER_PMD; j++) { 143 + pmd_t *pmd = &pmd_dir[j]; 144 + pte_t *pte_dir; 145 + 146 + if (!pmd_present(*pmd)) 147 + continue; 148 + 149 + pte_dir = (pte_t *)__pmd_page(*pmd); 150 + init_pointer_table(pte_dir, TABLE_PTE); 151 + } 132 152 } 133 - 134 - /* insert also pointer table that we used to unmap the zero page */ 135 - if (zero_pgtable) 136 - init_pointer_table((unsigned long)zero_pgtable); 137 153 #endif 138 154 } 139 155

+16 -20

arch/m68k/mm/kmap.c

··· 24 24 25 25 #undef DEBUG 26 26 27 - #define PTRTREESIZE (256*1024) 28 - 29 27 /* 30 28 * For 040/060 we can use the virtual memory area like other architectures, 31 29 * but for 020/030 we want to use early termination page descriptors and we ··· 48 50 49 51 #else 50 52 51 - #define IO_SIZE (256*1024) 53 + #define IO_SIZE PMD_SIZE 52 54 53 55 static struct vm_struct *iolist; 54 56 ··· 79 81 80 82 #if CONFIG_PGTABLE_LEVELS == 3 81 83 if (CPU_IS_020_OR_030) { 82 - int pmd_off = (virtaddr/PTRTREESIZE) & 15; 83 - int pmd_type = pmd_dir->pmd[pmd_off] & _DESCTYPE_MASK; 84 + int pmd_type = pmd_val(*pmd_dir) & _DESCTYPE_MASK; 84 85 85 86 if (pmd_type == _PAGE_PRESENT) { 86 - pmd_dir->pmd[pmd_off] = 0; 87 - virtaddr += PTRTREESIZE; 88 - size -= PTRTREESIZE; 89 - continue; 87 + pmd_clear(pmd_dir); 88 + virtaddr += PMD_SIZE; 89 + size -= PMD_SIZE; 90 + 90 91 } else if (pmd_type == 0) 91 92 continue; 92 93 } ··· 246 249 247 250 while ((long)size > 0) { 248 251 #ifdef DEBUG 249 - if (!(virtaddr & (PTRTREESIZE-1))) 252 + if (!(virtaddr & (PMD_SIZE-1))) 250 253 printk ("\npa=%#lx va=%#lx ", physaddr, virtaddr); 251 254 #endif 252 255 pgd_dir = pgd_offset_k(virtaddr); ··· 260 263 261 264 #if CONFIG_PGTABLE_LEVELS == 3 262 265 if (CPU_IS_020_OR_030) { 263 - pmd_dir->pmd[(virtaddr/PTRTREESIZE) & 15] = physaddr; 264 - physaddr += PTRTREESIZE; 265 - virtaddr += PTRTREESIZE; 266 - size -= PTRTREESIZE; 266 + pmd_val(*pmd_dir) = physaddr; 267 + physaddr += PMD_SIZE; 268 + virtaddr += PMD_SIZE; 269 + size -= PMD_SIZE; 267 270 } else 268 271 #endif 269 272 { ··· 364 367 365 368 #if CONFIG_PGTABLE_LEVELS == 3 366 369 if (CPU_IS_020_OR_030) { 367 - int pmd_off = (virtaddr/PTRTREESIZE) & 15; 370 + unsigned long pmd = pmd_val(*pmd_dir); 368 371 369 - if ((pmd_dir->pmd[pmd_off] & _DESCTYPE_MASK) == _PAGE_PRESENT) { 370 - pmd_dir->pmd[pmd_off] = (pmd_dir->pmd[pmd_off] & 371 - _CACHEMASK040) | cmode; 372 - virtaddr += PTRTREESIZE; 373 - size -= PTRTREESIZE; 372 + if ((pmd & _DESCTYPE_MASK) == _PAGE_PRESENT) { 373 + *pmd_dir = __pmd((pmd & _CACHEMASK040) | cmode); 374 + virtaddr += PMD_SIZE; 375 + size -= PMD_SIZE; 374 376 continue; 375 377 } 376 378 }

-103

arch/m68k/mm/memory.c

··· 22 22 #include <asm/machdep.h> 23 23 24 24 25 - /* ++andreas: {get,free}_pointer_table rewritten to use unused fields from 26 - struct page instead of separately kmalloced struct. Stolen from 27 - arch/sparc/mm/srmmu.c ... */ 28 - 29 - typedef struct list_head ptable_desc; 30 - static LIST_HEAD(ptable_list); 31 - 32 - #define PD_PTABLE(page) ((ptable_desc *)&(virt_to_page(page)->lru)) 33 - #define PD_PAGE(ptable) (list_entry(ptable, struct page, lru)) 34 - #define PD_MARKBITS(dp) (*(unsigned char *)&PD_PAGE(dp)->index) 35 - 36 - #define PTABLE_SIZE (PTRS_PER_PMD * sizeof(pmd_t)) 37 - 38 - void __init init_pointer_table(unsigned long ptable) 39 - { 40 - ptable_desc *dp; 41 - unsigned long page = ptable & PAGE_MASK; 42 - unsigned char mask = 1 << ((ptable - page)/PTABLE_SIZE); 43 - 44 - dp = PD_PTABLE(page); 45 - if (!(PD_MARKBITS(dp) & mask)) { 46 - PD_MARKBITS(dp) = 0xff; 47 - list_add(dp, &ptable_list); 48 - } 49 - 50 - PD_MARKBITS(dp) &= ~mask; 51 - pr_debug("init_pointer_table: %lx, %x\n", ptable, PD_MARKBITS(dp)); 52 - 53 - /* unreserve the page so it's possible to free that page */ 54 - __ClearPageReserved(PD_PAGE(dp)); 55 - init_page_count(PD_PAGE(dp)); 56 - 57 - return; 58 - } 59 - 60 - pmd_t *get_pointer_table (void) 61 - { 62 - ptable_desc *dp = ptable_list.next; 63 - unsigned char mask = PD_MARKBITS (dp); 64 - unsigned char tmp; 65 - unsigned int off; 66 - 67 - /* 68 - * For a pointer table for a user process address space, a 69 - * table is taken from a page allocated for the purpose. Each 70 - * page can hold 8 pointer tables. The page is remapped in 71 - * virtual address space to be noncacheable. 72 - */ 73 - if (mask == 0) { 74 - void *page; 75 - ptable_desc *new; 76 - 77 - if (!(page = (void *)get_zeroed_page(GFP_KERNEL))) 78 - return NULL; 79 - 80 - flush_tlb_kernel_page(page); 81 - nocache_page(page); 82 - 83 - new = PD_PTABLE(page); 84 - PD_MARKBITS(new) = 0xfe; 85 - list_add_tail(new, dp); 86 - 87 - return (pmd_t *)page; 88 - } 89 - 90 - for (tmp = 1, off = 0; (mask & tmp) == 0; tmp <<= 1, off += PTABLE_SIZE) 91 - ; 92 - PD_MARKBITS(dp) = mask & ~tmp; 93 - if (!PD_MARKBITS(dp)) { 94 - /* move to end of list */ 95 - list_move_tail(dp, &ptable_list); 96 - } 97 - return (pmd_t *) (page_address(PD_PAGE(dp)) + off); 98 - } 99 - 100 - int free_pointer_table (pmd_t *ptable) 101 - { 102 - ptable_desc *dp; 103 - unsigned long page = (unsigned long)ptable & PAGE_MASK; 104 - unsigned char mask = 1 << (((unsigned long)ptable - page)/PTABLE_SIZE); 105 - 106 - dp = PD_PTABLE(page); 107 - if (PD_MARKBITS (dp) & mask) 108 - panic ("table already free!"); 109 - 110 - PD_MARKBITS (dp) |= mask; 111 - 112 - if (PD_MARKBITS(dp) == 0xff) { 113 - /* all tables in page are free, free page */ 114 - list_del(dp); 115 - cache_page((void *)page); 116 - free_page (page); 117 - return 1; 118 - } else if (ptable_list.next != dp) { 119 - /* 120 - * move this descriptor to the front of the list, since 121 - * it has one or more free tables. 122 - */ 123 - list_move(dp, &ptable_list); 124 - } 125 - return 0; 126 - } 127 - 128 25 /* invalidate page in both caches */ 129 26 static inline void clear040(unsigned long paddr) 130 27 {

+187 -41

arch/m68k/mm/motorola.c

··· 45 45 EXPORT_SYMBOL(mm_cachebits); 46 46 #endif 47 47 48 + 49 + /* 50 + * Motorola 680x0 user's manual recommends using uncached memory for address 51 + * translation tables. 52 + * 53 + * Seeing how the MMU can be external on (some of) these chips, that seems like 54 + * a very important recommendation to follow. Provide some helpers to combat 55 + * 'variation' amongst the users of this. 56 + */ 57 + 58 + void mmu_page_ctor(void *page) 59 + { 60 + __flush_page_to_ram(page); 61 + flush_tlb_kernel_page(page); 62 + nocache_page(page); 63 + } 64 + 65 + void mmu_page_dtor(void *page) 66 + { 67 + cache_page(page); 68 + } 69 + 70 + /* ++andreas: {get,free}_pointer_table rewritten to use unused fields from 71 + struct page instead of separately kmalloced struct. Stolen from 72 + arch/sparc/mm/srmmu.c ... */ 73 + 74 + typedef struct list_head ptable_desc; 75 + 76 + static struct list_head ptable_list[2] = { 77 + LIST_HEAD_INIT(ptable_list[0]), 78 + LIST_HEAD_INIT(ptable_list[1]), 79 + }; 80 + 81 + #define PD_PTABLE(page) ((ptable_desc *)&(virt_to_page(page)->lru)) 82 + #define PD_PAGE(ptable) (list_entry(ptable, struct page, lru)) 83 + #define PD_MARKBITS(dp) (*(unsigned int *)&PD_PAGE(dp)->index) 84 + 85 + static const int ptable_shift[2] = { 86 + 7+2, /* PGD, PMD */ 87 + 6+2, /* PTE */ 88 + }; 89 + 90 + #define ptable_size(type) (1U << ptable_shift[type]) 91 + #define ptable_mask(type) ((1U << (PAGE_SIZE / ptable_size(type))) - 1) 92 + 93 + void __init init_pointer_table(void *table, int type) 94 + { 95 + ptable_desc *dp; 96 + unsigned long ptable = (unsigned long)table; 97 + unsigned long page = ptable & PAGE_MASK; 98 + unsigned int mask = 1U << ((ptable - page)/ptable_size(type)); 99 + 100 + dp = PD_PTABLE(page); 101 + if (!(PD_MARKBITS(dp) & mask)) { 102 + PD_MARKBITS(dp) = ptable_mask(type); 103 + list_add(dp, &ptable_list[type]); 104 + } 105 + 106 + PD_MARKBITS(dp) &= ~mask; 107 + pr_debug("init_pointer_table: %lx, %x\n", ptable, PD_MARKBITS(dp)); 108 + 109 + /* unreserve the page so it's possible to free that page */ 110 + __ClearPageReserved(PD_PAGE(dp)); 111 + init_page_count(PD_PAGE(dp)); 112 + 113 + return; 114 + } 115 + 116 + void *get_pointer_table(int type) 117 + { 118 + ptable_desc *dp = ptable_list[type].next; 119 + unsigned int mask = list_empty(&ptable_list[type]) ? 0 : PD_MARKBITS(dp); 120 + unsigned int tmp, off; 121 + 122 + /* 123 + * For a pointer table for a user process address space, a 124 + * table is taken from a page allocated for the purpose. Each 125 + * page can hold 8 pointer tables. The page is remapped in 126 + * virtual address space to be noncacheable. 127 + */ 128 + if (mask == 0) { 129 + void *page; 130 + ptable_desc *new; 131 + 132 + if (!(page = (void *)get_zeroed_page(GFP_KERNEL))) 133 + return NULL; 134 + 135 + if (type == TABLE_PTE) { 136 + /* 137 + * m68k doesn't have SPLIT_PTE_PTLOCKS for not having 138 + * SMP. 139 + */ 140 + pgtable_pte_page_ctor(virt_to_page(page)); 141 + } 142 + 143 + mmu_page_ctor(page); 144 + 145 + new = PD_PTABLE(page); 146 + PD_MARKBITS(new) = ptable_mask(type) - 1; 147 + list_add_tail(new, dp); 148 + 149 + return (pmd_t *)page; 150 + } 151 + 152 + for (tmp = 1, off = 0; (mask & tmp) == 0; tmp <<= 1, off += ptable_size(type)) 153 + ; 154 + PD_MARKBITS(dp) = mask & ~tmp; 155 + if (!PD_MARKBITS(dp)) { 156 + /* move to end of list */ 157 + list_move_tail(dp, &ptable_list[type]); 158 + } 159 + return page_address(PD_PAGE(dp)) + off; 160 + } 161 + 162 + int free_pointer_table(void *table, int type) 163 + { 164 + ptable_desc *dp; 165 + unsigned long ptable = (unsigned long)table; 166 + unsigned long page = ptable & PAGE_MASK; 167 + unsigned int mask = 1U << ((ptable - page)/ptable_size(type)); 168 + 169 + dp = PD_PTABLE(page); 170 + if (PD_MARKBITS (dp) & mask) 171 + panic ("table already free!"); 172 + 173 + PD_MARKBITS (dp) |= mask; 174 + 175 + if (PD_MARKBITS(dp) == ptable_mask(type)) { 176 + /* all tables in page are free, free page */ 177 + list_del(dp); 178 + mmu_page_dtor((void *)page); 179 + if (type == TABLE_PTE) 180 + pgtable_pte_page_dtor(virt_to_page(page)); 181 + free_page (page); 182 + return 1; 183 + } else if (ptable_list[type].next != dp) { 184 + /* 185 + * move this descriptor to the front of the list, since 186 + * it has one or more free tables. 187 + */ 188 + list_move(dp, &ptable_list[type]); 189 + } 190 + return 0; 191 + } 192 + 48 193 /* size of memory already mapped in head.S */ 49 194 extern __initdata unsigned long m68k_init_mapped_size; 50 195 51 196 extern unsigned long availmem; 52 197 198 + static pte_t *last_pte_table __initdata = NULL; 199 + 53 200 static pte_t * __init kernel_page_table(void) 54 201 { 55 - pte_t *ptablep; 202 + pte_t *pte_table = last_pte_table; 56 203 57 - ptablep = (pte_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); 58 - if (!ptablep) 59 - panic("%s: Failed to allocate %lu bytes align=%lx\n", 60 - __func__, PAGE_SIZE, PAGE_SIZE); 204 + if (((unsigned long)last_pte_table & ~PAGE_MASK) == 0) { 205 + pte_table = (pte_t *)memblock_alloc_low(PAGE_SIZE, PAGE_SIZE); 206 + if (!pte_table) { 207 + panic("%s: Failed to allocate %lu bytes align=%lx\n", 208 + __func__, PAGE_SIZE, PAGE_SIZE); 209 + } 61 210 62 - clear_page(ptablep); 63 - __flush_page_to_ram(ptablep); 64 - flush_tlb_kernel_page(ptablep); 65 - nocache_page(ptablep); 211 + clear_page(pte_table); 212 + mmu_page_ctor(pte_table); 66 213 67 - return ptablep; 214 + last_pte_table = pte_table; 215 + } 216 + 217 + last_pte_table += PTRS_PER_PTE; 218 + 219 + return pte_table; 68 220 } 69 221 70 - static pmd_t *last_pgtable __initdata = NULL; 71 - pmd_t *zero_pgtable __initdata = NULL; 222 + static pmd_t *last_pmd_table __initdata = NULL; 72 223 73 224 static pmd_t * __init kernel_ptr_table(void) 74 225 { 75 - if (!last_pgtable) { 226 + if (!last_pmd_table) { 76 227 unsigned long pmd, last; 77 228 int i; 78 229 ··· 242 91 last = pmd; 243 92 } 244 93 245 - last_pgtable = (pmd_t *)last; 94 + last_pmd_table = (pmd_t *)last; 246 95 #ifdef DEBUG 247 - printk("kernel_ptr_init: %p\n", last_pgtable); 96 + printk("kernel_ptr_init: %p\n", last_pmd_table); 248 97 #endif 249 98 } 250 99 251 - last_pgtable += PTRS_PER_PMD; 252 - if (((unsigned long)last_pgtable & ~PAGE_MASK) == 0) { 253 - last_pgtable = (pmd_t *)memblock_alloc_low(PAGE_SIZE, 100 + last_pmd_table += PTRS_PER_PMD; 101 + if (((unsigned long)last_pmd_table & ~PAGE_MASK) == 0) { 102 + last_pmd_table = (pmd_t *)memblock_alloc_low(PAGE_SIZE, 254 103 PAGE_SIZE); 255 - if (!last_pgtable) 104 + if (!last_pmd_table) 256 105 panic("%s: Failed to allocate %lu bytes align=%lx\n", 257 106 __func__, PAGE_SIZE, PAGE_SIZE); 258 107 259 - clear_page(last_pgtable); 260 - __flush_page_to_ram(last_pgtable); 261 - flush_tlb_kernel_page(last_pgtable); 262 - nocache_page(last_pgtable); 108 + clear_page(last_pmd_table); 109 + mmu_page_ctor(last_pmd_table); 263 110 } 264 111 265 - return last_pgtable; 112 + return last_pmd_table; 266 113 } 267 114 268 115 static void __init map_node(int node) 269 116 { 270 - #define PTRTREESIZE (256*1024) 271 - #define ROOTTREESIZE (32*1024*1024) 272 117 unsigned long physaddr, virtaddr, size; 273 118 pgd_t *pgd_dir; 274 119 p4d_t *p4d_dir; ··· 282 135 283 136 while (size > 0) { 284 137 #ifdef DEBUG 285 - if (!(virtaddr & (PTRTREESIZE-1))) 138 + if (!(virtaddr & (PMD_SIZE-1))) 286 139 printk ("\npa=%#lx va=%#lx ", physaddr & PAGE_MASK, 287 140 virtaddr); 288 141 #endif 289 142 pgd_dir = pgd_offset_k(virtaddr); 290 143 if (virtaddr && CPU_IS_020_OR_030) { 291 - if (!(virtaddr & (ROOTTREESIZE-1)) && 292 - size >= ROOTTREESIZE) { 144 + if (!(virtaddr & (PGDIR_SIZE-1)) && 145 + size >= PGDIR_SIZE) { 293 146 #ifdef DEBUG 294 147 printk ("[very early term]"); 295 148 #endif 296 149 pgd_val(*pgd_dir) = physaddr; 297 - size -= ROOTTREESIZE; 298 - virtaddr += ROOTTREESIZE; 299 - physaddr += ROOTTREESIZE; 150 + size -= PGDIR_SIZE; 151 + virtaddr += PGDIR_SIZE; 152 + physaddr += PGDIR_SIZE; 300 153 continue; 301 154 } 302 155 } ··· 316 169 #ifdef DEBUG 317 170 printk ("[early term]"); 318 171 #endif 319 - pmd_dir->pmd[(virtaddr/PTRTREESIZE) & 15] = physaddr; 320 - physaddr += PTRTREESIZE; 172 + pmd_val(*pmd_dir) = physaddr; 173 + physaddr += PMD_SIZE; 321 174 } else { 322 175 int i; 323 176 #ifdef DEBUG 324 177 printk ("[zero map]"); 325 178 #endif 326 - zero_pgtable = kernel_ptr_table(); 327 - pte_dir = (pte_t *)zero_pgtable; 328 - pmd_dir->pmd[0] = virt_to_phys(pte_dir) | 329 - _PAGE_TABLE | _PAGE_ACCESSED; 179 + pte_dir = kernel_page_table(); 180 + pmd_set(pmd_dir, pte_dir); 181 + 330 182 pte_val(*pte_dir++) = 0; 331 183 physaddr += PAGE_SIZE; 332 - for (i = 1; i < 64; physaddr += PAGE_SIZE, i++) 184 + for (i = 1; i < PTRS_PER_PTE; physaddr += PAGE_SIZE, i++) 333 185 pte_val(*pte_dir++) = physaddr; 334 186 } 335 - size -= PTRTREESIZE; 336 - virtaddr += PTRTREESIZE; 187 + size -= PMD_SIZE; 188 + virtaddr += PMD_SIZE; 337 189 } else { 338 190 if (!pmd_present(*pmd_dir)) { 339 191 #ifdef DEBUG