LoongArch: Adjust memory management for 32BIT/64BIT

+13

arch/loongarch/include/asm/addrspace.h

··· 38 38 #endif 39 39 40 40 #ifndef WRITECOMBINE_BASE 41 + #ifdef CONFIG_32BIT 42 + #define WRITECOMBINE_BASE CSR_DMW0_BASE 43 + #else 41 44 #define WRITECOMBINE_BASE CSR_DMW2_BASE 42 45 #endif 46 + #endif 43 47 48 + #ifdef CONFIG_32BIT 49 + #define DMW_PABITS 29 50 + #define TO_PHYS_MASK ((_UL(1) << _UL(DMW_PABITS)) - 1) 51 + #else 44 52 #define DMW_PABITS 48 45 53 #define TO_PHYS_MASK ((_ULL(1) << _ULL(DMW_PABITS)) - 1) 54 + #endif 46 55 47 56 /* 48 57 * Memory above this physical address will be considered highmem. ··· 121 112 /* 122 113 * Returns the physical address of a KPRANGEx / XKPRANGE address 123 114 */ 115 + #ifdef CONFIG_32BIT 116 + #define PHYSADDR(a) ((_ACAST32_(a)) & TO_PHYS_MASK) 117 + #else 124 118 #define PHYSADDR(a) ((_ACAST64_(a)) & TO_PHYS_MASK) 119 + #endif 125 120 126 121 /* 127 122 * On LoongArch, I/O ports mappring is following:

-3

arch/loongarch/include/asm/cpu-features.h

··· 20 20 #define cpu_has_loongarch64 (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_64BIT) 21 21 22 22 #ifdef CONFIG_32BIT 23 - # define cpu_has_64bits (cpu_data[0].isa_level & LOONGARCH_CPU_ISA_64BIT) 24 23 # define cpu_vabits 31 25 24 # define cpu_pabits 31 26 25 #endif 27 26 28 27 #ifdef CONFIG_64BIT 29 - # define cpu_has_64bits 1 30 28 # define cpu_vabits cpu_data[0].vabits 31 29 # define cpu_pabits cpu_data[0].pabits 32 - # define __NEED_ADDRBITS_PROBE 33 30 #endif 34 31 35 32 /*

+24

arch/loongarch/include/asm/loongarch.h

··· 912 912 #define LOONGARCH_CSR_DMWIN3 0x183 /* 64 direct map win3: MEM */ 913 913 914 914 /* Direct Map window 0/1/2/3 */ 915 + 916 + #ifdef CONFIG_32BIT 917 + 918 + #define CSR_DMW0_PLV0 (1 << 0) 919 + #define CSR_DMW0_VSEG (0x4) 920 + #define CSR_DMW0_BASE (CSR_DMW0_VSEG << DMW_PABITS) 921 + #define CSR_DMW0_INIT (CSR_DMW0_BASE | CSR_DMW0_PLV0) 922 + 923 + #define CSR_DMW1_PLV0 (1 << 0) 924 + #define CSR_DMW1_MAT (1 << 4) 925 + #define CSR_DMW1_VSEG (0x5) 926 + #define CSR_DMW1_BASE (CSR_DMW1_VSEG << DMW_PABITS) 927 + #define CSR_DMW1_INIT (CSR_DMW1_BASE | CSR_DMW1_MAT | CSR_DMW1_PLV0) 928 + 929 + #define CSR_DMW2_INIT 0x0 930 + 931 + #define CSR_DMW3_INIT 0x0 932 + 933 + #else 934 + 915 935 #define CSR_DMW0_PLV0 _CONST64_(1 << 0) 916 936 #define CSR_DMW0_VSEG _CONST64_(0x8000) 917 937 #define CSR_DMW0_BASE (CSR_DMW0_VSEG << DMW_PABITS) ··· 950 930 #define CSR_DMW2_INIT (CSR_DMW2_BASE | CSR_DMW2_MAT | CSR_DMW2_PLV0) 951 931 952 932 #define CSR_DMW3_INIT 0x0 933 + 934 + #endif 953 935 954 936 /* Performance Counter registers */ 955 937 #define LOONGARCH_CSR_PERFCTRL0 0x200 /* 32 perf event 0 config */ ··· 1410 1388 #define ENTRYLO_C_SHIFT 4 1411 1389 #define ENTRYLO_C (_ULCAST_(3) << ENTRYLO_C_SHIFT) 1412 1390 #define ENTRYLO_G (_ULCAST_(1) << 6) 1391 + #ifdef CONFIG_64BIT 1413 1392 #define ENTRYLO_NR (_ULCAST_(1) << 61) 1414 1393 #define ENTRYLO_NX (_ULCAST_(1) << 62) 1394 + #endif 1415 1395 1416 1396 /* Values for PageSize register */ 1417 1397 #define PS_4K 0x0000000c

+1 -1

arch/loongarch/include/asm/page.h

··· 10 10 11 11 #include <vdso/page.h> 12 12 13 - #define HPAGE_SHIFT (PAGE_SHIFT + PAGE_SHIFT - 3) 13 + #define HPAGE_SHIFT (PAGE_SHIFT + PAGE_SHIFT - PTRLOG) 14 14 #define HPAGE_SIZE (_AC(1, UL) << HPAGE_SHIFT) 15 15 #define HPAGE_MASK (~(HPAGE_SIZE - 1)) 16 16 #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT)

+34 -2

arch/loongarch/include/asm/pgtable-bits.h

··· 6 6 #define _ASM_PGTABLE_BITS_H 7 7 8 8 /* Page table bits */ 9 + 10 + #ifdef CONFIG_32BIT 11 + #define _PAGE_VALID_SHIFT 0 12 + #define _PAGE_ACCESSED_SHIFT 0 /* Reuse Valid for Accessed */ 13 + #define _PAGE_DIRTY_SHIFT 1 14 + #define _PAGE_PLV_SHIFT 2 /* 2~3, two bits */ 15 + #define _CACHE_SHIFT 4 /* 4~5, two bits */ 16 + #define _PAGE_GLOBAL_SHIFT 6 17 + #define _PAGE_HUGE_SHIFT 6 /* HUGE is a PMD bit */ 18 + #define _PAGE_PRESENT_SHIFT 7 19 + #define _PAGE_PFN_SHIFT 8 20 + #define _PAGE_HGLOBAL_SHIFT 12 /* HGlobal is a PMD bit */ 21 + #define _PAGE_SWP_EXCLUSIVE_SHIFT 13 22 + #define _PAGE_PFN_END_SHIFT 28 23 + #define _PAGE_WRITE_SHIFT 29 24 + #define _PAGE_MODIFIED_SHIFT 30 25 + #define _PAGE_PRESENT_INVALID_SHIFT 31 26 + #endif 27 + 28 + #ifdef CONFIG_64BIT 9 29 #define _PAGE_VALID_SHIFT 0 10 30 #define _PAGE_ACCESSED_SHIFT 0 /* Reuse Valid for Accessed */ 11 31 #define _PAGE_DIRTY_SHIFT 1 ··· 38 18 #define _PAGE_MODIFIED_SHIFT 9 39 19 #define _PAGE_PROTNONE_SHIFT 10 40 20 #define _PAGE_SPECIAL_SHIFT 11 41 - #define _PAGE_HGLOBAL_SHIFT 12 /* HGlobal is a PMD bit */ 42 21 #define _PAGE_PFN_SHIFT 12 22 + #define _PAGE_HGLOBAL_SHIFT 12 /* HGlobal is a PMD bit */ 43 23 #define _PAGE_SWP_EXCLUSIVE_SHIFT 23 44 24 #define _PAGE_PFN_END_SHIFT 48 45 25 #define _PAGE_PRESENT_INVALID_SHIFT 60 46 26 #define _PAGE_NO_READ_SHIFT 61 47 27 #define _PAGE_NO_EXEC_SHIFT 62 48 28 #define _PAGE_RPLV_SHIFT 63 29 + #endif 49 30 50 31 /* Used by software */ 51 32 #define _PAGE_PRESENT (_ULCAST_(1) << _PAGE_PRESENT_SHIFT) ··· 54 33 #define _PAGE_WRITE (_ULCAST_(1) << _PAGE_WRITE_SHIFT) 55 34 #define _PAGE_ACCESSED (_ULCAST_(1) << _PAGE_ACCESSED_SHIFT) 56 35 #define _PAGE_MODIFIED (_ULCAST_(1) << _PAGE_MODIFIED_SHIFT) 36 + #ifdef CONFIG_32BIT 37 + #define _PAGE_PROTNONE 0 38 + #define _PAGE_SPECIAL 0 39 + #else 57 40 #define _PAGE_PROTNONE (_ULCAST_(1) << _PAGE_PROTNONE_SHIFT) 58 41 #define _PAGE_SPECIAL (_ULCAST_(1) << _PAGE_SPECIAL_SHIFT) 42 + #endif 59 43 60 - /* We borrow bit 23 to store the exclusive marker in swap PTEs. */ 44 + /* We borrow bit 13/23 to store the exclusive marker in swap PTEs. */ 61 45 #define _PAGE_SWP_EXCLUSIVE (_ULCAST_(1) << _PAGE_SWP_EXCLUSIVE_SHIFT) 62 46 63 47 /* Used by TLB hardware (placed in EntryLo*) */ ··· 72 46 #define _PAGE_GLOBAL (_ULCAST_(1) << _PAGE_GLOBAL_SHIFT) 73 47 #define _PAGE_HUGE (_ULCAST_(1) << _PAGE_HUGE_SHIFT) 74 48 #define _PAGE_HGLOBAL (_ULCAST_(1) << _PAGE_HGLOBAL_SHIFT) 49 + #ifdef CONFIG_32BIT 50 + #define _PAGE_NO_READ 0 51 + #define _PAGE_NO_EXEC 0 52 + #define _PAGE_RPLV 0 53 + #else 75 54 #define _PAGE_NO_READ (_ULCAST_(1) << _PAGE_NO_READ_SHIFT) 76 55 #define _PAGE_NO_EXEC (_ULCAST_(1) << _PAGE_NO_EXEC_SHIFT) 77 56 #define _PAGE_RPLV (_ULCAST_(1) << _PAGE_RPLV_SHIFT) 57 + #endif 78 58 #define _CACHE_MASK (_ULCAST_(3) << _CACHE_SHIFT) 79 59 #define PFN_PTE_SHIFT (PAGE_SHIFT - 12 + _PAGE_PFN_SHIFT) 80 60

+58 -23

arch/loongarch/include/asm/pgtable.h

··· 11 11 12 12 #include <linux/compiler.h> 13 13 #include <asm/addrspace.h> 14 + #include <asm/asm.h> 14 15 #include <asm/page.h> 15 16 #include <asm/pgtable-bits.h> 16 17 ··· 24 23 #endif 25 24 26 25 #if CONFIG_PGTABLE_LEVELS == 2 27 - #define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - 3)) 26 + #define PGDIR_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - PTRLOG)) 28 27 #elif CONFIG_PGTABLE_LEVELS == 3 29 - #define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - 3)) 28 + #define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - PTRLOG)) 30 29 #define PMD_SIZE (1UL << PMD_SHIFT) 31 30 #define PMD_MASK (~(PMD_SIZE-1)) 32 - #define PGDIR_SHIFT (PMD_SHIFT + (PAGE_SHIFT - 3)) 31 + #define PGDIR_SHIFT (PMD_SHIFT + (PAGE_SHIFT - PTRLOG)) 33 32 #elif CONFIG_PGTABLE_LEVELS == 4 34 - #define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - 3)) 33 + #define PMD_SHIFT (PAGE_SHIFT + (PAGE_SHIFT - PTRLOG)) 35 34 #define PMD_SIZE (1UL << PMD_SHIFT) 36 35 #define PMD_MASK (~(PMD_SIZE-1)) 37 - #define PUD_SHIFT (PMD_SHIFT + (PAGE_SHIFT - 3)) 36 + #define PUD_SHIFT (PMD_SHIFT + (PAGE_SHIFT - PTRLOG)) 38 37 #define PUD_SIZE (1UL << PUD_SHIFT) 39 38 #define PUD_MASK (~(PUD_SIZE-1)) 40 - #define PGDIR_SHIFT (PUD_SHIFT + (PAGE_SHIFT - 3)) 39 + #define PGDIR_SHIFT (PUD_SHIFT + (PAGE_SHIFT - PTRLOG)) 41 40 #endif 42 41 43 42 #define PGDIR_SIZE (1UL << PGDIR_SHIFT) 44 43 #define PGDIR_MASK (~(PGDIR_SIZE-1)) 45 44 46 - #define VA_BITS (PGDIR_SHIFT + (PAGE_SHIFT - 3)) 45 + #ifdef CONFIG_32BIT 46 + #define VA_BITS 32 47 + #else 48 + #define VA_BITS (PGDIR_SHIFT + (PAGE_SHIFT - PTRLOG)) 49 + #endif 47 50 48 - #define PTRS_PER_PGD (PAGE_SIZE >> 3) 51 + #define PTRS_PER_PGD (PAGE_SIZE >> PTRLOG) 49 52 #if CONFIG_PGTABLE_LEVELS > 3 50 - #define PTRS_PER_PUD (PAGE_SIZE >> 3) 53 + #define PTRS_PER_PUD (PAGE_SIZE >> PTRLOG) 51 54 #endif 52 55 #if CONFIG_PGTABLE_LEVELS > 2 53 - #define PTRS_PER_PMD (PAGE_SIZE >> 3) 56 + #define PTRS_PER_PMD (PAGE_SIZE >> PTRLOG) 54 57 #endif 55 - #define PTRS_PER_PTE (PAGE_SIZE >> 3) 58 + #define PTRS_PER_PTE (PAGE_SIZE >> PTRLOG) 56 59 60 + #ifdef CONFIG_32BIT 61 + #define USER_PTRS_PER_PGD (TASK_SIZE / PGDIR_SIZE) 62 + #else 57 63 #define USER_PTRS_PER_PGD ((TASK_SIZE64 / PGDIR_SIZE)?(TASK_SIZE64 / PGDIR_SIZE):1) 64 + #endif 58 65 59 66 #ifndef __ASSEMBLER__ 60 67 ··· 83 74 84 75 #define ZERO_PAGE(vaddr) virt_to_page(empty_zero_page) 85 76 86 - /* 87 - * TLB refill handlers may also map the vmalloc area into xkvrange. 88 - * Avoid the first couple of pages so NULL pointer dereferences will 89 - * still reliably trap. 90 - */ 77 + #ifdef CONFIG_32BIT 78 + 79 + #define VMALLOC_START (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE)) 80 + #define VMALLOC_END (FIXADDR_START - (2 * PAGE_SIZE)) 81 + 82 + #endif 83 + 84 + #ifdef CONFIG_64BIT 85 + 91 86 #define MODULES_VADDR (vm_map_base + PCI_IOSIZE + (2 * PAGE_SIZE)) 92 87 #define MODULES_END (MODULES_VADDR + SZ_256M) 93 88 ··· 118 105 119 106 #define KFENCE_AREA_START (VMEMMAP_END + 1) 120 107 #define KFENCE_AREA_END (KFENCE_AREA_START + KFENCE_AREA_SIZE - 1) 108 + 109 + #endif 121 110 122 111 #define ptep_get(ptep) READ_ONCE(*(ptep)) 123 112 #define pmdp_get(pmdp) READ_ONCE(*(pmdp)) ··· 292 277 * Encode/decode swap entries and swap PTEs. Swap PTEs are all PTEs that 293 278 * are !pte_none() && !pte_present(). 294 279 * 295 - * Format of swap PTEs: 280 + * Format of 32bit swap PTEs: 281 + * 282 + * 3 3 2 2 2 2 2 2 2 2 2 2 1 1 1 1 1 1 1 1 1 1 283 + * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 284 + * <------------ offset -------------> E <- type -> <-- zeroes --> 285 + * 286 + * E is the exclusive marker that is not stored in swap entries. 287 + * The zero'ed bits include _PAGE_PRESENT. 288 + * 289 + * Format of 64bit swap PTEs: 296 290 * 297 291 * 6 6 6 6 5 5 5 5 5 5 5 5 5 5 4 4 4 4 4 4 4 4 4 4 3 3 3 3 3 3 3 3 298 292 * 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 ··· 314 290 * E is the exclusive marker that is not stored in swap entries. 315 291 * The zero'ed bits include _PAGE_PRESENT and _PAGE_PROTNONE. 316 292 */ 317 - static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) 318 - { pte_t pte; pte_val(pte) = ((type & 0x7f) << 16) | (offset << 24); return pte; } 319 293 320 - #define __swp_type(x) (((x).val >> 16) & 0x7f) 321 - #define __swp_offset(x) ((x).val >> 24) 294 + #define __SWP_TYPE_BITS (IS_ENABLED(CONFIG_32BIT) ? 5 : 7) 295 + #define __SWP_TYPE_MASK ((1UL << __SWP_TYPE_BITS) - 1) 296 + #define __SWP_TYPE_SHIFT (IS_ENABLED(CONFIG_32BIT) ? 8 : 16) 297 + #define __SWP_OFFSET_SHIFT (__SWP_TYPE_BITS + __SWP_TYPE_SHIFT + 1) 298 + 299 + static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) 300 + { 301 + pte_t pte; 302 + pte_val(pte) = ((type & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT) | (offset << __SWP_OFFSET_SHIFT); 303 + return pte; 304 + } 305 + 306 + #define __swp_type(x) (((x).val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK) 307 + #define __swp_offset(x) ((x).val >> __SWP_OFFSET_SHIFT) 322 308 #define __swp_entry(type, offset) ((swp_entry_t) { pte_val(mk_swap_pte((type), (offset))) }) 323 - #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) 309 + 324 310 #define __swp_entry_to_pte(x) __pte((x).val) 325 - #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) 326 311 #define __swp_entry_to_pmd(x) __pmd((x).val | _PAGE_HUGE) 312 + #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) 313 + #define __pmd_to_swp_entry(pmd) ((swp_entry_t) { pmd_val(pmd) }) 327 314 328 315 static inline bool pte_swp_exclusive(pte_t pte) 329 316 {

+5 -1

arch/loongarch/kernel/cpu-probe.c

··· 106 106 107 107 static void cpu_probe_addrbits(struct cpuinfo_loongarch *c) 108 108 { 109 - #ifdef __NEED_ADDRBITS_PROBE 109 + #ifdef CONFIG_32BIT 110 + c->pabits = cpu_pabits; 111 + c->vabits = cpu_vabits; 112 + vm_map_base = KVRANGE; 113 + #else 110 114 c->pabits = (read_cpucfg(LOONGARCH_CPUCFG1) & CPUCFG1_PABITS) >> 4; 111 115 c->vabits = (read_cpucfg(LOONGARCH_CPUCFG1) & CPUCFG1_VABITS) >> 12; 112 116 vm_map_base = 0UL - (1UL << c->vabits);

+8

arch/loongarch/lib/dump_tlb.c

··· 73 73 vwidth, (entryhi & ~0x1fffUL), asidwidth, asid & asidmask); 74 74 75 75 /* NR/NX are in awkward places, so mask them off separately */ 76 + #ifdef CONFIG_64BIT 76 77 pa = entrylo0 & ~(ENTRYLO_NR | ENTRYLO_NX); 78 + #endif 77 79 pa = pa & PAGE_MASK; 78 80 pr_cont("\n\t["); 81 + #ifdef CONFIG_64BIT 79 82 pr_cont("nr=%d nx=%d ", 80 83 (entrylo0 & ENTRYLO_NR) ? 1 : 0, 81 84 (entrylo0 & ENTRYLO_NX) ? 1 : 0); 85 + #endif 82 86 pr_cont("pa=0x%0*llx c=%d d=%d v=%d g=%d plv=%lld] [", 83 87 pwidth, pa, c0, 84 88 (entrylo0 & ENTRYLO_D) ? 1 : 0, ··· 90 86 (entrylo0 & ENTRYLO_G) ? 1 : 0, 91 87 (entrylo0 & ENTRYLO_PLV) >> ENTRYLO_PLV_SHIFT); 92 88 /* NR/NX are in awkward places, so mask them off separately */ 89 + #ifdef CONFIG_64BIT 93 90 pa = entrylo1 & ~(ENTRYLO_NR | ENTRYLO_NX); 91 + #endif 94 92 pa = pa & PAGE_MASK; 93 + #ifdef CONFIG_64BIT 95 94 pr_cont("nr=%d nx=%d ", 96 95 (entrylo1 & ENTRYLO_NR) ? 1 : 0, 97 96 (entrylo1 & ENTRYLO_NX) ? 1 : 0); 97 + #endif 98 98 pr_cont("pa=0x%0*llx c=%d d=%d v=%d g=%d plv=%lld]\n", 99 99 pwidth, pa, c1, 100 100 (entrylo1 & ENTRYLO_D) ? 1 : 0,

+2 -2

arch/loongarch/mm/init.c

··· 224 224 pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss; 225 225 EXPORT_SYMBOL(invalid_pte_table); 226 226 227 - #ifdef CONFIG_EXECMEM 227 + #if defined(CONFIG_EXECMEM) && defined(MODULES_VADDR) 228 228 static struct execmem_info execmem_info __ro_after_init; 229 229 230 230 struct execmem_info __init *execmem_arch_setup(void) ··· 242 242 243 243 return &execmem_info; 244 244 } 245 - #endif /* CONFIG_EXECMEM */ 245 + #endif /* CONFIG_EXECMEM && MODULES_VADDR */

+59 -59

arch/loongarch/mm/page.S

··· 10 10 11 11 .align 5 12 12 SYM_FUNC_START(clear_page) 13 - lu12i.w t0, 1 << (PAGE_SHIFT - 12) 14 - add.d t0, t0, a0 13 + lu12i.w t0, 1 << (PAGE_SHIFT - 12) 14 + PTR_ADD t0, t0, a0 15 15 1: 16 - st.d zero, a0, 0 17 - st.d zero, a0, 8 18 - st.d zero, a0, 16 19 - st.d zero, a0, 24 20 - st.d zero, a0, 32 21 - st.d zero, a0, 40 22 - st.d zero, a0, 48 23 - st.d zero, a0, 56 24 - addi.d a0, a0, 128 25 - st.d zero, a0, -64 26 - st.d zero, a0, -56 27 - st.d zero, a0, -48 28 - st.d zero, a0, -40 29 - st.d zero, a0, -32 30 - st.d zero, a0, -24 31 - st.d zero, a0, -16 32 - st.d zero, a0, -8 33 - bne t0, a0, 1b 16 + LONG_S zero, a0, (LONGSIZE * 0) 17 + LONG_S zero, a0, (LONGSIZE * 1) 18 + LONG_S zero, a0, (LONGSIZE * 2) 19 + LONG_S zero, a0, (LONGSIZE * 3) 20 + LONG_S zero, a0, (LONGSIZE * 4) 21 + LONG_S zero, a0, (LONGSIZE * 5) 22 + LONG_S zero, a0, (LONGSIZE * 6) 23 + LONG_S zero, a0, (LONGSIZE * 7) 24 + PTR_ADDI a0, a0, (LONGSIZE * 16) 25 + LONG_S zero, a0, -(LONGSIZE * 8) 26 + LONG_S zero, a0, -(LONGSIZE * 7) 27 + LONG_S zero, a0, -(LONGSIZE * 6) 28 + LONG_S zero, a0, -(LONGSIZE * 5) 29 + LONG_S zero, a0, -(LONGSIZE * 4) 30 + LONG_S zero, a0, -(LONGSIZE * 3) 31 + LONG_S zero, a0, -(LONGSIZE * 2) 32 + LONG_S zero, a0, -(LONGSIZE * 1) 33 + bne t0, a0, 1b 34 34 35 - jr ra 35 + jr ra 36 36 SYM_FUNC_END(clear_page) 37 37 EXPORT_SYMBOL(clear_page) 38 38 39 39 .align 5 40 40 SYM_FUNC_START(copy_page) 41 - lu12i.w t8, 1 << (PAGE_SHIFT - 12) 42 - add.d t8, t8, a0 41 + lu12i.w t8, 1 << (PAGE_SHIFT - 12) 42 + PTR_ADD t8, t8, a0 43 43 1: 44 - ld.d t0, a1, 0 45 - ld.d t1, a1, 8 46 - ld.d t2, a1, 16 47 - ld.d t3, a1, 24 48 - ld.d t4, a1, 32 49 - ld.d t5, a1, 40 50 - ld.d t6, a1, 48 51 - ld.d t7, a1, 56 44 + LONG_L t0, a1, (LONGSIZE * 0) 45 + LONG_L t1, a1, (LONGSIZE * 1) 46 + LONG_L t2, a1, (LONGSIZE * 2) 47 + LONG_L t3, a1, (LONGSIZE * 3) 48 + LONG_L t4, a1, (LONGSIZE * 4) 49 + LONG_L t5, a1, (LONGSIZE * 5) 50 + LONG_L t6, a1, (LONGSIZE * 6) 51 + LONG_L t7, a1, (LONGSIZE * 7) 52 52 53 - st.d t0, a0, 0 54 - st.d t1, a0, 8 55 - ld.d t0, a1, 64 56 - ld.d t1, a1, 72 57 - st.d t2, a0, 16 58 - st.d t3, a0, 24 59 - ld.d t2, a1, 80 60 - ld.d t3, a1, 88 61 - st.d t4, a0, 32 62 - st.d t5, a0, 40 63 - ld.d t4, a1, 96 64 - ld.d t5, a1, 104 65 - st.d t6, a0, 48 66 - st.d t7, a0, 56 67 - ld.d t6, a1, 112 68 - ld.d t7, a1, 120 69 - addi.d a0, a0, 128 70 - addi.d a1, a1, 128 53 + LONG_S t0, a0, (LONGSIZE * 0) 54 + LONG_S t1, a0, (LONGSIZE * 1) 55 + LONG_L t0, a1, (LONGSIZE * 8) 56 + LONG_L t1, a1, (LONGSIZE * 9) 57 + LONG_S t2, a0, (LONGSIZE * 2) 58 + LONG_S t3, a0, (LONGSIZE * 3) 59 + LONG_L t2, a1, (LONGSIZE * 10) 60 + LONG_L t3, a1, (LONGSIZE * 11) 61 + LONG_S t4, a0, (LONGSIZE * 4) 62 + LONG_S t5, a0, (LONGSIZE * 5) 63 + LONG_L t4, a1, (LONGSIZE * 12) 64 + LONG_L t5, a1, (LONGSIZE * 13) 65 + LONG_S t6, a0, (LONGSIZE * 6) 66 + LONG_S t7, a0, (LONGSIZE * 7) 67 + LONG_L t6, a1, (LONGSIZE * 14) 68 + LONG_L t7, a1, (LONGSIZE * 15) 69 + PTR_ADDI a0, a0, (LONGSIZE * 16) 70 + PTR_ADDI a1, a1, (LONGSIZE * 16) 71 71 72 - st.d t0, a0, -64 73 - st.d t1, a0, -56 74 - st.d t2, a0, -48 75 - st.d t3, a0, -40 76 - st.d t4, a0, -32 77 - st.d t5, a0, -24 78 - st.d t6, a0, -16 79 - st.d t7, a0, -8 72 + LONG_S t0, a0, -(LONGSIZE * 8) 73 + LONG_S t1, a0, -(LONGSIZE * 7) 74 + LONG_S t2, a0, -(LONGSIZE * 6) 75 + LONG_S t3, a0, -(LONGSIZE * 5) 76 + LONG_S t4, a0, -(LONGSIZE * 4) 77 + LONG_S t5, a0, -(LONGSIZE * 3) 78 + LONG_S t6, a0, -(LONGSIZE * 2) 79 + LONG_S t7, a0, -(LONGSIZE * 1) 80 80 81 - bne t8, a0, 1b 82 - jr ra 81 + bne t8, a0, 1b 82 + jr ra 83 83 SYM_FUNC_END(copy_page) 84 84 EXPORT_SYMBOL(copy_page)

+2

arch/loongarch/mm/tlb.c

··· 251 251 pr_define("_PAGE_GLOBAL_SHIFT %d\n", _PAGE_GLOBAL_SHIFT); 252 252 pr_define("_PAGE_PRESENT_SHIFT %d\n", _PAGE_PRESENT_SHIFT); 253 253 pr_define("_PAGE_WRITE_SHIFT %d\n", _PAGE_WRITE_SHIFT); 254 + #ifdef CONFIG_64BIT 254 255 pr_define("_PAGE_NO_READ_SHIFT %d\n", _PAGE_NO_READ_SHIFT); 255 256 pr_define("_PAGE_NO_EXEC_SHIFT %d\n", _PAGE_NO_EXEC_SHIFT); 257 + #endif 256 258 pr_define("PFN_PTE_SHIFT %d\n", PFN_PTE_SHIFT); 257 259 pr_debug("\n"); 258 260 }

+216 -106

arch/loongarch/mm/tlbex.S

··· 11 11 12 12 #define INVTLB_ADDR_GFALSE_AND_ASID 5 13 13 14 - #define PTRS_PER_PGD_BITS (PAGE_SHIFT - 3) 15 - #define PTRS_PER_PUD_BITS (PAGE_SHIFT - 3) 16 - #define PTRS_PER_PMD_BITS (PAGE_SHIFT - 3) 17 - #define PTRS_PER_PTE_BITS (PAGE_SHIFT - 3) 14 + #define PTRS_PER_PGD_BITS (PAGE_SHIFT - PTRLOG) 15 + #define PTRS_PER_PUD_BITS (PAGE_SHIFT - PTRLOG) 16 + #define PTRS_PER_PMD_BITS (PAGE_SHIFT - PTRLOG) 17 + #define PTRS_PER_PTE_BITS (PAGE_SHIFT - PTRLOG) 18 + 19 + #ifdef CONFIG_32BIT 20 + #define PTE_LL ll.w 21 + #define PTE_SC sc.w 22 + #else 23 + #define PTE_LL ll.d 24 + #define PTE_SC sc.d 25 + #endif 18 26 19 27 .macro tlb_do_page_fault, write 20 28 SYM_CODE_START(tlb_do_page_fault_\write) ··· 68 60 69 61 vmalloc_done_load: 70 62 /* Get PGD offset in bytes */ 71 - bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 72 - alsl.d t1, ra, t1, 3 63 + #ifdef CONFIG_32BIT 64 + PTR_BSTRPICK ra, t0, 31, PGDIR_SHIFT 65 + #else 66 + PTR_BSTRPICK ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 67 + #endif 68 + PTR_ALSL t1, ra, t1, _PGD_T_LOG2 69 + 73 70 #if CONFIG_PGTABLE_LEVELS > 3 74 - ld.d t1, t1, 0 75 - bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 76 - alsl.d t1, ra, t1, 3 71 + PTR_L t1, t1, 0 72 + PTR_BSTRPICK ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 73 + PTR_ALSL t1, ra, t1, _PMD_T_LOG2 74 + 77 75 #endif 78 76 #if CONFIG_PGTABLE_LEVELS > 2 79 - ld.d t1, t1, 0 80 - bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 81 - alsl.d t1, ra, t1, 3 77 + PTR_L t1, t1, 0 78 + PTR_BSTRPICK ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 79 + PTR_ALSL t1, ra, t1, _PMD_T_LOG2 80 + 82 81 #endif 83 - ld.d ra, t1, 0 82 + PTR_L ra, t1, 0 84 83 85 84 /* 86 85 * For huge tlb entries, pmde doesn't contain an address but 87 86 * instead contains the tlb pte. Check the PAGE_HUGE bit and 88 87 * see if we need to jump to huge tlb processing. 89 88 */ 90 - rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 89 + PTR_ROTRI ra, ra, _PAGE_HUGE_SHIFT + 1 91 90 bltz ra, tlb_huge_update_load 92 91 93 - rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 94 - bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 95 - alsl.d t1, t0, ra, _PTE_T_LOG2 92 + PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) 93 + PTR_BSTRPICK t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 94 + PTR_ALSL t1, t0, ra, _PTE_T_LOG2 96 95 97 96 #ifdef CONFIG_SMP 98 97 smp_pgtable_change_load: 99 - ll.d t0, t1, 0 98 + PTE_LL t0, t1, 0 100 99 #else 101 - ld.d t0, t1, 0 100 + PTR_L t0, t1, 0 102 101 #endif 103 102 andi ra, t0, _PAGE_PRESENT 104 103 beqz ra, nopage_tlb_load 105 104 106 105 ori t0, t0, _PAGE_VALID 106 + 107 107 #ifdef CONFIG_SMP 108 - sc.d t0, t1, 0 108 + PTE_SC t0, t1, 0 109 109 beqz t0, smp_pgtable_change_load 110 110 #else 111 - st.d t0, t1, 0 111 + PTR_S t0, t1, 0 112 112 #endif 113 + 113 114 tlbsrch 114 - bstrins.d t1, zero, 3, 3 115 - ld.d t0, t1, 0 116 - ld.d t1, t1, 8 115 + PTR_BSTRINS t1, zero, _PTE_T_LOG2, _PTE_T_LOG2 116 + PTR_L t0, t1, 0 117 + PTR_L t1, t1, _PTE_T_SIZE 117 118 csrwr t0, LOONGARCH_CSR_TLBELO0 118 119 csrwr t1, LOONGARCH_CSR_TLBELO1 119 120 tlbwr ··· 132 115 csrrd ra, EXCEPTION_KS2 133 116 ertn 134 117 135 - #ifdef CONFIG_64BIT 136 118 vmalloc_load: 137 119 la_abs t1, swapper_pg_dir 138 120 b vmalloc_done_load 139 - #endif 140 121 141 122 /* This is the entry point of a huge page. */ 142 123 tlb_huge_update_load: 143 124 #ifdef CONFIG_SMP 144 - ll.d ra, t1, 0 125 + PTE_LL ra, t1, 0 145 126 #else 146 - rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 127 + PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) 147 128 #endif 148 129 andi t0, ra, _PAGE_PRESENT 149 130 beqz t0, nopage_tlb_load 150 131 151 132 #ifdef CONFIG_SMP 152 133 ori t0, ra, _PAGE_VALID 153 - sc.d t0, t1, 0 134 + PTE_SC t0, t1, 0 154 135 beqz t0, tlb_huge_update_load 155 136 ori t0, ra, _PAGE_VALID 156 137 #else 157 138 ori t0, ra, _PAGE_VALID 158 - st.d t0, t1, 0 139 + PTR_S t0, t1, 0 159 140 #endif 160 141 csrrd ra, LOONGARCH_CSR_ASID 161 142 csrrd t1, LOONGARCH_CSR_BADV ··· 173 158 xori t0, t0, _PAGE_HUGE 174 159 lu12i.w t1, _PAGE_HGLOBAL >> 12 175 160 and t1, t0, t1 176 - srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 161 + PTR_SRLI t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 177 162 or t0, t0, t1 178 163 179 164 move ra, t0 180 165 csrwr ra, LOONGARCH_CSR_TLBELO0 181 166 182 167 /* Convert to entrylo1 */ 183 - addi.d t1, zero, 1 184 - slli.d t1, t1, (HPAGE_SHIFT - 1) 185 - add.d t0, t0, t1 168 + PTR_ADDI t1, zero, 1 169 + PTR_SLLI t1, t1, (HPAGE_SHIFT - 1) 170 + PTR_ADD t0, t0, t1 186 171 csrwr t0, LOONGARCH_CSR_TLBELO1 187 172 188 173 /* Set huge page tlb entry size */ 189 - addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 190 - addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 174 + PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 175 + PTR_LI t1, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT)) 191 176 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 192 177 193 178 tlbfill 194 179 195 - addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 196 - addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 180 + PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 181 + PTR_LI t1, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT)) 197 182 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 198 183 199 184 csrrd t0, EXCEPTION_KS0 ··· 231 216 232 217 vmalloc_done_store: 233 218 /* Get PGD offset in bytes */ 234 - bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 235 - alsl.d t1, ra, t1, 3 219 + #ifdef CONFIG_32BIT 220 + PTR_BSTRPICK ra, t0, 31, PGDIR_SHIFT 221 + #else 222 + PTR_BSTRPICK ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 223 + #endif 224 + PTR_ALSL t1, ra, t1, _PGD_T_LOG2 225 + 236 226 #if CONFIG_PGTABLE_LEVELS > 3 237 - ld.d t1, t1, 0 238 - bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 239 - alsl.d t1, ra, t1, 3 227 + PTR_L t1, t1, 0 228 + PTR_BSTRPICK ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 229 + PTR_ALSL t1, ra, t1, _PMD_T_LOG2 240 230 #endif 241 231 #if CONFIG_PGTABLE_LEVELS > 2 242 - ld.d t1, t1, 0 243 - bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 244 - alsl.d t1, ra, t1, 3 232 + PTR_L t1, t1, 0 233 + PTR_BSTRPICK ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 234 + PTR_ALSL t1, ra, t1, _PMD_T_LOG2 245 235 #endif 246 - ld.d ra, t1, 0 236 + PTR_L ra, t1, 0 247 237 248 238 /* 249 239 * For huge tlb entries, pmde doesn't contain an address but 250 240 * instead contains the tlb pte. Check the PAGE_HUGE bit and 251 241 * see if we need to jump to huge tlb processing. 252 242 */ 253 - rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 243 + PTR_ROTRI ra, ra, _PAGE_HUGE_SHIFT + 1 254 244 bltz ra, tlb_huge_update_store 255 245 256 - rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 257 - bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 258 - alsl.d t1, t0, ra, _PTE_T_LOG2 246 + PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) 247 + PTR_BSTRPICK t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 248 + PTR_ALSL t1, t0, ra, _PTE_T_LOG2 259 249 260 250 #ifdef CONFIG_SMP 261 251 smp_pgtable_change_store: 262 - ll.d t0, t1, 0 252 + PTE_LL t0, t1, 0 263 253 #else 264 - ld.d t0, t1, 0 254 + PTR_L t0, t1, 0 265 255 #endif 256 + 257 + #ifdef CONFIG_64BIT 266 258 andi ra, t0, _PAGE_PRESENT | _PAGE_WRITE 267 259 xori ra, ra, _PAGE_PRESENT | _PAGE_WRITE 260 + #else 261 + PTR_LI ra, _PAGE_PRESENT | _PAGE_WRITE 262 + and ra, ra, t0 263 + nor ra, ra, zero 264 + #endif 268 265 bnez ra, nopage_tlb_store 269 266 267 + #ifdef CONFIG_64BIT 270 268 ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 269 + #else 270 + PTR_LI ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 271 + or t0, ra, t0 272 + #endif 273 + 271 274 #ifdef CONFIG_SMP 272 - sc.d t0, t1, 0 275 + PTE_SC t0, t1, 0 273 276 beqz t0, smp_pgtable_change_store 274 277 #else 275 - st.d t0, t1, 0 278 + PTR_S t0, t1, 0 276 279 #endif 277 280 tlbsrch 278 - bstrins.d t1, zero, 3, 3 279 - ld.d t0, t1, 0 280 - ld.d t1, t1, 8 281 + PTR_BSTRINS t1, zero, _PTE_T_LOG2, _PTE_T_LOG2 282 + PTR_L t0, t1, 0 283 + PTR_L t1, t1, _PTE_T_SIZE 281 284 csrwr t0, LOONGARCH_CSR_TLBELO0 282 285 csrwr t1, LOONGARCH_CSR_TLBELO1 283 286 tlbwr ··· 305 272 csrrd ra, EXCEPTION_KS2 306 273 ertn 307 274 308 - #ifdef CONFIG_64BIT 309 275 vmalloc_store: 310 276 la_abs t1, swapper_pg_dir 311 277 b vmalloc_done_store 312 - #endif 313 278 314 279 /* This is the entry point of a huge page. */ 315 280 tlb_huge_update_store: 316 281 #ifdef CONFIG_SMP 317 - ll.d ra, t1, 0 282 + PTE_LL ra, t1, 0 318 283 #else 319 - rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 284 + PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) 320 285 #endif 286 + 287 + #ifdef CONFIG_64BIT 321 288 andi t0, ra, _PAGE_PRESENT | _PAGE_WRITE 322 289 xori t0, t0, _PAGE_PRESENT | _PAGE_WRITE 290 + #else 291 + PTR_LI t0, _PAGE_PRESENT | _PAGE_WRITE 292 + and t0, t0, ra 293 + nor t0, t0, zero 294 + #endif 295 + 323 296 bnez t0, nopage_tlb_store 324 297 325 298 #ifdef CONFIG_SMP 326 299 ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 327 - sc.d t0, t1, 0 300 + PTE_SC t0, t1, 0 328 301 beqz t0, tlb_huge_update_store 329 302 ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 330 303 #else 304 + #ifdef CONFIG_64BIT 331 305 ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 332 - st.d t0, t1, 0 306 + #else 307 + PTR_LI t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 308 + or t0, ra, t0 309 + #endif 310 + PTR_S t0, t1, 0 333 311 #endif 334 312 csrrd ra, LOONGARCH_CSR_ASID 335 313 csrrd t1, LOONGARCH_CSR_BADV ··· 360 316 xori t0, t0, _PAGE_HUGE 361 317 lu12i.w t1, _PAGE_HGLOBAL >> 12 362 318 and t1, t0, t1 363 - srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 319 + PTR_SRLI t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 364 320 or t0, t0, t1 365 321 366 322 move ra, t0 367 323 csrwr ra, LOONGARCH_CSR_TLBELO0 368 324 369 325 /* Convert to entrylo1 */ 370 - addi.d t1, zero, 1 371 - slli.d t1, t1, (HPAGE_SHIFT - 1) 372 - add.d t0, t0, t1 326 + PTR_ADDI t1, zero, 1 327 + PTR_SLLI t1, t1, (HPAGE_SHIFT - 1) 328 + PTR_ADD t0, t0, t1 373 329 csrwr t0, LOONGARCH_CSR_TLBELO1 374 330 375 331 /* Set huge page tlb entry size */ 376 - addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 377 - addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 332 + PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 333 + PTR_LI t1, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT)) 378 334 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 379 335 380 336 tlbfill 381 337 382 338 /* Reset default page size */ 383 - addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 384 - addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 339 + PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 340 + PTR_LI t1, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT)) 385 341 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 386 342 387 343 csrrd t0, EXCEPTION_KS0 ··· 419 375 420 376 vmalloc_done_modify: 421 377 /* Get PGD offset in bytes */ 422 - bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 423 - alsl.d t1, ra, t1, 3 378 + #ifdef CONFIG_32BIT 379 + PTR_BSTRPICK ra, t0, 31, PGDIR_SHIFT 380 + #else 381 + PTR_BSTRPICK ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 382 + #endif 383 + PTR_ALSL t1, ra, t1, _PGD_T_LOG2 384 + 424 385 #if CONFIG_PGTABLE_LEVELS > 3 425 - ld.d t1, t1, 0 426 - bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 427 - alsl.d t1, ra, t1, 3 386 + PTR_L t1, t1, 0 387 + PTR_BSTRPICK ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 388 + PTR_ALSL t1, ra, t1, _PMD_T_LOG2 428 389 #endif 429 390 #if CONFIG_PGTABLE_LEVELS > 2 430 - ld.d t1, t1, 0 431 - bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 432 - alsl.d t1, ra, t1, 3 391 + PTR_L t1, t1, 0 392 + PTR_BSTRPICK ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 393 + PTR_ALSL t1, ra, t1, _PMD_T_LOG2 433 394 #endif 434 - ld.d ra, t1, 0 395 + PTR_L ra, t1, 0 435 396 436 397 /* 437 398 * For huge tlb entries, pmde doesn't contain an address but 438 399 * instead contains the tlb pte. Check the PAGE_HUGE bit and 439 400 * see if we need to jump to huge tlb processing. 440 401 */ 441 - rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 402 + PTR_ROTRI ra, ra, _PAGE_HUGE_SHIFT + 1 442 403 bltz ra, tlb_huge_update_modify 443 404 444 - rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 445 - bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 446 - alsl.d t1, t0, ra, _PTE_T_LOG2 405 + PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) 406 + PTR_BSTRPICK t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 407 + PTR_ALSL t1, t0, ra, _PTE_T_LOG2 447 408 448 409 #ifdef CONFIG_SMP 449 410 smp_pgtable_change_modify: 450 - ll.d t0, t1, 0 411 + PTE_LL t0, t1, 0 451 412 #else 452 - ld.d t0, t1, 0 413 + PTR_L t0, t1, 0 453 414 #endif 415 + #ifdef CONFIG_64BIT 454 416 andi ra, t0, _PAGE_WRITE 417 + #else 418 + PTR_LI ra, _PAGE_WRITE 419 + and ra, t0, ra 420 + #endif 421 + 455 422 beqz ra, nopage_tlb_modify 456 423 424 + #ifdef CONFIG_64BIT 457 425 ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 426 + #else 427 + PTR_LI ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 428 + or t0, ra, t0 429 + #endif 430 + 458 431 #ifdef CONFIG_SMP 459 - sc.d t0, t1, 0 432 + PTE_SC t0, t1, 0 460 433 beqz t0, smp_pgtable_change_modify 461 434 #else 462 - st.d t0, t1, 0 435 + PTR_S t0, t1, 0 463 436 #endif 464 437 tlbsrch 465 - bstrins.d t1, zero, 3, 3 466 - ld.d t0, t1, 0 467 - ld.d t1, t1, 8 438 + PTR_BSTRINS t1, zero, _PTE_T_LOG2, _PTE_T_LOG2 439 + PTR_L t0, t1, 0 440 + PTR_L t1, t1, _PTE_T_SIZE 468 441 csrwr t0, LOONGARCH_CSR_TLBELO0 469 442 csrwr t1, LOONGARCH_CSR_TLBELO1 470 443 tlbwr ··· 491 430 csrrd ra, EXCEPTION_KS2 492 431 ertn 493 432 494 - #ifdef CONFIG_64BIT 495 433 vmalloc_modify: 496 434 la_abs t1, swapper_pg_dir 497 435 b vmalloc_done_modify 498 - #endif 499 436 500 437 /* This is the entry point of a huge page. */ 501 438 tlb_huge_update_modify: 502 439 #ifdef CONFIG_SMP 503 - ll.d ra, t1, 0 440 + PTE_LL ra, t1, 0 504 441 #else 505 - rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 442 + PTR_ROTRI ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1) 506 443 #endif 444 + 445 + #ifdef CONFIG_64BIT 507 446 andi t0, ra, _PAGE_WRITE 447 + #else 448 + PTR_LI t0, _PAGE_WRITE 449 + and t0, ra, t0 450 + #endif 451 + 508 452 beqz t0, nopage_tlb_modify 509 453 510 454 #ifdef CONFIG_SMP 511 455 ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 512 - sc.d t0, t1, 0 456 + PTE_SC t0, t1, 0 513 457 beqz t0, tlb_huge_update_modify 514 458 ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 515 459 #else 460 + #ifdef CONFIG_64BIT 516 461 ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 517 - st.d t0, t1, 0 462 + #else 463 + PTR_LI t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 464 + or t0, ra, t0 465 + #endif 466 + PTR_S t0, t1, 0 518 467 #endif 519 468 csrrd ra, LOONGARCH_CSR_ASID 520 469 csrrd t1, LOONGARCH_CSR_BADV ··· 544 473 xori t0, t0, _PAGE_HUGE 545 474 lu12i.w t1, _PAGE_HGLOBAL >> 12 546 475 and t1, t0, t1 547 - srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 476 + PTR_SRLI t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 548 477 or t0, t0, t1 549 478 550 479 move ra, t0 551 480 csrwr ra, LOONGARCH_CSR_TLBELO0 552 481 553 482 /* Convert to entrylo1 */ 554 - addi.d t1, zero, 1 555 - slli.d t1, t1, (HPAGE_SHIFT - 1) 556 - add.d t0, t0, t1 483 + PTR_ADDI t1, zero, 1 484 + PTR_SLLI t1, t1, (HPAGE_SHIFT - 1) 485 + PTR_ADD t0, t0, t1 557 486 csrwr t0, LOONGARCH_CSR_TLBELO1 558 487 559 488 /* Set huge page tlb entry size */ 560 - addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 561 - addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 489 + PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 490 + PTR_LI t1, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT)) 562 491 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 563 492 564 493 tlbfill 565 494 566 495 /* Reset default page size */ 567 - addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 568 - addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 496 + PTR_LI t0, (CSR_TLBIDX_PS >> 16) << 16 497 + PTR_LI t1, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT)) 569 498 csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 570 499 571 500 csrrd t0, EXCEPTION_KS0 ··· 588 517 jr t0 589 518 SYM_CODE_END(handle_tlb_modify_ptw) 590 519 520 + #ifdef CONFIG_32BIT 521 + SYM_CODE_START(handle_tlb_refill) 522 + UNWIND_HINT_UNDEFINED 523 + csrwr t0, EXCEPTION_KS0 524 + csrwr t1, EXCEPTION_KS1 525 + csrwr ra, EXCEPTION_KS2 526 + li.w ra, 0x1fffffff 527 + 528 + csrrd t0, LOONGARCH_CSR_PGD 529 + csrrd t1, LOONGARCH_CSR_TLBRBADV 530 + srli.w t1, t1, PGDIR_SHIFT 531 + slli.w t1, t1, 0x2 532 + add.w t0, t0, t1 533 + and t0, t0, ra 534 + 535 + ld.w t0, t0, 0 536 + csrrd t1, LOONGARCH_CSR_TLBRBADV 537 + slli.w t1, t1, (32 - PGDIR_SHIFT) 538 + srli.w t1, t1, (32 - PGDIR_SHIFT + PAGE_SHIFT + 1) 539 + slli.w t1, t1, (0x2 + 1) 540 + add.w t0, t0, t1 541 + and t0, t0, ra 542 + 543 + ld.w t1, t0, 0x0 544 + csrwr t1, LOONGARCH_CSR_TLBRELO0 545 + 546 + ld.w t1, t0, 0x4 547 + csrwr t1, LOONGARCH_CSR_TLBRELO1 548 + 549 + tlbfill 550 + csrrd t0, EXCEPTION_KS0 551 + csrrd t1, EXCEPTION_KS1 552 + csrrd ra, EXCEPTION_KS2 553 + ertn 554 + SYM_CODE_END(handle_tlb_refill) 555 + #endif 556 + 557 + #ifdef CONFIG_64BIT 591 558 SYM_CODE_START(handle_tlb_refill) 592 559 UNWIND_HINT_UNDEFINED 593 560 csrwr t0, LOONGARCH_CSR_TLBRSAVE ··· 643 534 csrrd t0, LOONGARCH_CSR_TLBRSAVE 644 535 ertn 645 536 SYM_CODE_END(handle_tlb_refill) 537 + #endif