tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
arm64: TLB maintenance functionality
This patch adds the TLB maintenance functions. There is no distinction
made between the I and D TLBs. TLB maintenance operations are
automatically broadcast between CPUs in hardware. The inner-shareable
operations are always present, even on UP systems.
NOTE: Large part of this patch to be dropped once Peter Z's generic
mmu_gather patches are merged.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Acked-by: Tony Lindgren <tony@atomide.com>
Acked-by: Nicolas Pitre <nico@linaro.org>
Acked-by: Olof Johansson <olof@lixom.net>
Acked-by: Santosh Shilimkar <santosh.shilimkar@ti.com>
Acked-by: Arnd Bergmann <arnd@arndb.de>
+383
+190
arch/arm64/include/asm/tlb.h
+190
arch/arm64/include/asm/tlb.h
···
1
+
/*
2
+
* Based on arch/arm/include/asm/tlb.h
3
+
*
4
+
* Copyright (C) 2002 Russell King
5
+
* Copyright (C) 2012 ARM Ltd.
6
+
*
7
+
* This program is free software; you can redistribute it and/or modify
8
+
* it under the terms of the GNU General Public License version 2 as
9
+
* published by the Free Software Foundation.
10
+
*
11
+
* This program is distributed in the hope that it will be useful,
12
+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+
* GNU General Public License for more details.
15
+
*
16
+
* You should have received a copy of the GNU General Public License
17
+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
18
+
*/
19
+
#ifndef __ASM_TLB_H
20
+
#define __ASM_TLB_H
21
+
22
+
#include <linux/pagemap.h>
23
+
#include <linux/swap.h>
24
+
25
+
#include <asm/pgalloc.h>
26
+
#include <asm/tlbflush.h>
27
+
28
+
#define MMU_GATHER_BUNDLE 8
29
+
30
+
/*
31
+
* TLB handling. This allows us to remove pages from the page
32
+
* tables, and efficiently handle the TLB issues.
33
+
*/
34
+
struct mmu_gather {
35
+
struct mm_struct *mm;
36
+
unsigned int fullmm;
37
+
struct vm_area_struct *vma;
38
+
unsigned long range_start;
39
+
unsigned long range_end;
40
+
unsigned int nr;
41
+
unsigned int max;
42
+
struct page **pages;
43
+
struct page *local[MMU_GATHER_BUNDLE];
44
+
};
45
+
46
+
/*
47
+
* This is unnecessarily complex. There's three ways the TLB shootdown
48
+
* code is used:
49
+
* 1. Unmapping a range of vmas. See zap_page_range(), unmap_region().
50
+
* tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called.
51
+
* tlb->vma will be non-NULL.
52
+
* 2. Unmapping all vmas. See exit_mmap().
53
+
* tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called.
54
+
* tlb->vma will be non-NULL. Additionally, page tables will be freed.
55
+
* 3. Unmapping argument pages. See shift_arg_pages().
56
+
* tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called.
57
+
* tlb->vma will be NULL.
58
+
*/
59
+
static inline void tlb_flush(struct mmu_gather *tlb)
60
+
{
61
+
if (tlb->fullmm || !tlb->vma)
62
+
flush_tlb_mm(tlb->mm);
63
+
else if (tlb->range_end > 0) {
64
+
flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end);
65
+
tlb->range_start = TASK_SIZE;
66
+
tlb->range_end = 0;
67
+
}
68
+
}
69
+
70
+
static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr)
71
+
{
72
+
if (!tlb->fullmm) {
73
+
if (addr < tlb->range_start)
74
+
tlb->range_start = addr;
75
+
if (addr + PAGE_SIZE > tlb->range_end)
76
+
tlb->range_end = addr + PAGE_SIZE;
77
+
}
78
+
}
79
+
80
+
static inline void __tlb_alloc_page(struct mmu_gather *tlb)
81
+
{
82
+
unsigned long addr = __get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
83
+
84
+
if (addr) {
85
+
tlb->pages = (void *)addr;
86
+
tlb->max = PAGE_SIZE / sizeof(struct page *);
87
+
}
88
+
}
89
+
90
+
static inline void tlb_flush_mmu(struct mmu_gather *tlb)
91
+
{
92
+
tlb_flush(tlb);
93
+
free_pages_and_swap_cache(tlb->pages, tlb->nr);
94
+
tlb->nr = 0;
95
+
if (tlb->pages == tlb->local)
96
+
__tlb_alloc_page(tlb);
97
+
}
98
+
99
+
static inline void
100
+
tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm, unsigned int fullmm)
101
+
{
102
+
tlb->mm = mm;
103
+
tlb->fullmm = fullmm;
104
+
tlb->vma = NULL;
105
+
tlb->max = ARRAY_SIZE(tlb->local);
106
+
tlb->pages = tlb->local;
107
+
tlb->nr = 0;
108
+
__tlb_alloc_page(tlb);
109
+
}
110
+
111
+
static inline void
112
+
tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
113
+
{
114
+
tlb_flush_mmu(tlb);
115
+
116
+
/* keep the page table cache within bounds */
117
+
check_pgt_cache();
118
+
119
+
if (tlb->pages != tlb->local)
120
+
free_pages((unsigned long)tlb->pages, 0);
121
+
}
122
+
123
+
/*
124
+
* Memorize the range for the TLB flush.
125
+
*/
126
+
static inline void
127
+
tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
128
+
{
129
+
tlb_add_flush(tlb, addr);
130
+
}
131
+
132
+
/*
133
+
* In the case of tlb vma handling, we can optimise these away in the
134
+
* case where we're doing a full MM flush. When we're doing a munmap,
135
+
* the vmas are adjusted to only cover the region to be torn down.
136
+
*/
137
+
static inline void
138
+
tlb_start_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
139
+
{
140
+
if (!tlb->fullmm) {
141
+
tlb->vma = vma;
142
+
tlb->range_start = TASK_SIZE;
143
+
tlb->range_end = 0;
144
+
}
145
+
}
146
+
147
+
static inline void
148
+
tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
149
+
{
150
+
if (!tlb->fullmm)
151
+
tlb_flush(tlb);
152
+
}
153
+
154
+
static inline int __tlb_remove_page(struct mmu_gather *tlb, struct page *page)
155
+
{
156
+
tlb->pages[tlb->nr++] = page;
157
+
VM_BUG_ON(tlb->nr > tlb->max);
158
+
return tlb->max - tlb->nr;
159
+
}
160
+
161
+
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
162
+
{
163
+
if (!__tlb_remove_page(tlb, page))
164
+
tlb_flush_mmu(tlb);
165
+
}
166
+
167
+
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
168
+
unsigned long addr)
169
+
{
170
+
pgtable_page_dtor(pte);
171
+
tlb_add_flush(tlb, addr);
172
+
tlb_remove_page(tlb, pte);
173
+
}
174
+
175
+
#ifndef CONFIG_ARM64_64K_PAGES
176
+
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
177
+
unsigned long addr)
178
+
{
179
+
tlb_add_flush(tlb, addr);
180
+
tlb_remove_page(tlb, virt_to_page(pmdp));
181
+
}
182
+
#endif
183
+
184
+
#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
185
+
#define pmd_free_tlb(tlb, pmdp, addr) __pmd_free_tlb(tlb, pmdp, addr)
186
+
#define pud_free_tlb(tlb, pudp, addr) pud_free((tlb)->mm, pudp)
187
+
188
+
#define tlb_migrate_finish(mm) do { } while (0)
189
+
190
+
#endif
+122
arch/arm64/include/asm/tlbflush.h
+122
arch/arm64/include/asm/tlbflush.h
···
1
+
/*
2
+
* Based on arch/arm/include/asm/tlbflush.h
3
+
*
4
+
* Copyright (C) 1999-2003 Russell King
5
+
* Copyright (C) 2012 ARM Ltd.
6
+
*
7
+
* This program is free software; you can redistribute it and/or modify
8
+
* it under the terms of the GNU General Public License version 2 as
9
+
* published by the Free Software Foundation.
10
+
*
11
+
* This program is distributed in the hope that it will be useful,
12
+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13
+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14
+
* GNU General Public License for more details.
15
+
*
16
+
* You should have received a copy of the GNU General Public License
17
+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
18
+
*/
19
+
#ifndef __ASM_TLBFLUSH_H
20
+
#define __ASM_TLBFLUSH_H
21
+
22
+
#ifndef __ASSEMBLY__
23
+
24
+
#include <linux/sched.h>
25
+
#include <asm/cputype.h>
26
+
27
+
extern void __cpu_flush_user_tlb_range(unsigned long, unsigned long, struct vm_area_struct *);
28
+
extern void __cpu_flush_kern_tlb_range(unsigned long, unsigned long);
29
+
30
+
extern struct cpu_tlb_fns cpu_tlb;
31
+
32
+
/*
33
+
* TLB Management
34
+
* ==============
35
+
*
36
+
* The arch/arm64/mm/tlb.S files implement these methods.
37
+
*
38
+
* The TLB specific code is expected to perform whatever tests it needs
39
+
* to determine if it should invalidate the TLB for each call. Start
40
+
* addresses are inclusive and end addresses are exclusive; it is safe to
41
+
* round these addresses down.
42
+
*
43
+
* flush_tlb_all()
44
+
*
45
+
* Invalidate the entire TLB.
46
+
*
47
+
* flush_tlb_mm(mm)
48
+
*
49
+
* Invalidate all TLB entries in a particular address space.
50
+
* - mm - mm_struct describing address space
51
+
*
52
+
* flush_tlb_range(mm,start,end)
53
+
*
54
+
* Invalidate a range of TLB entries in the specified address
55
+
* space.
56
+
* - mm - mm_struct describing address space
57
+
* - start - start address (may not be aligned)
58
+
* - end - end address (exclusive, may not be aligned)
59
+
*
60
+
* flush_tlb_page(vaddr,vma)
61
+
*
62
+
* Invalidate the specified page in the specified address range.
63
+
* - vaddr - virtual address (may not be aligned)
64
+
* - vma - vma_struct describing address range
65
+
*
66
+
* flush_kern_tlb_page(kaddr)
67
+
*
68
+
* Invalidate the TLB entry for the specified page. The address
69
+
* will be in the kernels virtual memory space. Current uses
70
+
* only require the D-TLB to be invalidated.
71
+
* - kaddr - Kernel virtual memory address
72
+
*/
73
+
static inline void flush_tlb_all(void)
74
+
{
75
+
dsb();
76
+
asm("tlbi vmalle1is");
77
+
dsb();
78
+
isb();
79
+
}
80
+
81
+
static inline void flush_tlb_mm(struct mm_struct *mm)
82
+
{
83
+
unsigned long asid = (unsigned long)ASID(mm) << 48;
84
+
85
+
dsb();
86
+
asm("tlbi aside1is, %0" : : "r" (asid));
87
+
dsb();
88
+
}
89
+
90
+
static inline void flush_tlb_page(struct vm_area_struct *vma,
91
+
unsigned long uaddr)
92
+
{
93
+
unsigned long addr = uaddr >> 12 |
94
+
((unsigned long)ASID(vma->vm_mm) << 48);
95
+
96
+
dsb();
97
+
asm("tlbi vae1is, %0" : : "r" (addr));
98
+
dsb();
99
+
}
100
+
101
+
/*
102
+
* Convert calls to our calling convention.
103
+
*/
104
+
#define flush_tlb_range(vma,start,end) __cpu_flush_user_tlb_range(start,end,vma)
105
+
#define flush_tlb_kernel_range(s,e) __cpu_flush_kern_tlb_range(s,e)
106
+
107
+
/*
108
+
* On AArch64, the cache coherency is handled via the set_pte_at() function.
109
+
*/
110
+
static inline void update_mmu_cache(struct vm_area_struct *vma,
111
+
unsigned long addr, pte_t *ptep)
112
+
{
113
+
/*
114
+
* set_pte() does not have a DSB, so make sure that the page table
115
+
* write is visible.
116
+
*/
117
+
dsb();
118
+
}
119
+
120
+
#endif
121
+
122
+
#endif
+71
arch/arm64/mm/tlb.S
+71
arch/arm64/mm/tlb.S
···
1
+
/*
2
+
* Based on arch/arm/mm/tlb.S
3
+
*
4
+
* Copyright (C) 1997-2002 Russell King
5
+
* Copyright (C) 2012 ARM Ltd.
6
+
* Written by Catalin Marinas <catalin.marinas@arm.com>
7
+
*
8
+
* This program is free software; you can redistribute it and/or modify
9
+
* it under the terms of the GNU General Public License version 2 as
10
+
* published by the Free Software Foundation.
11
+
*
12
+
* This program is distributed in the hope that it will be useful,
13
+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
14
+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15
+
* GNU General Public License for more details.
16
+
*
17
+
* You should have received a copy of the GNU General Public License
18
+
* along with this program. If not, see <http://www.gnu.org/licenses/>.
19
+
*/
20
+
#include <linux/linkage.h>
21
+
#include <asm/assembler.h>
22
+
#include <asm/asm-offsets.h>
23
+
#include <asm/page.h>
24
+
#include <asm/tlbflush.h>
25
+
#include "proc-macros.S"
26
+
27
+
/*
28
+
* __cpu_flush_user_tlb_range(start, end, vma)
29
+
*
30
+
* Invalidate a range of TLB entries in the specified address space.
31
+
*
32
+
* - start - start address (may not be aligned)
33
+
* - end - end address (exclusive, may not be aligned)
34
+
* - vma - vma_struct describing address range
35
+
*/
36
+
ENTRY(__cpu_flush_user_tlb_range)
37
+
vma_vm_mm x3, x2 // get vma->vm_mm
38
+
mmid x3, x3 // get vm_mm->context.id
39
+
dsb sy
40
+
lsr x0, x0, #12 // align address
41
+
lsr x1, x1, #12
42
+
bfi x0, x3, #48, #16 // start VA and ASID
43
+
bfi x1, x3, #48, #16 // end VA and ASID
44
+
1: tlbi vae1is, x0 // TLB invalidate by address and ASID
45
+
add x0, x0, #1
46
+
cmp x0, x1
47
+
b.lo 1b
48
+
dsb sy
49
+
ret
50
+
ENDPROC(__cpu_flush_user_tlb_range)
51
+
52
+
/*
53
+
* __cpu_flush_kern_tlb_range(start,end)
54
+
*
55
+
* Invalidate a range of kernel TLB entries.
56
+
*
57
+
* - start - start address (may not be aligned)
58
+
* - end - end address (exclusive, may not be aligned)
59
+
*/
60
+
ENTRY(__cpu_flush_kern_tlb_range)
61
+
dsb sy
62
+
lsr x0, x0, #12 // align address
63
+
lsr x1, x1, #12
64
+
1: tlbi vaae1is, x0 // TLB invalidate by address
65
+
add x0, x0, #1
66
+
cmp x0, x1
67
+
b.lo 1b
68
+
dsb sy
69
+
isb
70
+
ret
71
+
ENDPROC(__cpu_flush_kern_tlb_range)