Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
ARM: tlb: delay page freeing for SMP and ARMv7 CPUs
We need to delay freeing any mapped page on SMP and ARMv7 systems to
ensure that the data is not accessed by other CPUs, or is used for
speculative prefetch with ARMv7. This includes not only mapped pages
but also pages used for the page tables themselves.
This avoids races with the MMU/other CPUs accessing pages after they've
been freed but before we've invalidated the TLB.
Signed-off-by: Russell King <rmk+kernel@arm.linux.org.uk>
+89
-13
arch/arm/include/asm/tlb.h
+89
-13
arch/arm/include/asm/tlb.h
···
18
18
#define __ASMARM_TLB_H
19
19
20
20
#include <asm/cacheflush.h>
21
-
#include <asm/tlbflush.h>
22
21
23
22
#ifndef CONFIG_MMU
24
23
···
26
27
27
28
#else /* !CONFIG_MMU */
28
29
30
+
#include <linux/swap.h>
29
31
#include <asm/pgalloc.h>
32
+
#include <asm/tlbflush.h>
33
+
34
+
/*
35
+
* We need to delay page freeing for SMP as other CPUs can access pages
36
+
* which have been removed but not yet had their TLB entries invalidated.
37
+
* Also, as ARMv7 speculative prefetch can drag new entries into the TLB,
38
+
* we need to apply this same delaying tactic to ensure correct operation.
39
+
*/
40
+
#if defined(CONFIG_SMP) || defined(CONFIG_CPU_32v7)
41
+
#define tlb_fast_mode(tlb) 0
42
+
#define FREE_PTE_NR 500
43
+
#else
44
+
#define tlb_fast_mode(tlb) 1
45
+
#define FREE_PTE_NR 0
46
+
#endif
30
47
31
48
/*
32
49
* TLB handling. This allows us to remove pages from the page
···
51
36
struct mmu_gather {
52
37
struct mm_struct *mm;
53
38
unsigned int fullmm;
39
+
struct vm_area_struct *vma;
54
40
unsigned long range_start;
55
41
unsigned long range_end;
42
+
unsigned int nr;
43
+
struct page *pages[FREE_PTE_NR];
56
44
};
57
45
58
46
DECLARE_PER_CPU(struct mmu_gather, mmu_gathers);
47
+
48
+
/*
49
+
* This is unnecessarily complex. There's three ways the TLB shootdown
50
+
* code is used:
51
+
* 1. Unmapping a range of vmas. See zap_page_range(), unmap_region().
52
+
* tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called.
53
+
* tlb->vma will be non-NULL.
54
+
* 2. Unmapping all vmas. See exit_mmap().
55
+
* tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called.
56
+
* tlb->vma will be non-NULL. Additionally, page tables will be freed.
57
+
* 3. Unmapping argument pages. See shift_arg_pages().
58
+
* tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called.
59
+
* tlb->vma will be NULL.
60
+
*/
61
+
static inline void tlb_flush(struct mmu_gather *tlb)
62
+
{
63
+
if (tlb->fullmm || !tlb->vma)
64
+
flush_tlb_mm(tlb->mm);
65
+
else if (tlb->range_end > 0) {
66
+
flush_tlb_range(tlb->vma, tlb->range_start, tlb->range_end);
67
+
tlb->range_start = TASK_SIZE;
68
+
tlb->range_end = 0;
69
+
}
70
+
}
71
+
72
+
static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr)
73
+
{
74
+
if (!tlb->fullmm) {
75
+
if (addr < tlb->range_start)
76
+
tlb->range_start = addr;
77
+
if (addr + PAGE_SIZE > tlb->range_end)
78
+
tlb->range_end = addr + PAGE_SIZE;
79
+
}
80
+
}
81
+
82
+
static inline void tlb_flush_mmu(struct mmu_gather *tlb)
83
+
{
84
+
tlb_flush(tlb);
85
+
if (!tlb_fast_mode(tlb)) {
86
+
free_pages_and_swap_cache(tlb->pages, tlb->nr);
87
+
tlb->nr = 0;
88
+
}
89
+
}
59
90
60
91
static inline struct mmu_gather *
61
92
tlb_gather_mmu(struct mm_struct *mm, unsigned int full_mm_flush)
···
110
49
111
50
tlb->mm = mm;
112
51
tlb->fullmm = full_mm_flush;
52
+
tlb->vma = NULL;
53
+
tlb->nr = 0;
113
54
114
55
return tlb;
115
56
}
···
119
56
static inline void
120
57
tlb_finish_mmu(struct mmu_gather *tlb, unsigned long start, unsigned long end)
121
58
{
122
-
if (tlb->fullmm)
123
-
flush_tlb_mm(tlb->mm);
59
+
tlb_flush_mmu(tlb);
124
60
125
61
/* keep the page table cache within bounds */
126
62
check_pgt_cache();
···
133
71
static inline void
134
72
tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, unsigned long addr)
135
73
{
136
-
if (!tlb->fullmm) {
137
-
if (addr < tlb->range_start)
138
-
tlb->range_start = addr;
139
-
if (addr + PAGE_SIZE > tlb->range_end)
140
-
tlb->range_end = addr + PAGE_SIZE;
141
-
}
74
+
tlb_add_flush(tlb, addr);
142
75
}
143
76
144
77
/*
···
146
89
{
147
90
if (!tlb->fullmm) {
148
91
flush_cache_range(vma, vma->vm_start, vma->vm_end);
92
+
tlb->vma = vma;
149
93
tlb->range_start = TASK_SIZE;
150
94
tlb->range_end = 0;
151
95
}
···
155
97
static inline void
156
98
tlb_end_vma(struct mmu_gather *tlb, struct vm_area_struct *vma)
157
99
{
158
-
if (!tlb->fullmm && tlb->range_end > 0)
159
-
flush_tlb_range(vma, tlb->range_start, tlb->range_end);
100
+
if (!tlb->fullmm)
101
+
tlb_flush(tlb);
160
102
}
161
103
162
-
#define tlb_remove_page(tlb,page) free_page_and_swap_cache(page)
163
-
#define pte_free_tlb(tlb, ptep, addr) pte_free((tlb)->mm, ptep)
104
+
static inline void tlb_remove_page(struct mmu_gather *tlb, struct page *page)
105
+
{
106
+
if (tlb_fast_mode(tlb)) {
107
+
free_page_and_swap_cache(page);
108
+
} else {
109
+
tlb->pages[tlb->nr++] = page;
110
+
if (tlb->nr >= FREE_PTE_NR)
111
+
tlb_flush_mmu(tlb);
112
+
}
113
+
}
114
+
115
+
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
116
+
unsigned long addr)
117
+
{
118
+
pgtable_page_dtor(pte);
119
+
tlb_add_flush(tlb, addr);
120
+
tlb_remove_page(tlb, pte);
121
+
}
122
+
123
+
#define pte_free_tlb(tlb, ptep, addr) __pte_free_tlb(tlb, ptep, addr)
164
124
#define pmd_free_tlb(tlb, pmdp, addr) pmd_free((tlb)->mm, pmdp)
165
125
166
126
#define tlb_migrate_finish(mm) do { } while (0)