tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * Based on arch/arm/include/asm/tlbflush.h
3 *
4 * Copyright (C) 1999-2003 Russell King
5 * Copyright (C) 2012 ARM Ltd.
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License version 2 as
9 * published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
15 *
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
18 */
19#ifndef __ASM_TLBFLUSH_H
20#define __ASM_TLBFLUSH_H
21
22#ifndef __ASSEMBLY__
23
24#include <linux/sched.h>
25#include <asm/cputype.h>
26
27/*
28 * TLB Management
29 * ==============
30 *
31 * The TLB specific code is expected to perform whatever tests it needs
32 * to determine if it should invalidate the TLB for each call. Start
33 * addresses are inclusive and end addresses are exclusive; it is safe to
34 * round these addresses down.
35 *
36 * flush_tlb_all()
37 *
38 * Invalidate the entire TLB.
39 *
40 * flush_tlb_mm(mm)
41 *
42 * Invalidate all TLB entries in a particular address space.
43 * - mm - mm_struct describing address space
44 *
45 * flush_tlb_range(mm,start,end)
46 *
47 * Invalidate a range of TLB entries in the specified address
48 * space.
49 * - mm - mm_struct describing address space
50 * - start - start address (may not be aligned)
51 * - end - end address (exclusive, may not be aligned)
52 *
53 * flush_tlb_page(vaddr,vma)
54 *
55 * Invalidate the specified page in the specified address range.
56 * - vaddr - virtual address (may not be aligned)
57 * - vma - vma_struct describing address range
58 *
59 * flush_kern_tlb_page(kaddr)
60 *
61 * Invalidate the TLB entry for the specified page. The address
62 * will be in the kernels virtual memory space. Current uses
63 * only require the D-TLB to be invalidated.
64 * - kaddr - Kernel virtual memory address
65 */
66static inline void local_flush_tlb_all(void)
67{
68 dsb(nshst);
69 asm("tlbi vmalle1");
70 dsb(nsh);
71 isb();
72}
73
74static inline void flush_tlb_all(void)
75{
76 dsb(ishst);
77 asm("tlbi vmalle1is");
78 dsb(ish);
79 isb();
80}
81
82static inline void flush_tlb_mm(struct mm_struct *mm)
83{
84 unsigned long asid = ASID(mm) << 48;
85
86 dsb(ishst);
87 asm("tlbi aside1is, %0" : : "r" (asid));
88 dsb(ish);
89}
90
91static inline void flush_tlb_page(struct vm_area_struct *vma,
92 unsigned long uaddr)
93{
94 unsigned long addr = uaddr >> 12 | (ASID(vma->vm_mm) << 48);
95
96 dsb(ishst);
97 asm("tlbi vale1is, %0" : : "r" (addr));
98 dsb(ish);
99}
100
101/*
102 * This is meant to avoid soft lock-ups on large TLB flushing ranges and not
103 * necessarily a performance improvement.
104 */
105#define MAX_TLB_RANGE (1024UL << PAGE_SHIFT)
106
107static inline void __flush_tlb_range(struct vm_area_struct *vma,
108 unsigned long start, unsigned long end,
109 bool last_level)
110{
111 unsigned long asid = ASID(vma->vm_mm) << 48;
112 unsigned long addr;
113
114 if ((end - start) > MAX_TLB_RANGE) {
115 flush_tlb_mm(vma->vm_mm);
116 return;
117 }
118
119 start = asid | (start >> 12);
120 end = asid | (end >> 12);
121
122 dsb(ishst);
123 for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12)) {
124 if (last_level)
125 asm("tlbi vale1is, %0" : : "r"(addr));
126 else
127 asm("tlbi vae1is, %0" : : "r"(addr));
128 }
129 dsb(ish);
130}
131
132static inline void flush_tlb_range(struct vm_area_struct *vma,
133 unsigned long start, unsigned long end)
134{
135 __flush_tlb_range(vma, start, end, false);
136}
137
138static inline void flush_tlb_kernel_range(unsigned long start, unsigned long end)
139{
140 unsigned long addr;
141
142 if ((end - start) > MAX_TLB_RANGE) {
143 flush_tlb_all();
144 return;
145 }
146
147 start >>= 12;
148 end >>= 12;
149
150 dsb(ishst);
151 for (addr = start; addr < end; addr += 1 << (PAGE_SHIFT - 12))
152 asm("tlbi vaae1is, %0" : : "r"(addr));
153 dsb(ish);
154 isb();
155}
156
157/*
158 * Used to invalidate the TLB (walk caches) corresponding to intermediate page
159 * table levels (pgd/pud/pmd).
160 */
161static inline void __flush_tlb_pgtable(struct mm_struct *mm,
162 unsigned long uaddr)
163{
164 unsigned long addr = uaddr >> 12 | (ASID(mm) << 48);
165
166 asm("tlbi vae1is, %0" : : "r" (addr));
167 dsb(ish);
168}
169
170#endif
171
172#endif