Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/*
2 * linux/arch/arm/mm/context.c
3 *
4 * Copyright (C) 2002-2003 Deep Blue Solutions Ltd, all rights reserved.
5 * Copyright (C) 2012 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This program is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License version 2 as
11 * published by the Free Software Foundation.
12 */
13#include <linux/init.h>
14#include <linux/sched.h>
15#include <linux/mm.h>
16#include <linux/smp.h>
17#include <linux/percpu.h>
18
19#include <asm/mmu_context.h>
20#include <asm/smp_plat.h>
21#include <asm/thread_notify.h>
22#include <asm/tlbflush.h>
23#include <asm/proc-fns.h>
24
25/*
26 * On ARMv6, we have the following structure in the Context ID:
27 *
28 * 31 7 0
29 * +-------------------------+-----------+
30 * | process ID | ASID |
31 * +-------------------------+-----------+
32 * | context ID |
33 * +-------------------------------------+
34 *
35 * The ASID is used to tag entries in the CPU caches and TLBs.
36 * The context ID is used by debuggers and trace logic, and
37 * should be unique within all running processes.
38 *
39 * In big endian operation, the two 32 bit words are swapped if accesed by
40 * non 64-bit operations.
41 */
42#define ASID_FIRST_VERSION (1ULL << ASID_BITS)
43#define NUM_USER_ASIDS ASID_FIRST_VERSION
44
45static DEFINE_RAW_SPINLOCK(cpu_asid_lock);
46static atomic64_t asid_generation = ATOMIC64_INIT(ASID_FIRST_VERSION);
47static DECLARE_BITMAP(asid_map, NUM_USER_ASIDS);
48
49static DEFINE_PER_CPU(atomic64_t, active_asids);
50static DEFINE_PER_CPU(u64, reserved_asids);
51static cpumask_t tlb_flush_pending;
52
53#ifdef CONFIG_ARM_ERRATA_798181
54void a15_erratum_get_cpumask(int this_cpu, struct mm_struct *mm,
55 cpumask_t *mask)
56{
57 int cpu;
58 unsigned long flags;
59 u64 context_id, asid;
60
61 raw_spin_lock_irqsave(&cpu_asid_lock, flags);
62 context_id = mm->context.id.counter;
63 for_each_online_cpu(cpu) {
64 if (cpu == this_cpu)
65 continue;
66 /*
67 * We only need to send an IPI if the other CPUs are
68 * running the same ASID as the one being invalidated.
69 */
70 asid = per_cpu(active_asids, cpu).counter;
71 if (asid == 0)
72 asid = per_cpu(reserved_asids, cpu);
73 if (context_id == asid)
74 cpumask_set_cpu(cpu, mask);
75 }
76 raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
77}
78#endif
79
80#ifdef CONFIG_ARM_LPAE
81static void cpu_set_reserved_ttbr0(void)
82{
83 /*
84 * Set TTBR0 to swapper_pg_dir which contains only global entries. The
85 * ASID is set to 0.
86 */
87 cpu_set_ttbr(0, __pa(swapper_pg_dir));
88 isb();
89}
90#else
91static void cpu_set_reserved_ttbr0(void)
92{
93 u32 ttb;
94 /* Copy TTBR1 into TTBR0 */
95 asm volatile(
96 " mrc p15, 0, %0, c2, c0, 1 @ read TTBR1\n"
97 " mcr p15, 0, %0, c2, c0, 0 @ set TTBR0\n"
98 : "=r" (ttb));
99 isb();
100}
101#endif
102
103#ifdef CONFIG_PID_IN_CONTEXTIDR
104static int contextidr_notifier(struct notifier_block *unused, unsigned long cmd,
105 void *t)
106{
107 u32 contextidr;
108 pid_t pid;
109 struct thread_info *thread = t;
110
111 if (cmd != THREAD_NOTIFY_SWITCH)
112 return NOTIFY_DONE;
113
114 pid = task_pid_nr(thread->task) << ASID_BITS;
115 asm volatile(
116 " mrc p15, 0, %0, c13, c0, 1\n"
117 " and %0, %0, %2\n"
118 " orr %0, %0, %1\n"
119 " mcr p15, 0, %0, c13, c0, 1\n"
120 : "=r" (contextidr), "+r" (pid)
121 : "I" (~ASID_MASK));
122 isb();
123
124 return NOTIFY_OK;
125}
126
127static struct notifier_block contextidr_notifier_block = {
128 .notifier_call = contextidr_notifier,
129};
130
131static int __init contextidr_notifier_init(void)
132{
133 return thread_register_notifier(&contextidr_notifier_block);
134}
135arch_initcall(contextidr_notifier_init);
136#endif
137
138static void flush_context(unsigned int cpu)
139{
140 int i;
141 u64 asid;
142
143 /* Update the list of reserved ASIDs and the ASID bitmap. */
144 bitmap_clear(asid_map, 0, NUM_USER_ASIDS);
145 for_each_possible_cpu(i) {
146 if (i == cpu) {
147 asid = 0;
148 } else {
149 asid = atomic64_xchg(&per_cpu(active_asids, i), 0);
150 /*
151 * If this CPU has already been through a
152 * rollover, but hasn't run another task in
153 * the meantime, we must preserve its reserved
154 * ASID, as this is the only trace we have of
155 * the process it is still running.
156 */
157 if (asid == 0)
158 asid = per_cpu(reserved_asids, i);
159 __set_bit(asid & ~ASID_MASK, asid_map);
160 }
161 per_cpu(reserved_asids, i) = asid;
162 }
163
164 /* Queue a TLB invalidate and flush the I-cache if necessary. */
165 cpumask_setall(&tlb_flush_pending);
166
167 if (icache_is_vivt_asid_tagged())
168 __flush_icache_all();
169}
170
171static int is_reserved_asid(u64 asid)
172{
173 int cpu;
174 for_each_possible_cpu(cpu)
175 if (per_cpu(reserved_asids, cpu) == asid)
176 return 1;
177 return 0;
178}
179
180static u64 new_context(struct mm_struct *mm, unsigned int cpu)
181{
182 u64 asid = atomic64_read(&mm->context.id);
183 u64 generation = atomic64_read(&asid_generation);
184
185 if (asid != 0 && is_reserved_asid(asid)) {
186 /*
187 * Our current ASID was active during a rollover, we can
188 * continue to use it and this was just a false alarm.
189 */
190 asid = generation | (asid & ~ASID_MASK);
191 } else {
192 /*
193 * Allocate a free ASID. If we can't find one, take a
194 * note of the currently active ASIDs and mark the TLBs
195 * as requiring flushes. We always count from ASID #1,
196 * as we reserve ASID #0 to switch via TTBR0 and indicate
197 * rollover events.
198 */
199 asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
200 if (asid == NUM_USER_ASIDS) {
201 generation = atomic64_add_return(ASID_FIRST_VERSION,
202 &asid_generation);
203 flush_context(cpu);
204 asid = find_next_zero_bit(asid_map, NUM_USER_ASIDS, 1);
205 }
206 __set_bit(asid, asid_map);
207 asid |= generation;
208 cpumask_clear(mm_cpumask(mm));
209 }
210
211 return asid;
212}
213
214void check_and_switch_context(struct mm_struct *mm, struct task_struct *tsk)
215{
216 unsigned long flags;
217 unsigned int cpu = smp_processor_id();
218 u64 asid;
219
220 if (unlikely(mm->context.vmalloc_seq != init_mm.context.vmalloc_seq))
221 __check_vmalloc_seq(mm);
222
223 /*
224 * Required during context switch to avoid speculative page table
225 * walking with the wrong TTBR.
226 */
227 cpu_set_reserved_ttbr0();
228
229 asid = atomic64_read(&mm->context.id);
230 if (!((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS)
231 && atomic64_xchg(&per_cpu(active_asids, cpu), asid))
232 goto switch_mm_fastpath;
233
234 raw_spin_lock_irqsave(&cpu_asid_lock, flags);
235 /* Check that our ASID belongs to the current generation. */
236 asid = atomic64_read(&mm->context.id);
237 if ((asid ^ atomic64_read(&asid_generation)) >> ASID_BITS) {
238 asid = new_context(mm, cpu);
239 atomic64_set(&mm->context.id, asid);
240 }
241
242 if (cpumask_test_and_clear_cpu(cpu, &tlb_flush_pending)) {
243 local_flush_bp_all();
244 local_flush_tlb_all();
245 }
246
247 atomic64_set(&per_cpu(active_asids, cpu), asid);
248 cpumask_set_cpu(cpu, mm_cpumask(mm));
249 raw_spin_unlock_irqrestore(&cpu_asid_lock, flags);
250
251switch_mm_fastpath:
252 cpu_switch_mm(mm->pgd, mm);
253}