tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012,2013 - ARM Ltd
4 * Author: Marc Zyngier <marc.zyngier@arm.com>
5 */
6
7#ifndef __ARM_KVM_INIT_H__
8#define __ARM_KVM_INIT_H__
9
10#ifndef __ASSEMBLY__
11#error Assembly-only header
12#endif
13
14#include <asm/kvm_arm.h>
15#include <asm/ptrace.h>
16#include <asm/sysreg.h>
17#include <linux/irqchip/arm-gic-v3.h>
18
19.macro __init_el2_sctlr
20 mov_q x0, INIT_SCTLR_EL2_MMU_OFF
21 msr sctlr_el2, x0
22 isb
23.endm
24
25.macro __init_el2_hcrx
26 mrs x0, id_aa64mmfr1_el1
27 ubfx x0, x0, #ID_AA64MMFR1_EL1_HCX_SHIFT, #4
28 cbz x0, .Lskip_hcrx_\@
29 mov_q x0, HCRX_HOST_FLAGS
30
31 /* Enable GCS if supported */
32 mrs_s x1, SYS_ID_AA64PFR1_EL1
33 ubfx x1, x1, #ID_AA64PFR1_EL1_GCS_SHIFT, #4
34 cbz x1, .Lset_hcrx_\@
35 orr x0, x0, #HCRX_EL2_GCSEn
36
37.Lset_hcrx_\@:
38 msr_s SYS_HCRX_EL2, x0
39.Lskip_hcrx_\@:
40.endm
41
42/* Check if running in host at EL2 mode, i.e., (h)VHE. Jump to fail if not. */
43.macro __check_hvhe fail, tmp
44 mrs \tmp, hcr_el2
45 and \tmp, \tmp, #HCR_E2H
46 cbz \tmp, \fail
47.endm
48
49/*
50 * Allow Non-secure EL1 and EL0 to access physical timer and counter.
51 * This is not necessary for VHE, since the host kernel runs in EL2,
52 * and EL0 accesses are configured in the later stage of boot process.
53 * Note that when HCR_EL2.E2H == 1, CNTHCTL_EL2 has the same bit layout
54 * as CNTKCTL_EL1, and CNTKCTL_EL1 accessing instructions are redefined
55 * to access CNTHCTL_EL2. This allows the kernel designed to run at EL1
56 * to transparently mess with the EL0 bits via CNTKCTL_EL1 access in
57 * EL2.
58 */
59.macro __init_el2_timers
60 mov x0, #3 // Enable EL1 physical timers
61 __check_hvhe .LnVHE_\@, x1
62 lsl x0, x0, #10
63.LnVHE_\@:
64 msr cnthctl_el2, x0
65 msr cntvoff_el2, xzr // Clear virtual offset
66.endm
67
68.macro __init_el2_debug
69 mrs x1, id_aa64dfr0_el1
70 ubfx x0, x1, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4
71 cmp x0, #ID_AA64DFR0_EL1_PMUVer_NI
72 ccmp x0, #ID_AA64DFR0_EL1_PMUVer_IMP_DEF, #4, ne
73 b.eq .Lskip_pmu_\@ // Skip if no PMU present or IMP_DEF
74 mrs x0, pmcr_el0 // Disable debug access traps
75 ubfx x0, x0, #11, #5 // to EL2 and allow access to
76.Lskip_pmu_\@:
77 csel x2, xzr, x0, eq // all PMU counters from EL1
78
79 /* Statistical profiling */
80 ubfx x0, x1, #ID_AA64DFR0_EL1_PMSVer_SHIFT, #4
81 cbz x0, .Lskip_spe_\@ // Skip if SPE not present
82
83 mrs_s x0, SYS_PMBIDR_EL1 // If SPE available at EL2,
84 and x0, x0, #(1 << PMBIDR_EL1_P_SHIFT)
85 cbnz x0, .Lskip_spe_el2_\@ // then permit sampling of physical
86 mov x0, #(1 << PMSCR_EL2_PCT_SHIFT | \
87 1 << PMSCR_EL2_PA_SHIFT)
88 msr_s SYS_PMSCR_EL2, x0 // addresses and physical counter
89.Lskip_spe_el2_\@:
90 mov x0, #(MDCR_EL2_E2PB_MASK << MDCR_EL2_E2PB_SHIFT)
91 orr x2, x2, x0 // If we don't have VHE, then
92 // use EL1&0 translation.
93
94.Lskip_spe_\@:
95 /* Trace buffer */
96 ubfx x0, x1, #ID_AA64DFR0_EL1_TraceBuffer_SHIFT, #4
97 cbz x0, .Lskip_trace_\@ // Skip if TraceBuffer is not present
98
99 mrs_s x0, SYS_TRBIDR_EL1
100 and x0, x0, TRBIDR_EL1_P
101 cbnz x0, .Lskip_trace_\@ // If TRBE is available at EL2
102
103 mov x0, #(MDCR_EL2_E2TB_MASK << MDCR_EL2_E2TB_SHIFT)
104 orr x2, x2, x0 // allow the EL1&0 translation
105 // to own it.
106
107.Lskip_trace_\@:
108 msr mdcr_el2, x2 // Configure debug traps
109.endm
110
111/* LORegions */
112.macro __init_el2_lor
113 mrs x1, id_aa64mmfr1_el1
114 ubfx x0, x1, #ID_AA64MMFR1_EL1_LO_SHIFT, 4
115 cbz x0, .Lskip_lor_\@
116 msr_s SYS_LORC_EL1, xzr
117.Lskip_lor_\@:
118.endm
119
120/* Stage-2 translation */
121.macro __init_el2_stage2
122 msr vttbr_el2, xzr
123.endm
124
125/* GICv3 system register access */
126.macro __init_el2_gicv3
127 mrs x0, id_aa64pfr0_el1
128 ubfx x0, x0, #ID_AA64PFR0_EL1_GIC_SHIFT, #4
129 cbz x0, .Lskip_gicv3_\@
130
131 mrs_s x0, SYS_ICC_SRE_EL2
132 orr x0, x0, #ICC_SRE_EL2_SRE // Set ICC_SRE_EL2.SRE==1
133 orr x0, x0, #ICC_SRE_EL2_ENABLE // Set ICC_SRE_EL2.Enable==1
134 msr_s SYS_ICC_SRE_EL2, x0
135 isb // Make sure SRE is now set
136 mrs_s x0, SYS_ICC_SRE_EL2 // Read SRE back,
137 tbz x0, #0, .Lskip_gicv3_\@ // and check that it sticks
138 msr_s SYS_ICH_HCR_EL2, xzr // Reset ICH_HCR_EL2 to defaults
139.Lskip_gicv3_\@:
140.endm
141
142.macro __init_el2_hstr
143 msr hstr_el2, xzr // Disable CP15 traps to EL2
144.endm
145
146/* Virtual CPU ID registers */
147.macro __init_el2_nvhe_idregs
148 mrs x0, midr_el1
149 mrs x1, mpidr_el1
150 msr vpidr_el2, x0
151 msr vmpidr_el2, x1
152.endm
153
154/* Coprocessor traps */
155.macro __init_el2_cptr
156 __check_hvhe .LnVHE_\@, x1
157 mov x0, #CPACR_ELx_FPEN
158 msr cpacr_el1, x0
159 b .Lskip_set_cptr_\@
160.LnVHE_\@:
161 mov x0, #0x33ff
162 msr cptr_el2, x0 // Disable copro. traps to EL2
163.Lskip_set_cptr_\@:
164.endm
165
166/* Disable any fine grained traps */
167.macro __init_el2_fgt
168 mrs x1, id_aa64mmfr0_el1
169 ubfx x1, x1, #ID_AA64MMFR0_EL1_FGT_SHIFT, #4
170 cbz x1, .Lskip_fgt_\@
171
172 mov x0, xzr
173 mrs x1, id_aa64dfr0_el1
174 ubfx x1, x1, #ID_AA64DFR0_EL1_PMSVer_SHIFT, #4
175 cmp x1, #3
176 b.lt .Lskip_spe_fgt_\@
177 /* Disable PMSNEVFR_EL1 read and write traps */
178 orr x0, x0, #(1 << 62)
179
180.Lskip_spe_fgt_\@:
181 msr_s SYS_HDFGRTR_EL2, x0
182 msr_s SYS_HDFGWTR_EL2, x0
183
184 mov x0, xzr
185 mrs x1, id_aa64pfr1_el1
186 ubfx x1, x1, #ID_AA64PFR1_EL1_SME_SHIFT, #4
187 cbz x1, .Lskip_debug_fgt_\@
188
189 /* Disable nVHE traps of TPIDR2 and SMPRI */
190 orr x0, x0, #HFGxTR_EL2_nSMPRI_EL1_MASK
191 orr x0, x0, #HFGxTR_EL2_nTPIDR2_EL0_MASK
192
193.Lskip_debug_fgt_\@:
194 mrs_s x1, SYS_ID_AA64MMFR3_EL1
195 ubfx x1, x1, #ID_AA64MMFR3_EL1_S1PIE_SHIFT, #4
196 cbz x1, .Lskip_pie_fgt_\@
197
198 /* Disable trapping of PIR_EL1 / PIRE0_EL1 */
199 orr x0, x0, #HFGxTR_EL2_nPIR_EL1
200 orr x0, x0, #HFGxTR_EL2_nPIRE0_EL1
201
202.Lskip_pie_fgt_\@:
203 mrs_s x1, SYS_ID_AA64MMFR3_EL1
204 ubfx x1, x1, #ID_AA64MMFR3_EL1_S1POE_SHIFT, #4
205 cbz x1, .Lskip_poe_fgt_\@
206
207 /* Disable trapping of POR_EL0 */
208 orr x0, x0, #HFGxTR_EL2_nPOR_EL0
209
210.Lskip_poe_fgt_\@:
211 /* GCS depends on PIE so we don't check it if PIE is absent */
212 mrs_s x1, SYS_ID_AA64PFR1_EL1
213 ubfx x1, x1, #ID_AA64PFR1_EL1_GCS_SHIFT, #4
214 cbz x1, .Lset_fgt_\@
215
216 /* Disable traps of access to GCS registers at EL0 and EL1 */
217 orr x0, x0, #HFGxTR_EL2_nGCS_EL1_MASK
218 orr x0, x0, #HFGxTR_EL2_nGCS_EL0_MASK
219
220.Lset_fgt_\@:
221 msr_s SYS_HFGRTR_EL2, x0
222 msr_s SYS_HFGWTR_EL2, x0
223 msr_s SYS_HFGITR_EL2, xzr
224
225 mrs x1, id_aa64pfr0_el1 // AMU traps UNDEF without AMU
226 ubfx x1, x1, #ID_AA64PFR0_EL1_AMU_SHIFT, #4
227 cbz x1, .Lskip_amu_fgt_\@
228
229 msr_s SYS_HAFGRTR_EL2, xzr
230
231.Lskip_amu_fgt_\@:
232
233.Lskip_fgt_\@:
234.endm
235
236.macro __init_el2_gcs
237 mrs_s x1, SYS_ID_AA64PFR1_EL1
238 ubfx x1, x1, #ID_AA64PFR1_EL1_GCS_SHIFT, #4
239 cbz x1, .Lskip_gcs_\@
240
241 /* Ensure GCS is not enabled when we start trying to do BLs */
242 msr_s SYS_GCSCR_EL1, xzr
243 msr_s SYS_GCSCRE0_EL1, xzr
244.Lskip_gcs_\@:
245.endm
246
247.macro __init_el2_nvhe_prepare_eret
248 mov x0, #INIT_PSTATE_EL1
249 msr spsr_el2, x0
250.endm
251
252.macro __init_el2_mpam
253 /* Memory Partitioning And Monitoring: disable EL2 traps */
254 mrs x1, id_aa64pfr0_el1
255 ubfx x0, x1, #ID_AA64PFR0_EL1_MPAM_SHIFT, #4
256 cbz x0, .Lskip_mpam_\@ // skip if no MPAM
257 msr_s SYS_MPAM2_EL2, xzr // use the default partition
258 // and disable lower traps
259 mrs_s x0, SYS_MPAMIDR_EL1
260 tbz x0, #MPAMIDR_EL1_HAS_HCR_SHIFT, .Lskip_mpam_\@ // skip if no MPAMHCR reg
261 msr_s SYS_MPAMHCR_EL2, xzr // clear TRAP_MPAMIDR_EL1 -> EL2
262.Lskip_mpam_\@:
263.endm
264
265/**
266 * Initialize EL2 registers to sane values. This should be called early on all
267 * cores that were booted in EL2. Note that everything gets initialised as
268 * if VHE was not available. The kernel context will be upgraded to VHE
269 * if possible later on in the boot process
270 *
271 * Regs: x0, x1 and x2 are clobbered.
272 */
273.macro init_el2_state
274 __init_el2_sctlr
275 __init_el2_hcrx
276 __init_el2_timers
277 __init_el2_debug
278 __init_el2_lor
279 __init_el2_stage2
280 __init_el2_gicv3
281 __init_el2_hstr
282 __init_el2_mpam
283 __init_el2_nvhe_idregs
284 __init_el2_cptr
285 __init_el2_fgt
286 __init_el2_gcs
287.endm
288
289#ifndef __KVM_NVHE_HYPERVISOR__
290// This will clobber tmp1 and tmp2, and expect tmp1 to contain
291// the id register value as read from the HW
292.macro __check_override idreg, fld, width, pass, fail, tmp1, tmp2
293 ubfx \tmp1, \tmp1, #\fld, #\width
294 cbz \tmp1, \fail
295
296 adr_l \tmp1, \idreg\()_override
297 ldr \tmp2, [\tmp1, FTR_OVR_VAL_OFFSET]
298 ldr \tmp1, [\tmp1, FTR_OVR_MASK_OFFSET]
299 ubfx \tmp2, \tmp2, #\fld, #\width
300 ubfx \tmp1, \tmp1, #\fld, #\width
301 cmp \tmp1, xzr
302 and \tmp2, \tmp2, \tmp1
303 csinv \tmp2, \tmp2, xzr, ne
304 cbnz \tmp2, \pass
305 b \fail
306.endm
307
308// This will clobber tmp1 and tmp2
309.macro check_override idreg, fld, pass, fail, tmp1, tmp2
310 mrs \tmp1, \idreg\()_el1
311 __check_override \idreg \fld 4 \pass \fail \tmp1 \tmp2
312.endm
313#else
314// This will clobber tmp
315.macro __check_override idreg, fld, width, pass, fail, tmp, ignore
316 ldr_l \tmp, \idreg\()_el1_sys_val
317 ubfx \tmp, \tmp, #\fld, #\width
318 cbnz \tmp, \pass
319 b \fail
320.endm
321
322.macro check_override idreg, fld, pass, fail, tmp, ignore
323 __check_override \idreg \fld 4 \pass \fail \tmp \ignore
324.endm
325#endif
326
327.macro finalise_el2_state
328 check_override id_aa64pfr0, ID_AA64PFR0_EL1_SVE_SHIFT, .Linit_sve_\@, .Lskip_sve_\@, x1, x2
329
330.Linit_sve_\@: /* SVE register access */
331 __check_hvhe .Lcptr_nvhe_\@, x1
332
333 // (h)VHE case
334 mrs x0, cpacr_el1 // Disable SVE traps
335 orr x0, x0, #CPACR_ELx_ZEN
336 msr cpacr_el1, x0
337 b .Lskip_set_cptr_\@
338
339.Lcptr_nvhe_\@: // nVHE case
340 mrs x0, cptr_el2 // Disable SVE traps
341 bic x0, x0, #CPTR_EL2_TZ
342 msr cptr_el2, x0
343.Lskip_set_cptr_\@:
344 isb
345 mov x1, #ZCR_ELx_LEN_MASK // SVE: Enable full vector
346 msr_s SYS_ZCR_EL2, x1 // length for EL1.
347
348.Lskip_sve_\@:
349 check_override id_aa64pfr1, ID_AA64PFR1_EL1_SME_SHIFT, .Linit_sme_\@, .Lskip_sme_\@, x1, x2
350
351.Linit_sme_\@: /* SME register access and priority mapping */
352 __check_hvhe .Lcptr_nvhe_sme_\@, x1
353
354 // (h)VHE case
355 mrs x0, cpacr_el1 // Disable SME traps
356 orr x0, x0, #CPACR_ELx_SMEN
357 msr cpacr_el1, x0
358 b .Lskip_set_cptr_sme_\@
359
360.Lcptr_nvhe_sme_\@: // nVHE case
361 mrs x0, cptr_el2 // Disable SME traps
362 bic x0, x0, #CPTR_EL2_TSM
363 msr cptr_el2, x0
364.Lskip_set_cptr_sme_\@:
365 isb
366
367 mrs x1, sctlr_el2
368 orr x1, x1, #SCTLR_ELx_ENTP2 // Disable TPIDR2 traps
369 msr sctlr_el2, x1
370 isb
371
372 mov x0, #0 // SMCR controls
373
374 // Full FP in SM?
375 mrs_s x1, SYS_ID_AA64SMFR0_EL1
376 __check_override id_aa64smfr0, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, .Linit_sme_fa64_\@, .Lskip_sme_fa64_\@, x1, x2
377
378.Linit_sme_fa64_\@:
379 orr x0, x0, SMCR_ELx_FA64_MASK
380.Lskip_sme_fa64_\@:
381
382 // ZT0 available?
383 mrs_s x1, SYS_ID_AA64SMFR0_EL1
384 __check_override id_aa64smfr0, ID_AA64SMFR0_EL1_SMEver_SHIFT, 4, .Linit_sme_zt0_\@, .Lskip_sme_zt0_\@, x1, x2
385.Linit_sme_zt0_\@:
386 orr x0, x0, SMCR_ELx_EZT0_MASK
387.Lskip_sme_zt0_\@:
388
389 orr x0, x0, #SMCR_ELx_LEN_MASK // Enable full SME vector
390 msr_s SYS_SMCR_EL2, x0 // length for EL1.
391
392 mrs_s x1, SYS_SMIDR_EL1 // Priority mapping supported?
393 ubfx x1, x1, #SMIDR_EL1_SMPS_SHIFT, #1
394 cbz x1, .Lskip_sme_\@
395
396 msr_s SYS_SMPRIMAP_EL2, xzr // Make all priorities equal
397.Lskip_sme_\@:
398.endm
399
400#endif /* __ARM_KVM_INIT_H__ */