tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 ARM Ltd.
4 */
5#ifndef __ASM_FP_H
6#define __ASM_FP_H
7
8#include <asm/errno.h>
9#include <asm/ptrace.h>
10#include <asm/processor.h>
11#include <asm/sigcontext.h>
12#include <asm/sysreg.h>
13
14#ifndef __ASSEMBLY__
15
16#include <linux/bitmap.h>
17#include <linux/build_bug.h>
18#include <linux/bug.h>
19#include <linux/cache.h>
20#include <linux/init.h>
21#include <linux/stddef.h>
22#include <linux/types.h>
23
24/* Masks for extracting the FPSR and FPCR from the FPSCR */
25#define VFP_FPSCR_STAT_MASK 0xf800009f
26#define VFP_FPSCR_CTRL_MASK 0x07f79f00
27/*
28 * The VFP state has 32x64-bit registers and a single 32-bit
29 * control/status register.
30 */
31#define VFP_STATE_SIZE ((32 * 8) + 4)
32
33static inline unsigned long cpacr_save_enable_kernel_sve(void)
34{
35 unsigned long old = read_sysreg(cpacr_el1);
36 unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_ZEN_EL1EN;
37
38 write_sysreg(old | set, cpacr_el1);
39 isb();
40 return old;
41}
42
43static inline unsigned long cpacr_save_enable_kernel_sme(void)
44{
45 unsigned long old = read_sysreg(cpacr_el1);
46 unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_SMEN_EL1EN;
47
48 write_sysreg(old | set, cpacr_el1);
49 isb();
50 return old;
51}
52
53static inline void cpacr_restore(unsigned long cpacr)
54{
55 write_sysreg(cpacr, cpacr_el1);
56 isb();
57}
58
59/*
60 * When we defined the maximum SVE vector length we defined the ABI so
61 * that the maximum vector length included all the reserved for future
62 * expansion bits in ZCR rather than those just currently defined by
63 * the architecture. Using this length to allocate worst size buffers
64 * results in excessively large allocations, and this effect is even
65 * more pronounced for SME due to ZA. Define more suitable VLs for
66 * these situations.
67 */
68#define ARCH_SVE_VQ_MAX ((ZCR_ELx_LEN_MASK >> ZCR_ELx_LEN_SHIFT) + 1)
69#define SME_VQ_MAX ((SMCR_ELx_LEN_MASK >> SMCR_ELx_LEN_SHIFT) + 1)
70
71struct task_struct;
72
73extern void fpsimd_save_state(struct user_fpsimd_state *state);
74extern void fpsimd_load_state(struct user_fpsimd_state *state);
75
76extern void fpsimd_thread_switch(struct task_struct *next);
77extern void fpsimd_flush_thread(void);
78
79extern void fpsimd_signal_preserve_current_state(void);
80extern void fpsimd_preserve_current_state(void);
81extern void fpsimd_restore_current_state(void);
82extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
83extern void fpsimd_kvm_prepare(void);
84
85struct cpu_fp_state {
86 struct user_fpsimd_state *st;
87 void *sve_state;
88 void *sme_state;
89 u64 *svcr;
90 u64 *fpmr;
91 unsigned int sve_vl;
92 unsigned int sme_vl;
93 enum fp_type *fp_type;
94 enum fp_type to_save;
95};
96
97extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);
98
99extern void fpsimd_flush_task_state(struct task_struct *target);
100extern void fpsimd_save_and_flush_cpu_state(void);
101
102static inline bool thread_sm_enabled(struct thread_struct *thread)
103{
104 return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
105}
106
107static inline bool thread_za_enabled(struct thread_struct *thread)
108{
109 return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
110}
111
112/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
113#define VL_ARCH_MAX 0x100
114
115/* Offset of FFR in the SVE register dump */
116static inline size_t sve_ffr_offset(int vl)
117{
118 return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
119}
120
121static inline void *sve_pffr(struct thread_struct *thread)
122{
123 unsigned int vl;
124
125 if (system_supports_sme() && thread_sm_enabled(thread))
126 vl = thread_get_sme_vl(thread);
127 else
128 vl = thread_get_sve_vl(thread);
129
130 return (char *)thread->sve_state + sve_ffr_offset(vl);
131}
132
133static inline void *thread_zt_state(struct thread_struct *thread)
134{
135 /* The ZT register state is stored immediately after the ZA state */
136 unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread));
137 return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
138}
139
140extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);
141extern void sve_load_state(void const *state, u32 const *pfpsr,
142 int restore_ffr);
143extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
144extern unsigned int sve_get_vl(void);
145extern void sve_set_vq(unsigned long vq_minus_1);
146extern void sme_set_vq(unsigned long vq_minus_1);
147extern void sme_save_state(void *state, int zt);
148extern void sme_load_state(void const *state, int zt);
149
150struct arm64_cpu_capabilities;
151extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused);
152extern void cpu_enable_sve(const struct arm64_cpu_capabilities *__unused);
153extern void cpu_enable_sme(const struct arm64_cpu_capabilities *__unused);
154extern void cpu_enable_sme2(const struct arm64_cpu_capabilities *__unused);
155extern void cpu_enable_fa64(const struct arm64_cpu_capabilities *__unused);
156extern void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__unused);
157
158/*
159 * Helpers to translate bit indices in sve_vq_map to VQ values (and
160 * vice versa). This allows find_next_bit() to be used to find the
161 * _maximum_ VQ not exceeding a certain value.
162 */
163static inline unsigned int __vq_to_bit(unsigned int vq)
164{
165 return SVE_VQ_MAX - vq;
166}
167
168static inline unsigned int __bit_to_vq(unsigned int bit)
169{
170 return SVE_VQ_MAX - bit;
171}
172
173
174struct vl_info {
175 enum vec_type type;
176 const char *name; /* For display purposes */
177
178 /* Minimum supported vector length across all CPUs */
179 int min_vl;
180
181 /* Maximum supported vector length across all CPUs */
182 int max_vl;
183 int max_virtualisable_vl;
184
185 /*
186 * Set of available vector lengths,
187 * where length vq encoded as bit __vq_to_bit(vq):
188 */
189 DECLARE_BITMAP(vq_map, SVE_VQ_MAX);
190
191 /* Set of vector lengths present on at least one cpu: */
192 DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
193};
194
195#ifdef CONFIG_ARM64_SVE
196
197extern void sve_alloc(struct task_struct *task, bool flush);
198extern void fpsimd_release_task(struct task_struct *task);
199extern void fpsimd_sync_to_sve(struct task_struct *task);
200extern void fpsimd_force_sync_to_sve(struct task_struct *task);
201extern void sve_sync_to_fpsimd(struct task_struct *task);
202extern void sve_sync_from_fpsimd_zeropad(struct task_struct *task);
203
204extern int vec_set_vector_length(struct task_struct *task, enum vec_type type,
205 unsigned long vl, unsigned long flags);
206
207extern int sve_set_current_vl(unsigned long arg);
208extern int sve_get_current_vl(void);
209
210static inline void sve_user_disable(void)
211{
212 sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
213}
214
215static inline void sve_user_enable(void)
216{
217 sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
218}
219
220#define sve_cond_update_zcr_vq(val, reg) \
221 do { \
222 u64 __zcr = read_sysreg_s((reg)); \
223 u64 __new = __zcr & ~ZCR_ELx_LEN_MASK; \
224 __new |= (val) & ZCR_ELx_LEN_MASK; \
225 if (__zcr != __new) \
226 write_sysreg_s(__new, (reg)); \
227 } while (0)
228
229/*
230 * Probing and setup functions.
231 * Calls to these functions must be serialised with one another.
232 */
233enum vec_type;
234
235extern void __init vec_init_vq_map(enum vec_type type);
236extern void vec_update_vq_map(enum vec_type type);
237extern int vec_verify_vq_map(enum vec_type type);
238extern void __init sve_setup(void);
239
240extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];
241
242static inline void write_vl(enum vec_type type, u64 val)
243{
244 u64 tmp;
245
246 switch (type) {
247#ifdef CONFIG_ARM64_SVE
248 case ARM64_VEC_SVE:
249 tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
250 write_sysreg_s(tmp | val, SYS_ZCR_EL1);
251 break;
252#endif
253#ifdef CONFIG_ARM64_SME
254 case ARM64_VEC_SME:
255 tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;
256 write_sysreg_s(tmp | val, SYS_SMCR_EL1);
257 break;
258#endif
259 default:
260 WARN_ON_ONCE(1);
261 break;
262 }
263}
264
265static inline int vec_max_vl(enum vec_type type)
266{
267 return vl_info[type].max_vl;
268}
269
270static inline int vec_max_virtualisable_vl(enum vec_type type)
271{
272 return vl_info[type].max_virtualisable_vl;
273}
274
275static inline int sve_max_vl(void)
276{
277 return vec_max_vl(ARM64_VEC_SVE);
278}
279
280static inline int sve_max_virtualisable_vl(void)
281{
282 return vec_max_virtualisable_vl(ARM64_VEC_SVE);
283}
284
285/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
286static inline bool vq_available(enum vec_type type, unsigned int vq)
287{
288 return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
289}
290
291static inline bool sve_vq_available(unsigned int vq)
292{
293 return vq_available(ARM64_VEC_SVE, vq);
294}
295
296size_t sve_state_size(struct task_struct const *task);
297
298#else /* ! CONFIG_ARM64_SVE */
299
300static inline void sve_alloc(struct task_struct *task, bool flush) { }
301static inline void fpsimd_release_task(struct task_struct *task) { }
302static inline void sve_sync_to_fpsimd(struct task_struct *task) { }
303static inline void sve_sync_from_fpsimd_zeropad(struct task_struct *task) { }
304
305static inline int sve_max_virtualisable_vl(void)
306{
307 return 0;
308}
309
310static inline int sve_set_current_vl(unsigned long arg)
311{
312 return -EINVAL;
313}
314
315static inline int sve_get_current_vl(void)
316{
317 return -EINVAL;
318}
319
320static inline int sve_max_vl(void)
321{
322 return -EINVAL;
323}
324
325static inline bool sve_vq_available(unsigned int vq) { return false; }
326
327static inline void sve_user_disable(void) { BUILD_BUG(); }
328static inline void sve_user_enable(void) { BUILD_BUG(); }
329
330#define sve_cond_update_zcr_vq(val, reg) do { } while (0)
331
332static inline void vec_init_vq_map(enum vec_type t) { }
333static inline void vec_update_vq_map(enum vec_type t) { }
334static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
335static inline void sve_setup(void) { }
336
337static inline size_t sve_state_size(struct task_struct const *task)
338{
339 return 0;
340}
341
342#endif /* ! CONFIG_ARM64_SVE */
343
344#ifdef CONFIG_ARM64_SME
345
346static inline void sme_user_disable(void)
347{
348 sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0);
349}
350
351static inline void sme_user_enable(void)
352{
353 sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN);
354}
355
356static inline void sme_smstart_sm(void)
357{
358 asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr"));
359}
360
361static inline void sme_smstop_sm(void)
362{
363 asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr"));
364}
365
366static inline void sme_smstop(void)
367{
368 asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));
369}
370
371extern void __init sme_setup(void);
372
373static inline int sme_max_vl(void)
374{
375 return vec_max_vl(ARM64_VEC_SME);
376}
377
378static inline int sme_max_virtualisable_vl(void)
379{
380 return vec_max_virtualisable_vl(ARM64_VEC_SME);
381}
382
383extern void sme_alloc(struct task_struct *task, bool flush);
384extern unsigned int sme_get_vl(void);
385extern int sme_set_current_vl(unsigned long arg);
386extern int sme_get_current_vl(void);
387extern void sme_suspend_exit(void);
388
389/*
390 * Return how many bytes of memory are required to store the full SME
391 * specific state for task, given task's currently configured vector
392 * length.
393 */
394static inline size_t sme_state_size(struct task_struct const *task)
395{
396 unsigned int vl = task_get_sme_vl(task);
397 size_t size;
398
399 size = ZA_SIG_REGS_SIZE(sve_vq_from_vl(vl));
400
401 if (system_supports_sme2())
402 size += ZT_SIG_REG_SIZE;
403
404 return size;
405}
406
407#else
408
409static inline void sme_user_disable(void) { BUILD_BUG(); }
410static inline void sme_user_enable(void) { BUILD_BUG(); }
411
412static inline void sme_smstart_sm(void) { }
413static inline void sme_smstop_sm(void) { }
414static inline void sme_smstop(void) { }
415
416static inline void sme_alloc(struct task_struct *task, bool flush) { }
417static inline void sme_setup(void) { }
418static inline unsigned int sme_get_vl(void) { return 0; }
419static inline int sme_max_vl(void) { return 0; }
420static inline int sme_max_virtualisable_vl(void) { return 0; }
421static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
422static inline int sme_get_current_vl(void) { return -EINVAL; }
423static inline void sme_suspend_exit(void) { }
424
425static inline size_t sme_state_size(struct task_struct const *task)
426{
427 return 0;
428}
429
430#endif /* ! CONFIG_ARM64_SME */
431
432/* For use by EFI runtime services calls only */
433extern void __efi_fpsimd_begin(void);
434extern void __efi_fpsimd_end(void);
435
436#endif
437
438#endif