tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
fork
Configure Feed
Select the types of activity you want to include in your feed.
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2012 ARM Ltd.
4 */
5#ifndef __ASM_FP_H
6#define __ASM_FP_H
7
8#include <asm/errno.h>
9#include <asm/percpu.h>
10#include <asm/ptrace.h>
11#include <asm/processor.h>
12#include <asm/sigcontext.h>
13#include <asm/sysreg.h>
14
15#ifndef __ASSEMBLER__
16
17#include <linux/bitmap.h>
18#include <linux/build_bug.h>
19#include <linux/bug.h>
20#include <linux/cache.h>
21#include <linux/init.h>
22#include <linux/stddef.h>
23#include <linux/types.h>
24
25/* Masks for extracting the FPSR and FPCR from the FPSCR */
26#define VFP_FPSCR_STAT_MASK 0xf800009f
27#define VFP_FPSCR_CTRL_MASK 0x07f79f00
28/*
29 * The VFP state has 32x64-bit registers and a single 32-bit
30 * control/status register.
31 */
32#define VFP_STATE_SIZE ((32 * 8) + 4)
33
34static inline unsigned long cpacr_save_enable_kernel_sve(void)
35{
36 unsigned long old = read_sysreg(cpacr_el1);
37 unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_ZEN_EL1EN;
38
39 write_sysreg(old | set, cpacr_el1);
40 isb();
41 return old;
42}
43
44static inline unsigned long cpacr_save_enable_kernel_sme(void)
45{
46 unsigned long old = read_sysreg(cpacr_el1);
47 unsigned long set = CPACR_EL1_FPEN_EL1EN | CPACR_EL1_SMEN_EL1EN;
48
49 write_sysreg(old | set, cpacr_el1);
50 isb();
51 return old;
52}
53
54static inline void cpacr_restore(unsigned long cpacr)
55{
56 write_sysreg(cpacr, cpacr_el1);
57 isb();
58}
59
60/*
61 * When we defined the maximum SVE vector length we defined the ABI so
62 * that the maximum vector length included all the reserved for future
63 * expansion bits in ZCR rather than those just currently defined by
64 * the architecture. Using this length to allocate worst size buffers
65 * results in excessively large allocations, and this effect is even
66 * more pronounced for SME due to ZA. Define more suitable VLs for
67 * these situations.
68 */
69#define ARCH_SVE_VQ_MAX ((ZCR_ELx_LEN_MASK >> ZCR_ELx_LEN_SHIFT) + 1)
70#define SME_VQ_MAX ((SMCR_ELx_LEN_MASK >> SMCR_ELx_LEN_SHIFT) + 1)
71
72struct task_struct;
73
74extern void fpsimd_save_state(struct user_fpsimd_state *state);
75extern void fpsimd_load_state(struct user_fpsimd_state *state);
76
77extern void fpsimd_thread_switch(struct task_struct *next);
78extern void fpsimd_flush_thread(void);
79
80extern void fpsimd_preserve_current_state(void);
81extern void fpsimd_restore_current_state(void);
82extern void fpsimd_update_current_state(struct user_fpsimd_state const *state);
83
84struct cpu_fp_state {
85 struct user_fpsimd_state *st;
86 void *sve_state;
87 void *sme_state;
88 u64 *svcr;
89 u64 *fpmr;
90 unsigned int sve_vl;
91 unsigned int sme_vl;
92 enum fp_type *fp_type;
93 enum fp_type to_save;
94};
95
96DECLARE_PER_CPU(struct cpu_fp_state, fpsimd_last_state);
97
98extern void fpsimd_bind_state_to_cpu(struct cpu_fp_state *fp_state);
99
100extern void fpsimd_flush_task_state(struct task_struct *target);
101extern void fpsimd_save_and_flush_current_state(void);
102extern void fpsimd_save_and_flush_cpu_state(void);
103
104static inline bool thread_sm_enabled(struct thread_struct *thread)
105{
106 return system_supports_sme() && (thread->svcr & SVCR_SM_MASK);
107}
108
109static inline bool thread_za_enabled(struct thread_struct *thread)
110{
111 return system_supports_sme() && (thread->svcr & SVCR_ZA_MASK);
112}
113
114extern void task_smstop_sm(struct task_struct *task);
115
116/* Maximum VL that SVE/SME VL-agnostic software can transparently support */
117#define VL_ARCH_MAX 0x100
118
119/* Offset of FFR in the SVE register dump */
120static inline size_t sve_ffr_offset(int vl)
121{
122 return SVE_SIG_FFR_OFFSET(sve_vq_from_vl(vl)) - SVE_SIG_REGS_OFFSET;
123}
124
125static inline void *sve_pffr(struct thread_struct *thread)
126{
127 unsigned int vl;
128
129 if (system_supports_sme() && thread_sm_enabled(thread))
130 vl = thread_get_sme_vl(thread);
131 else
132 vl = thread_get_sve_vl(thread);
133
134 return (char *)thread->sve_state + sve_ffr_offset(vl);
135}
136
137static inline void *thread_zt_state(struct thread_struct *thread)
138{
139 /* The ZT register state is stored immediately after the ZA state */
140 unsigned int sme_vq = sve_vq_from_vl(thread_get_sme_vl(thread));
141 return thread->sme_state + ZA_SIG_REGS_SIZE(sme_vq);
142}
143
144extern void sve_save_state(void *state, u32 *pfpsr, int save_ffr);
145extern void sve_load_state(void const *state, u32 const *pfpsr,
146 int restore_ffr);
147extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
148extern unsigned int sve_get_vl(void);
149extern void sve_set_vq(unsigned long vq_minus_1);
150extern void sme_set_vq(unsigned long vq_minus_1);
151extern void sme_save_state(void *state, int zt);
152extern void sme_load_state(void const *state, int zt);
153
154struct arm64_cpu_capabilities;
155extern void cpu_enable_fpsimd(const struct arm64_cpu_capabilities *__unused);
156extern void cpu_enable_sve(const struct arm64_cpu_capabilities *__unused);
157extern void cpu_enable_sme(const struct arm64_cpu_capabilities *__unused);
158extern void cpu_enable_sme2(const struct arm64_cpu_capabilities *__unused);
159extern void cpu_enable_fa64(const struct arm64_cpu_capabilities *__unused);
160extern void cpu_enable_fpmr(const struct arm64_cpu_capabilities *__unused);
161
162/*
163 * Helpers to translate bit indices in sve_vq_map to VQ values (and
164 * vice versa). This allows find_next_bit() to be used to find the
165 * _maximum_ VQ not exceeding a certain value.
166 */
167static inline unsigned int __vq_to_bit(unsigned int vq)
168{
169 return SVE_VQ_MAX - vq;
170}
171
172static inline unsigned int __bit_to_vq(unsigned int bit)
173{
174 return SVE_VQ_MAX - bit;
175}
176
177
178struct vl_info {
179 enum vec_type type;
180 const char *name; /* For display purposes */
181
182 /* Minimum supported vector length across all CPUs */
183 int min_vl;
184
185 /* Maximum supported vector length across all CPUs */
186 int max_vl;
187 int max_virtualisable_vl;
188
189 /*
190 * Set of available vector lengths,
191 * where length vq encoded as bit __vq_to_bit(vq):
192 */
193 DECLARE_BITMAP(vq_map, SVE_VQ_MAX);
194
195 /* Set of vector lengths present on at least one cpu: */
196 DECLARE_BITMAP(vq_partial_map, SVE_VQ_MAX);
197};
198
199#ifdef CONFIG_ARM64_SVE
200
201extern void sve_alloc(struct task_struct *task, bool flush);
202extern void fpsimd_release_task(struct task_struct *task);
203extern void fpsimd_sync_from_effective_state(struct task_struct *task);
204extern void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task);
205
206extern int vec_set_vector_length(struct task_struct *task, enum vec_type type,
207 unsigned long vl, unsigned long flags);
208
209extern int sve_set_current_vl(unsigned long arg);
210extern int sve_get_current_vl(void);
211
212static inline void sve_user_disable(void)
213{
214 sysreg_clear_set(cpacr_el1, CPACR_EL1_ZEN_EL0EN, 0);
215}
216
217static inline void sve_user_enable(void)
218{
219 sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_ZEN_EL0EN);
220}
221
222#define sve_cond_update_zcr_vq(val, reg) \
223 do { \
224 u64 __zcr = read_sysreg_s((reg)); \
225 u64 __new = __zcr & ~ZCR_ELx_LEN_MASK; \
226 __new |= (val) & ZCR_ELx_LEN_MASK; \
227 if (__zcr != __new) \
228 write_sysreg_s(__new, (reg)); \
229 } while (0)
230
231/*
232 * Probing and setup functions.
233 * Calls to these functions must be serialised with one another.
234 */
235enum vec_type;
236
237extern void __init vec_init_vq_map(enum vec_type type);
238extern void vec_update_vq_map(enum vec_type type);
239extern int vec_verify_vq_map(enum vec_type type);
240extern void __init sve_setup(void);
241
242extern __ro_after_init struct vl_info vl_info[ARM64_VEC_MAX];
243
244static inline void write_vl(enum vec_type type, u64 val)
245{
246 u64 tmp;
247
248 switch (type) {
249#ifdef CONFIG_ARM64_SVE
250 case ARM64_VEC_SVE:
251 tmp = read_sysreg_s(SYS_ZCR_EL1) & ~ZCR_ELx_LEN_MASK;
252 write_sysreg_s(tmp | val, SYS_ZCR_EL1);
253 break;
254#endif
255#ifdef CONFIG_ARM64_SME
256 case ARM64_VEC_SME:
257 tmp = read_sysreg_s(SYS_SMCR_EL1) & ~SMCR_ELx_LEN_MASK;
258 write_sysreg_s(tmp | val, SYS_SMCR_EL1);
259 break;
260#endif
261 default:
262 WARN_ON_ONCE(1);
263 break;
264 }
265}
266
267static inline int vec_max_vl(enum vec_type type)
268{
269 return vl_info[type].max_vl;
270}
271
272static inline int vec_max_virtualisable_vl(enum vec_type type)
273{
274 return vl_info[type].max_virtualisable_vl;
275}
276
277static inline int sve_max_vl(void)
278{
279 return vec_max_vl(ARM64_VEC_SVE);
280}
281
282static inline int sve_max_virtualisable_vl(void)
283{
284 return vec_max_virtualisable_vl(ARM64_VEC_SVE);
285}
286
287/* Ensure vq >= SVE_VQ_MIN && vq <= SVE_VQ_MAX before calling this function */
288static inline bool vq_available(enum vec_type type, unsigned int vq)
289{
290 return test_bit(__vq_to_bit(vq), vl_info[type].vq_map);
291}
292
293static inline bool sve_vq_available(unsigned int vq)
294{
295 return vq_available(ARM64_VEC_SVE, vq);
296}
297
298static inline size_t __sve_state_size(unsigned int sve_vl, unsigned int sme_vl)
299{
300 unsigned int vl = max(sve_vl, sme_vl);
301 return SVE_SIG_REGS_SIZE(sve_vq_from_vl(vl));
302}
303
304/*
305 * Return how many bytes of memory are required to store the full SVE
306 * state for task, given task's currently configured vector length.
307 */
308static inline size_t sve_state_size(struct task_struct const *task)
309{
310 unsigned int sve_vl = task_get_sve_vl(task);
311 unsigned int sme_vl = task_get_sme_vl(task);
312 return __sve_state_size(sve_vl, sme_vl);
313}
314
315#else /* ! CONFIG_ARM64_SVE */
316
317static inline void sve_alloc(struct task_struct *task, bool flush) { }
318static inline void fpsimd_release_task(struct task_struct *task) { }
319static inline void fpsimd_sync_from_effective_state(struct task_struct *task) { }
320static inline void fpsimd_sync_to_effective_state_zeropad(struct task_struct *task) { }
321
322static inline int sve_max_virtualisable_vl(void)
323{
324 return 0;
325}
326
327static inline int sve_set_current_vl(unsigned long arg)
328{
329 return -EINVAL;
330}
331
332static inline int sve_get_current_vl(void)
333{
334 return -EINVAL;
335}
336
337static inline int sve_max_vl(void)
338{
339 return -EINVAL;
340}
341
342static inline bool sve_vq_available(unsigned int vq) { return false; }
343
344static inline void sve_user_disable(void) { BUILD_BUG(); }
345static inline void sve_user_enable(void) { BUILD_BUG(); }
346
347#define sve_cond_update_zcr_vq(val, reg) do { } while (0)
348
349static inline void vec_init_vq_map(enum vec_type t) { }
350static inline void vec_update_vq_map(enum vec_type t) { }
351static inline int vec_verify_vq_map(enum vec_type t) { return 0; }
352static inline void sve_setup(void) { }
353
354static inline size_t __sve_state_size(unsigned int sve_vl, unsigned int sme_vl)
355{
356 return 0;
357}
358
359static inline size_t sve_state_size(struct task_struct const *task)
360{
361 return 0;
362}
363
364#endif /* ! CONFIG_ARM64_SVE */
365
366#ifdef CONFIG_ARM64_SME
367
368static inline void sme_user_disable(void)
369{
370 sysreg_clear_set(cpacr_el1, CPACR_EL1_SMEN_EL0EN, 0);
371}
372
373static inline void sme_user_enable(void)
374{
375 sysreg_clear_set(cpacr_el1, 0, CPACR_EL1_SMEN_EL0EN);
376}
377
378static inline void sme_smstart_sm(void)
379{
380 asm volatile(__msr_s(SYS_SVCR_SMSTART_SM_EL0, "xzr"));
381}
382
383static inline void sme_smstop_sm(void)
384{
385 asm volatile(__msr_s(SYS_SVCR_SMSTOP_SM_EL0, "xzr"));
386}
387
388static inline void sme_smstop(void)
389{
390 asm volatile(__msr_s(SYS_SVCR_SMSTOP_SMZA_EL0, "xzr"));
391}
392
393extern void __init sme_setup(void);
394
395static inline int sme_max_vl(void)
396{
397 return vec_max_vl(ARM64_VEC_SME);
398}
399
400static inline int sme_max_virtualisable_vl(void)
401{
402 return vec_max_virtualisable_vl(ARM64_VEC_SME);
403}
404
405extern void sme_alloc(struct task_struct *task, bool flush);
406extern unsigned int sme_get_vl(void);
407extern int sme_set_current_vl(unsigned long arg);
408extern int sme_get_current_vl(void);
409extern void sme_suspend_exit(void);
410
411static inline size_t __sme_state_size(unsigned int sme_vl)
412{
413 size_t size = ZA_SIG_REGS_SIZE(sve_vq_from_vl(sme_vl));
414
415 if (system_supports_sme2())
416 size += ZT_SIG_REG_SIZE;
417
418 return size;
419}
420
421/*
422 * Return how many bytes of memory are required to store the full SME
423 * specific state for task, given task's currently configured vector
424 * length.
425 */
426static inline size_t sme_state_size(struct task_struct const *task)
427{
428 return __sme_state_size(task_get_sme_vl(task));
429}
430
431#else
432
433static inline void sme_user_disable(void) { BUILD_BUG(); }
434static inline void sme_user_enable(void) { BUILD_BUG(); }
435
436static inline void sme_smstart_sm(void) { }
437static inline void sme_smstop_sm(void) { }
438static inline void sme_smstop(void) { }
439
440static inline void sme_alloc(struct task_struct *task, bool flush) { }
441static inline void sme_setup(void) { }
442static inline unsigned int sme_get_vl(void) { return 0; }
443static inline int sme_max_vl(void) { return 0; }
444static inline int sme_max_virtualisable_vl(void) { return 0; }
445static inline int sme_set_current_vl(unsigned long arg) { return -EINVAL; }
446static inline int sme_get_current_vl(void) { return -EINVAL; }
447static inline void sme_suspend_exit(void) { }
448
449static inline size_t __sme_state_size(unsigned int sme_vl)
450{
451 return 0;
452}
453
454static inline size_t sme_state_size(struct task_struct const *task)
455{
456 return 0;
457}
458
459#endif /* ! CONFIG_ARM64_SME */
460
461/* For use by EFI runtime services calls only */
462extern void __efi_fpsimd_begin(void);
463extern void __efi_fpsimd_end(void);
464
465#endif
466
467#endif