tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (c) 2024 Ventana Micro Systems Inc.
4 */
5
6#ifndef __KVM_NACL_H
7#define __KVM_NACL_H
8
9#include <linux/jump_label.h>
10#include <linux/percpu.h>
11#include <asm/byteorder.h>
12#include <asm/csr.h>
13#include <asm/sbi.h>
14
15struct kvm_vcpu_arch;
16
17DECLARE_STATIC_KEY_FALSE(kvm_riscv_nacl_available);
18#define kvm_riscv_nacl_available() \
19 static_branch_unlikely(&kvm_riscv_nacl_available)
20
21DECLARE_STATIC_KEY_FALSE(kvm_riscv_nacl_sync_csr_available);
22#define kvm_riscv_nacl_sync_csr_available() \
23 static_branch_unlikely(&kvm_riscv_nacl_sync_csr_available)
24
25DECLARE_STATIC_KEY_FALSE(kvm_riscv_nacl_sync_hfence_available);
26#define kvm_riscv_nacl_sync_hfence_available() \
27 static_branch_unlikely(&kvm_riscv_nacl_sync_hfence_available)
28
29DECLARE_STATIC_KEY_FALSE(kvm_riscv_nacl_sync_sret_available);
30#define kvm_riscv_nacl_sync_sret_available() \
31 static_branch_unlikely(&kvm_riscv_nacl_sync_sret_available)
32
33DECLARE_STATIC_KEY_FALSE(kvm_riscv_nacl_autoswap_csr_available);
34#define kvm_riscv_nacl_autoswap_csr_available() \
35 static_branch_unlikely(&kvm_riscv_nacl_autoswap_csr_available)
36
37struct kvm_riscv_nacl {
38 void *shmem;
39 phys_addr_t shmem_phys;
40};
41DECLARE_PER_CPU(struct kvm_riscv_nacl, kvm_riscv_nacl);
42
43void __kvm_riscv_nacl_hfence(void *shmem,
44 unsigned long control,
45 unsigned long page_num,
46 unsigned long page_count);
47
48void __kvm_riscv_nacl_switch_to(struct kvm_vcpu_arch *vcpu_arch,
49 unsigned long sbi_ext_id,
50 unsigned long sbi_func_id);
51
52int kvm_riscv_nacl_enable(void);
53
54void kvm_riscv_nacl_disable(void);
55
56void kvm_riscv_nacl_exit(void);
57
58int kvm_riscv_nacl_init(void);
59
60#ifdef CONFIG_32BIT
61#define lelong_to_cpu(__x) le32_to_cpu(__x)
62#define cpu_to_lelong(__x) cpu_to_le32(__x)
63#else
64#define lelong_to_cpu(__x) le64_to_cpu(__x)
65#define cpu_to_lelong(__x) cpu_to_le64(__x)
66#endif
67
68#define nacl_shmem() \
69 this_cpu_ptr(&kvm_riscv_nacl)->shmem
70
71#define nacl_scratch_read_long(__shmem, __offset) \
72({ \
73 unsigned long *__p = (__shmem) + \
74 SBI_NACL_SHMEM_SCRATCH_OFFSET + \
75 (__offset); \
76 lelong_to_cpu(*__p); \
77})
78
79#define nacl_scratch_write_long(__shmem, __offset, __val) \
80do { \
81 unsigned long *__p = (__shmem) + \
82 SBI_NACL_SHMEM_SCRATCH_OFFSET + \
83 (__offset); \
84 *__p = cpu_to_lelong(__val); \
85} while (0)
86
87#define nacl_scratch_write_longs(__shmem, __offset, __array, __count) \
88do { \
89 unsigned int __i; \
90 unsigned long *__p = (__shmem) + \
91 SBI_NACL_SHMEM_SCRATCH_OFFSET + \
92 (__offset); \
93 for (__i = 0; __i < (__count); __i++) \
94 __p[__i] = cpu_to_lelong((__array)[__i]); \
95} while (0)
96
97#define nacl_sync_hfence(__e) \
98 sbi_ecall(SBI_EXT_NACL, SBI_EXT_NACL_SYNC_HFENCE, \
99 (__e), 0, 0, 0, 0, 0)
100
101#define nacl_hfence_mkconfig(__type, __order, __vmid, __asid) \
102({ \
103 unsigned long __c = SBI_NACL_SHMEM_HFENCE_CONFIG_PEND; \
104 __c |= ((__type) & SBI_NACL_SHMEM_HFENCE_CONFIG_TYPE_MASK) \
105 << SBI_NACL_SHMEM_HFENCE_CONFIG_TYPE_SHIFT; \
106 __c |= (((__order) - SBI_NACL_SHMEM_HFENCE_ORDER_BASE) & \
107 SBI_NACL_SHMEM_HFENCE_CONFIG_ORDER_MASK) \
108 << SBI_NACL_SHMEM_HFENCE_CONFIG_ORDER_SHIFT; \
109 __c |= ((__vmid) & SBI_NACL_SHMEM_HFENCE_CONFIG_VMID_MASK) \
110 << SBI_NACL_SHMEM_HFENCE_CONFIG_VMID_SHIFT; \
111 __c |= ((__asid) & SBI_NACL_SHMEM_HFENCE_CONFIG_ASID_MASK); \
112 __c; \
113})
114
115#define nacl_hfence_mkpnum(__order, __addr) \
116 ((__addr) >> (__order))
117
118#define nacl_hfence_mkpcount(__order, __size) \
119 ((__size) >> (__order))
120
121#define nacl_hfence_gvma(__shmem, __gpa, __gpsz, __order) \
122__kvm_riscv_nacl_hfence(__shmem, \
123 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_GVMA, \
124 __order, 0, 0), \
125 nacl_hfence_mkpnum(__order, __gpa), \
126 nacl_hfence_mkpcount(__order, __gpsz))
127
128#define nacl_hfence_gvma_all(__shmem) \
129__kvm_riscv_nacl_hfence(__shmem, \
130 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_GVMA_ALL, \
131 0, 0, 0), 0, 0)
132
133#define nacl_hfence_gvma_vmid(__shmem, __vmid, __gpa, __gpsz, __order) \
134__kvm_riscv_nacl_hfence(__shmem, \
135 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_GVMA_VMID, \
136 __order, __vmid, 0), \
137 nacl_hfence_mkpnum(__order, __gpa), \
138 nacl_hfence_mkpcount(__order, __gpsz))
139
140#define nacl_hfence_gvma_vmid_all(__shmem, __vmid) \
141__kvm_riscv_nacl_hfence(__shmem, \
142 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_GVMA_VMID_ALL, \
143 0, __vmid, 0), 0, 0)
144
145#define nacl_hfence_vvma(__shmem, __vmid, __gva, __gvsz, __order) \
146__kvm_riscv_nacl_hfence(__shmem, \
147 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_VVMA, \
148 __order, __vmid, 0), \
149 nacl_hfence_mkpnum(__order, __gva), \
150 nacl_hfence_mkpcount(__order, __gvsz))
151
152#define nacl_hfence_vvma_all(__shmem, __vmid) \
153__kvm_riscv_nacl_hfence(__shmem, \
154 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_VVMA_ALL, \
155 0, __vmid, 0), 0, 0)
156
157#define nacl_hfence_vvma_asid(__shmem, __vmid, __asid, __gva, __gvsz, __order)\
158__kvm_riscv_nacl_hfence(__shmem, \
159 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_VVMA_ASID, \
160 __order, __vmid, __asid), \
161 nacl_hfence_mkpnum(__order, __gva), \
162 nacl_hfence_mkpcount(__order, __gvsz))
163
164#define nacl_hfence_vvma_asid_all(__shmem, __vmid, __asid) \
165__kvm_riscv_nacl_hfence(__shmem, \
166 nacl_hfence_mkconfig(SBI_NACL_SHMEM_HFENCE_TYPE_VVMA_ASID_ALL, \
167 0, __vmid, __asid), 0, 0)
168
169#define nacl_csr_read(__shmem, __csr) \
170({ \
171 unsigned long *__a = (__shmem) + SBI_NACL_SHMEM_CSR_OFFSET; \
172 lelong_to_cpu(__a[SBI_NACL_SHMEM_CSR_INDEX(__csr)]); \
173})
174
175#define nacl_csr_write(__shmem, __csr, __val) \
176do { \
177 void *__s = (__shmem); \
178 unsigned int __i = SBI_NACL_SHMEM_CSR_INDEX(__csr); \
179 unsigned long *__a = (__s) + SBI_NACL_SHMEM_CSR_OFFSET; \
180 u8 *__b = (__s) + SBI_NACL_SHMEM_DBITMAP_OFFSET; \
181 __a[__i] = cpu_to_lelong(__val); \
182 __b[__i >> 3] |= 1U << (__i & 0x7); \
183} while (0)
184
185#define nacl_csr_swap(__shmem, __csr, __val) \
186({ \
187 void *__s = (__shmem); \
188 unsigned int __i = SBI_NACL_SHMEM_CSR_INDEX(__csr); \
189 unsigned long *__a = (__s) + SBI_NACL_SHMEM_CSR_OFFSET; \
190 u8 *__b = (__s) + SBI_NACL_SHMEM_DBITMAP_OFFSET; \
191 unsigned long __r = lelong_to_cpu(__a[__i]); \
192 __a[__i] = cpu_to_lelong(__val); \
193 __b[__i >> 3] |= 1U << (__i & 0x7); \
194 __r; \
195})
196
197#define nacl_sync_csr(__csr) \
198 sbi_ecall(SBI_EXT_NACL, SBI_EXT_NACL_SYNC_CSR, \
199 (__csr), 0, 0, 0, 0, 0)
200
201/*
202 * Each ncsr_xyz() macro defined below has it's own static-branch so every
203 * use of ncsr_xyz() macro emits a patchable direct jump. This means multiple
204 * back-to-back ncsr_xyz() macro usage will emit multiple patchable direct
205 * jumps which is sub-optimal.
206 *
207 * Based on the above, it is recommended to avoid multiple back-to-back
208 * ncsr_xyz() macro usage.
209 */
210
211#define ncsr_read(__csr) \
212({ \
213 unsigned long __r; \
214 if (kvm_riscv_nacl_available()) \
215 __r = nacl_csr_read(nacl_shmem(), __csr); \
216 else \
217 __r = csr_read(__csr); \
218 __r; \
219})
220
221#define ncsr_write(__csr, __val) \
222do { \
223 if (kvm_riscv_nacl_sync_csr_available()) \
224 nacl_csr_write(nacl_shmem(), __csr, __val); \
225 else \
226 csr_write(__csr, __val); \
227} while (0)
228
229#define ncsr_swap(__csr, __val) \
230({ \
231 unsigned long __r; \
232 if (kvm_riscv_nacl_sync_csr_available()) \
233 __r = nacl_csr_swap(nacl_shmem(), __csr, __val); \
234 else \
235 __r = csr_swap(__csr, __val); \
236 __r; \
237})
238
239#define nsync_csr(__csr) \
240do { \
241 if (kvm_riscv_nacl_sync_csr_available()) \
242 nacl_sync_csr(__csr); \
243} while (0)
244
245#endif