tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2
3#include "rseq-bits-template.h"
4
5#if defined(RSEQ_TEMPLATE_MO_RELAXED) && \
6 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
7
8static inline __always_inline
9int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv,
10 int cpu)
11{
12 RSEQ_INJECT_C(9)
13
14 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
15 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
16#ifdef RSEQ_COMPARE_TWICE
17 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
18 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
19#endif
20 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
21 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
22 RSEQ_INJECT_ASM(3)
23 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
24 RSEQ_INJECT_ASM(4)
25#ifdef RSEQ_COMPARE_TWICE
26 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
27 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
28#endif
29 RSEQ_ASM_OP_FINAL_STORE(v, newv, 3)
30 RSEQ_INJECT_ASM(5)
31 RSEQ_ASM_DEFINE_ABORT(4, abort)
32 : /* gcc asm goto does not allow outputs */
33 : [cpu_id] "r" (cpu),
34 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
35 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
36 [v] "m" (*v),
37 [expect] "r" (expect),
38 [newv] "r" (newv)
39 RSEQ_INJECT_INPUT
40 : "memory", RSEQ_ASM_TMP_REG_1
41 RSEQ_INJECT_CLOBBER
42 : abort, cmpfail
43#ifdef RSEQ_COMPARE_TWICE
44 , error1, error2
45#endif
46 );
47
48 return 0;
49abort:
50 RSEQ_INJECT_FAILED
51 return -1;
52cmpfail:
53 return 1;
54#ifdef RSEQ_COMPARE_TWICE
55error1:
56 rseq_bug("cpu_id comparison failed");
57error2:
58 rseq_bug("expected value comparison failed");
59#endif
60}
61
62static inline __always_inline
63int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot,
64 off_t voffp, intptr_t *load, int cpu)
65{
66 RSEQ_INJECT_C(9)
67
68 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
69 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
70#ifdef RSEQ_COMPARE_TWICE
71 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
72 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
73#endif
74 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
75 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
76 RSEQ_INJECT_ASM(3)
77 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")
78 RSEQ_INJECT_ASM(4)
79#ifdef RSEQ_COMPARE_TWICE
80 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
81 RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")
82#endif
83 RSEQ_ASM_OP_R_LOAD(v)
84 RSEQ_ASM_OP_R_STORE(load)
85 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
86 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
87 RSEQ_INJECT_ASM(5)
88 RSEQ_ASM_DEFINE_ABORT(4, abort)
89 : /* gcc asm goto does not allow outputs */
90 : [cpu_id] "r" (cpu),
91 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
92 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
93 [v] "m" (*v),
94 [expectnot] "r" (expectnot),
95 [load] "m" (*load),
96 [voffp] "Ir" (voffp)
97 RSEQ_INJECT_INPUT
98 : "memory", RSEQ_ASM_TMP_REG_1
99 RSEQ_INJECT_CLOBBER
100 : abort, cmpfail
101#ifdef RSEQ_COMPARE_TWICE
102 , error1, error2
103#endif
104 );
105 return 0;
106abort:
107 RSEQ_INJECT_FAILED
108 return -1;
109cmpfail:
110 return 1;
111#ifdef RSEQ_COMPARE_TWICE
112error1:
113 rseq_bug("cpu_id comparison failed");
114error2:
115 rseq_bug("expected value comparison failed");
116#endif
117}
118
119static inline __always_inline
120int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu)
121{
122 RSEQ_INJECT_C(9)
123
124 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
125#ifdef RSEQ_COMPARE_TWICE
126 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
127#endif
128 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
129 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
130 RSEQ_INJECT_ASM(3)
131#ifdef RSEQ_COMPARE_TWICE
132 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
133#endif
134 RSEQ_ASM_OP_R_LOAD(v)
135 RSEQ_ASM_OP_R_ADD(count)
136 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
137 RSEQ_INJECT_ASM(4)
138 RSEQ_ASM_DEFINE_ABORT(4, abort)
139 : /* gcc asm goto does not allow outputs */
140 : [cpu_id] "r" (cpu),
141 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
142 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
143 [v] "m" (*v),
144 [count] "r" (count)
145 RSEQ_INJECT_INPUT
146 : "memory", RSEQ_ASM_TMP_REG_1
147 RSEQ_INJECT_CLOBBER
148 : abort
149#ifdef RSEQ_COMPARE_TWICE
150 , error1
151#endif
152 );
153 return 0;
154abort:
155 RSEQ_INJECT_FAILED
156 return -1;
157#ifdef RSEQ_COMPARE_TWICE
158error1:
159 rseq_bug("cpu_id comparison failed");
160#endif
161}
162
163static inline __always_inline
164int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect,
165 intptr_t *v2, intptr_t expect2,
166 intptr_t newv, int cpu)
167{
168 RSEQ_INJECT_C(9)
169
170 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
171 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
172#ifdef RSEQ_COMPARE_TWICE
173 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
174 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
175 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
176#endif
177 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
178 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
179 RSEQ_INJECT_ASM(3)
180 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
181 RSEQ_INJECT_ASM(4)
182 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")
183 RSEQ_INJECT_ASM(5)
184#ifdef RSEQ_COMPARE_TWICE
185 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
186 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
187 RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")
188#endif
189 RSEQ_ASM_OP_FINAL_STORE(v, newv, 3)
190 RSEQ_INJECT_ASM(6)
191 RSEQ_ASM_DEFINE_ABORT(4, abort)
192 : /* gcc asm goto does not allow outputs */
193 : [cpu_id] "r" (cpu),
194 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
195 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
196 [v] "m" (*v),
197 [expect] "r" (expect),
198 [v2] "m" (*v2),
199 [expect2] "r" (expect2),
200 [newv] "r" (newv)
201 RSEQ_INJECT_INPUT
202 : "memory", RSEQ_ASM_TMP_REG_1
203 RSEQ_INJECT_CLOBBER
204 : abort, cmpfail
205#ifdef RSEQ_COMPARE_TWICE
206 , error1, error2, error3
207#endif
208 );
209
210 return 0;
211abort:
212 RSEQ_INJECT_FAILED
213 return -1;
214cmpfail:
215 return 1;
216#ifdef RSEQ_COMPARE_TWICE
217error1:
218 rseq_bug("cpu_id comparison failed");
219error2:
220 rseq_bug("expected value comparison failed");
221error3:
222 rseq_bug("2nd expected value comparison failed");
223#endif
224}
225
226#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
227
228/*
229 * pval = *(ptr+off)
230 * *pval += inc;
231 */
232static inline __always_inline
233int RSEQ_TEMPLATE_IDENTIFIER(rseq_offset_deref_addv)(intptr_t *ptr, off_t off, intptr_t inc,
234 int cpu)
235{
236 RSEQ_INJECT_C(9)
237
238 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
239#ifdef RSEQ_COMPARE_TWICE
240 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
241#endif
242 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
243 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
244 RSEQ_INJECT_ASM(3)
245#ifdef RSEQ_COMPARE_TWICE
246 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
247#endif
248 RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, inc, 3)
249 RSEQ_INJECT_ASM(4)
250 RSEQ_ASM_DEFINE_ABORT(4, abort)
251 : /* gcc asm goto does not allow outputs */
252 : [cpu_id] "r" (cpu),
253 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
254 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
255 [ptr] "r" (ptr),
256 [off] "r" (off),
257 [inc] "r" (inc)
258 RSEQ_INJECT_INPUT
259 : "memory", RSEQ_ASM_TMP_REG_1
260 RSEQ_INJECT_CLOBBER
261 : abort
262#ifdef RSEQ_COMPARE_TWICE
263 , error1
264#endif
265 );
266 return 0;
267abort:
268 RSEQ_INJECT_FAILED
269 return -1;
270#ifdef RSEQ_COMPARE_TWICE
271error1:
272 rseq_bug("cpu_id comparison failed");
273#endif
274}
275
276#endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) &&
277 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
278
279#if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \
280 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID))
281
282static inline __always_inline
283int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect,
284 intptr_t *v2, intptr_t newv2,
285 intptr_t newv, int cpu)
286{
287 RSEQ_INJECT_C(9)
288
289 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
290 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
291#ifdef RSEQ_COMPARE_TWICE
292 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
293 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
294#endif
295 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
296 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
297 RSEQ_INJECT_ASM(3)
298 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
299 RSEQ_INJECT_ASM(4)
300#ifdef RSEQ_COMPARE_TWICE
301 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
302 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
303#endif
304 RSEQ_ASM_OP_STORE(v2, newv2)
305 RSEQ_INJECT_ASM(5)
306#ifdef RSEQ_TEMPLATE_MO_RELEASE
307 RSEQ_ASM_OP_FINAL_STORE_RELEASE(v, newv, 3)
308#else
309 RSEQ_ASM_OP_FINAL_STORE(v, newv, 3)
310#endif
311 RSEQ_INJECT_ASM(6)
312 RSEQ_ASM_DEFINE_ABORT(4, abort)
313 : /* gcc asm goto does not allow outputs */
314 : [cpu_id] "r" (cpu),
315 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
316 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
317 [expect] "r" (expect),
318 [v] "m" (*v),
319 [newv] "r" (newv),
320 [v2] "m" (*v2),
321 [newv2] "r" (newv2)
322 RSEQ_INJECT_INPUT
323 : "memory", RSEQ_ASM_TMP_REG_1
324 RSEQ_INJECT_CLOBBER
325 : abort, cmpfail
326#ifdef RSEQ_COMPARE_TWICE
327 , error1, error2
328#endif
329 );
330
331 return 0;
332abort:
333 RSEQ_INJECT_FAILED
334 return -1;
335cmpfail:
336 return 1;
337#ifdef RSEQ_COMPARE_TWICE
338error1:
339 rseq_bug("cpu_id comparison failed");
340error2:
341 rseq_bug("expected value comparison failed");
342#endif
343}
344
345static inline __always_inline
346int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect,
347 void *dst, void *src, size_t len,
348 intptr_t newv, int cpu)
349{
350 RSEQ_INJECT_C(9)
351 __asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
352 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
353#ifdef RSEQ_COMPARE_TWICE
354 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
355 RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
356#endif
357 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
358 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
359 RSEQ_INJECT_ASM(3)
360 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
361 RSEQ_INJECT_ASM(4)
362#ifdef RSEQ_COMPARE_TWICE
363 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
364 RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
365#endif
366 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
367 RSEQ_INJECT_ASM(5)
368#ifdef RSEQ_TEMPLATE_MO_RELEASE
369 RSEQ_ASM_OP_FINAL_STORE_RELEASE(v, newv, 3)
370#else
371 RSEQ_ASM_OP_FINAL_STORE(v, newv, 3)
372#endif
373 RSEQ_INJECT_ASM(6)
374 RSEQ_ASM_DEFINE_ABORT(4, abort)
375 : /* gcc asm goto does not allow outputs */
376 : [cpu_id] "r" (cpu),
377 [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD),
378 [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr),
379 [expect] "r" (expect),
380 [v] "m" (*v),
381 [newv] "r" (newv),
382 [dst] "r" (dst),
383 [src] "r" (src),
384 [len] "r" (len)
385 RSEQ_INJECT_INPUT
386 : "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
387 RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
388 RSEQ_INJECT_CLOBBER
389 : abort, cmpfail
390#ifdef RSEQ_COMPARE_TWICE
391 , error1, error2
392#endif
393 );
394
395 return 0;
396abort:
397 RSEQ_INJECT_FAILED
398 return -1;
399cmpfail:
400 return 1;
401#ifdef RSEQ_COMPARE_TWICE
402error1:
403 rseq_bug("cpu_id comparison failed");
404error2:
405 rseq_bug("expected value comparison failed");
406#endif
407}
408
409#endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) &&
410 (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */
411
412#include "rseq-bits-reset.h"