tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-arm64.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
7 */
8
9#define RSEQ_SIG 0xd428bc00 /* BRK #0x45E0 */
10
11#define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
12#define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
13#define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
14
15#define rseq_smp_load_acquire(p) \
16__extension__ ({ \
17 __typeof(*p) ____p1; \
18 switch (sizeof(*p)) { \
19 case 1: \
20 asm volatile ("ldarb %w0, %1" \
21 : "=r" (*(__u8 *)p) \
22 : "Q" (*p) : "memory"); \
23 break; \
24 case 2: \
25 asm volatile ("ldarh %w0, %1" \
26 : "=r" (*(__u16 *)p) \
27 : "Q" (*p) : "memory"); \
28 break; \
29 case 4: \
30 asm volatile ("ldar %w0, %1" \
31 : "=r" (*(__u32 *)p) \
32 : "Q" (*p) : "memory"); \
33 break; \
34 case 8: \
35 asm volatile ("ldar %0, %1" \
36 : "=r" (*(__u64 *)p) \
37 : "Q" (*p) : "memory"); \
38 break; \
39 } \
40 ____p1; \
41})
42
43#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
44
45#define rseq_smp_store_release(p, v) \
46do { \
47 switch (sizeof(*p)) { \
48 case 1: \
49 asm volatile ("stlrb %w1, %0" \
50 : "=Q" (*p) \
51 : "r" ((__u8)v) \
52 : "memory"); \
53 break; \
54 case 2: \
55 asm volatile ("stlrh %w1, %0" \
56 : "=Q" (*p) \
57 : "r" ((__u16)v) \
58 : "memory"); \
59 break; \
60 case 4: \
61 asm volatile ("stlr %w1, %0" \
62 : "=Q" (*p) \
63 : "r" ((__u32)v) \
64 : "memory"); \
65 break; \
66 case 8: \
67 asm volatile ("stlr %1, %0" \
68 : "=Q" (*p) \
69 : "r" ((__u64)v) \
70 : "memory"); \
71 break; \
72 } \
73} while (0)
74
75#ifdef RSEQ_SKIP_FASTPATH
76#include "rseq-skip.h"
77#else /* !RSEQ_SKIP_FASTPATH */
78
79#define RSEQ_ASM_TMP_REG32 "w15"
80#define RSEQ_ASM_TMP_REG "x15"
81#define RSEQ_ASM_TMP_REG_2 "x14"
82
83#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
84 post_commit_offset, abort_ip) \
85 " .pushsection __rseq_table, \"aw\"\n" \
86 " .balign 32\n" \
87 __rseq_str(label) ":\n" \
88 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
89 " .quad " __rseq_str(start_ip) ", " \
90 __rseq_str(post_commit_offset) ", " \
91 __rseq_str(abort_ip) "\n" \
92 " .popsection\n"
93
94#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
95 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
96 (post_commit_ip - start_ip), abort_ip)
97
98#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
99 RSEQ_INJECT_ASM(1) \
100 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
101 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
102 ", :lo12:" __rseq_str(cs_label) "\n" \
103 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
104 __rseq_str(label) ":\n"
105
106#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
107 " b 222f\n" \
108 " .inst " __rseq_str(RSEQ_SIG) "\n" \
109 __rseq_str(label) ":\n" \
110 " b %l[" __rseq_str(abort_label) "]\n" \
111 "222:\n"
112
113#define RSEQ_ASM_OP_STORE(value, var) \
114 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
115
116#define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
117 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
118
119#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
120 RSEQ_ASM_OP_STORE(value, var) \
121 __rseq_str(post_commit_label) ":\n"
122
123#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
124 RSEQ_ASM_OP_STORE_RELEASE(value, var) \
125 __rseq_str(post_commit_label) ":\n"
126
127#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
128 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
129 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
130 ", %[" __rseq_str(expect) "]\n" \
131 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
132
133#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
134 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
135 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
136 ", %w[" __rseq_str(expect) "]\n" \
137 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
138
139#define RSEQ_ASM_OP_CMPNE(var, expect, label) \
140 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
141 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
142 ", %[" __rseq_str(expect) "]\n" \
143 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
144
145#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
146 RSEQ_INJECT_ASM(2) \
147 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
148
149#define RSEQ_ASM_OP_R_LOAD(var) \
150 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
151
152#define RSEQ_ASM_OP_R_STORE(var) \
153 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
154
155#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
156 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
157 ", %[" __rseq_str(offset) "]]\n"
158
159#define RSEQ_ASM_OP_R_ADD(count) \
160 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
161 ", %[" __rseq_str(count) "]\n"
162
163#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
164 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
165 __rseq_str(post_commit_label) ":\n"
166
167#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
168 " cbz %[" __rseq_str(len) "], 333f\n" \
169 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
170 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
171 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
172 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
173 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
174 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
175 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
176 "333:\n"
177
178static inline __attribute__((always_inline))
179int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
180{
181 RSEQ_INJECT_C(9)
182
183 __asm__ __volatile__ goto (
184 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
185 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
186 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
187 RSEQ_INJECT_ASM(3)
188 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
189 RSEQ_INJECT_ASM(4)
190#ifdef RSEQ_COMPARE_TWICE
191 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
192 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
193#endif
194 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
195 RSEQ_INJECT_ASM(5)
196 RSEQ_ASM_DEFINE_ABORT(4, abort)
197 : /* gcc asm goto does not allow outputs */
198 : [cpu_id] "r" (cpu),
199 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
200 [rseq_cs] "m" (__rseq_abi.rseq_cs),
201 [v] "Qo" (*v),
202 [expect] "r" (expect),
203 [newv] "r" (newv)
204 RSEQ_INJECT_INPUT
205 : "memory", RSEQ_ASM_TMP_REG
206 : abort, cmpfail
207#ifdef RSEQ_COMPARE_TWICE
208 , error1, error2
209#endif
210 );
211
212 return 0;
213abort:
214 RSEQ_INJECT_FAILED
215 return -1;
216cmpfail:
217 return 1;
218#ifdef RSEQ_COMPARE_TWICE
219error1:
220 rseq_bug("cpu_id comparison failed");
221error2:
222 rseq_bug("expected value comparison failed");
223#endif
224}
225
226static inline __attribute__((always_inline))
227int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
228 off_t voffp, intptr_t *load, int cpu)
229{
230 RSEQ_INJECT_C(9)
231
232 __asm__ __volatile__ goto (
233 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
234 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
235 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
236 RSEQ_INJECT_ASM(3)
237 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
238 RSEQ_INJECT_ASM(4)
239#ifdef RSEQ_COMPARE_TWICE
240 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
241 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
242#endif
243 RSEQ_ASM_OP_R_LOAD(v)
244 RSEQ_ASM_OP_R_STORE(load)
245 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
246 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
247 RSEQ_INJECT_ASM(5)
248 RSEQ_ASM_DEFINE_ABORT(4, abort)
249 : /* gcc asm goto does not allow outputs */
250 : [cpu_id] "r" (cpu),
251 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
252 [rseq_cs] "m" (__rseq_abi.rseq_cs),
253 [v] "Qo" (*v),
254 [expectnot] "r" (expectnot),
255 [load] "Qo" (*load),
256 [voffp] "r" (voffp)
257 RSEQ_INJECT_INPUT
258 : "memory", RSEQ_ASM_TMP_REG
259 : abort, cmpfail
260#ifdef RSEQ_COMPARE_TWICE
261 , error1, error2
262#endif
263 );
264 return 0;
265abort:
266 RSEQ_INJECT_FAILED
267 return -1;
268cmpfail:
269 return 1;
270#ifdef RSEQ_COMPARE_TWICE
271error1:
272 rseq_bug("cpu_id comparison failed");
273error2:
274 rseq_bug("expected value comparison failed");
275#endif
276}
277
278static inline __attribute__((always_inline))
279int rseq_addv(intptr_t *v, intptr_t count, int cpu)
280{
281 RSEQ_INJECT_C(9)
282
283 __asm__ __volatile__ goto (
284 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
285 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
286 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
287 RSEQ_INJECT_ASM(3)
288#ifdef RSEQ_COMPARE_TWICE
289 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
290#endif
291 RSEQ_ASM_OP_R_LOAD(v)
292 RSEQ_ASM_OP_R_ADD(count)
293 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
294 RSEQ_INJECT_ASM(4)
295 RSEQ_ASM_DEFINE_ABORT(4, abort)
296 : /* gcc asm goto does not allow outputs */
297 : [cpu_id] "r" (cpu),
298 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
299 [rseq_cs] "m" (__rseq_abi.rseq_cs),
300 [v] "Qo" (*v),
301 [count] "r" (count)
302 RSEQ_INJECT_INPUT
303 : "memory", RSEQ_ASM_TMP_REG
304 : abort
305#ifdef RSEQ_COMPARE_TWICE
306 , error1
307#endif
308 );
309 return 0;
310abort:
311 RSEQ_INJECT_FAILED
312 return -1;
313#ifdef RSEQ_COMPARE_TWICE
314error1:
315 rseq_bug("cpu_id comparison failed");
316#endif
317}
318
319static inline __attribute__((always_inline))
320int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
321 intptr_t *v2, intptr_t newv2,
322 intptr_t newv, int cpu)
323{
324 RSEQ_INJECT_C(9)
325
326 __asm__ __volatile__ goto (
327 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
328 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
329 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
330 RSEQ_INJECT_ASM(3)
331 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
332 RSEQ_INJECT_ASM(4)
333#ifdef RSEQ_COMPARE_TWICE
334 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
335 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
336#endif
337 RSEQ_ASM_OP_STORE(newv2, v2)
338 RSEQ_INJECT_ASM(5)
339 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
340 RSEQ_INJECT_ASM(6)
341 RSEQ_ASM_DEFINE_ABORT(4, abort)
342 : /* gcc asm goto does not allow outputs */
343 : [cpu_id] "r" (cpu),
344 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
345 [rseq_cs] "m" (__rseq_abi.rseq_cs),
346 [expect] "r" (expect),
347 [v] "Qo" (*v),
348 [newv] "r" (newv),
349 [v2] "Qo" (*v2),
350 [newv2] "r" (newv2)
351 RSEQ_INJECT_INPUT
352 : "memory", RSEQ_ASM_TMP_REG
353 : abort, cmpfail
354#ifdef RSEQ_COMPARE_TWICE
355 , error1, error2
356#endif
357 );
358
359 return 0;
360abort:
361 RSEQ_INJECT_FAILED
362 return -1;
363cmpfail:
364 return 1;
365#ifdef RSEQ_COMPARE_TWICE
366error1:
367 rseq_bug("cpu_id comparison failed");
368error2:
369 rseq_bug("expected value comparison failed");
370#endif
371}
372
373static inline __attribute__((always_inline))
374int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
375 intptr_t *v2, intptr_t newv2,
376 intptr_t newv, int cpu)
377{
378 RSEQ_INJECT_C(9)
379
380 __asm__ __volatile__ goto (
381 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
382 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
383 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
384 RSEQ_INJECT_ASM(3)
385 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
386 RSEQ_INJECT_ASM(4)
387#ifdef RSEQ_COMPARE_TWICE
388 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
389 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
390#endif
391 RSEQ_ASM_OP_STORE(newv2, v2)
392 RSEQ_INJECT_ASM(5)
393 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
394 RSEQ_INJECT_ASM(6)
395 RSEQ_ASM_DEFINE_ABORT(4, abort)
396 : /* gcc asm goto does not allow outputs */
397 : [cpu_id] "r" (cpu),
398 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
399 [rseq_cs] "m" (__rseq_abi.rseq_cs),
400 [expect] "r" (expect),
401 [v] "Qo" (*v),
402 [newv] "r" (newv),
403 [v2] "Qo" (*v2),
404 [newv2] "r" (newv2)
405 RSEQ_INJECT_INPUT
406 : "memory", RSEQ_ASM_TMP_REG
407 : abort, cmpfail
408#ifdef RSEQ_COMPARE_TWICE
409 , error1, error2
410#endif
411 );
412
413 return 0;
414abort:
415 RSEQ_INJECT_FAILED
416 return -1;
417cmpfail:
418 return 1;
419#ifdef RSEQ_COMPARE_TWICE
420error1:
421 rseq_bug("cpu_id comparison failed");
422error2:
423 rseq_bug("expected value comparison failed");
424#endif
425}
426
427static inline __attribute__((always_inline))
428int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
429 intptr_t *v2, intptr_t expect2,
430 intptr_t newv, int cpu)
431{
432 RSEQ_INJECT_C(9)
433
434 __asm__ __volatile__ goto (
435 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
436 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
437 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
438 RSEQ_INJECT_ASM(3)
439 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
440 RSEQ_INJECT_ASM(4)
441 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
442 RSEQ_INJECT_ASM(5)
443#ifdef RSEQ_COMPARE_TWICE
444 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
445 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
446 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
447#endif
448 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
449 RSEQ_INJECT_ASM(6)
450 RSEQ_ASM_DEFINE_ABORT(4, abort)
451 : /* gcc asm goto does not allow outputs */
452 : [cpu_id] "r" (cpu),
453 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
454 [rseq_cs] "m" (__rseq_abi.rseq_cs),
455 [v] "Qo" (*v),
456 [expect] "r" (expect),
457 [v2] "Qo" (*v2),
458 [expect2] "r" (expect2),
459 [newv] "r" (newv)
460 RSEQ_INJECT_INPUT
461 : "memory", RSEQ_ASM_TMP_REG
462 : abort, cmpfail
463#ifdef RSEQ_COMPARE_TWICE
464 , error1, error2, error3
465#endif
466 );
467
468 return 0;
469abort:
470 RSEQ_INJECT_FAILED
471 return -1;
472cmpfail:
473 return 1;
474#ifdef RSEQ_COMPARE_TWICE
475error1:
476 rseq_bug("cpu_id comparison failed");
477error2:
478 rseq_bug("expected value comparison failed");
479error3:
480 rseq_bug("2nd expected value comparison failed");
481#endif
482}
483
484static inline __attribute__((always_inline))
485int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
486 void *dst, void *src, size_t len,
487 intptr_t newv, int cpu)
488{
489 RSEQ_INJECT_C(9)
490
491 __asm__ __volatile__ goto (
492 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
493 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
494 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
495 RSEQ_INJECT_ASM(3)
496 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
497 RSEQ_INJECT_ASM(4)
498#ifdef RSEQ_COMPARE_TWICE
499 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
500 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
501#endif
502 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
503 RSEQ_INJECT_ASM(5)
504 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
505 RSEQ_INJECT_ASM(6)
506 RSEQ_ASM_DEFINE_ABORT(4, abort)
507 : /* gcc asm goto does not allow outputs */
508 : [cpu_id] "r" (cpu),
509 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
510 [rseq_cs] "m" (__rseq_abi.rseq_cs),
511 [expect] "r" (expect),
512 [v] "Qo" (*v),
513 [newv] "r" (newv),
514 [dst] "r" (dst),
515 [src] "r" (src),
516 [len] "r" (len)
517 RSEQ_INJECT_INPUT
518 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
519 : abort, cmpfail
520#ifdef RSEQ_COMPARE_TWICE
521 , error1, error2
522#endif
523 );
524
525 return 0;
526abort:
527 RSEQ_INJECT_FAILED
528 return -1;
529cmpfail:
530 return 1;
531#ifdef RSEQ_COMPARE_TWICE
532error1:
533 rseq_bug("cpu_id comparison failed");
534error2:
535 rseq_bug("expected value comparison failed");
536#endif
537}
538
539static inline __attribute__((always_inline))
540int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
541 void *dst, void *src, size_t len,
542 intptr_t newv, int cpu)
543{
544 RSEQ_INJECT_C(9)
545
546 __asm__ __volatile__ goto (
547 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
548 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
549 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
550 RSEQ_INJECT_ASM(3)
551 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
552 RSEQ_INJECT_ASM(4)
553#ifdef RSEQ_COMPARE_TWICE
554 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
555 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
556#endif
557 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
558 RSEQ_INJECT_ASM(5)
559 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
560 RSEQ_INJECT_ASM(6)
561 RSEQ_ASM_DEFINE_ABORT(4, abort)
562 : /* gcc asm goto does not allow outputs */
563 : [cpu_id] "r" (cpu),
564 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
565 [rseq_cs] "m" (__rseq_abi.rseq_cs),
566 [expect] "r" (expect),
567 [v] "Qo" (*v),
568 [newv] "r" (newv),
569 [dst] "r" (dst),
570 [src] "r" (src),
571 [len] "r" (len)
572 RSEQ_INJECT_INPUT
573 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
574 : abort, cmpfail
575#ifdef RSEQ_COMPARE_TWICE
576 , error1, error2
577#endif
578 );
579
580 return 0;
581abort:
582 RSEQ_INJECT_FAILED
583 return -1;
584cmpfail:
585 return 1;
586#ifdef RSEQ_COMPARE_TWICE
587error1:
588 rseq_bug("cpu_id comparison failed");
589error2:
590 rseq_bug("expected value comparison failed");
591#endif
592}
593
594#endif /* !RSEQ_SKIP_FASTPATH */