tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-arm64.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com>
7 */
8
9/*
10 * aarch64 -mbig-endian generates mixed endianness code vs data:
11 * little-endian code and big-endian data. Ensure the RSEQ_SIG signature
12 * matches code endianness.
13 */
14#define RSEQ_SIG_CODE 0xd428bc00 /* BRK #0x45E0. */
15
16#ifdef __AARCH64EB__
17#define RSEQ_SIG_DATA 0x00bc28d4 /* BRK #0x45E0. */
18#else
19#define RSEQ_SIG_DATA RSEQ_SIG_CODE
20#endif
21
22#define RSEQ_SIG RSEQ_SIG_DATA
23
24#define rseq_smp_mb() __asm__ __volatile__ ("dmb ish" ::: "memory")
25#define rseq_smp_rmb() __asm__ __volatile__ ("dmb ishld" ::: "memory")
26#define rseq_smp_wmb() __asm__ __volatile__ ("dmb ishst" ::: "memory")
27
28#define rseq_smp_load_acquire(p) \
29__extension__ ({ \
30 __typeof(*p) ____p1; \
31 switch (sizeof(*p)) { \
32 case 1: \
33 asm volatile ("ldarb %w0, %1" \
34 : "=r" (*(__u8 *)p) \
35 : "Q" (*p) : "memory"); \
36 break; \
37 case 2: \
38 asm volatile ("ldarh %w0, %1" \
39 : "=r" (*(__u16 *)p) \
40 : "Q" (*p) : "memory"); \
41 break; \
42 case 4: \
43 asm volatile ("ldar %w0, %1" \
44 : "=r" (*(__u32 *)p) \
45 : "Q" (*p) : "memory"); \
46 break; \
47 case 8: \
48 asm volatile ("ldar %0, %1" \
49 : "=r" (*(__u64 *)p) \
50 : "Q" (*p) : "memory"); \
51 break; \
52 } \
53 ____p1; \
54})
55
56#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
57
58#define rseq_smp_store_release(p, v) \
59do { \
60 switch (sizeof(*p)) { \
61 case 1: \
62 asm volatile ("stlrb %w1, %0" \
63 : "=Q" (*p) \
64 : "r" ((__u8)v) \
65 : "memory"); \
66 break; \
67 case 2: \
68 asm volatile ("stlrh %w1, %0" \
69 : "=Q" (*p) \
70 : "r" ((__u16)v) \
71 : "memory"); \
72 break; \
73 case 4: \
74 asm volatile ("stlr %w1, %0" \
75 : "=Q" (*p) \
76 : "r" ((__u32)v) \
77 : "memory"); \
78 break; \
79 case 8: \
80 asm volatile ("stlr %1, %0" \
81 : "=Q" (*p) \
82 : "r" ((__u64)v) \
83 : "memory"); \
84 break; \
85 } \
86} while (0)
87
88#ifdef RSEQ_SKIP_FASTPATH
89#include "rseq-skip.h"
90#else /* !RSEQ_SKIP_FASTPATH */
91
92#define RSEQ_ASM_TMP_REG32 "w15"
93#define RSEQ_ASM_TMP_REG "x15"
94#define RSEQ_ASM_TMP_REG_2 "x14"
95
96#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
97 post_commit_offset, abort_ip) \
98 " .pushsection __rseq_cs, \"aw\"\n" \
99 " .balign 32\n" \
100 __rseq_str(label) ":\n" \
101 " .long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
102 " .quad " __rseq_str(start_ip) ", " \
103 __rseq_str(post_commit_offset) ", " \
104 __rseq_str(abort_ip) "\n" \
105 " .popsection\n\t" \
106 " .pushsection __rseq_cs_ptr_array, \"aw\"\n" \
107 " .quad " __rseq_str(label) "b\n" \
108 " .popsection\n"
109
110#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
111 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
112 (post_commit_ip - start_ip), abort_ip)
113
114/*
115 * Exit points of a rseq critical section consist of all instructions outside
116 * of the critical section where a critical section can either branch to or
117 * reach through the normal course of its execution. The abort IP and the
118 * post-commit IP are already part of the __rseq_cs section and should not be
119 * explicitly defined as additional exit points. Knowing all exit points is
120 * useful to assist debuggers stepping over the critical section.
121 */
122#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
123 " .pushsection __rseq_exit_point_array, \"aw\"\n" \
124 " .quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
125 " .popsection\n"
126
127#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
128 RSEQ_INJECT_ASM(1) \
129 " adrp " RSEQ_ASM_TMP_REG ", " __rseq_str(cs_label) "\n" \
130 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
131 ", :lo12:" __rseq_str(cs_label) "\n" \
132 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(rseq_cs) "]\n" \
133 __rseq_str(label) ":\n"
134
135#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
136 " b 222f\n" \
137 " .inst " __rseq_str(RSEQ_SIG_CODE) "\n" \
138 __rseq_str(label) ":\n" \
139 " b %l[" __rseq_str(abort_label) "]\n" \
140 "222:\n"
141
142#define RSEQ_ASM_OP_STORE(value, var) \
143 " str %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
144
145#define RSEQ_ASM_OP_STORE_RELEASE(value, var) \
146 " stlr %[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
147
148#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
149 RSEQ_ASM_OP_STORE(value, var) \
150 __rseq_str(post_commit_label) ":\n"
151
152#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
153 RSEQ_ASM_OP_STORE_RELEASE(value, var) \
154 __rseq_str(post_commit_label) ":\n"
155
156#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
157 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
158 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
159 ", %[" __rseq_str(expect) "]\n" \
160 " cbnz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
161
162#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
163 " ldr " RSEQ_ASM_TMP_REG32 ", %[" __rseq_str(var) "]\n" \
164 " sub " RSEQ_ASM_TMP_REG32 ", " RSEQ_ASM_TMP_REG32 \
165 ", %w[" __rseq_str(expect) "]\n" \
166 " cbnz " RSEQ_ASM_TMP_REG32 ", " __rseq_str(label) "\n"
167
168#define RSEQ_ASM_OP_CMPNE(var, expect, label) \
169 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
170 " sub " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
171 ", %[" __rseq_str(expect) "]\n" \
172 " cbz " RSEQ_ASM_TMP_REG ", " __rseq_str(label) "\n"
173
174#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
175 RSEQ_INJECT_ASM(2) \
176 RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
177
178#define RSEQ_ASM_OP_R_LOAD(var) \
179 " ldr " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
180
181#define RSEQ_ASM_OP_R_STORE(var) \
182 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n"
183
184#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
185 " ldr " RSEQ_ASM_TMP_REG ", [" RSEQ_ASM_TMP_REG \
186 ", %[" __rseq_str(offset) "]]\n"
187
188#define RSEQ_ASM_OP_R_ADD(count) \
189 " add " RSEQ_ASM_TMP_REG ", " RSEQ_ASM_TMP_REG \
190 ", %[" __rseq_str(count) "]\n"
191
192#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
193 " str " RSEQ_ASM_TMP_REG ", %[" __rseq_str(var) "]\n" \
194 __rseq_str(post_commit_label) ":\n"
195
196#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
197 " cbz %[" __rseq_str(len) "], 333f\n" \
198 " mov " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(len) "]\n" \
199 "222: sub " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", #1\n" \
200 " ldrb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(src) "]" \
201 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
202 " strb " RSEQ_ASM_TMP_REG32 ", [%[" __rseq_str(dst) "]" \
203 ", " RSEQ_ASM_TMP_REG_2 "]\n" \
204 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \
205 "333:\n"
206
207static inline __attribute__((always_inline))
208int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
209{
210 RSEQ_INJECT_C(9)
211
212 __asm__ __volatile__ goto (
213 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
214 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
215#ifdef RSEQ_COMPARE_TWICE
216 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
217 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
218#endif
219 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
220 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
221 RSEQ_INJECT_ASM(3)
222 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
223 RSEQ_INJECT_ASM(4)
224#ifdef RSEQ_COMPARE_TWICE
225 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
226 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
227#endif
228 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
229 RSEQ_INJECT_ASM(5)
230 RSEQ_ASM_DEFINE_ABORT(4, abort)
231 : /* gcc asm goto does not allow outputs */
232 : [cpu_id] "r" (cpu),
233 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
234 [rseq_cs] "m" (__rseq_abi.rseq_cs),
235 [v] "Qo" (*v),
236 [expect] "r" (expect),
237 [newv] "r" (newv)
238 RSEQ_INJECT_INPUT
239 : "memory", RSEQ_ASM_TMP_REG
240 : abort, cmpfail
241#ifdef RSEQ_COMPARE_TWICE
242 , error1, error2
243#endif
244 );
245
246 return 0;
247abort:
248 RSEQ_INJECT_FAILED
249 return -1;
250cmpfail:
251 return 1;
252#ifdef RSEQ_COMPARE_TWICE
253error1:
254 rseq_bug("cpu_id comparison failed");
255error2:
256 rseq_bug("expected value comparison failed");
257#endif
258}
259
260static inline __attribute__((always_inline))
261int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
262 off_t voffp, intptr_t *load, int cpu)
263{
264 RSEQ_INJECT_C(9)
265
266 __asm__ __volatile__ goto (
267 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
268 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
269#ifdef RSEQ_COMPARE_TWICE
270 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
271 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
272#endif
273 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
274 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
275 RSEQ_INJECT_ASM(3)
276 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
277 RSEQ_INJECT_ASM(4)
278#ifdef RSEQ_COMPARE_TWICE
279 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
280 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
281#endif
282 RSEQ_ASM_OP_R_LOAD(v)
283 RSEQ_ASM_OP_R_STORE(load)
284 RSEQ_ASM_OP_R_LOAD_OFF(voffp)
285 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
286 RSEQ_INJECT_ASM(5)
287 RSEQ_ASM_DEFINE_ABORT(4, abort)
288 : /* gcc asm goto does not allow outputs */
289 : [cpu_id] "r" (cpu),
290 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
291 [rseq_cs] "m" (__rseq_abi.rseq_cs),
292 [v] "Qo" (*v),
293 [expectnot] "r" (expectnot),
294 [load] "Qo" (*load),
295 [voffp] "r" (voffp)
296 RSEQ_INJECT_INPUT
297 : "memory", RSEQ_ASM_TMP_REG
298 : abort, cmpfail
299#ifdef RSEQ_COMPARE_TWICE
300 , error1, error2
301#endif
302 );
303 return 0;
304abort:
305 RSEQ_INJECT_FAILED
306 return -1;
307cmpfail:
308 return 1;
309#ifdef RSEQ_COMPARE_TWICE
310error1:
311 rseq_bug("cpu_id comparison failed");
312error2:
313 rseq_bug("expected value comparison failed");
314#endif
315}
316
317static inline __attribute__((always_inline))
318int rseq_addv(intptr_t *v, intptr_t count, int cpu)
319{
320 RSEQ_INJECT_C(9)
321
322 __asm__ __volatile__ goto (
323 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
324#ifdef RSEQ_COMPARE_TWICE
325 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
326#endif
327 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
328 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
329 RSEQ_INJECT_ASM(3)
330#ifdef RSEQ_COMPARE_TWICE
331 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
332#endif
333 RSEQ_ASM_OP_R_LOAD(v)
334 RSEQ_ASM_OP_R_ADD(count)
335 RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
336 RSEQ_INJECT_ASM(4)
337 RSEQ_ASM_DEFINE_ABORT(4, abort)
338 : /* gcc asm goto does not allow outputs */
339 : [cpu_id] "r" (cpu),
340 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
341 [rseq_cs] "m" (__rseq_abi.rseq_cs),
342 [v] "Qo" (*v),
343 [count] "r" (count)
344 RSEQ_INJECT_INPUT
345 : "memory", RSEQ_ASM_TMP_REG
346 : abort
347#ifdef RSEQ_COMPARE_TWICE
348 , error1
349#endif
350 );
351 return 0;
352abort:
353 RSEQ_INJECT_FAILED
354 return -1;
355#ifdef RSEQ_COMPARE_TWICE
356error1:
357 rseq_bug("cpu_id comparison failed");
358#endif
359}
360
361static inline __attribute__((always_inline))
362int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
363 intptr_t *v2, intptr_t newv2,
364 intptr_t newv, int cpu)
365{
366 RSEQ_INJECT_C(9)
367
368 __asm__ __volatile__ goto (
369 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
370 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
371#ifdef RSEQ_COMPARE_TWICE
372 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
373 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
374#endif
375 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
376 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
377 RSEQ_INJECT_ASM(3)
378 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
379 RSEQ_INJECT_ASM(4)
380#ifdef RSEQ_COMPARE_TWICE
381 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
382 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
383#endif
384 RSEQ_ASM_OP_STORE(newv2, v2)
385 RSEQ_INJECT_ASM(5)
386 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
387 RSEQ_INJECT_ASM(6)
388 RSEQ_ASM_DEFINE_ABORT(4, abort)
389 : /* gcc asm goto does not allow outputs */
390 : [cpu_id] "r" (cpu),
391 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
392 [rseq_cs] "m" (__rseq_abi.rseq_cs),
393 [expect] "r" (expect),
394 [v] "Qo" (*v),
395 [newv] "r" (newv),
396 [v2] "Qo" (*v2),
397 [newv2] "r" (newv2)
398 RSEQ_INJECT_INPUT
399 : "memory", RSEQ_ASM_TMP_REG
400 : abort, cmpfail
401#ifdef RSEQ_COMPARE_TWICE
402 , error1, error2
403#endif
404 );
405
406 return 0;
407abort:
408 RSEQ_INJECT_FAILED
409 return -1;
410cmpfail:
411 return 1;
412#ifdef RSEQ_COMPARE_TWICE
413error1:
414 rseq_bug("cpu_id comparison failed");
415error2:
416 rseq_bug("expected value comparison failed");
417#endif
418}
419
420static inline __attribute__((always_inline))
421int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
422 intptr_t *v2, intptr_t newv2,
423 intptr_t newv, int cpu)
424{
425 RSEQ_INJECT_C(9)
426
427 __asm__ __volatile__ goto (
428 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
429 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
430#ifdef RSEQ_COMPARE_TWICE
431 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
432 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
433#endif
434 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
435 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
436 RSEQ_INJECT_ASM(3)
437 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
438 RSEQ_INJECT_ASM(4)
439#ifdef RSEQ_COMPARE_TWICE
440 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
441 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
442#endif
443 RSEQ_ASM_OP_STORE(newv2, v2)
444 RSEQ_INJECT_ASM(5)
445 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
446 RSEQ_INJECT_ASM(6)
447 RSEQ_ASM_DEFINE_ABORT(4, abort)
448 : /* gcc asm goto does not allow outputs */
449 : [cpu_id] "r" (cpu),
450 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
451 [rseq_cs] "m" (__rseq_abi.rseq_cs),
452 [expect] "r" (expect),
453 [v] "Qo" (*v),
454 [newv] "r" (newv),
455 [v2] "Qo" (*v2),
456 [newv2] "r" (newv2)
457 RSEQ_INJECT_INPUT
458 : "memory", RSEQ_ASM_TMP_REG
459 : abort, cmpfail
460#ifdef RSEQ_COMPARE_TWICE
461 , error1, error2
462#endif
463 );
464
465 return 0;
466abort:
467 RSEQ_INJECT_FAILED
468 return -1;
469cmpfail:
470 return 1;
471#ifdef RSEQ_COMPARE_TWICE
472error1:
473 rseq_bug("cpu_id comparison failed");
474error2:
475 rseq_bug("expected value comparison failed");
476#endif
477}
478
479static inline __attribute__((always_inline))
480int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
481 intptr_t *v2, intptr_t expect2,
482 intptr_t newv, int cpu)
483{
484 RSEQ_INJECT_C(9)
485
486 __asm__ __volatile__ goto (
487 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
488 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
489#ifdef RSEQ_COMPARE_TWICE
490 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
491 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
492 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3])
493#endif
494 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
495 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
496 RSEQ_INJECT_ASM(3)
497 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
498 RSEQ_INJECT_ASM(4)
499 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
500 RSEQ_INJECT_ASM(5)
501#ifdef RSEQ_COMPARE_TWICE
502 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
503 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
504 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
505#endif
506 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
507 RSEQ_INJECT_ASM(6)
508 RSEQ_ASM_DEFINE_ABORT(4, abort)
509 : /* gcc asm goto does not allow outputs */
510 : [cpu_id] "r" (cpu),
511 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
512 [rseq_cs] "m" (__rseq_abi.rseq_cs),
513 [v] "Qo" (*v),
514 [expect] "r" (expect),
515 [v2] "Qo" (*v2),
516 [expect2] "r" (expect2),
517 [newv] "r" (newv)
518 RSEQ_INJECT_INPUT
519 : "memory", RSEQ_ASM_TMP_REG
520 : abort, cmpfail
521#ifdef RSEQ_COMPARE_TWICE
522 , error1, error2, error3
523#endif
524 );
525
526 return 0;
527abort:
528 RSEQ_INJECT_FAILED
529 return -1;
530cmpfail:
531 return 1;
532#ifdef RSEQ_COMPARE_TWICE
533error1:
534 rseq_bug("cpu_id comparison failed");
535error2:
536 rseq_bug("expected value comparison failed");
537error3:
538 rseq_bug("2nd expected value comparison failed");
539#endif
540}
541
542static inline __attribute__((always_inline))
543int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
544 void *dst, void *src, size_t len,
545 intptr_t newv, int cpu)
546{
547 RSEQ_INJECT_C(9)
548
549 __asm__ __volatile__ goto (
550 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
551 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
552#ifdef RSEQ_COMPARE_TWICE
553 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
554 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
555#endif
556 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
557 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
558 RSEQ_INJECT_ASM(3)
559 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
560 RSEQ_INJECT_ASM(4)
561#ifdef RSEQ_COMPARE_TWICE
562 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
563 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
564#endif
565 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
566 RSEQ_INJECT_ASM(5)
567 RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
568 RSEQ_INJECT_ASM(6)
569 RSEQ_ASM_DEFINE_ABORT(4, abort)
570 : /* gcc asm goto does not allow outputs */
571 : [cpu_id] "r" (cpu),
572 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
573 [rseq_cs] "m" (__rseq_abi.rseq_cs),
574 [expect] "r" (expect),
575 [v] "Qo" (*v),
576 [newv] "r" (newv),
577 [dst] "r" (dst),
578 [src] "r" (src),
579 [len] "r" (len)
580 RSEQ_INJECT_INPUT
581 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
582 : abort, cmpfail
583#ifdef RSEQ_COMPARE_TWICE
584 , error1, error2
585#endif
586 );
587
588 return 0;
589abort:
590 RSEQ_INJECT_FAILED
591 return -1;
592cmpfail:
593 return 1;
594#ifdef RSEQ_COMPARE_TWICE
595error1:
596 rseq_bug("cpu_id comparison failed");
597error2:
598 rseq_bug("expected value comparison failed");
599#endif
600}
601
602static inline __attribute__((always_inline))
603int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
604 void *dst, void *src, size_t len,
605 intptr_t newv, int cpu)
606{
607 RSEQ_INJECT_C(9)
608
609 __asm__ __volatile__ goto (
610 RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
611 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail])
612#ifdef RSEQ_COMPARE_TWICE
613 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1])
614 RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2])
615#endif
616 RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
617 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
618 RSEQ_INJECT_ASM(3)
619 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
620 RSEQ_INJECT_ASM(4)
621#ifdef RSEQ_COMPARE_TWICE
622 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
623 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
624#endif
625 RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
626 RSEQ_INJECT_ASM(5)
627 RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
628 RSEQ_INJECT_ASM(6)
629 RSEQ_ASM_DEFINE_ABORT(4, abort)
630 : /* gcc asm goto does not allow outputs */
631 : [cpu_id] "r" (cpu),
632 [current_cpu_id] "Qo" (__rseq_abi.cpu_id),
633 [rseq_cs] "m" (__rseq_abi.rseq_cs),
634 [expect] "r" (expect),
635 [v] "Qo" (*v),
636 [newv] "r" (newv),
637 [dst] "r" (dst),
638 [src] "r" (src),
639 [len] "r" (len)
640 RSEQ_INJECT_INPUT
641 : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2
642 : abort, cmpfail
643#ifdef RSEQ_COMPARE_TWICE
644 , error1, error2
645#endif
646 );
647
648 return 0;
649abort:
650 RSEQ_INJECT_FAILED
651 return -1;
652cmpfail:
653 return 1;
654#ifdef RSEQ_COMPARE_TWICE
655error1:
656 rseq_bug("cpu_id comparison failed");
657error2:
658 rseq_bug("expected value comparison failed");
659#endif
660}
661
662#endif /* !RSEQ_SKIP_FASTPATH */