tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2
3/*
4 * RSEQ_SIG uses the trap4 instruction. As Linux does not make use of the
5 * access-register mode nor the linkage stack this instruction will always
6 * cause a special-operation exception (the trap-enabled bit in the DUCT
7 * is and will stay 0). The instruction pattern is
8 * b2 ff 0f ff trap4 4095(%r0)
9 */
10#define RSEQ_SIG 0xB2FF0FFF
11
12#define rseq_smp_mb() __asm__ __volatile__ ("bcr 15,0" ::: "memory")
13#define rseq_smp_rmb() rseq_smp_mb()
14#define rseq_smp_wmb() rseq_smp_mb()
15
16#define rseq_smp_load_acquire(p) \
17__extension__ ({ \
18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
19 rseq_barrier(); \
20 ____p1; \
21})
22
23#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
24
25#define rseq_smp_store_release(p, v) \
26do { \
27 rseq_barrier(); \
28 RSEQ_WRITE_ONCE(*p, v); \
29} while (0)
30
31#ifdef RSEQ_SKIP_FASTPATH
32#include "rseq-skip.h"
33#else /* !RSEQ_SKIP_FASTPATH */
34
35#ifdef __s390x__
36
37#define LONG_L "lg"
38#define LONG_S "stg"
39#define LONG_LT_R "ltgr"
40#define LONG_CMP "cg"
41#define LONG_CMP_R "cgr"
42#define LONG_ADDI "aghi"
43#define LONG_ADD_R "agr"
44
45#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
46 start_ip, post_commit_offset, abort_ip) \
47 ".pushsection __rseq_cs, \"aw\"\n\t" \
48 ".balign 32\n\t" \
49 __rseq_str(label) ":\n\t" \
50 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
51 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
52 ".popsection\n\t" \
53 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
54 ".quad " __rseq_str(label) "b\n\t" \
55 ".popsection\n\t"
56
57/*
58 * Exit points of a rseq critical section consist of all instructions outside
59 * of the critical section where a critical section can either branch to or
60 * reach through the normal course of its execution. The abort IP and the
61 * post-commit IP are already part of the __rseq_cs section and should not be
62 * explicitly defined as additional exit points. Knowing all exit points is
63 * useful to assist debuggers stepping over the critical section.
64 */
65#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
66 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
67 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
68 ".popsection\n\t"
69
70#elif __s390__
71
72#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
73 start_ip, post_commit_offset, abort_ip) \
74 ".pushsection __rseq_cs, \"aw\"\n\t" \
75 ".balign 32\n\t" \
76 __rseq_str(label) ":\n\t" \
77 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
78 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
79 ".popsection\n\t" \
80 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
81 ".long 0x0, " __rseq_str(label) "b\n\t" \
82 ".popsection\n\t"
83
84/*
85 * Exit points of a rseq critical section consist of all instructions outside
86 * of the critical section where a critical section can either branch to or
87 * reach through the normal course of its execution. The abort IP and the
88 * post-commit IP are already part of the __rseq_cs section and should not be
89 * explicitly defined as additional exit points. Knowing all exit points is
90 * useful to assist debuggers stepping over the critical section.
91 */
92#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
93 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
94 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
95 ".popsection\n\t"
96
97#define LONG_L "l"
98#define LONG_S "st"
99#define LONG_LT_R "ltr"
100#define LONG_CMP "c"
101#define LONG_CMP_R "cr"
102#define LONG_ADDI "ahi"
103#define LONG_ADD_R "ar"
104
105#endif
106
107#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
108 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
109 (post_commit_ip - start_ip), abort_ip)
110
111#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
112 RSEQ_INJECT_ASM(1) \
113 "larl %%r0, " __rseq_str(cs_label) "\n\t" \
114 LONG_S " %%r0, %[" __rseq_str(rseq_cs) "]\n\t" \
115 __rseq_str(label) ":\n\t"
116
117#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
118 RSEQ_INJECT_ASM(2) \
119 "c %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
120 "jnz " __rseq_str(label) "\n\t"
121
122#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
123 ".pushsection __rseq_failure, \"ax\"\n\t" \
124 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
125 __rseq_str(label) ":\n\t" \
126 teardown \
127 "jg %l[" __rseq_str(abort_label) "]\n\t" \
128 ".popsection\n\t"
129
130#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
131 ".pushsection __rseq_failure, \"ax\"\n\t" \
132 __rseq_str(label) ":\n\t" \
133 teardown \
134 "jg %l[" __rseq_str(cmpfail_label) "]\n\t" \
135 ".popsection\n\t"
136
137static inline __attribute__((always_inline))
138int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
139{
140 RSEQ_INJECT_C(9)
141
142 __asm__ __volatile__ goto (
143 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
144 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
145#ifdef RSEQ_COMPARE_TWICE
146 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
147 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
148#endif
149 /* Start rseq by storing table entry pointer into rseq_cs. */
150 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
151 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
152 RSEQ_INJECT_ASM(3)
153 LONG_CMP " %[expect], %[v]\n\t"
154 "jnz %l[cmpfail]\n\t"
155 RSEQ_INJECT_ASM(4)
156#ifdef RSEQ_COMPARE_TWICE
157 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
158 LONG_CMP " %[expect], %[v]\n\t"
159 "jnz %l[error2]\n\t"
160#endif
161 /* final store */
162 LONG_S " %[newv], %[v]\n\t"
163 "2:\n\t"
164 RSEQ_INJECT_ASM(5)
165 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
166 : /* gcc asm goto does not allow outputs */
167 : [cpu_id] "r" (cpu),
168 [current_cpu_id] "m" (__rseq_abi.cpu_id),
169 [rseq_cs] "m" (__rseq_abi.rseq_cs),
170 [v] "m" (*v),
171 [expect] "r" (expect),
172 [newv] "r" (newv)
173 RSEQ_INJECT_INPUT
174 : "memory", "cc", "r0"
175 RSEQ_INJECT_CLOBBER
176 : abort, cmpfail
177#ifdef RSEQ_COMPARE_TWICE
178 , error1, error2
179#endif
180 );
181 return 0;
182abort:
183 RSEQ_INJECT_FAILED
184 return -1;
185cmpfail:
186 return 1;
187#ifdef RSEQ_COMPARE_TWICE
188error1:
189 rseq_bug("cpu_id comparison failed");
190error2:
191 rseq_bug("expected value comparison failed");
192#endif
193}
194
195/*
196 * Compare @v against @expectnot. When it does _not_ match, load @v
197 * into @load, and store the content of *@v + voffp into @v.
198 */
199static inline __attribute__((always_inline))
200int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
201 off_t voffp, intptr_t *load, int cpu)
202{
203 RSEQ_INJECT_C(9)
204
205 __asm__ __volatile__ goto (
206 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
207 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
208#ifdef RSEQ_COMPARE_TWICE
209 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
210 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
211#endif
212 /* Start rseq by storing table entry pointer into rseq_cs. */
213 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
214 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
215 RSEQ_INJECT_ASM(3)
216 LONG_L " %%r1, %[v]\n\t"
217 LONG_CMP_R " %%r1, %[expectnot]\n\t"
218 "je %l[cmpfail]\n\t"
219 RSEQ_INJECT_ASM(4)
220#ifdef RSEQ_COMPARE_TWICE
221 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
222 LONG_L " %%r1, %[v]\n\t"
223 LONG_CMP_R " %%r1, %[expectnot]\n\t"
224 "je %l[error2]\n\t"
225#endif
226 LONG_S " %%r1, %[load]\n\t"
227 LONG_ADD_R " %%r1, %[voffp]\n\t"
228 LONG_L " %%r1, 0(%%r1)\n\t"
229 /* final store */
230 LONG_S " %%r1, %[v]\n\t"
231 "2:\n\t"
232 RSEQ_INJECT_ASM(5)
233 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
234 : /* gcc asm goto does not allow outputs */
235 : [cpu_id] "r" (cpu),
236 [current_cpu_id] "m" (__rseq_abi.cpu_id),
237 [rseq_cs] "m" (__rseq_abi.rseq_cs),
238 /* final store input */
239 [v] "m" (*v),
240 [expectnot] "r" (expectnot),
241 [voffp] "r" (voffp),
242 [load] "m" (*load)
243 RSEQ_INJECT_INPUT
244 : "memory", "cc", "r0", "r1"
245 RSEQ_INJECT_CLOBBER
246 : abort, cmpfail
247#ifdef RSEQ_COMPARE_TWICE
248 , error1, error2
249#endif
250 );
251 return 0;
252abort:
253 RSEQ_INJECT_FAILED
254 return -1;
255cmpfail:
256 return 1;
257#ifdef RSEQ_COMPARE_TWICE
258error1:
259 rseq_bug("cpu_id comparison failed");
260error2:
261 rseq_bug("expected value comparison failed");
262#endif
263}
264
265static inline __attribute__((always_inline))
266int rseq_addv(intptr_t *v, intptr_t count, int cpu)
267{
268 RSEQ_INJECT_C(9)
269
270 __asm__ __volatile__ goto (
271 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
272#ifdef RSEQ_COMPARE_TWICE
273 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
274#endif
275 /* Start rseq by storing table entry pointer into rseq_cs. */
276 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
277 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
278 RSEQ_INJECT_ASM(3)
279#ifdef RSEQ_COMPARE_TWICE
280 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
281#endif
282 LONG_L " %%r0, %[v]\n\t"
283 LONG_ADD_R " %%r0, %[count]\n\t"
284 /* final store */
285 LONG_S " %%r0, %[v]\n\t"
286 "2:\n\t"
287 RSEQ_INJECT_ASM(4)
288 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
289 : /* gcc asm goto does not allow outputs */
290 : [cpu_id] "r" (cpu),
291 [current_cpu_id] "m" (__rseq_abi.cpu_id),
292 [rseq_cs] "m" (__rseq_abi.rseq_cs),
293 /* final store input */
294 [v] "m" (*v),
295 [count] "r" (count)
296 RSEQ_INJECT_INPUT
297 : "memory", "cc", "r0"
298 RSEQ_INJECT_CLOBBER
299 : abort
300#ifdef RSEQ_COMPARE_TWICE
301 , error1
302#endif
303 );
304 return 0;
305abort:
306 RSEQ_INJECT_FAILED
307 return -1;
308#ifdef RSEQ_COMPARE_TWICE
309error1:
310 rseq_bug("cpu_id comparison failed");
311#endif
312}
313
314static inline __attribute__((always_inline))
315int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
316 intptr_t *v2, intptr_t newv2,
317 intptr_t newv, int cpu)
318{
319 RSEQ_INJECT_C(9)
320
321 __asm__ __volatile__ goto (
322 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
323 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
324#ifdef RSEQ_COMPARE_TWICE
325 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
326 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
327#endif
328 /* Start rseq by storing table entry pointer into rseq_cs. */
329 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
330 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
331 RSEQ_INJECT_ASM(3)
332 LONG_CMP " %[expect], %[v]\n\t"
333 "jnz %l[cmpfail]\n\t"
334 RSEQ_INJECT_ASM(4)
335#ifdef RSEQ_COMPARE_TWICE
336 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
337 LONG_CMP " %[expect], %[v]\n\t"
338 "jnz %l[error2]\n\t"
339#endif
340 /* try store */
341 LONG_S " %[newv2], %[v2]\n\t"
342 RSEQ_INJECT_ASM(5)
343 /* final store */
344 LONG_S " %[newv], %[v]\n\t"
345 "2:\n\t"
346 RSEQ_INJECT_ASM(6)
347 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
348 : /* gcc asm goto does not allow outputs */
349 : [cpu_id] "r" (cpu),
350 [current_cpu_id] "m" (__rseq_abi.cpu_id),
351 [rseq_cs] "m" (__rseq_abi.rseq_cs),
352 /* try store input */
353 [v2] "m" (*v2),
354 [newv2] "r" (newv2),
355 /* final store input */
356 [v] "m" (*v),
357 [expect] "r" (expect),
358 [newv] "r" (newv)
359 RSEQ_INJECT_INPUT
360 : "memory", "cc", "r0"
361 RSEQ_INJECT_CLOBBER
362 : abort, cmpfail
363#ifdef RSEQ_COMPARE_TWICE
364 , error1, error2
365#endif
366 );
367 return 0;
368abort:
369 RSEQ_INJECT_FAILED
370 return -1;
371cmpfail:
372 return 1;
373#ifdef RSEQ_COMPARE_TWICE
374error1:
375 rseq_bug("cpu_id comparison failed");
376error2:
377 rseq_bug("expected value comparison failed");
378#endif
379}
380
381/* s390 is TSO. */
382static inline __attribute__((always_inline))
383int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
384 intptr_t *v2, intptr_t newv2,
385 intptr_t newv, int cpu)
386{
387 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
388}
389
390static inline __attribute__((always_inline))
391int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
392 intptr_t *v2, intptr_t expect2,
393 intptr_t newv, int cpu)
394{
395 RSEQ_INJECT_C(9)
396
397 __asm__ __volatile__ goto (
398 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
399 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
400#ifdef RSEQ_COMPARE_TWICE
401 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
402 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
403 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
404#endif
405 /* Start rseq by storing table entry pointer into rseq_cs. */
406 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
407 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
408 RSEQ_INJECT_ASM(3)
409 LONG_CMP " %[expect], %[v]\n\t"
410 "jnz %l[cmpfail]\n\t"
411 RSEQ_INJECT_ASM(4)
412 LONG_CMP " %[expect2], %[v2]\n\t"
413 "jnz %l[cmpfail]\n\t"
414 RSEQ_INJECT_ASM(5)
415#ifdef RSEQ_COMPARE_TWICE
416 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
417 LONG_CMP " %[expect], %[v]\n\t"
418 "jnz %l[error2]\n\t"
419 LONG_CMP " %[expect2], %[v2]\n\t"
420 "jnz %l[error3]\n\t"
421#endif
422 /* final store */
423 LONG_S " %[newv], %[v]\n\t"
424 "2:\n\t"
425 RSEQ_INJECT_ASM(6)
426 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
427 : /* gcc asm goto does not allow outputs */
428 : [cpu_id] "r" (cpu),
429 [current_cpu_id] "m" (__rseq_abi.cpu_id),
430 [rseq_cs] "m" (__rseq_abi.rseq_cs),
431 /* cmp2 input */
432 [v2] "m" (*v2),
433 [expect2] "r" (expect2),
434 /* final store input */
435 [v] "m" (*v),
436 [expect] "r" (expect),
437 [newv] "r" (newv)
438 RSEQ_INJECT_INPUT
439 : "memory", "cc", "r0"
440 RSEQ_INJECT_CLOBBER
441 : abort, cmpfail
442#ifdef RSEQ_COMPARE_TWICE
443 , error1, error2, error3
444#endif
445 );
446 return 0;
447abort:
448 RSEQ_INJECT_FAILED
449 return -1;
450cmpfail:
451 return 1;
452#ifdef RSEQ_COMPARE_TWICE
453error1:
454 rseq_bug("cpu_id comparison failed");
455error2:
456 rseq_bug("1st expected value comparison failed");
457error3:
458 rseq_bug("2nd expected value comparison failed");
459#endif
460}
461
462static inline __attribute__((always_inline))
463int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
464 void *dst, void *src, size_t len,
465 intptr_t newv, int cpu)
466{
467 uint64_t rseq_scratch[3];
468
469 RSEQ_INJECT_C(9)
470
471 __asm__ __volatile__ goto (
472 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
473 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
474#ifdef RSEQ_COMPARE_TWICE
475 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
476 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
477#endif
478 LONG_S " %[src], %[rseq_scratch0]\n\t"
479 LONG_S " %[dst], %[rseq_scratch1]\n\t"
480 LONG_S " %[len], %[rseq_scratch2]\n\t"
481 /* Start rseq by storing table entry pointer into rseq_cs. */
482 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
483 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
484 RSEQ_INJECT_ASM(3)
485 LONG_CMP " %[expect], %[v]\n\t"
486 "jnz 5f\n\t"
487 RSEQ_INJECT_ASM(4)
488#ifdef RSEQ_COMPARE_TWICE
489 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
490 LONG_CMP " %[expect], %[v]\n\t"
491 "jnz 7f\n\t"
492#endif
493 /* try memcpy */
494 LONG_LT_R " %[len], %[len]\n\t"
495 "jz 333f\n\t"
496 "222:\n\t"
497 "ic %%r0,0(%[src])\n\t"
498 "stc %%r0,0(%[dst])\n\t"
499 LONG_ADDI " %[src], 1\n\t"
500 LONG_ADDI " %[dst], 1\n\t"
501 LONG_ADDI " %[len], -1\n\t"
502 "jnz 222b\n\t"
503 "333:\n\t"
504 RSEQ_INJECT_ASM(5)
505 /* final store */
506 LONG_S " %[newv], %[v]\n\t"
507 "2:\n\t"
508 RSEQ_INJECT_ASM(6)
509 /* teardown */
510 LONG_L " %[len], %[rseq_scratch2]\n\t"
511 LONG_L " %[dst], %[rseq_scratch1]\n\t"
512 LONG_L " %[src], %[rseq_scratch0]\n\t"
513 RSEQ_ASM_DEFINE_ABORT(4,
514 LONG_L " %[len], %[rseq_scratch2]\n\t"
515 LONG_L " %[dst], %[rseq_scratch1]\n\t"
516 LONG_L " %[src], %[rseq_scratch0]\n\t",
517 abort)
518 RSEQ_ASM_DEFINE_CMPFAIL(5,
519 LONG_L " %[len], %[rseq_scratch2]\n\t"
520 LONG_L " %[dst], %[rseq_scratch1]\n\t"
521 LONG_L " %[src], %[rseq_scratch0]\n\t",
522 cmpfail)
523#ifdef RSEQ_COMPARE_TWICE
524 RSEQ_ASM_DEFINE_CMPFAIL(6,
525 LONG_L " %[len], %[rseq_scratch2]\n\t"
526 LONG_L " %[dst], %[rseq_scratch1]\n\t"
527 LONG_L " %[src], %[rseq_scratch0]\n\t",
528 error1)
529 RSEQ_ASM_DEFINE_CMPFAIL(7,
530 LONG_L " %[len], %[rseq_scratch2]\n\t"
531 LONG_L " %[dst], %[rseq_scratch1]\n\t"
532 LONG_L " %[src], %[rseq_scratch0]\n\t",
533 error2)
534#endif
535 : /* gcc asm goto does not allow outputs */
536 : [cpu_id] "r" (cpu),
537 [current_cpu_id] "m" (__rseq_abi.cpu_id),
538 [rseq_cs] "m" (__rseq_abi.rseq_cs),
539 /* final store input */
540 [v] "m" (*v),
541 [expect] "r" (expect),
542 [newv] "r" (newv),
543 /* try memcpy input */
544 [dst] "r" (dst),
545 [src] "r" (src),
546 [len] "r" (len),
547 [rseq_scratch0] "m" (rseq_scratch[0]),
548 [rseq_scratch1] "m" (rseq_scratch[1]),
549 [rseq_scratch2] "m" (rseq_scratch[2])
550 RSEQ_INJECT_INPUT
551 : "memory", "cc", "r0"
552 RSEQ_INJECT_CLOBBER
553 : abort, cmpfail
554#ifdef RSEQ_COMPARE_TWICE
555 , error1, error2
556#endif
557 );
558 return 0;
559abort:
560 RSEQ_INJECT_FAILED
561 return -1;
562cmpfail:
563 return 1;
564#ifdef RSEQ_COMPARE_TWICE
565error1:
566 rseq_bug("cpu_id comparison failed");
567error2:
568 rseq_bug("expected value comparison failed");
569#endif
570}
571
572/* s390 is TSO. */
573static inline __attribute__((always_inline))
574int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
575 void *dst, void *src, size_t len,
576 intptr_t newv, int cpu)
577{
578 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
579 newv, cpu);
580}
581#endif /* !RSEQ_SKIP_FASTPATH */