tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-ppc.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com>
7 */
8
9/*
10 * RSEQ_SIG is used with the following trap instruction:
11 *
12 * powerpc-be: 0f e5 00 0b twui r5,11
13 * powerpc64-le: 0b 00 e5 0f twui r5,11
14 * powerpc64-be: 0f e5 00 0b twui r5,11
15 */
16
17#define RSEQ_SIG 0x0fe5000b
18
19#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory", "cc")
20#define rseq_smp_lwsync() __asm__ __volatile__ ("lwsync" ::: "memory", "cc")
21#define rseq_smp_rmb() rseq_smp_lwsync()
22#define rseq_smp_wmb() rseq_smp_lwsync()
23
24#define rseq_smp_load_acquire(p) \
25__extension__ ({ \
26 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
27 rseq_smp_lwsync(); \
28 ____p1; \
29})
30
31#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_lwsync()
32
33#define rseq_smp_store_release(p, v) \
34do { \
35 rseq_smp_lwsync(); \
36 RSEQ_WRITE_ONCE(*p, v); \
37} while (0)
38
39#ifdef RSEQ_SKIP_FASTPATH
40#include "rseq-skip.h"
41#else /* !RSEQ_SKIP_FASTPATH */
42
43/*
44 * The __rseq_cs_ptr_array and __rseq_cs sections can be used by debuggers to
45 * better handle single-stepping through the restartable critical sections.
46 */
47
48#ifdef __PPC64__
49
50#define STORE_WORD "std "
51#define LOAD_WORD "ld "
52#define LOADX_WORD "ldx "
53#define CMP_WORD "cmpd "
54
55#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
56 start_ip, post_commit_offset, abort_ip) \
57 ".pushsection __rseq_cs, \"aw\"\n\t" \
58 ".balign 32\n\t" \
59 __rseq_str(label) ":\n\t" \
60 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
61 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
62 ".popsection\n\t" \
63 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
64 ".quad " __rseq_str(label) "b\n\t" \
65 ".popsection\n\t"
66
67#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
68 RSEQ_INJECT_ASM(1) \
69 "lis %%r17, (" __rseq_str(cs_label) ")@highest\n\t" \
70 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@higher\n\t" \
71 "rldicr %%r17, %%r17, 32, 31\n\t" \
72 "oris %%r17, %%r17, (" __rseq_str(cs_label) ")@high\n\t" \
73 "ori %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
74 "std %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
75 __rseq_str(label) ":\n\t"
76
77/*
78 * Exit points of a rseq critical section consist of all instructions outside
79 * of the critical section where a critical section can either branch to or
80 * reach through the normal course of its execution. The abort IP and the
81 * post-commit IP are already part of the __rseq_cs section and should not be
82 * explicitly defined as additional exit points. Knowing all exit points is
83 * useful to assist debuggers stepping over the critical section.
84 */
85#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
86 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
87 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
88 ".popsection\n\t"
89
90#else /* #ifdef __PPC64__ */
91
92#define STORE_WORD "stw "
93#define LOAD_WORD "lwz "
94#define LOADX_WORD "lwzx "
95#define CMP_WORD "cmpw "
96
97#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
98 start_ip, post_commit_offset, abort_ip) \
99 ".pushsection __rseq_cs, \"aw\"\n\t" \
100 ".balign 32\n\t" \
101 __rseq_str(label) ":\n\t" \
102 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
103 /* 32-bit only supported on BE */ \
104 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) "\n\t" \
105 ".popsection\n\t" \
106 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
107 ".long 0x0, " __rseq_str(label) "b\n\t" \
108 ".popsection\n\t"
109
110/*
111 * Exit points of a rseq critical section consist of all instructions outside
112 * of the critical section where a critical section can either branch to or
113 * reach through the normal course of its execution. The abort IP and the
114 * post-commit IP are already part of the __rseq_cs section and should not be
115 * explicitly defined as additional exit points. Knowing all exit points is
116 * useful to assist debuggers stepping over the critical section.
117 */
118#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
119 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
120 /* 32-bit only supported on BE */ \
121 ".long 0x0, " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) "\n\t" \
122 ".popsection\n\t"
123
124#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
125 RSEQ_INJECT_ASM(1) \
126 "lis %%r17, (" __rseq_str(cs_label) ")@ha\n\t" \
127 "addi %%r17, %%r17, (" __rseq_str(cs_label) ")@l\n\t" \
128 "stw %%r17, %[" __rseq_str(rseq_cs) "]\n\t" \
129 __rseq_str(label) ":\n\t"
130
131#endif /* #ifdef __PPC64__ */
132
133#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
134 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
135 (post_commit_ip - start_ip), abort_ip)
136
137#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
138 RSEQ_INJECT_ASM(2) \
139 "lwz %%r17, %[" __rseq_str(current_cpu_id) "]\n\t" \
140 "cmpw cr7, %[" __rseq_str(cpu_id) "], %%r17\n\t" \
141 "bne- cr7, " __rseq_str(label) "\n\t"
142
143#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
144 ".pushsection __rseq_failure, \"ax\"\n\t" \
145 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
146 __rseq_str(label) ":\n\t" \
147 "b %l[" __rseq_str(abort_label) "]\n\t" \
148 ".popsection\n\t"
149
150/*
151 * RSEQ_ASM_OPs: asm operations for rseq
152 * RSEQ_ASM_OP_R_*: has hard-code registers in it
153 * RSEQ_ASM_OP_* (else): doesn't have hard-code registers(unless cr7)
154 */
155#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
156 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
157 CMP_WORD "cr7, %%r17, %[" __rseq_str(expect) "]\n\t" \
158 "bne- cr7, " __rseq_str(label) "\n\t"
159
160#define RSEQ_ASM_OP_CMPNE(var, expectnot, label) \
161 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
162 CMP_WORD "cr7, %%r17, %[" __rseq_str(expectnot) "]\n\t" \
163 "beq- cr7, " __rseq_str(label) "\n\t"
164
165#define RSEQ_ASM_OP_STORE(value, var) \
166 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t"
167
168/* Load @var to r17 */
169#define RSEQ_ASM_OP_R_LOAD(var) \
170 LOAD_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
171
172/* Store r17 to @var */
173#define RSEQ_ASM_OP_R_STORE(var) \
174 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t"
175
176/* Add @count to r17 */
177#define RSEQ_ASM_OP_R_ADD(count) \
178 "add %%r17, %[" __rseq_str(count) "], %%r17\n\t"
179
180/* Load (r17 + voffp) to r17 */
181#define RSEQ_ASM_OP_R_LOADX(voffp) \
182 LOADX_WORD "%%r17, %[" __rseq_str(voffp) "], %%r17\n\t"
183
184/* TODO: implement a faster memcpy. */
185#define RSEQ_ASM_OP_R_MEMCPY() \
186 "cmpdi %%r19, 0\n\t" \
187 "beq 333f\n\t" \
188 "addi %%r20, %%r20, -1\n\t" \
189 "addi %%r21, %%r21, -1\n\t" \
190 "222:\n\t" \
191 "lbzu %%r18, 1(%%r20)\n\t" \
192 "stbu %%r18, 1(%%r21)\n\t" \
193 "addi %%r19, %%r19, -1\n\t" \
194 "cmpdi %%r19, 0\n\t" \
195 "bne 222b\n\t" \
196 "333:\n\t" \
197
198#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
199 STORE_WORD "%%r17, %[" __rseq_str(var) "]\n\t" \
200 __rseq_str(post_commit_label) ":\n\t"
201
202#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
203 STORE_WORD "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \
204 __rseq_str(post_commit_label) ":\n\t"
205
206static inline __attribute__((always_inline))
207int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
208{
209 RSEQ_INJECT_C(9)
210
211 __asm__ __volatile__ goto (
212 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
213 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
214#ifdef RSEQ_COMPARE_TWICE
215 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
216 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
217#endif
218 /* Start rseq by storing table entry pointer into rseq_cs. */
219 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
220 /* cmp cpuid */
221 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
222 RSEQ_INJECT_ASM(3)
223 /* cmp @v equal to @expect */
224 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
225 RSEQ_INJECT_ASM(4)
226#ifdef RSEQ_COMPARE_TWICE
227 /* cmp cpuid */
228 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
229 /* cmp @v equal to @expect */
230 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
231#endif
232 /* final store */
233 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
234 RSEQ_INJECT_ASM(5)
235 RSEQ_ASM_DEFINE_ABORT(4, abort)
236 : /* gcc asm goto does not allow outputs */
237 : [cpu_id] "r" (cpu),
238 [current_cpu_id] "m" (__rseq_abi.cpu_id),
239 [rseq_cs] "m" (__rseq_abi.rseq_cs),
240 [v] "m" (*v),
241 [expect] "r" (expect),
242 [newv] "r" (newv)
243 RSEQ_INJECT_INPUT
244 : "memory", "cc", "r17"
245 RSEQ_INJECT_CLOBBER
246 : abort, cmpfail
247#ifdef RSEQ_COMPARE_TWICE
248 , error1, error2
249#endif
250 );
251 return 0;
252abort:
253 RSEQ_INJECT_FAILED
254 return -1;
255cmpfail:
256 return 1;
257#ifdef RSEQ_COMPARE_TWICE
258error1:
259 rseq_bug("cpu_id comparison failed");
260error2:
261 rseq_bug("expected value comparison failed");
262#endif
263}
264
265static inline __attribute__((always_inline))
266int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
267 off_t voffp, intptr_t *load, int cpu)
268{
269 RSEQ_INJECT_C(9)
270
271 __asm__ __volatile__ goto (
272 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
273 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
274#ifdef RSEQ_COMPARE_TWICE
275 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
276 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
277#endif
278 /* Start rseq by storing table entry pointer into rseq_cs. */
279 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
280 /* cmp cpuid */
281 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
282 RSEQ_INJECT_ASM(3)
283 /* cmp @v not equal to @expectnot */
284 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail])
285 RSEQ_INJECT_ASM(4)
286#ifdef RSEQ_COMPARE_TWICE
287 /* cmp cpuid */
288 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
289 /* cmp @v not equal to @expectnot */
290 RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2])
291#endif
292 /* load the value of @v */
293 RSEQ_ASM_OP_R_LOAD(v)
294 /* store it in @load */
295 RSEQ_ASM_OP_R_STORE(load)
296 /* dereference voffp(v) */
297 RSEQ_ASM_OP_R_LOADX(voffp)
298 /* final store the value at voffp(v) */
299 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
300 RSEQ_INJECT_ASM(5)
301 RSEQ_ASM_DEFINE_ABORT(4, abort)
302 : /* gcc asm goto does not allow outputs */
303 : [cpu_id] "r" (cpu),
304 [current_cpu_id] "m" (__rseq_abi.cpu_id),
305 [rseq_cs] "m" (__rseq_abi.rseq_cs),
306 /* final store input */
307 [v] "m" (*v),
308 [expectnot] "r" (expectnot),
309 [voffp] "b" (voffp),
310 [load] "m" (*load)
311 RSEQ_INJECT_INPUT
312 : "memory", "cc", "r17"
313 RSEQ_INJECT_CLOBBER
314 : abort, cmpfail
315#ifdef RSEQ_COMPARE_TWICE
316 , error1, error2
317#endif
318 );
319 return 0;
320abort:
321 RSEQ_INJECT_FAILED
322 return -1;
323cmpfail:
324 return 1;
325#ifdef RSEQ_COMPARE_TWICE
326error1:
327 rseq_bug("cpu_id comparison failed");
328error2:
329 rseq_bug("expected value comparison failed");
330#endif
331}
332
333static inline __attribute__((always_inline))
334int rseq_addv(intptr_t *v, intptr_t count, int cpu)
335{
336 RSEQ_INJECT_C(9)
337
338 __asm__ __volatile__ goto (
339 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
340#ifdef RSEQ_COMPARE_TWICE
341 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
342#endif
343 /* Start rseq by storing table entry pointer into rseq_cs. */
344 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
345 /* cmp cpuid */
346 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
347 RSEQ_INJECT_ASM(3)
348#ifdef RSEQ_COMPARE_TWICE
349 /* cmp cpuid */
350 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
351#endif
352 /* load the value of @v */
353 RSEQ_ASM_OP_R_LOAD(v)
354 /* add @count to it */
355 RSEQ_ASM_OP_R_ADD(count)
356 /* final store */
357 RSEQ_ASM_OP_R_FINAL_STORE(v, 2)
358 RSEQ_INJECT_ASM(4)
359 RSEQ_ASM_DEFINE_ABORT(4, abort)
360 : /* gcc asm goto does not allow outputs */
361 : [cpu_id] "r" (cpu),
362 [current_cpu_id] "m" (__rseq_abi.cpu_id),
363 [rseq_cs] "m" (__rseq_abi.rseq_cs),
364 /* final store input */
365 [v] "m" (*v),
366 [count] "r" (count)
367 RSEQ_INJECT_INPUT
368 : "memory", "cc", "r17"
369 RSEQ_INJECT_CLOBBER
370 : abort
371#ifdef RSEQ_COMPARE_TWICE
372 , error1
373#endif
374 );
375 return 0;
376abort:
377 RSEQ_INJECT_FAILED
378 return -1;
379#ifdef RSEQ_COMPARE_TWICE
380error1:
381 rseq_bug("cpu_id comparison failed");
382#endif
383}
384
385static inline __attribute__((always_inline))
386int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
387 intptr_t *v2, intptr_t newv2,
388 intptr_t newv, int cpu)
389{
390 RSEQ_INJECT_C(9)
391
392 __asm__ __volatile__ goto (
393 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
394 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
395#ifdef RSEQ_COMPARE_TWICE
396 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
397 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
398#endif
399 /* Start rseq by storing table entry pointer into rseq_cs. */
400 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
401 /* cmp cpuid */
402 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
403 RSEQ_INJECT_ASM(3)
404 /* cmp @v equal to @expect */
405 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
406 RSEQ_INJECT_ASM(4)
407#ifdef RSEQ_COMPARE_TWICE
408 /* cmp cpuid */
409 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
410 /* cmp @v equal to @expect */
411 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
412#endif
413 /* try store */
414 RSEQ_ASM_OP_STORE(newv2, v2)
415 RSEQ_INJECT_ASM(5)
416 /* final store */
417 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
418 RSEQ_INJECT_ASM(6)
419 RSEQ_ASM_DEFINE_ABORT(4, abort)
420 : /* gcc asm goto does not allow outputs */
421 : [cpu_id] "r" (cpu),
422 [current_cpu_id] "m" (__rseq_abi.cpu_id),
423 [rseq_cs] "m" (__rseq_abi.rseq_cs),
424 /* try store input */
425 [v2] "m" (*v2),
426 [newv2] "r" (newv2),
427 /* final store input */
428 [v] "m" (*v),
429 [expect] "r" (expect),
430 [newv] "r" (newv)
431 RSEQ_INJECT_INPUT
432 : "memory", "cc", "r17"
433 RSEQ_INJECT_CLOBBER
434 : abort, cmpfail
435#ifdef RSEQ_COMPARE_TWICE
436 , error1, error2
437#endif
438 );
439 return 0;
440abort:
441 RSEQ_INJECT_FAILED
442 return -1;
443cmpfail:
444 return 1;
445#ifdef RSEQ_COMPARE_TWICE
446error1:
447 rseq_bug("cpu_id comparison failed");
448error2:
449 rseq_bug("expected value comparison failed");
450#endif
451}
452
453static inline __attribute__((always_inline))
454int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
455 intptr_t *v2, intptr_t newv2,
456 intptr_t newv, int cpu)
457{
458 RSEQ_INJECT_C(9)
459
460 __asm__ __volatile__ goto (
461 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
462 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
463#ifdef RSEQ_COMPARE_TWICE
464 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
465 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
466#endif
467 /* Start rseq by storing table entry pointer into rseq_cs. */
468 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
469 /* cmp cpuid */
470 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
471 RSEQ_INJECT_ASM(3)
472 /* cmp @v equal to @expect */
473 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
474 RSEQ_INJECT_ASM(4)
475#ifdef RSEQ_COMPARE_TWICE
476 /* cmp cpuid */
477 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
478 /* cmp @v equal to @expect */
479 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
480#endif
481 /* try store */
482 RSEQ_ASM_OP_STORE(newv2, v2)
483 RSEQ_INJECT_ASM(5)
484 /* for 'release' */
485 "lwsync\n\t"
486 /* final store */
487 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
488 RSEQ_INJECT_ASM(6)
489 RSEQ_ASM_DEFINE_ABORT(4, abort)
490 : /* gcc asm goto does not allow outputs */
491 : [cpu_id] "r" (cpu),
492 [current_cpu_id] "m" (__rseq_abi.cpu_id),
493 [rseq_cs] "m" (__rseq_abi.rseq_cs),
494 /* try store input */
495 [v2] "m" (*v2),
496 [newv2] "r" (newv2),
497 /* final store input */
498 [v] "m" (*v),
499 [expect] "r" (expect),
500 [newv] "r" (newv)
501 RSEQ_INJECT_INPUT
502 : "memory", "cc", "r17"
503 RSEQ_INJECT_CLOBBER
504 : abort, cmpfail
505#ifdef RSEQ_COMPARE_TWICE
506 , error1, error2
507#endif
508 );
509 return 0;
510abort:
511 RSEQ_INJECT_FAILED
512 return -1;
513cmpfail:
514 return 1;
515#ifdef RSEQ_COMPARE_TWICE
516error1:
517 rseq_bug("cpu_id comparison failed");
518error2:
519 rseq_bug("expected value comparison failed");
520#endif
521}
522
523static inline __attribute__((always_inline))
524int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
525 intptr_t *v2, intptr_t expect2,
526 intptr_t newv, int cpu)
527{
528 RSEQ_INJECT_C(9)
529
530 __asm__ __volatile__ goto (
531 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
532 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
533#ifdef RSEQ_COMPARE_TWICE
534 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
535 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
536 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
537#endif
538 /* Start rseq by storing table entry pointer into rseq_cs. */
539 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
540 /* cmp cpuid */
541 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
542 RSEQ_INJECT_ASM(3)
543 /* cmp @v equal to @expect */
544 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
545 RSEQ_INJECT_ASM(4)
546 /* cmp @v2 equal to @expct2 */
547 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail])
548 RSEQ_INJECT_ASM(5)
549#ifdef RSEQ_COMPARE_TWICE
550 /* cmp cpuid */
551 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
552 /* cmp @v equal to @expect */
553 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
554 /* cmp @v2 equal to @expct2 */
555 RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3])
556#endif
557 /* final store */
558 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
559 RSEQ_INJECT_ASM(6)
560 RSEQ_ASM_DEFINE_ABORT(4, abort)
561 : /* gcc asm goto does not allow outputs */
562 : [cpu_id] "r" (cpu),
563 [current_cpu_id] "m" (__rseq_abi.cpu_id),
564 [rseq_cs] "m" (__rseq_abi.rseq_cs),
565 /* cmp2 input */
566 [v2] "m" (*v2),
567 [expect2] "r" (expect2),
568 /* final store input */
569 [v] "m" (*v),
570 [expect] "r" (expect),
571 [newv] "r" (newv)
572 RSEQ_INJECT_INPUT
573 : "memory", "cc", "r17"
574 RSEQ_INJECT_CLOBBER
575 : abort, cmpfail
576#ifdef RSEQ_COMPARE_TWICE
577 , error1, error2, error3
578#endif
579 );
580 return 0;
581abort:
582 RSEQ_INJECT_FAILED
583 return -1;
584cmpfail:
585 return 1;
586#ifdef RSEQ_COMPARE_TWICE
587error1:
588 rseq_bug("cpu_id comparison failed");
589error2:
590 rseq_bug("1st expected value comparison failed");
591error3:
592 rseq_bug("2nd expected value comparison failed");
593#endif
594}
595
596static inline __attribute__((always_inline))
597int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
598 void *dst, void *src, size_t len,
599 intptr_t newv, int cpu)
600{
601 RSEQ_INJECT_C(9)
602
603 __asm__ __volatile__ goto (
604 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
605 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
606#ifdef RSEQ_COMPARE_TWICE
607 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
608 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
609#endif
610 /* setup for mempcy */
611 "mr %%r19, %[len]\n\t"
612 "mr %%r20, %[src]\n\t"
613 "mr %%r21, %[dst]\n\t"
614 /* Start rseq by storing table entry pointer into rseq_cs. */
615 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
616 /* cmp cpuid */
617 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
618 RSEQ_INJECT_ASM(3)
619 /* cmp @v equal to @expect */
620 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
621 RSEQ_INJECT_ASM(4)
622#ifdef RSEQ_COMPARE_TWICE
623 /* cmp cpuid */
624 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
625 /* cmp @v equal to @expect */
626 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
627#endif
628 /* try memcpy */
629 RSEQ_ASM_OP_R_MEMCPY()
630 RSEQ_INJECT_ASM(5)
631 /* final store */
632 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
633 RSEQ_INJECT_ASM(6)
634 /* teardown */
635 RSEQ_ASM_DEFINE_ABORT(4, abort)
636 : /* gcc asm goto does not allow outputs */
637 : [cpu_id] "r" (cpu),
638 [current_cpu_id] "m" (__rseq_abi.cpu_id),
639 [rseq_cs] "m" (__rseq_abi.rseq_cs),
640 /* final store input */
641 [v] "m" (*v),
642 [expect] "r" (expect),
643 [newv] "r" (newv),
644 /* try memcpy input */
645 [dst] "r" (dst),
646 [src] "r" (src),
647 [len] "r" (len)
648 RSEQ_INJECT_INPUT
649 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
650 RSEQ_INJECT_CLOBBER
651 : abort, cmpfail
652#ifdef RSEQ_COMPARE_TWICE
653 , error1, error2
654#endif
655 );
656 return 0;
657abort:
658 RSEQ_INJECT_FAILED
659 return -1;
660cmpfail:
661 return 1;
662#ifdef RSEQ_COMPARE_TWICE
663error1:
664 rseq_bug("cpu_id comparison failed");
665error2:
666 rseq_bug("expected value comparison failed");
667#endif
668}
669
670static inline __attribute__((always_inline))
671int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
672 void *dst, void *src, size_t len,
673 intptr_t newv, int cpu)
674{
675 RSEQ_INJECT_C(9)
676
677 __asm__ __volatile__ goto (
678 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
679 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
680#ifdef RSEQ_COMPARE_TWICE
681 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
682 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
683#endif
684 /* setup for mempcy */
685 "mr %%r19, %[len]\n\t"
686 "mr %%r20, %[src]\n\t"
687 "mr %%r21, %[dst]\n\t"
688 /* Start rseq by storing table entry pointer into rseq_cs. */
689 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
690 /* cmp cpuid */
691 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
692 RSEQ_INJECT_ASM(3)
693 /* cmp @v equal to @expect */
694 RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail])
695 RSEQ_INJECT_ASM(4)
696#ifdef RSEQ_COMPARE_TWICE
697 /* cmp cpuid */
698 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
699 /* cmp @v equal to @expect */
700 RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2])
701#endif
702 /* try memcpy */
703 RSEQ_ASM_OP_R_MEMCPY()
704 RSEQ_INJECT_ASM(5)
705 /* for 'release' */
706 "lwsync\n\t"
707 /* final store */
708 RSEQ_ASM_OP_FINAL_STORE(newv, v, 2)
709 RSEQ_INJECT_ASM(6)
710 /* teardown */
711 RSEQ_ASM_DEFINE_ABORT(4, abort)
712 : /* gcc asm goto does not allow outputs */
713 : [cpu_id] "r" (cpu),
714 [current_cpu_id] "m" (__rseq_abi.cpu_id),
715 [rseq_cs] "m" (__rseq_abi.rseq_cs),
716 /* final store input */
717 [v] "m" (*v),
718 [expect] "r" (expect),
719 [newv] "r" (newv),
720 /* try memcpy input */
721 [dst] "r" (dst),
722 [src] "r" (src),
723 [len] "r" (len)
724 RSEQ_INJECT_INPUT
725 : "memory", "cc", "r17", "r18", "r19", "r20", "r21"
726 RSEQ_INJECT_CLOBBER
727 : abort, cmpfail
728#ifdef RSEQ_COMPARE_TWICE
729 , error1, error2
730#endif
731 );
732 return 0;
733abort:
734 RSEQ_INJECT_FAILED
735 return -1;
736cmpfail:
737 return 1;
738#ifdef RSEQ_COMPARE_TWICE
739error1:
740 rseq_bug("cpu_id comparison failed");
741error2:
742 rseq_bug("expected value comparison failed");
743#endif
744}
745
746#undef STORE_WORD
747#undef LOAD_WORD
748#undef LOADX_WORD
749#undef CMP_WORD
750
751#endif /* !RSEQ_SKIP_FASTPATH */