tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * Author: Paul Burton <paul.burton@mips.com>
4 * (C) Copyright 2018 MIPS Tech LLC
5 *
6 * Based on rseq-arm.h:
7 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
8 */
9
10#define RSEQ_SIG 0x53053053
11
12#define rseq_smp_mb() __asm__ __volatile__ ("sync" ::: "memory")
13#define rseq_smp_rmb() rseq_smp_mb()
14#define rseq_smp_wmb() rseq_smp_mb()
15
16#define rseq_smp_load_acquire(p) \
17__extension__ ({ \
18 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
19 rseq_smp_mb(); \
20 ____p1; \
21})
22
23#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
24
25#define rseq_smp_store_release(p, v) \
26do { \
27 rseq_smp_mb(); \
28 RSEQ_WRITE_ONCE(*p, v); \
29} while (0)
30
31#ifdef RSEQ_SKIP_FASTPATH
32#include "rseq-skip.h"
33#else /* !RSEQ_SKIP_FASTPATH */
34
35#if _MIPS_SZLONG == 64
36# define LONG ".dword"
37# define LONG_LA "dla"
38# define LONG_L "ld"
39# define LONG_S "sd"
40# define LONG_ADDI "daddiu"
41# define U32_U64_PAD(x) x
42#elif _MIPS_SZLONG == 32
43# define LONG ".word"
44# define LONG_LA "la"
45# define LONG_L "lw"
46# define LONG_S "sw"
47# define LONG_ADDI "addiu"
48# ifdef __BIG_ENDIAN
49# define U32_U64_PAD(x) "0x0, " x
50# else
51# define U32_U64_PAD(x) x ", 0x0"
52# endif
53#else
54# error unsupported _MIPS_SZLONG
55#endif
56
57#define __RSEQ_ASM_DEFINE_TABLE(version, flags, start_ip, \
58 post_commit_offset, abort_ip) \
59 ".pushsection __rseq_table, \"aw\"\n\t" \
60 ".balign 32\n\t" \
61 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
62 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
63 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
64 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
65 ".popsection\n\t"
66
67#define RSEQ_ASM_DEFINE_TABLE(start_ip, post_commit_ip, abort_ip) \
68 __RSEQ_ASM_DEFINE_TABLE(0x0, 0x0, start_ip, \
69 (post_commit_ip - start_ip), abort_ip)
70
71#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
72 RSEQ_INJECT_ASM(1) \
73 LONG_LA " $4, " __rseq_str(cs_label) "\n\t" \
74 LONG_S " $4, %[" __rseq_str(rseq_cs) "]\n\t" \
75 __rseq_str(label) ":\n\t"
76
77#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
78 RSEQ_INJECT_ASM(2) \
79 "lw $4, %[" __rseq_str(current_cpu_id) "]\n\t" \
80 "bne $4, %[" __rseq_str(cpu_id) "], " __rseq_str(label) "\n\t"
81
82#define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
83 abort_label, version, flags, \
84 start_ip, post_commit_offset, abort_ip) \
85 ".balign 32\n\t" \
86 __rseq_str(table_label) ":\n\t" \
87 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
88 LONG " " U32_U64_PAD(__rseq_str(start_ip)) "\n\t" \
89 LONG " " U32_U64_PAD(__rseq_str(post_commit_offset)) "\n\t" \
90 LONG " " U32_U64_PAD(__rseq_str(abort_ip)) "\n\t" \
91 ".word " __rseq_str(RSEQ_SIG) "\n\t" \
92 __rseq_str(label) ":\n\t" \
93 teardown \
94 "b %l[" __rseq_str(abort_label) "]\n\t"
95
96#define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
97 start_ip, post_commit_ip, abort_ip) \
98 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
99 abort_label, 0x0, 0x0, start_ip, \
100 (post_commit_ip - start_ip), abort_ip)
101
102#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
103 __rseq_str(label) ":\n\t" \
104 teardown \
105 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
106
107#define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("")
108
109static inline __attribute__((always_inline))
110int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
111{
112 RSEQ_INJECT_C(9)
113
114 rseq_workaround_gcc_asm_size_guess();
115 __asm__ __volatile__ goto (
116 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
117 /* Start rseq by storing table entry pointer into rseq_cs. */
118 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
119 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
120 RSEQ_INJECT_ASM(3)
121 LONG_L " $4, %[v]\n\t"
122 "bne $4, %[expect], %l[cmpfail]\n\t"
123 RSEQ_INJECT_ASM(4)
124#ifdef RSEQ_COMPARE_TWICE
125 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
126 LONG_L " $4, %[v]\n\t"
127 "bne $4, %[expect], %l[error2]\n\t"
128#endif
129 /* final store */
130 LONG_S " %[newv], %[v]\n\t"
131 "2:\n\t"
132 RSEQ_INJECT_ASM(5)
133 "b 5f\n\t"
134 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
135 "5:\n\t"
136 : /* gcc asm goto does not allow outputs */
137 : [cpu_id] "r" (cpu),
138 [current_cpu_id] "m" (__rseq_abi.cpu_id),
139 [rseq_cs] "m" (__rseq_abi.rseq_cs),
140 [v] "m" (*v),
141 [expect] "r" (expect),
142 [newv] "r" (newv)
143 RSEQ_INJECT_INPUT
144 : "$4", "memory"
145 RSEQ_INJECT_CLOBBER
146 : abort, cmpfail
147#ifdef RSEQ_COMPARE_TWICE
148 , error1, error2
149#endif
150 );
151 rseq_workaround_gcc_asm_size_guess();
152 return 0;
153abort:
154 rseq_workaround_gcc_asm_size_guess();
155 RSEQ_INJECT_FAILED
156 return -1;
157cmpfail:
158 rseq_workaround_gcc_asm_size_guess();
159 return 1;
160#ifdef RSEQ_COMPARE_TWICE
161error1:
162 rseq_bug("cpu_id comparison failed");
163error2:
164 rseq_bug("expected value comparison failed");
165#endif
166}
167
168static inline __attribute__((always_inline))
169int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
170 off_t voffp, intptr_t *load, int cpu)
171{
172 RSEQ_INJECT_C(9)
173
174 rseq_workaround_gcc_asm_size_guess();
175 __asm__ __volatile__ goto (
176 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
177 /* Start rseq by storing table entry pointer into rseq_cs. */
178 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
179 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
180 RSEQ_INJECT_ASM(3)
181 LONG_L " $4, %[v]\n\t"
182 "beq $4, %[expectnot], %l[cmpfail]\n\t"
183 RSEQ_INJECT_ASM(4)
184#ifdef RSEQ_COMPARE_TWICE
185 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
186 LONG_L " $4, %[v]\n\t"
187 "beq $4, %[expectnot], %l[error2]\n\t"
188#endif
189 LONG_S " $4, %[load]\n\t"
190 LONG_ADDI " $4, %[voffp]\n\t"
191 LONG_L " $4, 0($4)\n\t"
192 /* final store */
193 LONG_S " $4, %[v]\n\t"
194 "2:\n\t"
195 RSEQ_INJECT_ASM(5)
196 "b 5f\n\t"
197 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
198 "5:\n\t"
199 : /* gcc asm goto does not allow outputs */
200 : [cpu_id] "r" (cpu),
201 [current_cpu_id] "m" (__rseq_abi.cpu_id),
202 [rseq_cs] "m" (__rseq_abi.rseq_cs),
203 /* final store input */
204 [v] "m" (*v),
205 [expectnot] "r" (expectnot),
206 [voffp] "Ir" (voffp),
207 [load] "m" (*load)
208 RSEQ_INJECT_INPUT
209 : "$4", "memory"
210 RSEQ_INJECT_CLOBBER
211 : abort, cmpfail
212#ifdef RSEQ_COMPARE_TWICE
213 , error1, error2
214#endif
215 );
216 rseq_workaround_gcc_asm_size_guess();
217 return 0;
218abort:
219 rseq_workaround_gcc_asm_size_guess();
220 RSEQ_INJECT_FAILED
221 return -1;
222cmpfail:
223 rseq_workaround_gcc_asm_size_guess();
224 return 1;
225#ifdef RSEQ_COMPARE_TWICE
226error1:
227 rseq_bug("cpu_id comparison failed");
228error2:
229 rseq_bug("expected value comparison failed");
230#endif
231}
232
233static inline __attribute__((always_inline))
234int rseq_addv(intptr_t *v, intptr_t count, int cpu)
235{
236 RSEQ_INJECT_C(9)
237
238 rseq_workaround_gcc_asm_size_guess();
239 __asm__ __volatile__ goto (
240 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
241 /* Start rseq by storing table entry pointer into rseq_cs. */
242 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
243 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
244 RSEQ_INJECT_ASM(3)
245#ifdef RSEQ_COMPARE_TWICE
246 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
247#endif
248 LONG_L " $4, %[v]\n\t"
249 LONG_ADDI " $4, %[count]\n\t"
250 /* final store */
251 LONG_S " $4, %[v]\n\t"
252 "2:\n\t"
253 RSEQ_INJECT_ASM(4)
254 "b 5f\n\t"
255 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
256 "5:\n\t"
257 : /* gcc asm goto does not allow outputs */
258 : [cpu_id] "r" (cpu),
259 [current_cpu_id] "m" (__rseq_abi.cpu_id),
260 [rseq_cs] "m" (__rseq_abi.rseq_cs),
261 [v] "m" (*v),
262 [count] "Ir" (count)
263 RSEQ_INJECT_INPUT
264 : "$4", "memory"
265 RSEQ_INJECT_CLOBBER
266 : abort
267#ifdef RSEQ_COMPARE_TWICE
268 , error1
269#endif
270 );
271 rseq_workaround_gcc_asm_size_guess();
272 return 0;
273abort:
274 rseq_workaround_gcc_asm_size_guess();
275 RSEQ_INJECT_FAILED
276 return -1;
277#ifdef RSEQ_COMPARE_TWICE
278error1:
279 rseq_bug("cpu_id comparison failed");
280#endif
281}
282
283static inline __attribute__((always_inline))
284int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
285 intptr_t *v2, intptr_t newv2,
286 intptr_t newv, int cpu)
287{
288 RSEQ_INJECT_C(9)
289
290 rseq_workaround_gcc_asm_size_guess();
291 __asm__ __volatile__ goto (
292 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
293 /* Start rseq by storing table entry pointer into rseq_cs. */
294 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
295 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
296 RSEQ_INJECT_ASM(3)
297 LONG_L " $4, %[v]\n\t"
298 "bne $4, %[expect], %l[cmpfail]\n\t"
299 RSEQ_INJECT_ASM(4)
300#ifdef RSEQ_COMPARE_TWICE
301 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
302 LONG_L " $4, %[v]\n\t"
303 "bne $4, %[expect], %l[error2]\n\t"
304#endif
305 /* try store */
306 LONG_S " %[newv2], %[v2]\n\t"
307 RSEQ_INJECT_ASM(5)
308 /* final store */
309 LONG_S " %[newv], %[v]\n\t"
310 "2:\n\t"
311 RSEQ_INJECT_ASM(6)
312 "b 5f\n\t"
313 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
314 "5:\n\t"
315 : /* gcc asm goto does not allow outputs */
316 : [cpu_id] "r" (cpu),
317 [current_cpu_id] "m" (__rseq_abi.cpu_id),
318 [rseq_cs] "m" (__rseq_abi.rseq_cs),
319 /* try store input */
320 [v2] "m" (*v2),
321 [newv2] "r" (newv2),
322 /* final store input */
323 [v] "m" (*v),
324 [expect] "r" (expect),
325 [newv] "r" (newv)
326 RSEQ_INJECT_INPUT
327 : "$4", "memory"
328 RSEQ_INJECT_CLOBBER
329 : abort, cmpfail
330#ifdef RSEQ_COMPARE_TWICE
331 , error1, error2
332#endif
333 );
334 rseq_workaround_gcc_asm_size_guess();
335 return 0;
336abort:
337 rseq_workaround_gcc_asm_size_guess();
338 RSEQ_INJECT_FAILED
339 return -1;
340cmpfail:
341 rseq_workaround_gcc_asm_size_guess();
342 return 1;
343#ifdef RSEQ_COMPARE_TWICE
344error1:
345 rseq_bug("cpu_id comparison failed");
346error2:
347 rseq_bug("expected value comparison failed");
348#endif
349}
350
351static inline __attribute__((always_inline))
352int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
353 intptr_t *v2, intptr_t newv2,
354 intptr_t newv, int cpu)
355{
356 RSEQ_INJECT_C(9)
357
358 rseq_workaround_gcc_asm_size_guess();
359 __asm__ __volatile__ goto (
360 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
361 /* Start rseq by storing table entry pointer into rseq_cs. */
362 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
363 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
364 RSEQ_INJECT_ASM(3)
365 LONG_L " $4, %[v]\n\t"
366 "bne $4, %[expect], %l[cmpfail]\n\t"
367 RSEQ_INJECT_ASM(4)
368#ifdef RSEQ_COMPARE_TWICE
369 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
370 LONG_L " $4, %[v]\n\t"
371 "bne $4, %[expect], %l[error2]\n\t"
372#endif
373 /* try store */
374 LONG_S " %[newv2], %[v2]\n\t"
375 RSEQ_INJECT_ASM(5)
376 "sync\n\t" /* full sync provides store-release */
377 /* final store */
378 LONG_S " %[newv], %[v]\n\t"
379 "2:\n\t"
380 RSEQ_INJECT_ASM(6)
381 "b 5f\n\t"
382 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
383 "5:\n\t"
384 : /* gcc asm goto does not allow outputs */
385 : [cpu_id] "r" (cpu),
386 [current_cpu_id] "m" (__rseq_abi.cpu_id),
387 [rseq_cs] "m" (__rseq_abi.rseq_cs),
388 /* try store input */
389 [v2] "m" (*v2),
390 [newv2] "r" (newv2),
391 /* final store input */
392 [v] "m" (*v),
393 [expect] "r" (expect),
394 [newv] "r" (newv)
395 RSEQ_INJECT_INPUT
396 : "$4", "memory"
397 RSEQ_INJECT_CLOBBER
398 : abort, cmpfail
399#ifdef RSEQ_COMPARE_TWICE
400 , error1, error2
401#endif
402 );
403 rseq_workaround_gcc_asm_size_guess();
404 return 0;
405abort:
406 rseq_workaround_gcc_asm_size_guess();
407 RSEQ_INJECT_FAILED
408 return -1;
409cmpfail:
410 rseq_workaround_gcc_asm_size_guess();
411 return 1;
412#ifdef RSEQ_COMPARE_TWICE
413error1:
414 rseq_bug("cpu_id comparison failed");
415error2:
416 rseq_bug("expected value comparison failed");
417#endif
418}
419
420static inline __attribute__((always_inline))
421int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
422 intptr_t *v2, intptr_t expect2,
423 intptr_t newv, int cpu)
424{
425 RSEQ_INJECT_C(9)
426
427 rseq_workaround_gcc_asm_size_guess();
428 __asm__ __volatile__ goto (
429 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
430 /* Start rseq by storing table entry pointer into rseq_cs. */
431 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
432 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
433 RSEQ_INJECT_ASM(3)
434 LONG_L " $4, %[v]\n\t"
435 "bne $4, %[expect], %l[cmpfail]\n\t"
436 RSEQ_INJECT_ASM(4)
437 LONG_L " $4, %[v2]\n\t"
438 "bne $4, %[expect2], %l[cmpfail]\n\t"
439 RSEQ_INJECT_ASM(5)
440#ifdef RSEQ_COMPARE_TWICE
441 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
442 LONG_L " $4, %[v]\n\t"
443 "bne $4, %[expect], %l[error2]\n\t"
444 LONG_L " $4, %[v2]\n\t"
445 "bne $4, %[expect2], %l[error3]\n\t"
446#endif
447 /* final store */
448 LONG_S " %[newv], %[v]\n\t"
449 "2:\n\t"
450 RSEQ_INJECT_ASM(6)
451 "b 5f\n\t"
452 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
453 "5:\n\t"
454 : /* gcc asm goto does not allow outputs */
455 : [cpu_id] "r" (cpu),
456 [current_cpu_id] "m" (__rseq_abi.cpu_id),
457 [rseq_cs] "m" (__rseq_abi.rseq_cs),
458 /* cmp2 input */
459 [v2] "m" (*v2),
460 [expect2] "r" (expect2),
461 /* final store input */
462 [v] "m" (*v),
463 [expect] "r" (expect),
464 [newv] "r" (newv)
465 RSEQ_INJECT_INPUT
466 : "$4", "memory"
467 RSEQ_INJECT_CLOBBER
468 : abort, cmpfail
469#ifdef RSEQ_COMPARE_TWICE
470 , error1, error2, error3
471#endif
472 );
473 rseq_workaround_gcc_asm_size_guess();
474 return 0;
475abort:
476 rseq_workaround_gcc_asm_size_guess();
477 RSEQ_INJECT_FAILED
478 return -1;
479cmpfail:
480 rseq_workaround_gcc_asm_size_guess();
481 return 1;
482#ifdef RSEQ_COMPARE_TWICE
483error1:
484 rseq_bug("cpu_id comparison failed");
485error2:
486 rseq_bug("1st expected value comparison failed");
487error3:
488 rseq_bug("2nd expected value comparison failed");
489#endif
490}
491
492static inline __attribute__((always_inline))
493int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
494 void *dst, void *src, size_t len,
495 intptr_t newv, int cpu)
496{
497 uintptr_t rseq_scratch[3];
498
499 RSEQ_INJECT_C(9)
500
501 rseq_workaround_gcc_asm_size_guess();
502 __asm__ __volatile__ goto (
503 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
504 LONG_S " %[src], %[rseq_scratch0]\n\t"
505 LONG_S " %[dst], %[rseq_scratch1]\n\t"
506 LONG_S " %[len], %[rseq_scratch2]\n\t"
507 /* Start rseq by storing table entry pointer into rseq_cs. */
508 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
509 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
510 RSEQ_INJECT_ASM(3)
511 LONG_L " $4, %[v]\n\t"
512 "bne $4, %[expect], 5f\n\t"
513 RSEQ_INJECT_ASM(4)
514#ifdef RSEQ_COMPARE_TWICE
515 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
516 LONG_L " $4, %[v]\n\t"
517 "bne $4, %[expect], 7f\n\t"
518#endif
519 /* try memcpy */
520 "beqz %[len], 333f\n\t" \
521 "222:\n\t" \
522 "lb $4, 0(%[src])\n\t" \
523 "sb $4, 0(%[dst])\n\t" \
524 LONG_ADDI " %[src], 1\n\t" \
525 LONG_ADDI " %[dst], 1\n\t" \
526 LONG_ADDI " %[len], -1\n\t" \
527 "bnez %[len], 222b\n\t" \
528 "333:\n\t" \
529 RSEQ_INJECT_ASM(5)
530 /* final store */
531 LONG_S " %[newv], %[v]\n\t"
532 "2:\n\t"
533 RSEQ_INJECT_ASM(6)
534 /* teardown */
535 LONG_L " %[len], %[rseq_scratch2]\n\t"
536 LONG_L " %[dst], %[rseq_scratch1]\n\t"
537 LONG_L " %[src], %[rseq_scratch0]\n\t"
538 "b 8f\n\t"
539 RSEQ_ASM_DEFINE_ABORT(3, 4,
540 /* teardown */
541 LONG_L " %[len], %[rseq_scratch2]\n\t"
542 LONG_L " %[dst], %[rseq_scratch1]\n\t"
543 LONG_L " %[src], %[rseq_scratch0]\n\t",
544 abort, 1b, 2b, 4f)
545 RSEQ_ASM_DEFINE_CMPFAIL(5,
546 /* teardown */
547 LONG_L " %[len], %[rseq_scratch2]\n\t"
548 LONG_L " %[dst], %[rseq_scratch1]\n\t"
549 LONG_L " %[src], %[rseq_scratch0]\n\t",
550 cmpfail)
551#ifdef RSEQ_COMPARE_TWICE
552 RSEQ_ASM_DEFINE_CMPFAIL(6,
553 /* teardown */
554 LONG_L " %[len], %[rseq_scratch2]\n\t"
555 LONG_L " %[dst], %[rseq_scratch1]\n\t"
556 LONG_L " %[src], %[rseq_scratch0]\n\t",
557 error1)
558 RSEQ_ASM_DEFINE_CMPFAIL(7,
559 /* teardown */
560 LONG_L " %[len], %[rseq_scratch2]\n\t"
561 LONG_L " %[dst], %[rseq_scratch1]\n\t"
562 LONG_L " %[src], %[rseq_scratch0]\n\t",
563 error2)
564#endif
565 "8:\n\t"
566 : /* gcc asm goto does not allow outputs */
567 : [cpu_id] "r" (cpu),
568 [current_cpu_id] "m" (__rseq_abi.cpu_id),
569 [rseq_cs] "m" (__rseq_abi.rseq_cs),
570 /* final store input */
571 [v] "m" (*v),
572 [expect] "r" (expect),
573 [newv] "r" (newv),
574 /* try memcpy input */
575 [dst] "r" (dst),
576 [src] "r" (src),
577 [len] "r" (len),
578 [rseq_scratch0] "m" (rseq_scratch[0]),
579 [rseq_scratch1] "m" (rseq_scratch[1]),
580 [rseq_scratch2] "m" (rseq_scratch[2])
581 RSEQ_INJECT_INPUT
582 : "$4", "memory"
583 RSEQ_INJECT_CLOBBER
584 : abort, cmpfail
585#ifdef RSEQ_COMPARE_TWICE
586 , error1, error2
587#endif
588 );
589 rseq_workaround_gcc_asm_size_guess();
590 return 0;
591abort:
592 rseq_workaround_gcc_asm_size_guess();
593 RSEQ_INJECT_FAILED
594 return -1;
595cmpfail:
596 rseq_workaround_gcc_asm_size_guess();
597 return 1;
598#ifdef RSEQ_COMPARE_TWICE
599error1:
600 rseq_workaround_gcc_asm_size_guess();
601 rseq_bug("cpu_id comparison failed");
602error2:
603 rseq_workaround_gcc_asm_size_guess();
604 rseq_bug("expected value comparison failed");
605#endif
606}
607
608static inline __attribute__((always_inline))
609int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
610 void *dst, void *src, size_t len,
611 intptr_t newv, int cpu)
612{
613 uintptr_t rseq_scratch[3];
614
615 RSEQ_INJECT_C(9)
616
617 rseq_workaround_gcc_asm_size_guess();
618 __asm__ __volatile__ goto (
619 RSEQ_ASM_DEFINE_TABLE(1f, 2f, 4f) /* start, commit, abort */
620 LONG_S " %[src], %[rseq_scratch0]\n\t"
621 LONG_S " %[dst], %[rseq_scratch1]\n\t"
622 LONG_S " %[len], %[rseq_scratch2]\n\t"
623 /* Start rseq by storing table entry pointer into rseq_cs. */
624 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
625 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
626 RSEQ_INJECT_ASM(3)
627 LONG_L " $4, %[v]\n\t"
628 "bne $4, %[expect], 5f\n\t"
629 RSEQ_INJECT_ASM(4)
630#ifdef RSEQ_COMPARE_TWICE
631 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
632 LONG_L " $4, %[v]\n\t"
633 "bne $4, %[expect], 7f\n\t"
634#endif
635 /* try memcpy */
636 "beqz %[len], 333f\n\t" \
637 "222:\n\t" \
638 "lb $4, 0(%[src])\n\t" \
639 "sb $4, 0(%[dst])\n\t" \
640 LONG_ADDI " %[src], 1\n\t" \
641 LONG_ADDI " %[dst], 1\n\t" \
642 LONG_ADDI " %[len], -1\n\t" \
643 "bnez %[len], 222b\n\t" \
644 "333:\n\t" \
645 RSEQ_INJECT_ASM(5)
646 "sync\n\t" /* full sync provides store-release */
647 /* final store */
648 LONG_S " %[newv], %[v]\n\t"
649 "2:\n\t"
650 RSEQ_INJECT_ASM(6)
651 /* teardown */
652 LONG_L " %[len], %[rseq_scratch2]\n\t"
653 LONG_L " %[dst], %[rseq_scratch1]\n\t"
654 LONG_L " %[src], %[rseq_scratch0]\n\t"
655 "b 8f\n\t"
656 RSEQ_ASM_DEFINE_ABORT(3, 4,
657 /* teardown */
658 LONG_L " %[len], %[rseq_scratch2]\n\t"
659 LONG_L " %[dst], %[rseq_scratch1]\n\t"
660 LONG_L " %[src], %[rseq_scratch0]\n\t",
661 abort, 1b, 2b, 4f)
662 RSEQ_ASM_DEFINE_CMPFAIL(5,
663 /* teardown */
664 LONG_L " %[len], %[rseq_scratch2]\n\t"
665 LONG_L " %[dst], %[rseq_scratch1]\n\t"
666 LONG_L " %[src], %[rseq_scratch0]\n\t",
667 cmpfail)
668#ifdef RSEQ_COMPARE_TWICE
669 RSEQ_ASM_DEFINE_CMPFAIL(6,
670 /* teardown */
671 LONG_L " %[len], %[rseq_scratch2]\n\t"
672 LONG_L " %[dst], %[rseq_scratch1]\n\t"
673 LONG_L " %[src], %[rseq_scratch0]\n\t",
674 error1)
675 RSEQ_ASM_DEFINE_CMPFAIL(7,
676 /* teardown */
677 LONG_L " %[len], %[rseq_scratch2]\n\t"
678 LONG_L " %[dst], %[rseq_scratch1]\n\t"
679 LONG_L " %[src], %[rseq_scratch0]\n\t",
680 error2)
681#endif
682 "8:\n\t"
683 : /* gcc asm goto does not allow outputs */
684 : [cpu_id] "r" (cpu),
685 [current_cpu_id] "m" (__rseq_abi.cpu_id),
686 [rseq_cs] "m" (__rseq_abi.rseq_cs),
687 /* final store input */
688 [v] "m" (*v),
689 [expect] "r" (expect),
690 [newv] "r" (newv),
691 /* try memcpy input */
692 [dst] "r" (dst),
693 [src] "r" (src),
694 [len] "r" (len),
695 [rseq_scratch0] "m" (rseq_scratch[0]),
696 [rseq_scratch1] "m" (rseq_scratch[1]),
697 [rseq_scratch2] "m" (rseq_scratch[2])
698 RSEQ_INJECT_INPUT
699 : "$4", "memory"
700 RSEQ_INJECT_CLOBBER
701 : abort, cmpfail
702#ifdef RSEQ_COMPARE_TWICE
703 , error1, error2
704#endif
705 );
706 rseq_workaround_gcc_asm_size_guess();
707 return 0;
708abort:
709 rseq_workaround_gcc_asm_size_guess();
710 RSEQ_INJECT_FAILED
711 return -1;
712cmpfail:
713 rseq_workaround_gcc_asm_size_guess();
714 return 1;
715#ifdef RSEQ_COMPARE_TWICE
716error1:
717 rseq_workaround_gcc_asm_size_guess();
718 rseq_bug("cpu_id comparison failed");
719error2:
720 rseq_workaround_gcc_asm_size_guess();
721 rseq_bug("expected value comparison failed");
722#endif
723}
724
725#endif /* !RSEQ_SKIP_FASTPATH */