tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
rseq/selftests: Add support for RISC-V
Add support for RISC-V in the rseq selftests, which covers both
64-bit and 32-bit ISA with little endian mode.
Signed-off-by: Vincent Chen <vincent.chen@sifive.com>
Tested-by: Eric Lin <eric.lin@sifive.com>
Reviewed-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>
authored by
Vincent Chen
and committed by
Palmer Dabbelt
6d1a6f46
93917ad5
+702
+23
tools/testing/selftests/rseq/param_test.c
+23
tools/testing/selftests/rseq/param_test.c
···
207
207
"addiu " INJECT_ASM_REG ", -1\n\t" \
208
208
"bnez " INJECT_ASM_REG ", 222b\n\t" \
209
209
"333:\n\t"
210
+
#elif defined(__riscv)
211
+
212
+
#define RSEQ_INJECT_INPUT \
213
+
, [loop_cnt_1]"m"(loop_cnt[1]) \
214
+
, [loop_cnt_2]"m"(loop_cnt[2]) \
215
+
, [loop_cnt_3]"m"(loop_cnt[3]) \
216
+
, [loop_cnt_4]"m"(loop_cnt[4]) \
217
+
, [loop_cnt_5]"m"(loop_cnt[5]) \
218
+
, [loop_cnt_6]"m"(loop_cnt[6])
219
+
220
+
#define INJECT_ASM_REG "t1"
221
+
222
+
#define RSEQ_INJECT_CLOBBER \
223
+
, INJECT_ASM_REG
224
+
225
+
#define RSEQ_INJECT_ASM(n) \
226
+
"lw " INJECT_ASM_REG ", %[loop_cnt_" #n "]\n\t" \
227
+
"beqz " INJECT_ASM_REG ", 333f\n\t" \
228
+
"222:\n\t" \
229
+
"addi " INJECT_ASM_REG "," INJECT_ASM_REG ", -1\n\t" \
230
+
"bnez " INJECT_ASM_REG ", 222b\n\t" \
231
+
"333:\n\t"
232
+
210
233
211
234
#else
212
235
#error unsupported target
+677
tools/testing/selftests/rseq/rseq-riscv.h
+677
tools/testing/selftests/rseq/rseq-riscv.h
···
1
+
/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2
+
/*
3
+
* Select the instruction "csrw mhartid, x0" as the RSEQ_SIG. Unlike
4
+
* other architectures, the ebreak instruction has no immediate field for
5
+
* distinguishing purposes. Hence, ebreak is not suitable as RSEQ_SIG.
6
+
* "csrw mhartid, x0" can also satisfy the RSEQ requirement because it
7
+
* is an uncommon instruction and will raise an illegal instruction
8
+
* exception when executed in all modes.
9
+
*/
10
+
#include <endian.h>
11
+
12
+
#if defined(__BYTE_ORDER) ? (__BYTE_ORDER == __LITTLE_ENDIAN) : defined(__LITTLE_ENDIAN)
13
+
#define RSEQ_SIG 0xf1401073 /* csrr mhartid, x0 */
14
+
#else
15
+
#error "Currently, RSEQ only supports Little-Endian version"
16
+
#endif
17
+
18
+
#if __riscv_xlen == 64
19
+
#define __REG_SEL(a, b) a
20
+
#elif __riscv_xlen == 32
21
+
#define __REG_SEL(a, b) b
22
+
#endif
23
+
24
+
#define REG_L __REG_SEL("ld ", "lw ")
25
+
#define REG_S __REG_SEL("sd ", "sw ")
26
+
27
+
#define RISCV_FENCE(p, s) \
28
+
__asm__ __volatile__ ("fence " #p "," #s : : : "memory")
29
+
#define rseq_smp_mb() RISCV_FENCE(rw, rw)
30
+
#define rseq_smp_rmb() RISCV_FENCE(r, r)
31
+
#define rseq_smp_wmb() RISCV_FENCE(w, w)
32
+
#define RSEQ_ASM_TMP_REG_1 "t6"
33
+
#define RSEQ_ASM_TMP_REG_2 "t5"
34
+
#define RSEQ_ASM_TMP_REG_3 "t4"
35
+
#define RSEQ_ASM_TMP_REG_4 "t3"
36
+
37
+
#define rseq_smp_load_acquire(p) \
38
+
__extension__ ({ \
39
+
__typeof(*(p)) ____p1 = RSEQ_READ_ONCE(*(p)); \
40
+
RISCV_FENCE(r, rw) \
41
+
____p1; \
42
+
})
43
+
44
+
#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
45
+
46
+
#define rseq_smp_store_release(p, v) \
47
+
do { \
48
+
RISCV_FENCE(rw, w); \
49
+
RSEQ_WRITE_ONCE(*(p), v); \
50
+
} while (0)
51
+
52
+
#ifdef RSEQ_SKIP_FASTPATH
53
+
#include "rseq-skip.h"
54
+
#else /* !RSEQ_SKIP_FASTPATH */
55
+
56
+
#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
57
+
post_commit_offset, abort_ip) \
58
+
".pushsection __rseq_cs, \"aw\"\n" \
59
+
".balign 32\n" \
60
+
__rseq_str(label) ":\n" \
61
+
".long " __rseq_str(version) ", " __rseq_str(flags) "\n" \
62
+
".quad " __rseq_str(start_ip) ", " \
63
+
__rseq_str(post_commit_offset) ", " \
64
+
__rseq_str(abort_ip) "\n" \
65
+
".popsection\n\t" \
66
+
".pushsection __rseq_cs_ptr_array, \"aw\"\n" \
67
+
".quad " __rseq_str(label) "b\n" \
68
+
".popsection\n"
69
+
70
+
#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
71
+
__RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
72
+
((post_commit_ip) - (start_ip)), abort_ip)
73
+
74
+
/*
75
+
* Exit points of a rseq critical section consist of all instructions outside
76
+
* of the critical section where a critical section can either branch to or
77
+
* reach through the normal course of its execution. The abort IP and the
78
+
* post-commit IP are already part of the __rseq_cs section and should not be
79
+
* explicitly defined as additional exit points. Knowing all exit points is
80
+
* useful to assist debuggers stepping over the critical section.
81
+
*/
82
+
#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
83
+
".pushsection __rseq_exit_point_array, \"aw\"\n" \
84
+
".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n" \
85
+
".popsection\n"
86
+
87
+
#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
88
+
RSEQ_INJECT_ASM(1) \
89
+
"la "RSEQ_ASM_TMP_REG_1 ", " __rseq_str(cs_label) "\n" \
90
+
REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(rseq_cs) "]\n" \
91
+
__rseq_str(label) ":\n"
92
+
93
+
#define RSEQ_ASM_DEFINE_ABORT(label, abort_label) \
94
+
"j 222f\n" \
95
+
".balign 4\n" \
96
+
".long " __rseq_str(RSEQ_SIG) "\n" \
97
+
__rseq_str(label) ":\n" \
98
+
"j %l[" __rseq_str(abort_label) "]\n" \
99
+
"222:\n"
100
+
101
+
#define RSEQ_ASM_OP_STORE(value, var) \
102
+
REG_S "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n"
103
+
104
+
#define RSEQ_ASM_OP_CMPEQ(var, expect, label) \
105
+
REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
106
+
"bne "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
107
+
__rseq_str(label) "\n"
108
+
109
+
#define RSEQ_ASM_OP_CMPEQ32(var, expect, label) \
110
+
"lw "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
111
+
"bne "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
112
+
__rseq_str(label) "\n"
113
+
114
+
#define RSEQ_ASM_OP_CMPNE(var, expect, label) \
115
+
REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
116
+
"beq "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(expect) "] ," \
117
+
__rseq_str(label) "\n"
118
+
119
+
#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
120
+
RSEQ_INJECT_ASM(2) \
121
+
RSEQ_ASM_OP_CMPEQ32(current_cpu_id, cpu_id, label)
122
+
123
+
#define RSEQ_ASM_OP_R_LOAD(var) \
124
+
REG_L RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
125
+
126
+
#define RSEQ_ASM_OP_R_STORE(var) \
127
+
REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n"
128
+
129
+
#define RSEQ_ASM_OP_R_LOAD_OFF(offset) \
130
+
"add "RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(offset) "], " \
131
+
RSEQ_ASM_TMP_REG_1 "\n" \
132
+
REG_L RSEQ_ASM_TMP_REG_1 ", (" RSEQ_ASM_TMP_REG_1 ")\n"
133
+
134
+
#define RSEQ_ASM_OP_R_ADD(count) \
135
+
"add "RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 \
136
+
", %[" __rseq_str(count) "]\n"
137
+
138
+
#define RSEQ_ASM_OP_FINAL_STORE(value, var, post_commit_label) \
139
+
RSEQ_ASM_OP_STORE(value, var) \
140
+
__rseq_str(post_commit_label) ":\n"
141
+
142
+
#define RSEQ_ASM_OP_FINAL_STORE_RELEASE(value, var, post_commit_label) \
143
+
"fence rw, w\n" \
144
+
RSEQ_ASM_OP_STORE(value, var) \
145
+
__rseq_str(post_commit_label) ":\n"
146
+
147
+
#define RSEQ_ASM_OP_R_FINAL_STORE(var, post_commit_label) \
148
+
REG_S RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(var) "]\n" \
149
+
__rseq_str(post_commit_label) ":\n"
150
+
151
+
#define RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) \
152
+
"beqz %[" __rseq_str(len) "], 333f\n" \
153
+
"mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(len) "]\n" \
154
+
"mv " RSEQ_ASM_TMP_REG_2 ", %[" __rseq_str(src) "]\n" \
155
+
"mv " RSEQ_ASM_TMP_REG_3 ", %[" __rseq_str(dst) "]\n" \
156
+
"222:\n" \
157
+
"lb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_2 ")\n" \
158
+
"sb " RSEQ_ASM_TMP_REG_4 ", 0(" RSEQ_ASM_TMP_REG_3 ")\n" \
159
+
"addi " RSEQ_ASM_TMP_REG_1 ", " RSEQ_ASM_TMP_REG_1 ", -1\n" \
160
+
"addi " RSEQ_ASM_TMP_REG_2 ", " RSEQ_ASM_TMP_REG_2 ", 1\n" \
161
+
"addi " RSEQ_ASM_TMP_REG_3 ", " RSEQ_ASM_TMP_REG_3 ", 1\n" \
162
+
"bnez " RSEQ_ASM_TMP_REG_1 ", 222b\n" \
163
+
"333:\n"
164
+
165
+
#define RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, post_commit_label) \
166
+
"mv " RSEQ_ASM_TMP_REG_1 ", %[" __rseq_str(ptr) "]\n" \
167
+
RSEQ_ASM_OP_R_ADD(off) \
168
+
REG_L RSEQ_ASM_TMP_REG_1 ", 0(" RSEQ_ASM_TMP_REG_1 ")\n" \
169
+
RSEQ_ASM_OP_R_ADD(inc) \
170
+
__rseq_str(post_commit_label) ":\n"
171
+
172
+
static inline __always_inline
173
+
int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
174
+
{
175
+
RSEQ_INJECT_C(9)
176
+
177
+
__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
178
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
179
+
#ifdef RSEQ_COMPARE_TWICE
180
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
181
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
182
+
#endif
183
+
RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
184
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
185
+
RSEQ_INJECT_ASM(3)
186
+
RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
187
+
RSEQ_INJECT_ASM(4)
188
+
#ifdef RSEQ_COMPARE_TWICE
189
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
190
+
RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
191
+
#endif
192
+
RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
193
+
RSEQ_INJECT_ASM(5)
194
+
RSEQ_ASM_DEFINE_ABORT(4, abort)
195
+
: /* gcc asm goto does not allow outputs */
196
+
: [cpu_id] "r" (cpu),
197
+
[current_cpu_id] "m" (__rseq_abi.cpu_id),
198
+
[rseq_cs] "m" (__rseq_abi.rseq_cs),
199
+
[v] "m" (*v),
200
+
[expect] "r" (expect),
201
+
[newv] "r" (newv)
202
+
RSEQ_INJECT_INPUT
203
+
: "memory", RSEQ_ASM_TMP_REG_1
204
+
RSEQ_INJECT_CLOBBER
205
+
: abort, cmpfail
206
+
#ifdef RSEQ_COMPARE_TWICE
207
+
, error1, error2
208
+
#endif
209
+
);
210
+
211
+
return 0;
212
+
abort:
213
+
RSEQ_INJECT_FAILED
214
+
return -1;
215
+
cmpfail:
216
+
return 1;
217
+
#ifdef RSEQ_COMPARE_TWICE
218
+
error1:
219
+
rseq_bug("cpu_id comparison failed");
220
+
error2:
221
+
rseq_bug("expected value comparison failed");
222
+
#endif
223
+
}
224
+
225
+
static inline __always_inline
226
+
int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
227
+
off_t voffp, intptr_t *load, int cpu)
228
+
{
229
+
RSEQ_INJECT_C(9)
230
+
231
+
__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
232
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
233
+
#ifdef RSEQ_COMPARE_TWICE
234
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
235
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
236
+
#endif
237
+
RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
238
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
239
+
RSEQ_INJECT_ASM(3)
240
+
RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[cmpfail]")
241
+
RSEQ_INJECT_ASM(4)
242
+
#ifdef RSEQ_COMPARE_TWICE
243
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
244
+
RSEQ_ASM_OP_CMPNE(v, expectnot, "%l[error2]")
245
+
#endif
246
+
RSEQ_ASM_OP_R_LOAD(v)
247
+
RSEQ_ASM_OP_R_STORE(load)
248
+
RSEQ_ASM_OP_R_LOAD_OFF(voffp)
249
+
RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
250
+
RSEQ_INJECT_ASM(5)
251
+
RSEQ_ASM_DEFINE_ABORT(4, abort)
252
+
: /* gcc asm goto does not allow outputs */
253
+
: [cpu_id] "r" (cpu),
254
+
[current_cpu_id] "m" (__rseq_abi.cpu_id),
255
+
[rseq_cs] "m" (__rseq_abi.rseq_cs),
256
+
[v] "m" (*v),
257
+
[expectnot] "r" (expectnot),
258
+
[load] "m" (*load),
259
+
[voffp] "r" (voffp)
260
+
RSEQ_INJECT_INPUT
261
+
: "memory", RSEQ_ASM_TMP_REG_1
262
+
RSEQ_INJECT_CLOBBER
263
+
: abort, cmpfail
264
+
#ifdef RSEQ_COMPARE_TWICE
265
+
, error1, error2
266
+
#endif
267
+
);
268
+
return 0;
269
+
abort:
270
+
RSEQ_INJECT_FAILED
271
+
return -1;
272
+
cmpfail:
273
+
return 1;
274
+
#ifdef RSEQ_COMPARE_TWICE
275
+
error1:
276
+
rseq_bug("cpu_id comparison failed");
277
+
error2:
278
+
rseq_bug("expected value comparison failed");
279
+
#endif
280
+
}
281
+
282
+
static inline __always_inline
283
+
int rseq_addv(intptr_t *v, intptr_t count, int cpu)
284
+
{
285
+
RSEQ_INJECT_C(9)
286
+
287
+
__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
288
+
#ifdef RSEQ_COMPARE_TWICE
289
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
290
+
#endif
291
+
RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
292
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
293
+
RSEQ_INJECT_ASM(3)
294
+
#ifdef RSEQ_COMPARE_TWICE
295
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
296
+
#endif
297
+
RSEQ_ASM_OP_R_LOAD(v)
298
+
RSEQ_ASM_OP_R_ADD(count)
299
+
RSEQ_ASM_OP_R_FINAL_STORE(v, 3)
300
+
RSEQ_INJECT_ASM(4)
301
+
RSEQ_ASM_DEFINE_ABORT(4, abort)
302
+
: /* gcc asm goto does not allow outputs */
303
+
: [cpu_id] "r" (cpu),
304
+
[current_cpu_id] "m" (__rseq_abi.cpu_id),
305
+
[rseq_cs] "m" (__rseq_abi.rseq_cs),
306
+
[v] "m" (*v),
307
+
[count] "r" (count)
308
+
RSEQ_INJECT_INPUT
309
+
: "memory", RSEQ_ASM_TMP_REG_1
310
+
RSEQ_INJECT_CLOBBER
311
+
: abort
312
+
#ifdef RSEQ_COMPARE_TWICE
313
+
, error1
314
+
#endif
315
+
);
316
+
return 0;
317
+
abort:
318
+
RSEQ_INJECT_FAILED
319
+
return -1;
320
+
#ifdef RSEQ_COMPARE_TWICE
321
+
error1:
322
+
rseq_bug("cpu_id comparison failed");
323
+
#endif
324
+
}
325
+
326
+
static inline __always_inline
327
+
int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
328
+
intptr_t *v2, intptr_t newv2,
329
+
intptr_t newv, int cpu)
330
+
{
331
+
RSEQ_INJECT_C(9)
332
+
333
+
__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
334
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
335
+
#ifdef RSEQ_COMPARE_TWICE
336
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
337
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
338
+
#endif
339
+
RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
340
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
341
+
RSEQ_INJECT_ASM(3)
342
+
RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
343
+
RSEQ_INJECT_ASM(4)
344
+
#ifdef RSEQ_COMPARE_TWICE
345
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
346
+
RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
347
+
#endif
348
+
RSEQ_ASM_OP_STORE(newv2, v2)
349
+
RSEQ_INJECT_ASM(5)
350
+
RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
351
+
RSEQ_INJECT_ASM(6)
352
+
RSEQ_ASM_DEFINE_ABORT(4, abort)
353
+
: /* gcc asm goto does not allow outputs */
354
+
: [cpu_id] "r" (cpu),
355
+
[current_cpu_id] "m" (__rseq_abi.cpu_id),
356
+
[rseq_cs] "m" (__rseq_abi.rseq_cs),
357
+
[expect] "r" (expect),
358
+
[v] "m" (*v),
359
+
[newv] "r" (newv),
360
+
[v2] "m" (*v2),
361
+
[newv2] "r" (newv2)
362
+
RSEQ_INJECT_INPUT
363
+
: "memory", RSEQ_ASM_TMP_REG_1
364
+
RSEQ_INJECT_CLOBBER
365
+
: abort, cmpfail
366
+
#ifdef RSEQ_COMPARE_TWICE
367
+
, error1, error2
368
+
#endif
369
+
);
370
+
371
+
return 0;
372
+
abort:
373
+
RSEQ_INJECT_FAILED
374
+
return -1;
375
+
cmpfail:
376
+
return 1;
377
+
#ifdef RSEQ_COMPARE_TWICE
378
+
error1:
379
+
rseq_bug("cpu_id comparison failed");
380
+
error2:
381
+
rseq_bug("expected value comparison failed");
382
+
#endif
383
+
}
384
+
385
+
static inline __always_inline
386
+
int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
387
+
intptr_t *v2, intptr_t newv2,
388
+
intptr_t newv, int cpu)
389
+
{
390
+
RSEQ_INJECT_C(9)
391
+
392
+
__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
393
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
394
+
#ifdef RSEQ_COMPARE_TWICE
395
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
396
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
397
+
#endif
398
+
RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
399
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
400
+
RSEQ_INJECT_ASM(3)
401
+
RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
402
+
RSEQ_INJECT_ASM(4)
403
+
#ifdef RSEQ_COMPARE_TWICE
404
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
405
+
RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
406
+
#endif
407
+
RSEQ_ASM_OP_STORE(newv2, v2)
408
+
RSEQ_INJECT_ASM(5)
409
+
RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
410
+
RSEQ_INJECT_ASM(6)
411
+
RSEQ_ASM_DEFINE_ABORT(4, abort)
412
+
: /* gcc asm goto does not allow outputs */
413
+
: [cpu_id] "r" (cpu),
414
+
[current_cpu_id] "m" (__rseq_abi.cpu_id),
415
+
[rseq_cs] "m" (__rseq_abi.rseq_cs),
416
+
[expect] "r" (expect),
417
+
[v] "m" (*v),
418
+
[newv] "r" (newv),
419
+
[v2] "m" (*v2),
420
+
[newv2] "r" (newv2)
421
+
RSEQ_INJECT_INPUT
422
+
: "memory", RSEQ_ASM_TMP_REG_1
423
+
RSEQ_INJECT_CLOBBER
424
+
: abort, cmpfail
425
+
#ifdef RSEQ_COMPARE_TWICE
426
+
, error1, error2
427
+
#endif
428
+
);
429
+
430
+
return 0;
431
+
abort:
432
+
RSEQ_INJECT_FAILED
433
+
return -1;
434
+
cmpfail:
435
+
return 1;
436
+
#ifdef RSEQ_COMPARE_TWICE
437
+
error1:
438
+
rseq_bug("cpu_id comparison failed");
439
+
error2:
440
+
rseq_bug("expected value comparison failed");
441
+
#endif
442
+
}
443
+
444
+
static inline __always_inline
445
+
int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
446
+
intptr_t *v2, intptr_t expect2,
447
+
intptr_t newv, int cpu)
448
+
{
449
+
RSEQ_INJECT_C(9)
450
+
451
+
__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
452
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
453
+
#ifdef RSEQ_COMPARE_TWICE
454
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
455
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
456
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error3]")
457
+
#endif
458
+
RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
459
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
460
+
RSEQ_INJECT_ASM(3)
461
+
RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
462
+
RSEQ_INJECT_ASM(4)
463
+
RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[cmpfail]")
464
+
RSEQ_INJECT_ASM(5)
465
+
#ifdef RSEQ_COMPARE_TWICE
466
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
467
+
RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
468
+
RSEQ_ASM_OP_CMPEQ(v2, expect2, "%l[error3]")
469
+
#endif
470
+
RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
471
+
RSEQ_INJECT_ASM(6)
472
+
RSEQ_ASM_DEFINE_ABORT(4, abort)
473
+
: /* gcc asm goto does not allow outputs */
474
+
: [cpu_id] "r" (cpu),
475
+
[current_cpu_id] "m" (__rseq_abi.cpu_id),
476
+
[rseq_cs] "m" (__rseq_abi.rseq_cs),
477
+
[v] "m" (*v),
478
+
[expect] "r" (expect),
479
+
[v2] "m" (*v2),
480
+
[expect2] "r" (expect2),
481
+
[newv] "r" (newv)
482
+
RSEQ_INJECT_INPUT
483
+
: "memory", RSEQ_ASM_TMP_REG_1
484
+
RSEQ_INJECT_CLOBBER
485
+
: abort, cmpfail
486
+
#ifdef RSEQ_COMPARE_TWICE
487
+
, error1, error2, error3
488
+
#endif
489
+
);
490
+
491
+
return 0;
492
+
abort:
493
+
RSEQ_INJECT_FAILED
494
+
return -1;
495
+
cmpfail:
496
+
return 1;
497
+
#ifdef RSEQ_COMPARE_TWICE
498
+
error1:
499
+
rseq_bug("cpu_id comparison failed");
500
+
error2:
501
+
rseq_bug("expected value comparison failed");
502
+
error3:
503
+
rseq_bug("2nd expected value comparison failed");
504
+
#endif
505
+
}
506
+
507
+
static inline __always_inline
508
+
int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
509
+
void *dst, void *src, size_t len,
510
+
intptr_t newv, int cpu)
511
+
{
512
+
RSEQ_INJECT_C(9)
513
+
__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
514
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
515
+
#ifdef RSEQ_COMPARE_TWICE
516
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
517
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
518
+
#endif
519
+
RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
520
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
521
+
RSEQ_INJECT_ASM(3)
522
+
RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
523
+
RSEQ_INJECT_ASM(4)
524
+
#ifdef RSEQ_COMPARE_TWICE
525
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
526
+
RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
527
+
#endif
528
+
RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
529
+
RSEQ_INJECT_ASM(5)
530
+
RSEQ_ASM_OP_FINAL_STORE(newv, v, 3)
531
+
RSEQ_INJECT_ASM(6)
532
+
RSEQ_ASM_DEFINE_ABORT(4, abort)
533
+
: /* gcc asm goto does not allow outputs */
534
+
: [cpu_id] "r" (cpu),
535
+
[current_cpu_id] "m" (__rseq_abi.cpu_id),
536
+
[rseq_cs] "m" (__rseq_abi.rseq_cs),
537
+
[expect] "r" (expect),
538
+
[v] "m" (*v),
539
+
[newv] "r" (newv),
540
+
[dst] "r" (dst),
541
+
[src] "r" (src),
542
+
[len] "r" (len)
543
+
RSEQ_INJECT_INPUT
544
+
: "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
545
+
RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
546
+
RSEQ_INJECT_CLOBBER
547
+
: abort, cmpfail
548
+
#ifdef RSEQ_COMPARE_TWICE
549
+
, error1, error2
550
+
#endif
551
+
);
552
+
553
+
return 0;
554
+
abort:
555
+
RSEQ_INJECT_FAILED
556
+
return -1;
557
+
cmpfail:
558
+
return 1;
559
+
#ifdef RSEQ_COMPARE_TWICE
560
+
error1:
561
+
rseq_bug("cpu_id comparison failed");
562
+
error2:
563
+
rseq_bug("expected value comparison failed");
564
+
#endif
565
+
}
566
+
567
+
static inline __always_inline
568
+
int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
569
+
void *dst, void *src, size_t len,
570
+
intptr_t newv, int cpu)
571
+
{
572
+
RSEQ_INJECT_C(9)
573
+
574
+
__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
575
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[cmpfail]")
576
+
#ifdef RSEQ_COMPARE_TWICE
577
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
578
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error2]")
579
+
#endif
580
+
RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
581
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
582
+
RSEQ_INJECT_ASM(3)
583
+
RSEQ_ASM_OP_CMPEQ(v, expect, "%l[cmpfail]")
584
+
RSEQ_INJECT_ASM(4)
585
+
#ifdef RSEQ_COMPARE_TWICE
586
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
587
+
RSEQ_ASM_OP_CMPEQ(v, expect, "%l[error2]")
588
+
#endif
589
+
RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len)
590
+
RSEQ_INJECT_ASM(5)
591
+
RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3)
592
+
RSEQ_INJECT_ASM(6)
593
+
RSEQ_ASM_DEFINE_ABORT(4, abort)
594
+
: /* gcc asm goto does not allow outputs */
595
+
: [cpu_id] "r" (cpu),
596
+
[current_cpu_id] "m" (__rseq_abi.cpu_id),
597
+
[rseq_cs] "m" (__rseq_abi.rseq_cs),
598
+
[expect] "r" (expect),
599
+
[v] "m" (*v),
600
+
[newv] "r" (newv),
601
+
[dst] "r" (dst),
602
+
[src] "r" (src),
603
+
[len] "r" (len)
604
+
RSEQ_INJECT_INPUT
605
+
: "memory", RSEQ_ASM_TMP_REG_1, RSEQ_ASM_TMP_REG_2,
606
+
RSEQ_ASM_TMP_REG_3, RSEQ_ASM_TMP_REG_4
607
+
RSEQ_INJECT_CLOBBER
608
+
: abort, cmpfail
609
+
#ifdef RSEQ_COMPARE_TWICE
610
+
, error1, error2
611
+
#endif
612
+
);
613
+
614
+
return 0;
615
+
abort:
616
+
RSEQ_INJECT_FAILED
617
+
return -1;
618
+
cmpfail:
619
+
return 1;
620
+
#ifdef RSEQ_COMPARE_TWICE
621
+
error1:
622
+
rseq_bug("cpu_id comparison failed");
623
+
error2:
624
+
rseq_bug("expected value comparison failed");
625
+
#endif
626
+
}
627
+
628
+
#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
629
+
630
+
/*
631
+
* pval = *(ptr+off)
632
+
* *pval += inc;
633
+
*/
634
+
static inline __always_inline
635
+
int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
636
+
{
637
+
RSEQ_INJECT_C(9)
638
+
639
+
__asm__ __volatile__ goto(RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f)
640
+
#ifdef RSEQ_COMPARE_TWICE
641
+
RSEQ_ASM_DEFINE_EXIT_POINT(2f, "%l[error1]")
642
+
#endif
643
+
RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs)
644
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
645
+
RSEQ_INJECT_ASM(3)
646
+
#ifdef RSEQ_COMPARE_TWICE
647
+
RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, "%l[error1]")
648
+
#endif
649
+
RSEQ_ASM_OP_R_DEREF_ADDV(ptr, off, 3)
650
+
RSEQ_INJECT_ASM(4)
651
+
RSEQ_ASM_DEFINE_ABORT(4, abort)
652
+
: /* gcc asm goto does not allow outputs */
653
+
: [cpu_id] "r" (cpu),
654
+
[current_cpu_id] "m" (__rseq_abi.cpu_id),
655
+
[rseq_cs] "m" (__rseq_abi.rseq_cs),
656
+
[ptr] "r" (ptr),
657
+
[off] "er" (off),
658
+
[inc] "er" (inc)
659
+
RSEQ_INJECT_INPUT
660
+
: "memory", RSEQ_ASM_TMP_REG_1
661
+
RSEQ_INJECT_CLOBBER
662
+
: abort
663
+
#ifdef RSEQ_COMPARE_TWICE
664
+
, error1
665
+
#endif
666
+
);
667
+
return 0;
668
+
abort:
669
+
RSEQ_INJECT_FAILED
670
+
return -1;
671
+
#ifdef RSEQ_COMPARE_TWICE
672
+
error1:
673
+
rseq_bug("cpu_id comparison failed");
674
+
#endif
675
+
}
676
+
677
+
#endif /* !RSEQ_SKIP_FASTPATH */