tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-arm.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8/*
9 * RSEQ_SIG uses the udf A32 instruction with an uncommon immediate operand
10 * value 0x5de3. This traps if user-space reaches this instruction by mistake,
11 * and the uncommon operand ensures the kernel does not move the instruction
12 * pointer to attacker-controlled code on rseq abort.
13 *
14 * The instruction pattern in the A32 instruction set is:
15 *
16 * e7f5def3 udf #24035 ; 0x5de3
17 *
18 * This translates to the following instruction pattern in the T16 instruction
19 * set:
20 *
21 * little endian:
22 * def3 udf #243 ; 0xf3
23 * e7f5 b.n <7f5>
24 *
25 * pre-ARMv6 big endian code:
26 * e7f5 b.n <7f5>
27 * def3 udf #243 ; 0xf3
28 *
29 * ARMv6+ -mbig-endian generates mixed endianness code vs data: little-endian
30 * code and big-endian data. Ensure the RSEQ_SIG data signature matches code
31 * endianness. Prior to ARMv6, -mbig-endian generates big-endian code and data
32 * (which match), so there is no need to reverse the endianness of the data
33 * representation of the signature. However, the choice between BE32 and BE8
34 * is done by the linker, so we cannot know whether code and data endianness
35 * will be mixed before the linker is invoked.
36 */
37
38#define RSEQ_SIG_CODE 0xe7f5def3
39
40#ifndef __ASSEMBLER__
41
42#define RSEQ_SIG_DATA \
43 ({ \
44 int sig; \
45 asm volatile ("b 2f\n\t" \
46 "1: .inst " __rseq_str(RSEQ_SIG_CODE) "\n\t" \
47 "2:\n\t" \
48 "ldr %[sig], 1b\n\t" \
49 : [sig] "=r" (sig)); \
50 sig; \
51 })
52
53#define RSEQ_SIG RSEQ_SIG_DATA
54
55#endif
56
57#define rseq_smp_mb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
58#define rseq_smp_rmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
59#define rseq_smp_wmb() __asm__ __volatile__ ("dmb" ::: "memory", "cc")
60
61#define rseq_smp_load_acquire(p) \
62__extension__ ({ \
63 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
64 rseq_smp_mb(); \
65 ____p1; \
66})
67
68#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
69
70#define rseq_smp_store_release(p, v) \
71do { \
72 rseq_smp_mb(); \
73 RSEQ_WRITE_ONCE(*p, v); \
74} while (0)
75
76#ifdef RSEQ_SKIP_FASTPATH
77#include "rseq-skip.h"
78#else /* !RSEQ_SKIP_FASTPATH */
79
80#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, start_ip, \
81 post_commit_offset, abort_ip) \
82 ".pushsection __rseq_cs, \"aw\"\n\t" \
83 ".balign 32\n\t" \
84 __rseq_str(label) ":\n\t" \
85 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
86 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
87 ".popsection\n\t" \
88 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
89 ".word " __rseq_str(label) "b, 0x0\n\t" \
90 ".popsection\n\t"
91
92#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
93 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
94 (post_commit_ip - start_ip), abort_ip)
95
96/*
97 * Exit points of a rseq critical section consist of all instructions outside
98 * of the critical section where a critical section can either branch to or
99 * reach through the normal course of its execution. The abort IP and the
100 * post-commit IP are already part of the __rseq_cs section and should not be
101 * explicitly defined as additional exit points. Knowing all exit points is
102 * useful to assist debuggers stepping over the critical section.
103 */
104#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
105 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
106 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
107 ".popsection\n\t"
108
109#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
110 RSEQ_INJECT_ASM(1) \
111 "adr r0, " __rseq_str(cs_label) "\n\t" \
112 "str r0, %[" __rseq_str(rseq_cs) "]\n\t" \
113 __rseq_str(label) ":\n\t"
114
115#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
116 RSEQ_INJECT_ASM(2) \
117 "ldr r0, %[" __rseq_str(current_cpu_id) "]\n\t" \
118 "cmp %[" __rseq_str(cpu_id) "], r0\n\t" \
119 "bne " __rseq_str(label) "\n\t"
120
121#define __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
122 abort_label, version, flags, \
123 start_ip, post_commit_offset, abort_ip) \
124 ".balign 32\n\t" \
125 __rseq_str(table_label) ":\n\t" \
126 ".word " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
127 ".word " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
128 ".arm\n\t" \
129 ".inst " __rseq_str(RSEQ_SIG_CODE) "\n\t" \
130 __rseq_str(label) ":\n\t" \
131 teardown \
132 "b %l[" __rseq_str(abort_label) "]\n\t"
133
134#define RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, abort_label, \
135 start_ip, post_commit_ip, abort_ip) \
136 __RSEQ_ASM_DEFINE_ABORT(table_label, label, teardown, \
137 abort_label, 0x0, 0x0, start_ip, \
138 (post_commit_ip - start_ip), abort_ip)
139
140#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
141 __rseq_str(label) ":\n\t" \
142 teardown \
143 "b %l[" __rseq_str(cmpfail_label) "]\n\t"
144
145#define rseq_workaround_gcc_asm_size_guess() __asm__ __volatile__("")
146
147static inline __attribute__((always_inline))
148int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
149{
150 RSEQ_INJECT_C(9)
151
152 rseq_workaround_gcc_asm_size_guess();
153 __asm__ __volatile__ goto (
154 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
155 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
156#ifdef RSEQ_COMPARE_TWICE
157 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
158 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
159#endif
160 /* Start rseq by storing table entry pointer into rseq_cs. */
161 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
162 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
163 RSEQ_INJECT_ASM(3)
164 "ldr r0, %[v]\n\t"
165 "cmp %[expect], r0\n\t"
166 "bne %l[cmpfail]\n\t"
167 RSEQ_INJECT_ASM(4)
168#ifdef RSEQ_COMPARE_TWICE
169 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
170 "ldr r0, %[v]\n\t"
171 "cmp %[expect], r0\n\t"
172 "bne %l[error2]\n\t"
173#endif
174 /* final store */
175 "str %[newv], %[v]\n\t"
176 "2:\n\t"
177 RSEQ_INJECT_ASM(5)
178 "b 5f\n\t"
179 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
180 "5:\n\t"
181 : /* gcc asm goto does not allow outputs */
182 : [cpu_id] "r" (cpu),
183 [current_cpu_id] "m" (__rseq_abi.cpu_id),
184 [rseq_cs] "m" (__rseq_abi.rseq_cs),
185 [v] "m" (*v),
186 [expect] "r" (expect),
187 [newv] "r" (newv)
188 RSEQ_INJECT_INPUT
189 : "r0", "memory", "cc"
190 RSEQ_INJECT_CLOBBER
191 : abort, cmpfail
192#ifdef RSEQ_COMPARE_TWICE
193 , error1, error2
194#endif
195 );
196 rseq_workaround_gcc_asm_size_guess();
197 return 0;
198abort:
199 rseq_workaround_gcc_asm_size_guess();
200 RSEQ_INJECT_FAILED
201 return -1;
202cmpfail:
203 rseq_workaround_gcc_asm_size_guess();
204 return 1;
205#ifdef RSEQ_COMPARE_TWICE
206error1:
207 rseq_bug("cpu_id comparison failed");
208error2:
209 rseq_bug("expected value comparison failed");
210#endif
211}
212
213static inline __attribute__((always_inline))
214int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
215 off_t voffp, intptr_t *load, int cpu)
216{
217 RSEQ_INJECT_C(9)
218
219 rseq_workaround_gcc_asm_size_guess();
220 __asm__ __volatile__ goto (
221 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
222 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
223#ifdef RSEQ_COMPARE_TWICE
224 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
225 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
226#endif
227 /* Start rseq by storing table entry pointer into rseq_cs. */
228 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
229 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
230 RSEQ_INJECT_ASM(3)
231 "ldr r0, %[v]\n\t"
232 "cmp %[expectnot], r0\n\t"
233 "beq %l[cmpfail]\n\t"
234 RSEQ_INJECT_ASM(4)
235#ifdef RSEQ_COMPARE_TWICE
236 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
237 "ldr r0, %[v]\n\t"
238 "cmp %[expectnot], r0\n\t"
239 "beq %l[error2]\n\t"
240#endif
241 "str r0, %[load]\n\t"
242 "add r0, %[voffp]\n\t"
243 "ldr r0, [r0]\n\t"
244 /* final store */
245 "str r0, %[v]\n\t"
246 "2:\n\t"
247 RSEQ_INJECT_ASM(5)
248 "b 5f\n\t"
249 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
250 "5:\n\t"
251 : /* gcc asm goto does not allow outputs */
252 : [cpu_id] "r" (cpu),
253 [current_cpu_id] "m" (__rseq_abi.cpu_id),
254 [rseq_cs] "m" (__rseq_abi.rseq_cs),
255 /* final store input */
256 [v] "m" (*v),
257 [expectnot] "r" (expectnot),
258 [voffp] "Ir" (voffp),
259 [load] "m" (*load)
260 RSEQ_INJECT_INPUT
261 : "r0", "memory", "cc"
262 RSEQ_INJECT_CLOBBER
263 : abort, cmpfail
264#ifdef RSEQ_COMPARE_TWICE
265 , error1, error2
266#endif
267 );
268 rseq_workaround_gcc_asm_size_guess();
269 return 0;
270abort:
271 rseq_workaround_gcc_asm_size_guess();
272 RSEQ_INJECT_FAILED
273 return -1;
274cmpfail:
275 rseq_workaround_gcc_asm_size_guess();
276 return 1;
277#ifdef RSEQ_COMPARE_TWICE
278error1:
279 rseq_bug("cpu_id comparison failed");
280error2:
281 rseq_bug("expected value comparison failed");
282#endif
283}
284
285static inline __attribute__((always_inline))
286int rseq_addv(intptr_t *v, intptr_t count, int cpu)
287{
288 RSEQ_INJECT_C(9)
289
290 rseq_workaround_gcc_asm_size_guess();
291 __asm__ __volatile__ goto (
292 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
293#ifdef RSEQ_COMPARE_TWICE
294 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
295#endif
296 /* Start rseq by storing table entry pointer into rseq_cs. */
297 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
298 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
299 RSEQ_INJECT_ASM(3)
300#ifdef RSEQ_COMPARE_TWICE
301 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
302#endif
303 "ldr r0, %[v]\n\t"
304 "add r0, %[count]\n\t"
305 /* final store */
306 "str r0, %[v]\n\t"
307 "2:\n\t"
308 RSEQ_INJECT_ASM(4)
309 "b 5f\n\t"
310 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
311 "5:\n\t"
312 : /* gcc asm goto does not allow outputs */
313 : [cpu_id] "r" (cpu),
314 [current_cpu_id] "m" (__rseq_abi.cpu_id),
315 [rseq_cs] "m" (__rseq_abi.rseq_cs),
316 [v] "m" (*v),
317 [count] "Ir" (count)
318 RSEQ_INJECT_INPUT
319 : "r0", "memory", "cc"
320 RSEQ_INJECT_CLOBBER
321 : abort
322#ifdef RSEQ_COMPARE_TWICE
323 , error1
324#endif
325 );
326 rseq_workaround_gcc_asm_size_guess();
327 return 0;
328abort:
329 rseq_workaround_gcc_asm_size_guess();
330 RSEQ_INJECT_FAILED
331 return -1;
332#ifdef RSEQ_COMPARE_TWICE
333error1:
334 rseq_bug("cpu_id comparison failed");
335#endif
336}
337
338static inline __attribute__((always_inline))
339int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
340 intptr_t *v2, intptr_t newv2,
341 intptr_t newv, int cpu)
342{
343 RSEQ_INJECT_C(9)
344
345 rseq_workaround_gcc_asm_size_guess();
346 __asm__ __volatile__ goto (
347 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
348 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
349#ifdef RSEQ_COMPARE_TWICE
350 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
351 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
352#endif
353 /* Start rseq by storing table entry pointer into rseq_cs. */
354 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
355 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
356 RSEQ_INJECT_ASM(3)
357 "ldr r0, %[v]\n\t"
358 "cmp %[expect], r0\n\t"
359 "bne %l[cmpfail]\n\t"
360 RSEQ_INJECT_ASM(4)
361#ifdef RSEQ_COMPARE_TWICE
362 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
363 "ldr r0, %[v]\n\t"
364 "cmp %[expect], r0\n\t"
365 "bne %l[error2]\n\t"
366#endif
367 /* try store */
368 "str %[newv2], %[v2]\n\t"
369 RSEQ_INJECT_ASM(5)
370 /* final store */
371 "str %[newv], %[v]\n\t"
372 "2:\n\t"
373 RSEQ_INJECT_ASM(6)
374 "b 5f\n\t"
375 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
376 "5:\n\t"
377 : /* gcc asm goto does not allow outputs */
378 : [cpu_id] "r" (cpu),
379 [current_cpu_id] "m" (__rseq_abi.cpu_id),
380 [rseq_cs] "m" (__rseq_abi.rseq_cs),
381 /* try store input */
382 [v2] "m" (*v2),
383 [newv2] "r" (newv2),
384 /* final store input */
385 [v] "m" (*v),
386 [expect] "r" (expect),
387 [newv] "r" (newv)
388 RSEQ_INJECT_INPUT
389 : "r0", "memory", "cc"
390 RSEQ_INJECT_CLOBBER
391 : abort, cmpfail
392#ifdef RSEQ_COMPARE_TWICE
393 , error1, error2
394#endif
395 );
396 rseq_workaround_gcc_asm_size_guess();
397 return 0;
398abort:
399 rseq_workaround_gcc_asm_size_guess();
400 RSEQ_INJECT_FAILED
401 return -1;
402cmpfail:
403 rseq_workaround_gcc_asm_size_guess();
404 return 1;
405#ifdef RSEQ_COMPARE_TWICE
406error1:
407 rseq_bug("cpu_id comparison failed");
408error2:
409 rseq_bug("expected value comparison failed");
410#endif
411}
412
413static inline __attribute__((always_inline))
414int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
415 intptr_t *v2, intptr_t newv2,
416 intptr_t newv, int cpu)
417{
418 RSEQ_INJECT_C(9)
419
420 rseq_workaround_gcc_asm_size_guess();
421 __asm__ __volatile__ goto (
422 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
423 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
424#ifdef RSEQ_COMPARE_TWICE
425 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
426 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
427#endif
428 /* Start rseq by storing table entry pointer into rseq_cs. */
429 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
430 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
431 RSEQ_INJECT_ASM(3)
432 "ldr r0, %[v]\n\t"
433 "cmp %[expect], r0\n\t"
434 "bne %l[cmpfail]\n\t"
435 RSEQ_INJECT_ASM(4)
436#ifdef RSEQ_COMPARE_TWICE
437 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
438 "ldr r0, %[v]\n\t"
439 "cmp %[expect], r0\n\t"
440 "bne %l[error2]\n\t"
441#endif
442 /* try store */
443 "str %[newv2], %[v2]\n\t"
444 RSEQ_INJECT_ASM(5)
445 "dmb\n\t" /* full mb provides store-release */
446 /* final store */
447 "str %[newv], %[v]\n\t"
448 "2:\n\t"
449 RSEQ_INJECT_ASM(6)
450 "b 5f\n\t"
451 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
452 "5:\n\t"
453 : /* gcc asm goto does not allow outputs */
454 : [cpu_id] "r" (cpu),
455 [current_cpu_id] "m" (__rseq_abi.cpu_id),
456 [rseq_cs] "m" (__rseq_abi.rseq_cs),
457 /* try store input */
458 [v2] "m" (*v2),
459 [newv2] "r" (newv2),
460 /* final store input */
461 [v] "m" (*v),
462 [expect] "r" (expect),
463 [newv] "r" (newv)
464 RSEQ_INJECT_INPUT
465 : "r0", "memory", "cc"
466 RSEQ_INJECT_CLOBBER
467 : abort, cmpfail
468#ifdef RSEQ_COMPARE_TWICE
469 , error1, error2
470#endif
471 );
472 rseq_workaround_gcc_asm_size_guess();
473 return 0;
474abort:
475 rseq_workaround_gcc_asm_size_guess();
476 RSEQ_INJECT_FAILED
477 return -1;
478cmpfail:
479 rseq_workaround_gcc_asm_size_guess();
480 return 1;
481#ifdef RSEQ_COMPARE_TWICE
482error1:
483 rseq_bug("cpu_id comparison failed");
484error2:
485 rseq_bug("expected value comparison failed");
486#endif
487}
488
489static inline __attribute__((always_inline))
490int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
491 intptr_t *v2, intptr_t expect2,
492 intptr_t newv, int cpu)
493{
494 RSEQ_INJECT_C(9)
495
496 rseq_workaround_gcc_asm_size_guess();
497 __asm__ __volatile__ goto (
498 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
499 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
500#ifdef RSEQ_COMPARE_TWICE
501 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
502 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
503 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
504#endif
505 /* Start rseq by storing table entry pointer into rseq_cs. */
506 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
507 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
508 RSEQ_INJECT_ASM(3)
509 "ldr r0, %[v]\n\t"
510 "cmp %[expect], r0\n\t"
511 "bne %l[cmpfail]\n\t"
512 RSEQ_INJECT_ASM(4)
513 "ldr r0, %[v2]\n\t"
514 "cmp %[expect2], r0\n\t"
515 "bne %l[cmpfail]\n\t"
516 RSEQ_INJECT_ASM(5)
517#ifdef RSEQ_COMPARE_TWICE
518 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
519 "ldr r0, %[v]\n\t"
520 "cmp %[expect], r0\n\t"
521 "bne %l[error2]\n\t"
522 "ldr r0, %[v2]\n\t"
523 "cmp %[expect2], r0\n\t"
524 "bne %l[error3]\n\t"
525#endif
526 /* final store */
527 "str %[newv], %[v]\n\t"
528 "2:\n\t"
529 RSEQ_INJECT_ASM(6)
530 "b 5f\n\t"
531 RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f)
532 "5:\n\t"
533 : /* gcc asm goto does not allow outputs */
534 : [cpu_id] "r" (cpu),
535 [current_cpu_id] "m" (__rseq_abi.cpu_id),
536 [rseq_cs] "m" (__rseq_abi.rseq_cs),
537 /* cmp2 input */
538 [v2] "m" (*v2),
539 [expect2] "r" (expect2),
540 /* final store input */
541 [v] "m" (*v),
542 [expect] "r" (expect),
543 [newv] "r" (newv)
544 RSEQ_INJECT_INPUT
545 : "r0", "memory", "cc"
546 RSEQ_INJECT_CLOBBER
547 : abort, cmpfail
548#ifdef RSEQ_COMPARE_TWICE
549 , error1, error2, error3
550#endif
551 );
552 rseq_workaround_gcc_asm_size_guess();
553 return 0;
554abort:
555 rseq_workaround_gcc_asm_size_guess();
556 RSEQ_INJECT_FAILED
557 return -1;
558cmpfail:
559 rseq_workaround_gcc_asm_size_guess();
560 return 1;
561#ifdef RSEQ_COMPARE_TWICE
562error1:
563 rseq_bug("cpu_id comparison failed");
564error2:
565 rseq_bug("1st expected value comparison failed");
566error3:
567 rseq_bug("2nd expected value comparison failed");
568#endif
569}
570
571static inline __attribute__((always_inline))
572int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
573 void *dst, void *src, size_t len,
574 intptr_t newv, int cpu)
575{
576 uint32_t rseq_scratch[3];
577
578 RSEQ_INJECT_C(9)
579
580 rseq_workaround_gcc_asm_size_guess();
581 __asm__ __volatile__ goto (
582 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
583 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
584#ifdef RSEQ_COMPARE_TWICE
585 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
586 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
587#endif
588 "str %[src], %[rseq_scratch0]\n\t"
589 "str %[dst], %[rseq_scratch1]\n\t"
590 "str %[len], %[rseq_scratch2]\n\t"
591 /* Start rseq by storing table entry pointer into rseq_cs. */
592 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
593 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
594 RSEQ_INJECT_ASM(3)
595 "ldr r0, %[v]\n\t"
596 "cmp %[expect], r0\n\t"
597 "bne 5f\n\t"
598 RSEQ_INJECT_ASM(4)
599#ifdef RSEQ_COMPARE_TWICE
600 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
601 "ldr r0, %[v]\n\t"
602 "cmp %[expect], r0\n\t"
603 "bne 7f\n\t"
604#endif
605 /* try memcpy */
606 "cmp %[len], #0\n\t" \
607 "beq 333f\n\t" \
608 "222:\n\t" \
609 "ldrb %%r0, [%[src]]\n\t" \
610 "strb %%r0, [%[dst]]\n\t" \
611 "adds %[src], #1\n\t" \
612 "adds %[dst], #1\n\t" \
613 "subs %[len], #1\n\t" \
614 "bne 222b\n\t" \
615 "333:\n\t" \
616 RSEQ_INJECT_ASM(5)
617 /* final store */
618 "str %[newv], %[v]\n\t"
619 "2:\n\t"
620 RSEQ_INJECT_ASM(6)
621 /* teardown */
622 "ldr %[len], %[rseq_scratch2]\n\t"
623 "ldr %[dst], %[rseq_scratch1]\n\t"
624 "ldr %[src], %[rseq_scratch0]\n\t"
625 "b 8f\n\t"
626 RSEQ_ASM_DEFINE_ABORT(3, 4,
627 /* teardown */
628 "ldr %[len], %[rseq_scratch2]\n\t"
629 "ldr %[dst], %[rseq_scratch1]\n\t"
630 "ldr %[src], %[rseq_scratch0]\n\t",
631 abort, 1b, 2b, 4f)
632 RSEQ_ASM_DEFINE_CMPFAIL(5,
633 /* teardown */
634 "ldr %[len], %[rseq_scratch2]\n\t"
635 "ldr %[dst], %[rseq_scratch1]\n\t"
636 "ldr %[src], %[rseq_scratch0]\n\t",
637 cmpfail)
638#ifdef RSEQ_COMPARE_TWICE
639 RSEQ_ASM_DEFINE_CMPFAIL(6,
640 /* teardown */
641 "ldr %[len], %[rseq_scratch2]\n\t"
642 "ldr %[dst], %[rseq_scratch1]\n\t"
643 "ldr %[src], %[rseq_scratch0]\n\t",
644 error1)
645 RSEQ_ASM_DEFINE_CMPFAIL(7,
646 /* teardown */
647 "ldr %[len], %[rseq_scratch2]\n\t"
648 "ldr %[dst], %[rseq_scratch1]\n\t"
649 "ldr %[src], %[rseq_scratch0]\n\t",
650 error2)
651#endif
652 "8:\n\t"
653 : /* gcc asm goto does not allow outputs */
654 : [cpu_id] "r" (cpu),
655 [current_cpu_id] "m" (__rseq_abi.cpu_id),
656 [rseq_cs] "m" (__rseq_abi.rseq_cs),
657 /* final store input */
658 [v] "m" (*v),
659 [expect] "r" (expect),
660 [newv] "r" (newv),
661 /* try memcpy input */
662 [dst] "r" (dst),
663 [src] "r" (src),
664 [len] "r" (len),
665 [rseq_scratch0] "m" (rseq_scratch[0]),
666 [rseq_scratch1] "m" (rseq_scratch[1]),
667 [rseq_scratch2] "m" (rseq_scratch[2])
668 RSEQ_INJECT_INPUT
669 : "r0", "memory", "cc"
670 RSEQ_INJECT_CLOBBER
671 : abort, cmpfail
672#ifdef RSEQ_COMPARE_TWICE
673 , error1, error2
674#endif
675 );
676 rseq_workaround_gcc_asm_size_guess();
677 return 0;
678abort:
679 rseq_workaround_gcc_asm_size_guess();
680 RSEQ_INJECT_FAILED
681 return -1;
682cmpfail:
683 rseq_workaround_gcc_asm_size_guess();
684 return 1;
685#ifdef RSEQ_COMPARE_TWICE
686error1:
687 rseq_workaround_gcc_asm_size_guess();
688 rseq_bug("cpu_id comparison failed");
689error2:
690 rseq_workaround_gcc_asm_size_guess();
691 rseq_bug("expected value comparison failed");
692#endif
693}
694
695static inline __attribute__((always_inline))
696int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
697 void *dst, void *src, size_t len,
698 intptr_t newv, int cpu)
699{
700 uint32_t rseq_scratch[3];
701
702 RSEQ_INJECT_C(9)
703
704 rseq_workaround_gcc_asm_size_guess();
705 __asm__ __volatile__ goto (
706 RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */
707 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
708#ifdef RSEQ_COMPARE_TWICE
709 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
710 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
711#endif
712 "str %[src], %[rseq_scratch0]\n\t"
713 "str %[dst], %[rseq_scratch1]\n\t"
714 "str %[len], %[rseq_scratch2]\n\t"
715 /* Start rseq by storing table entry pointer into rseq_cs. */
716 RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs)
717 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
718 RSEQ_INJECT_ASM(3)
719 "ldr r0, %[v]\n\t"
720 "cmp %[expect], r0\n\t"
721 "bne 5f\n\t"
722 RSEQ_INJECT_ASM(4)
723#ifdef RSEQ_COMPARE_TWICE
724 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
725 "ldr r0, %[v]\n\t"
726 "cmp %[expect], r0\n\t"
727 "bne 7f\n\t"
728#endif
729 /* try memcpy */
730 "cmp %[len], #0\n\t" \
731 "beq 333f\n\t" \
732 "222:\n\t" \
733 "ldrb %%r0, [%[src]]\n\t" \
734 "strb %%r0, [%[dst]]\n\t" \
735 "adds %[src], #1\n\t" \
736 "adds %[dst], #1\n\t" \
737 "subs %[len], #1\n\t" \
738 "bne 222b\n\t" \
739 "333:\n\t" \
740 RSEQ_INJECT_ASM(5)
741 "dmb\n\t" /* full mb provides store-release */
742 /* final store */
743 "str %[newv], %[v]\n\t"
744 "2:\n\t"
745 RSEQ_INJECT_ASM(6)
746 /* teardown */
747 "ldr %[len], %[rseq_scratch2]\n\t"
748 "ldr %[dst], %[rseq_scratch1]\n\t"
749 "ldr %[src], %[rseq_scratch0]\n\t"
750 "b 8f\n\t"
751 RSEQ_ASM_DEFINE_ABORT(3, 4,
752 /* teardown */
753 "ldr %[len], %[rseq_scratch2]\n\t"
754 "ldr %[dst], %[rseq_scratch1]\n\t"
755 "ldr %[src], %[rseq_scratch0]\n\t",
756 abort, 1b, 2b, 4f)
757 RSEQ_ASM_DEFINE_CMPFAIL(5,
758 /* teardown */
759 "ldr %[len], %[rseq_scratch2]\n\t"
760 "ldr %[dst], %[rseq_scratch1]\n\t"
761 "ldr %[src], %[rseq_scratch0]\n\t",
762 cmpfail)
763#ifdef RSEQ_COMPARE_TWICE
764 RSEQ_ASM_DEFINE_CMPFAIL(6,
765 /* teardown */
766 "ldr %[len], %[rseq_scratch2]\n\t"
767 "ldr %[dst], %[rseq_scratch1]\n\t"
768 "ldr %[src], %[rseq_scratch0]\n\t",
769 error1)
770 RSEQ_ASM_DEFINE_CMPFAIL(7,
771 /* teardown */
772 "ldr %[len], %[rseq_scratch2]\n\t"
773 "ldr %[dst], %[rseq_scratch1]\n\t"
774 "ldr %[src], %[rseq_scratch0]\n\t",
775 error2)
776#endif
777 "8:\n\t"
778 : /* gcc asm goto does not allow outputs */
779 : [cpu_id] "r" (cpu),
780 [current_cpu_id] "m" (__rseq_abi.cpu_id),
781 [rseq_cs] "m" (__rseq_abi.rseq_cs),
782 /* final store input */
783 [v] "m" (*v),
784 [expect] "r" (expect),
785 [newv] "r" (newv),
786 /* try memcpy input */
787 [dst] "r" (dst),
788 [src] "r" (src),
789 [len] "r" (len),
790 [rseq_scratch0] "m" (rseq_scratch[0]),
791 [rseq_scratch1] "m" (rseq_scratch[1]),
792 [rseq_scratch2] "m" (rseq_scratch[2])
793 RSEQ_INJECT_INPUT
794 : "r0", "memory", "cc"
795 RSEQ_INJECT_CLOBBER
796 : abort, cmpfail
797#ifdef RSEQ_COMPARE_TWICE
798 , error1, error2
799#endif
800 );
801 rseq_workaround_gcc_asm_size_guess();
802 return 0;
803abort:
804 rseq_workaround_gcc_asm_size_guess();
805 RSEQ_INJECT_FAILED
806 return -1;
807cmpfail:
808 rseq_workaround_gcc_asm_size_guess();
809 return 1;
810#ifdef RSEQ_COMPARE_TWICE
811error1:
812 rseq_workaround_gcc_asm_size_guess();
813 rseq_bug("cpu_id comparison failed");
814error2:
815 rseq_workaround_gcc_asm_size_guess();
816 rseq_bug("expected value comparison failed");
817#endif
818}
819
820#endif /* !RSEQ_SKIP_FASTPATH */