tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-x86.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8#include <stdint.h>
9
10/*
11 * RSEQ_SIG is used with the following reserved undefined instructions, which
12 * trap in user-space:
13 *
14 * x86-32: 0f b9 3d 53 30 05 53 ud1 0x53053053,%edi
15 * x86-64: 0f b9 3d 53 30 05 53 ud1 0x53053053(%rip),%edi
16 */
17#define RSEQ_SIG 0x53053053
18
19/*
20 * Due to a compiler optimization bug in gcc-8 with asm goto and TLS asm input
21 * operands, we cannot use "m" input operands, and rather pass the __rseq_abi
22 * address through a "r" input operand.
23 */
24
25/* Offset of cpu_id and rseq_cs fields in struct rseq. */
26#define RSEQ_CPU_ID_OFFSET 4
27#define RSEQ_CS_OFFSET 8
28
29#ifdef __x86_64__
30
31#define rseq_smp_mb() \
32 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
33#define rseq_smp_rmb() rseq_barrier()
34#define rseq_smp_wmb() rseq_barrier()
35
36#define rseq_smp_load_acquire(p) \
37__extension__ ({ \
38 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
39 rseq_barrier(); \
40 ____p1; \
41})
42
43#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
44
45#define rseq_smp_store_release(p, v) \
46do { \
47 rseq_barrier(); \
48 RSEQ_WRITE_ONCE(*p, v); \
49} while (0)
50
51#ifdef RSEQ_SKIP_FASTPATH
52#include "rseq-skip.h"
53#else /* !RSEQ_SKIP_FASTPATH */
54
55#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
56 start_ip, post_commit_offset, abort_ip) \
57 ".pushsection __rseq_cs, \"aw\"\n\t" \
58 ".balign 32\n\t" \
59 __rseq_str(label) ":\n\t" \
60 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
61 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
62 ".popsection\n\t" \
63 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
64 ".quad " __rseq_str(label) "b\n\t" \
65 ".popsection\n\t"
66
67
68#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
69 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
70 (post_commit_ip - start_ip), abort_ip)
71
72/*
73 * Exit points of a rseq critical section consist of all instructions outside
74 * of the critical section where a critical section can either branch to or
75 * reach through the normal course of its execution. The abort IP and the
76 * post-commit IP are already part of the __rseq_cs section and should not be
77 * explicitly defined as additional exit points. Knowing all exit points is
78 * useful to assist debuggers stepping over the critical section.
79 */
80#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
81 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
82 ".quad " __rseq_str(start_ip) ", " __rseq_str(exit_ip) "\n\t" \
83 ".popsection\n\t"
84
85#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
86 RSEQ_INJECT_ASM(1) \
87 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
88 "movq %%rax, " __rseq_str(rseq_cs) "\n\t" \
89 __rseq_str(label) ":\n\t"
90
91#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
92 RSEQ_INJECT_ASM(2) \
93 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
94 "jnz " __rseq_str(label) "\n\t"
95
96#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
97 ".pushsection __rseq_failure, \"ax\"\n\t" \
98 /* Disassembler-friendly signature: ud1 <sig>(%rip),%edi. */ \
99 ".byte 0x0f, 0xb9, 0x3d\n\t" \
100 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
101 __rseq_str(label) ":\n\t" \
102 teardown \
103 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
104 ".popsection\n\t"
105
106#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
107 ".pushsection __rseq_failure, \"ax\"\n\t" \
108 __rseq_str(label) ":\n\t" \
109 teardown \
110 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
111 ".popsection\n\t"
112
113static inline __attribute__((always_inline))
114int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
115{
116 RSEQ_INJECT_C(9)
117
118 __asm__ __volatile__ goto (
119 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
120 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
121#ifdef RSEQ_COMPARE_TWICE
122 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
123 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
124#endif
125 /* Start rseq by storing table entry pointer into rseq_cs. */
126 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
127 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
128 RSEQ_INJECT_ASM(3)
129 "cmpq %[v], %[expect]\n\t"
130 "jnz %l[cmpfail]\n\t"
131 RSEQ_INJECT_ASM(4)
132#ifdef RSEQ_COMPARE_TWICE
133 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
134 "cmpq %[v], %[expect]\n\t"
135 "jnz %l[error2]\n\t"
136#endif
137 /* final store */
138 "movq %[newv], %[v]\n\t"
139 "2:\n\t"
140 RSEQ_INJECT_ASM(5)
141 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
142 : /* gcc asm goto does not allow outputs */
143 : [cpu_id] "r" (cpu),
144 [rseq_abi] "r" (&__rseq_abi),
145 [v] "m" (*v),
146 [expect] "r" (expect),
147 [newv] "r" (newv)
148 : "memory", "cc", "rax"
149 RSEQ_INJECT_CLOBBER
150 : abort, cmpfail
151#ifdef RSEQ_COMPARE_TWICE
152 , error1, error2
153#endif
154 );
155 return 0;
156abort:
157 RSEQ_INJECT_FAILED
158 return -1;
159cmpfail:
160 return 1;
161#ifdef RSEQ_COMPARE_TWICE
162error1:
163 rseq_bug("cpu_id comparison failed");
164error2:
165 rseq_bug("expected value comparison failed");
166#endif
167}
168
169/*
170 * Compare @v against @expectnot. When it does _not_ match, load @v
171 * into @load, and store the content of *@v + voffp into @v.
172 */
173static inline __attribute__((always_inline))
174int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
175 off_t voffp, intptr_t *load, int cpu)
176{
177 RSEQ_INJECT_C(9)
178
179 __asm__ __volatile__ goto (
180 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
181 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
182#ifdef RSEQ_COMPARE_TWICE
183 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
184 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
185#endif
186 /* Start rseq by storing table entry pointer into rseq_cs. */
187 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
188 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
189 RSEQ_INJECT_ASM(3)
190 "movq %[v], %%rbx\n\t"
191 "cmpq %%rbx, %[expectnot]\n\t"
192 "je %l[cmpfail]\n\t"
193 RSEQ_INJECT_ASM(4)
194#ifdef RSEQ_COMPARE_TWICE
195 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
196 "movq %[v], %%rbx\n\t"
197 "cmpq %%rbx, %[expectnot]\n\t"
198 "je %l[error2]\n\t"
199#endif
200 "movq %%rbx, %[load]\n\t"
201 "addq %[voffp], %%rbx\n\t"
202 "movq (%%rbx), %%rbx\n\t"
203 /* final store */
204 "movq %%rbx, %[v]\n\t"
205 "2:\n\t"
206 RSEQ_INJECT_ASM(5)
207 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
208 : /* gcc asm goto does not allow outputs */
209 : [cpu_id] "r" (cpu),
210 [rseq_abi] "r" (&__rseq_abi),
211 /* final store input */
212 [v] "m" (*v),
213 [expectnot] "r" (expectnot),
214 [voffp] "er" (voffp),
215 [load] "m" (*load)
216 : "memory", "cc", "rax", "rbx"
217 RSEQ_INJECT_CLOBBER
218 : abort, cmpfail
219#ifdef RSEQ_COMPARE_TWICE
220 , error1, error2
221#endif
222 );
223 return 0;
224abort:
225 RSEQ_INJECT_FAILED
226 return -1;
227cmpfail:
228 return 1;
229#ifdef RSEQ_COMPARE_TWICE
230error1:
231 rseq_bug("cpu_id comparison failed");
232error2:
233 rseq_bug("expected value comparison failed");
234#endif
235}
236
237static inline __attribute__((always_inline))
238int rseq_addv(intptr_t *v, intptr_t count, int cpu)
239{
240 RSEQ_INJECT_C(9)
241
242 __asm__ __volatile__ goto (
243 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
244#ifdef RSEQ_COMPARE_TWICE
245 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
246#endif
247 /* Start rseq by storing table entry pointer into rseq_cs. */
248 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
249 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
250 RSEQ_INJECT_ASM(3)
251#ifdef RSEQ_COMPARE_TWICE
252 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
253#endif
254 /* final store */
255 "addq %[count], %[v]\n\t"
256 "2:\n\t"
257 RSEQ_INJECT_ASM(4)
258 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
259 : /* gcc asm goto does not allow outputs */
260 : [cpu_id] "r" (cpu),
261 [rseq_abi] "r" (&__rseq_abi),
262 /* final store input */
263 [v] "m" (*v),
264 [count] "er" (count)
265 : "memory", "cc", "rax"
266 RSEQ_INJECT_CLOBBER
267 : abort
268#ifdef RSEQ_COMPARE_TWICE
269 , error1
270#endif
271 );
272 return 0;
273abort:
274 RSEQ_INJECT_FAILED
275 return -1;
276#ifdef RSEQ_COMPARE_TWICE
277error1:
278 rseq_bug("cpu_id comparison failed");
279#endif
280}
281
282#define RSEQ_ARCH_HAS_OFFSET_DEREF_ADDV
283
284/*
285 * pval = *(ptr+off)
286 * *pval += inc;
287 */
288static inline __attribute__((always_inline))
289int rseq_offset_deref_addv(intptr_t *ptr, off_t off, intptr_t inc, int cpu)
290{
291 RSEQ_INJECT_C(9)
292
293 __asm__ __volatile__ goto (
294 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
295#ifdef RSEQ_COMPARE_TWICE
296 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
297#endif
298 /* Start rseq by storing table entry pointer into rseq_cs. */
299 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
300 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
301 RSEQ_INJECT_ASM(3)
302#ifdef RSEQ_COMPARE_TWICE
303 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
304#endif
305 /* get p+v */
306 "movq %[ptr], %%rbx\n\t"
307 "addq %[off], %%rbx\n\t"
308 /* get pv */
309 "movq (%%rbx), %%rcx\n\t"
310 /* *pv += inc */
311 "addq %[inc], (%%rcx)\n\t"
312 "2:\n\t"
313 RSEQ_INJECT_ASM(4)
314 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
315 : /* gcc asm goto does not allow outputs */
316 : [cpu_id] "r" (cpu),
317 [rseq_abi] "r" (&__rseq_abi),
318 /* final store input */
319 [ptr] "m" (*ptr),
320 [off] "er" (off),
321 [inc] "er" (inc)
322 : "memory", "cc", "rax", "rbx", "rcx"
323 RSEQ_INJECT_CLOBBER
324 : abort
325#ifdef RSEQ_COMPARE_TWICE
326 , error1
327#endif
328 );
329 return 0;
330abort:
331 RSEQ_INJECT_FAILED
332 return -1;
333#ifdef RSEQ_COMPARE_TWICE
334error1:
335 rseq_bug("cpu_id comparison failed");
336#endif
337}
338
339static inline __attribute__((always_inline))
340int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
341 intptr_t *v2, intptr_t newv2,
342 intptr_t newv, int cpu)
343{
344 RSEQ_INJECT_C(9)
345
346 __asm__ __volatile__ goto (
347 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
348 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
349#ifdef RSEQ_COMPARE_TWICE
350 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
351 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
352#endif
353 /* Start rseq by storing table entry pointer into rseq_cs. */
354 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
355 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
356 RSEQ_INJECT_ASM(3)
357 "cmpq %[v], %[expect]\n\t"
358 "jnz %l[cmpfail]\n\t"
359 RSEQ_INJECT_ASM(4)
360#ifdef RSEQ_COMPARE_TWICE
361 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
362 "cmpq %[v], %[expect]\n\t"
363 "jnz %l[error2]\n\t"
364#endif
365 /* try store */
366 "movq %[newv2], %[v2]\n\t"
367 RSEQ_INJECT_ASM(5)
368 /* final store */
369 "movq %[newv], %[v]\n\t"
370 "2:\n\t"
371 RSEQ_INJECT_ASM(6)
372 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
373 : /* gcc asm goto does not allow outputs */
374 : [cpu_id] "r" (cpu),
375 [rseq_abi] "r" (&__rseq_abi),
376 /* try store input */
377 [v2] "m" (*v2),
378 [newv2] "r" (newv2),
379 /* final store input */
380 [v] "m" (*v),
381 [expect] "r" (expect),
382 [newv] "r" (newv)
383 : "memory", "cc", "rax"
384 RSEQ_INJECT_CLOBBER
385 : abort, cmpfail
386#ifdef RSEQ_COMPARE_TWICE
387 , error1, error2
388#endif
389 );
390 return 0;
391abort:
392 RSEQ_INJECT_FAILED
393 return -1;
394cmpfail:
395 return 1;
396#ifdef RSEQ_COMPARE_TWICE
397error1:
398 rseq_bug("cpu_id comparison failed");
399error2:
400 rseq_bug("expected value comparison failed");
401#endif
402}
403
404/* x86-64 is TSO. */
405static inline __attribute__((always_inline))
406int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
407 intptr_t *v2, intptr_t newv2,
408 intptr_t newv, int cpu)
409{
410 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
411}
412
413static inline __attribute__((always_inline))
414int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
415 intptr_t *v2, intptr_t expect2,
416 intptr_t newv, int cpu)
417{
418 RSEQ_INJECT_C(9)
419
420 __asm__ __volatile__ goto (
421 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
422 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
423#ifdef RSEQ_COMPARE_TWICE
424 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
425 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
426 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
427#endif
428 /* Start rseq by storing table entry pointer into rseq_cs. */
429 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
430 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
431 RSEQ_INJECT_ASM(3)
432 "cmpq %[v], %[expect]\n\t"
433 "jnz %l[cmpfail]\n\t"
434 RSEQ_INJECT_ASM(4)
435 "cmpq %[v2], %[expect2]\n\t"
436 "jnz %l[cmpfail]\n\t"
437 RSEQ_INJECT_ASM(5)
438#ifdef RSEQ_COMPARE_TWICE
439 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
440 "cmpq %[v], %[expect]\n\t"
441 "jnz %l[error2]\n\t"
442 "cmpq %[v2], %[expect2]\n\t"
443 "jnz %l[error3]\n\t"
444#endif
445 /* final store */
446 "movq %[newv], %[v]\n\t"
447 "2:\n\t"
448 RSEQ_INJECT_ASM(6)
449 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
450 : /* gcc asm goto does not allow outputs */
451 : [cpu_id] "r" (cpu),
452 [rseq_abi] "r" (&__rseq_abi),
453 /* cmp2 input */
454 [v2] "m" (*v2),
455 [expect2] "r" (expect2),
456 /* final store input */
457 [v] "m" (*v),
458 [expect] "r" (expect),
459 [newv] "r" (newv)
460 : "memory", "cc", "rax"
461 RSEQ_INJECT_CLOBBER
462 : abort, cmpfail
463#ifdef RSEQ_COMPARE_TWICE
464 , error1, error2, error3
465#endif
466 );
467 return 0;
468abort:
469 RSEQ_INJECT_FAILED
470 return -1;
471cmpfail:
472 return 1;
473#ifdef RSEQ_COMPARE_TWICE
474error1:
475 rseq_bug("cpu_id comparison failed");
476error2:
477 rseq_bug("1st expected value comparison failed");
478error3:
479 rseq_bug("2nd expected value comparison failed");
480#endif
481}
482
483static inline __attribute__((always_inline))
484int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
485 void *dst, void *src, size_t len,
486 intptr_t newv, int cpu)
487{
488 uint64_t rseq_scratch[3];
489
490 RSEQ_INJECT_C(9)
491
492 __asm__ __volatile__ goto (
493 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
494 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
495#ifdef RSEQ_COMPARE_TWICE
496 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
497 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
498#endif
499 "movq %[src], %[rseq_scratch0]\n\t"
500 "movq %[dst], %[rseq_scratch1]\n\t"
501 "movq %[len], %[rseq_scratch2]\n\t"
502 /* Start rseq by storing table entry pointer into rseq_cs. */
503 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
504 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
505 RSEQ_INJECT_ASM(3)
506 "cmpq %[v], %[expect]\n\t"
507 "jnz 5f\n\t"
508 RSEQ_INJECT_ASM(4)
509#ifdef RSEQ_COMPARE_TWICE
510 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
511 "cmpq %[v], %[expect]\n\t"
512 "jnz 7f\n\t"
513#endif
514 /* try memcpy */
515 "test %[len], %[len]\n\t" \
516 "jz 333f\n\t" \
517 "222:\n\t" \
518 "movb (%[src]), %%al\n\t" \
519 "movb %%al, (%[dst])\n\t" \
520 "inc %[src]\n\t" \
521 "inc %[dst]\n\t" \
522 "dec %[len]\n\t" \
523 "jnz 222b\n\t" \
524 "333:\n\t" \
525 RSEQ_INJECT_ASM(5)
526 /* final store */
527 "movq %[newv], %[v]\n\t"
528 "2:\n\t"
529 RSEQ_INJECT_ASM(6)
530 /* teardown */
531 "movq %[rseq_scratch2], %[len]\n\t"
532 "movq %[rseq_scratch1], %[dst]\n\t"
533 "movq %[rseq_scratch0], %[src]\n\t"
534 RSEQ_ASM_DEFINE_ABORT(4,
535 "movq %[rseq_scratch2], %[len]\n\t"
536 "movq %[rseq_scratch1], %[dst]\n\t"
537 "movq %[rseq_scratch0], %[src]\n\t",
538 abort)
539 RSEQ_ASM_DEFINE_CMPFAIL(5,
540 "movq %[rseq_scratch2], %[len]\n\t"
541 "movq %[rseq_scratch1], %[dst]\n\t"
542 "movq %[rseq_scratch0], %[src]\n\t",
543 cmpfail)
544#ifdef RSEQ_COMPARE_TWICE
545 RSEQ_ASM_DEFINE_CMPFAIL(6,
546 "movq %[rseq_scratch2], %[len]\n\t"
547 "movq %[rseq_scratch1], %[dst]\n\t"
548 "movq %[rseq_scratch0], %[src]\n\t",
549 error1)
550 RSEQ_ASM_DEFINE_CMPFAIL(7,
551 "movq %[rseq_scratch2], %[len]\n\t"
552 "movq %[rseq_scratch1], %[dst]\n\t"
553 "movq %[rseq_scratch0], %[src]\n\t",
554 error2)
555#endif
556 : /* gcc asm goto does not allow outputs */
557 : [cpu_id] "r" (cpu),
558 [rseq_abi] "r" (&__rseq_abi),
559 /* final store input */
560 [v] "m" (*v),
561 [expect] "r" (expect),
562 [newv] "r" (newv),
563 /* try memcpy input */
564 [dst] "r" (dst),
565 [src] "r" (src),
566 [len] "r" (len),
567 [rseq_scratch0] "m" (rseq_scratch[0]),
568 [rseq_scratch1] "m" (rseq_scratch[1]),
569 [rseq_scratch2] "m" (rseq_scratch[2])
570 : "memory", "cc", "rax"
571 RSEQ_INJECT_CLOBBER
572 : abort, cmpfail
573#ifdef RSEQ_COMPARE_TWICE
574 , error1, error2
575#endif
576 );
577 return 0;
578abort:
579 RSEQ_INJECT_FAILED
580 return -1;
581cmpfail:
582 return 1;
583#ifdef RSEQ_COMPARE_TWICE
584error1:
585 rseq_bug("cpu_id comparison failed");
586error2:
587 rseq_bug("expected value comparison failed");
588#endif
589}
590
591/* x86-64 is TSO. */
592static inline __attribute__((always_inline))
593int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
594 void *dst, void *src, size_t len,
595 intptr_t newv, int cpu)
596{
597 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
598 newv, cpu);
599}
600
601#endif /* !RSEQ_SKIP_FASTPATH */
602
603#elif __i386__
604
605#define rseq_smp_mb() \
606 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
607#define rseq_smp_rmb() \
608 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
609#define rseq_smp_wmb() \
610 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
611
612#define rseq_smp_load_acquire(p) \
613__extension__ ({ \
614 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
615 rseq_smp_mb(); \
616 ____p1; \
617})
618
619#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
620
621#define rseq_smp_store_release(p, v) \
622do { \
623 rseq_smp_mb(); \
624 RSEQ_WRITE_ONCE(*p, v); \
625} while (0)
626
627#ifdef RSEQ_SKIP_FASTPATH
628#include "rseq-skip.h"
629#else /* !RSEQ_SKIP_FASTPATH */
630
631/*
632 * Use eax as scratch register and take memory operands as input to
633 * lessen register pressure. Especially needed when compiling in O0.
634 */
635#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
636 start_ip, post_commit_offset, abort_ip) \
637 ".pushsection __rseq_cs, \"aw\"\n\t" \
638 ".balign 32\n\t" \
639 __rseq_str(label) ":\n\t" \
640 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
641 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
642 ".popsection\n\t" \
643 ".pushsection __rseq_cs_ptr_array, \"aw\"\n\t" \
644 ".long " __rseq_str(label) "b, 0x0\n\t" \
645 ".popsection\n\t"
646
647#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
648 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
649 (post_commit_ip - start_ip), abort_ip)
650
651/*
652 * Exit points of a rseq critical section consist of all instructions outside
653 * of the critical section where a critical section can either branch to or
654 * reach through the normal course of its execution. The abort IP and the
655 * post-commit IP are already part of the __rseq_cs section and should not be
656 * explicitly defined as additional exit points. Knowing all exit points is
657 * useful to assist debuggers stepping over the critical section.
658 */
659#define RSEQ_ASM_DEFINE_EXIT_POINT(start_ip, exit_ip) \
660 ".pushsection __rseq_exit_point_array, \"aw\"\n\t" \
661 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(exit_ip) ", 0x0\n\t" \
662 ".popsection\n\t"
663
664#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
665 RSEQ_INJECT_ASM(1) \
666 "movl $" __rseq_str(cs_label) ", " __rseq_str(rseq_cs) "\n\t" \
667 __rseq_str(label) ":\n\t"
668
669#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
670 RSEQ_INJECT_ASM(2) \
671 "cmpl %[" __rseq_str(cpu_id) "], " __rseq_str(current_cpu_id) "\n\t" \
672 "jnz " __rseq_str(label) "\n\t"
673
674#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
675 ".pushsection __rseq_failure, \"ax\"\n\t" \
676 /* Disassembler-friendly signature: ud1 <sig>,%edi. */ \
677 ".byte 0x0f, 0xb9, 0x3d\n\t" \
678 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
679 __rseq_str(label) ":\n\t" \
680 teardown \
681 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
682 ".popsection\n\t"
683
684#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
685 ".pushsection __rseq_failure, \"ax\"\n\t" \
686 __rseq_str(label) ":\n\t" \
687 teardown \
688 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
689 ".popsection\n\t"
690
691static inline __attribute__((always_inline))
692int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
693{
694 RSEQ_INJECT_C(9)
695
696 __asm__ __volatile__ goto (
697 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
698 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
699#ifdef RSEQ_COMPARE_TWICE
700 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
701 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
702#endif
703 /* Start rseq by storing table entry pointer into rseq_cs. */
704 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
705 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
706 RSEQ_INJECT_ASM(3)
707 "cmpl %[v], %[expect]\n\t"
708 "jnz %l[cmpfail]\n\t"
709 RSEQ_INJECT_ASM(4)
710#ifdef RSEQ_COMPARE_TWICE
711 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
712 "cmpl %[v], %[expect]\n\t"
713 "jnz %l[error2]\n\t"
714#endif
715 /* final store */
716 "movl %[newv], %[v]\n\t"
717 "2:\n\t"
718 RSEQ_INJECT_ASM(5)
719 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
720 : /* gcc asm goto does not allow outputs */
721 : [cpu_id] "r" (cpu),
722 [rseq_abi] "r" (&__rseq_abi),
723 [v] "m" (*v),
724 [expect] "r" (expect),
725 [newv] "r" (newv)
726 : "memory", "cc", "eax"
727 RSEQ_INJECT_CLOBBER
728 : abort, cmpfail
729#ifdef RSEQ_COMPARE_TWICE
730 , error1, error2
731#endif
732 );
733 return 0;
734abort:
735 RSEQ_INJECT_FAILED
736 return -1;
737cmpfail:
738 return 1;
739#ifdef RSEQ_COMPARE_TWICE
740error1:
741 rseq_bug("cpu_id comparison failed");
742error2:
743 rseq_bug("expected value comparison failed");
744#endif
745}
746
747/*
748 * Compare @v against @expectnot. When it does _not_ match, load @v
749 * into @load, and store the content of *@v + voffp into @v.
750 */
751static inline __attribute__((always_inline))
752int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
753 off_t voffp, intptr_t *load, int cpu)
754{
755 RSEQ_INJECT_C(9)
756
757 __asm__ __volatile__ goto (
758 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
759 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
760#ifdef RSEQ_COMPARE_TWICE
761 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
762 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
763#endif
764 /* Start rseq by storing table entry pointer into rseq_cs. */
765 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
766 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
767 RSEQ_INJECT_ASM(3)
768 "movl %[v], %%ebx\n\t"
769 "cmpl %%ebx, %[expectnot]\n\t"
770 "je %l[cmpfail]\n\t"
771 RSEQ_INJECT_ASM(4)
772#ifdef RSEQ_COMPARE_TWICE
773 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
774 "movl %[v], %%ebx\n\t"
775 "cmpl %%ebx, %[expectnot]\n\t"
776 "je %l[error2]\n\t"
777#endif
778 "movl %%ebx, %[load]\n\t"
779 "addl %[voffp], %%ebx\n\t"
780 "movl (%%ebx), %%ebx\n\t"
781 /* final store */
782 "movl %%ebx, %[v]\n\t"
783 "2:\n\t"
784 RSEQ_INJECT_ASM(5)
785 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
786 : /* gcc asm goto does not allow outputs */
787 : [cpu_id] "r" (cpu),
788 [rseq_abi] "r" (&__rseq_abi),
789 /* final store input */
790 [v] "m" (*v),
791 [expectnot] "r" (expectnot),
792 [voffp] "ir" (voffp),
793 [load] "m" (*load)
794 : "memory", "cc", "eax", "ebx"
795 RSEQ_INJECT_CLOBBER
796 : abort, cmpfail
797#ifdef RSEQ_COMPARE_TWICE
798 , error1, error2
799#endif
800 );
801 return 0;
802abort:
803 RSEQ_INJECT_FAILED
804 return -1;
805cmpfail:
806 return 1;
807#ifdef RSEQ_COMPARE_TWICE
808error1:
809 rseq_bug("cpu_id comparison failed");
810error2:
811 rseq_bug("expected value comparison failed");
812#endif
813}
814
815static inline __attribute__((always_inline))
816int rseq_addv(intptr_t *v, intptr_t count, int cpu)
817{
818 RSEQ_INJECT_C(9)
819
820 __asm__ __volatile__ goto (
821 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
822#ifdef RSEQ_COMPARE_TWICE
823 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
824#endif
825 /* Start rseq by storing table entry pointer into rseq_cs. */
826 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
827 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
828 RSEQ_INJECT_ASM(3)
829#ifdef RSEQ_COMPARE_TWICE
830 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
831#endif
832 /* final store */
833 "addl %[count], %[v]\n\t"
834 "2:\n\t"
835 RSEQ_INJECT_ASM(4)
836 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
837 : /* gcc asm goto does not allow outputs */
838 : [cpu_id] "r" (cpu),
839 [rseq_abi] "r" (&__rseq_abi),
840 /* final store input */
841 [v] "m" (*v),
842 [count] "ir" (count)
843 : "memory", "cc", "eax"
844 RSEQ_INJECT_CLOBBER
845 : abort
846#ifdef RSEQ_COMPARE_TWICE
847 , error1
848#endif
849 );
850 return 0;
851abort:
852 RSEQ_INJECT_FAILED
853 return -1;
854#ifdef RSEQ_COMPARE_TWICE
855error1:
856 rseq_bug("cpu_id comparison failed");
857#endif
858}
859
860static inline __attribute__((always_inline))
861int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
862 intptr_t *v2, intptr_t newv2,
863 intptr_t newv, int cpu)
864{
865 RSEQ_INJECT_C(9)
866
867 __asm__ __volatile__ goto (
868 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
869 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
870#ifdef RSEQ_COMPARE_TWICE
871 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
872 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
873#endif
874 /* Start rseq by storing table entry pointer into rseq_cs. */
875 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
876 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
877 RSEQ_INJECT_ASM(3)
878 "cmpl %[v], %[expect]\n\t"
879 "jnz %l[cmpfail]\n\t"
880 RSEQ_INJECT_ASM(4)
881#ifdef RSEQ_COMPARE_TWICE
882 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
883 "cmpl %[v], %[expect]\n\t"
884 "jnz %l[error2]\n\t"
885#endif
886 /* try store */
887 "movl %[newv2], %%eax\n\t"
888 "movl %%eax, %[v2]\n\t"
889 RSEQ_INJECT_ASM(5)
890 /* final store */
891 "movl %[newv], %[v]\n\t"
892 "2:\n\t"
893 RSEQ_INJECT_ASM(6)
894 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
895 : /* gcc asm goto does not allow outputs */
896 : [cpu_id] "r" (cpu),
897 [rseq_abi] "r" (&__rseq_abi),
898 /* try store input */
899 [v2] "m" (*v2),
900 [newv2] "m" (newv2),
901 /* final store input */
902 [v] "m" (*v),
903 [expect] "r" (expect),
904 [newv] "r" (newv)
905 : "memory", "cc", "eax"
906 RSEQ_INJECT_CLOBBER
907 : abort, cmpfail
908#ifdef RSEQ_COMPARE_TWICE
909 , error1, error2
910#endif
911 );
912 return 0;
913abort:
914 RSEQ_INJECT_FAILED
915 return -1;
916cmpfail:
917 return 1;
918#ifdef RSEQ_COMPARE_TWICE
919error1:
920 rseq_bug("cpu_id comparison failed");
921error2:
922 rseq_bug("expected value comparison failed");
923#endif
924}
925
926static inline __attribute__((always_inline))
927int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
928 intptr_t *v2, intptr_t newv2,
929 intptr_t newv, int cpu)
930{
931 RSEQ_INJECT_C(9)
932
933 __asm__ __volatile__ goto (
934 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
935 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
936#ifdef RSEQ_COMPARE_TWICE
937 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
938 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
939#endif
940 /* Start rseq by storing table entry pointer into rseq_cs. */
941 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
942 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
943 RSEQ_INJECT_ASM(3)
944 "movl %[expect], %%eax\n\t"
945 "cmpl %[v], %%eax\n\t"
946 "jnz %l[cmpfail]\n\t"
947 RSEQ_INJECT_ASM(4)
948#ifdef RSEQ_COMPARE_TWICE
949 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
950 "movl %[expect], %%eax\n\t"
951 "cmpl %[v], %%eax\n\t"
952 "jnz %l[error2]\n\t"
953#endif
954 /* try store */
955 "movl %[newv2], %[v2]\n\t"
956 RSEQ_INJECT_ASM(5)
957 "lock; addl $0,-128(%%esp)\n\t"
958 /* final store */
959 "movl %[newv], %[v]\n\t"
960 "2:\n\t"
961 RSEQ_INJECT_ASM(6)
962 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
963 : /* gcc asm goto does not allow outputs */
964 : [cpu_id] "r" (cpu),
965 [rseq_abi] "r" (&__rseq_abi),
966 /* try store input */
967 [v2] "m" (*v2),
968 [newv2] "r" (newv2),
969 /* final store input */
970 [v] "m" (*v),
971 [expect] "m" (expect),
972 [newv] "r" (newv)
973 : "memory", "cc", "eax"
974 RSEQ_INJECT_CLOBBER
975 : abort, cmpfail
976#ifdef RSEQ_COMPARE_TWICE
977 , error1, error2
978#endif
979 );
980 return 0;
981abort:
982 RSEQ_INJECT_FAILED
983 return -1;
984cmpfail:
985 return 1;
986#ifdef RSEQ_COMPARE_TWICE
987error1:
988 rseq_bug("cpu_id comparison failed");
989error2:
990 rseq_bug("expected value comparison failed");
991#endif
992
993}
994
995static inline __attribute__((always_inline))
996int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
997 intptr_t *v2, intptr_t expect2,
998 intptr_t newv, int cpu)
999{
1000 RSEQ_INJECT_C(9)
1001
1002 __asm__ __volatile__ goto (
1003 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1004 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1005#ifdef RSEQ_COMPARE_TWICE
1006 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1007 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1008 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3])
1009#endif
1010 /* Start rseq by storing table entry pointer into rseq_cs. */
1011 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1012 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1013 RSEQ_INJECT_ASM(3)
1014 "cmpl %[v], %[expect]\n\t"
1015 "jnz %l[cmpfail]\n\t"
1016 RSEQ_INJECT_ASM(4)
1017 "cmpl %[expect2], %[v2]\n\t"
1018 "jnz %l[cmpfail]\n\t"
1019 RSEQ_INJECT_ASM(5)
1020#ifdef RSEQ_COMPARE_TWICE
1021 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), %l[error1])
1022 "cmpl %[v], %[expect]\n\t"
1023 "jnz %l[error2]\n\t"
1024 "cmpl %[expect2], %[v2]\n\t"
1025 "jnz %l[error3]\n\t"
1026#endif
1027 "movl %[newv], %%eax\n\t"
1028 /* final store */
1029 "movl %%eax, %[v]\n\t"
1030 "2:\n\t"
1031 RSEQ_INJECT_ASM(6)
1032 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
1033 : /* gcc asm goto does not allow outputs */
1034 : [cpu_id] "r" (cpu),
1035 [rseq_abi] "r" (&__rseq_abi),
1036 /* cmp2 input */
1037 [v2] "m" (*v2),
1038 [expect2] "r" (expect2),
1039 /* final store input */
1040 [v] "m" (*v),
1041 [expect] "r" (expect),
1042 [newv] "m" (newv)
1043 : "memory", "cc", "eax"
1044 RSEQ_INJECT_CLOBBER
1045 : abort, cmpfail
1046#ifdef RSEQ_COMPARE_TWICE
1047 , error1, error2, error3
1048#endif
1049 );
1050 return 0;
1051abort:
1052 RSEQ_INJECT_FAILED
1053 return -1;
1054cmpfail:
1055 return 1;
1056#ifdef RSEQ_COMPARE_TWICE
1057error1:
1058 rseq_bug("cpu_id comparison failed");
1059error2:
1060 rseq_bug("1st expected value comparison failed");
1061error3:
1062 rseq_bug("2nd expected value comparison failed");
1063#endif
1064}
1065
1066/* TODO: implement a faster memcpy. */
1067static inline __attribute__((always_inline))
1068int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
1069 void *dst, void *src, size_t len,
1070 intptr_t newv, int cpu)
1071{
1072 uint32_t rseq_scratch[3];
1073
1074 RSEQ_INJECT_C(9)
1075
1076 __asm__ __volatile__ goto (
1077 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1078 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1079#ifdef RSEQ_COMPARE_TWICE
1080 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1081 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1082#endif
1083 "movl %[src], %[rseq_scratch0]\n\t"
1084 "movl %[dst], %[rseq_scratch1]\n\t"
1085 "movl %[len], %[rseq_scratch2]\n\t"
1086 /* Start rseq by storing table entry pointer into rseq_cs. */
1087 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1088 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1089 RSEQ_INJECT_ASM(3)
1090 "movl %[expect], %%eax\n\t"
1091 "cmpl %%eax, %[v]\n\t"
1092 "jnz 5f\n\t"
1093 RSEQ_INJECT_ASM(4)
1094#ifdef RSEQ_COMPARE_TWICE
1095 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1096 "movl %[expect], %%eax\n\t"
1097 "cmpl %%eax, %[v]\n\t"
1098 "jnz 7f\n\t"
1099#endif
1100 /* try memcpy */
1101 "test %[len], %[len]\n\t" \
1102 "jz 333f\n\t" \
1103 "222:\n\t" \
1104 "movb (%[src]), %%al\n\t" \
1105 "movb %%al, (%[dst])\n\t" \
1106 "inc %[src]\n\t" \
1107 "inc %[dst]\n\t" \
1108 "dec %[len]\n\t" \
1109 "jnz 222b\n\t" \
1110 "333:\n\t" \
1111 RSEQ_INJECT_ASM(5)
1112 "movl %[newv], %%eax\n\t"
1113 /* final store */
1114 "movl %%eax, %[v]\n\t"
1115 "2:\n\t"
1116 RSEQ_INJECT_ASM(6)
1117 /* teardown */
1118 "movl %[rseq_scratch2], %[len]\n\t"
1119 "movl %[rseq_scratch1], %[dst]\n\t"
1120 "movl %[rseq_scratch0], %[src]\n\t"
1121 RSEQ_ASM_DEFINE_ABORT(4,
1122 "movl %[rseq_scratch2], %[len]\n\t"
1123 "movl %[rseq_scratch1], %[dst]\n\t"
1124 "movl %[rseq_scratch0], %[src]\n\t",
1125 abort)
1126 RSEQ_ASM_DEFINE_CMPFAIL(5,
1127 "movl %[rseq_scratch2], %[len]\n\t"
1128 "movl %[rseq_scratch1], %[dst]\n\t"
1129 "movl %[rseq_scratch0], %[src]\n\t",
1130 cmpfail)
1131#ifdef RSEQ_COMPARE_TWICE
1132 RSEQ_ASM_DEFINE_CMPFAIL(6,
1133 "movl %[rseq_scratch2], %[len]\n\t"
1134 "movl %[rseq_scratch1], %[dst]\n\t"
1135 "movl %[rseq_scratch0], %[src]\n\t",
1136 error1)
1137 RSEQ_ASM_DEFINE_CMPFAIL(7,
1138 "movl %[rseq_scratch2], %[len]\n\t"
1139 "movl %[rseq_scratch1], %[dst]\n\t"
1140 "movl %[rseq_scratch0], %[src]\n\t",
1141 error2)
1142#endif
1143 : /* gcc asm goto does not allow outputs */
1144 : [cpu_id] "r" (cpu),
1145 [rseq_abi] "r" (&__rseq_abi),
1146 /* final store input */
1147 [v] "m" (*v),
1148 [expect] "m" (expect),
1149 [newv] "m" (newv),
1150 /* try memcpy input */
1151 [dst] "r" (dst),
1152 [src] "r" (src),
1153 [len] "r" (len),
1154 [rseq_scratch0] "m" (rseq_scratch[0]),
1155 [rseq_scratch1] "m" (rseq_scratch[1]),
1156 [rseq_scratch2] "m" (rseq_scratch[2])
1157 : "memory", "cc", "eax"
1158 RSEQ_INJECT_CLOBBER
1159 : abort, cmpfail
1160#ifdef RSEQ_COMPARE_TWICE
1161 , error1, error2
1162#endif
1163 );
1164 return 0;
1165abort:
1166 RSEQ_INJECT_FAILED
1167 return -1;
1168cmpfail:
1169 return 1;
1170#ifdef RSEQ_COMPARE_TWICE
1171error1:
1172 rseq_bug("cpu_id comparison failed");
1173error2:
1174 rseq_bug("expected value comparison failed");
1175#endif
1176}
1177
1178/* TODO: implement a faster memcpy. */
1179static inline __attribute__((always_inline))
1180int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1181 void *dst, void *src, size_t len,
1182 intptr_t newv, int cpu)
1183{
1184 uint32_t rseq_scratch[3];
1185
1186 RSEQ_INJECT_C(9)
1187
1188 __asm__ __volatile__ goto (
1189 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1190 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail])
1191#ifdef RSEQ_COMPARE_TWICE
1192 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1])
1193 RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2])
1194#endif
1195 "movl %[src], %[rseq_scratch0]\n\t"
1196 "movl %[dst], %[rseq_scratch1]\n\t"
1197 "movl %[len], %[rseq_scratch2]\n\t"
1198 /* Start rseq by storing table entry pointer into rseq_cs. */
1199 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, RSEQ_CS_OFFSET(%[rseq_abi]))
1200 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 4f)
1201 RSEQ_INJECT_ASM(3)
1202 "movl %[expect], %%eax\n\t"
1203 "cmpl %%eax, %[v]\n\t"
1204 "jnz 5f\n\t"
1205 RSEQ_INJECT_ASM(4)
1206#ifdef RSEQ_COMPARE_TWICE
1207 RSEQ_ASM_CMP_CPU_ID(cpu_id, RSEQ_CPU_ID_OFFSET(%[rseq_abi]), 6f)
1208 "movl %[expect], %%eax\n\t"
1209 "cmpl %%eax, %[v]\n\t"
1210 "jnz 7f\n\t"
1211#endif
1212 /* try memcpy */
1213 "test %[len], %[len]\n\t" \
1214 "jz 333f\n\t" \
1215 "222:\n\t" \
1216 "movb (%[src]), %%al\n\t" \
1217 "movb %%al, (%[dst])\n\t" \
1218 "inc %[src]\n\t" \
1219 "inc %[dst]\n\t" \
1220 "dec %[len]\n\t" \
1221 "jnz 222b\n\t" \
1222 "333:\n\t" \
1223 RSEQ_INJECT_ASM(5)
1224 "lock; addl $0,-128(%%esp)\n\t"
1225 "movl %[newv], %%eax\n\t"
1226 /* final store */
1227 "movl %%eax, %[v]\n\t"
1228 "2:\n\t"
1229 RSEQ_INJECT_ASM(6)
1230 /* teardown */
1231 "movl %[rseq_scratch2], %[len]\n\t"
1232 "movl %[rseq_scratch1], %[dst]\n\t"
1233 "movl %[rseq_scratch0], %[src]\n\t"
1234 RSEQ_ASM_DEFINE_ABORT(4,
1235 "movl %[rseq_scratch2], %[len]\n\t"
1236 "movl %[rseq_scratch1], %[dst]\n\t"
1237 "movl %[rseq_scratch0], %[src]\n\t",
1238 abort)
1239 RSEQ_ASM_DEFINE_CMPFAIL(5,
1240 "movl %[rseq_scratch2], %[len]\n\t"
1241 "movl %[rseq_scratch1], %[dst]\n\t"
1242 "movl %[rseq_scratch0], %[src]\n\t",
1243 cmpfail)
1244#ifdef RSEQ_COMPARE_TWICE
1245 RSEQ_ASM_DEFINE_CMPFAIL(6,
1246 "movl %[rseq_scratch2], %[len]\n\t"
1247 "movl %[rseq_scratch1], %[dst]\n\t"
1248 "movl %[rseq_scratch0], %[src]\n\t",
1249 error1)
1250 RSEQ_ASM_DEFINE_CMPFAIL(7,
1251 "movl %[rseq_scratch2], %[len]\n\t"
1252 "movl %[rseq_scratch1], %[dst]\n\t"
1253 "movl %[rseq_scratch0], %[src]\n\t",
1254 error2)
1255#endif
1256 : /* gcc asm goto does not allow outputs */
1257 : [cpu_id] "r" (cpu),
1258 [rseq_abi] "r" (&__rseq_abi),
1259 /* final store input */
1260 [v] "m" (*v),
1261 [expect] "m" (expect),
1262 [newv] "m" (newv),
1263 /* try memcpy input */
1264 [dst] "r" (dst),
1265 [src] "r" (src),
1266 [len] "r" (len),
1267 [rseq_scratch0] "m" (rseq_scratch[0]),
1268 [rseq_scratch1] "m" (rseq_scratch[1]),
1269 [rseq_scratch2] "m" (rseq_scratch[2])
1270 : "memory", "cc", "eax"
1271 RSEQ_INJECT_CLOBBER
1272 : abort, cmpfail
1273#ifdef RSEQ_COMPARE_TWICE
1274 , error1, error2
1275#endif
1276 );
1277 return 0;
1278abort:
1279 RSEQ_INJECT_FAILED
1280 return -1;
1281cmpfail:
1282 return 1;
1283#ifdef RSEQ_COMPARE_TWICE
1284error1:
1285 rseq_bug("cpu_id comparison failed");
1286error2:
1287 rseq_bug("expected value comparison failed");
1288#endif
1289}
1290
1291#endif /* !RSEQ_SKIP_FASTPATH */
1292
1293#endif