tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: LGPL-2.1 OR MIT */
2/*
3 * rseq-x86.h
4 *
5 * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 */
7
8#include <stdint.h>
9
10#define RSEQ_SIG 0x53053053
11
12#ifdef __x86_64__
13
14#define rseq_smp_mb() \
15 __asm__ __volatile__ ("lock; addl $0,-128(%%rsp)" ::: "memory", "cc")
16#define rseq_smp_rmb() rseq_barrier()
17#define rseq_smp_wmb() rseq_barrier()
18
19#define rseq_smp_load_acquire(p) \
20__extension__ ({ \
21 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
22 rseq_barrier(); \
23 ____p1; \
24})
25
26#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
27
28#define rseq_smp_store_release(p, v) \
29do { \
30 rseq_barrier(); \
31 RSEQ_WRITE_ONCE(*p, v); \
32} while (0)
33
34#ifdef RSEQ_SKIP_FASTPATH
35#include "rseq-skip.h"
36#else /* !RSEQ_SKIP_FASTPATH */
37
38#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
39 start_ip, post_commit_offset, abort_ip) \
40 ".pushsection __rseq_table, \"aw\"\n\t" \
41 ".balign 32\n\t" \
42 __rseq_str(label) ":\n\t" \
43 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
44 ".quad " __rseq_str(start_ip) ", " __rseq_str(post_commit_offset) ", " __rseq_str(abort_ip) "\n\t" \
45 ".popsection\n\t"
46
47#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
48 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
49 (post_commit_ip - start_ip), abort_ip)
50
51#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
52 RSEQ_INJECT_ASM(1) \
53 "leaq " __rseq_str(cs_label) "(%%rip), %%rax\n\t" \
54 "movq %%rax, %[" __rseq_str(rseq_cs) "]\n\t" \
55 __rseq_str(label) ":\n\t"
56
57#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
58 RSEQ_INJECT_ASM(2) \
59 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
60 "jnz " __rseq_str(label) "\n\t"
61
62#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
63 ".pushsection __rseq_failure, \"ax\"\n\t" \
64 /* Disassembler-friendly signature: nopl <sig>(%rip). */\
65 ".byte 0x0f, 0x1f, 0x05\n\t" \
66 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
67 __rseq_str(label) ":\n\t" \
68 teardown \
69 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
70 ".popsection\n\t"
71
72#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
73 ".pushsection __rseq_failure, \"ax\"\n\t" \
74 __rseq_str(label) ":\n\t" \
75 teardown \
76 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
77 ".popsection\n\t"
78
79static inline __attribute__((always_inline))
80int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
81{
82 RSEQ_INJECT_C(9)
83
84 __asm__ __volatile__ goto (
85 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
86 /* Start rseq by storing table entry pointer into rseq_cs. */
87 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
88 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
89 RSEQ_INJECT_ASM(3)
90 "cmpq %[v], %[expect]\n\t"
91 "jnz %l[cmpfail]\n\t"
92 RSEQ_INJECT_ASM(4)
93#ifdef RSEQ_COMPARE_TWICE
94 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
95 "cmpq %[v], %[expect]\n\t"
96 "jnz %l[error2]\n\t"
97#endif
98 /* final store */
99 "movq %[newv], %[v]\n\t"
100 "2:\n\t"
101 RSEQ_INJECT_ASM(5)
102 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
103 : /* gcc asm goto does not allow outputs */
104 : [cpu_id] "r" (cpu),
105 [current_cpu_id] "m" (__rseq_abi.cpu_id),
106 [rseq_cs] "m" (__rseq_abi.rseq_cs),
107 [v] "m" (*v),
108 [expect] "r" (expect),
109 [newv] "r" (newv)
110 : "memory", "cc", "rax"
111 RSEQ_INJECT_CLOBBER
112 : abort, cmpfail
113#ifdef RSEQ_COMPARE_TWICE
114 , error1, error2
115#endif
116 );
117 return 0;
118abort:
119 RSEQ_INJECT_FAILED
120 return -1;
121cmpfail:
122 return 1;
123#ifdef RSEQ_COMPARE_TWICE
124error1:
125 rseq_bug("cpu_id comparison failed");
126error2:
127 rseq_bug("expected value comparison failed");
128#endif
129}
130
131/*
132 * Compare @v against @expectnot. When it does _not_ match, load @v
133 * into @load, and store the content of *@v + voffp into @v.
134 */
135static inline __attribute__((always_inline))
136int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
137 off_t voffp, intptr_t *load, int cpu)
138{
139 RSEQ_INJECT_C(9)
140
141 __asm__ __volatile__ goto (
142 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
143 /* Start rseq by storing table entry pointer into rseq_cs. */
144 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
145 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
146 RSEQ_INJECT_ASM(3)
147 "movq %[v], %%rbx\n\t"
148 "cmpq %%rbx, %[expectnot]\n\t"
149 "je %l[cmpfail]\n\t"
150 RSEQ_INJECT_ASM(4)
151#ifdef RSEQ_COMPARE_TWICE
152 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
153 "movq %[v], %%rbx\n\t"
154 "cmpq %%rbx, %[expectnot]\n\t"
155 "je %l[error2]\n\t"
156#endif
157 "movq %%rbx, %[load]\n\t"
158 "addq %[voffp], %%rbx\n\t"
159 "movq (%%rbx), %%rbx\n\t"
160 /* final store */
161 "movq %%rbx, %[v]\n\t"
162 "2:\n\t"
163 RSEQ_INJECT_ASM(5)
164 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
165 : /* gcc asm goto does not allow outputs */
166 : [cpu_id] "r" (cpu),
167 [current_cpu_id] "m" (__rseq_abi.cpu_id),
168 [rseq_cs] "m" (__rseq_abi.rseq_cs),
169 /* final store input */
170 [v] "m" (*v),
171 [expectnot] "r" (expectnot),
172 [voffp] "er" (voffp),
173 [load] "m" (*load)
174 : "memory", "cc", "rax", "rbx"
175 RSEQ_INJECT_CLOBBER
176 : abort, cmpfail
177#ifdef RSEQ_COMPARE_TWICE
178 , error1, error2
179#endif
180 );
181 return 0;
182abort:
183 RSEQ_INJECT_FAILED
184 return -1;
185cmpfail:
186 return 1;
187#ifdef RSEQ_COMPARE_TWICE
188error1:
189 rseq_bug("cpu_id comparison failed");
190error2:
191 rseq_bug("expected value comparison failed");
192#endif
193}
194
195static inline __attribute__((always_inline))
196int rseq_addv(intptr_t *v, intptr_t count, int cpu)
197{
198 RSEQ_INJECT_C(9)
199
200 __asm__ __volatile__ goto (
201 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
202 /* Start rseq by storing table entry pointer into rseq_cs. */
203 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
204 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
205 RSEQ_INJECT_ASM(3)
206#ifdef RSEQ_COMPARE_TWICE
207 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
208#endif
209 /* final store */
210 "addq %[count], %[v]\n\t"
211 "2:\n\t"
212 RSEQ_INJECT_ASM(4)
213 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
214 : /* gcc asm goto does not allow outputs */
215 : [cpu_id] "r" (cpu),
216 [current_cpu_id] "m" (__rseq_abi.cpu_id),
217 [rseq_cs] "m" (__rseq_abi.rseq_cs),
218 /* final store input */
219 [v] "m" (*v),
220 [count] "er" (count)
221 : "memory", "cc", "rax"
222 RSEQ_INJECT_CLOBBER
223 : abort
224#ifdef RSEQ_COMPARE_TWICE
225 , error1
226#endif
227 );
228 return 0;
229abort:
230 RSEQ_INJECT_FAILED
231 return -1;
232#ifdef RSEQ_COMPARE_TWICE
233error1:
234 rseq_bug("cpu_id comparison failed");
235#endif
236}
237
238static inline __attribute__((always_inline))
239int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
240 intptr_t *v2, intptr_t newv2,
241 intptr_t newv, int cpu)
242{
243 RSEQ_INJECT_C(9)
244
245 __asm__ __volatile__ goto (
246 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
247 /* Start rseq by storing table entry pointer into rseq_cs. */
248 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
249 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
250 RSEQ_INJECT_ASM(3)
251 "cmpq %[v], %[expect]\n\t"
252 "jnz %l[cmpfail]\n\t"
253 RSEQ_INJECT_ASM(4)
254#ifdef RSEQ_COMPARE_TWICE
255 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
256 "cmpq %[v], %[expect]\n\t"
257 "jnz %l[error2]\n\t"
258#endif
259 /* try store */
260 "movq %[newv2], %[v2]\n\t"
261 RSEQ_INJECT_ASM(5)
262 /* final store */
263 "movq %[newv], %[v]\n\t"
264 "2:\n\t"
265 RSEQ_INJECT_ASM(6)
266 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
267 : /* gcc asm goto does not allow outputs */
268 : [cpu_id] "r" (cpu),
269 [current_cpu_id] "m" (__rseq_abi.cpu_id),
270 [rseq_cs] "m" (__rseq_abi.rseq_cs),
271 /* try store input */
272 [v2] "m" (*v2),
273 [newv2] "r" (newv2),
274 /* final store input */
275 [v] "m" (*v),
276 [expect] "r" (expect),
277 [newv] "r" (newv)
278 : "memory", "cc", "rax"
279 RSEQ_INJECT_CLOBBER
280 : abort, cmpfail
281#ifdef RSEQ_COMPARE_TWICE
282 , error1, error2
283#endif
284 );
285 return 0;
286abort:
287 RSEQ_INJECT_FAILED
288 return -1;
289cmpfail:
290 return 1;
291#ifdef RSEQ_COMPARE_TWICE
292error1:
293 rseq_bug("cpu_id comparison failed");
294error2:
295 rseq_bug("expected value comparison failed");
296#endif
297}
298
299/* x86-64 is TSO. */
300static inline __attribute__((always_inline))
301int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
302 intptr_t *v2, intptr_t newv2,
303 intptr_t newv, int cpu)
304{
305 return rseq_cmpeqv_trystorev_storev(v, expect, v2, newv2, newv, cpu);
306}
307
308static inline __attribute__((always_inline))
309int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
310 intptr_t *v2, intptr_t expect2,
311 intptr_t newv, int cpu)
312{
313 RSEQ_INJECT_C(9)
314
315 __asm__ __volatile__ goto (
316 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
317 /* Start rseq by storing table entry pointer into rseq_cs. */
318 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
319 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
320 RSEQ_INJECT_ASM(3)
321 "cmpq %[v], %[expect]\n\t"
322 "jnz %l[cmpfail]\n\t"
323 RSEQ_INJECT_ASM(4)
324 "cmpq %[v2], %[expect2]\n\t"
325 "jnz %l[cmpfail]\n\t"
326 RSEQ_INJECT_ASM(5)
327#ifdef RSEQ_COMPARE_TWICE
328 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
329 "cmpq %[v], %[expect]\n\t"
330 "jnz %l[error2]\n\t"
331 "cmpq %[v2], %[expect2]\n\t"
332 "jnz %l[error3]\n\t"
333#endif
334 /* final store */
335 "movq %[newv], %[v]\n\t"
336 "2:\n\t"
337 RSEQ_INJECT_ASM(6)
338 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
339 : /* gcc asm goto does not allow outputs */
340 : [cpu_id] "r" (cpu),
341 [current_cpu_id] "m" (__rseq_abi.cpu_id),
342 [rseq_cs] "m" (__rseq_abi.rseq_cs),
343 /* cmp2 input */
344 [v2] "m" (*v2),
345 [expect2] "r" (expect2),
346 /* final store input */
347 [v] "m" (*v),
348 [expect] "r" (expect),
349 [newv] "r" (newv)
350 : "memory", "cc", "rax"
351 RSEQ_INJECT_CLOBBER
352 : abort, cmpfail
353#ifdef RSEQ_COMPARE_TWICE
354 , error1, error2, error3
355#endif
356 );
357 return 0;
358abort:
359 RSEQ_INJECT_FAILED
360 return -1;
361cmpfail:
362 return 1;
363#ifdef RSEQ_COMPARE_TWICE
364error1:
365 rseq_bug("cpu_id comparison failed");
366error2:
367 rseq_bug("1st expected value comparison failed");
368error3:
369 rseq_bug("2nd expected value comparison failed");
370#endif
371}
372
373static inline __attribute__((always_inline))
374int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
375 void *dst, void *src, size_t len,
376 intptr_t newv, int cpu)
377{
378 uint64_t rseq_scratch[3];
379
380 RSEQ_INJECT_C(9)
381
382 __asm__ __volatile__ goto (
383 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
384 "movq %[src], %[rseq_scratch0]\n\t"
385 "movq %[dst], %[rseq_scratch1]\n\t"
386 "movq %[len], %[rseq_scratch2]\n\t"
387 /* Start rseq by storing table entry pointer into rseq_cs. */
388 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
389 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
390 RSEQ_INJECT_ASM(3)
391 "cmpq %[v], %[expect]\n\t"
392 "jnz 5f\n\t"
393 RSEQ_INJECT_ASM(4)
394#ifdef RSEQ_COMPARE_TWICE
395 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
396 "cmpq %[v], %[expect]\n\t"
397 "jnz 7f\n\t"
398#endif
399 /* try memcpy */
400 "test %[len], %[len]\n\t" \
401 "jz 333f\n\t" \
402 "222:\n\t" \
403 "movb (%[src]), %%al\n\t" \
404 "movb %%al, (%[dst])\n\t" \
405 "inc %[src]\n\t" \
406 "inc %[dst]\n\t" \
407 "dec %[len]\n\t" \
408 "jnz 222b\n\t" \
409 "333:\n\t" \
410 RSEQ_INJECT_ASM(5)
411 /* final store */
412 "movq %[newv], %[v]\n\t"
413 "2:\n\t"
414 RSEQ_INJECT_ASM(6)
415 /* teardown */
416 "movq %[rseq_scratch2], %[len]\n\t"
417 "movq %[rseq_scratch1], %[dst]\n\t"
418 "movq %[rseq_scratch0], %[src]\n\t"
419 RSEQ_ASM_DEFINE_ABORT(4,
420 "movq %[rseq_scratch2], %[len]\n\t"
421 "movq %[rseq_scratch1], %[dst]\n\t"
422 "movq %[rseq_scratch0], %[src]\n\t",
423 abort)
424 RSEQ_ASM_DEFINE_CMPFAIL(5,
425 "movq %[rseq_scratch2], %[len]\n\t"
426 "movq %[rseq_scratch1], %[dst]\n\t"
427 "movq %[rseq_scratch0], %[src]\n\t",
428 cmpfail)
429#ifdef RSEQ_COMPARE_TWICE
430 RSEQ_ASM_DEFINE_CMPFAIL(6,
431 "movq %[rseq_scratch2], %[len]\n\t"
432 "movq %[rseq_scratch1], %[dst]\n\t"
433 "movq %[rseq_scratch0], %[src]\n\t",
434 error1)
435 RSEQ_ASM_DEFINE_CMPFAIL(7,
436 "movq %[rseq_scratch2], %[len]\n\t"
437 "movq %[rseq_scratch1], %[dst]\n\t"
438 "movq %[rseq_scratch0], %[src]\n\t",
439 error2)
440#endif
441 : /* gcc asm goto does not allow outputs */
442 : [cpu_id] "r" (cpu),
443 [current_cpu_id] "m" (__rseq_abi.cpu_id),
444 [rseq_cs] "m" (__rseq_abi.rseq_cs),
445 /* final store input */
446 [v] "m" (*v),
447 [expect] "r" (expect),
448 [newv] "r" (newv),
449 /* try memcpy input */
450 [dst] "r" (dst),
451 [src] "r" (src),
452 [len] "r" (len),
453 [rseq_scratch0] "m" (rseq_scratch[0]),
454 [rseq_scratch1] "m" (rseq_scratch[1]),
455 [rseq_scratch2] "m" (rseq_scratch[2])
456 : "memory", "cc", "rax"
457 RSEQ_INJECT_CLOBBER
458 : abort, cmpfail
459#ifdef RSEQ_COMPARE_TWICE
460 , error1, error2
461#endif
462 );
463 return 0;
464abort:
465 RSEQ_INJECT_FAILED
466 return -1;
467cmpfail:
468 return 1;
469#ifdef RSEQ_COMPARE_TWICE
470error1:
471 rseq_bug("cpu_id comparison failed");
472error2:
473 rseq_bug("expected value comparison failed");
474#endif
475}
476
477/* x86-64 is TSO. */
478static inline __attribute__((always_inline))
479int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
480 void *dst, void *src, size_t len,
481 intptr_t newv, int cpu)
482{
483 return rseq_cmpeqv_trymemcpy_storev(v, expect, dst, src, len,
484 newv, cpu);
485}
486
487#endif /* !RSEQ_SKIP_FASTPATH */
488
489#elif __i386__
490
491#define rseq_smp_mb() \
492 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
493#define rseq_smp_rmb() \
494 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
495#define rseq_smp_wmb() \
496 __asm__ __volatile__ ("lock; addl $0,-128(%%esp)" ::: "memory", "cc")
497
498#define rseq_smp_load_acquire(p) \
499__extension__ ({ \
500 __typeof(*p) ____p1 = RSEQ_READ_ONCE(*p); \
501 rseq_smp_mb(); \
502 ____p1; \
503})
504
505#define rseq_smp_acquire__after_ctrl_dep() rseq_smp_rmb()
506
507#define rseq_smp_store_release(p, v) \
508do { \
509 rseq_smp_mb(); \
510 RSEQ_WRITE_ONCE(*p, v); \
511} while (0)
512
513#ifdef RSEQ_SKIP_FASTPATH
514#include "rseq-skip.h"
515#else /* !RSEQ_SKIP_FASTPATH */
516
517/*
518 * Use eax as scratch register and take memory operands as input to
519 * lessen register pressure. Especially needed when compiling in O0.
520 */
521#define __RSEQ_ASM_DEFINE_TABLE(label, version, flags, \
522 start_ip, post_commit_offset, abort_ip) \
523 ".pushsection __rseq_table, \"aw\"\n\t" \
524 ".balign 32\n\t" \
525 __rseq_str(label) ":\n\t" \
526 ".long " __rseq_str(version) ", " __rseq_str(flags) "\n\t" \
527 ".long " __rseq_str(start_ip) ", 0x0, " __rseq_str(post_commit_offset) ", 0x0, " __rseq_str(abort_ip) ", 0x0\n\t" \
528 ".popsection\n\t"
529
530#define RSEQ_ASM_DEFINE_TABLE(label, start_ip, post_commit_ip, abort_ip) \
531 __RSEQ_ASM_DEFINE_TABLE(label, 0x0, 0x0, start_ip, \
532 (post_commit_ip - start_ip), abort_ip)
533
534#define RSEQ_ASM_STORE_RSEQ_CS(label, cs_label, rseq_cs) \
535 RSEQ_INJECT_ASM(1) \
536 "movl $" __rseq_str(cs_label) ", %[rseq_cs]\n\t" \
537 __rseq_str(label) ":\n\t"
538
539#define RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, label) \
540 RSEQ_INJECT_ASM(2) \
541 "cmpl %[" __rseq_str(cpu_id) "], %[" __rseq_str(current_cpu_id) "]\n\t" \
542 "jnz " __rseq_str(label) "\n\t"
543
544#define RSEQ_ASM_DEFINE_ABORT(label, teardown, abort_label) \
545 ".pushsection __rseq_failure, \"ax\"\n\t" \
546 /* Disassembler-friendly signature: nopl <sig>. */ \
547 ".byte 0x0f, 0x1f, 0x05\n\t" \
548 ".long " __rseq_str(RSEQ_SIG) "\n\t" \
549 __rseq_str(label) ":\n\t" \
550 teardown \
551 "jmp %l[" __rseq_str(abort_label) "]\n\t" \
552 ".popsection\n\t"
553
554#define RSEQ_ASM_DEFINE_CMPFAIL(label, teardown, cmpfail_label) \
555 ".pushsection __rseq_failure, \"ax\"\n\t" \
556 __rseq_str(label) ":\n\t" \
557 teardown \
558 "jmp %l[" __rseq_str(cmpfail_label) "]\n\t" \
559 ".popsection\n\t"
560
561static inline __attribute__((always_inline))
562int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu)
563{
564 RSEQ_INJECT_C(9)
565
566 __asm__ __volatile__ goto (
567 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
568 /* Start rseq by storing table entry pointer into rseq_cs. */
569 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
570 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
571 RSEQ_INJECT_ASM(3)
572 "cmpl %[v], %[expect]\n\t"
573 "jnz %l[cmpfail]\n\t"
574 RSEQ_INJECT_ASM(4)
575#ifdef RSEQ_COMPARE_TWICE
576 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
577 "cmpl %[v], %[expect]\n\t"
578 "jnz %l[error2]\n\t"
579#endif
580 /* final store */
581 "movl %[newv], %[v]\n\t"
582 "2:\n\t"
583 RSEQ_INJECT_ASM(5)
584 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
585 : /* gcc asm goto does not allow outputs */
586 : [cpu_id] "r" (cpu),
587 [current_cpu_id] "m" (__rseq_abi.cpu_id),
588 [rseq_cs] "m" (__rseq_abi.rseq_cs),
589 [v] "m" (*v),
590 [expect] "r" (expect),
591 [newv] "r" (newv)
592 : "memory", "cc", "eax"
593 RSEQ_INJECT_CLOBBER
594 : abort, cmpfail
595#ifdef RSEQ_COMPARE_TWICE
596 , error1, error2
597#endif
598 );
599 return 0;
600abort:
601 RSEQ_INJECT_FAILED
602 return -1;
603cmpfail:
604 return 1;
605#ifdef RSEQ_COMPARE_TWICE
606error1:
607 rseq_bug("cpu_id comparison failed");
608error2:
609 rseq_bug("expected value comparison failed");
610#endif
611}
612
613/*
614 * Compare @v against @expectnot. When it does _not_ match, load @v
615 * into @load, and store the content of *@v + voffp into @v.
616 */
617static inline __attribute__((always_inline))
618int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot,
619 off_t voffp, intptr_t *load, int cpu)
620{
621 RSEQ_INJECT_C(9)
622
623 __asm__ __volatile__ goto (
624 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
625 /* Start rseq by storing table entry pointer into rseq_cs. */
626 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
627 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
628 RSEQ_INJECT_ASM(3)
629 "movl %[v], %%ebx\n\t"
630 "cmpl %%ebx, %[expectnot]\n\t"
631 "je %l[cmpfail]\n\t"
632 RSEQ_INJECT_ASM(4)
633#ifdef RSEQ_COMPARE_TWICE
634 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
635 "movl %[v], %%ebx\n\t"
636 "cmpl %%ebx, %[expectnot]\n\t"
637 "je %l[error2]\n\t"
638#endif
639 "movl %%ebx, %[load]\n\t"
640 "addl %[voffp], %%ebx\n\t"
641 "movl (%%ebx), %%ebx\n\t"
642 /* final store */
643 "movl %%ebx, %[v]\n\t"
644 "2:\n\t"
645 RSEQ_INJECT_ASM(5)
646 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
647 : /* gcc asm goto does not allow outputs */
648 : [cpu_id] "r" (cpu),
649 [current_cpu_id] "m" (__rseq_abi.cpu_id),
650 [rseq_cs] "m" (__rseq_abi.rseq_cs),
651 /* final store input */
652 [v] "m" (*v),
653 [expectnot] "r" (expectnot),
654 [voffp] "ir" (voffp),
655 [load] "m" (*load)
656 : "memory", "cc", "eax", "ebx"
657 RSEQ_INJECT_CLOBBER
658 : abort, cmpfail
659#ifdef RSEQ_COMPARE_TWICE
660 , error1, error2
661#endif
662 );
663 return 0;
664abort:
665 RSEQ_INJECT_FAILED
666 return -1;
667cmpfail:
668 return 1;
669#ifdef RSEQ_COMPARE_TWICE
670error1:
671 rseq_bug("cpu_id comparison failed");
672error2:
673 rseq_bug("expected value comparison failed");
674#endif
675}
676
677static inline __attribute__((always_inline))
678int rseq_addv(intptr_t *v, intptr_t count, int cpu)
679{
680 RSEQ_INJECT_C(9)
681
682 __asm__ __volatile__ goto (
683 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
684 /* Start rseq by storing table entry pointer into rseq_cs. */
685 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
686 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
687 RSEQ_INJECT_ASM(3)
688#ifdef RSEQ_COMPARE_TWICE
689 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
690#endif
691 /* final store */
692 "addl %[count], %[v]\n\t"
693 "2:\n\t"
694 RSEQ_INJECT_ASM(4)
695 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
696 : /* gcc asm goto does not allow outputs */
697 : [cpu_id] "r" (cpu),
698 [current_cpu_id] "m" (__rseq_abi.cpu_id),
699 [rseq_cs] "m" (__rseq_abi.rseq_cs),
700 /* final store input */
701 [v] "m" (*v),
702 [count] "ir" (count)
703 : "memory", "cc", "eax"
704 RSEQ_INJECT_CLOBBER
705 : abort
706#ifdef RSEQ_COMPARE_TWICE
707 , error1
708#endif
709 );
710 return 0;
711abort:
712 RSEQ_INJECT_FAILED
713 return -1;
714#ifdef RSEQ_COMPARE_TWICE
715error1:
716 rseq_bug("cpu_id comparison failed");
717#endif
718}
719
720static inline __attribute__((always_inline))
721int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect,
722 intptr_t *v2, intptr_t newv2,
723 intptr_t newv, int cpu)
724{
725 RSEQ_INJECT_C(9)
726
727 __asm__ __volatile__ goto (
728 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
729 /* Start rseq by storing table entry pointer into rseq_cs. */
730 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
731 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
732 RSEQ_INJECT_ASM(3)
733 "cmpl %[v], %[expect]\n\t"
734 "jnz %l[cmpfail]\n\t"
735 RSEQ_INJECT_ASM(4)
736#ifdef RSEQ_COMPARE_TWICE
737 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
738 "cmpl %[v], %[expect]\n\t"
739 "jnz %l[error2]\n\t"
740#endif
741 /* try store */
742 "movl %[newv2], %%eax\n\t"
743 "movl %%eax, %[v2]\n\t"
744 RSEQ_INJECT_ASM(5)
745 /* final store */
746 "movl %[newv], %[v]\n\t"
747 "2:\n\t"
748 RSEQ_INJECT_ASM(6)
749 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
750 : /* gcc asm goto does not allow outputs */
751 : [cpu_id] "r" (cpu),
752 [current_cpu_id] "m" (__rseq_abi.cpu_id),
753 [rseq_cs] "m" (__rseq_abi.rseq_cs),
754 /* try store input */
755 [v2] "m" (*v2),
756 [newv2] "m" (newv2),
757 /* final store input */
758 [v] "m" (*v),
759 [expect] "r" (expect),
760 [newv] "r" (newv)
761 : "memory", "cc", "eax"
762 RSEQ_INJECT_CLOBBER
763 : abort, cmpfail
764#ifdef RSEQ_COMPARE_TWICE
765 , error1, error2
766#endif
767 );
768 return 0;
769abort:
770 RSEQ_INJECT_FAILED
771 return -1;
772cmpfail:
773 return 1;
774#ifdef RSEQ_COMPARE_TWICE
775error1:
776 rseq_bug("cpu_id comparison failed");
777error2:
778 rseq_bug("expected value comparison failed");
779#endif
780}
781
782static inline __attribute__((always_inline))
783int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect,
784 intptr_t *v2, intptr_t newv2,
785 intptr_t newv, int cpu)
786{
787 RSEQ_INJECT_C(9)
788
789 __asm__ __volatile__ goto (
790 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
791 /* Start rseq by storing table entry pointer into rseq_cs. */
792 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
793 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
794 RSEQ_INJECT_ASM(3)
795 "movl %[expect], %%eax\n\t"
796 "cmpl %[v], %%eax\n\t"
797 "jnz %l[cmpfail]\n\t"
798 RSEQ_INJECT_ASM(4)
799#ifdef RSEQ_COMPARE_TWICE
800 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
801 "movl %[expect], %%eax\n\t"
802 "cmpl %[v], %%eax\n\t"
803 "jnz %l[error2]\n\t"
804#endif
805 /* try store */
806 "movl %[newv2], %[v2]\n\t"
807 RSEQ_INJECT_ASM(5)
808 "lock; addl $0,-128(%%esp)\n\t"
809 /* final store */
810 "movl %[newv], %[v]\n\t"
811 "2:\n\t"
812 RSEQ_INJECT_ASM(6)
813 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
814 : /* gcc asm goto does not allow outputs */
815 : [cpu_id] "r" (cpu),
816 [current_cpu_id] "m" (__rseq_abi.cpu_id),
817 [rseq_cs] "m" (__rseq_abi.rseq_cs),
818 /* try store input */
819 [v2] "m" (*v2),
820 [newv2] "r" (newv2),
821 /* final store input */
822 [v] "m" (*v),
823 [expect] "m" (expect),
824 [newv] "r" (newv)
825 : "memory", "cc", "eax"
826 RSEQ_INJECT_CLOBBER
827 : abort, cmpfail
828#ifdef RSEQ_COMPARE_TWICE
829 , error1, error2
830#endif
831 );
832 return 0;
833abort:
834 RSEQ_INJECT_FAILED
835 return -1;
836cmpfail:
837 return 1;
838#ifdef RSEQ_COMPARE_TWICE
839error1:
840 rseq_bug("cpu_id comparison failed");
841error2:
842 rseq_bug("expected value comparison failed");
843#endif
844
845}
846
847static inline __attribute__((always_inline))
848int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect,
849 intptr_t *v2, intptr_t expect2,
850 intptr_t newv, int cpu)
851{
852 RSEQ_INJECT_C(9)
853
854 __asm__ __volatile__ goto (
855 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
856 /* Start rseq by storing table entry pointer into rseq_cs. */
857 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
858 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
859 RSEQ_INJECT_ASM(3)
860 "cmpl %[v], %[expect]\n\t"
861 "jnz %l[cmpfail]\n\t"
862 RSEQ_INJECT_ASM(4)
863 "cmpl %[expect2], %[v2]\n\t"
864 "jnz %l[cmpfail]\n\t"
865 RSEQ_INJECT_ASM(5)
866#ifdef RSEQ_COMPARE_TWICE
867 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1])
868 "cmpl %[v], %[expect]\n\t"
869 "jnz %l[error2]\n\t"
870 "cmpl %[expect2], %[v2]\n\t"
871 "jnz %l[error3]\n\t"
872#endif
873 "movl %[newv], %%eax\n\t"
874 /* final store */
875 "movl %%eax, %[v]\n\t"
876 "2:\n\t"
877 RSEQ_INJECT_ASM(6)
878 RSEQ_ASM_DEFINE_ABORT(4, "", abort)
879 : /* gcc asm goto does not allow outputs */
880 : [cpu_id] "r" (cpu),
881 [current_cpu_id] "m" (__rseq_abi.cpu_id),
882 [rseq_cs] "m" (__rseq_abi.rseq_cs),
883 /* cmp2 input */
884 [v2] "m" (*v2),
885 [expect2] "r" (expect2),
886 /* final store input */
887 [v] "m" (*v),
888 [expect] "r" (expect),
889 [newv] "m" (newv)
890 : "memory", "cc", "eax"
891 RSEQ_INJECT_CLOBBER
892 : abort, cmpfail
893#ifdef RSEQ_COMPARE_TWICE
894 , error1, error2, error3
895#endif
896 );
897 return 0;
898abort:
899 RSEQ_INJECT_FAILED
900 return -1;
901cmpfail:
902 return 1;
903#ifdef RSEQ_COMPARE_TWICE
904error1:
905 rseq_bug("cpu_id comparison failed");
906error2:
907 rseq_bug("1st expected value comparison failed");
908error3:
909 rseq_bug("2nd expected value comparison failed");
910#endif
911}
912
913/* TODO: implement a faster memcpy. */
914static inline __attribute__((always_inline))
915int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect,
916 void *dst, void *src, size_t len,
917 intptr_t newv, int cpu)
918{
919 uint32_t rseq_scratch[3];
920
921 RSEQ_INJECT_C(9)
922
923 __asm__ __volatile__ goto (
924 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
925 "movl %[src], %[rseq_scratch0]\n\t"
926 "movl %[dst], %[rseq_scratch1]\n\t"
927 "movl %[len], %[rseq_scratch2]\n\t"
928 /* Start rseq by storing table entry pointer into rseq_cs. */
929 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
930 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
931 RSEQ_INJECT_ASM(3)
932 "movl %[expect], %%eax\n\t"
933 "cmpl %%eax, %[v]\n\t"
934 "jnz 5f\n\t"
935 RSEQ_INJECT_ASM(4)
936#ifdef RSEQ_COMPARE_TWICE
937 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
938 "movl %[expect], %%eax\n\t"
939 "cmpl %%eax, %[v]\n\t"
940 "jnz 7f\n\t"
941#endif
942 /* try memcpy */
943 "test %[len], %[len]\n\t" \
944 "jz 333f\n\t" \
945 "222:\n\t" \
946 "movb (%[src]), %%al\n\t" \
947 "movb %%al, (%[dst])\n\t" \
948 "inc %[src]\n\t" \
949 "inc %[dst]\n\t" \
950 "dec %[len]\n\t" \
951 "jnz 222b\n\t" \
952 "333:\n\t" \
953 RSEQ_INJECT_ASM(5)
954 "movl %[newv], %%eax\n\t"
955 /* final store */
956 "movl %%eax, %[v]\n\t"
957 "2:\n\t"
958 RSEQ_INJECT_ASM(6)
959 /* teardown */
960 "movl %[rseq_scratch2], %[len]\n\t"
961 "movl %[rseq_scratch1], %[dst]\n\t"
962 "movl %[rseq_scratch0], %[src]\n\t"
963 RSEQ_ASM_DEFINE_ABORT(4,
964 "movl %[rseq_scratch2], %[len]\n\t"
965 "movl %[rseq_scratch1], %[dst]\n\t"
966 "movl %[rseq_scratch0], %[src]\n\t",
967 abort)
968 RSEQ_ASM_DEFINE_CMPFAIL(5,
969 "movl %[rseq_scratch2], %[len]\n\t"
970 "movl %[rseq_scratch1], %[dst]\n\t"
971 "movl %[rseq_scratch0], %[src]\n\t",
972 cmpfail)
973#ifdef RSEQ_COMPARE_TWICE
974 RSEQ_ASM_DEFINE_CMPFAIL(6,
975 "movl %[rseq_scratch2], %[len]\n\t"
976 "movl %[rseq_scratch1], %[dst]\n\t"
977 "movl %[rseq_scratch0], %[src]\n\t",
978 error1)
979 RSEQ_ASM_DEFINE_CMPFAIL(7,
980 "movl %[rseq_scratch2], %[len]\n\t"
981 "movl %[rseq_scratch1], %[dst]\n\t"
982 "movl %[rseq_scratch0], %[src]\n\t",
983 error2)
984#endif
985 : /* gcc asm goto does not allow outputs */
986 : [cpu_id] "r" (cpu),
987 [current_cpu_id] "m" (__rseq_abi.cpu_id),
988 [rseq_cs] "m" (__rseq_abi.rseq_cs),
989 /* final store input */
990 [v] "m" (*v),
991 [expect] "m" (expect),
992 [newv] "m" (newv),
993 /* try memcpy input */
994 [dst] "r" (dst),
995 [src] "r" (src),
996 [len] "r" (len),
997 [rseq_scratch0] "m" (rseq_scratch[0]),
998 [rseq_scratch1] "m" (rseq_scratch[1]),
999 [rseq_scratch2] "m" (rseq_scratch[2])
1000 : "memory", "cc", "eax"
1001 RSEQ_INJECT_CLOBBER
1002 : abort, cmpfail
1003#ifdef RSEQ_COMPARE_TWICE
1004 , error1, error2
1005#endif
1006 );
1007 return 0;
1008abort:
1009 RSEQ_INJECT_FAILED
1010 return -1;
1011cmpfail:
1012 return 1;
1013#ifdef RSEQ_COMPARE_TWICE
1014error1:
1015 rseq_bug("cpu_id comparison failed");
1016error2:
1017 rseq_bug("expected value comparison failed");
1018#endif
1019}
1020
1021/* TODO: implement a faster memcpy. */
1022static inline __attribute__((always_inline))
1023int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect,
1024 void *dst, void *src, size_t len,
1025 intptr_t newv, int cpu)
1026{
1027 uint32_t rseq_scratch[3];
1028
1029 RSEQ_INJECT_C(9)
1030
1031 __asm__ __volatile__ goto (
1032 RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */
1033 "movl %[src], %[rseq_scratch0]\n\t"
1034 "movl %[dst], %[rseq_scratch1]\n\t"
1035 "movl %[len], %[rseq_scratch2]\n\t"
1036 /* Start rseq by storing table entry pointer into rseq_cs. */
1037 RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs)
1038 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f)
1039 RSEQ_INJECT_ASM(3)
1040 "movl %[expect], %%eax\n\t"
1041 "cmpl %%eax, %[v]\n\t"
1042 "jnz 5f\n\t"
1043 RSEQ_INJECT_ASM(4)
1044#ifdef RSEQ_COMPARE_TWICE
1045 RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f)
1046 "movl %[expect], %%eax\n\t"
1047 "cmpl %%eax, %[v]\n\t"
1048 "jnz 7f\n\t"
1049#endif
1050 /* try memcpy */
1051 "test %[len], %[len]\n\t" \
1052 "jz 333f\n\t" \
1053 "222:\n\t" \
1054 "movb (%[src]), %%al\n\t" \
1055 "movb %%al, (%[dst])\n\t" \
1056 "inc %[src]\n\t" \
1057 "inc %[dst]\n\t" \
1058 "dec %[len]\n\t" \
1059 "jnz 222b\n\t" \
1060 "333:\n\t" \
1061 RSEQ_INJECT_ASM(5)
1062 "lock; addl $0,-128(%%esp)\n\t"
1063 "movl %[newv], %%eax\n\t"
1064 /* final store */
1065 "movl %%eax, %[v]\n\t"
1066 "2:\n\t"
1067 RSEQ_INJECT_ASM(6)
1068 /* teardown */
1069 "movl %[rseq_scratch2], %[len]\n\t"
1070 "movl %[rseq_scratch1], %[dst]\n\t"
1071 "movl %[rseq_scratch0], %[src]\n\t"
1072 RSEQ_ASM_DEFINE_ABORT(4,
1073 "movl %[rseq_scratch2], %[len]\n\t"
1074 "movl %[rseq_scratch1], %[dst]\n\t"
1075 "movl %[rseq_scratch0], %[src]\n\t",
1076 abort)
1077 RSEQ_ASM_DEFINE_CMPFAIL(5,
1078 "movl %[rseq_scratch2], %[len]\n\t"
1079 "movl %[rseq_scratch1], %[dst]\n\t"
1080 "movl %[rseq_scratch0], %[src]\n\t",
1081 cmpfail)
1082#ifdef RSEQ_COMPARE_TWICE
1083 RSEQ_ASM_DEFINE_CMPFAIL(6,
1084 "movl %[rseq_scratch2], %[len]\n\t"
1085 "movl %[rseq_scratch1], %[dst]\n\t"
1086 "movl %[rseq_scratch0], %[src]\n\t",
1087 error1)
1088 RSEQ_ASM_DEFINE_CMPFAIL(7,
1089 "movl %[rseq_scratch2], %[len]\n\t"
1090 "movl %[rseq_scratch1], %[dst]\n\t"
1091 "movl %[rseq_scratch0], %[src]\n\t",
1092 error2)
1093#endif
1094 : /* gcc asm goto does not allow outputs */
1095 : [cpu_id] "r" (cpu),
1096 [current_cpu_id] "m" (__rseq_abi.cpu_id),
1097 [rseq_cs] "m" (__rseq_abi.rseq_cs),
1098 /* final store input */
1099 [v] "m" (*v),
1100 [expect] "m" (expect),
1101 [newv] "m" (newv),
1102 /* try memcpy input */
1103 [dst] "r" (dst),
1104 [src] "r" (src),
1105 [len] "r" (len),
1106 [rseq_scratch0] "m" (rseq_scratch[0]),
1107 [rseq_scratch1] "m" (rseq_scratch[1]),
1108 [rseq_scratch2] "m" (rseq_scratch[2])
1109 : "memory", "cc", "eax"
1110 RSEQ_INJECT_CLOBBER
1111 : abort, cmpfail
1112#ifdef RSEQ_COMPARE_TWICE
1113 , error1, error2
1114#endif
1115 );
1116 return 0;
1117abort:
1118 RSEQ_INJECT_FAILED
1119 return -1;
1120cmpfail:
1121 return 1;
1122#ifdef RSEQ_COMPARE_TWICE
1123error1:
1124 rseq_bug("cpu_id comparison failed");
1125error2:
1126 rseq_bug("expected value comparison failed");
1127#endif
1128}
1129
1130#endif /* !RSEQ_SKIP_FASTPATH */
1131
1132#endif