tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
fork atom

selftests/rseq: ppc: Template memory ordering and percpu access mode

Introduce a rseq-ppc-bits.h template header which is internally included
to generate the static inline functions covering:

- relaxed and release memory ordering,
- per-cpu-id and per-mm-cid per-cpu data access.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20221122203932.231377-16-mathieu.desnoyers@efficios.com

authored by

Mathieu Desnoyers and committed by
Peter Zijlstra e61bd94c 431b6328

+479 -572
+454
tools/testing/selftests/rseq/rseq-ppc-bits.h
··· 1 + /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 + /* 3 + * rseq-ppc-bits.h 4 + * 5 + * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 + * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> 7 + */ 8 + 9 + #include "rseq-bits-template.h" 10 + 11 + #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 12 + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 13 + 14 + static inline __attribute__((always_inline)) 15 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 16 + { 17 + RSEQ_INJECT_C(9) 18 + 19 + __asm__ __volatile__ goto ( 20 + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 21 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 22 + #ifdef RSEQ_COMPARE_TWICE 23 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 24 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 25 + #endif 26 + /* Start rseq by storing table entry pointer into rseq_cs. */ 27 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 28 + /* cmp cpuid */ 29 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 30 + RSEQ_INJECT_ASM(3) 31 + /* cmp @v equal to @expect */ 32 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 33 + RSEQ_INJECT_ASM(4) 34 + #ifdef RSEQ_COMPARE_TWICE 35 + /* cmp cpuid */ 36 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 37 + /* cmp @v equal to @expect */ 38 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 39 + #endif 40 + /* final store */ 41 + RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 42 + RSEQ_INJECT_ASM(5) 43 + RSEQ_ASM_DEFINE_ABORT(4, abort) 44 + : /* gcc asm goto does not allow outputs */ 45 + : [cpu_id] "r" (cpu), 46 + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 47 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 48 + [v] "m" (*v), 49 + [expect] "r" (expect), 50 + [newv] "r" (newv) 51 + RSEQ_INJECT_INPUT 52 + : "memory", "cc", "r17" 53 + RSEQ_INJECT_CLOBBER 54 + : abort, cmpfail 55 + #ifdef RSEQ_COMPARE_TWICE 56 + , error1, error2 57 + #endif 58 + ); 59 + rseq_after_asm_goto(); 60 + return 0; 61 + abort: 62 + rseq_after_asm_goto(); 63 + RSEQ_INJECT_FAILED 64 + return -1; 65 + cmpfail: 66 + rseq_after_asm_goto(); 67 + return 1; 68 + #ifdef RSEQ_COMPARE_TWICE 69 + error1: 70 + rseq_after_asm_goto(); 71 + rseq_bug("cpu_id comparison failed"); 72 + error2: 73 + rseq_after_asm_goto(); 74 + rseq_bug("expected value comparison failed"); 75 + #endif 76 + } 77 + 78 + static inline __attribute__((always_inline)) 79 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 80 + long voffp, intptr_t *load, int cpu) 81 + { 82 + RSEQ_INJECT_C(9) 83 + 84 + __asm__ __volatile__ goto ( 85 + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 86 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 87 + #ifdef RSEQ_COMPARE_TWICE 88 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 89 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 90 + #endif 91 + /* Start rseq by storing table entry pointer into rseq_cs. */ 92 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 93 + /* cmp cpuid */ 94 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 95 + RSEQ_INJECT_ASM(3) 96 + /* cmp @v not equal to @expectnot */ 97 + RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 98 + RSEQ_INJECT_ASM(4) 99 + #ifdef RSEQ_COMPARE_TWICE 100 + /* cmp cpuid */ 101 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 102 + /* cmp @v not equal to @expectnot */ 103 + RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 104 + #endif 105 + /* load the value of @v */ 106 + RSEQ_ASM_OP_R_LOAD(v) 107 + /* store it in @load */ 108 + RSEQ_ASM_OP_R_STORE(load) 109 + /* dereference voffp(v) */ 110 + RSEQ_ASM_OP_R_LOADX(voffp) 111 + /* final store the value at voffp(v) */ 112 + RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 113 + RSEQ_INJECT_ASM(5) 114 + RSEQ_ASM_DEFINE_ABORT(4, abort) 115 + : /* gcc asm goto does not allow outputs */ 116 + : [cpu_id] "r" (cpu), 117 + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 118 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 119 + /* final store input */ 120 + [v] "m" (*v), 121 + [expectnot] "r" (expectnot), 122 + [voffp] "b" (voffp), 123 + [load] "m" (*load) 124 + RSEQ_INJECT_INPUT 125 + : "memory", "cc", "r17" 126 + RSEQ_INJECT_CLOBBER 127 + : abort, cmpfail 128 + #ifdef RSEQ_COMPARE_TWICE 129 + , error1, error2 130 + #endif 131 + ); 132 + rseq_after_asm_goto(); 133 + return 0; 134 + abort: 135 + rseq_after_asm_goto(); 136 + RSEQ_INJECT_FAILED 137 + return -1; 138 + cmpfail: 139 + rseq_after_asm_goto(); 140 + return 1; 141 + #ifdef RSEQ_COMPARE_TWICE 142 + error1: 143 + rseq_after_asm_goto(); 144 + rseq_bug("cpu_id comparison failed"); 145 + error2: 146 + rseq_after_asm_goto(); 147 + rseq_bug("expected value comparison failed"); 148 + #endif 149 + } 150 + 151 + static inline __attribute__((always_inline)) 152 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 153 + { 154 + RSEQ_INJECT_C(9) 155 + 156 + __asm__ __volatile__ goto ( 157 + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 158 + #ifdef RSEQ_COMPARE_TWICE 159 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 160 + #endif 161 + /* Start rseq by storing table entry pointer into rseq_cs. */ 162 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 163 + /* cmp cpuid */ 164 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 165 + RSEQ_INJECT_ASM(3) 166 + #ifdef RSEQ_COMPARE_TWICE 167 + /* cmp cpuid */ 168 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 169 + #endif 170 + /* load the value of @v */ 171 + RSEQ_ASM_OP_R_LOAD(v) 172 + /* add @count to it */ 173 + RSEQ_ASM_OP_R_ADD(count) 174 + /* final store */ 175 + RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 176 + RSEQ_INJECT_ASM(4) 177 + RSEQ_ASM_DEFINE_ABORT(4, abort) 178 + : /* gcc asm goto does not allow outputs */ 179 + : [cpu_id] "r" (cpu), 180 + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 181 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 182 + /* final store input */ 183 + [v] "m" (*v), 184 + [count] "r" (count) 185 + RSEQ_INJECT_INPUT 186 + : "memory", "cc", "r17" 187 + RSEQ_INJECT_CLOBBER 188 + : abort 189 + #ifdef RSEQ_COMPARE_TWICE 190 + , error1 191 + #endif 192 + ); 193 + rseq_after_asm_goto(); 194 + return 0; 195 + abort: 196 + rseq_after_asm_goto(); 197 + RSEQ_INJECT_FAILED 198 + return -1; 199 + #ifdef RSEQ_COMPARE_TWICE 200 + error1: 201 + rseq_after_asm_goto(); 202 + rseq_bug("cpu_id comparison failed"); 203 + #endif 204 + } 205 + 206 + static inline __attribute__((always_inline)) 207 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 208 + intptr_t *v2, intptr_t expect2, 209 + intptr_t newv, int cpu) 210 + { 211 + RSEQ_INJECT_C(9) 212 + 213 + __asm__ __volatile__ goto ( 214 + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 215 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 216 + #ifdef RSEQ_COMPARE_TWICE 217 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 218 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 219 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 220 + #endif 221 + /* Start rseq by storing table entry pointer into rseq_cs. */ 222 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 223 + /* cmp cpuid */ 224 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 225 + RSEQ_INJECT_ASM(3) 226 + /* cmp @v equal to @expect */ 227 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 228 + RSEQ_INJECT_ASM(4) 229 + /* cmp @v2 equal to @expct2 */ 230 + RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 231 + RSEQ_INJECT_ASM(5) 232 + #ifdef RSEQ_COMPARE_TWICE 233 + /* cmp cpuid */ 234 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 235 + /* cmp @v equal to @expect */ 236 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 237 + /* cmp @v2 equal to @expct2 */ 238 + RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 239 + #endif 240 + /* final store */ 241 + RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 242 + RSEQ_INJECT_ASM(6) 243 + RSEQ_ASM_DEFINE_ABORT(4, abort) 244 + : /* gcc asm goto does not allow outputs */ 245 + : [cpu_id] "r" (cpu), 246 + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 247 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 248 + /* cmp2 input */ 249 + [v2] "m" (*v2), 250 + [expect2] "r" (expect2), 251 + /* final store input */ 252 + [v] "m" (*v), 253 + [expect] "r" (expect), 254 + [newv] "r" (newv) 255 + RSEQ_INJECT_INPUT 256 + : "memory", "cc", "r17" 257 + RSEQ_INJECT_CLOBBER 258 + : abort, cmpfail 259 + #ifdef RSEQ_COMPARE_TWICE 260 + , error1, error2, error3 261 + #endif 262 + ); 263 + rseq_after_asm_goto(); 264 + return 0; 265 + abort: 266 + rseq_after_asm_goto(); 267 + RSEQ_INJECT_FAILED 268 + return -1; 269 + cmpfail: 270 + rseq_after_asm_goto(); 271 + return 1; 272 + #ifdef RSEQ_COMPARE_TWICE 273 + error1: 274 + rseq_after_asm_goto(); 275 + rseq_bug("cpu_id comparison failed"); 276 + error2: 277 + rseq_after_asm_goto(); 278 + rseq_bug("1st expected value comparison failed"); 279 + error3: 280 + rseq_after_asm_goto(); 281 + rseq_bug("2nd expected value comparison failed"); 282 + #endif 283 + } 284 + 285 + #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 286 + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 287 + 288 + #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 289 + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 290 + 291 + static inline __attribute__((always_inline)) 292 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 293 + intptr_t *v2, intptr_t newv2, 294 + intptr_t newv, int cpu) 295 + { 296 + RSEQ_INJECT_C(9) 297 + 298 + __asm__ __volatile__ goto ( 299 + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 300 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 301 + #ifdef RSEQ_COMPARE_TWICE 302 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 303 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 304 + #endif 305 + /* Start rseq by storing table entry pointer into rseq_cs. */ 306 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 307 + /* cmp cpuid */ 308 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 309 + RSEQ_INJECT_ASM(3) 310 + /* cmp @v equal to @expect */ 311 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 312 + RSEQ_INJECT_ASM(4) 313 + #ifdef RSEQ_COMPARE_TWICE 314 + /* cmp cpuid */ 315 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 316 + /* cmp @v equal to @expect */ 317 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 318 + #endif 319 + /* try store */ 320 + RSEQ_ASM_OP_STORE(newv2, v2) 321 + RSEQ_INJECT_ASM(5) 322 + #ifdef RSEQ_TEMPLATE_MO_RELEASE 323 + /* for 'release' */ 324 + "lwsync\n\t" 325 + #endif 326 + /* final store */ 327 + RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 328 + RSEQ_INJECT_ASM(6) 329 + RSEQ_ASM_DEFINE_ABORT(4, abort) 330 + : /* gcc asm goto does not allow outputs */ 331 + : [cpu_id] "r" (cpu), 332 + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 333 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 334 + /* try store input */ 335 + [v2] "m" (*v2), 336 + [newv2] "r" (newv2), 337 + /* final store input */ 338 + [v] "m" (*v), 339 + [expect] "r" (expect), 340 + [newv] "r" (newv) 341 + RSEQ_INJECT_INPUT 342 + : "memory", "cc", "r17" 343 + RSEQ_INJECT_CLOBBER 344 + : abort, cmpfail 345 + #ifdef RSEQ_COMPARE_TWICE 346 + , error1, error2 347 + #endif 348 + ); 349 + rseq_after_asm_goto(); 350 + return 0; 351 + abort: 352 + rseq_after_asm_goto(); 353 + RSEQ_INJECT_FAILED 354 + return -1; 355 + cmpfail: 356 + rseq_after_asm_goto(); 357 + return 1; 358 + #ifdef RSEQ_COMPARE_TWICE 359 + error1: 360 + rseq_after_asm_goto(); 361 + rseq_bug("cpu_id comparison failed"); 362 + error2: 363 + rseq_after_asm_goto(); 364 + rseq_bug("expected value comparison failed"); 365 + #endif 366 + } 367 + 368 + static inline __attribute__((always_inline)) 369 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 370 + void *dst, void *src, size_t len, 371 + intptr_t newv, int cpu) 372 + { 373 + RSEQ_INJECT_C(9) 374 + 375 + __asm__ __volatile__ goto ( 376 + RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 377 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 378 + #ifdef RSEQ_COMPARE_TWICE 379 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 380 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 381 + #endif 382 + /* setup for mempcy */ 383 + "mr %%r19, %[len]\n\t" 384 + "mr %%r20, %[src]\n\t" 385 + "mr %%r21, %[dst]\n\t" 386 + /* Start rseq by storing table entry pointer into rseq_cs. */ 387 + RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 388 + /* cmp cpuid */ 389 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 390 + RSEQ_INJECT_ASM(3) 391 + /* cmp @v equal to @expect */ 392 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 393 + RSEQ_INJECT_ASM(4) 394 + #ifdef RSEQ_COMPARE_TWICE 395 + /* cmp cpuid */ 396 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 397 + /* cmp @v equal to @expect */ 398 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 399 + #endif 400 + /* try memcpy */ 401 + RSEQ_ASM_OP_R_MEMCPY() 402 + RSEQ_INJECT_ASM(5) 403 + #ifdef RSEQ_TEMPLATE_MO_RELEASE 404 + /* for 'release' */ 405 + "lwsync\n\t" 406 + #endif 407 + /* final store */ 408 + RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 409 + RSEQ_INJECT_ASM(6) 410 + /* teardown */ 411 + RSEQ_ASM_DEFINE_ABORT(4, abort) 412 + : /* gcc asm goto does not allow outputs */ 413 + : [cpu_id] "r" (cpu), 414 + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 415 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 416 + /* final store input */ 417 + [v] "m" (*v), 418 + [expect] "r" (expect), 419 + [newv] "r" (newv), 420 + /* try memcpy input */ 421 + [dst] "r" (dst), 422 + [src] "r" (src), 423 + [len] "r" (len) 424 + RSEQ_INJECT_INPUT 425 + : "memory", "cc", "r17", "r18", "r19", "r20", "r21" 426 + RSEQ_INJECT_CLOBBER 427 + : abort, cmpfail 428 + #ifdef RSEQ_COMPARE_TWICE 429 + , error1, error2 430 + #endif 431 + ); 432 + rseq_after_asm_goto(); 433 + return 0; 434 + abort: 435 + rseq_after_asm_goto(); 436 + RSEQ_INJECT_FAILED 437 + return -1; 438 + cmpfail: 439 + rseq_after_asm_goto(); 440 + return 1; 441 + #ifdef RSEQ_COMPARE_TWICE 442 + error1: 443 + rseq_after_asm_goto(); 444 + rseq_bug("cpu_id comparison failed"); 445 + error2: 446 + rseq_after_asm_goto(); 447 + rseq_bug("expected value comparison failed"); 448 + #endif 449 + } 450 + 451 + #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 452 + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 453 + 454 + #include "rseq-bits-reset.h"
+25 -572
tools/testing/selftests/rseq/rseq-ppc.h
··· 2 2 /* 3 3 * rseq-ppc.h 4 4 * 5 - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 5 + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 6 * (C) Copyright 2016-2018 - Boqun Feng <boqun.feng@gmail.com> 7 7 */ 8 8 ··· 205 205 RSEQ_STORE_LONG(var) "%[" __rseq_str(value) "], %[" __rseq_str(var) "]\n\t" \ 206 206 __rseq_str(post_commit_label) ":\n\t" 207 207 208 - static inline __attribute__((always_inline)) 209 - int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 210 - { 211 - RSEQ_INJECT_C(9) 208 + /* Per-cpu-id indexing. */ 212 209 213 - __asm__ __volatile__ goto ( 214 - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 215 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 216 - #ifdef RSEQ_COMPARE_TWICE 217 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 218 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 219 - #endif 220 - /* Start rseq by storing table entry pointer into rseq_cs. */ 221 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 222 - /* cmp cpuid */ 223 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 224 - RSEQ_INJECT_ASM(3) 225 - /* cmp @v equal to @expect */ 226 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 227 - RSEQ_INJECT_ASM(4) 228 - #ifdef RSEQ_COMPARE_TWICE 229 - /* cmp cpuid */ 230 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 231 - /* cmp @v equal to @expect */ 232 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 233 - #endif 234 - /* final store */ 235 - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 236 - RSEQ_INJECT_ASM(5) 237 - RSEQ_ASM_DEFINE_ABORT(4, abort) 238 - : /* gcc asm goto does not allow outputs */ 239 - : [cpu_id] "r" (cpu), 240 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 241 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 242 - [v] "m" (*v), 243 - [expect] "r" (expect), 244 - [newv] "r" (newv) 245 - RSEQ_INJECT_INPUT 246 - : "memory", "cc", "r17" 247 - RSEQ_INJECT_CLOBBER 248 - : abort, cmpfail 249 - #ifdef RSEQ_COMPARE_TWICE 250 - , error1, error2 251 - #endif 252 - ); 253 - rseq_after_asm_goto(); 254 - return 0; 255 - abort: 256 - rseq_after_asm_goto(); 257 - RSEQ_INJECT_FAILED 258 - return -1; 259 - cmpfail: 260 - rseq_after_asm_goto(); 261 - return 1; 262 - #ifdef RSEQ_COMPARE_TWICE 263 - error1: 264 - rseq_after_asm_goto(); 265 - rseq_bug("cpu_id comparison failed"); 266 - error2: 267 - rseq_after_asm_goto(); 268 - rseq_bug("expected value comparison failed"); 269 - #endif 270 - } 210 + #define RSEQ_TEMPLATE_CPU_ID 211 + #define RSEQ_TEMPLATE_MO_RELAXED 212 + #include "rseq-ppc-bits.h" 213 + #undef RSEQ_TEMPLATE_MO_RELAXED 271 214 272 - static inline __attribute__((always_inline)) 273 - int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 274 - long voffp, intptr_t *load, int cpu) 275 - { 276 - RSEQ_INJECT_C(9) 215 + #define RSEQ_TEMPLATE_MO_RELEASE 216 + #include "rseq-ppc-bits.h" 217 + #undef RSEQ_TEMPLATE_MO_RELEASE 218 + #undef RSEQ_TEMPLATE_CPU_ID 277 219 278 - __asm__ __volatile__ goto ( 279 - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 280 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 281 - #ifdef RSEQ_COMPARE_TWICE 282 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 283 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 284 - #endif 285 - /* Start rseq by storing table entry pointer into rseq_cs. */ 286 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 287 - /* cmp cpuid */ 288 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 289 - RSEQ_INJECT_ASM(3) 290 - /* cmp @v not equal to @expectnot */ 291 - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 292 - RSEQ_INJECT_ASM(4) 293 - #ifdef RSEQ_COMPARE_TWICE 294 - /* cmp cpuid */ 295 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 296 - /* cmp @v not equal to @expectnot */ 297 - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 298 - #endif 299 - /* load the value of @v */ 300 - RSEQ_ASM_OP_R_LOAD(v) 301 - /* store it in @load */ 302 - RSEQ_ASM_OP_R_STORE(load) 303 - /* dereference voffp(v) */ 304 - RSEQ_ASM_OP_R_LOADX(voffp) 305 - /* final store the value at voffp(v) */ 306 - RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 307 - RSEQ_INJECT_ASM(5) 308 - RSEQ_ASM_DEFINE_ABORT(4, abort) 309 - : /* gcc asm goto does not allow outputs */ 310 - : [cpu_id] "r" (cpu), 311 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 312 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 313 - /* final store input */ 314 - [v] "m" (*v), 315 - [expectnot] "r" (expectnot), 316 - [voffp] "b" (voffp), 317 - [load] "m" (*load) 318 - RSEQ_INJECT_INPUT 319 - : "memory", "cc", "r17" 320 - RSEQ_INJECT_CLOBBER 321 - : abort, cmpfail 322 - #ifdef RSEQ_COMPARE_TWICE 323 - , error1, error2 324 - #endif 325 - ); 326 - rseq_after_asm_goto(); 327 - return 0; 328 - abort: 329 - rseq_after_asm_goto(); 330 - RSEQ_INJECT_FAILED 331 - return -1; 332 - cmpfail: 333 - rseq_after_asm_goto(); 334 - return 1; 335 - #ifdef RSEQ_COMPARE_TWICE 336 - error1: 337 - rseq_after_asm_goto(); 338 - rseq_bug("cpu_id comparison failed"); 339 - error2: 340 - rseq_after_asm_goto(); 341 - rseq_bug("expected value comparison failed"); 342 - #endif 343 - } 220 + /* Per-mm-cid indexing. */ 344 221 345 - static inline __attribute__((always_inline)) 346 - int rseq_addv(intptr_t *v, intptr_t count, int cpu) 347 - { 348 - RSEQ_INJECT_C(9) 222 + #define RSEQ_TEMPLATE_MM_CID 223 + #define RSEQ_TEMPLATE_MO_RELAXED 224 + #include "rseq-ppc-bits.h" 225 + #undef RSEQ_TEMPLATE_MO_RELAXED 349 226 350 - __asm__ __volatile__ goto ( 351 - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 352 - #ifdef RSEQ_COMPARE_TWICE 353 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 354 - #endif 355 - /* Start rseq by storing table entry pointer into rseq_cs. */ 356 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 357 - /* cmp cpuid */ 358 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 359 - RSEQ_INJECT_ASM(3) 360 - #ifdef RSEQ_COMPARE_TWICE 361 - /* cmp cpuid */ 362 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 363 - #endif 364 - /* load the value of @v */ 365 - RSEQ_ASM_OP_R_LOAD(v) 366 - /* add @count to it */ 367 - RSEQ_ASM_OP_R_ADD(count) 368 - /* final store */ 369 - RSEQ_ASM_OP_R_FINAL_STORE(v, 2) 370 - RSEQ_INJECT_ASM(4) 371 - RSEQ_ASM_DEFINE_ABORT(4, abort) 372 - : /* gcc asm goto does not allow outputs */ 373 - : [cpu_id] "r" (cpu), 374 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 375 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 376 - /* final store input */ 377 - [v] "m" (*v), 378 - [count] "r" (count) 379 - RSEQ_INJECT_INPUT 380 - : "memory", "cc", "r17" 381 - RSEQ_INJECT_CLOBBER 382 - : abort 383 - #ifdef RSEQ_COMPARE_TWICE 384 - , error1 385 - #endif 386 - ); 387 - rseq_after_asm_goto(); 388 - return 0; 389 - abort: 390 - rseq_after_asm_goto(); 391 - RSEQ_INJECT_FAILED 392 - return -1; 393 - #ifdef RSEQ_COMPARE_TWICE 394 - error1: 395 - rseq_after_asm_goto(); 396 - rseq_bug("cpu_id comparison failed"); 397 - #endif 398 - } 227 + #define RSEQ_TEMPLATE_MO_RELEASE 228 + #include "rseq-ppc-bits.h" 229 + #undef RSEQ_TEMPLATE_MO_RELEASE 230 + #undef RSEQ_TEMPLATE_MM_CID 399 231 400 - static inline __attribute__((always_inline)) 401 - int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 402 - intptr_t *v2, intptr_t newv2, 403 - intptr_t newv, int cpu) 404 - { 405 - RSEQ_INJECT_C(9) 232 + /* APIs which are not based on cpu ids. */ 406 233 407 - __asm__ __volatile__ goto ( 408 - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 409 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 410 - #ifdef RSEQ_COMPARE_TWICE 411 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 412 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 413 - #endif 414 - /* Start rseq by storing table entry pointer into rseq_cs. */ 415 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 416 - /* cmp cpuid */ 417 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 418 - RSEQ_INJECT_ASM(3) 419 - /* cmp @v equal to @expect */ 420 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 421 - RSEQ_INJECT_ASM(4) 422 - #ifdef RSEQ_COMPARE_TWICE 423 - /* cmp cpuid */ 424 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 425 - /* cmp @v equal to @expect */ 426 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 427 - #endif 428 - /* try store */ 429 - RSEQ_ASM_OP_STORE(newv2, v2) 430 - RSEQ_INJECT_ASM(5) 431 - /* final store */ 432 - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 433 - RSEQ_INJECT_ASM(6) 434 - RSEQ_ASM_DEFINE_ABORT(4, abort) 435 - : /* gcc asm goto does not allow outputs */ 436 - : [cpu_id] "r" (cpu), 437 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 438 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 439 - /* try store input */ 440 - [v2] "m" (*v2), 441 - [newv2] "r" (newv2), 442 - /* final store input */ 443 - [v] "m" (*v), 444 - [expect] "r" (expect), 445 - [newv] "r" (newv) 446 - RSEQ_INJECT_INPUT 447 - : "memory", "cc", "r17" 448 - RSEQ_INJECT_CLOBBER 449 - : abort, cmpfail 450 - #ifdef RSEQ_COMPARE_TWICE 451 - , error1, error2 452 - #endif 453 - ); 454 - rseq_after_asm_goto(); 455 - return 0; 456 - abort: 457 - rseq_after_asm_goto(); 458 - RSEQ_INJECT_FAILED 459 - return -1; 460 - cmpfail: 461 - rseq_after_asm_goto(); 462 - return 1; 463 - #ifdef RSEQ_COMPARE_TWICE 464 - error1: 465 - rseq_after_asm_goto(); 466 - rseq_bug("cpu_id comparison failed"); 467 - error2: 468 - rseq_after_asm_goto(); 469 - rseq_bug("expected value comparison failed"); 470 - #endif 471 - } 472 - 473 - static inline __attribute__((always_inline)) 474 - int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 475 - intptr_t *v2, intptr_t newv2, 476 - intptr_t newv, int cpu) 477 - { 478 - RSEQ_INJECT_C(9) 479 - 480 - __asm__ __volatile__ goto ( 481 - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 482 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 483 - #ifdef RSEQ_COMPARE_TWICE 484 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 485 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 486 - #endif 487 - /* Start rseq by storing table entry pointer into rseq_cs. */ 488 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 489 - /* cmp cpuid */ 490 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 491 - RSEQ_INJECT_ASM(3) 492 - /* cmp @v equal to @expect */ 493 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 494 - RSEQ_INJECT_ASM(4) 495 - #ifdef RSEQ_COMPARE_TWICE 496 - /* cmp cpuid */ 497 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 498 - /* cmp @v equal to @expect */ 499 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 500 - #endif 501 - /* try store */ 502 - RSEQ_ASM_OP_STORE(newv2, v2) 503 - RSEQ_INJECT_ASM(5) 504 - /* for 'release' */ 505 - "lwsync\n\t" 506 - /* final store */ 507 - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 508 - RSEQ_INJECT_ASM(6) 509 - RSEQ_ASM_DEFINE_ABORT(4, abort) 510 - : /* gcc asm goto does not allow outputs */ 511 - : [cpu_id] "r" (cpu), 512 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 513 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 514 - /* try store input */ 515 - [v2] "m" (*v2), 516 - [newv2] "r" (newv2), 517 - /* final store input */ 518 - [v] "m" (*v), 519 - [expect] "r" (expect), 520 - [newv] "r" (newv) 521 - RSEQ_INJECT_INPUT 522 - : "memory", "cc", "r17" 523 - RSEQ_INJECT_CLOBBER 524 - : abort, cmpfail 525 - #ifdef RSEQ_COMPARE_TWICE 526 - , error1, error2 527 - #endif 528 - ); 529 - rseq_after_asm_goto(); 530 - return 0; 531 - abort: 532 - rseq_after_asm_goto(); 533 - RSEQ_INJECT_FAILED 534 - return -1; 535 - cmpfail: 536 - rseq_after_asm_goto(); 537 - return 1; 538 - #ifdef RSEQ_COMPARE_TWICE 539 - error1: 540 - rseq_after_asm_goto(); 541 - rseq_bug("cpu_id comparison failed"); 542 - error2: 543 - rseq_after_asm_goto(); 544 - rseq_bug("expected value comparison failed"); 545 - #endif 546 - } 547 - 548 - static inline __attribute__((always_inline)) 549 - int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 550 - intptr_t *v2, intptr_t expect2, 551 - intptr_t newv, int cpu) 552 - { 553 - RSEQ_INJECT_C(9) 554 - 555 - __asm__ __volatile__ goto ( 556 - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 557 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 558 - #ifdef RSEQ_COMPARE_TWICE 559 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 560 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 561 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 562 - #endif 563 - /* Start rseq by storing table entry pointer into rseq_cs. */ 564 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 565 - /* cmp cpuid */ 566 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 567 - RSEQ_INJECT_ASM(3) 568 - /* cmp @v equal to @expect */ 569 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 570 - RSEQ_INJECT_ASM(4) 571 - /* cmp @v2 equal to @expct2 */ 572 - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 573 - RSEQ_INJECT_ASM(5) 574 - #ifdef RSEQ_COMPARE_TWICE 575 - /* cmp cpuid */ 576 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 577 - /* cmp @v equal to @expect */ 578 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 579 - /* cmp @v2 equal to @expct2 */ 580 - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 581 - #endif 582 - /* final store */ 583 - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 584 - RSEQ_INJECT_ASM(6) 585 - RSEQ_ASM_DEFINE_ABORT(4, abort) 586 - : /* gcc asm goto does not allow outputs */ 587 - : [cpu_id] "r" (cpu), 588 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 589 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 590 - /* cmp2 input */ 591 - [v2] "m" (*v2), 592 - [expect2] "r" (expect2), 593 - /* final store input */ 594 - [v] "m" (*v), 595 - [expect] "r" (expect), 596 - [newv] "r" (newv) 597 - RSEQ_INJECT_INPUT 598 - : "memory", "cc", "r17" 599 - RSEQ_INJECT_CLOBBER 600 - : abort, cmpfail 601 - #ifdef RSEQ_COMPARE_TWICE 602 - , error1, error2, error3 603 - #endif 604 - ); 605 - rseq_after_asm_goto(); 606 - return 0; 607 - abort: 608 - rseq_after_asm_goto(); 609 - RSEQ_INJECT_FAILED 610 - return -1; 611 - cmpfail: 612 - rseq_after_asm_goto(); 613 - return 1; 614 - #ifdef RSEQ_COMPARE_TWICE 615 - error1: 616 - rseq_after_asm_goto(); 617 - rseq_bug("cpu_id comparison failed"); 618 - error2: 619 - rseq_after_asm_goto(); 620 - rseq_bug("1st expected value comparison failed"); 621 - error3: 622 - rseq_after_asm_goto(); 623 - rseq_bug("2nd expected value comparison failed"); 624 - #endif 625 - } 626 - 627 - static inline __attribute__((always_inline)) 628 - int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 629 - void *dst, void *src, size_t len, 630 - intptr_t newv, int cpu) 631 - { 632 - RSEQ_INJECT_C(9) 633 - 634 - __asm__ __volatile__ goto ( 635 - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 636 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 637 - #ifdef RSEQ_COMPARE_TWICE 638 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 639 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 640 - #endif 641 - /* setup for mempcy */ 642 - "mr %%r19, %[len]\n\t" 643 - "mr %%r20, %[src]\n\t" 644 - "mr %%r21, %[dst]\n\t" 645 - /* Start rseq by storing table entry pointer into rseq_cs. */ 646 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 647 - /* cmp cpuid */ 648 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 649 - RSEQ_INJECT_ASM(3) 650 - /* cmp @v equal to @expect */ 651 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 652 - RSEQ_INJECT_ASM(4) 653 - #ifdef RSEQ_COMPARE_TWICE 654 - /* cmp cpuid */ 655 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 656 - /* cmp @v equal to @expect */ 657 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 658 - #endif 659 - /* try memcpy */ 660 - RSEQ_ASM_OP_R_MEMCPY() 661 - RSEQ_INJECT_ASM(5) 662 - /* final store */ 663 - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 664 - RSEQ_INJECT_ASM(6) 665 - /* teardown */ 666 - RSEQ_ASM_DEFINE_ABORT(4, abort) 667 - : /* gcc asm goto does not allow outputs */ 668 - : [cpu_id] "r" (cpu), 669 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 670 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 671 - /* final store input */ 672 - [v] "m" (*v), 673 - [expect] "r" (expect), 674 - [newv] "r" (newv), 675 - /* try memcpy input */ 676 - [dst] "r" (dst), 677 - [src] "r" (src), 678 - [len] "r" (len) 679 - RSEQ_INJECT_INPUT 680 - : "memory", "cc", "r17", "r18", "r19", "r20", "r21" 681 - RSEQ_INJECT_CLOBBER 682 - : abort, cmpfail 683 - #ifdef RSEQ_COMPARE_TWICE 684 - , error1, error2 685 - #endif 686 - ); 687 - rseq_after_asm_goto(); 688 - return 0; 689 - abort: 690 - rseq_after_asm_goto(); 691 - RSEQ_INJECT_FAILED 692 - return -1; 693 - cmpfail: 694 - rseq_after_asm_goto(); 695 - return 1; 696 - #ifdef RSEQ_COMPARE_TWICE 697 - error1: 698 - rseq_after_asm_goto(); 699 - rseq_bug("cpu_id comparison failed"); 700 - error2: 701 - rseq_after_asm_goto(); 702 - rseq_bug("expected value comparison failed"); 703 - #endif 704 - } 705 - 706 - static inline __attribute__((always_inline)) 707 - int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 708 - void *dst, void *src, size_t len, 709 - intptr_t newv, int cpu) 710 - { 711 - RSEQ_INJECT_C(9) 712 - 713 - __asm__ __volatile__ goto ( 714 - RSEQ_ASM_DEFINE_TABLE(3, 1f, 2f, 4f) /* start, commit, abort */ 715 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 716 - #ifdef RSEQ_COMPARE_TWICE 717 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 718 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 719 - #endif 720 - /* setup for mempcy */ 721 - "mr %%r19, %[len]\n\t" 722 - "mr %%r20, %[src]\n\t" 723 - "mr %%r21, %[dst]\n\t" 724 - /* Start rseq by storing table entry pointer into rseq_cs. */ 725 - RSEQ_ASM_STORE_RSEQ_CS(1, 3b, rseq_cs) 726 - /* cmp cpuid */ 727 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 728 - RSEQ_INJECT_ASM(3) 729 - /* cmp @v equal to @expect */ 730 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 731 - RSEQ_INJECT_ASM(4) 732 - #ifdef RSEQ_COMPARE_TWICE 733 - /* cmp cpuid */ 734 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 735 - /* cmp @v equal to @expect */ 736 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 737 - #endif 738 - /* try memcpy */ 739 - RSEQ_ASM_OP_R_MEMCPY() 740 - RSEQ_INJECT_ASM(5) 741 - /* for 'release' */ 742 - "lwsync\n\t" 743 - /* final store */ 744 - RSEQ_ASM_OP_FINAL_STORE(newv, v, 2) 745 - RSEQ_INJECT_ASM(6) 746 - /* teardown */ 747 - RSEQ_ASM_DEFINE_ABORT(4, abort) 748 - : /* gcc asm goto does not allow outputs */ 749 - : [cpu_id] "r" (cpu), 750 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 751 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 752 - /* final store input */ 753 - [v] "m" (*v), 754 - [expect] "r" (expect), 755 - [newv] "r" (newv), 756 - /* try memcpy input */ 757 - [dst] "r" (dst), 758 - [src] "r" (src), 759 - [len] "r" (len) 760 - RSEQ_INJECT_INPUT 761 - : "memory", "cc", "r17", "r18", "r19", "r20", "r21" 762 - RSEQ_INJECT_CLOBBER 763 - : abort, cmpfail 764 - #ifdef RSEQ_COMPARE_TWICE 765 - , error1, error2 766 - #endif 767 - ); 768 - rseq_after_asm_goto(); 769 - return 0; 770 - abort: 771 - rseq_after_asm_goto(); 772 - RSEQ_INJECT_FAILED 773 - return -1; 774 - cmpfail: 775 - rseq_after_asm_goto(); 776 - return 1; 777 - #ifdef RSEQ_COMPARE_TWICE 778 - error1: 779 - rseq_after_asm_goto(); 780 - rseq_bug("cpu_id comparison failed"); 781 - error2: 782 - rseq_after_asm_goto(); 783 - rseq_bug("expected value comparison failed"); 784 - #endif 785 - } 234 + #define RSEQ_TEMPLATE_CPU_ID_NONE 235 + #define RSEQ_TEMPLATE_MO_RELAXED 236 + #include "rseq-ppc-bits.h" 237 + #undef RSEQ_TEMPLATE_MO_RELAXED 238 + #undef RSEQ_TEMPLATE_CPU_ID_NONE