selftests/rseq: arm: Template memory ordering and percpu access mode

Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git

kernel os linux

Introduce a rseq-arm-bits.h template header which is internally included
to generate the static inline functions covering:

- relaxed and release memory ordering,
- per-cpu-id and per-mm-cid per-cpu data access.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20221122203932.231377-13-mathieu.desnoyers@efficios.com

authored by

Mathieu Desnoyers and committed by

Peter Zijlstra 3 years ago 5bf4aba3 ae315738

+530 -670

2 changed files

expand all

tools

testing

selftests

rseq

rseq-arm-bits.h

rseq-arm.h

+505

tools/testing/selftests/rseq/rseq-arm-bits.h

··· 1 + /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 + /* 3 + * rseq-arm-bits.h 4 + * 5 + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 + */ 7 + 8 + #include "rseq-bits-template.h" 9 + 10 + #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 11 + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 12 + 13 + static inline __attribute__((always_inline)) 14 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 15 + { 16 + RSEQ_INJECT_C(9) 17 + 18 + __asm__ __volatile__ goto ( 19 + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 20 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 21 + #ifdef RSEQ_COMPARE_TWICE 22 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 23 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 24 + #endif 25 + /* Start rseq by storing table entry pointer into rseq_cs. */ 26 + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 27 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 28 + RSEQ_INJECT_ASM(3) 29 + "ldr r0, %[v]\n\t" 30 + "cmp %[expect], r0\n\t" 31 + "bne %l[cmpfail]\n\t" 32 + RSEQ_INJECT_ASM(4) 33 + #ifdef RSEQ_COMPARE_TWICE 34 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 35 + "ldr r0, %[v]\n\t" 36 + "cmp %[expect], r0\n\t" 37 + "bne %l[error2]\n\t" 38 + #endif 39 + /* final store */ 40 + "str %[newv], %[v]\n\t" 41 + "2:\n\t" 42 + RSEQ_INJECT_ASM(5) 43 + "b 5f\n\t" 44 + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 45 + "5:\n\t" 46 + : /* gcc asm goto does not allow outputs */ 47 + : [cpu_id] "r" (cpu), 48 + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 49 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 50 + [v] "m" (*v), 51 + [expect] "r" (expect), 52 + [newv] "r" (newv) 53 + RSEQ_INJECT_INPUT 54 + : "r0", "memory", "cc" 55 + RSEQ_INJECT_CLOBBER 56 + : abort, cmpfail 57 + #ifdef RSEQ_COMPARE_TWICE 58 + , error1, error2 59 + #endif 60 + ); 61 + rseq_after_asm_goto(); 62 + return 0; 63 + abort: 64 + rseq_after_asm_goto(); 65 + RSEQ_INJECT_FAILED 66 + return -1; 67 + cmpfail: 68 + rseq_after_asm_goto(); 69 + return 1; 70 + #ifdef RSEQ_COMPARE_TWICE 71 + error1: 72 + rseq_after_asm_goto(); 73 + rseq_bug("cpu_id comparison failed"); 74 + error2: 75 + rseq_after_asm_goto(); 76 + rseq_bug("expected value comparison failed"); 77 + #endif 78 + } 79 + 80 + static inline __attribute__((always_inline)) 81 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 82 + long voffp, intptr_t *load, int cpu) 83 + { 84 + RSEQ_INJECT_C(9) 85 + 86 + __asm__ __volatile__ goto ( 87 + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 88 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 89 + #ifdef RSEQ_COMPARE_TWICE 90 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 91 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 92 + #endif 93 + /* Start rseq by storing table entry pointer into rseq_cs. */ 94 + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 95 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 96 + RSEQ_INJECT_ASM(3) 97 + "ldr r0, %[v]\n\t" 98 + "cmp %[expectnot], r0\n\t" 99 + "beq %l[cmpfail]\n\t" 100 + RSEQ_INJECT_ASM(4) 101 + #ifdef RSEQ_COMPARE_TWICE 102 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 103 + "ldr r0, %[v]\n\t" 104 + "cmp %[expectnot], r0\n\t" 105 + "beq %l[error2]\n\t" 106 + #endif 107 + "str r0, %[load]\n\t" 108 + "add r0, %[voffp]\n\t" 109 + "ldr r0, [r0]\n\t" 110 + /* final store */ 111 + "str r0, %[v]\n\t" 112 + "2:\n\t" 113 + RSEQ_INJECT_ASM(5) 114 + "b 5f\n\t" 115 + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 116 + "5:\n\t" 117 + : /* gcc asm goto does not allow outputs */ 118 + : [cpu_id] "r" (cpu), 119 + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 120 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 121 + /* final store input */ 122 + [v] "m" (*v), 123 + [expectnot] "r" (expectnot), 124 + [voffp] "Ir" (voffp), 125 + [load] "m" (*load) 126 + RSEQ_INJECT_INPUT 127 + : "r0", "memory", "cc" 128 + RSEQ_INJECT_CLOBBER 129 + : abort, cmpfail 130 + #ifdef RSEQ_COMPARE_TWICE 131 + , error1, error2 132 + #endif 133 + ); 134 + rseq_after_asm_goto(); 135 + return 0; 136 + abort: 137 + rseq_after_asm_goto(); 138 + RSEQ_INJECT_FAILED 139 + return -1; 140 + cmpfail: 141 + rseq_after_asm_goto(); 142 + return 1; 143 + #ifdef RSEQ_COMPARE_TWICE 144 + error1: 145 + rseq_after_asm_goto(); 146 + rseq_bug("cpu_id comparison failed"); 147 + error2: 148 + rseq_after_asm_goto(); 149 + rseq_bug("expected value comparison failed"); 150 + #endif 151 + } 152 + 153 + static inline __attribute__((always_inline)) 154 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 155 + { 156 + RSEQ_INJECT_C(9) 157 + 158 + __asm__ __volatile__ goto ( 159 + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 160 + #ifdef RSEQ_COMPARE_TWICE 161 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 162 + #endif 163 + /* Start rseq by storing table entry pointer into rseq_cs. */ 164 + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 165 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 166 + RSEQ_INJECT_ASM(3) 167 + #ifdef RSEQ_COMPARE_TWICE 168 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 169 + #endif 170 + "ldr r0, %[v]\n\t" 171 + "add r0, %[count]\n\t" 172 + /* final store */ 173 + "str r0, %[v]\n\t" 174 + "2:\n\t" 175 + RSEQ_INJECT_ASM(4) 176 + "b 5f\n\t" 177 + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 178 + "5:\n\t" 179 + : /* gcc asm goto does not allow outputs */ 180 + : [cpu_id] "r" (cpu), 181 + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 182 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 183 + [v] "m" (*v), 184 + [count] "Ir" (count) 185 + RSEQ_INJECT_INPUT 186 + : "r0", "memory", "cc" 187 + RSEQ_INJECT_CLOBBER 188 + : abort 189 + #ifdef RSEQ_COMPARE_TWICE 190 + , error1 191 + #endif 192 + ); 193 + rseq_after_asm_goto(); 194 + return 0; 195 + abort: 196 + rseq_after_asm_goto(); 197 + RSEQ_INJECT_FAILED 198 + return -1; 199 + #ifdef RSEQ_COMPARE_TWICE 200 + error1: 201 + rseq_after_asm_goto(); 202 + rseq_bug("cpu_id comparison failed"); 203 + #endif 204 + } 205 + 206 + static inline __attribute__((always_inline)) 207 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 208 + intptr_t *v2, intptr_t expect2, 209 + intptr_t newv, int cpu) 210 + { 211 + RSEQ_INJECT_C(9) 212 + 213 + __asm__ __volatile__ goto ( 214 + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 215 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 216 + #ifdef RSEQ_COMPARE_TWICE 217 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 218 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 219 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 220 + #endif 221 + /* Start rseq by storing table entry pointer into rseq_cs. */ 222 + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 223 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 224 + RSEQ_INJECT_ASM(3) 225 + "ldr r0, %[v]\n\t" 226 + "cmp %[expect], r0\n\t" 227 + "bne %l[cmpfail]\n\t" 228 + RSEQ_INJECT_ASM(4) 229 + "ldr r0, %[v2]\n\t" 230 + "cmp %[expect2], r0\n\t" 231 + "bne %l[cmpfail]\n\t" 232 + RSEQ_INJECT_ASM(5) 233 + #ifdef RSEQ_COMPARE_TWICE 234 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 235 + "ldr r0, %[v]\n\t" 236 + "cmp %[expect], r0\n\t" 237 + "bne %l[error2]\n\t" 238 + "ldr r0, %[v2]\n\t" 239 + "cmp %[expect2], r0\n\t" 240 + "bne %l[error3]\n\t" 241 + #endif 242 + /* final store */ 243 + "str %[newv], %[v]\n\t" 244 + "2:\n\t" 245 + RSEQ_INJECT_ASM(6) 246 + "b 5f\n\t" 247 + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 248 + "5:\n\t" 249 + : /* gcc asm goto does not allow outputs */ 250 + : [cpu_id] "r" (cpu), 251 + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 252 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 253 + /* cmp2 input */ 254 + [v2] "m" (*v2), 255 + [expect2] "r" (expect2), 256 + /* final store input */ 257 + [v] "m" (*v), 258 + [expect] "r" (expect), 259 + [newv] "r" (newv) 260 + RSEQ_INJECT_INPUT 261 + : "r0", "memory", "cc" 262 + RSEQ_INJECT_CLOBBER 263 + : abort, cmpfail 264 + #ifdef RSEQ_COMPARE_TWICE 265 + , error1, error2, error3 266 + #endif 267 + ); 268 + rseq_after_asm_goto(); 269 + return 0; 270 + abort: 271 + rseq_after_asm_goto(); 272 + RSEQ_INJECT_FAILED 273 + return -1; 274 + cmpfail: 275 + rseq_after_asm_goto(); 276 + return 1; 277 + #ifdef RSEQ_COMPARE_TWICE 278 + error1: 279 + rseq_after_asm_goto(); 280 + rseq_bug("cpu_id comparison failed"); 281 + error2: 282 + rseq_after_asm_goto(); 283 + rseq_bug("1st expected value comparison failed"); 284 + error3: 285 + rseq_after_asm_goto(); 286 + rseq_bug("2nd expected value comparison failed"); 287 + #endif 288 + } 289 + 290 + #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 291 + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 292 + 293 + #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 294 + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 295 + 296 + static inline __attribute__((always_inline)) 297 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 298 + intptr_t *v2, intptr_t newv2, 299 + intptr_t newv, int cpu) 300 + { 301 + RSEQ_INJECT_C(9) 302 + 303 + __asm__ __volatile__ goto ( 304 + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 305 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 306 + #ifdef RSEQ_COMPARE_TWICE 307 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 308 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 309 + #endif 310 + /* Start rseq by storing table entry pointer into rseq_cs. */ 311 + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 312 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 313 + RSEQ_INJECT_ASM(3) 314 + "ldr r0, %[v]\n\t" 315 + "cmp %[expect], r0\n\t" 316 + "bne %l[cmpfail]\n\t" 317 + RSEQ_INJECT_ASM(4) 318 + #ifdef RSEQ_COMPARE_TWICE 319 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 320 + "ldr r0, %[v]\n\t" 321 + "cmp %[expect], r0\n\t" 322 + "bne %l[error2]\n\t" 323 + #endif 324 + /* try store */ 325 + "str %[newv2], %[v2]\n\t" 326 + RSEQ_INJECT_ASM(5) 327 + #ifdef RSEQ_TEMPLATE_MO_RELEASE 328 + "dmb\n\t" /* full mb provides store-release */ 329 + #endif 330 + /* final store */ 331 + "str %[newv], %[v]\n\t" 332 + "2:\n\t" 333 + RSEQ_INJECT_ASM(6) 334 + "b 5f\n\t" 335 + RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 336 + "5:\n\t" 337 + : /* gcc asm goto does not allow outputs */ 338 + : [cpu_id] "r" (cpu), 339 + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 340 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 341 + /* try store input */ 342 + [v2] "m" (*v2), 343 + [newv2] "r" (newv2), 344 + /* final store input */ 345 + [v] "m" (*v), 346 + [expect] "r" (expect), 347 + [newv] "r" (newv) 348 + RSEQ_INJECT_INPUT 349 + : "r0", "memory", "cc" 350 + RSEQ_INJECT_CLOBBER 351 + : abort, cmpfail 352 + #ifdef RSEQ_COMPARE_TWICE 353 + , error1, error2 354 + #endif 355 + ); 356 + rseq_after_asm_goto(); 357 + return 0; 358 + abort: 359 + rseq_after_asm_goto(); 360 + RSEQ_INJECT_FAILED 361 + return -1; 362 + cmpfail: 363 + rseq_after_asm_goto(); 364 + return 1; 365 + #ifdef RSEQ_COMPARE_TWICE 366 + error1: 367 + rseq_after_asm_goto(); 368 + rseq_bug("cpu_id comparison failed"); 369 + error2: 370 + rseq_after_asm_goto(); 371 + rseq_bug("expected value comparison failed"); 372 + #endif 373 + } 374 + 375 + 376 + static inline __attribute__((always_inline)) 377 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 378 + void *dst, void *src, size_t len, 379 + intptr_t newv, int cpu) 380 + { 381 + uint32_t rseq_scratch[3]; 382 + 383 + RSEQ_INJECT_C(9) 384 + 385 + __asm__ __volatile__ goto ( 386 + RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 387 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 388 + #ifdef RSEQ_COMPARE_TWICE 389 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 390 + RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 391 + #endif 392 + "str %[src], %[rseq_scratch0]\n\t" 393 + "str %[dst], %[rseq_scratch1]\n\t" 394 + "str %[len], %[rseq_scratch2]\n\t" 395 + /* Start rseq by storing table entry pointer into rseq_cs. */ 396 + RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 397 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 398 + RSEQ_INJECT_ASM(3) 399 + "ldr r0, %[v]\n\t" 400 + "cmp %[expect], r0\n\t" 401 + "bne 5f\n\t" 402 + RSEQ_INJECT_ASM(4) 403 + #ifdef RSEQ_COMPARE_TWICE 404 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 405 + "ldr r0, %[v]\n\t" 406 + "cmp %[expect], r0\n\t" 407 + "bne 7f\n\t" 408 + #endif 409 + /* try memcpy */ 410 + "cmp %[len], #0\n\t" \ 411 + "beq 333f\n\t" \ 412 + "222:\n\t" \ 413 + "ldrb %%r0, [%[src]]\n\t" \ 414 + "strb %%r0, [%[dst]]\n\t" \ 415 + "adds %[src], #1\n\t" \ 416 + "adds %[dst], #1\n\t" \ 417 + "subs %[len], #1\n\t" \ 418 + "bne 222b\n\t" \ 419 + "333:\n\t" \ 420 + RSEQ_INJECT_ASM(5) 421 + #ifdef RSEQ_TEMPLATE_MO_RELEASE 422 + "dmb\n\t" /* full mb provides store-release */ 423 + #endif 424 + /* final store */ 425 + "str %[newv], %[v]\n\t" 426 + "2:\n\t" 427 + RSEQ_INJECT_ASM(6) 428 + /* teardown */ 429 + "ldr %[len], %[rseq_scratch2]\n\t" 430 + "ldr %[dst], %[rseq_scratch1]\n\t" 431 + "ldr %[src], %[rseq_scratch0]\n\t" 432 + "b 8f\n\t" 433 + RSEQ_ASM_DEFINE_ABORT(3, 4, 434 + /* teardown */ 435 + "ldr %[len], %[rseq_scratch2]\n\t" 436 + "ldr %[dst], %[rseq_scratch1]\n\t" 437 + "ldr %[src], %[rseq_scratch0]\n\t", 438 + abort, 1b, 2b, 4f) 439 + RSEQ_ASM_DEFINE_CMPFAIL(5, 440 + /* teardown */ 441 + "ldr %[len], %[rseq_scratch2]\n\t" 442 + "ldr %[dst], %[rseq_scratch1]\n\t" 443 + "ldr %[src], %[rseq_scratch0]\n\t", 444 + cmpfail) 445 + #ifdef RSEQ_COMPARE_TWICE 446 + RSEQ_ASM_DEFINE_CMPFAIL(6, 447 + /* teardown */ 448 + "ldr %[len], %[rseq_scratch2]\n\t" 449 + "ldr %[dst], %[rseq_scratch1]\n\t" 450 + "ldr %[src], %[rseq_scratch0]\n\t", 451 + error1) 452 + RSEQ_ASM_DEFINE_CMPFAIL(7, 453 + /* teardown */ 454 + "ldr %[len], %[rseq_scratch2]\n\t" 455 + "ldr %[dst], %[rseq_scratch1]\n\t" 456 + "ldr %[src], %[rseq_scratch0]\n\t", 457 + error2) 458 + #endif 459 + "8:\n\t" 460 + : /* gcc asm goto does not allow outputs */ 461 + : [cpu_id] "r" (cpu), 462 + [current_cpu_id] "m" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 463 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 464 + /* final store input */ 465 + [v] "m" (*v), 466 + [expect] "r" (expect), 467 + [newv] "r" (newv), 468 + /* try memcpy input */ 469 + [dst] "r" (dst), 470 + [src] "r" (src), 471 + [len] "r" (len), 472 + [rseq_scratch0] "m" (rseq_scratch[0]), 473 + [rseq_scratch1] "m" (rseq_scratch[1]), 474 + [rseq_scratch2] "m" (rseq_scratch[2]) 475 + RSEQ_INJECT_INPUT 476 + : "r0", "memory", "cc" 477 + RSEQ_INJECT_CLOBBER 478 + : abort, cmpfail 479 + #ifdef RSEQ_COMPARE_TWICE 480 + , error1, error2 481 + #endif 482 + ); 483 + rseq_after_asm_goto(); 484 + return 0; 485 + abort: 486 + rseq_after_asm_goto(); 487 + RSEQ_INJECT_FAILED 488 + return -1; 489 + cmpfail: 490 + rseq_after_asm_goto(); 491 + return 1; 492 + #ifdef RSEQ_COMPARE_TWICE 493 + error1: 494 + rseq_after_asm_goto(); 495 + rseq_bug("cpu_id comparison failed"); 496 + error2: 497 + rseq_after_asm_goto(); 498 + rseq_bug("expected value comparison failed"); 499 + #endif 500 + } 501 + 502 + #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 503 + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 504 + 505 + #include "rseq-bits-reset.h"

+25 -670

tools/testing/selftests/rseq/rseq-arm.h

··· 2 2 /* 3 3 * rseq-arm.h 4 4 * 5 - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 5 + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 6 */ 7 7 8 8 /* ··· 143 143 teardown \ 144 144 "b %l[" __rseq_str(cmpfail_label) "]\n\t" 145 145 146 - static inline __attribute__((always_inline)) 147 - int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 148 - { 149 - RSEQ_INJECT_C(9) 146 + /* Per-cpu-id indexing. */ 150 147 151 - __asm__ __volatile__ goto ( 152 - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 153 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 154 - #ifdef RSEQ_COMPARE_TWICE 155 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 156 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 157 - #endif 158 - /* Start rseq by storing table entry pointer into rseq_cs. */ 159 - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 160 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 161 - RSEQ_INJECT_ASM(3) 162 - "ldr r0, %[v]\n\t" 163 - "cmp %[expect], r0\n\t" 164 - "bne %l[cmpfail]\n\t" 165 - RSEQ_INJECT_ASM(4) 166 - #ifdef RSEQ_COMPARE_TWICE 167 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 168 - "ldr r0, %[v]\n\t" 169 - "cmp %[expect], r0\n\t" 170 - "bne %l[error2]\n\t" 171 - #endif 172 - /* final store */ 173 - "str %[newv], %[v]\n\t" 174 - "2:\n\t" 175 - RSEQ_INJECT_ASM(5) 176 - "b 5f\n\t" 177 - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 178 - "5:\n\t" 179 - : /* gcc asm goto does not allow outputs */ 180 - : [cpu_id] "r" (cpu), 181 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 182 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 183 - [v] "m" (*v), 184 - [expect] "r" (expect), 185 - [newv] "r" (newv) 186 - RSEQ_INJECT_INPUT 187 - : "r0", "memory", "cc" 188 - RSEQ_INJECT_CLOBBER 189 - : abort, cmpfail 190 - #ifdef RSEQ_COMPARE_TWICE 191 - , error1, error2 192 - #endif 193 - ); 194 - rseq_after_asm_goto(); 195 - return 0; 196 - abort: 197 - rseq_after_asm_goto(); 198 - RSEQ_INJECT_FAILED 199 - return -1; 200 - cmpfail: 201 - rseq_after_asm_goto(); 202 - return 1; 203 - #ifdef RSEQ_COMPARE_TWICE 204 - error1: 205 - rseq_after_asm_goto(); 206 - rseq_bug("cpu_id comparison failed"); 207 - error2: 208 - rseq_after_asm_goto(); 209 - rseq_bug("expected value comparison failed"); 210 - #endif 211 - } 148 + #define RSEQ_TEMPLATE_CPU_ID 149 + #define RSEQ_TEMPLATE_MO_RELAXED 150 + #include "rseq-arm-bits.h" 151 + #undef RSEQ_TEMPLATE_MO_RELAXED 212 152 213 - static inline __attribute__((always_inline)) 214 - int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 215 - long voffp, intptr_t *load, int cpu) 216 - { 217 - RSEQ_INJECT_C(9) 153 + #define RSEQ_TEMPLATE_MO_RELEASE 154 + #include "rseq-arm-bits.h" 155 + #undef RSEQ_TEMPLATE_MO_RELEASE 156 + #undef RSEQ_TEMPLATE_CPU_ID 218 157 219 - __asm__ __volatile__ goto ( 220 - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 221 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 222 - #ifdef RSEQ_COMPARE_TWICE 223 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 224 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 225 - #endif 226 - /* Start rseq by storing table entry pointer into rseq_cs. */ 227 - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 228 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 229 - RSEQ_INJECT_ASM(3) 230 - "ldr r0, %[v]\n\t" 231 - "cmp %[expectnot], r0\n\t" 232 - "beq %l[cmpfail]\n\t" 233 - RSEQ_INJECT_ASM(4) 234 - #ifdef RSEQ_COMPARE_TWICE 235 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 236 - "ldr r0, %[v]\n\t" 237 - "cmp %[expectnot], r0\n\t" 238 - "beq %l[error2]\n\t" 239 - #endif 240 - "str r0, %[load]\n\t" 241 - "add r0, %[voffp]\n\t" 242 - "ldr r0, [r0]\n\t" 243 - /* final store */ 244 - "str r0, %[v]\n\t" 245 - "2:\n\t" 246 - RSEQ_INJECT_ASM(5) 247 - "b 5f\n\t" 248 - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 249 - "5:\n\t" 250 - : /* gcc asm goto does not allow outputs */ 251 - : [cpu_id] "r" (cpu), 252 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 253 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 254 - /* final store input */ 255 - [v] "m" (*v), 256 - [expectnot] "r" (expectnot), 257 - [voffp] "Ir" (voffp), 258 - [load] "m" (*load) 259 - RSEQ_INJECT_INPUT 260 - : "r0", "memory", "cc" 261 - RSEQ_INJECT_CLOBBER 262 - : abort, cmpfail 263 - #ifdef RSEQ_COMPARE_TWICE 264 - , error1, error2 265 - #endif 266 - ); 267 - rseq_after_asm_goto(); 268 - return 0; 269 - abort: 270 - rseq_after_asm_goto(); 271 - RSEQ_INJECT_FAILED 272 - return -1; 273 - cmpfail: 274 - rseq_after_asm_goto(); 275 - return 1; 276 - #ifdef RSEQ_COMPARE_TWICE 277 - error1: 278 - rseq_after_asm_goto(); 279 - rseq_bug("cpu_id comparison failed"); 280 - error2: 281 - rseq_after_asm_goto(); 282 - rseq_bug("expected value comparison failed"); 283 - #endif 284 - } 158 + /* Per-mm-cid indexing. */ 285 159 286 - static inline __attribute__((always_inline)) 287 - int rseq_addv(intptr_t *v, intptr_t count, int cpu) 288 - { 289 - RSEQ_INJECT_C(9) 160 + #define RSEQ_TEMPLATE_MM_CID 161 + #define RSEQ_TEMPLATE_MO_RELAXED 162 + #include "rseq-arm-bits.h" 163 + #undef RSEQ_TEMPLATE_MO_RELAXED 290 164 291 - __asm__ __volatile__ goto ( 292 - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 293 - #ifdef RSEQ_COMPARE_TWICE 294 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 295 - #endif 296 - /* Start rseq by storing table entry pointer into rseq_cs. */ 297 - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 298 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 299 - RSEQ_INJECT_ASM(3) 300 - #ifdef RSEQ_COMPARE_TWICE 301 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 302 - #endif 303 - "ldr r0, %[v]\n\t" 304 - "add r0, %[count]\n\t" 305 - /* final store */ 306 - "str r0, %[v]\n\t" 307 - "2:\n\t" 308 - RSEQ_INJECT_ASM(4) 309 - "b 5f\n\t" 310 - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 311 - "5:\n\t" 312 - : /* gcc asm goto does not allow outputs */ 313 - : [cpu_id] "r" (cpu), 314 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 315 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 316 - [v] "m" (*v), 317 - [count] "Ir" (count) 318 - RSEQ_INJECT_INPUT 319 - : "r0", "memory", "cc" 320 - RSEQ_INJECT_CLOBBER 321 - : abort 322 - #ifdef RSEQ_COMPARE_TWICE 323 - , error1 324 - #endif 325 - ); 326 - rseq_after_asm_goto(); 327 - return 0; 328 - abort: 329 - rseq_after_asm_goto(); 330 - RSEQ_INJECT_FAILED 331 - return -1; 332 - #ifdef RSEQ_COMPARE_TWICE 333 - error1: 334 - rseq_after_asm_goto(); 335 - rseq_bug("cpu_id comparison failed"); 336 - #endif 337 - } 165 + #define RSEQ_TEMPLATE_MO_RELEASE 166 + #include "rseq-arm-bits.h" 167 + #undef RSEQ_TEMPLATE_MO_RELEASE 168 + #undef RSEQ_TEMPLATE_MM_CID 338 169 339 - static inline __attribute__((always_inline)) 340 - int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 341 - intptr_t *v2, intptr_t newv2, 342 - intptr_t newv, int cpu) 343 - { 344 - RSEQ_INJECT_C(9) 170 + /* APIs which are not based on cpu ids. */ 345 171 346 - __asm__ __volatile__ goto ( 347 - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 348 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 349 - #ifdef RSEQ_COMPARE_TWICE 350 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 351 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 352 - #endif 353 - /* Start rseq by storing table entry pointer into rseq_cs. */ 354 - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 355 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 356 - RSEQ_INJECT_ASM(3) 357 - "ldr r0, %[v]\n\t" 358 - "cmp %[expect], r0\n\t" 359 - "bne %l[cmpfail]\n\t" 360 - RSEQ_INJECT_ASM(4) 361 - #ifdef RSEQ_COMPARE_TWICE 362 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 363 - "ldr r0, %[v]\n\t" 364 - "cmp %[expect], r0\n\t" 365 - "bne %l[error2]\n\t" 366 - #endif 367 - /* try store */ 368 - "str %[newv2], %[v2]\n\t" 369 - RSEQ_INJECT_ASM(5) 370 - /* final store */ 371 - "str %[newv], %[v]\n\t" 372 - "2:\n\t" 373 - RSEQ_INJECT_ASM(6) 374 - "b 5f\n\t" 375 - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 376 - "5:\n\t" 377 - : /* gcc asm goto does not allow outputs */ 378 - : [cpu_id] "r" (cpu), 379 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 380 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 381 - /* try store input */ 382 - [v2] "m" (*v2), 383 - [newv2] "r" (newv2), 384 - /* final store input */ 385 - [v] "m" (*v), 386 - [expect] "r" (expect), 387 - [newv] "r" (newv) 388 - RSEQ_INJECT_INPUT 389 - : "r0", "memory", "cc" 390 - RSEQ_INJECT_CLOBBER 391 - : abort, cmpfail 392 - #ifdef RSEQ_COMPARE_TWICE 393 - , error1, error2 394 - #endif 395 - ); 396 - rseq_after_asm_goto(); 397 - return 0; 398 - abort: 399 - rseq_after_asm_goto(); 400 - RSEQ_INJECT_FAILED 401 - return -1; 402 - cmpfail: 403 - rseq_after_asm_goto(); 404 - return 1; 405 - #ifdef RSEQ_COMPARE_TWICE 406 - error1: 407 - rseq_after_asm_goto(); 408 - rseq_bug("cpu_id comparison failed"); 409 - error2: 410 - rseq_after_asm_goto(); 411 - rseq_bug("expected value comparison failed"); 412 - #endif 413 - } 414 - 415 - static inline __attribute__((always_inline)) 416 - int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 417 - intptr_t *v2, intptr_t newv2, 418 - intptr_t newv, int cpu) 419 - { 420 - RSEQ_INJECT_C(9) 421 - 422 - __asm__ __volatile__ goto ( 423 - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 424 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 425 - #ifdef RSEQ_COMPARE_TWICE 426 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 427 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 428 - #endif 429 - /* Start rseq by storing table entry pointer into rseq_cs. */ 430 - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 431 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 432 - RSEQ_INJECT_ASM(3) 433 - "ldr r0, %[v]\n\t" 434 - "cmp %[expect], r0\n\t" 435 - "bne %l[cmpfail]\n\t" 436 - RSEQ_INJECT_ASM(4) 437 - #ifdef RSEQ_COMPARE_TWICE 438 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 439 - "ldr r0, %[v]\n\t" 440 - "cmp %[expect], r0\n\t" 441 - "bne %l[error2]\n\t" 442 - #endif 443 - /* try store */ 444 - "str %[newv2], %[v2]\n\t" 445 - RSEQ_INJECT_ASM(5) 446 - "dmb\n\t" /* full mb provides store-release */ 447 - /* final store */ 448 - "str %[newv], %[v]\n\t" 449 - "2:\n\t" 450 - RSEQ_INJECT_ASM(6) 451 - "b 5f\n\t" 452 - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 453 - "5:\n\t" 454 - : /* gcc asm goto does not allow outputs */ 455 - : [cpu_id] "r" (cpu), 456 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 457 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 458 - /* try store input */ 459 - [v2] "m" (*v2), 460 - [newv2] "r" (newv2), 461 - /* final store input */ 462 - [v] "m" (*v), 463 - [expect] "r" (expect), 464 - [newv] "r" (newv) 465 - RSEQ_INJECT_INPUT 466 - : "r0", "memory", "cc" 467 - RSEQ_INJECT_CLOBBER 468 - : abort, cmpfail 469 - #ifdef RSEQ_COMPARE_TWICE 470 - , error1, error2 471 - #endif 472 - ); 473 - rseq_after_asm_goto(); 474 - return 0; 475 - abort: 476 - rseq_after_asm_goto(); 477 - RSEQ_INJECT_FAILED 478 - return -1; 479 - cmpfail: 480 - rseq_after_asm_goto(); 481 - return 1; 482 - #ifdef RSEQ_COMPARE_TWICE 483 - error1: 484 - rseq_after_asm_goto(); 485 - rseq_bug("cpu_id comparison failed"); 486 - error2: 487 - rseq_after_asm_goto(); 488 - rseq_bug("expected value comparison failed"); 489 - #endif 490 - } 491 - 492 - static inline __attribute__((always_inline)) 493 - int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 494 - intptr_t *v2, intptr_t expect2, 495 - intptr_t newv, int cpu) 496 - { 497 - RSEQ_INJECT_C(9) 498 - 499 - __asm__ __volatile__ goto ( 500 - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 501 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 502 - #ifdef RSEQ_COMPARE_TWICE 503 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 504 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 505 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error3]) 506 - #endif 507 - /* Start rseq by storing table entry pointer into rseq_cs. */ 508 - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 509 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 510 - RSEQ_INJECT_ASM(3) 511 - "ldr r0, %[v]\n\t" 512 - "cmp %[expect], r0\n\t" 513 - "bne %l[cmpfail]\n\t" 514 - RSEQ_INJECT_ASM(4) 515 - "ldr r0, %[v2]\n\t" 516 - "cmp %[expect2], r0\n\t" 517 - "bne %l[cmpfail]\n\t" 518 - RSEQ_INJECT_ASM(5) 519 - #ifdef RSEQ_COMPARE_TWICE 520 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 521 - "ldr r0, %[v]\n\t" 522 - "cmp %[expect], r0\n\t" 523 - "bne %l[error2]\n\t" 524 - "ldr r0, %[v2]\n\t" 525 - "cmp %[expect2], r0\n\t" 526 - "bne %l[error3]\n\t" 527 - #endif 528 - /* final store */ 529 - "str %[newv], %[v]\n\t" 530 - "2:\n\t" 531 - RSEQ_INJECT_ASM(6) 532 - "b 5f\n\t" 533 - RSEQ_ASM_DEFINE_ABORT(3, 4, "", abort, 1b, 2b, 4f) 534 - "5:\n\t" 535 - : /* gcc asm goto does not allow outputs */ 536 - : [cpu_id] "r" (cpu), 537 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 538 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 539 - /* cmp2 input */ 540 - [v2] "m" (*v2), 541 - [expect2] "r" (expect2), 542 - /* final store input */ 543 - [v] "m" (*v), 544 - [expect] "r" (expect), 545 - [newv] "r" (newv) 546 - RSEQ_INJECT_INPUT 547 - : "r0", "memory", "cc" 548 - RSEQ_INJECT_CLOBBER 549 - : abort, cmpfail 550 - #ifdef RSEQ_COMPARE_TWICE 551 - , error1, error2, error3 552 - #endif 553 - ); 554 - rseq_after_asm_goto(); 555 - return 0; 556 - abort: 557 - rseq_after_asm_goto(); 558 - RSEQ_INJECT_FAILED 559 - return -1; 560 - cmpfail: 561 - rseq_after_asm_goto(); 562 - return 1; 563 - #ifdef RSEQ_COMPARE_TWICE 564 - error1: 565 - rseq_after_asm_goto(); 566 - rseq_bug("cpu_id comparison failed"); 567 - error2: 568 - rseq_after_asm_goto(); 569 - rseq_bug("1st expected value comparison failed"); 570 - error3: 571 - rseq_after_asm_goto(); 572 - rseq_bug("2nd expected value comparison failed"); 573 - #endif 574 - } 575 - 576 - static inline __attribute__((always_inline)) 577 - int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 578 - void *dst, void *src, size_t len, 579 - intptr_t newv, int cpu) 580 - { 581 - uint32_t rseq_scratch[3]; 582 - 583 - RSEQ_INJECT_C(9) 584 - 585 - __asm__ __volatile__ goto ( 586 - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 587 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 588 - #ifdef RSEQ_COMPARE_TWICE 589 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 590 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 591 - #endif 592 - "str %[src], %[rseq_scratch0]\n\t" 593 - "str %[dst], %[rseq_scratch1]\n\t" 594 - "str %[len], %[rseq_scratch2]\n\t" 595 - /* Start rseq by storing table entry pointer into rseq_cs. */ 596 - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 597 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 598 - RSEQ_INJECT_ASM(3) 599 - "ldr r0, %[v]\n\t" 600 - "cmp %[expect], r0\n\t" 601 - "bne 5f\n\t" 602 - RSEQ_INJECT_ASM(4) 603 - #ifdef RSEQ_COMPARE_TWICE 604 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 605 - "ldr r0, %[v]\n\t" 606 - "cmp %[expect], r0\n\t" 607 - "bne 7f\n\t" 608 - #endif 609 - /* try memcpy */ 610 - "cmp %[len], #0\n\t" \ 611 - "beq 333f\n\t" \ 612 - "222:\n\t" \ 613 - "ldrb %%r0, [%[src]]\n\t" \ 614 - "strb %%r0, [%[dst]]\n\t" \ 615 - "adds %[src], #1\n\t" \ 616 - "adds %[dst], #1\n\t" \ 617 - "subs %[len], #1\n\t" \ 618 - "bne 222b\n\t" \ 619 - "333:\n\t" \ 620 - RSEQ_INJECT_ASM(5) 621 - /* final store */ 622 - "str %[newv], %[v]\n\t" 623 - "2:\n\t" 624 - RSEQ_INJECT_ASM(6) 625 - /* teardown */ 626 - "ldr %[len], %[rseq_scratch2]\n\t" 627 - "ldr %[dst], %[rseq_scratch1]\n\t" 628 - "ldr %[src], %[rseq_scratch0]\n\t" 629 - "b 8f\n\t" 630 - RSEQ_ASM_DEFINE_ABORT(3, 4, 631 - /* teardown */ 632 - "ldr %[len], %[rseq_scratch2]\n\t" 633 - "ldr %[dst], %[rseq_scratch1]\n\t" 634 - "ldr %[src], %[rseq_scratch0]\n\t", 635 - abort, 1b, 2b, 4f) 636 - RSEQ_ASM_DEFINE_CMPFAIL(5, 637 - /* teardown */ 638 - "ldr %[len], %[rseq_scratch2]\n\t" 639 - "ldr %[dst], %[rseq_scratch1]\n\t" 640 - "ldr %[src], %[rseq_scratch0]\n\t", 641 - cmpfail) 642 - #ifdef RSEQ_COMPARE_TWICE 643 - RSEQ_ASM_DEFINE_CMPFAIL(6, 644 - /* teardown */ 645 - "ldr %[len], %[rseq_scratch2]\n\t" 646 - "ldr %[dst], %[rseq_scratch1]\n\t" 647 - "ldr %[src], %[rseq_scratch0]\n\t", 648 - error1) 649 - RSEQ_ASM_DEFINE_CMPFAIL(7, 650 - /* teardown */ 651 - "ldr %[len], %[rseq_scratch2]\n\t" 652 - "ldr %[dst], %[rseq_scratch1]\n\t" 653 - "ldr %[src], %[rseq_scratch0]\n\t", 654 - error2) 655 - #endif 656 - "8:\n\t" 657 - : /* gcc asm goto does not allow outputs */ 658 - : [cpu_id] "r" (cpu), 659 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 660 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 661 - /* final store input */ 662 - [v] "m" (*v), 663 - [expect] "r" (expect), 664 - [newv] "r" (newv), 665 - /* try memcpy input */ 666 - [dst] "r" (dst), 667 - [src] "r" (src), 668 - [len] "r" (len), 669 - [rseq_scratch0] "m" (rseq_scratch[0]), 670 - [rseq_scratch1] "m" (rseq_scratch[1]), 671 - [rseq_scratch2] "m" (rseq_scratch[2]) 672 - RSEQ_INJECT_INPUT 673 - : "r0", "memory", "cc" 674 - RSEQ_INJECT_CLOBBER 675 - : abort, cmpfail 676 - #ifdef RSEQ_COMPARE_TWICE 677 - , error1, error2 678 - #endif 679 - ); 680 - rseq_after_asm_goto(); 681 - return 0; 682 - abort: 683 - rseq_after_asm_goto(); 684 - RSEQ_INJECT_FAILED 685 - return -1; 686 - cmpfail: 687 - rseq_after_asm_goto(); 688 - return 1; 689 - #ifdef RSEQ_COMPARE_TWICE 690 - error1: 691 - rseq_after_asm_goto(); 692 - rseq_bug("cpu_id comparison failed"); 693 - error2: 694 - rseq_after_asm_goto(); 695 - rseq_bug("expected value comparison failed"); 696 - #endif 697 - } 698 - 699 - static inline __attribute__((always_inline)) 700 - int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 701 - void *dst, void *src, size_t len, 702 - intptr_t newv, int cpu) 703 - { 704 - uint32_t rseq_scratch[3]; 705 - 706 - RSEQ_INJECT_C(9) 707 - 708 - __asm__ __volatile__ goto ( 709 - RSEQ_ASM_DEFINE_TABLE(9, 1f, 2f, 4f) /* start, commit, abort */ 710 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[cmpfail]) 711 - #ifdef RSEQ_COMPARE_TWICE 712 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error1]) 713 - RSEQ_ASM_DEFINE_EXIT_POINT(1f, %l[error2]) 714 - #endif 715 - "str %[src], %[rseq_scratch0]\n\t" 716 - "str %[dst], %[rseq_scratch1]\n\t" 717 - "str %[len], %[rseq_scratch2]\n\t" 718 - /* Start rseq by storing table entry pointer into rseq_cs. */ 719 - RSEQ_ASM_STORE_RSEQ_CS(1, 3f, rseq_cs) 720 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 721 - RSEQ_INJECT_ASM(3) 722 - "ldr r0, %[v]\n\t" 723 - "cmp %[expect], r0\n\t" 724 - "bne 5f\n\t" 725 - RSEQ_INJECT_ASM(4) 726 - #ifdef RSEQ_COMPARE_TWICE 727 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 6f) 728 - "ldr r0, %[v]\n\t" 729 - "cmp %[expect], r0\n\t" 730 - "bne 7f\n\t" 731 - #endif 732 - /* try memcpy */ 733 - "cmp %[len], #0\n\t" \ 734 - "beq 333f\n\t" \ 735 - "222:\n\t" \ 736 - "ldrb %%r0, [%[src]]\n\t" \ 737 - "strb %%r0, [%[dst]]\n\t" \ 738 - "adds %[src], #1\n\t" \ 739 - "adds %[dst], #1\n\t" \ 740 - "subs %[len], #1\n\t" \ 741 - "bne 222b\n\t" \ 742 - "333:\n\t" \ 743 - RSEQ_INJECT_ASM(5) 744 - "dmb\n\t" /* full mb provides store-release */ 745 - /* final store */ 746 - "str %[newv], %[v]\n\t" 747 - "2:\n\t" 748 - RSEQ_INJECT_ASM(6) 749 - /* teardown */ 750 - "ldr %[len], %[rseq_scratch2]\n\t" 751 - "ldr %[dst], %[rseq_scratch1]\n\t" 752 - "ldr %[src], %[rseq_scratch0]\n\t" 753 - "b 8f\n\t" 754 - RSEQ_ASM_DEFINE_ABORT(3, 4, 755 - /* teardown */ 756 - "ldr %[len], %[rseq_scratch2]\n\t" 757 - "ldr %[dst], %[rseq_scratch1]\n\t" 758 - "ldr %[src], %[rseq_scratch0]\n\t", 759 - abort, 1b, 2b, 4f) 760 - RSEQ_ASM_DEFINE_CMPFAIL(5, 761 - /* teardown */ 762 - "ldr %[len], %[rseq_scratch2]\n\t" 763 - "ldr %[dst], %[rseq_scratch1]\n\t" 764 - "ldr %[src], %[rseq_scratch0]\n\t", 765 - cmpfail) 766 - #ifdef RSEQ_COMPARE_TWICE 767 - RSEQ_ASM_DEFINE_CMPFAIL(6, 768 - /* teardown */ 769 - "ldr %[len], %[rseq_scratch2]\n\t" 770 - "ldr %[dst], %[rseq_scratch1]\n\t" 771 - "ldr %[src], %[rseq_scratch0]\n\t", 772 - error1) 773 - RSEQ_ASM_DEFINE_CMPFAIL(7, 774 - /* teardown */ 775 - "ldr %[len], %[rseq_scratch2]\n\t" 776 - "ldr %[dst], %[rseq_scratch1]\n\t" 777 - "ldr %[src], %[rseq_scratch0]\n\t", 778 - error2) 779 - #endif 780 - "8:\n\t" 781 - : /* gcc asm goto does not allow outputs */ 782 - : [cpu_id] "r" (cpu), 783 - [current_cpu_id] "m" (rseq_get_abi()->cpu_id), 784 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 785 - /* final store input */ 786 - [v] "m" (*v), 787 - [expect] "r" (expect), 788 - [newv] "r" (newv), 789 - /* try memcpy input */ 790 - [dst] "r" (dst), 791 - [src] "r" (src), 792 - [len] "r" (len), 793 - [rseq_scratch0] "m" (rseq_scratch[0]), 794 - [rseq_scratch1] "m" (rseq_scratch[1]), 795 - [rseq_scratch2] "m" (rseq_scratch[2]) 796 - RSEQ_INJECT_INPUT 797 - : "r0", "memory", "cc" 798 - RSEQ_INJECT_CLOBBER 799 - : abort, cmpfail 800 - #ifdef RSEQ_COMPARE_TWICE 801 - , error1, error2 802 - #endif 803 - ); 804 - rseq_after_asm_goto(); 805 - return 0; 806 - abort: 807 - rseq_after_asm_goto(); 808 - RSEQ_INJECT_FAILED 809 - return -1; 810 - cmpfail: 811 - rseq_after_asm_goto(); 812 - return 1; 813 - #ifdef RSEQ_COMPARE_TWICE 814 - error1: 815 - rseq_after_asm_goto(); 816 - rseq_bug("cpu_id comparison failed"); 817 - error2: 818 - rseq_after_asm_goto(); 819 - rseq_bug("expected value comparison failed"); 820 - #endif 821 - } 172 + #define RSEQ_TEMPLATE_CPU_ID_NONE 173 + #define RSEQ_TEMPLATE_MO_RELAXED 174 + #include "rseq-arm-bits.h" 175 + #undef RSEQ_TEMPLATE_MO_RELAXED 176 + #undef RSEQ_TEMPLATE_CPU_ID_NONE