tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
fork atom

selftests/rseq: arm64: Template memory ordering and percpu access mode

Introduce a rseq-arm64-bits.h template header which is internally
included to generate the static inline functions covering:

- relaxed and release memory ordering,
- per-cpu-id and per-mm-cid per-cpu data access.

Signed-off-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20221122203932.231377-14-mathieu.desnoyers@efficios.com

authored by

Mathieu Desnoyers and committed by
Peter Zijlstra 8d4eeb8b 5bf4aba3

+417 -481
+392
tools/testing/selftests/rseq/rseq-arm64-bits.h
··· 1 + /* SPDX-License-Identifier: LGPL-2.1 OR MIT */ 2 + /* 3 + * rseq-arm64-bits.h 4 + * 5 + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 + * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> 7 + */ 8 + 9 + #include "rseq-bits-template.h" 10 + 11 + #if defined(RSEQ_TEMPLATE_MO_RELAXED) && \ 12 + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 13 + 14 + static inline __attribute__((always_inline)) 15 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_storev)(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 16 + { 17 + RSEQ_INJECT_C(9) 18 + 19 + __asm__ __volatile__ goto ( 20 + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 21 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 22 + #ifdef RSEQ_COMPARE_TWICE 23 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 24 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 25 + #endif 26 + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 27 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 28 + RSEQ_INJECT_ASM(3) 29 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 30 + RSEQ_INJECT_ASM(4) 31 + #ifdef RSEQ_COMPARE_TWICE 32 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 33 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 34 + #endif 35 + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 36 + RSEQ_INJECT_ASM(5) 37 + RSEQ_ASM_DEFINE_ABORT(4, abort) 38 + : /* gcc asm goto does not allow outputs */ 39 + : [cpu_id] "r" (cpu), 40 + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 41 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 42 + [v] "Qo" (*v), 43 + [expect] "r" (expect), 44 + [newv] "r" (newv) 45 + RSEQ_INJECT_INPUT 46 + : "memory", RSEQ_ASM_TMP_REG 47 + : abort, cmpfail 48 + #ifdef RSEQ_COMPARE_TWICE 49 + , error1, error2 50 + #endif 51 + ); 52 + rseq_after_asm_goto(); 53 + return 0; 54 + abort: 55 + rseq_after_asm_goto(); 56 + RSEQ_INJECT_FAILED 57 + return -1; 58 + cmpfail: 59 + rseq_after_asm_goto(); 60 + return 1; 61 + #ifdef RSEQ_COMPARE_TWICE 62 + error1: 63 + rseq_after_asm_goto(); 64 + rseq_bug("cpu_id comparison failed"); 65 + error2: 66 + rseq_after_asm_goto(); 67 + rseq_bug("expected value comparison failed"); 68 + #endif 69 + } 70 + 71 + static inline __attribute__((always_inline)) 72 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpnev_storeoffp_load)(intptr_t *v, intptr_t expectnot, 73 + long voffp, intptr_t *load, int cpu) 74 + { 75 + RSEQ_INJECT_C(9) 76 + 77 + __asm__ __volatile__ goto ( 78 + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 79 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 80 + #ifdef RSEQ_COMPARE_TWICE 81 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 82 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 83 + #endif 84 + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 85 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 86 + RSEQ_INJECT_ASM(3) 87 + RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 88 + RSEQ_INJECT_ASM(4) 89 + #ifdef RSEQ_COMPARE_TWICE 90 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 91 + RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 92 + #endif 93 + RSEQ_ASM_OP_R_LOAD(v) 94 + RSEQ_ASM_OP_R_STORE(load) 95 + RSEQ_ASM_OP_R_LOAD_OFF(voffp) 96 + RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 97 + RSEQ_INJECT_ASM(5) 98 + RSEQ_ASM_DEFINE_ABORT(4, abort) 99 + : /* gcc asm goto does not allow outputs */ 100 + : [cpu_id] "r" (cpu), 101 + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 102 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 103 + [v] "Qo" (*v), 104 + [expectnot] "r" (expectnot), 105 + [load] "Qo" (*load), 106 + [voffp] "r" (voffp) 107 + RSEQ_INJECT_INPUT 108 + : "memory", RSEQ_ASM_TMP_REG 109 + : abort, cmpfail 110 + #ifdef RSEQ_COMPARE_TWICE 111 + , error1, error2 112 + #endif 113 + ); 114 + rseq_after_asm_goto(); 115 + return 0; 116 + abort: 117 + rseq_after_asm_goto(); 118 + RSEQ_INJECT_FAILED 119 + return -1; 120 + cmpfail: 121 + rseq_after_asm_goto(); 122 + return 1; 123 + #ifdef RSEQ_COMPARE_TWICE 124 + error1: 125 + rseq_after_asm_goto(); 126 + rseq_bug("cpu_id comparison failed"); 127 + error2: 128 + rseq_after_asm_goto(); 129 + rseq_bug("expected value comparison failed"); 130 + #endif 131 + } 132 + 133 + static inline __attribute__((always_inline)) 134 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_addv)(intptr_t *v, intptr_t count, int cpu) 135 + { 136 + RSEQ_INJECT_C(9) 137 + 138 + __asm__ __volatile__ goto ( 139 + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 140 + #ifdef RSEQ_COMPARE_TWICE 141 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 142 + #endif 143 + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 144 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 145 + RSEQ_INJECT_ASM(3) 146 + #ifdef RSEQ_COMPARE_TWICE 147 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 148 + #endif 149 + RSEQ_ASM_OP_R_LOAD(v) 150 + RSEQ_ASM_OP_R_ADD(count) 151 + RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 152 + RSEQ_INJECT_ASM(4) 153 + RSEQ_ASM_DEFINE_ABORT(4, abort) 154 + : /* gcc asm goto does not allow outputs */ 155 + : [cpu_id] "r" (cpu), 156 + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 157 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 158 + [v] "Qo" (*v), 159 + [count] "r" (count) 160 + RSEQ_INJECT_INPUT 161 + : "memory", RSEQ_ASM_TMP_REG 162 + : abort 163 + #ifdef RSEQ_COMPARE_TWICE 164 + , error1 165 + #endif 166 + ); 167 + rseq_after_asm_goto(); 168 + return 0; 169 + abort: 170 + rseq_after_asm_goto(); 171 + RSEQ_INJECT_FAILED 172 + return -1; 173 + #ifdef RSEQ_COMPARE_TWICE 174 + error1: 175 + rseq_after_asm_goto(); 176 + rseq_bug("cpu_id comparison failed"); 177 + #endif 178 + } 179 + 180 + static inline __attribute__((always_inline)) 181 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_cmpeqv_storev)(intptr_t *v, intptr_t expect, 182 + intptr_t *v2, intptr_t expect2, 183 + intptr_t newv, int cpu) 184 + { 185 + RSEQ_INJECT_C(9) 186 + 187 + __asm__ __volatile__ goto ( 188 + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 189 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 190 + #ifdef RSEQ_COMPARE_TWICE 191 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 192 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 193 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) 194 + #endif 195 + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 196 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 197 + RSEQ_INJECT_ASM(3) 198 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 199 + RSEQ_INJECT_ASM(4) 200 + RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 201 + RSEQ_INJECT_ASM(5) 202 + #ifdef RSEQ_COMPARE_TWICE 203 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 204 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 205 + RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 206 + #endif 207 + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 208 + RSEQ_INJECT_ASM(6) 209 + RSEQ_ASM_DEFINE_ABORT(4, abort) 210 + : /* gcc asm goto does not allow outputs */ 211 + : [cpu_id] "r" (cpu), 212 + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 213 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 214 + [v] "Qo" (*v), 215 + [expect] "r" (expect), 216 + [v2] "Qo" (*v2), 217 + [expect2] "r" (expect2), 218 + [newv] "r" (newv) 219 + RSEQ_INJECT_INPUT 220 + : "memory", RSEQ_ASM_TMP_REG 221 + : abort, cmpfail 222 + #ifdef RSEQ_COMPARE_TWICE 223 + , error1, error2, error3 224 + #endif 225 + ); 226 + rseq_after_asm_goto(); 227 + return 0; 228 + abort: 229 + rseq_after_asm_goto(); 230 + RSEQ_INJECT_FAILED 231 + return -1; 232 + cmpfail: 233 + rseq_after_asm_goto(); 234 + return 1; 235 + #ifdef RSEQ_COMPARE_TWICE 236 + error1: 237 + rseq_after_asm_goto(); 238 + rseq_bug("cpu_id comparison failed"); 239 + error2: 240 + rseq_after_asm_goto(); 241 + rseq_bug("expected value comparison failed"); 242 + error3: 243 + rseq_after_asm_goto(); 244 + rseq_bug("2nd expected value comparison failed"); 245 + #endif 246 + } 247 + 248 + #endif /* #if defined(RSEQ_TEMPLATE_MO_RELAXED) && 249 + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 250 + 251 + #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && \ 252 + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) 253 + 254 + static inline __attribute__((always_inline)) 255 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trystorev_storev)(intptr_t *v, intptr_t expect, 256 + intptr_t *v2, intptr_t newv2, 257 + intptr_t newv, int cpu) 258 + { 259 + RSEQ_INJECT_C(9) 260 + 261 + __asm__ __volatile__ goto ( 262 + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 263 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 264 + #ifdef RSEQ_COMPARE_TWICE 265 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 266 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 267 + #endif 268 + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 269 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 270 + RSEQ_INJECT_ASM(3) 271 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 272 + RSEQ_INJECT_ASM(4) 273 + #ifdef RSEQ_COMPARE_TWICE 274 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 275 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 276 + #endif 277 + RSEQ_ASM_OP_STORE(newv2, v2) 278 + RSEQ_INJECT_ASM(5) 279 + #ifdef RSEQ_TEMPLATE_MO_RELEASE 280 + RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 281 + #else 282 + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 283 + #endif 284 + RSEQ_INJECT_ASM(6) 285 + RSEQ_ASM_DEFINE_ABORT(4, abort) 286 + : /* gcc asm goto does not allow outputs */ 287 + : [cpu_id] "r" (cpu), 288 + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 289 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 290 + [expect] "r" (expect), 291 + [v] "Qo" (*v), 292 + [newv] "r" (newv), 293 + [v2] "Qo" (*v2), 294 + [newv2] "r" (newv2) 295 + RSEQ_INJECT_INPUT 296 + : "memory", RSEQ_ASM_TMP_REG 297 + : abort, cmpfail 298 + #ifdef RSEQ_COMPARE_TWICE 299 + , error1, error2 300 + #endif 301 + ); 302 + rseq_after_asm_goto(); 303 + return 0; 304 + abort: 305 + rseq_after_asm_goto(); 306 + RSEQ_INJECT_FAILED 307 + return -1; 308 + cmpfail: 309 + rseq_after_asm_goto(); 310 + return 1; 311 + #ifdef RSEQ_COMPARE_TWICE 312 + error1: 313 + rseq_after_asm_goto(); 314 + rseq_bug("cpu_id comparison failed"); 315 + error2: 316 + rseq_after_asm_goto(); 317 + rseq_bug("expected value comparison failed"); 318 + #endif 319 + } 320 + 321 + static inline __attribute__((always_inline)) 322 + int RSEQ_TEMPLATE_IDENTIFIER(rseq_cmpeqv_trymemcpy_storev)(intptr_t *v, intptr_t expect, 323 + void *dst, void *src, size_t len, 324 + intptr_t newv, int cpu) 325 + { 326 + RSEQ_INJECT_C(9) 327 + 328 + __asm__ __volatile__ goto ( 329 + RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 330 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 331 + #ifdef RSEQ_COMPARE_TWICE 332 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 333 + RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 334 + #endif 335 + RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 336 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 337 + RSEQ_INJECT_ASM(3) 338 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 339 + RSEQ_INJECT_ASM(4) 340 + #ifdef RSEQ_COMPARE_TWICE 341 + RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 342 + RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 343 + #endif 344 + RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 345 + RSEQ_INJECT_ASM(5) 346 + #ifdef RSEQ_TEMPLATE_MO_RELEASE 347 + RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 348 + #else 349 + RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 350 + #endif 351 + RSEQ_INJECT_ASM(6) 352 + RSEQ_ASM_DEFINE_ABORT(4, abort) 353 + : /* gcc asm goto does not allow outputs */ 354 + : [cpu_id] "r" (cpu), 355 + [current_cpu_id] "Qo" (rseq_get_abi()->RSEQ_TEMPLATE_CPU_ID_FIELD), 356 + [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 357 + [expect] "r" (expect), 358 + [v] "Qo" (*v), 359 + [newv] "r" (newv), 360 + [dst] "r" (dst), 361 + [src] "r" (src), 362 + [len] "r" (len) 363 + RSEQ_INJECT_INPUT 364 + : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 365 + : abort, cmpfail 366 + #ifdef RSEQ_COMPARE_TWICE 367 + , error1, error2 368 + #endif 369 + ); 370 + rseq_after_asm_goto(); 371 + return 0; 372 + abort: 373 + rseq_after_asm_goto(); 374 + RSEQ_INJECT_FAILED 375 + return -1; 376 + cmpfail: 377 + rseq_after_asm_goto(); 378 + return 1; 379 + #ifdef RSEQ_COMPARE_TWICE 380 + error1: 381 + rseq_after_asm_goto(); 382 + rseq_bug("cpu_id comparison failed"); 383 + error2: 384 + rseq_after_asm_goto(); 385 + rseq_bug("expected value comparison failed"); 386 + #endif 387 + } 388 + 389 + #endif /* #if (defined(RSEQ_TEMPLATE_MO_RELAXED) || defined(RSEQ_TEMPLATE_MO_RELEASE)) && 390 + (defined(RSEQ_TEMPLATE_CPU_ID) || defined(RSEQ_TEMPLATE_MM_CID)) */ 391 + 392 + #include "rseq-bits-reset.h"
+25 -481
tools/testing/selftests/rseq/rseq-arm64.h
··· 2 2 /* 3 3 * rseq-arm64.h 4 4 * 5 - * (C) Copyright 2016-2018 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 5 + * (C) Copyright 2016-2022 - Mathieu Desnoyers <mathieu.desnoyers@efficios.com> 6 6 * (C) Copyright 2018 - Will Deacon <will.deacon@arm.com> 7 7 */ 8 8 ··· 200 200 " cbnz " RSEQ_ASM_TMP_REG_2 ", 222b\n" \ 201 201 "333:\n" 202 202 203 - static inline __attribute__((always_inline)) 204 - int rseq_cmpeqv_storev(intptr_t *v, intptr_t expect, intptr_t newv, int cpu) 205 - { 206 - RSEQ_INJECT_C(9) 203 + /* Per-cpu-id indexing. */ 207 204 208 - __asm__ __volatile__ goto ( 209 - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 210 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 211 - #ifdef RSEQ_COMPARE_TWICE 212 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 213 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 214 - #endif 215 - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 216 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 217 - RSEQ_INJECT_ASM(3) 218 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 219 - RSEQ_INJECT_ASM(4) 220 - #ifdef RSEQ_COMPARE_TWICE 221 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 222 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 223 - #endif 224 - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 225 - RSEQ_INJECT_ASM(5) 226 - RSEQ_ASM_DEFINE_ABORT(4, abort) 227 - : /* gcc asm goto does not allow outputs */ 228 - : [cpu_id] "r" (cpu), 229 - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 230 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 231 - [v] "Qo" (*v), 232 - [expect] "r" (expect), 233 - [newv] "r" (newv) 234 - RSEQ_INJECT_INPUT 235 - : "memory", RSEQ_ASM_TMP_REG 236 - : abort, cmpfail 237 - #ifdef RSEQ_COMPARE_TWICE 238 - , error1, error2 239 - #endif 240 - ); 241 - rseq_after_asm_goto(); 242 - return 0; 243 - abort: 244 - rseq_after_asm_goto(); 245 - RSEQ_INJECT_FAILED 246 - return -1; 247 - cmpfail: 248 - rseq_after_asm_goto(); 249 - return 1; 250 - #ifdef RSEQ_COMPARE_TWICE 251 - error1: 252 - rseq_after_asm_goto(); 253 - rseq_bug("cpu_id comparison failed"); 254 - error2: 255 - rseq_after_asm_goto(); 256 - rseq_bug("expected value comparison failed"); 257 - #endif 258 - } 205 + #define RSEQ_TEMPLATE_CPU_ID 206 + #define RSEQ_TEMPLATE_MO_RELAXED 207 + #include "rseq-arm64-bits.h" 208 + #undef RSEQ_TEMPLATE_MO_RELAXED 259 209 260 - static inline __attribute__((always_inline)) 261 - int rseq_cmpnev_storeoffp_load(intptr_t *v, intptr_t expectnot, 262 - long voffp, intptr_t *load, int cpu) 263 - { 264 - RSEQ_INJECT_C(9) 210 + #define RSEQ_TEMPLATE_MO_RELEASE 211 + #include "rseq-arm64-bits.h" 212 + #undef RSEQ_TEMPLATE_MO_RELEASE 213 + #undef RSEQ_TEMPLATE_CPU_ID 265 214 266 - __asm__ __volatile__ goto ( 267 - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 268 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 269 - #ifdef RSEQ_COMPARE_TWICE 270 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 271 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 272 - #endif 273 - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 274 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 275 - RSEQ_INJECT_ASM(3) 276 - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[cmpfail]) 277 - RSEQ_INJECT_ASM(4) 278 - #ifdef RSEQ_COMPARE_TWICE 279 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 280 - RSEQ_ASM_OP_CMPNE(v, expectnot, %l[error2]) 281 - #endif 282 - RSEQ_ASM_OP_R_LOAD(v) 283 - RSEQ_ASM_OP_R_STORE(load) 284 - RSEQ_ASM_OP_R_LOAD_OFF(voffp) 285 - RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 286 - RSEQ_INJECT_ASM(5) 287 - RSEQ_ASM_DEFINE_ABORT(4, abort) 288 - : /* gcc asm goto does not allow outputs */ 289 - : [cpu_id] "r" (cpu), 290 - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 291 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 292 - [v] "Qo" (*v), 293 - [expectnot] "r" (expectnot), 294 - [load] "Qo" (*load), 295 - [voffp] "r" (voffp) 296 - RSEQ_INJECT_INPUT 297 - : "memory", RSEQ_ASM_TMP_REG 298 - : abort, cmpfail 299 - #ifdef RSEQ_COMPARE_TWICE 300 - , error1, error2 301 - #endif 302 - ); 303 - rseq_after_asm_goto(); 304 - return 0; 305 - abort: 306 - rseq_after_asm_goto(); 307 - RSEQ_INJECT_FAILED 308 - return -1; 309 - cmpfail: 310 - rseq_after_asm_goto(); 311 - return 1; 312 - #ifdef RSEQ_COMPARE_TWICE 313 - error1: 314 - rseq_after_asm_goto(); 315 - rseq_bug("cpu_id comparison failed"); 316 - error2: 317 - rseq_after_asm_goto(); 318 - rseq_bug("expected value comparison failed"); 319 - #endif 320 - } 215 + /* Per-mm-cid indexing. */ 321 216 322 - static inline __attribute__((always_inline)) 323 - int rseq_addv(intptr_t *v, intptr_t count, int cpu) 324 - { 325 - RSEQ_INJECT_C(9) 217 + #define RSEQ_TEMPLATE_MM_CID 218 + #define RSEQ_TEMPLATE_MO_RELAXED 219 + #include "rseq-arm64-bits.h" 220 + #undef RSEQ_TEMPLATE_MO_RELAXED 326 221 327 - __asm__ __volatile__ goto ( 328 - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 329 - #ifdef RSEQ_COMPARE_TWICE 330 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 331 - #endif 332 - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 333 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 334 - RSEQ_INJECT_ASM(3) 335 - #ifdef RSEQ_COMPARE_TWICE 336 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 337 - #endif 338 - RSEQ_ASM_OP_R_LOAD(v) 339 - RSEQ_ASM_OP_R_ADD(count) 340 - RSEQ_ASM_OP_R_FINAL_STORE(v, 3) 341 - RSEQ_INJECT_ASM(4) 342 - RSEQ_ASM_DEFINE_ABORT(4, abort) 343 - : /* gcc asm goto does not allow outputs */ 344 - : [cpu_id] "r" (cpu), 345 - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 346 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 347 - [v] "Qo" (*v), 348 - [count] "r" (count) 349 - RSEQ_INJECT_INPUT 350 - : "memory", RSEQ_ASM_TMP_REG 351 - : abort 352 - #ifdef RSEQ_COMPARE_TWICE 353 - , error1 354 - #endif 355 - ); 356 - rseq_after_asm_goto(); 357 - return 0; 358 - abort: 359 - rseq_after_asm_goto(); 360 - RSEQ_INJECT_FAILED 361 - return -1; 362 - #ifdef RSEQ_COMPARE_TWICE 363 - error1: 364 - rseq_after_asm_goto(); 365 - rseq_bug("cpu_id comparison failed"); 366 - #endif 367 - } 222 + #define RSEQ_TEMPLATE_MO_RELEASE 223 + #include "rseq-arm64-bits.h" 224 + #undef RSEQ_TEMPLATE_MO_RELEASE 225 + #undef RSEQ_TEMPLATE_MM_CID 368 226 369 - static inline __attribute__((always_inline)) 370 - int rseq_cmpeqv_trystorev_storev(intptr_t *v, intptr_t expect, 371 - intptr_t *v2, intptr_t newv2, 372 - intptr_t newv, int cpu) 373 - { 374 - RSEQ_INJECT_C(9) 227 + /* APIs which are not based on cpu ids. */ 375 228 376 - __asm__ __volatile__ goto ( 377 - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 378 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 379 - #ifdef RSEQ_COMPARE_TWICE 380 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 381 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 382 - #endif 383 - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 384 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 385 - RSEQ_INJECT_ASM(3) 386 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 387 - RSEQ_INJECT_ASM(4) 388 - #ifdef RSEQ_COMPARE_TWICE 389 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 390 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 391 - #endif 392 - RSEQ_ASM_OP_STORE(newv2, v2) 393 - RSEQ_INJECT_ASM(5) 394 - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 395 - RSEQ_INJECT_ASM(6) 396 - RSEQ_ASM_DEFINE_ABORT(4, abort) 397 - : /* gcc asm goto does not allow outputs */ 398 - : [cpu_id] "r" (cpu), 399 - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 400 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 401 - [expect] "r" (expect), 402 - [v] "Qo" (*v), 403 - [newv] "r" (newv), 404 - [v2] "Qo" (*v2), 405 - [newv2] "r" (newv2) 406 - RSEQ_INJECT_INPUT 407 - : "memory", RSEQ_ASM_TMP_REG 408 - : abort, cmpfail 409 - #ifdef RSEQ_COMPARE_TWICE 410 - , error1, error2 411 - #endif 412 - ); 413 - rseq_after_asm_goto(); 414 - return 0; 415 - abort: 416 - rseq_after_asm_goto(); 417 - RSEQ_INJECT_FAILED 418 - return -1; 419 - cmpfail: 420 - rseq_after_asm_goto(); 421 - return 1; 422 - #ifdef RSEQ_COMPARE_TWICE 423 - error1: 424 - rseq_after_asm_goto(); 425 - rseq_bug("cpu_id comparison failed"); 426 - error2: 427 - rseq_after_asm_goto(); 428 - rseq_bug("expected value comparison failed"); 429 - #endif 430 - } 431 - 432 - static inline __attribute__((always_inline)) 433 - int rseq_cmpeqv_trystorev_storev_release(intptr_t *v, intptr_t expect, 434 - intptr_t *v2, intptr_t newv2, 435 - intptr_t newv, int cpu) 436 - { 437 - RSEQ_INJECT_C(9) 438 - 439 - __asm__ __volatile__ goto ( 440 - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 441 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 442 - #ifdef RSEQ_COMPARE_TWICE 443 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 444 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 445 - #endif 446 - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 447 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 448 - RSEQ_INJECT_ASM(3) 449 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 450 - RSEQ_INJECT_ASM(4) 451 - #ifdef RSEQ_COMPARE_TWICE 452 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 453 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 454 - #endif 455 - RSEQ_ASM_OP_STORE(newv2, v2) 456 - RSEQ_INJECT_ASM(5) 457 - RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 458 - RSEQ_INJECT_ASM(6) 459 - RSEQ_ASM_DEFINE_ABORT(4, abort) 460 - : /* gcc asm goto does not allow outputs */ 461 - : [cpu_id] "r" (cpu), 462 - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 463 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 464 - [expect] "r" (expect), 465 - [v] "Qo" (*v), 466 - [newv] "r" (newv), 467 - [v2] "Qo" (*v2), 468 - [newv2] "r" (newv2) 469 - RSEQ_INJECT_INPUT 470 - : "memory", RSEQ_ASM_TMP_REG 471 - : abort, cmpfail 472 - #ifdef RSEQ_COMPARE_TWICE 473 - , error1, error2 474 - #endif 475 - ); 476 - rseq_after_asm_goto(); 477 - return 0; 478 - abort: 479 - rseq_after_asm_goto(); 480 - RSEQ_INJECT_FAILED 481 - return -1; 482 - cmpfail: 483 - rseq_after_asm_goto(); 484 - return 1; 485 - #ifdef RSEQ_COMPARE_TWICE 486 - error1: 487 - rseq_after_asm_goto(); 488 - rseq_bug("cpu_id comparison failed"); 489 - error2: 490 - rseq_after_asm_goto(); 491 - rseq_bug("expected value comparison failed"); 492 - #endif 493 - } 494 - 495 - static inline __attribute__((always_inline)) 496 - int rseq_cmpeqv_cmpeqv_storev(intptr_t *v, intptr_t expect, 497 - intptr_t *v2, intptr_t expect2, 498 - intptr_t newv, int cpu) 499 - { 500 - RSEQ_INJECT_C(9) 501 - 502 - __asm__ __volatile__ goto ( 503 - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 504 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 505 - #ifdef RSEQ_COMPARE_TWICE 506 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 507 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 508 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error3]) 509 - #endif 510 - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 511 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 512 - RSEQ_INJECT_ASM(3) 513 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 514 - RSEQ_INJECT_ASM(4) 515 - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[cmpfail]) 516 - RSEQ_INJECT_ASM(5) 517 - #ifdef RSEQ_COMPARE_TWICE 518 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 519 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 520 - RSEQ_ASM_OP_CMPEQ(v2, expect2, %l[error3]) 521 - #endif 522 - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 523 - RSEQ_INJECT_ASM(6) 524 - RSEQ_ASM_DEFINE_ABORT(4, abort) 525 - : /* gcc asm goto does not allow outputs */ 526 - : [cpu_id] "r" (cpu), 527 - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 528 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 529 - [v] "Qo" (*v), 530 - [expect] "r" (expect), 531 - [v2] "Qo" (*v2), 532 - [expect2] "r" (expect2), 533 - [newv] "r" (newv) 534 - RSEQ_INJECT_INPUT 535 - : "memory", RSEQ_ASM_TMP_REG 536 - : abort, cmpfail 537 - #ifdef RSEQ_COMPARE_TWICE 538 - , error1, error2, error3 539 - #endif 540 - ); 541 - rseq_after_asm_goto(); 542 - return 0; 543 - abort: 544 - rseq_after_asm_goto(); 545 - RSEQ_INJECT_FAILED 546 - return -1; 547 - cmpfail: 548 - rseq_after_asm_goto(); 549 - return 1; 550 - #ifdef RSEQ_COMPARE_TWICE 551 - error1: 552 - rseq_after_asm_goto(); 553 - rseq_bug("cpu_id comparison failed"); 554 - error2: 555 - rseq_after_asm_goto(); 556 - rseq_bug("expected value comparison failed"); 557 - error3: 558 - rseq_after_asm_goto(); 559 - rseq_bug("2nd expected value comparison failed"); 560 - #endif 561 - } 562 - 563 - static inline __attribute__((always_inline)) 564 - int rseq_cmpeqv_trymemcpy_storev(intptr_t *v, intptr_t expect, 565 - void *dst, void *src, size_t len, 566 - intptr_t newv, int cpu) 567 - { 568 - RSEQ_INJECT_C(9) 569 - 570 - __asm__ __volatile__ goto ( 571 - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 572 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 573 - #ifdef RSEQ_COMPARE_TWICE 574 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 575 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 576 - #endif 577 - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 578 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 579 - RSEQ_INJECT_ASM(3) 580 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 581 - RSEQ_INJECT_ASM(4) 582 - #ifdef RSEQ_COMPARE_TWICE 583 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 584 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 585 - #endif 586 - RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 587 - RSEQ_INJECT_ASM(5) 588 - RSEQ_ASM_OP_FINAL_STORE(newv, v, 3) 589 - RSEQ_INJECT_ASM(6) 590 - RSEQ_ASM_DEFINE_ABORT(4, abort) 591 - : /* gcc asm goto does not allow outputs */ 592 - : [cpu_id] "r" (cpu), 593 - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 594 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 595 - [expect] "r" (expect), 596 - [v] "Qo" (*v), 597 - [newv] "r" (newv), 598 - [dst] "r" (dst), 599 - [src] "r" (src), 600 - [len] "r" (len) 601 - RSEQ_INJECT_INPUT 602 - : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 603 - : abort, cmpfail 604 - #ifdef RSEQ_COMPARE_TWICE 605 - , error1, error2 606 - #endif 607 - ); 608 - rseq_after_asm_goto(); 609 - return 0; 610 - abort: 611 - rseq_after_asm_goto(); 612 - RSEQ_INJECT_FAILED 613 - return -1; 614 - cmpfail: 615 - rseq_after_asm_goto(); 616 - return 1; 617 - #ifdef RSEQ_COMPARE_TWICE 618 - error1: 619 - rseq_after_asm_goto(); 620 - rseq_bug("cpu_id comparison failed"); 621 - error2: 622 - rseq_after_asm_goto(); 623 - rseq_bug("expected value comparison failed"); 624 - #endif 625 - } 626 - 627 - static inline __attribute__((always_inline)) 628 - int rseq_cmpeqv_trymemcpy_storev_release(intptr_t *v, intptr_t expect, 629 - void *dst, void *src, size_t len, 630 - intptr_t newv, int cpu) 631 - { 632 - RSEQ_INJECT_C(9) 633 - 634 - __asm__ __volatile__ goto ( 635 - RSEQ_ASM_DEFINE_TABLE(1, 2f, 3f, 4f) 636 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[cmpfail]) 637 - #ifdef RSEQ_COMPARE_TWICE 638 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error1]) 639 - RSEQ_ASM_DEFINE_EXIT_POINT(2f, %l[error2]) 640 - #endif 641 - RSEQ_ASM_STORE_RSEQ_CS(2, 1b, rseq_cs) 642 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, 4f) 643 - RSEQ_INJECT_ASM(3) 644 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[cmpfail]) 645 - RSEQ_INJECT_ASM(4) 646 - #ifdef RSEQ_COMPARE_TWICE 647 - RSEQ_ASM_CMP_CPU_ID(cpu_id, current_cpu_id, %l[error1]) 648 - RSEQ_ASM_OP_CMPEQ(v, expect, %l[error2]) 649 - #endif 650 - RSEQ_ASM_OP_R_BAD_MEMCPY(dst, src, len) 651 - RSEQ_INJECT_ASM(5) 652 - RSEQ_ASM_OP_FINAL_STORE_RELEASE(newv, v, 3) 653 - RSEQ_INJECT_ASM(6) 654 - RSEQ_ASM_DEFINE_ABORT(4, abort) 655 - : /* gcc asm goto does not allow outputs */ 656 - : [cpu_id] "r" (cpu), 657 - [current_cpu_id] "Qo" (rseq_get_abi()->cpu_id), 658 - [rseq_cs] "m" (rseq_get_abi()->rseq_cs.arch.ptr), 659 - [expect] "r" (expect), 660 - [v] "Qo" (*v), 661 - [newv] "r" (newv), 662 - [dst] "r" (dst), 663 - [src] "r" (src), 664 - [len] "r" (len) 665 - RSEQ_INJECT_INPUT 666 - : "memory", RSEQ_ASM_TMP_REG, RSEQ_ASM_TMP_REG_2 667 - : abort, cmpfail 668 - #ifdef RSEQ_COMPARE_TWICE 669 - , error1, error2 670 - #endif 671 - ); 672 - rseq_after_asm_goto(); 673 - return 0; 674 - abort: 675 - rseq_after_asm_goto(); 676 - RSEQ_INJECT_FAILED 677 - return -1; 678 - cmpfail: 679 - rseq_after_asm_goto(); 680 - return 1; 681 - #ifdef RSEQ_COMPARE_TWICE 682 - error1: 683 - rseq_after_asm_goto(); 684 - rseq_bug("cpu_id comparison failed"); 685 - error2: 686 - rseq_after_asm_goto(); 687 - rseq_bug("expected value comparison failed"); 688 - #endif 689 - } 229 + #define RSEQ_TEMPLATE_CPU_ID_NONE 230 + #define RSEQ_TEMPLATE_MO_RELAXED 231 + #include "rseq-arm64-bits.h" 232 + #undef RSEQ_TEMPLATE_MO_RELAXED 233 + #undef RSEQ_TEMPLATE_CPU_ID_NONE