Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux

locking/atomics: atomic-instrumented: simplify ifdeffery

Now that all architectures implement ARCH_ATOMIC, the fallbacks are
generated before the instrumented wrappers are generated. Due to this,
in atomic-instrumented.h we can assume that the whole set of atomic
functions has been generated. Likewise, atomic-instrumented.h doesn't
need to provide a preprocessor definition for every atomic it wraps.

This patch removes the redundant ifdeffery.

There should be no functional change as a result of this patch.

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Boqun Feng <boqun.feng@gmail.com>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20210525140232.53872-34-mark.rutland@arm.com

authored by

Mark Rutland and committed by
Peter Zijlstra
bccf1ec3 3c188518

+3 -546
+1 -497
include/asm-generic/atomic-instrumented.h
··· 27 27 instrument_atomic_read(v, sizeof(*v)); 28 28 return arch_atomic_read(v); 29 29 } 30 - #define atomic_read atomic_read 31 30 32 - #if defined(arch_atomic_read_acquire) 33 31 static __always_inline int 34 32 atomic_read_acquire(const atomic_t *v) 35 33 { 36 34 instrument_atomic_read(v, sizeof(*v)); 37 35 return arch_atomic_read_acquire(v); 38 36 } 39 - #define atomic_read_acquire atomic_read_acquire 40 - #endif 41 37 42 38 static __always_inline void 43 39 atomic_set(atomic_t *v, int i) ··· 41 45 instrument_atomic_write(v, sizeof(*v)); 42 46 arch_atomic_set(v, i); 43 47 } 44 - #define atomic_set atomic_set 45 48 46 - #if defined(arch_atomic_set_release) 47 49 static __always_inline void 48 50 atomic_set_release(atomic_t *v, int i) 49 51 { 50 52 instrument_atomic_write(v, sizeof(*v)); 51 53 arch_atomic_set_release(v, i); 52 54 } 53 - #define atomic_set_release atomic_set_release 54 - #endif 55 55 56 56 static __always_inline void 57 57 atomic_add(int i, atomic_t *v) ··· 55 63 instrument_atomic_read_write(v, sizeof(*v)); 56 64 arch_atomic_add(i, v); 57 65 } 58 - #define atomic_add atomic_add 59 66 60 - #if !defined(arch_atomic_add_return_relaxed) || defined(arch_atomic_add_return) 61 67 static __always_inline int 62 68 atomic_add_return(int i, atomic_t *v) 63 69 { 64 70 instrument_atomic_read_write(v, sizeof(*v)); 65 71 return arch_atomic_add_return(i, v); 66 72 } 67 - #define atomic_add_return atomic_add_return 68 - #endif 69 73 70 - #if defined(arch_atomic_add_return_acquire) 71 74 static __always_inline int 72 75 atomic_add_return_acquire(int i, atomic_t *v) 73 76 { 74 77 instrument_atomic_read_write(v, sizeof(*v)); 75 78 return arch_atomic_add_return_acquire(i, v); 76 79 } 77 - #define atomic_add_return_acquire atomic_add_return_acquire 78 - #endif 79 80 80 - #if defined(arch_atomic_add_return_release) 81 81 static __always_inline int 82 82 atomic_add_return_release(int i, atomic_t *v) 83 83 { 84 84 instrument_atomic_read_write(v, sizeof(*v)); 85 85 return arch_atomic_add_return_release(i, v); 86 86 } 87 - #define atomic_add_return_release atomic_add_return_release 88 - #endif 89 87 90 - #if defined(arch_atomic_add_return_relaxed) 91 88 static __always_inline int 92 89 atomic_add_return_relaxed(int i, atomic_t *v) 93 90 { 94 91 instrument_atomic_read_write(v, sizeof(*v)); 95 92 return arch_atomic_add_return_relaxed(i, v); 96 93 } 97 - #define atomic_add_return_relaxed atomic_add_return_relaxed 98 - #endif 99 94 100 - #if !defined(arch_atomic_fetch_add_relaxed) || defined(arch_atomic_fetch_add) 101 95 static __always_inline int 102 96 atomic_fetch_add(int i, atomic_t *v) 103 97 { 104 98 instrument_atomic_read_write(v, sizeof(*v)); 105 99 return arch_atomic_fetch_add(i, v); 106 100 } 107 - #define atomic_fetch_add atomic_fetch_add 108 - #endif 109 101 110 - #if defined(arch_atomic_fetch_add_acquire) 111 102 static __always_inline int 112 103 atomic_fetch_add_acquire(int i, atomic_t *v) 113 104 { 114 105 instrument_atomic_read_write(v, sizeof(*v)); 115 106 return arch_atomic_fetch_add_acquire(i, v); 116 107 } 117 - #define atomic_fetch_add_acquire atomic_fetch_add_acquire 118 - #endif 119 108 120 - #if defined(arch_atomic_fetch_add_release) 121 109 static __always_inline int 122 110 atomic_fetch_add_release(int i, atomic_t *v) 123 111 { 124 112 instrument_atomic_read_write(v, sizeof(*v)); 125 113 return arch_atomic_fetch_add_release(i, v); 126 114 } 127 - #define atomic_fetch_add_release atomic_fetch_add_release 128 - #endif 129 115 130 - #if defined(arch_atomic_fetch_add_relaxed) 131 116 static __always_inline int 132 117 atomic_fetch_add_relaxed(int i, atomic_t *v) 133 118 { 134 119 instrument_atomic_read_write(v, sizeof(*v)); 135 120 return arch_atomic_fetch_add_relaxed(i, v); 136 121 } 137 - #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed 138 - #endif 139 122 140 123 static __always_inline void 141 124 atomic_sub(int i, atomic_t *v) ··· 118 151 instrument_atomic_read_write(v, sizeof(*v)); 119 152 arch_atomic_sub(i, v); 120 153 } 121 - #define atomic_sub atomic_sub 122 154 123 - #if !defined(arch_atomic_sub_return_relaxed) || defined(arch_atomic_sub_return) 124 155 static __always_inline int 125 156 atomic_sub_return(int i, atomic_t *v) 126 157 { 127 158 instrument_atomic_read_write(v, sizeof(*v)); 128 159 return arch_atomic_sub_return(i, v); 129 160 } 130 - #define atomic_sub_return atomic_sub_return 131 - #endif 132 161 133 - #if defined(arch_atomic_sub_return_acquire) 134 162 static __always_inline int 135 163 atomic_sub_return_acquire(int i, atomic_t *v) 136 164 { 137 165 instrument_atomic_read_write(v, sizeof(*v)); 138 166 return arch_atomic_sub_return_acquire(i, v); 139 167 } 140 - #define atomic_sub_return_acquire atomic_sub_return_acquire 141 - #endif 142 168 143 - #if defined(arch_atomic_sub_return_release) 144 169 static __always_inline int 145 170 atomic_sub_return_release(int i, atomic_t *v) 146 171 { 147 172 instrument_atomic_read_write(v, sizeof(*v)); 148 173 return arch_atomic_sub_return_release(i, v); 149 174 } 150 - #define atomic_sub_return_release atomic_sub_return_release 151 - #endif 152 175 153 - #if defined(arch_atomic_sub_return_relaxed) 154 176 static __always_inline int 155 177 atomic_sub_return_relaxed(int i, atomic_t *v) 156 178 { 157 179 instrument_atomic_read_write(v, sizeof(*v)); 158 180 return arch_atomic_sub_return_relaxed(i, v); 159 181 } 160 - #define atomic_sub_return_relaxed atomic_sub_return_relaxed 161 - #endif 162 182 163 - #if !defined(arch_atomic_fetch_sub_relaxed) || defined(arch_atomic_fetch_sub) 164 183 static __always_inline int 165 184 atomic_fetch_sub(int i, atomic_t *v) 166 185 { 167 186 instrument_atomic_read_write(v, sizeof(*v)); 168 187 return arch_atomic_fetch_sub(i, v); 169 188 } 170 - #define atomic_fetch_sub atomic_fetch_sub 171 - #endif 172 189 173 - #if defined(arch_atomic_fetch_sub_acquire) 174 190 static __always_inline int 175 191 atomic_fetch_sub_acquire(int i, atomic_t *v) 176 192 { 177 193 instrument_atomic_read_write(v, sizeof(*v)); 178 194 return arch_atomic_fetch_sub_acquire(i, v); 179 195 } 180 - #define atomic_fetch_sub_acquire atomic_fetch_sub_acquire 181 - #endif 182 196 183 - #if defined(arch_atomic_fetch_sub_release) 184 197 static __always_inline int 185 198 atomic_fetch_sub_release(int i, atomic_t *v) 186 199 { 187 200 instrument_atomic_read_write(v, sizeof(*v)); 188 201 return arch_atomic_fetch_sub_release(i, v); 189 202 } 190 - #define atomic_fetch_sub_release atomic_fetch_sub_release 191 - #endif 192 203 193 - #if defined(arch_atomic_fetch_sub_relaxed) 194 204 static __always_inline int 195 205 atomic_fetch_sub_relaxed(int i, atomic_t *v) 196 206 { 197 207 instrument_atomic_read_write(v, sizeof(*v)); 198 208 return arch_atomic_fetch_sub_relaxed(i, v); 199 209 } 200 - #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed 201 - #endif 202 210 203 - #if defined(arch_atomic_inc) 204 211 static __always_inline void 205 212 atomic_inc(atomic_t *v) 206 213 { 207 214 instrument_atomic_read_write(v, sizeof(*v)); 208 215 arch_atomic_inc(v); 209 216 } 210 - #define atomic_inc atomic_inc 211 - #endif 212 217 213 - #if defined(arch_atomic_inc_return) 214 218 static __always_inline int 215 219 atomic_inc_return(atomic_t *v) 216 220 { 217 221 instrument_atomic_read_write(v, sizeof(*v)); 218 222 return arch_atomic_inc_return(v); 219 223 } 220 - #define atomic_inc_return atomic_inc_return 221 - #endif 222 224 223 - #if defined(arch_atomic_inc_return_acquire) 224 225 static __always_inline int 225 226 atomic_inc_return_acquire(atomic_t *v) 226 227 { 227 228 instrument_atomic_read_write(v, sizeof(*v)); 228 229 return arch_atomic_inc_return_acquire(v); 229 230 } 230 - #define atomic_inc_return_acquire atomic_inc_return_acquire 231 - #endif 232 231 233 - #if defined(arch_atomic_inc_return_release) 234 232 static __always_inline int 235 233 atomic_inc_return_release(atomic_t *v) 236 234 { 237 235 instrument_atomic_read_write(v, sizeof(*v)); 238 236 return arch_atomic_inc_return_release(v); 239 237 } 240 - #define atomic_inc_return_release atomic_inc_return_release 241 - #endif 242 238 243 - #if defined(arch_atomic_inc_return_relaxed) 244 239 static __always_inline int 245 240 atomic_inc_return_relaxed(atomic_t *v) 246 241 { 247 242 instrument_atomic_read_write(v, sizeof(*v)); 248 243 return arch_atomic_inc_return_relaxed(v); 249 244 } 250 - #define atomic_inc_return_relaxed atomic_inc_return_relaxed 251 - #endif 252 245 253 - #if defined(arch_atomic_fetch_inc) 254 246 static __always_inline int 255 247 atomic_fetch_inc(atomic_t *v) 256 248 { 257 249 instrument_atomic_read_write(v, sizeof(*v)); 258 250 return arch_atomic_fetch_inc(v); 259 251 } 260 - #define atomic_fetch_inc atomic_fetch_inc 261 - #endif 262 252 263 - #if defined(arch_atomic_fetch_inc_acquire) 264 253 static __always_inline int 265 254 atomic_fetch_inc_acquire(atomic_t *v) 266 255 { 267 256 instrument_atomic_read_write(v, sizeof(*v)); 268 257 return arch_atomic_fetch_inc_acquire(v); 269 258 } 270 - #define atomic_fetch_inc_acquire atomic_fetch_inc_acquire 271 - #endif 272 259 273 - #if defined(arch_atomic_fetch_inc_release) 274 260 static __always_inline int 275 261 atomic_fetch_inc_release(atomic_t *v) 276 262 { 277 263 instrument_atomic_read_write(v, sizeof(*v)); 278 264 return arch_atomic_fetch_inc_release(v); 279 265 } 280 - #define atomic_fetch_inc_release atomic_fetch_inc_release 281 - #endif 282 266 283 - #if defined(arch_atomic_fetch_inc_relaxed) 284 267 static __always_inline int 285 268 atomic_fetch_inc_relaxed(atomic_t *v) 286 269 { 287 270 instrument_atomic_read_write(v, sizeof(*v)); 288 271 return arch_atomic_fetch_inc_relaxed(v); 289 272 } 290 - #define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed 291 - #endif 292 273 293 - #if defined(arch_atomic_dec) 294 274 static __always_inline void 295 275 atomic_dec(atomic_t *v) 296 276 { 297 277 instrument_atomic_read_write(v, sizeof(*v)); 298 278 arch_atomic_dec(v); 299 279 } 300 - #define atomic_dec atomic_dec 301 - #endif 302 280 303 - #if defined(arch_atomic_dec_return) 304 281 static __always_inline int 305 282 atomic_dec_return(atomic_t *v) 306 283 { 307 284 instrument_atomic_read_write(v, sizeof(*v)); 308 285 return arch_atomic_dec_return(v); 309 286 } 310 - #define atomic_dec_return atomic_dec_return 311 - #endif 312 287 313 - #if defined(arch_atomic_dec_return_acquire) 314 288 static __always_inline int 315 289 atomic_dec_return_acquire(atomic_t *v) 316 290 { 317 291 instrument_atomic_read_write(v, sizeof(*v)); 318 292 return arch_atomic_dec_return_acquire(v); 319 293 } 320 - #define atomic_dec_return_acquire atomic_dec_return_acquire 321 - #endif 322 294 323 - #if defined(arch_atomic_dec_return_release) 324 295 static __always_inline int 325 296 atomic_dec_return_release(atomic_t *v) 326 297 { 327 298 instrument_atomic_read_write(v, sizeof(*v)); 328 299 return arch_atomic_dec_return_release(v); 329 300 } 330 - #define atomic_dec_return_release atomic_dec_return_release 331 - #endif 332 301 333 - #if defined(arch_atomic_dec_return_relaxed) 334 302 static __always_inline int 335 303 atomic_dec_return_relaxed(atomic_t *v) 336 304 { 337 305 instrument_atomic_read_write(v, sizeof(*v)); 338 306 return arch_atomic_dec_return_relaxed(v); 339 307 } 340 - #define atomic_dec_return_relaxed atomic_dec_return_relaxed 341 - #endif 342 308 343 - #if defined(arch_atomic_fetch_dec) 344 309 static __always_inline int 345 310 atomic_fetch_dec(atomic_t *v) 346 311 { 347 312 instrument_atomic_read_write(v, sizeof(*v)); 348 313 return arch_atomic_fetch_dec(v); 349 314 } 350 - #define atomic_fetch_dec atomic_fetch_dec 351 - #endif 352 315 353 - #if defined(arch_atomic_fetch_dec_acquire) 354 316 static __always_inline int 355 317 atomic_fetch_dec_acquire(atomic_t *v) 356 318 { 357 319 instrument_atomic_read_write(v, sizeof(*v)); 358 320 return arch_atomic_fetch_dec_acquire(v); 359 321 } 360 - #define atomic_fetch_dec_acquire atomic_fetch_dec_acquire 361 - #endif 362 322 363 - #if defined(arch_atomic_fetch_dec_release) 364 323 static __always_inline int 365 324 atomic_fetch_dec_release(atomic_t *v) 366 325 { 367 326 instrument_atomic_read_write(v, sizeof(*v)); 368 327 return arch_atomic_fetch_dec_release(v); 369 328 } 370 - #define atomic_fetch_dec_release atomic_fetch_dec_release 371 - #endif 372 329 373 - #if defined(arch_atomic_fetch_dec_relaxed) 374 330 static __always_inline int 375 331 atomic_fetch_dec_relaxed(atomic_t *v) 376 332 { 377 333 instrument_atomic_read_write(v, sizeof(*v)); 378 334 return arch_atomic_fetch_dec_relaxed(v); 379 335 } 380 - #define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed 381 - #endif 382 336 383 337 static __always_inline void 384 338 atomic_and(int i, atomic_t *v) ··· 307 419 instrument_atomic_read_write(v, sizeof(*v)); 308 420 arch_atomic_and(i, v); 309 421 } 310 - #define atomic_and atomic_and 311 422 312 - #if !defined(arch_atomic_fetch_and_relaxed) || defined(arch_atomic_fetch_and) 313 423 static __always_inline int 314 424 atomic_fetch_and(int i, atomic_t *v) 315 425 { 316 426 instrument_atomic_read_write(v, sizeof(*v)); 317 427 return arch_atomic_fetch_and(i, v); 318 428 } 319 - #define atomic_fetch_and atomic_fetch_and 320 - #endif 321 429 322 - #if defined(arch_atomic_fetch_and_acquire) 323 430 static __always_inline int 324 431 atomic_fetch_and_acquire(int i, atomic_t *v) 325 432 { 326 433 instrument_atomic_read_write(v, sizeof(*v)); 327 434 return arch_atomic_fetch_and_acquire(i, v); 328 435 } 329 - #define atomic_fetch_and_acquire atomic_fetch_and_acquire 330 - #endif 331 436 332 - #if defined(arch_atomic_fetch_and_release) 333 437 static __always_inline int 334 438 atomic_fetch_and_release(int i, atomic_t *v) 335 439 { 336 440 instrument_atomic_read_write(v, sizeof(*v)); 337 441 return arch_atomic_fetch_and_release(i, v); 338 442 } 339 - #define atomic_fetch_and_release atomic_fetch_and_release 340 - #endif 341 443 342 - #if defined(arch_atomic_fetch_and_relaxed) 343 444 static __always_inline int 344 445 atomic_fetch_and_relaxed(int i, atomic_t *v) 345 446 { 346 447 instrument_atomic_read_write(v, sizeof(*v)); 347 448 return arch_atomic_fetch_and_relaxed(i, v); 348 449 } 349 - #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed 350 - #endif 351 450 352 - #if defined(arch_atomic_andnot) 353 451 static __always_inline void 354 452 atomic_andnot(int i, atomic_t *v) 355 453 { 356 454 instrument_atomic_read_write(v, sizeof(*v)); 357 455 arch_atomic_andnot(i, v); 358 456 } 359 - #define atomic_andnot atomic_andnot 360 - #endif 361 457 362 - #if defined(arch_atomic_fetch_andnot) 363 458 static __always_inline int 364 459 atomic_fetch_andnot(int i, atomic_t *v) 365 460 { 366 461 instrument_atomic_read_write(v, sizeof(*v)); 367 462 return arch_atomic_fetch_andnot(i, v); 368 463 } 369 - #define atomic_fetch_andnot atomic_fetch_andnot 370 - #endif 371 464 372 - #if defined(arch_atomic_fetch_andnot_acquire) 373 465 static __always_inline int 374 466 atomic_fetch_andnot_acquire(int i, atomic_t *v) 375 467 { 376 468 instrument_atomic_read_write(v, sizeof(*v)); 377 469 return arch_atomic_fetch_andnot_acquire(i, v); 378 470 } 379 - #define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire 380 - #endif 381 471 382 - #if defined(arch_atomic_fetch_andnot_release) 383 472 static __always_inline int 384 473 atomic_fetch_andnot_release(int i, atomic_t *v) 385 474 { 386 475 instrument_atomic_read_write(v, sizeof(*v)); 387 476 return arch_atomic_fetch_andnot_release(i, v); 388 477 } 389 - #define atomic_fetch_andnot_release atomic_fetch_andnot_release 390 - #endif 391 478 392 - #if defined(arch_atomic_fetch_andnot_relaxed) 393 479 static __always_inline int 394 480 atomic_fetch_andnot_relaxed(int i, atomic_t *v) 395 481 { 396 482 instrument_atomic_read_write(v, sizeof(*v)); 397 483 return arch_atomic_fetch_andnot_relaxed(i, v); 398 484 } 399 - #define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed 400 - #endif 401 485 402 486 static __always_inline void 403 487 atomic_or(int i, atomic_t *v) ··· 377 517 instrument_atomic_read_write(v, sizeof(*v)); 378 518 arch_atomic_or(i, v); 379 519 } 380 - #define atomic_or atomic_or 381 520 382 - #if !defined(arch_atomic_fetch_or_relaxed) || defined(arch_atomic_fetch_or) 383 521 static __always_inline int 384 522 atomic_fetch_or(int i, atomic_t *v) 385 523 { 386 524 instrument_atomic_read_write(v, sizeof(*v)); 387 525 return arch_atomic_fetch_or(i, v); 388 526 } 389 - #define atomic_fetch_or atomic_fetch_or 390 - #endif 391 527 392 - #if defined(arch_atomic_fetch_or_acquire) 393 528 static __always_inline int 394 529 atomic_fetch_or_acquire(int i, atomic_t *v) 395 530 { 396 531 instrument_atomic_read_write(v, sizeof(*v)); 397 532 return arch_atomic_fetch_or_acquire(i, v); 398 533 } 399 - #define atomic_fetch_or_acquire atomic_fetch_or_acquire 400 - #endif 401 534 402 - #if defined(arch_atomic_fetch_or_release) 403 535 static __always_inline int 404 536 atomic_fetch_or_release(int i, atomic_t *v) 405 537 { 406 538 instrument_atomic_read_write(v, sizeof(*v)); 407 539 return arch_atomic_fetch_or_release(i, v); 408 540 } 409 - #define atomic_fetch_or_release atomic_fetch_or_release 410 - #endif 411 541 412 - #if defined(arch_atomic_fetch_or_relaxed) 413 542 static __always_inline int 414 543 atomic_fetch_or_relaxed(int i, atomic_t *v) 415 544 { 416 545 instrument_atomic_read_write(v, sizeof(*v)); 417 546 return arch_atomic_fetch_or_relaxed(i, v); 418 547 } 419 - #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed 420 - #endif 421 548 422 549 static __always_inline void 423 550 atomic_xor(int i, atomic_t *v) ··· 412 565 instrument_atomic_read_write(v, sizeof(*v)); 413 566 arch_atomic_xor(i, v); 414 567 } 415 - #define atomic_xor atomic_xor 416 568 417 - #if !defined(arch_atomic_fetch_xor_relaxed) || defined(arch_atomic_fetch_xor) 418 569 static __always_inline int 419 570 atomic_fetch_xor(int i, atomic_t *v) 420 571 { 421 572 instrument_atomic_read_write(v, sizeof(*v)); 422 573 return arch_atomic_fetch_xor(i, v); 423 574 } 424 - #define atomic_fetch_xor atomic_fetch_xor 425 - #endif 426 575 427 - #if defined(arch_atomic_fetch_xor_acquire) 428 576 static __always_inline int 429 577 atomic_fetch_xor_acquire(int i, atomic_t *v) 430 578 { 431 579 instrument_atomic_read_write(v, sizeof(*v)); 432 580 return arch_atomic_fetch_xor_acquire(i, v); 433 581 } 434 - #define atomic_fetch_xor_acquire atomic_fetch_xor_acquire 435 - #endif 436 582 437 - #if defined(arch_atomic_fetch_xor_release) 438 583 static __always_inline int 439 584 atomic_fetch_xor_release(int i, atomic_t *v) 440 585 { 441 586 instrument_atomic_read_write(v, sizeof(*v)); 442 587 return arch_atomic_fetch_xor_release(i, v); 443 588 } 444 - #define atomic_fetch_xor_release atomic_fetch_xor_release 445 - #endif 446 589 447 - #if defined(arch_atomic_fetch_xor_relaxed) 448 590 static __always_inline int 449 591 atomic_fetch_xor_relaxed(int i, atomic_t *v) 450 592 { 451 593 instrument_atomic_read_write(v, sizeof(*v)); 452 594 return arch_atomic_fetch_xor_relaxed(i, v); 453 595 } 454 - #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed 455 - #endif 456 596 457 - #if !defined(arch_atomic_xchg_relaxed) || defined(arch_atomic_xchg) 458 597 static __always_inline int 459 598 atomic_xchg(atomic_t *v, int i) 460 599 { 461 600 instrument_atomic_read_write(v, sizeof(*v)); 462 601 return arch_atomic_xchg(v, i); 463 602 } 464 - #define atomic_xchg atomic_xchg 465 - #endif 466 603 467 - #if defined(arch_atomic_xchg_acquire) 468 604 static __always_inline int 469 605 atomic_xchg_acquire(atomic_t *v, int i) 470 606 { 471 607 instrument_atomic_read_write(v, sizeof(*v)); 472 608 return arch_atomic_xchg_acquire(v, i); 473 609 } 474 - #define atomic_xchg_acquire atomic_xchg_acquire 475 - #endif 476 610 477 - #if defined(arch_atomic_xchg_release) 478 611 static __always_inline int 479 612 atomic_xchg_release(atomic_t *v, int i) 480 613 { 481 614 instrument_atomic_read_write(v, sizeof(*v)); 482 615 return arch_atomic_xchg_release(v, i); 483 616 } 484 - #define atomic_xchg_release atomic_xchg_release 485 - #endif 486 617 487 - #if defined(arch_atomic_xchg_relaxed) 488 618 static __always_inline int 489 619 atomic_xchg_relaxed(atomic_t *v, int i) 490 620 { 491 621 instrument_atomic_read_write(v, sizeof(*v)); 492 622 return arch_atomic_xchg_relaxed(v, i); 493 623 } 494 - #define atomic_xchg_relaxed atomic_xchg_relaxed 495 - #endif 496 624 497 - #if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg) 498 625 static __always_inline int 499 626 atomic_cmpxchg(atomic_t *v, int old, int new) 500 627 { 501 628 instrument_atomic_read_write(v, sizeof(*v)); 502 629 return arch_atomic_cmpxchg(v, old, new); 503 630 } 504 - #define atomic_cmpxchg atomic_cmpxchg 505 - #endif 506 631 507 - #if defined(arch_atomic_cmpxchg_acquire) 508 632 static __always_inline int 509 633 atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 510 634 { 511 635 instrument_atomic_read_write(v, sizeof(*v)); 512 636 return arch_atomic_cmpxchg_acquire(v, old, new); 513 637 } 514 - #define atomic_cmpxchg_acquire atomic_cmpxchg_acquire 515 - #endif 516 638 517 - #if defined(arch_atomic_cmpxchg_release) 518 639 static __always_inline int 519 640 atomic_cmpxchg_release(atomic_t *v, int old, int new) 520 641 { 521 642 instrument_atomic_read_write(v, sizeof(*v)); 522 643 return arch_atomic_cmpxchg_release(v, old, new); 523 644 } 524 - #define atomic_cmpxchg_release atomic_cmpxchg_release 525 - #endif 526 645 527 - #if defined(arch_atomic_cmpxchg_relaxed) 528 646 static __always_inline int 529 647 atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) 530 648 { 531 649 instrument_atomic_read_write(v, sizeof(*v)); 532 650 return arch_atomic_cmpxchg_relaxed(v, old, new); 533 651 } 534 - #define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed 535 - #endif 536 652 537 - #if defined(arch_atomic_try_cmpxchg) 538 653 static __always_inline bool 539 654 atomic_try_cmpxchg(atomic_t *v, int *old, int new) 540 655 { ··· 504 695 instrument_atomic_read_write(old, sizeof(*old)); 505 696 return arch_atomic_try_cmpxchg(v, old, new); 506 697 } 507 - #define atomic_try_cmpxchg atomic_try_cmpxchg 508 - #endif 509 698 510 - #if defined(arch_atomic_try_cmpxchg_acquire) 511 699 static __always_inline bool 512 700 atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 513 701 { ··· 512 706 instrument_atomic_read_write(old, sizeof(*old)); 513 707 return arch_atomic_try_cmpxchg_acquire(v, old, new); 514 708 } 515 - #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire 516 - #endif 517 709 518 - #if defined(arch_atomic_try_cmpxchg_release) 519 710 static __always_inline bool 520 711 atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 521 712 { ··· 520 717 instrument_atomic_read_write(old, sizeof(*old)); 521 718 return arch_atomic_try_cmpxchg_release(v, old, new); 522 719 } 523 - #define atomic_try_cmpxchg_release atomic_try_cmpxchg_release 524 - #endif 525 720 526 - #if defined(arch_atomic_try_cmpxchg_relaxed) 527 721 static __always_inline bool 528 722 atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 529 723 { ··· 528 728 instrument_atomic_read_write(old, sizeof(*old)); 529 729 return arch_atomic_try_cmpxchg_relaxed(v, old, new); 530 730 } 531 - #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed 532 - #endif 533 731 534 - #if defined(arch_atomic_sub_and_test) 535 732 static __always_inline bool 536 733 atomic_sub_and_test(int i, atomic_t *v) 537 734 { 538 735 instrument_atomic_read_write(v, sizeof(*v)); 539 736 return arch_atomic_sub_and_test(i, v); 540 737 } 541 - #define atomic_sub_and_test atomic_sub_and_test 542 - #endif 543 738 544 - #if defined(arch_atomic_dec_and_test) 545 739 static __always_inline bool 546 740 atomic_dec_and_test(atomic_t *v) 547 741 { 548 742 instrument_atomic_read_write(v, sizeof(*v)); 549 743 return arch_atomic_dec_and_test(v); 550 744 } 551 - #define atomic_dec_and_test atomic_dec_and_test 552 - #endif 553 745 554 - #if defined(arch_atomic_inc_and_test) 555 746 static __always_inline bool 556 747 atomic_inc_and_test(atomic_t *v) 557 748 { 558 749 instrument_atomic_read_write(v, sizeof(*v)); 559 750 return arch_atomic_inc_and_test(v); 560 751 } 561 - #define atomic_inc_and_test atomic_inc_and_test 562 - #endif 563 752 564 - #if defined(arch_atomic_add_negative) 565 753 static __always_inline bool 566 754 atomic_add_negative(int i, atomic_t *v) 567 755 { 568 756 instrument_atomic_read_write(v, sizeof(*v)); 569 757 return arch_atomic_add_negative(i, v); 570 758 } 571 - #define atomic_add_negative atomic_add_negative 572 - #endif 573 759 574 - #if defined(arch_atomic_fetch_add_unless) 575 760 static __always_inline int 576 761 atomic_fetch_add_unless(atomic_t *v, int a, int u) 577 762 { 578 763 instrument_atomic_read_write(v, sizeof(*v)); 579 764 return arch_atomic_fetch_add_unless(v, a, u); 580 765 } 581 - #define atomic_fetch_add_unless atomic_fetch_add_unless 582 - #endif 583 766 584 - #if defined(arch_atomic_add_unless) 585 767 static __always_inline bool 586 768 atomic_add_unless(atomic_t *v, int a, int u) 587 769 { 588 770 instrument_atomic_read_write(v, sizeof(*v)); 589 771 return arch_atomic_add_unless(v, a, u); 590 772 } 591 - #define atomic_add_unless atomic_add_unless 592 - #endif 593 773 594 - #if defined(arch_atomic_inc_not_zero) 595 774 static __always_inline bool 596 775 atomic_inc_not_zero(atomic_t *v) 597 776 { 598 777 instrument_atomic_read_write(v, sizeof(*v)); 599 778 return arch_atomic_inc_not_zero(v); 600 779 } 601 - #define atomic_inc_not_zero atomic_inc_not_zero 602 - #endif 603 780 604 - #if defined(arch_atomic_inc_unless_negative) 605 781 static __always_inline bool 606 782 atomic_inc_unless_negative(atomic_t *v) 607 783 { 608 784 instrument_atomic_read_write(v, sizeof(*v)); 609 785 return arch_atomic_inc_unless_negative(v); 610 786 } 611 - #define atomic_inc_unless_negative atomic_inc_unless_negative 612 - #endif 613 787 614 - #if defined(arch_atomic_dec_unless_positive) 615 788 static __always_inline bool 616 789 atomic_dec_unless_positive(atomic_t *v) 617 790 { 618 791 instrument_atomic_read_write(v, sizeof(*v)); 619 792 return arch_atomic_dec_unless_positive(v); 620 793 } 621 - #define atomic_dec_unless_positive atomic_dec_unless_positive 622 - #endif 623 794 624 - #if defined(arch_atomic_dec_if_positive) 625 795 static __always_inline int 626 796 atomic_dec_if_positive(atomic_t *v) 627 797 { 628 798 instrument_atomic_read_write(v, sizeof(*v)); 629 799 return arch_atomic_dec_if_positive(v); 630 800 } 631 - #define atomic_dec_if_positive atomic_dec_if_positive 632 - #endif 633 801 634 802 static __always_inline s64 635 803 atomic64_read(const atomic64_t *v) ··· 605 837 instrument_atomic_read(v, sizeof(*v)); 606 838 return arch_atomic64_read(v); 607 839 } 608 - #define atomic64_read atomic64_read 609 840 610 - #if defined(arch_atomic64_read_acquire) 611 841 static __always_inline s64 612 842 atomic64_read_acquire(const atomic64_t *v) 613 843 { 614 844 instrument_atomic_read(v, sizeof(*v)); 615 845 return arch_atomic64_read_acquire(v); 616 846 } 617 - #define atomic64_read_acquire atomic64_read_acquire 618 - #endif 619 847 620 848 static __always_inline void 621 849 atomic64_set(atomic64_t *v, s64 i) ··· 619 855 instrument_atomic_write(v, sizeof(*v)); 620 856 arch_atomic64_set(v, i); 621 857 } 622 - #define atomic64_set atomic64_set 623 858 624 - #if defined(arch_atomic64_set_release) 625 859 static __always_inline void 626 860 atomic64_set_release(atomic64_t *v, s64 i) 627 861 { 628 862 instrument_atomic_write(v, sizeof(*v)); 629 863 arch_atomic64_set_release(v, i); 630 864 } 631 - #define atomic64_set_release atomic64_set_release 632 - #endif 633 865 634 866 static __always_inline void 635 867 atomic64_add(s64 i, atomic64_t *v) ··· 633 873 instrument_atomic_read_write(v, sizeof(*v)); 634 874 arch_atomic64_add(i, v); 635 875 } 636 - #define atomic64_add atomic64_add 637 876 638 - #if !defined(arch_atomic64_add_return_relaxed) || defined(arch_atomic64_add_return) 639 877 static __always_inline s64 640 878 atomic64_add_return(s64 i, atomic64_t *v) 641 879 { 642 880 instrument_atomic_read_write(v, sizeof(*v)); 643 881 return arch_atomic64_add_return(i, v); 644 882 } 645 - #define atomic64_add_return atomic64_add_return 646 - #endif 647 883 648 - #if defined(arch_atomic64_add_return_acquire) 649 884 static __always_inline s64 650 885 atomic64_add_return_acquire(s64 i, atomic64_t *v) 651 886 { 652 887 instrument_atomic_read_write(v, sizeof(*v)); 653 888 return arch_atomic64_add_return_acquire(i, v); 654 889 } 655 - #define atomic64_add_return_acquire atomic64_add_return_acquire 656 - #endif 657 890 658 - #if defined(arch_atomic64_add_return_release) 659 891 static __always_inline s64 660 892 atomic64_add_return_release(s64 i, atomic64_t *v) 661 893 { 662 894 instrument_atomic_read_write(v, sizeof(*v)); 663 895 return arch_atomic64_add_return_release(i, v); 664 896 } 665 - #define atomic64_add_return_release atomic64_add_return_release 666 - #endif 667 897 668 - #if defined(arch_atomic64_add_return_relaxed) 669 898 static __always_inline s64 670 899 atomic64_add_return_relaxed(s64 i, atomic64_t *v) 671 900 { 672 901 instrument_atomic_read_write(v, sizeof(*v)); 673 902 return arch_atomic64_add_return_relaxed(i, v); 674 903 } 675 - #define atomic64_add_return_relaxed atomic64_add_return_relaxed 676 - #endif 677 904 678 - #if !defined(arch_atomic64_fetch_add_relaxed) || defined(arch_atomic64_fetch_add) 679 905 static __always_inline s64 680 906 atomic64_fetch_add(s64 i, atomic64_t *v) 681 907 { 682 908 instrument_atomic_read_write(v, sizeof(*v)); 683 909 return arch_atomic64_fetch_add(i, v); 684 910 } 685 - #define atomic64_fetch_add atomic64_fetch_add 686 - #endif 687 911 688 - #if defined(arch_atomic64_fetch_add_acquire) 689 912 static __always_inline s64 690 913 atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 691 914 { 692 915 instrument_atomic_read_write(v, sizeof(*v)); 693 916 return arch_atomic64_fetch_add_acquire(i, v); 694 917 } 695 - #define atomic64_fetch_add_acquire atomic64_fetch_add_acquire 696 - #endif 697 918 698 - #if defined(arch_atomic64_fetch_add_release) 699 919 static __always_inline s64 700 920 atomic64_fetch_add_release(s64 i, atomic64_t *v) 701 921 { 702 922 instrument_atomic_read_write(v, sizeof(*v)); 703 923 return arch_atomic64_fetch_add_release(i, v); 704 924 } 705 - #define atomic64_fetch_add_release atomic64_fetch_add_release 706 - #endif 707 925 708 - #if defined(arch_atomic64_fetch_add_relaxed) 709 926 static __always_inline s64 710 927 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) 711 928 { 712 929 instrument_atomic_read_write(v, sizeof(*v)); 713 930 return arch_atomic64_fetch_add_relaxed(i, v); 714 931 } 715 - #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed 716 - #endif 717 932 718 933 static __always_inline void 719 934 atomic64_sub(s64 i, atomic64_t *v) ··· 696 961 instrument_atomic_read_write(v, sizeof(*v)); 697 962 arch_atomic64_sub(i, v); 698 963 } 699 - #define atomic64_sub atomic64_sub 700 964 701 - #if !defined(arch_atomic64_sub_return_relaxed) || defined(arch_atomic64_sub_return) 702 965 static __always_inline s64 703 966 atomic64_sub_return(s64 i, atomic64_t *v) 704 967 { 705 968 instrument_atomic_read_write(v, sizeof(*v)); 706 969 return arch_atomic64_sub_return(i, v); 707 970 } 708 - #define atomic64_sub_return atomic64_sub_return 709 - #endif 710 971 711 - #if defined(arch_atomic64_sub_return_acquire) 712 972 static __always_inline s64 713 973 atomic64_sub_return_acquire(s64 i, atomic64_t *v) 714 974 { 715 975 instrument_atomic_read_write(v, sizeof(*v)); 716 976 return arch_atomic64_sub_return_acquire(i, v); 717 977 } 718 - #define atomic64_sub_return_acquire atomic64_sub_return_acquire 719 - #endif 720 978 721 - #if defined(arch_atomic64_sub_return_release) 722 979 static __always_inline s64 723 980 atomic64_sub_return_release(s64 i, atomic64_t *v) 724 981 { 725 982 instrument_atomic_read_write(v, sizeof(*v)); 726 983 return arch_atomic64_sub_return_release(i, v); 727 984 } 728 - #define atomic64_sub_return_release atomic64_sub_return_release 729 - #endif 730 985 731 - #if defined(arch_atomic64_sub_return_relaxed) 732 986 static __always_inline s64 733 987 atomic64_sub_return_relaxed(s64 i, atomic64_t *v) 734 988 { 735 989 instrument_atomic_read_write(v, sizeof(*v)); 736 990 return arch_atomic64_sub_return_relaxed(i, v); 737 991 } 738 - #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed 739 - #endif 740 992 741 - #if !defined(arch_atomic64_fetch_sub_relaxed) || defined(arch_atomic64_fetch_sub) 742 993 static __always_inline s64 743 994 atomic64_fetch_sub(s64 i, atomic64_t *v) 744 995 { 745 996 instrument_atomic_read_write(v, sizeof(*v)); 746 997 return arch_atomic64_fetch_sub(i, v); 747 998 } 748 - #define atomic64_fetch_sub atomic64_fetch_sub 749 - #endif 750 999 751 - #if defined(arch_atomic64_fetch_sub_acquire) 752 1000 static __always_inline s64 753 1001 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 754 1002 { 755 1003 instrument_atomic_read_write(v, sizeof(*v)); 756 1004 return arch_atomic64_fetch_sub_acquire(i, v); 757 1005 } 758 - #define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire 759 - #endif 760 1006 761 - #if defined(arch_atomic64_fetch_sub_release) 762 1007 static __always_inline s64 763 1008 atomic64_fetch_sub_release(s64 i, atomic64_t *v) 764 1009 { 765 1010 instrument_atomic_read_write(v, sizeof(*v)); 766 1011 return arch_atomic64_fetch_sub_release(i, v); 767 1012 } 768 - #define atomic64_fetch_sub_release atomic64_fetch_sub_release 769 - #endif 770 1013 771 - #if defined(arch_atomic64_fetch_sub_relaxed) 772 1014 static __always_inline s64 773 1015 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) 774 1016 { 775 1017 instrument_atomic_read_write(v, sizeof(*v)); 776 1018 return arch_atomic64_fetch_sub_relaxed(i, v); 777 1019 } 778 - #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed 779 - #endif 780 1020 781 - #if defined(arch_atomic64_inc) 782 1021 static __always_inline void 783 1022 atomic64_inc(atomic64_t *v) 784 1023 { 785 1024 instrument_atomic_read_write(v, sizeof(*v)); 786 1025 arch_atomic64_inc(v); 787 1026 } 788 - #define atomic64_inc atomic64_inc 789 - #endif 790 1027 791 - #if defined(arch_atomic64_inc_return) 792 1028 static __always_inline s64 793 1029 atomic64_inc_return(atomic64_t *v) 794 1030 { 795 1031 instrument_atomic_read_write(v, sizeof(*v)); 796 1032 return arch_atomic64_inc_return(v); 797 1033 } 798 - #define atomic64_inc_return atomic64_inc_return 799 - #endif 800 1034 801 - #if defined(arch_atomic64_inc_return_acquire) 802 1035 static __always_inline s64 803 1036 atomic64_inc_return_acquire(atomic64_t *v) 804 1037 { 805 1038 instrument_atomic_read_write(v, sizeof(*v)); 806 1039 return arch_atomic64_inc_return_acquire(v); 807 1040 } 808 - #define atomic64_inc_return_acquire atomic64_inc_return_acquire 809 - #endif 810 1041 811 - #if defined(arch_atomic64_inc_return_release) 812 1042 static __always_inline s64 813 1043 atomic64_inc_return_release(atomic64_t *v) 814 1044 { 815 1045 instrument_atomic_read_write(v, sizeof(*v)); 816 1046 return arch_atomic64_inc_return_release(v); 817 1047 } 818 - #define atomic64_inc_return_release atomic64_inc_return_release 819 - #endif 820 1048 821 - #if defined(arch_atomic64_inc_return_relaxed) 822 1049 static __always_inline s64 823 1050 atomic64_inc_return_relaxed(atomic64_t *v) 824 1051 { 825 1052 instrument_atomic_read_write(v, sizeof(*v)); 826 1053 return arch_atomic64_inc_return_relaxed(v); 827 1054 } 828 - #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed 829 - #endif 830 1055 831 - #if defined(arch_atomic64_fetch_inc) 832 1056 static __always_inline s64 833 1057 atomic64_fetch_inc(atomic64_t *v) 834 1058 { 835 1059 instrument_atomic_read_write(v, sizeof(*v)); 836 1060 return arch_atomic64_fetch_inc(v); 837 1061 } 838 - #define atomic64_fetch_inc atomic64_fetch_inc 839 - #endif 840 1062 841 - #if defined(arch_atomic64_fetch_inc_acquire) 842 1063 static __always_inline s64 843 1064 atomic64_fetch_inc_acquire(atomic64_t *v) 844 1065 { 845 1066 instrument_atomic_read_write(v, sizeof(*v)); 846 1067 return arch_atomic64_fetch_inc_acquire(v); 847 1068 } 848 - #define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire 849 - #endif 850 1069 851 - #if defined(arch_atomic64_fetch_inc_release) 852 1070 static __always_inline s64 853 1071 atomic64_fetch_inc_release(atomic64_t *v) 854 1072 { 855 1073 instrument_atomic_read_write(v, sizeof(*v)); 856 1074 return arch_atomic64_fetch_inc_release(v); 857 1075 } 858 - #define atomic64_fetch_inc_release atomic64_fetch_inc_release 859 - #endif 860 1076 861 - #if defined(arch_atomic64_fetch_inc_relaxed) 862 1077 static __always_inline s64 863 1078 atomic64_fetch_inc_relaxed(atomic64_t *v) 864 1079 { 865 1080 instrument_atomic_read_write(v, sizeof(*v)); 866 1081 return arch_atomic64_fetch_inc_relaxed(v); 867 1082 } 868 - #define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed 869 - #endif 870 1083 871 - #if defined(arch_atomic64_dec) 872 1084 static __always_inline void 873 1085 atomic64_dec(atomic64_t *v) 874 1086 { 875 1087 instrument_atomic_read_write(v, sizeof(*v)); 876 1088 arch_atomic64_dec(v); 877 1089 } 878 - #define atomic64_dec atomic64_dec 879 - #endif 880 1090 881 - #if defined(arch_atomic64_dec_return) 882 1091 static __always_inline s64 883 1092 atomic64_dec_return(atomic64_t *v) 884 1093 { 885 1094 instrument_atomic_read_write(v, sizeof(*v)); 886 1095 return arch_atomic64_dec_return(v); 887 1096 } 888 - #define atomic64_dec_return atomic64_dec_return 889 - #endif 890 1097 891 - #if defined(arch_atomic64_dec_return_acquire) 892 1098 static __always_inline s64 893 1099 atomic64_dec_return_acquire(atomic64_t *v) 894 1100 { 895 1101 instrument_atomic_read_write(v, sizeof(*v)); 896 1102 return arch_atomic64_dec_return_acquire(v); 897 1103 } 898 - #define atomic64_dec_return_acquire atomic64_dec_return_acquire 899 - #endif 900 1104 901 - #if defined(arch_atomic64_dec_return_release) 902 1105 static __always_inline s64 903 1106 atomic64_dec_return_release(atomic64_t *v) 904 1107 { 905 1108 instrument_atomic_read_write(v, sizeof(*v)); 906 1109 return arch_atomic64_dec_return_release(v); 907 1110 } 908 - #define atomic64_dec_return_release atomic64_dec_return_release 909 - #endif 910 1111 911 - #if defined(arch_atomic64_dec_return_relaxed) 912 1112 static __always_inline s64 913 1113 atomic64_dec_return_relaxed(atomic64_t *v) 914 1114 { 915 1115 instrument_atomic_read_write(v, sizeof(*v)); 916 1116 return arch_atomic64_dec_return_relaxed(v); 917 1117 } 918 - #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed 919 - #endif 920 1118 921 - #if defined(arch_atomic64_fetch_dec) 922 1119 static __always_inline s64 923 1120 atomic64_fetch_dec(atomic64_t *v) 924 1121 { 925 1122 instrument_atomic_read_write(v, sizeof(*v)); 926 1123 return arch_atomic64_fetch_dec(v); 927 1124 } 928 - #define atomic64_fetch_dec atomic64_fetch_dec 929 - #endif 930 1125 931 - #if defined(arch_atomic64_fetch_dec_acquire) 932 1126 static __always_inline s64 933 1127 atomic64_fetch_dec_acquire(atomic64_t *v) 934 1128 { 935 1129 instrument_atomic_read_write(v, sizeof(*v)); 936 1130 return arch_atomic64_fetch_dec_acquire(v); 937 1131 } 938 - #define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire 939 - #endif 940 1132 941 - #if defined(arch_atomic64_fetch_dec_release) 942 1133 static __always_inline s64 943 1134 atomic64_fetch_dec_release(atomic64_t *v) 944 1135 { 945 1136 instrument_atomic_read_write(v, sizeof(*v)); 946 1137 return arch_atomic64_fetch_dec_release(v); 947 1138 } 948 - #define atomic64_fetch_dec_release atomic64_fetch_dec_release 949 - #endif 950 1139 951 - #if defined(arch_atomic64_fetch_dec_relaxed) 952 1140 static __always_inline s64 953 1141 atomic64_fetch_dec_relaxed(atomic64_t *v) 954 1142 { 955 1143 instrument_atomic_read_write(v, sizeof(*v)); 956 1144 return arch_atomic64_fetch_dec_relaxed(v); 957 1145 } 958 - #define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed 959 - #endif 960 1146 961 1147 static __always_inline void 962 1148 atomic64_and(s64 i, atomic64_t *v) ··· 885 1229 instrument_atomic_read_write(v, sizeof(*v)); 886 1230 arch_atomic64_and(i, v); 887 1231 } 888 - #define atomic64_and atomic64_and 889 1232 890 - #if !defined(arch_atomic64_fetch_and_relaxed) || defined(arch_atomic64_fetch_and) 891 1233 static __always_inline s64 892 1234 atomic64_fetch_and(s64 i, atomic64_t *v) 893 1235 { 894 1236 instrument_atomic_read_write(v, sizeof(*v)); 895 1237 return arch_atomic64_fetch_and(i, v); 896 1238 } 897 - #define atomic64_fetch_and atomic64_fetch_and 898 - #endif 899 1239 900 - #if defined(arch_atomic64_fetch_and_acquire) 901 1240 static __always_inline s64 902 1241 atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 903 1242 { 904 1243 instrument_atomic_read_write(v, sizeof(*v)); 905 1244 return arch_atomic64_fetch_and_acquire(i, v); 906 1245 } 907 - #define atomic64_fetch_and_acquire atomic64_fetch_and_acquire 908 - #endif 909 1246 910 - #if defined(arch_atomic64_fetch_and_release) 911 1247 static __always_inline s64 912 1248 atomic64_fetch_and_release(s64 i, atomic64_t *v) 913 1249 { 914 1250 instrument_atomic_read_write(v, sizeof(*v)); 915 1251 return arch_atomic64_fetch_and_release(i, v); 916 1252 } 917 - #define atomic64_fetch_and_release atomic64_fetch_and_release 918 - #endif 919 1253 920 - #if defined(arch_atomic64_fetch_and_relaxed) 921 1254 static __always_inline s64 922 1255 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) 923 1256 { 924 1257 instrument_atomic_read_write(v, sizeof(*v)); 925 1258 return arch_atomic64_fetch_and_relaxed(i, v); 926 1259 } 927 - #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed 928 - #endif 929 1260 930 - #if defined(arch_atomic64_andnot) 931 1261 static __always_inline void 932 1262 atomic64_andnot(s64 i, atomic64_t *v) 933 1263 { 934 1264 instrument_atomic_read_write(v, sizeof(*v)); 935 1265 arch_atomic64_andnot(i, v); 936 1266 } 937 - #define atomic64_andnot atomic64_andnot 938 - #endif 939 1267 940 - #if defined(arch_atomic64_fetch_andnot) 941 1268 static __always_inline s64 942 1269 atomic64_fetch_andnot(s64 i, atomic64_t *v) 943 1270 { 944 1271 instrument_atomic_read_write(v, sizeof(*v)); 945 1272 return arch_atomic64_fetch_andnot(i, v); 946 1273 } 947 - #define atomic64_fetch_andnot atomic64_fetch_andnot 948 - #endif 949 1274 950 - #if defined(arch_atomic64_fetch_andnot_acquire) 951 1275 static __always_inline s64 952 1276 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 953 1277 { 954 1278 instrument_atomic_read_write(v, sizeof(*v)); 955 1279 return arch_atomic64_fetch_andnot_acquire(i, v); 956 1280 } 957 - #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire 958 - #endif 959 1281 960 - #if defined(arch_atomic64_fetch_andnot_release) 961 1282 static __always_inline s64 962 1283 atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 963 1284 { 964 1285 instrument_atomic_read_write(v, sizeof(*v)); 965 1286 return arch_atomic64_fetch_andnot_release(i, v); 966 1287 } 967 - #define atomic64_fetch_andnot_release atomic64_fetch_andnot_release 968 - #endif 969 1288 970 - #if defined(arch_atomic64_fetch_andnot_relaxed) 971 1289 static __always_inline s64 972 1290 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 973 1291 { 974 1292 instrument_atomic_read_write(v, sizeof(*v)); 975 1293 return arch_atomic64_fetch_andnot_relaxed(i, v); 976 1294 } 977 - #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed 978 - #endif 979 1295 980 1296 static __always_inline void 981 1297 atomic64_or(s64 i, atomic64_t *v) ··· 955 1327 instrument_atomic_read_write(v, sizeof(*v)); 956 1328 arch_atomic64_or(i, v); 957 1329 } 958 - #define atomic64_or atomic64_or 959 1330 960 - #if !defined(arch_atomic64_fetch_or_relaxed) || defined(arch_atomic64_fetch_or) 961 1331 static __always_inline s64 962 1332 atomic64_fetch_or(s64 i, atomic64_t *v) 963 1333 { 964 1334 instrument_atomic_read_write(v, sizeof(*v)); 965 1335 return arch_atomic64_fetch_or(i, v); 966 1336 } 967 - #define atomic64_fetch_or atomic64_fetch_or 968 - #endif 969 1337 970 - #if defined(arch_atomic64_fetch_or_acquire) 971 1338 static __always_inline s64 972 1339 atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 973 1340 { 974 1341 instrument_atomic_read_write(v, sizeof(*v)); 975 1342 return arch_atomic64_fetch_or_acquire(i, v); 976 1343 } 977 - #define atomic64_fetch_or_acquire atomic64_fetch_or_acquire 978 - #endif 979 1344 980 - #if defined(arch_atomic64_fetch_or_release) 981 1345 static __always_inline s64 982 1346 atomic64_fetch_or_release(s64 i, atomic64_t *v) 983 1347 { 984 1348 instrument_atomic_read_write(v, sizeof(*v)); 985 1349 return arch_atomic64_fetch_or_release(i, v); 986 1350 } 987 - #define atomic64_fetch_or_release atomic64_fetch_or_release 988 - #endif 989 1351 990 - #if defined(arch_atomic64_fetch_or_relaxed) 991 1352 static __always_inline s64 992 1353 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) 993 1354 { 994 1355 instrument_atomic_read_write(v, sizeof(*v)); 995 1356 return arch_atomic64_fetch_or_relaxed(i, v); 996 1357 } 997 - #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed 998 - #endif 999 1358 1000 1359 static __always_inline void 1001 1360 atomic64_xor(s64 i, atomic64_t *v) ··· 990 1375 instrument_atomic_read_write(v, sizeof(*v)); 991 1376 arch_atomic64_xor(i, v); 992 1377 } 993 - #define atomic64_xor atomic64_xor 994 1378 995 - #if !defined(arch_atomic64_fetch_xor_relaxed) || defined(arch_atomic64_fetch_xor) 996 1379 static __always_inline s64 997 1380 atomic64_fetch_xor(s64 i, atomic64_t *v) 998 1381 { 999 1382 instrument_atomic_read_write(v, sizeof(*v)); 1000 1383 return arch_atomic64_fetch_xor(i, v); 1001 1384 } 1002 - #define atomic64_fetch_xor atomic64_fetch_xor 1003 - #endif 1004 1385 1005 - #if defined(arch_atomic64_fetch_xor_acquire) 1006 1386 static __always_inline s64 1007 1387 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 1008 1388 { 1009 1389 instrument_atomic_read_write(v, sizeof(*v)); 1010 1390 return arch_atomic64_fetch_xor_acquire(i, v); 1011 1391 } 1012 - #define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire 1013 - #endif 1014 1392 1015 - #if defined(arch_atomic64_fetch_xor_release) 1016 1393 static __always_inline s64 1017 1394 atomic64_fetch_xor_release(s64 i, atomic64_t *v) 1018 1395 { 1019 1396 instrument_atomic_read_write(v, sizeof(*v)); 1020 1397 return arch_atomic64_fetch_xor_release(i, v); 1021 1398 } 1022 - #define atomic64_fetch_xor_release atomic64_fetch_xor_release 1023 - #endif 1024 1399 1025 - #if defined(arch_atomic64_fetch_xor_relaxed) 1026 1400 static __always_inline s64 1027 1401 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) 1028 1402 { 1029 1403 instrument_atomic_read_write(v, sizeof(*v)); 1030 1404 return arch_atomic64_fetch_xor_relaxed(i, v); 1031 1405 } 1032 - #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed 1033 - #endif 1034 1406 1035 - #if !defined(arch_atomic64_xchg_relaxed) || defined(arch_atomic64_xchg) 1036 1407 static __always_inline s64 1037 1408 atomic64_xchg(atomic64_t *v, s64 i) 1038 1409 { 1039 1410 instrument_atomic_read_write(v, sizeof(*v)); 1040 1411 return arch_atomic64_xchg(v, i); 1041 1412 } 1042 - #define atomic64_xchg atomic64_xchg 1043 - #endif 1044 1413 1045 - #if defined(arch_atomic64_xchg_acquire) 1046 1414 static __always_inline s64 1047 1415 atomic64_xchg_acquire(atomic64_t *v, s64 i) 1048 1416 { 1049 1417 instrument_atomic_read_write(v, sizeof(*v)); 1050 1418 return arch_atomic64_xchg_acquire(v, i); 1051 1419 } 1052 - #define atomic64_xchg_acquire atomic64_xchg_acquire 1053 - #endif 1054 1420 1055 - #if defined(arch_atomic64_xchg_release) 1056 1421 static __always_inline s64 1057 1422 atomic64_xchg_release(atomic64_t *v, s64 i) 1058 1423 { 1059 1424 instrument_atomic_read_write(v, sizeof(*v)); 1060 1425 return arch_atomic64_xchg_release(v, i); 1061 1426 } 1062 - #define atomic64_xchg_release atomic64_xchg_release 1063 - #endif 1064 1427 1065 - #if defined(arch_atomic64_xchg_relaxed) 1066 1428 static __always_inline s64 1067 1429 atomic64_xchg_relaxed(atomic64_t *v, s64 i) 1068 1430 { 1069 1431 instrument_atomic_read_write(v, sizeof(*v)); 1070 1432 return arch_atomic64_xchg_relaxed(v, i); 1071 1433 } 1072 - #define atomic64_xchg_relaxed atomic64_xchg_relaxed 1073 - #endif 1074 1434 1075 - #if !defined(arch_atomic64_cmpxchg_relaxed) || defined(arch_atomic64_cmpxchg) 1076 1435 static __always_inline s64 1077 1436 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 1078 1437 { 1079 1438 instrument_atomic_read_write(v, sizeof(*v)); 1080 1439 return arch_atomic64_cmpxchg(v, old, new); 1081 1440 } 1082 - #define atomic64_cmpxchg atomic64_cmpxchg 1083 - #endif 1084 1441 1085 - #if defined(arch_atomic64_cmpxchg_acquire) 1086 1442 static __always_inline s64 1087 1443 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 1088 1444 { 1089 1445 instrument_atomic_read_write(v, sizeof(*v)); 1090 1446 return arch_atomic64_cmpxchg_acquire(v, old, new); 1091 1447 } 1092 - #define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire 1093 - #endif 1094 1448 1095 - #if defined(arch_atomic64_cmpxchg_release) 1096 1449 static __always_inline s64 1097 1450 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 1098 1451 { 1099 1452 instrument_atomic_read_write(v, sizeof(*v)); 1100 1453 return arch_atomic64_cmpxchg_release(v, old, new); 1101 1454 } 1102 - #define atomic64_cmpxchg_release atomic64_cmpxchg_release 1103 - #endif 1104 1455 1105 - #if defined(arch_atomic64_cmpxchg_relaxed) 1106 1456 static __always_inline s64 1107 1457 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) 1108 1458 { 1109 1459 instrument_atomic_read_write(v, sizeof(*v)); 1110 1460 return arch_atomic64_cmpxchg_relaxed(v, old, new); 1111 1461 } 1112 - #define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed 1113 - #endif 1114 1462 1115 - #if defined(arch_atomic64_try_cmpxchg) 1116 1463 static __always_inline bool 1117 1464 atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 1118 1465 { ··· 1082 1505 instrument_atomic_read_write(old, sizeof(*old)); 1083 1506 return arch_atomic64_try_cmpxchg(v, old, new); 1084 1507 } 1085 - #define atomic64_try_cmpxchg atomic64_try_cmpxchg 1086 - #endif 1087 1508 1088 - #if defined(arch_atomic64_try_cmpxchg_acquire) 1089 1509 static __always_inline bool 1090 1510 atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 1091 1511 { ··· 1090 1516 instrument_atomic_read_write(old, sizeof(*old)); 1091 1517 return arch_atomic64_try_cmpxchg_acquire(v, old, new); 1092 1518 } 1093 - #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire 1094 - #endif 1095 1519 1096 - #if defined(arch_atomic64_try_cmpxchg_release) 1097 1520 static __always_inline bool 1098 1521 atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 1099 1522 { ··· 1098 1527 instrument_atomic_read_write(old, sizeof(*old)); 1099 1528 return arch_atomic64_try_cmpxchg_release(v, old, new); 1100 1529 } 1101 - #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release 1102 - #endif 1103 1530 1104 - #if defined(arch_atomic64_try_cmpxchg_relaxed) 1105 1531 static __always_inline bool 1106 1532 atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 1107 1533 { ··· 1106 1538 instrument_atomic_read_write(old, sizeof(*old)); 1107 1539 return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 1108 1540 } 1109 - #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed 1110 - #endif 1111 1541 1112 - #if defined(arch_atomic64_sub_and_test) 1113 1542 static __always_inline bool 1114 1543 atomic64_sub_and_test(s64 i, atomic64_t *v) 1115 1544 { 1116 1545 instrument_atomic_read_write(v, sizeof(*v)); 1117 1546 return arch_atomic64_sub_and_test(i, v); 1118 1547 } 1119 - #define atomic64_sub_and_test atomic64_sub_and_test 1120 - #endif 1121 1548 1122 - #if defined(arch_atomic64_dec_and_test) 1123 1549 static __always_inline bool 1124 1550 atomic64_dec_and_test(atomic64_t *v) 1125 1551 { 1126 1552 instrument_atomic_read_write(v, sizeof(*v)); 1127 1553 return arch_atomic64_dec_and_test(v); 1128 1554 } 1129 - #define atomic64_dec_and_test atomic64_dec_and_test 1130 - #endif 1131 1555 1132 - #if defined(arch_atomic64_inc_and_test) 1133 1556 static __always_inline bool 1134 1557 atomic64_inc_and_test(atomic64_t *v) 1135 1558 { 1136 1559 instrument_atomic_read_write(v, sizeof(*v)); 1137 1560 return arch_atomic64_inc_and_test(v); 1138 1561 } 1139 - #define atomic64_inc_and_test atomic64_inc_and_test 1140 - #endif 1141 1562 1142 - #if defined(arch_atomic64_add_negative) 1143 1563 static __always_inline bool 1144 1564 atomic64_add_negative(s64 i, atomic64_t *v) 1145 1565 { 1146 1566 instrument_atomic_read_write(v, sizeof(*v)); 1147 1567 return arch_atomic64_add_negative(i, v); 1148 1568 } 1149 - #define atomic64_add_negative atomic64_add_negative 1150 - #endif 1151 1569 1152 - #if defined(arch_atomic64_fetch_add_unless) 1153 1570 static __always_inline s64 1154 1571 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 1155 1572 { 1156 1573 instrument_atomic_read_write(v, sizeof(*v)); 1157 1574 return arch_atomic64_fetch_add_unless(v, a, u); 1158 1575 } 1159 - #define atomic64_fetch_add_unless atomic64_fetch_add_unless 1160 - #endif 1161 1576 1162 - #if defined(arch_atomic64_add_unless) 1163 1577 static __always_inline bool 1164 1578 atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 1165 1579 { 1166 1580 instrument_atomic_read_write(v, sizeof(*v)); 1167 1581 return arch_atomic64_add_unless(v, a, u); 1168 1582 } 1169 - #define atomic64_add_unless atomic64_add_unless 1170 - #endif 1171 1583 1172 - #if defined(arch_atomic64_inc_not_zero) 1173 1584 static __always_inline bool 1174 1585 atomic64_inc_not_zero(atomic64_t *v) 1175 1586 { 1176 1587 instrument_atomic_read_write(v, sizeof(*v)); 1177 1588 return arch_atomic64_inc_not_zero(v); 1178 1589 } 1179 - #define atomic64_inc_not_zero atomic64_inc_not_zero 1180 - #endif 1181 1590 1182 - #if defined(arch_atomic64_inc_unless_negative) 1183 1591 static __always_inline bool 1184 1592 atomic64_inc_unless_negative(atomic64_t *v) 1185 1593 { 1186 1594 instrument_atomic_read_write(v, sizeof(*v)); 1187 1595 return arch_atomic64_inc_unless_negative(v); 1188 1596 } 1189 - #define atomic64_inc_unless_negative atomic64_inc_unless_negative 1190 - #endif 1191 1597 1192 - #if defined(arch_atomic64_dec_unless_positive) 1193 1598 static __always_inline bool 1194 1599 atomic64_dec_unless_positive(atomic64_t *v) 1195 1600 { 1196 1601 instrument_atomic_read_write(v, sizeof(*v)); 1197 1602 return arch_atomic64_dec_unless_positive(v); 1198 1603 } 1199 - #define atomic64_dec_unless_positive atomic64_dec_unless_positive 1200 - #endif 1201 1604 1202 - #if defined(arch_atomic64_dec_if_positive) 1203 1605 static __always_inline s64 1204 1606 atomic64_dec_if_positive(atomic64_t *v) 1205 1607 { 1206 1608 instrument_atomic_read_write(v, sizeof(*v)); 1207 1609 return arch_atomic64_dec_if_positive(v); 1208 1610 } 1209 - #define atomic64_dec_if_positive atomic64_dec_if_positive 1210 - #endif 1211 1611 1212 - #if !defined(arch_xchg_relaxed) || defined(arch_xchg) 1213 1612 #define xchg(ptr, ...) \ 1214 1613 ({ \ 1215 1614 typeof(ptr) __ai_ptr = (ptr); \ 1216 1615 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1217 1616 arch_xchg(__ai_ptr, __VA_ARGS__); \ 1218 1617 }) 1219 - #endif 1220 1618 1221 - #if defined(arch_xchg_acquire) 1222 1619 #define xchg_acquire(ptr, ...) \ 1223 1620 ({ \ 1224 1621 typeof(ptr) __ai_ptr = (ptr); \ 1225 1622 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1226 1623 arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ 1227 1624 }) 1228 - #endif 1229 1625 1230 - #if defined(arch_xchg_release) 1231 1626 #define xchg_release(ptr, ...) \ 1232 1627 ({ \ 1233 1628 typeof(ptr) __ai_ptr = (ptr); \ 1234 1629 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1235 1630 arch_xchg_release(__ai_ptr, __VA_ARGS__); \ 1236 1631 }) 1237 - #endif 1238 1632 1239 - #if defined(arch_xchg_relaxed) 1240 1633 #define xchg_relaxed(ptr, ...) \ 1241 1634 ({ \ 1242 1635 typeof(ptr) __ai_ptr = (ptr); \ 1243 1636 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1244 1637 arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ 1245 1638 }) 1246 - #endif 1247 1639 1248 - #if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg) 1249 1640 #define cmpxchg(ptr, ...) \ 1250 1641 ({ \ 1251 1642 typeof(ptr) __ai_ptr = (ptr); \ 1252 1643 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1253 1644 arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ 1254 1645 }) 1255 - #endif 1256 1646 1257 - #if defined(arch_cmpxchg_acquire) 1258 1647 #define cmpxchg_acquire(ptr, ...) \ 1259 1648 ({ \ 1260 1649 typeof(ptr) __ai_ptr = (ptr); \ 1261 1650 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1262 1651 arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ 1263 1652 }) 1264 - #endif 1265 1653 1266 - #if defined(arch_cmpxchg_release) 1267 1654 #define cmpxchg_release(ptr, ...) \ 1268 1655 ({ \ 1269 1656 typeof(ptr) __ai_ptr = (ptr); \ 1270 1657 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1271 1658 arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ 1272 1659 }) 1273 - #endif 1274 1660 1275 - #if defined(arch_cmpxchg_relaxed) 1276 1661 #define cmpxchg_relaxed(ptr, ...) \ 1277 1662 ({ \ 1278 1663 typeof(ptr) __ai_ptr = (ptr); \ 1279 1664 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1280 1665 arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ 1281 1666 }) 1282 - #endif 1283 1667 1284 - #if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64) 1285 1668 #define cmpxchg64(ptr, ...) \ 1286 1669 ({ \ 1287 1670 typeof(ptr) __ai_ptr = (ptr); \ 1288 1671 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1289 1672 arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ 1290 1673 }) 1291 - #endif 1292 1674 1293 - #if defined(arch_cmpxchg64_acquire) 1294 1675 #define cmpxchg64_acquire(ptr, ...) \ 1295 1676 ({ \ 1296 1677 typeof(ptr) __ai_ptr = (ptr); \ 1297 1678 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1298 1679 arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ 1299 1680 }) 1300 - #endif 1301 1681 1302 - #if defined(arch_cmpxchg64_release) 1303 1682 #define cmpxchg64_release(ptr, ...) \ 1304 1683 ({ \ 1305 1684 typeof(ptr) __ai_ptr = (ptr); \ 1306 1685 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1307 1686 arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ 1308 1687 }) 1309 - #endif 1310 1688 1311 - #if defined(arch_cmpxchg64_relaxed) 1312 1689 #define cmpxchg64_relaxed(ptr, ...) \ 1313 1690 ({ \ 1314 1691 typeof(ptr) __ai_ptr = (ptr); \ 1315 1692 instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ 1316 1693 arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ 1317 1694 }) 1318 - #endif 1319 1695 1320 - #if !defined(arch_try_cmpxchg_relaxed) || defined(arch_try_cmpxchg) 1321 1696 #define try_cmpxchg(ptr, oldp, ...) \ 1322 1697 ({ \ 1323 1698 typeof(ptr) __ai_ptr = (ptr); \ ··· 1269 1758 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ 1270 1759 arch_try_cmpxchg(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 1271 1760 }) 1272 - #endif 1273 1761 1274 - #if defined(arch_try_cmpxchg_acquire) 1275 1762 #define try_cmpxchg_acquire(ptr, oldp, ...) \ 1276 1763 ({ \ 1277 1764 typeof(ptr) __ai_ptr = (ptr); \ ··· 1278 1769 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ 1279 1770 arch_try_cmpxchg_acquire(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 1280 1771 }) 1281 - #endif 1282 1772 1283 - #if defined(arch_try_cmpxchg_release) 1284 1773 #define try_cmpxchg_release(ptr, oldp, ...) \ 1285 1774 ({ \ 1286 1775 typeof(ptr) __ai_ptr = (ptr); \ ··· 1287 1780 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ 1288 1781 arch_try_cmpxchg_release(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 1289 1782 }) 1290 - #endif 1291 1783 1292 - #if defined(arch_try_cmpxchg_relaxed) 1293 1784 #define try_cmpxchg_relaxed(ptr, oldp, ...) \ 1294 1785 ({ \ 1295 1786 typeof(ptr) __ai_ptr = (ptr); \ ··· 1296 1791 instrument_atomic_write(__ai_oldp, sizeof(*__ai_oldp)); \ 1297 1792 arch_try_cmpxchg_relaxed(__ai_ptr, __ai_oldp, __VA_ARGS__); \ 1298 1793 }) 1299 - #endif 1300 1794 1301 1795 #define cmpxchg_local(ptr, ...) \ 1302 1796 ({ \ ··· 1334 1830 }) 1335 1831 1336 1832 #endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */ 1337 - // 4bec382e44520f4d8267e42620054db26a659ea3 1833 + // 1d7c3a25aca5c7fb031c307be4c3d24c7b48fcd5
+2 -49
scripts/atomic/gen-atomic-instrumented.sh
··· 41 41 done 42 42 } 43 43 44 - # gen_guard(meta, atomic, pfx, name, sfx, order) 45 - gen_guard() 46 - { 47 - local meta="$1"; shift 48 - local atomic="$1"; shift 49 - local pfx="$1"; shift 50 - local name="$1"; shift 51 - local sfx="$1"; shift 52 - local order="$1"; shift 53 - 54 - local atomicname="arch_${atomic}_${pfx}${name}${sfx}${order}" 55 - 56 - local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")" 57 - 58 - # We definitely need a preprocessor symbol for this atomic if it is an 59 - # ordering variant, or if there's a generic fallback. 60 - if [ ! -z "${order}" ] || [ ! -z "${template}" ]; then 61 - printf "defined(${atomicname})" 62 - return 63 - fi 64 - 65 - # If this is a base variant, but a relaxed variant *may* exist, then we 66 - # only have a preprocessor symbol if the relaxed variant isn't defined 67 - if meta_has_relaxed "${meta}"; then 68 - printf "!defined(${atomicname}_relaxed) || defined(${atomicname})" 69 - fi 70 - } 71 - 72 44 #gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...) 73 45 gen_proto_order_variant() 74 46 { ··· 54 82 55 83 local atomicname="${atomic}_${pfx}${name}${sfx}${order}" 56 84 57 - local guard="$(gen_guard "${meta}" "${atomic}" "${pfx}" "${name}" "${sfx}" "${order}")" 58 - 59 85 local ret="$(gen_ret_type "${meta}" "${int}")" 60 86 local params="$(gen_params "${int}" "${atomic}" "$@")" 61 87 local checks="$(gen_params_checks "${meta}" "$@")" 62 88 local args="$(gen_args "$@")" 63 89 local retstmt="$(gen_ret_stmt "${meta}")" 64 - 65 - [ ! -z "${guard}" ] && printf "#if ${guard}\n" 66 90 67 91 cat <<EOF 68 92 static __always_inline ${ret} ··· 67 99 ${checks} 68 100 ${retstmt}arch_${atomicname}(${args}); 69 101 } 70 - #define ${atomicname} ${atomicname} 71 102 EOF 72 - 73 - [ ! -z "${guard}" ] && printf "#endif\n" 74 103 75 104 printf "\n" 76 105 } ··· 102 137 EOF 103 138 104 139 fi 105 - } 106 - 107 - gen_optional_xchg() 108 - { 109 - local name="$1"; shift 110 - local sfx="$1"; shift 111 - local guard="defined(arch_${name}${sfx})" 112 - 113 - [ -z "${sfx}" ] && guard="!defined(arch_${name}_relaxed) || defined(arch_${name})" 114 - 115 - printf "#if ${guard}\n" 116 - gen_xchg "${name}${sfx}" "" 117 - printf "#endif\n\n" 118 140 } 119 141 120 142 cat << EOF ··· 140 188 141 189 for xchg in "xchg" "cmpxchg" "cmpxchg64" "try_cmpxchg"; do 142 190 for order in "" "_acquire" "_release" "_relaxed"; do 143 - gen_optional_xchg "${xchg}" "${order}" 191 + gen_xchg "${xchg}${order}" "" 192 + printf "\n" 144 193 done 145 194 done 146 195