tjh.dev / kernel
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel os linux
fork atom
kernel / include / linux / atomic / atomic-arch-fallback.h
at v6.6 127 kB view raw
1// SPDX-License-Identifier: GPL-2.0 2 3// Generated by scripts/atomic/gen-atomic-fallback.sh 4// DO NOT MODIFY THIS FILE DIRECTLY 5 6#ifndef _LINUX_ATOMIC_FALLBACK_H 7#define _LINUX_ATOMIC_FALLBACK_H 8 9#include <linux/compiler.h> 10 11#if defined(arch_xchg) 12#define raw_xchg arch_xchg 13#elif defined(arch_xchg_relaxed) 14#define raw_xchg(...) \ 15 __atomic_op_fence(arch_xchg, __VA_ARGS__) 16#else 17extern void raw_xchg_not_implemented(void); 18#define raw_xchg(...) raw_xchg_not_implemented() 19#endif 20 21#if defined(arch_xchg_acquire) 22#define raw_xchg_acquire arch_xchg_acquire 23#elif defined(arch_xchg_relaxed) 24#define raw_xchg_acquire(...) \ 25 __atomic_op_acquire(arch_xchg, __VA_ARGS__) 26#elif defined(arch_xchg) 27#define raw_xchg_acquire arch_xchg 28#else 29extern void raw_xchg_acquire_not_implemented(void); 30#define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented() 31#endif 32 33#if defined(arch_xchg_release) 34#define raw_xchg_release arch_xchg_release 35#elif defined(arch_xchg_relaxed) 36#define raw_xchg_release(...) \ 37 __atomic_op_release(arch_xchg, __VA_ARGS__) 38#elif defined(arch_xchg) 39#define raw_xchg_release arch_xchg 40#else 41extern void raw_xchg_release_not_implemented(void); 42#define raw_xchg_release(...) raw_xchg_release_not_implemented() 43#endif 44 45#if defined(arch_xchg_relaxed) 46#define raw_xchg_relaxed arch_xchg_relaxed 47#elif defined(arch_xchg) 48#define raw_xchg_relaxed arch_xchg 49#else 50extern void raw_xchg_relaxed_not_implemented(void); 51#define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented() 52#endif 53 54#if defined(arch_cmpxchg) 55#define raw_cmpxchg arch_cmpxchg 56#elif defined(arch_cmpxchg_relaxed) 57#define raw_cmpxchg(...) \ 58 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__) 59#else 60extern void raw_cmpxchg_not_implemented(void); 61#define raw_cmpxchg(...) raw_cmpxchg_not_implemented() 62#endif 63 64#if defined(arch_cmpxchg_acquire) 65#define raw_cmpxchg_acquire arch_cmpxchg_acquire 66#elif defined(arch_cmpxchg_relaxed) 67#define raw_cmpxchg_acquire(...) \ 68 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__) 69#elif defined(arch_cmpxchg) 70#define raw_cmpxchg_acquire arch_cmpxchg 71#else 72extern void raw_cmpxchg_acquire_not_implemented(void); 73#define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented() 74#endif 75 76#if defined(arch_cmpxchg_release) 77#define raw_cmpxchg_release arch_cmpxchg_release 78#elif defined(arch_cmpxchg_relaxed) 79#define raw_cmpxchg_release(...) \ 80 __atomic_op_release(arch_cmpxchg, __VA_ARGS__) 81#elif defined(arch_cmpxchg) 82#define raw_cmpxchg_release arch_cmpxchg 83#else 84extern void raw_cmpxchg_release_not_implemented(void); 85#define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented() 86#endif 87 88#if defined(arch_cmpxchg_relaxed) 89#define raw_cmpxchg_relaxed arch_cmpxchg_relaxed 90#elif defined(arch_cmpxchg) 91#define raw_cmpxchg_relaxed arch_cmpxchg 92#else 93extern void raw_cmpxchg_relaxed_not_implemented(void); 94#define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented() 95#endif 96 97#if defined(arch_cmpxchg64) 98#define raw_cmpxchg64 arch_cmpxchg64 99#elif defined(arch_cmpxchg64_relaxed) 100#define raw_cmpxchg64(...) \ 101 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__) 102#else 103extern void raw_cmpxchg64_not_implemented(void); 104#define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented() 105#endif 106 107#if defined(arch_cmpxchg64_acquire) 108#define raw_cmpxchg64_acquire arch_cmpxchg64_acquire 109#elif defined(arch_cmpxchg64_relaxed) 110#define raw_cmpxchg64_acquire(...) \ 111 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__) 112#elif defined(arch_cmpxchg64) 113#define raw_cmpxchg64_acquire arch_cmpxchg64 114#else 115extern void raw_cmpxchg64_acquire_not_implemented(void); 116#define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented() 117#endif 118 119#if defined(arch_cmpxchg64_release) 120#define raw_cmpxchg64_release arch_cmpxchg64_release 121#elif defined(arch_cmpxchg64_relaxed) 122#define raw_cmpxchg64_release(...) \ 123 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__) 124#elif defined(arch_cmpxchg64) 125#define raw_cmpxchg64_release arch_cmpxchg64 126#else 127extern void raw_cmpxchg64_release_not_implemented(void); 128#define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented() 129#endif 130 131#if defined(arch_cmpxchg64_relaxed) 132#define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed 133#elif defined(arch_cmpxchg64) 134#define raw_cmpxchg64_relaxed arch_cmpxchg64 135#else 136extern void raw_cmpxchg64_relaxed_not_implemented(void); 137#define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented() 138#endif 139 140#if defined(arch_cmpxchg128) 141#define raw_cmpxchg128 arch_cmpxchg128 142#elif defined(arch_cmpxchg128_relaxed) 143#define raw_cmpxchg128(...) \ 144 __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__) 145#else 146extern void raw_cmpxchg128_not_implemented(void); 147#define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented() 148#endif 149 150#if defined(arch_cmpxchg128_acquire) 151#define raw_cmpxchg128_acquire arch_cmpxchg128_acquire 152#elif defined(arch_cmpxchg128_relaxed) 153#define raw_cmpxchg128_acquire(...) \ 154 __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__) 155#elif defined(arch_cmpxchg128) 156#define raw_cmpxchg128_acquire arch_cmpxchg128 157#else 158extern void raw_cmpxchg128_acquire_not_implemented(void); 159#define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented() 160#endif 161 162#if defined(arch_cmpxchg128_release) 163#define raw_cmpxchg128_release arch_cmpxchg128_release 164#elif defined(arch_cmpxchg128_relaxed) 165#define raw_cmpxchg128_release(...) \ 166 __atomic_op_release(arch_cmpxchg128, __VA_ARGS__) 167#elif defined(arch_cmpxchg128) 168#define raw_cmpxchg128_release arch_cmpxchg128 169#else 170extern void raw_cmpxchg128_release_not_implemented(void); 171#define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented() 172#endif 173 174#if defined(arch_cmpxchg128_relaxed) 175#define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed 176#elif defined(arch_cmpxchg128) 177#define raw_cmpxchg128_relaxed arch_cmpxchg128 178#else 179extern void raw_cmpxchg128_relaxed_not_implemented(void); 180#define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented() 181#endif 182 183#if defined(arch_try_cmpxchg) 184#define raw_try_cmpxchg arch_try_cmpxchg 185#elif defined(arch_try_cmpxchg_relaxed) 186#define raw_try_cmpxchg(...) \ 187 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__) 188#else 189#define raw_try_cmpxchg(_ptr, _oldp, _new) \ 190({ \ 191 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 192 ___r = raw_cmpxchg((_ptr), ___o, (_new)); \ 193 if (unlikely(___r != ___o)) \ 194 *___op = ___r; \ 195 likely(___r == ___o); \ 196}) 197#endif 198 199#if defined(arch_try_cmpxchg_acquire) 200#define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire 201#elif defined(arch_try_cmpxchg_relaxed) 202#define raw_try_cmpxchg_acquire(...) \ 203 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__) 204#elif defined(arch_try_cmpxchg) 205#define raw_try_cmpxchg_acquire arch_try_cmpxchg 206#else 207#define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \ 208({ \ 209 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 210 ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \ 211 if (unlikely(___r != ___o)) \ 212 *___op = ___r; \ 213 likely(___r == ___o); \ 214}) 215#endif 216 217#if defined(arch_try_cmpxchg_release) 218#define raw_try_cmpxchg_release arch_try_cmpxchg_release 219#elif defined(arch_try_cmpxchg_relaxed) 220#define raw_try_cmpxchg_release(...) \ 221 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__) 222#elif defined(arch_try_cmpxchg) 223#define raw_try_cmpxchg_release arch_try_cmpxchg 224#else 225#define raw_try_cmpxchg_release(_ptr, _oldp, _new) \ 226({ \ 227 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 228 ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \ 229 if (unlikely(___r != ___o)) \ 230 *___op = ___r; \ 231 likely(___r == ___o); \ 232}) 233#endif 234 235#if defined(arch_try_cmpxchg_relaxed) 236#define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed 237#elif defined(arch_try_cmpxchg) 238#define raw_try_cmpxchg_relaxed arch_try_cmpxchg 239#else 240#define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \ 241({ \ 242 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 243 ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \ 244 if (unlikely(___r != ___o)) \ 245 *___op = ___r; \ 246 likely(___r == ___o); \ 247}) 248#endif 249 250#if defined(arch_try_cmpxchg64) 251#define raw_try_cmpxchg64 arch_try_cmpxchg64 252#elif defined(arch_try_cmpxchg64_relaxed) 253#define raw_try_cmpxchg64(...) \ 254 __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__) 255#else 256#define raw_try_cmpxchg64(_ptr, _oldp, _new) \ 257({ \ 258 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 259 ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \ 260 if (unlikely(___r != ___o)) \ 261 *___op = ___r; \ 262 likely(___r == ___o); \ 263}) 264#endif 265 266#if defined(arch_try_cmpxchg64_acquire) 267#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire 268#elif defined(arch_try_cmpxchg64_relaxed) 269#define raw_try_cmpxchg64_acquire(...) \ 270 __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__) 271#elif defined(arch_try_cmpxchg64) 272#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64 273#else 274#define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \ 275({ \ 276 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 277 ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \ 278 if (unlikely(___r != ___o)) \ 279 *___op = ___r; \ 280 likely(___r == ___o); \ 281}) 282#endif 283 284#if defined(arch_try_cmpxchg64_release) 285#define raw_try_cmpxchg64_release arch_try_cmpxchg64_release 286#elif defined(arch_try_cmpxchg64_relaxed) 287#define raw_try_cmpxchg64_release(...) \ 288 __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__) 289#elif defined(arch_try_cmpxchg64) 290#define raw_try_cmpxchg64_release arch_try_cmpxchg64 291#else 292#define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \ 293({ \ 294 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 295 ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \ 296 if (unlikely(___r != ___o)) \ 297 *___op = ___r; \ 298 likely(___r == ___o); \ 299}) 300#endif 301 302#if defined(arch_try_cmpxchg64_relaxed) 303#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed 304#elif defined(arch_try_cmpxchg64) 305#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64 306#else 307#define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \ 308({ \ 309 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 310 ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \ 311 if (unlikely(___r != ___o)) \ 312 *___op = ___r; \ 313 likely(___r == ___o); \ 314}) 315#endif 316 317#if defined(arch_try_cmpxchg128) 318#define raw_try_cmpxchg128 arch_try_cmpxchg128 319#elif defined(arch_try_cmpxchg128_relaxed) 320#define raw_try_cmpxchg128(...) \ 321 __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__) 322#else 323#define raw_try_cmpxchg128(_ptr, _oldp, _new) \ 324({ \ 325 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 326 ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \ 327 if (unlikely(___r != ___o)) \ 328 *___op = ___r; \ 329 likely(___r == ___o); \ 330}) 331#endif 332 333#if defined(arch_try_cmpxchg128_acquire) 334#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire 335#elif defined(arch_try_cmpxchg128_relaxed) 336#define raw_try_cmpxchg128_acquire(...) \ 337 __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__) 338#elif defined(arch_try_cmpxchg128) 339#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128 340#else 341#define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \ 342({ \ 343 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 344 ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \ 345 if (unlikely(___r != ___o)) \ 346 *___op = ___r; \ 347 likely(___r == ___o); \ 348}) 349#endif 350 351#if defined(arch_try_cmpxchg128_release) 352#define raw_try_cmpxchg128_release arch_try_cmpxchg128_release 353#elif defined(arch_try_cmpxchg128_relaxed) 354#define raw_try_cmpxchg128_release(...) \ 355 __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__) 356#elif defined(arch_try_cmpxchg128) 357#define raw_try_cmpxchg128_release arch_try_cmpxchg128 358#else 359#define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \ 360({ \ 361 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 362 ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \ 363 if (unlikely(___r != ___o)) \ 364 *___op = ___r; \ 365 likely(___r == ___o); \ 366}) 367#endif 368 369#if defined(arch_try_cmpxchg128_relaxed) 370#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed 371#elif defined(arch_try_cmpxchg128) 372#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128 373#else 374#define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \ 375({ \ 376 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 377 ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \ 378 if (unlikely(___r != ___o)) \ 379 *___op = ___r; \ 380 likely(___r == ___o); \ 381}) 382#endif 383 384#define raw_cmpxchg_local arch_cmpxchg_local 385 386#ifdef arch_try_cmpxchg_local 387#define raw_try_cmpxchg_local arch_try_cmpxchg_local 388#else 389#define raw_try_cmpxchg_local(_ptr, _oldp, _new) \ 390({ \ 391 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 392 ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \ 393 if (unlikely(___r != ___o)) \ 394 *___op = ___r; \ 395 likely(___r == ___o); \ 396}) 397#endif 398 399#define raw_cmpxchg64_local arch_cmpxchg64_local 400 401#ifdef arch_try_cmpxchg64_local 402#define raw_try_cmpxchg64_local arch_try_cmpxchg64_local 403#else 404#define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \ 405({ \ 406 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 407 ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \ 408 if (unlikely(___r != ___o)) \ 409 *___op = ___r; \ 410 likely(___r == ___o); \ 411}) 412#endif 413 414#define raw_cmpxchg128_local arch_cmpxchg128_local 415 416#ifdef arch_try_cmpxchg128_local 417#define raw_try_cmpxchg128_local arch_try_cmpxchg128_local 418#else 419#define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \ 420({ \ 421 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \ 422 ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \ 423 if (unlikely(___r != ___o)) \ 424 *___op = ___r; \ 425 likely(___r == ___o); \ 426}) 427#endif 428 429#define raw_sync_cmpxchg arch_sync_cmpxchg 430 431/** 432 * raw_atomic_read() - atomic load with relaxed ordering 433 * @v: pointer to atomic_t 434 * 435 * Atomically loads the value of @v with relaxed ordering. 436 * 437 * Safe to use in noinstr code; prefer atomic_read() elsewhere. 438 * 439 * Return: The value loaded from @v. 440 */ 441static __always_inline int 442raw_atomic_read(const atomic_t *v) 443{ 444 return arch_atomic_read(v); 445} 446 447/** 448 * raw_atomic_read_acquire() - atomic load with acquire ordering 449 * @v: pointer to atomic_t 450 * 451 * Atomically loads the value of @v with acquire ordering. 452 * 453 * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere. 454 * 455 * Return: The value loaded from @v. 456 */ 457static __always_inline int 458raw_atomic_read_acquire(const atomic_t *v) 459{ 460#if defined(arch_atomic_read_acquire) 461 return arch_atomic_read_acquire(v); 462#else 463 int ret; 464 465 if (__native_word(atomic_t)) { 466 ret = smp_load_acquire(&(v)->counter); 467 } else { 468 ret = raw_atomic_read(v); 469 __atomic_acquire_fence(); 470 } 471 472 return ret; 473#endif 474} 475 476/** 477 * raw_atomic_set() - atomic set with relaxed ordering 478 * @v: pointer to atomic_t 479 * @i: int value to assign 480 * 481 * Atomically sets @v to @i with relaxed ordering. 482 * 483 * Safe to use in noinstr code; prefer atomic_set() elsewhere. 484 * 485 * Return: Nothing. 486 */ 487static __always_inline void 488raw_atomic_set(atomic_t *v, int i) 489{ 490 arch_atomic_set(v, i); 491} 492 493/** 494 * raw_atomic_set_release() - atomic set with release ordering 495 * @v: pointer to atomic_t 496 * @i: int value to assign 497 * 498 * Atomically sets @v to @i with release ordering. 499 * 500 * Safe to use in noinstr code; prefer atomic_set_release() elsewhere. 501 * 502 * Return: Nothing. 503 */ 504static __always_inline void 505raw_atomic_set_release(atomic_t *v, int i) 506{ 507#if defined(arch_atomic_set_release) 508 arch_atomic_set_release(v, i); 509#else 510 if (__native_word(atomic_t)) { 511 smp_store_release(&(v)->counter, i); 512 } else { 513 __atomic_release_fence(); 514 raw_atomic_set(v, i); 515 } 516#endif 517} 518 519/** 520 * raw_atomic_add() - atomic add with relaxed ordering 521 * @i: int value to add 522 * @v: pointer to atomic_t 523 * 524 * Atomically updates @v to (@v + @i) with relaxed ordering. 525 * 526 * Safe to use in noinstr code; prefer atomic_add() elsewhere. 527 * 528 * Return: Nothing. 529 */ 530static __always_inline void 531raw_atomic_add(int i, atomic_t *v) 532{ 533 arch_atomic_add(i, v); 534} 535 536/** 537 * raw_atomic_add_return() - atomic add with full ordering 538 * @i: int value to add 539 * @v: pointer to atomic_t 540 * 541 * Atomically updates @v to (@v + @i) with full ordering. 542 * 543 * Safe to use in noinstr code; prefer atomic_add_return() elsewhere. 544 * 545 * Return: The updated value of @v. 546 */ 547static __always_inline int 548raw_atomic_add_return(int i, atomic_t *v) 549{ 550#if defined(arch_atomic_add_return) 551 return arch_atomic_add_return(i, v); 552#elif defined(arch_atomic_add_return_relaxed) 553 int ret; 554 __atomic_pre_full_fence(); 555 ret = arch_atomic_add_return_relaxed(i, v); 556 __atomic_post_full_fence(); 557 return ret; 558#else 559#error "Unable to define raw_atomic_add_return" 560#endif 561} 562 563/** 564 * raw_atomic_add_return_acquire() - atomic add with acquire ordering 565 * @i: int value to add 566 * @v: pointer to atomic_t 567 * 568 * Atomically updates @v to (@v + @i) with acquire ordering. 569 * 570 * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere. 571 * 572 * Return: The updated value of @v. 573 */ 574static __always_inline int 575raw_atomic_add_return_acquire(int i, atomic_t *v) 576{ 577#if defined(arch_atomic_add_return_acquire) 578 return arch_atomic_add_return_acquire(i, v); 579#elif defined(arch_atomic_add_return_relaxed) 580 int ret = arch_atomic_add_return_relaxed(i, v); 581 __atomic_acquire_fence(); 582 return ret; 583#elif defined(arch_atomic_add_return) 584 return arch_atomic_add_return(i, v); 585#else 586#error "Unable to define raw_atomic_add_return_acquire" 587#endif 588} 589 590/** 591 * raw_atomic_add_return_release() - atomic add with release ordering 592 * @i: int value to add 593 * @v: pointer to atomic_t 594 * 595 * Atomically updates @v to (@v + @i) with release ordering. 596 * 597 * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere. 598 * 599 * Return: The updated value of @v. 600 */ 601static __always_inline int 602raw_atomic_add_return_release(int i, atomic_t *v) 603{ 604#if defined(arch_atomic_add_return_release) 605 return arch_atomic_add_return_release(i, v); 606#elif defined(arch_atomic_add_return_relaxed) 607 __atomic_release_fence(); 608 return arch_atomic_add_return_relaxed(i, v); 609#elif defined(arch_atomic_add_return) 610 return arch_atomic_add_return(i, v); 611#else 612#error "Unable to define raw_atomic_add_return_release" 613#endif 614} 615 616/** 617 * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering 618 * @i: int value to add 619 * @v: pointer to atomic_t 620 * 621 * Atomically updates @v to (@v + @i) with relaxed ordering. 622 * 623 * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere. 624 * 625 * Return: The updated value of @v. 626 */ 627static __always_inline int 628raw_atomic_add_return_relaxed(int i, atomic_t *v) 629{ 630#if defined(arch_atomic_add_return_relaxed) 631 return arch_atomic_add_return_relaxed(i, v); 632#elif defined(arch_atomic_add_return) 633 return arch_atomic_add_return(i, v); 634#else 635#error "Unable to define raw_atomic_add_return_relaxed" 636#endif 637} 638 639/** 640 * raw_atomic_fetch_add() - atomic add with full ordering 641 * @i: int value to add 642 * @v: pointer to atomic_t 643 * 644 * Atomically updates @v to (@v + @i) with full ordering. 645 * 646 * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere. 647 * 648 * Return: The original value of @v. 649 */ 650static __always_inline int 651raw_atomic_fetch_add(int i, atomic_t *v) 652{ 653#if defined(arch_atomic_fetch_add) 654 return arch_atomic_fetch_add(i, v); 655#elif defined(arch_atomic_fetch_add_relaxed) 656 int ret; 657 __atomic_pre_full_fence(); 658 ret = arch_atomic_fetch_add_relaxed(i, v); 659 __atomic_post_full_fence(); 660 return ret; 661#else 662#error "Unable to define raw_atomic_fetch_add" 663#endif 664} 665 666/** 667 * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering 668 * @i: int value to add 669 * @v: pointer to atomic_t 670 * 671 * Atomically updates @v to (@v + @i) with acquire ordering. 672 * 673 * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere. 674 * 675 * Return: The original value of @v. 676 */ 677static __always_inline int 678raw_atomic_fetch_add_acquire(int i, atomic_t *v) 679{ 680#if defined(arch_atomic_fetch_add_acquire) 681 return arch_atomic_fetch_add_acquire(i, v); 682#elif defined(arch_atomic_fetch_add_relaxed) 683 int ret = arch_atomic_fetch_add_relaxed(i, v); 684 __atomic_acquire_fence(); 685 return ret; 686#elif defined(arch_atomic_fetch_add) 687 return arch_atomic_fetch_add(i, v); 688#else 689#error "Unable to define raw_atomic_fetch_add_acquire" 690#endif 691} 692 693/** 694 * raw_atomic_fetch_add_release() - atomic add with release ordering 695 * @i: int value to add 696 * @v: pointer to atomic_t 697 * 698 * Atomically updates @v to (@v + @i) with release ordering. 699 * 700 * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere. 701 * 702 * Return: The original value of @v. 703 */ 704static __always_inline int 705raw_atomic_fetch_add_release(int i, atomic_t *v) 706{ 707#if defined(arch_atomic_fetch_add_release) 708 return arch_atomic_fetch_add_release(i, v); 709#elif defined(arch_atomic_fetch_add_relaxed) 710 __atomic_release_fence(); 711 return arch_atomic_fetch_add_relaxed(i, v); 712#elif defined(arch_atomic_fetch_add) 713 return arch_atomic_fetch_add(i, v); 714#else 715#error "Unable to define raw_atomic_fetch_add_release" 716#endif 717} 718 719/** 720 * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering 721 * @i: int value to add 722 * @v: pointer to atomic_t 723 * 724 * Atomically updates @v to (@v + @i) with relaxed ordering. 725 * 726 * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere. 727 * 728 * Return: The original value of @v. 729 */ 730static __always_inline int 731raw_atomic_fetch_add_relaxed(int i, atomic_t *v) 732{ 733#if defined(arch_atomic_fetch_add_relaxed) 734 return arch_atomic_fetch_add_relaxed(i, v); 735#elif defined(arch_atomic_fetch_add) 736 return arch_atomic_fetch_add(i, v); 737#else 738#error "Unable to define raw_atomic_fetch_add_relaxed" 739#endif 740} 741 742/** 743 * raw_atomic_sub() - atomic subtract with relaxed ordering 744 * @i: int value to subtract 745 * @v: pointer to atomic_t 746 * 747 * Atomically updates @v to (@v - @i) with relaxed ordering. 748 * 749 * Safe to use in noinstr code; prefer atomic_sub() elsewhere. 750 * 751 * Return: Nothing. 752 */ 753static __always_inline void 754raw_atomic_sub(int i, atomic_t *v) 755{ 756 arch_atomic_sub(i, v); 757} 758 759/** 760 * raw_atomic_sub_return() - atomic subtract with full ordering 761 * @i: int value to subtract 762 * @v: pointer to atomic_t 763 * 764 * Atomically updates @v to (@v - @i) with full ordering. 765 * 766 * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere. 767 * 768 * Return: The updated value of @v. 769 */ 770static __always_inline int 771raw_atomic_sub_return(int i, atomic_t *v) 772{ 773#if defined(arch_atomic_sub_return) 774 return arch_atomic_sub_return(i, v); 775#elif defined(arch_atomic_sub_return_relaxed) 776 int ret; 777 __atomic_pre_full_fence(); 778 ret = arch_atomic_sub_return_relaxed(i, v); 779 __atomic_post_full_fence(); 780 return ret; 781#else 782#error "Unable to define raw_atomic_sub_return" 783#endif 784} 785 786/** 787 * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering 788 * @i: int value to subtract 789 * @v: pointer to atomic_t 790 * 791 * Atomically updates @v to (@v - @i) with acquire ordering. 792 * 793 * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere. 794 * 795 * Return: The updated value of @v. 796 */ 797static __always_inline int 798raw_atomic_sub_return_acquire(int i, atomic_t *v) 799{ 800#if defined(arch_atomic_sub_return_acquire) 801 return arch_atomic_sub_return_acquire(i, v); 802#elif defined(arch_atomic_sub_return_relaxed) 803 int ret = arch_atomic_sub_return_relaxed(i, v); 804 __atomic_acquire_fence(); 805 return ret; 806#elif defined(arch_atomic_sub_return) 807 return arch_atomic_sub_return(i, v); 808#else 809#error "Unable to define raw_atomic_sub_return_acquire" 810#endif 811} 812 813/** 814 * raw_atomic_sub_return_release() - atomic subtract with release ordering 815 * @i: int value to subtract 816 * @v: pointer to atomic_t 817 * 818 * Atomically updates @v to (@v - @i) with release ordering. 819 * 820 * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere. 821 * 822 * Return: The updated value of @v. 823 */ 824static __always_inline int 825raw_atomic_sub_return_release(int i, atomic_t *v) 826{ 827#if defined(arch_atomic_sub_return_release) 828 return arch_atomic_sub_return_release(i, v); 829#elif defined(arch_atomic_sub_return_relaxed) 830 __atomic_release_fence(); 831 return arch_atomic_sub_return_relaxed(i, v); 832#elif defined(arch_atomic_sub_return) 833 return arch_atomic_sub_return(i, v); 834#else 835#error "Unable to define raw_atomic_sub_return_release" 836#endif 837} 838 839/** 840 * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering 841 * @i: int value to subtract 842 * @v: pointer to atomic_t 843 * 844 * Atomically updates @v to (@v - @i) with relaxed ordering. 845 * 846 * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere. 847 * 848 * Return: The updated value of @v. 849 */ 850static __always_inline int 851raw_atomic_sub_return_relaxed(int i, atomic_t *v) 852{ 853#if defined(arch_atomic_sub_return_relaxed) 854 return arch_atomic_sub_return_relaxed(i, v); 855#elif defined(arch_atomic_sub_return) 856 return arch_atomic_sub_return(i, v); 857#else 858#error "Unable to define raw_atomic_sub_return_relaxed" 859#endif 860} 861 862/** 863 * raw_atomic_fetch_sub() - atomic subtract with full ordering 864 * @i: int value to subtract 865 * @v: pointer to atomic_t 866 * 867 * Atomically updates @v to (@v - @i) with full ordering. 868 * 869 * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere. 870 * 871 * Return: The original value of @v. 872 */ 873static __always_inline int 874raw_atomic_fetch_sub(int i, atomic_t *v) 875{ 876#if defined(arch_atomic_fetch_sub) 877 return arch_atomic_fetch_sub(i, v); 878#elif defined(arch_atomic_fetch_sub_relaxed) 879 int ret; 880 __atomic_pre_full_fence(); 881 ret = arch_atomic_fetch_sub_relaxed(i, v); 882 __atomic_post_full_fence(); 883 return ret; 884#else 885#error "Unable to define raw_atomic_fetch_sub" 886#endif 887} 888 889/** 890 * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering 891 * @i: int value to subtract 892 * @v: pointer to atomic_t 893 * 894 * Atomically updates @v to (@v - @i) with acquire ordering. 895 * 896 * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere. 897 * 898 * Return: The original value of @v. 899 */ 900static __always_inline int 901raw_atomic_fetch_sub_acquire(int i, atomic_t *v) 902{ 903#if defined(arch_atomic_fetch_sub_acquire) 904 return arch_atomic_fetch_sub_acquire(i, v); 905#elif defined(arch_atomic_fetch_sub_relaxed) 906 int ret = arch_atomic_fetch_sub_relaxed(i, v); 907 __atomic_acquire_fence(); 908 return ret; 909#elif defined(arch_atomic_fetch_sub) 910 return arch_atomic_fetch_sub(i, v); 911#else 912#error "Unable to define raw_atomic_fetch_sub_acquire" 913#endif 914} 915 916/** 917 * raw_atomic_fetch_sub_release() - atomic subtract with release ordering 918 * @i: int value to subtract 919 * @v: pointer to atomic_t 920 * 921 * Atomically updates @v to (@v - @i) with release ordering. 922 * 923 * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere. 924 * 925 * Return: The original value of @v. 926 */ 927static __always_inline int 928raw_atomic_fetch_sub_release(int i, atomic_t *v) 929{ 930#if defined(arch_atomic_fetch_sub_release) 931 return arch_atomic_fetch_sub_release(i, v); 932#elif defined(arch_atomic_fetch_sub_relaxed) 933 __atomic_release_fence(); 934 return arch_atomic_fetch_sub_relaxed(i, v); 935#elif defined(arch_atomic_fetch_sub) 936 return arch_atomic_fetch_sub(i, v); 937#else 938#error "Unable to define raw_atomic_fetch_sub_release" 939#endif 940} 941 942/** 943 * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering 944 * @i: int value to subtract 945 * @v: pointer to atomic_t 946 * 947 * Atomically updates @v to (@v - @i) with relaxed ordering. 948 * 949 * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere. 950 * 951 * Return: The original value of @v. 952 */ 953static __always_inline int 954raw_atomic_fetch_sub_relaxed(int i, atomic_t *v) 955{ 956#if defined(arch_atomic_fetch_sub_relaxed) 957 return arch_atomic_fetch_sub_relaxed(i, v); 958#elif defined(arch_atomic_fetch_sub) 959 return arch_atomic_fetch_sub(i, v); 960#else 961#error "Unable to define raw_atomic_fetch_sub_relaxed" 962#endif 963} 964 965/** 966 * raw_atomic_inc() - atomic increment with relaxed ordering 967 * @v: pointer to atomic_t 968 * 969 * Atomically updates @v to (@v + 1) with relaxed ordering. 970 * 971 * Safe to use in noinstr code; prefer atomic_inc() elsewhere. 972 * 973 * Return: Nothing. 974 */ 975static __always_inline void 976raw_atomic_inc(atomic_t *v) 977{ 978#if defined(arch_atomic_inc) 979 arch_atomic_inc(v); 980#else 981 raw_atomic_add(1, v); 982#endif 983} 984 985/** 986 * raw_atomic_inc_return() - atomic increment with full ordering 987 * @v: pointer to atomic_t 988 * 989 * Atomically updates @v to (@v + 1) with full ordering. 990 * 991 * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere. 992 * 993 * Return: The updated value of @v. 994 */ 995static __always_inline int 996raw_atomic_inc_return(atomic_t *v) 997{ 998#if defined(arch_atomic_inc_return) 999 return arch_atomic_inc_return(v); 1000#elif defined(arch_atomic_inc_return_relaxed) 1001 int ret; 1002 __atomic_pre_full_fence(); 1003 ret = arch_atomic_inc_return_relaxed(v); 1004 __atomic_post_full_fence(); 1005 return ret; 1006#else 1007 return raw_atomic_add_return(1, v); 1008#endif 1009} 1010 1011/** 1012 * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering 1013 * @v: pointer to atomic_t 1014 * 1015 * Atomically updates @v to (@v + 1) with acquire ordering. 1016 * 1017 * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere. 1018 * 1019 * Return: The updated value of @v. 1020 */ 1021static __always_inline int 1022raw_atomic_inc_return_acquire(atomic_t *v) 1023{ 1024#if defined(arch_atomic_inc_return_acquire) 1025 return arch_atomic_inc_return_acquire(v); 1026#elif defined(arch_atomic_inc_return_relaxed) 1027 int ret = arch_atomic_inc_return_relaxed(v); 1028 __atomic_acquire_fence(); 1029 return ret; 1030#elif defined(arch_atomic_inc_return) 1031 return arch_atomic_inc_return(v); 1032#else 1033 return raw_atomic_add_return_acquire(1, v); 1034#endif 1035} 1036 1037/** 1038 * raw_atomic_inc_return_release() - atomic increment with release ordering 1039 * @v: pointer to atomic_t 1040 * 1041 * Atomically updates @v to (@v + 1) with release ordering. 1042 * 1043 * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere. 1044 * 1045 * Return: The updated value of @v. 1046 */ 1047static __always_inline int 1048raw_atomic_inc_return_release(atomic_t *v) 1049{ 1050#if defined(arch_atomic_inc_return_release) 1051 return arch_atomic_inc_return_release(v); 1052#elif defined(arch_atomic_inc_return_relaxed) 1053 __atomic_release_fence(); 1054 return arch_atomic_inc_return_relaxed(v); 1055#elif defined(arch_atomic_inc_return) 1056 return arch_atomic_inc_return(v); 1057#else 1058 return raw_atomic_add_return_release(1, v); 1059#endif 1060} 1061 1062/** 1063 * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering 1064 * @v: pointer to atomic_t 1065 * 1066 * Atomically updates @v to (@v + 1) with relaxed ordering. 1067 * 1068 * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere. 1069 * 1070 * Return: The updated value of @v. 1071 */ 1072static __always_inline int 1073raw_atomic_inc_return_relaxed(atomic_t *v) 1074{ 1075#if defined(arch_atomic_inc_return_relaxed) 1076 return arch_atomic_inc_return_relaxed(v); 1077#elif defined(arch_atomic_inc_return) 1078 return arch_atomic_inc_return(v); 1079#else 1080 return raw_atomic_add_return_relaxed(1, v); 1081#endif 1082} 1083 1084/** 1085 * raw_atomic_fetch_inc() - atomic increment with full ordering 1086 * @v: pointer to atomic_t 1087 * 1088 * Atomically updates @v to (@v + 1) with full ordering. 1089 * 1090 * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere. 1091 * 1092 * Return: The original value of @v. 1093 */ 1094static __always_inline int 1095raw_atomic_fetch_inc(atomic_t *v) 1096{ 1097#if defined(arch_atomic_fetch_inc) 1098 return arch_atomic_fetch_inc(v); 1099#elif defined(arch_atomic_fetch_inc_relaxed) 1100 int ret; 1101 __atomic_pre_full_fence(); 1102 ret = arch_atomic_fetch_inc_relaxed(v); 1103 __atomic_post_full_fence(); 1104 return ret; 1105#else 1106 return raw_atomic_fetch_add(1, v); 1107#endif 1108} 1109 1110/** 1111 * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering 1112 * @v: pointer to atomic_t 1113 * 1114 * Atomically updates @v to (@v + 1) with acquire ordering. 1115 * 1116 * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere. 1117 * 1118 * Return: The original value of @v. 1119 */ 1120static __always_inline int 1121raw_atomic_fetch_inc_acquire(atomic_t *v) 1122{ 1123#if defined(arch_atomic_fetch_inc_acquire) 1124 return arch_atomic_fetch_inc_acquire(v); 1125#elif defined(arch_atomic_fetch_inc_relaxed) 1126 int ret = arch_atomic_fetch_inc_relaxed(v); 1127 __atomic_acquire_fence(); 1128 return ret; 1129#elif defined(arch_atomic_fetch_inc) 1130 return arch_atomic_fetch_inc(v); 1131#else 1132 return raw_atomic_fetch_add_acquire(1, v); 1133#endif 1134} 1135 1136/** 1137 * raw_atomic_fetch_inc_release() - atomic increment with release ordering 1138 * @v: pointer to atomic_t 1139 * 1140 * Atomically updates @v to (@v + 1) with release ordering. 1141 * 1142 * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere. 1143 * 1144 * Return: The original value of @v. 1145 */ 1146static __always_inline int 1147raw_atomic_fetch_inc_release(atomic_t *v) 1148{ 1149#if defined(arch_atomic_fetch_inc_release) 1150 return arch_atomic_fetch_inc_release(v); 1151#elif defined(arch_atomic_fetch_inc_relaxed) 1152 __atomic_release_fence(); 1153 return arch_atomic_fetch_inc_relaxed(v); 1154#elif defined(arch_atomic_fetch_inc) 1155 return arch_atomic_fetch_inc(v); 1156#else 1157 return raw_atomic_fetch_add_release(1, v); 1158#endif 1159} 1160 1161/** 1162 * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering 1163 * @v: pointer to atomic_t 1164 * 1165 * Atomically updates @v to (@v + 1) with relaxed ordering. 1166 * 1167 * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere. 1168 * 1169 * Return: The original value of @v. 1170 */ 1171static __always_inline int 1172raw_atomic_fetch_inc_relaxed(atomic_t *v) 1173{ 1174#if defined(arch_atomic_fetch_inc_relaxed) 1175 return arch_atomic_fetch_inc_relaxed(v); 1176#elif defined(arch_atomic_fetch_inc) 1177 return arch_atomic_fetch_inc(v); 1178#else 1179 return raw_atomic_fetch_add_relaxed(1, v); 1180#endif 1181} 1182 1183/** 1184 * raw_atomic_dec() - atomic decrement with relaxed ordering 1185 * @v: pointer to atomic_t 1186 * 1187 * Atomically updates @v to (@v - 1) with relaxed ordering. 1188 * 1189 * Safe to use in noinstr code; prefer atomic_dec() elsewhere. 1190 * 1191 * Return: Nothing. 1192 */ 1193static __always_inline void 1194raw_atomic_dec(atomic_t *v) 1195{ 1196#if defined(arch_atomic_dec) 1197 arch_atomic_dec(v); 1198#else 1199 raw_atomic_sub(1, v); 1200#endif 1201} 1202 1203/** 1204 * raw_atomic_dec_return() - atomic decrement with full ordering 1205 * @v: pointer to atomic_t 1206 * 1207 * Atomically updates @v to (@v - 1) with full ordering. 1208 * 1209 * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere. 1210 * 1211 * Return: The updated value of @v. 1212 */ 1213static __always_inline int 1214raw_atomic_dec_return(atomic_t *v) 1215{ 1216#if defined(arch_atomic_dec_return) 1217 return arch_atomic_dec_return(v); 1218#elif defined(arch_atomic_dec_return_relaxed) 1219 int ret; 1220 __atomic_pre_full_fence(); 1221 ret = arch_atomic_dec_return_relaxed(v); 1222 __atomic_post_full_fence(); 1223 return ret; 1224#else 1225 return raw_atomic_sub_return(1, v); 1226#endif 1227} 1228 1229/** 1230 * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering 1231 * @v: pointer to atomic_t 1232 * 1233 * Atomically updates @v to (@v - 1) with acquire ordering. 1234 * 1235 * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere. 1236 * 1237 * Return: The updated value of @v. 1238 */ 1239static __always_inline int 1240raw_atomic_dec_return_acquire(atomic_t *v) 1241{ 1242#if defined(arch_atomic_dec_return_acquire) 1243 return arch_atomic_dec_return_acquire(v); 1244#elif defined(arch_atomic_dec_return_relaxed) 1245 int ret = arch_atomic_dec_return_relaxed(v); 1246 __atomic_acquire_fence(); 1247 return ret; 1248#elif defined(arch_atomic_dec_return) 1249 return arch_atomic_dec_return(v); 1250#else 1251 return raw_atomic_sub_return_acquire(1, v); 1252#endif 1253} 1254 1255/** 1256 * raw_atomic_dec_return_release() - atomic decrement with release ordering 1257 * @v: pointer to atomic_t 1258 * 1259 * Atomically updates @v to (@v - 1) with release ordering. 1260 * 1261 * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere. 1262 * 1263 * Return: The updated value of @v. 1264 */ 1265static __always_inline int 1266raw_atomic_dec_return_release(atomic_t *v) 1267{ 1268#if defined(arch_atomic_dec_return_release) 1269 return arch_atomic_dec_return_release(v); 1270#elif defined(arch_atomic_dec_return_relaxed) 1271 __atomic_release_fence(); 1272 return arch_atomic_dec_return_relaxed(v); 1273#elif defined(arch_atomic_dec_return) 1274 return arch_atomic_dec_return(v); 1275#else 1276 return raw_atomic_sub_return_release(1, v); 1277#endif 1278} 1279 1280/** 1281 * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering 1282 * @v: pointer to atomic_t 1283 * 1284 * Atomically updates @v to (@v - 1) with relaxed ordering. 1285 * 1286 * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere. 1287 * 1288 * Return: The updated value of @v. 1289 */ 1290static __always_inline int 1291raw_atomic_dec_return_relaxed(atomic_t *v) 1292{ 1293#if defined(arch_atomic_dec_return_relaxed) 1294 return arch_atomic_dec_return_relaxed(v); 1295#elif defined(arch_atomic_dec_return) 1296 return arch_atomic_dec_return(v); 1297#else 1298 return raw_atomic_sub_return_relaxed(1, v); 1299#endif 1300} 1301 1302/** 1303 * raw_atomic_fetch_dec() - atomic decrement with full ordering 1304 * @v: pointer to atomic_t 1305 * 1306 * Atomically updates @v to (@v - 1) with full ordering. 1307 * 1308 * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere. 1309 * 1310 * Return: The original value of @v. 1311 */ 1312static __always_inline int 1313raw_atomic_fetch_dec(atomic_t *v) 1314{ 1315#if defined(arch_atomic_fetch_dec) 1316 return arch_atomic_fetch_dec(v); 1317#elif defined(arch_atomic_fetch_dec_relaxed) 1318 int ret; 1319 __atomic_pre_full_fence(); 1320 ret = arch_atomic_fetch_dec_relaxed(v); 1321 __atomic_post_full_fence(); 1322 return ret; 1323#else 1324 return raw_atomic_fetch_sub(1, v); 1325#endif 1326} 1327 1328/** 1329 * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering 1330 * @v: pointer to atomic_t 1331 * 1332 * Atomically updates @v to (@v - 1) with acquire ordering. 1333 * 1334 * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere. 1335 * 1336 * Return: The original value of @v. 1337 */ 1338static __always_inline int 1339raw_atomic_fetch_dec_acquire(atomic_t *v) 1340{ 1341#if defined(arch_atomic_fetch_dec_acquire) 1342 return arch_atomic_fetch_dec_acquire(v); 1343#elif defined(arch_atomic_fetch_dec_relaxed) 1344 int ret = arch_atomic_fetch_dec_relaxed(v); 1345 __atomic_acquire_fence(); 1346 return ret; 1347#elif defined(arch_atomic_fetch_dec) 1348 return arch_atomic_fetch_dec(v); 1349#else 1350 return raw_atomic_fetch_sub_acquire(1, v); 1351#endif 1352} 1353 1354/** 1355 * raw_atomic_fetch_dec_release() - atomic decrement with release ordering 1356 * @v: pointer to atomic_t 1357 * 1358 * Atomically updates @v to (@v - 1) with release ordering. 1359 * 1360 * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere. 1361 * 1362 * Return: The original value of @v. 1363 */ 1364static __always_inline int 1365raw_atomic_fetch_dec_release(atomic_t *v) 1366{ 1367#if defined(arch_atomic_fetch_dec_release) 1368 return arch_atomic_fetch_dec_release(v); 1369#elif defined(arch_atomic_fetch_dec_relaxed) 1370 __atomic_release_fence(); 1371 return arch_atomic_fetch_dec_relaxed(v); 1372#elif defined(arch_atomic_fetch_dec) 1373 return arch_atomic_fetch_dec(v); 1374#else 1375 return raw_atomic_fetch_sub_release(1, v); 1376#endif 1377} 1378 1379/** 1380 * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering 1381 * @v: pointer to atomic_t 1382 * 1383 * Atomically updates @v to (@v - 1) with relaxed ordering. 1384 * 1385 * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere. 1386 * 1387 * Return: The original value of @v. 1388 */ 1389static __always_inline int 1390raw_atomic_fetch_dec_relaxed(atomic_t *v) 1391{ 1392#if defined(arch_atomic_fetch_dec_relaxed) 1393 return arch_atomic_fetch_dec_relaxed(v); 1394#elif defined(arch_atomic_fetch_dec) 1395 return arch_atomic_fetch_dec(v); 1396#else 1397 return raw_atomic_fetch_sub_relaxed(1, v); 1398#endif 1399} 1400 1401/** 1402 * raw_atomic_and() - atomic bitwise AND with relaxed ordering 1403 * @i: int value 1404 * @v: pointer to atomic_t 1405 * 1406 * Atomically updates @v to (@v & @i) with relaxed ordering. 1407 * 1408 * Safe to use in noinstr code; prefer atomic_and() elsewhere. 1409 * 1410 * Return: Nothing. 1411 */ 1412static __always_inline void 1413raw_atomic_and(int i, atomic_t *v) 1414{ 1415 arch_atomic_and(i, v); 1416} 1417 1418/** 1419 * raw_atomic_fetch_and() - atomic bitwise AND with full ordering 1420 * @i: int value 1421 * @v: pointer to atomic_t 1422 * 1423 * Atomically updates @v to (@v & @i) with full ordering. 1424 * 1425 * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere. 1426 * 1427 * Return: The original value of @v. 1428 */ 1429static __always_inline int 1430raw_atomic_fetch_and(int i, atomic_t *v) 1431{ 1432#if defined(arch_atomic_fetch_and) 1433 return arch_atomic_fetch_and(i, v); 1434#elif defined(arch_atomic_fetch_and_relaxed) 1435 int ret; 1436 __atomic_pre_full_fence(); 1437 ret = arch_atomic_fetch_and_relaxed(i, v); 1438 __atomic_post_full_fence(); 1439 return ret; 1440#else 1441#error "Unable to define raw_atomic_fetch_and" 1442#endif 1443} 1444 1445/** 1446 * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering 1447 * @i: int value 1448 * @v: pointer to atomic_t 1449 * 1450 * Atomically updates @v to (@v & @i) with acquire ordering. 1451 * 1452 * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere. 1453 * 1454 * Return: The original value of @v. 1455 */ 1456static __always_inline int 1457raw_atomic_fetch_and_acquire(int i, atomic_t *v) 1458{ 1459#if defined(arch_atomic_fetch_and_acquire) 1460 return arch_atomic_fetch_and_acquire(i, v); 1461#elif defined(arch_atomic_fetch_and_relaxed) 1462 int ret = arch_atomic_fetch_and_relaxed(i, v); 1463 __atomic_acquire_fence(); 1464 return ret; 1465#elif defined(arch_atomic_fetch_and) 1466 return arch_atomic_fetch_and(i, v); 1467#else 1468#error "Unable to define raw_atomic_fetch_and_acquire" 1469#endif 1470} 1471 1472/** 1473 * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering 1474 * @i: int value 1475 * @v: pointer to atomic_t 1476 * 1477 * Atomically updates @v to (@v & @i) with release ordering. 1478 * 1479 * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere. 1480 * 1481 * Return: The original value of @v. 1482 */ 1483static __always_inline int 1484raw_atomic_fetch_and_release(int i, atomic_t *v) 1485{ 1486#if defined(arch_atomic_fetch_and_release) 1487 return arch_atomic_fetch_and_release(i, v); 1488#elif defined(arch_atomic_fetch_and_relaxed) 1489 __atomic_release_fence(); 1490 return arch_atomic_fetch_and_relaxed(i, v); 1491#elif defined(arch_atomic_fetch_and) 1492 return arch_atomic_fetch_and(i, v); 1493#else 1494#error "Unable to define raw_atomic_fetch_and_release" 1495#endif 1496} 1497 1498/** 1499 * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering 1500 * @i: int value 1501 * @v: pointer to atomic_t 1502 * 1503 * Atomically updates @v to (@v & @i) with relaxed ordering. 1504 * 1505 * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere. 1506 * 1507 * Return: The original value of @v. 1508 */ 1509static __always_inline int 1510raw_atomic_fetch_and_relaxed(int i, atomic_t *v) 1511{ 1512#if defined(arch_atomic_fetch_and_relaxed) 1513 return arch_atomic_fetch_and_relaxed(i, v); 1514#elif defined(arch_atomic_fetch_and) 1515 return arch_atomic_fetch_and(i, v); 1516#else 1517#error "Unable to define raw_atomic_fetch_and_relaxed" 1518#endif 1519} 1520 1521/** 1522 * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering 1523 * @i: int value 1524 * @v: pointer to atomic_t 1525 * 1526 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 1527 * 1528 * Safe to use in noinstr code; prefer atomic_andnot() elsewhere. 1529 * 1530 * Return: Nothing. 1531 */ 1532static __always_inline void 1533raw_atomic_andnot(int i, atomic_t *v) 1534{ 1535#if defined(arch_atomic_andnot) 1536 arch_atomic_andnot(i, v); 1537#else 1538 raw_atomic_and(~i, v); 1539#endif 1540} 1541 1542/** 1543 * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering 1544 * @i: int value 1545 * @v: pointer to atomic_t 1546 * 1547 * Atomically updates @v to (@v & ~@i) with full ordering. 1548 * 1549 * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere. 1550 * 1551 * Return: The original value of @v. 1552 */ 1553static __always_inline int 1554raw_atomic_fetch_andnot(int i, atomic_t *v) 1555{ 1556#if defined(arch_atomic_fetch_andnot) 1557 return arch_atomic_fetch_andnot(i, v); 1558#elif defined(arch_atomic_fetch_andnot_relaxed) 1559 int ret; 1560 __atomic_pre_full_fence(); 1561 ret = arch_atomic_fetch_andnot_relaxed(i, v); 1562 __atomic_post_full_fence(); 1563 return ret; 1564#else 1565 return raw_atomic_fetch_and(~i, v); 1566#endif 1567} 1568 1569/** 1570 * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering 1571 * @i: int value 1572 * @v: pointer to atomic_t 1573 * 1574 * Atomically updates @v to (@v & ~@i) with acquire ordering. 1575 * 1576 * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere. 1577 * 1578 * Return: The original value of @v. 1579 */ 1580static __always_inline int 1581raw_atomic_fetch_andnot_acquire(int i, atomic_t *v) 1582{ 1583#if defined(arch_atomic_fetch_andnot_acquire) 1584 return arch_atomic_fetch_andnot_acquire(i, v); 1585#elif defined(arch_atomic_fetch_andnot_relaxed) 1586 int ret = arch_atomic_fetch_andnot_relaxed(i, v); 1587 __atomic_acquire_fence(); 1588 return ret; 1589#elif defined(arch_atomic_fetch_andnot) 1590 return arch_atomic_fetch_andnot(i, v); 1591#else 1592 return raw_atomic_fetch_and_acquire(~i, v); 1593#endif 1594} 1595 1596/** 1597 * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering 1598 * @i: int value 1599 * @v: pointer to atomic_t 1600 * 1601 * Atomically updates @v to (@v & ~@i) with release ordering. 1602 * 1603 * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere. 1604 * 1605 * Return: The original value of @v. 1606 */ 1607static __always_inline int 1608raw_atomic_fetch_andnot_release(int i, atomic_t *v) 1609{ 1610#if defined(arch_atomic_fetch_andnot_release) 1611 return arch_atomic_fetch_andnot_release(i, v); 1612#elif defined(arch_atomic_fetch_andnot_relaxed) 1613 __atomic_release_fence(); 1614 return arch_atomic_fetch_andnot_relaxed(i, v); 1615#elif defined(arch_atomic_fetch_andnot) 1616 return arch_atomic_fetch_andnot(i, v); 1617#else 1618 return raw_atomic_fetch_and_release(~i, v); 1619#endif 1620} 1621 1622/** 1623 * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering 1624 * @i: int value 1625 * @v: pointer to atomic_t 1626 * 1627 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 1628 * 1629 * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere. 1630 * 1631 * Return: The original value of @v. 1632 */ 1633static __always_inline int 1634raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v) 1635{ 1636#if defined(arch_atomic_fetch_andnot_relaxed) 1637 return arch_atomic_fetch_andnot_relaxed(i, v); 1638#elif defined(arch_atomic_fetch_andnot) 1639 return arch_atomic_fetch_andnot(i, v); 1640#else 1641 return raw_atomic_fetch_and_relaxed(~i, v); 1642#endif 1643} 1644 1645/** 1646 * raw_atomic_or() - atomic bitwise OR with relaxed ordering 1647 * @i: int value 1648 * @v: pointer to atomic_t 1649 * 1650 * Atomically updates @v to (@v | @i) with relaxed ordering. 1651 * 1652 * Safe to use in noinstr code; prefer atomic_or() elsewhere. 1653 * 1654 * Return: Nothing. 1655 */ 1656static __always_inline void 1657raw_atomic_or(int i, atomic_t *v) 1658{ 1659 arch_atomic_or(i, v); 1660} 1661 1662/** 1663 * raw_atomic_fetch_or() - atomic bitwise OR with full ordering 1664 * @i: int value 1665 * @v: pointer to atomic_t 1666 * 1667 * Atomically updates @v to (@v | @i) with full ordering. 1668 * 1669 * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere. 1670 * 1671 * Return: The original value of @v. 1672 */ 1673static __always_inline int 1674raw_atomic_fetch_or(int i, atomic_t *v) 1675{ 1676#if defined(arch_atomic_fetch_or) 1677 return arch_atomic_fetch_or(i, v); 1678#elif defined(arch_atomic_fetch_or_relaxed) 1679 int ret; 1680 __atomic_pre_full_fence(); 1681 ret = arch_atomic_fetch_or_relaxed(i, v); 1682 __atomic_post_full_fence(); 1683 return ret; 1684#else 1685#error "Unable to define raw_atomic_fetch_or" 1686#endif 1687} 1688 1689/** 1690 * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering 1691 * @i: int value 1692 * @v: pointer to atomic_t 1693 * 1694 * Atomically updates @v to (@v | @i) with acquire ordering. 1695 * 1696 * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere. 1697 * 1698 * Return: The original value of @v. 1699 */ 1700static __always_inline int 1701raw_atomic_fetch_or_acquire(int i, atomic_t *v) 1702{ 1703#if defined(arch_atomic_fetch_or_acquire) 1704 return arch_atomic_fetch_or_acquire(i, v); 1705#elif defined(arch_atomic_fetch_or_relaxed) 1706 int ret = arch_atomic_fetch_or_relaxed(i, v); 1707 __atomic_acquire_fence(); 1708 return ret; 1709#elif defined(arch_atomic_fetch_or) 1710 return arch_atomic_fetch_or(i, v); 1711#else 1712#error "Unable to define raw_atomic_fetch_or_acquire" 1713#endif 1714} 1715 1716/** 1717 * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering 1718 * @i: int value 1719 * @v: pointer to atomic_t 1720 * 1721 * Atomically updates @v to (@v | @i) with release ordering. 1722 * 1723 * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere. 1724 * 1725 * Return: The original value of @v. 1726 */ 1727static __always_inline int 1728raw_atomic_fetch_or_release(int i, atomic_t *v) 1729{ 1730#if defined(arch_atomic_fetch_or_release) 1731 return arch_atomic_fetch_or_release(i, v); 1732#elif defined(arch_atomic_fetch_or_relaxed) 1733 __atomic_release_fence(); 1734 return arch_atomic_fetch_or_relaxed(i, v); 1735#elif defined(arch_atomic_fetch_or) 1736 return arch_atomic_fetch_or(i, v); 1737#else 1738#error "Unable to define raw_atomic_fetch_or_release" 1739#endif 1740} 1741 1742/** 1743 * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering 1744 * @i: int value 1745 * @v: pointer to atomic_t 1746 * 1747 * Atomically updates @v to (@v | @i) with relaxed ordering. 1748 * 1749 * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere. 1750 * 1751 * Return: The original value of @v. 1752 */ 1753static __always_inline int 1754raw_atomic_fetch_or_relaxed(int i, atomic_t *v) 1755{ 1756#if defined(arch_atomic_fetch_or_relaxed) 1757 return arch_atomic_fetch_or_relaxed(i, v); 1758#elif defined(arch_atomic_fetch_or) 1759 return arch_atomic_fetch_or(i, v); 1760#else 1761#error "Unable to define raw_atomic_fetch_or_relaxed" 1762#endif 1763} 1764 1765/** 1766 * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering 1767 * @i: int value 1768 * @v: pointer to atomic_t 1769 * 1770 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 1771 * 1772 * Safe to use in noinstr code; prefer atomic_xor() elsewhere. 1773 * 1774 * Return: Nothing. 1775 */ 1776static __always_inline void 1777raw_atomic_xor(int i, atomic_t *v) 1778{ 1779 arch_atomic_xor(i, v); 1780} 1781 1782/** 1783 * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering 1784 * @i: int value 1785 * @v: pointer to atomic_t 1786 * 1787 * Atomically updates @v to (@v ^ @i) with full ordering. 1788 * 1789 * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere. 1790 * 1791 * Return: The original value of @v. 1792 */ 1793static __always_inline int 1794raw_atomic_fetch_xor(int i, atomic_t *v) 1795{ 1796#if defined(arch_atomic_fetch_xor) 1797 return arch_atomic_fetch_xor(i, v); 1798#elif defined(arch_atomic_fetch_xor_relaxed) 1799 int ret; 1800 __atomic_pre_full_fence(); 1801 ret = arch_atomic_fetch_xor_relaxed(i, v); 1802 __atomic_post_full_fence(); 1803 return ret; 1804#else 1805#error "Unable to define raw_atomic_fetch_xor" 1806#endif 1807} 1808 1809/** 1810 * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering 1811 * @i: int value 1812 * @v: pointer to atomic_t 1813 * 1814 * Atomically updates @v to (@v ^ @i) with acquire ordering. 1815 * 1816 * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere. 1817 * 1818 * Return: The original value of @v. 1819 */ 1820static __always_inline int 1821raw_atomic_fetch_xor_acquire(int i, atomic_t *v) 1822{ 1823#if defined(arch_atomic_fetch_xor_acquire) 1824 return arch_atomic_fetch_xor_acquire(i, v); 1825#elif defined(arch_atomic_fetch_xor_relaxed) 1826 int ret = arch_atomic_fetch_xor_relaxed(i, v); 1827 __atomic_acquire_fence(); 1828 return ret; 1829#elif defined(arch_atomic_fetch_xor) 1830 return arch_atomic_fetch_xor(i, v); 1831#else 1832#error "Unable to define raw_atomic_fetch_xor_acquire" 1833#endif 1834} 1835 1836/** 1837 * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering 1838 * @i: int value 1839 * @v: pointer to atomic_t 1840 * 1841 * Atomically updates @v to (@v ^ @i) with release ordering. 1842 * 1843 * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere. 1844 * 1845 * Return: The original value of @v. 1846 */ 1847static __always_inline int 1848raw_atomic_fetch_xor_release(int i, atomic_t *v) 1849{ 1850#if defined(arch_atomic_fetch_xor_release) 1851 return arch_atomic_fetch_xor_release(i, v); 1852#elif defined(arch_atomic_fetch_xor_relaxed) 1853 __atomic_release_fence(); 1854 return arch_atomic_fetch_xor_relaxed(i, v); 1855#elif defined(arch_atomic_fetch_xor) 1856 return arch_atomic_fetch_xor(i, v); 1857#else 1858#error "Unable to define raw_atomic_fetch_xor_release" 1859#endif 1860} 1861 1862/** 1863 * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering 1864 * @i: int value 1865 * @v: pointer to atomic_t 1866 * 1867 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 1868 * 1869 * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere. 1870 * 1871 * Return: The original value of @v. 1872 */ 1873static __always_inline int 1874raw_atomic_fetch_xor_relaxed(int i, atomic_t *v) 1875{ 1876#if defined(arch_atomic_fetch_xor_relaxed) 1877 return arch_atomic_fetch_xor_relaxed(i, v); 1878#elif defined(arch_atomic_fetch_xor) 1879 return arch_atomic_fetch_xor(i, v); 1880#else 1881#error "Unable to define raw_atomic_fetch_xor_relaxed" 1882#endif 1883} 1884 1885/** 1886 * raw_atomic_xchg() - atomic exchange with full ordering 1887 * @v: pointer to atomic_t 1888 * @new: int value to assign 1889 * 1890 * Atomically updates @v to @new with full ordering. 1891 * 1892 * Safe to use in noinstr code; prefer atomic_xchg() elsewhere. 1893 * 1894 * Return: The original value of @v. 1895 */ 1896static __always_inline int 1897raw_atomic_xchg(atomic_t *v, int new) 1898{ 1899#if defined(arch_atomic_xchg) 1900 return arch_atomic_xchg(v, new); 1901#elif defined(arch_atomic_xchg_relaxed) 1902 int ret; 1903 __atomic_pre_full_fence(); 1904 ret = arch_atomic_xchg_relaxed(v, new); 1905 __atomic_post_full_fence(); 1906 return ret; 1907#else 1908 return raw_xchg(&v->counter, new); 1909#endif 1910} 1911 1912/** 1913 * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering 1914 * @v: pointer to atomic_t 1915 * @new: int value to assign 1916 * 1917 * Atomically updates @v to @new with acquire ordering. 1918 * 1919 * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere. 1920 * 1921 * Return: The original value of @v. 1922 */ 1923static __always_inline int 1924raw_atomic_xchg_acquire(atomic_t *v, int new) 1925{ 1926#if defined(arch_atomic_xchg_acquire) 1927 return arch_atomic_xchg_acquire(v, new); 1928#elif defined(arch_atomic_xchg_relaxed) 1929 int ret = arch_atomic_xchg_relaxed(v, new); 1930 __atomic_acquire_fence(); 1931 return ret; 1932#elif defined(arch_atomic_xchg) 1933 return arch_atomic_xchg(v, new); 1934#else 1935 return raw_xchg_acquire(&v->counter, new); 1936#endif 1937} 1938 1939/** 1940 * raw_atomic_xchg_release() - atomic exchange with release ordering 1941 * @v: pointer to atomic_t 1942 * @new: int value to assign 1943 * 1944 * Atomically updates @v to @new with release ordering. 1945 * 1946 * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere. 1947 * 1948 * Return: The original value of @v. 1949 */ 1950static __always_inline int 1951raw_atomic_xchg_release(atomic_t *v, int new) 1952{ 1953#if defined(arch_atomic_xchg_release) 1954 return arch_atomic_xchg_release(v, new); 1955#elif defined(arch_atomic_xchg_relaxed) 1956 __atomic_release_fence(); 1957 return arch_atomic_xchg_relaxed(v, new); 1958#elif defined(arch_atomic_xchg) 1959 return arch_atomic_xchg(v, new); 1960#else 1961 return raw_xchg_release(&v->counter, new); 1962#endif 1963} 1964 1965/** 1966 * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering 1967 * @v: pointer to atomic_t 1968 * @new: int value to assign 1969 * 1970 * Atomically updates @v to @new with relaxed ordering. 1971 * 1972 * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere. 1973 * 1974 * Return: The original value of @v. 1975 */ 1976static __always_inline int 1977raw_atomic_xchg_relaxed(atomic_t *v, int new) 1978{ 1979#if defined(arch_atomic_xchg_relaxed) 1980 return arch_atomic_xchg_relaxed(v, new); 1981#elif defined(arch_atomic_xchg) 1982 return arch_atomic_xchg(v, new); 1983#else 1984 return raw_xchg_relaxed(&v->counter, new); 1985#endif 1986} 1987 1988/** 1989 * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering 1990 * @v: pointer to atomic_t 1991 * @old: int value to compare with 1992 * @new: int value to assign 1993 * 1994 * If (@v == @old), atomically updates @v to @new with full ordering. 1995 * 1996 * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere. 1997 * 1998 * Return: The original value of @v. 1999 */ 2000static __always_inline int 2001raw_atomic_cmpxchg(atomic_t *v, int old, int new) 2002{ 2003#if defined(arch_atomic_cmpxchg) 2004 return arch_atomic_cmpxchg(v, old, new); 2005#elif defined(arch_atomic_cmpxchg_relaxed) 2006 int ret; 2007 __atomic_pre_full_fence(); 2008 ret = arch_atomic_cmpxchg_relaxed(v, old, new); 2009 __atomic_post_full_fence(); 2010 return ret; 2011#else 2012 return raw_cmpxchg(&v->counter, old, new); 2013#endif 2014} 2015 2016/** 2017 * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 2018 * @v: pointer to atomic_t 2019 * @old: int value to compare with 2020 * @new: int value to assign 2021 * 2022 * If (@v == @old), atomically updates @v to @new with acquire ordering. 2023 * 2024 * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere. 2025 * 2026 * Return: The original value of @v. 2027 */ 2028static __always_inline int 2029raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) 2030{ 2031#if defined(arch_atomic_cmpxchg_acquire) 2032 return arch_atomic_cmpxchg_acquire(v, old, new); 2033#elif defined(arch_atomic_cmpxchg_relaxed) 2034 int ret = arch_atomic_cmpxchg_relaxed(v, old, new); 2035 __atomic_acquire_fence(); 2036 return ret; 2037#elif defined(arch_atomic_cmpxchg) 2038 return arch_atomic_cmpxchg(v, old, new); 2039#else 2040 return raw_cmpxchg_acquire(&v->counter, old, new); 2041#endif 2042} 2043 2044/** 2045 * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering 2046 * @v: pointer to atomic_t 2047 * @old: int value to compare with 2048 * @new: int value to assign 2049 * 2050 * If (@v == @old), atomically updates @v to @new with release ordering. 2051 * 2052 * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere. 2053 * 2054 * Return: The original value of @v. 2055 */ 2056static __always_inline int 2057raw_atomic_cmpxchg_release(atomic_t *v, int old, int new) 2058{ 2059#if defined(arch_atomic_cmpxchg_release) 2060 return arch_atomic_cmpxchg_release(v, old, new); 2061#elif defined(arch_atomic_cmpxchg_relaxed) 2062 __atomic_release_fence(); 2063 return arch_atomic_cmpxchg_relaxed(v, old, new); 2064#elif defined(arch_atomic_cmpxchg) 2065 return arch_atomic_cmpxchg(v, old, new); 2066#else 2067 return raw_cmpxchg_release(&v->counter, old, new); 2068#endif 2069} 2070 2071/** 2072 * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 2073 * @v: pointer to atomic_t 2074 * @old: int value to compare with 2075 * @new: int value to assign 2076 * 2077 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 2078 * 2079 * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere. 2080 * 2081 * Return: The original value of @v. 2082 */ 2083static __always_inline int 2084raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) 2085{ 2086#if defined(arch_atomic_cmpxchg_relaxed) 2087 return arch_atomic_cmpxchg_relaxed(v, old, new); 2088#elif defined(arch_atomic_cmpxchg) 2089 return arch_atomic_cmpxchg(v, old, new); 2090#else 2091 return raw_cmpxchg_relaxed(&v->counter, old, new); 2092#endif 2093} 2094 2095/** 2096 * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering 2097 * @v: pointer to atomic_t 2098 * @old: pointer to int value to compare with 2099 * @new: int value to assign 2100 * 2101 * If (@v == @old), atomically updates @v to @new with full ordering. 2102 * Otherwise, updates @old to the current value of @v. 2103 * 2104 * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere. 2105 * 2106 * Return: @true if the exchange occured, @false otherwise. 2107 */ 2108static __always_inline bool 2109raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new) 2110{ 2111#if defined(arch_atomic_try_cmpxchg) 2112 return arch_atomic_try_cmpxchg(v, old, new); 2113#elif defined(arch_atomic_try_cmpxchg_relaxed) 2114 bool ret; 2115 __atomic_pre_full_fence(); 2116 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); 2117 __atomic_post_full_fence(); 2118 return ret; 2119#else 2120 int r, o = *old; 2121 r = raw_atomic_cmpxchg(v, o, new); 2122 if (unlikely(r != o)) 2123 *old = r; 2124 return likely(r == o); 2125#endif 2126} 2127 2128/** 2129 * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 2130 * @v: pointer to atomic_t 2131 * @old: pointer to int value to compare with 2132 * @new: int value to assign 2133 * 2134 * If (@v == @old), atomically updates @v to @new with acquire ordering. 2135 * Otherwise, updates @old to the current value of @v. 2136 * 2137 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere. 2138 * 2139 * Return: @true if the exchange occured, @false otherwise. 2140 */ 2141static __always_inline bool 2142raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) 2143{ 2144#if defined(arch_atomic_try_cmpxchg_acquire) 2145 return arch_atomic_try_cmpxchg_acquire(v, old, new); 2146#elif defined(arch_atomic_try_cmpxchg_relaxed) 2147 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); 2148 __atomic_acquire_fence(); 2149 return ret; 2150#elif defined(arch_atomic_try_cmpxchg) 2151 return arch_atomic_try_cmpxchg(v, old, new); 2152#else 2153 int r, o = *old; 2154 r = raw_atomic_cmpxchg_acquire(v, o, new); 2155 if (unlikely(r != o)) 2156 *old = r; 2157 return likely(r == o); 2158#endif 2159} 2160 2161/** 2162 * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering 2163 * @v: pointer to atomic_t 2164 * @old: pointer to int value to compare with 2165 * @new: int value to assign 2166 * 2167 * If (@v == @old), atomically updates @v to @new with release ordering. 2168 * Otherwise, updates @old to the current value of @v. 2169 * 2170 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere. 2171 * 2172 * Return: @true if the exchange occured, @false otherwise. 2173 */ 2174static __always_inline bool 2175raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) 2176{ 2177#if defined(arch_atomic_try_cmpxchg_release) 2178 return arch_atomic_try_cmpxchg_release(v, old, new); 2179#elif defined(arch_atomic_try_cmpxchg_relaxed) 2180 __atomic_release_fence(); 2181 return arch_atomic_try_cmpxchg_relaxed(v, old, new); 2182#elif defined(arch_atomic_try_cmpxchg) 2183 return arch_atomic_try_cmpxchg(v, old, new); 2184#else 2185 int r, o = *old; 2186 r = raw_atomic_cmpxchg_release(v, o, new); 2187 if (unlikely(r != o)) 2188 *old = r; 2189 return likely(r == o); 2190#endif 2191} 2192 2193/** 2194 * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 2195 * @v: pointer to atomic_t 2196 * @old: pointer to int value to compare with 2197 * @new: int value to assign 2198 * 2199 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 2200 * Otherwise, updates @old to the current value of @v. 2201 * 2202 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere. 2203 * 2204 * Return: @true if the exchange occured, @false otherwise. 2205 */ 2206static __always_inline bool 2207raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) 2208{ 2209#if defined(arch_atomic_try_cmpxchg_relaxed) 2210 return arch_atomic_try_cmpxchg_relaxed(v, old, new); 2211#elif defined(arch_atomic_try_cmpxchg) 2212 return arch_atomic_try_cmpxchg(v, old, new); 2213#else 2214 int r, o = *old; 2215 r = raw_atomic_cmpxchg_relaxed(v, o, new); 2216 if (unlikely(r != o)) 2217 *old = r; 2218 return likely(r == o); 2219#endif 2220} 2221 2222/** 2223 * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering 2224 * @i: int value to add 2225 * @v: pointer to atomic_t 2226 * 2227 * Atomically updates @v to (@v - @i) with full ordering. 2228 * 2229 * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere. 2230 * 2231 * Return: @true if the resulting value of @v is zero, @false otherwise. 2232 */ 2233static __always_inline bool 2234raw_atomic_sub_and_test(int i, atomic_t *v) 2235{ 2236#if defined(arch_atomic_sub_and_test) 2237 return arch_atomic_sub_and_test(i, v); 2238#else 2239 return raw_atomic_sub_return(i, v) == 0; 2240#endif 2241} 2242 2243/** 2244 * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering 2245 * @v: pointer to atomic_t 2246 * 2247 * Atomically updates @v to (@v - 1) with full ordering. 2248 * 2249 * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere. 2250 * 2251 * Return: @true if the resulting value of @v is zero, @false otherwise. 2252 */ 2253static __always_inline bool 2254raw_atomic_dec_and_test(atomic_t *v) 2255{ 2256#if defined(arch_atomic_dec_and_test) 2257 return arch_atomic_dec_and_test(v); 2258#else 2259 return raw_atomic_dec_return(v) == 0; 2260#endif 2261} 2262 2263/** 2264 * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering 2265 * @v: pointer to atomic_t 2266 * 2267 * Atomically updates @v to (@v + 1) with full ordering. 2268 * 2269 * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere. 2270 * 2271 * Return: @true if the resulting value of @v is zero, @false otherwise. 2272 */ 2273static __always_inline bool 2274raw_atomic_inc_and_test(atomic_t *v) 2275{ 2276#if defined(arch_atomic_inc_and_test) 2277 return arch_atomic_inc_and_test(v); 2278#else 2279 return raw_atomic_inc_return(v) == 0; 2280#endif 2281} 2282 2283/** 2284 * raw_atomic_add_negative() - atomic add and test if negative with full ordering 2285 * @i: int value to add 2286 * @v: pointer to atomic_t 2287 * 2288 * Atomically updates @v to (@v + @i) with full ordering. 2289 * 2290 * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere. 2291 * 2292 * Return: @true if the resulting value of @v is negative, @false otherwise. 2293 */ 2294static __always_inline bool 2295raw_atomic_add_negative(int i, atomic_t *v) 2296{ 2297#if defined(arch_atomic_add_negative) 2298 return arch_atomic_add_negative(i, v); 2299#elif defined(arch_atomic_add_negative_relaxed) 2300 bool ret; 2301 __atomic_pre_full_fence(); 2302 ret = arch_atomic_add_negative_relaxed(i, v); 2303 __atomic_post_full_fence(); 2304 return ret; 2305#else 2306 return raw_atomic_add_return(i, v) < 0; 2307#endif 2308} 2309 2310/** 2311 * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering 2312 * @i: int value to add 2313 * @v: pointer to atomic_t 2314 * 2315 * Atomically updates @v to (@v + @i) with acquire ordering. 2316 * 2317 * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere. 2318 * 2319 * Return: @true if the resulting value of @v is negative, @false otherwise. 2320 */ 2321static __always_inline bool 2322raw_atomic_add_negative_acquire(int i, atomic_t *v) 2323{ 2324#if defined(arch_atomic_add_negative_acquire) 2325 return arch_atomic_add_negative_acquire(i, v); 2326#elif defined(arch_atomic_add_negative_relaxed) 2327 bool ret = arch_atomic_add_negative_relaxed(i, v); 2328 __atomic_acquire_fence(); 2329 return ret; 2330#elif defined(arch_atomic_add_negative) 2331 return arch_atomic_add_negative(i, v); 2332#else 2333 return raw_atomic_add_return_acquire(i, v) < 0; 2334#endif 2335} 2336 2337/** 2338 * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering 2339 * @i: int value to add 2340 * @v: pointer to atomic_t 2341 * 2342 * Atomically updates @v to (@v + @i) with release ordering. 2343 * 2344 * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere. 2345 * 2346 * Return: @true if the resulting value of @v is negative, @false otherwise. 2347 */ 2348static __always_inline bool 2349raw_atomic_add_negative_release(int i, atomic_t *v) 2350{ 2351#if defined(arch_atomic_add_negative_release) 2352 return arch_atomic_add_negative_release(i, v); 2353#elif defined(arch_atomic_add_negative_relaxed) 2354 __atomic_release_fence(); 2355 return arch_atomic_add_negative_relaxed(i, v); 2356#elif defined(arch_atomic_add_negative) 2357 return arch_atomic_add_negative(i, v); 2358#else 2359 return raw_atomic_add_return_release(i, v) < 0; 2360#endif 2361} 2362 2363/** 2364 * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering 2365 * @i: int value to add 2366 * @v: pointer to atomic_t 2367 * 2368 * Atomically updates @v to (@v + @i) with relaxed ordering. 2369 * 2370 * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere. 2371 * 2372 * Return: @true if the resulting value of @v is negative, @false otherwise. 2373 */ 2374static __always_inline bool 2375raw_atomic_add_negative_relaxed(int i, atomic_t *v) 2376{ 2377#if defined(arch_atomic_add_negative_relaxed) 2378 return arch_atomic_add_negative_relaxed(i, v); 2379#elif defined(arch_atomic_add_negative) 2380 return arch_atomic_add_negative(i, v); 2381#else 2382 return raw_atomic_add_return_relaxed(i, v) < 0; 2383#endif 2384} 2385 2386/** 2387 * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering 2388 * @v: pointer to atomic_t 2389 * @a: int value to add 2390 * @u: int value to compare with 2391 * 2392 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 2393 * 2394 * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere. 2395 * 2396 * Return: The original value of @v. 2397 */ 2398static __always_inline int 2399raw_atomic_fetch_add_unless(atomic_t *v, int a, int u) 2400{ 2401#if defined(arch_atomic_fetch_add_unless) 2402 return arch_atomic_fetch_add_unless(v, a, u); 2403#else 2404 int c = raw_atomic_read(v); 2405 2406 do { 2407 if (unlikely(c == u)) 2408 break; 2409 } while (!raw_atomic_try_cmpxchg(v, &c, c + a)); 2410 2411 return c; 2412#endif 2413} 2414 2415/** 2416 * raw_atomic_add_unless() - atomic add unless value with full ordering 2417 * @v: pointer to atomic_t 2418 * @a: int value to add 2419 * @u: int value to compare with 2420 * 2421 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 2422 * 2423 * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere. 2424 * 2425 * Return: @true if @v was updated, @false otherwise. 2426 */ 2427static __always_inline bool 2428raw_atomic_add_unless(atomic_t *v, int a, int u) 2429{ 2430#if defined(arch_atomic_add_unless) 2431 return arch_atomic_add_unless(v, a, u); 2432#else 2433 return raw_atomic_fetch_add_unless(v, a, u) != u; 2434#endif 2435} 2436 2437/** 2438 * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering 2439 * @v: pointer to atomic_t 2440 * 2441 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. 2442 * 2443 * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere. 2444 * 2445 * Return: @true if @v was updated, @false otherwise. 2446 */ 2447static __always_inline bool 2448raw_atomic_inc_not_zero(atomic_t *v) 2449{ 2450#if defined(arch_atomic_inc_not_zero) 2451 return arch_atomic_inc_not_zero(v); 2452#else 2453 return raw_atomic_add_unless(v, 1, 0); 2454#endif 2455} 2456 2457/** 2458 * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering 2459 * @v: pointer to atomic_t 2460 * 2461 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. 2462 * 2463 * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere. 2464 * 2465 * Return: @true if @v was updated, @false otherwise. 2466 */ 2467static __always_inline bool 2468raw_atomic_inc_unless_negative(atomic_t *v) 2469{ 2470#if defined(arch_atomic_inc_unless_negative) 2471 return arch_atomic_inc_unless_negative(v); 2472#else 2473 int c = raw_atomic_read(v); 2474 2475 do { 2476 if (unlikely(c < 0)) 2477 return false; 2478 } while (!raw_atomic_try_cmpxchg(v, &c, c + 1)); 2479 2480 return true; 2481#endif 2482} 2483 2484/** 2485 * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering 2486 * @v: pointer to atomic_t 2487 * 2488 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. 2489 * 2490 * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere. 2491 * 2492 * Return: @true if @v was updated, @false otherwise. 2493 */ 2494static __always_inline bool 2495raw_atomic_dec_unless_positive(atomic_t *v) 2496{ 2497#if defined(arch_atomic_dec_unless_positive) 2498 return arch_atomic_dec_unless_positive(v); 2499#else 2500 int c = raw_atomic_read(v); 2501 2502 do { 2503 if (unlikely(c > 0)) 2504 return false; 2505 } while (!raw_atomic_try_cmpxchg(v, &c, c - 1)); 2506 2507 return true; 2508#endif 2509} 2510 2511/** 2512 * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering 2513 * @v: pointer to atomic_t 2514 * 2515 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. 2516 * 2517 * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere. 2518 * 2519 * Return: The old value of (@v - 1), regardless of whether @v was updated. 2520 */ 2521static __always_inline int 2522raw_atomic_dec_if_positive(atomic_t *v) 2523{ 2524#if defined(arch_atomic_dec_if_positive) 2525 return arch_atomic_dec_if_positive(v); 2526#else 2527 int dec, c = raw_atomic_read(v); 2528 2529 do { 2530 dec = c - 1; 2531 if (unlikely(dec < 0)) 2532 break; 2533 } while (!raw_atomic_try_cmpxchg(v, &c, dec)); 2534 2535 return dec; 2536#endif 2537} 2538 2539#ifdef CONFIG_GENERIC_ATOMIC64 2540#include <asm-generic/atomic64.h> 2541#endif 2542 2543/** 2544 * raw_atomic64_read() - atomic load with relaxed ordering 2545 * @v: pointer to atomic64_t 2546 * 2547 * Atomically loads the value of @v with relaxed ordering. 2548 * 2549 * Safe to use in noinstr code; prefer atomic64_read() elsewhere. 2550 * 2551 * Return: The value loaded from @v. 2552 */ 2553static __always_inline s64 2554raw_atomic64_read(const atomic64_t *v) 2555{ 2556 return arch_atomic64_read(v); 2557} 2558 2559/** 2560 * raw_atomic64_read_acquire() - atomic load with acquire ordering 2561 * @v: pointer to atomic64_t 2562 * 2563 * Atomically loads the value of @v with acquire ordering. 2564 * 2565 * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere. 2566 * 2567 * Return: The value loaded from @v. 2568 */ 2569static __always_inline s64 2570raw_atomic64_read_acquire(const atomic64_t *v) 2571{ 2572#if defined(arch_atomic64_read_acquire) 2573 return arch_atomic64_read_acquire(v); 2574#else 2575 s64 ret; 2576 2577 if (__native_word(atomic64_t)) { 2578 ret = smp_load_acquire(&(v)->counter); 2579 } else { 2580 ret = raw_atomic64_read(v); 2581 __atomic_acquire_fence(); 2582 } 2583 2584 return ret; 2585#endif 2586} 2587 2588/** 2589 * raw_atomic64_set() - atomic set with relaxed ordering 2590 * @v: pointer to atomic64_t 2591 * @i: s64 value to assign 2592 * 2593 * Atomically sets @v to @i with relaxed ordering. 2594 * 2595 * Safe to use in noinstr code; prefer atomic64_set() elsewhere. 2596 * 2597 * Return: Nothing. 2598 */ 2599static __always_inline void 2600raw_atomic64_set(atomic64_t *v, s64 i) 2601{ 2602 arch_atomic64_set(v, i); 2603} 2604 2605/** 2606 * raw_atomic64_set_release() - atomic set with release ordering 2607 * @v: pointer to atomic64_t 2608 * @i: s64 value to assign 2609 * 2610 * Atomically sets @v to @i with release ordering. 2611 * 2612 * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere. 2613 * 2614 * Return: Nothing. 2615 */ 2616static __always_inline void 2617raw_atomic64_set_release(atomic64_t *v, s64 i) 2618{ 2619#if defined(arch_atomic64_set_release) 2620 arch_atomic64_set_release(v, i); 2621#else 2622 if (__native_word(atomic64_t)) { 2623 smp_store_release(&(v)->counter, i); 2624 } else { 2625 __atomic_release_fence(); 2626 raw_atomic64_set(v, i); 2627 } 2628#endif 2629} 2630 2631/** 2632 * raw_atomic64_add() - atomic add with relaxed ordering 2633 * @i: s64 value to add 2634 * @v: pointer to atomic64_t 2635 * 2636 * Atomically updates @v to (@v + @i) with relaxed ordering. 2637 * 2638 * Safe to use in noinstr code; prefer atomic64_add() elsewhere. 2639 * 2640 * Return: Nothing. 2641 */ 2642static __always_inline void 2643raw_atomic64_add(s64 i, atomic64_t *v) 2644{ 2645 arch_atomic64_add(i, v); 2646} 2647 2648/** 2649 * raw_atomic64_add_return() - atomic add with full ordering 2650 * @i: s64 value to add 2651 * @v: pointer to atomic64_t 2652 * 2653 * Atomically updates @v to (@v + @i) with full ordering. 2654 * 2655 * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere. 2656 * 2657 * Return: The updated value of @v. 2658 */ 2659static __always_inline s64 2660raw_atomic64_add_return(s64 i, atomic64_t *v) 2661{ 2662#if defined(arch_atomic64_add_return) 2663 return arch_atomic64_add_return(i, v); 2664#elif defined(arch_atomic64_add_return_relaxed) 2665 s64 ret; 2666 __atomic_pre_full_fence(); 2667 ret = arch_atomic64_add_return_relaxed(i, v); 2668 __atomic_post_full_fence(); 2669 return ret; 2670#else 2671#error "Unable to define raw_atomic64_add_return" 2672#endif 2673} 2674 2675/** 2676 * raw_atomic64_add_return_acquire() - atomic add with acquire ordering 2677 * @i: s64 value to add 2678 * @v: pointer to atomic64_t 2679 * 2680 * Atomically updates @v to (@v + @i) with acquire ordering. 2681 * 2682 * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere. 2683 * 2684 * Return: The updated value of @v. 2685 */ 2686static __always_inline s64 2687raw_atomic64_add_return_acquire(s64 i, atomic64_t *v) 2688{ 2689#if defined(arch_atomic64_add_return_acquire) 2690 return arch_atomic64_add_return_acquire(i, v); 2691#elif defined(arch_atomic64_add_return_relaxed) 2692 s64 ret = arch_atomic64_add_return_relaxed(i, v); 2693 __atomic_acquire_fence(); 2694 return ret; 2695#elif defined(arch_atomic64_add_return) 2696 return arch_atomic64_add_return(i, v); 2697#else 2698#error "Unable to define raw_atomic64_add_return_acquire" 2699#endif 2700} 2701 2702/** 2703 * raw_atomic64_add_return_release() - atomic add with release ordering 2704 * @i: s64 value to add 2705 * @v: pointer to atomic64_t 2706 * 2707 * Atomically updates @v to (@v + @i) with release ordering. 2708 * 2709 * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere. 2710 * 2711 * Return: The updated value of @v. 2712 */ 2713static __always_inline s64 2714raw_atomic64_add_return_release(s64 i, atomic64_t *v) 2715{ 2716#if defined(arch_atomic64_add_return_release) 2717 return arch_atomic64_add_return_release(i, v); 2718#elif defined(arch_atomic64_add_return_relaxed) 2719 __atomic_release_fence(); 2720 return arch_atomic64_add_return_relaxed(i, v); 2721#elif defined(arch_atomic64_add_return) 2722 return arch_atomic64_add_return(i, v); 2723#else 2724#error "Unable to define raw_atomic64_add_return_release" 2725#endif 2726} 2727 2728/** 2729 * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering 2730 * @i: s64 value to add 2731 * @v: pointer to atomic64_t 2732 * 2733 * Atomically updates @v to (@v + @i) with relaxed ordering. 2734 * 2735 * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere. 2736 * 2737 * Return: The updated value of @v. 2738 */ 2739static __always_inline s64 2740raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v) 2741{ 2742#if defined(arch_atomic64_add_return_relaxed) 2743 return arch_atomic64_add_return_relaxed(i, v); 2744#elif defined(arch_atomic64_add_return) 2745 return arch_atomic64_add_return(i, v); 2746#else 2747#error "Unable to define raw_atomic64_add_return_relaxed" 2748#endif 2749} 2750 2751/** 2752 * raw_atomic64_fetch_add() - atomic add with full ordering 2753 * @i: s64 value to add 2754 * @v: pointer to atomic64_t 2755 * 2756 * Atomically updates @v to (@v + @i) with full ordering. 2757 * 2758 * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere. 2759 * 2760 * Return: The original value of @v. 2761 */ 2762static __always_inline s64 2763raw_atomic64_fetch_add(s64 i, atomic64_t *v) 2764{ 2765#if defined(arch_atomic64_fetch_add) 2766 return arch_atomic64_fetch_add(i, v); 2767#elif defined(arch_atomic64_fetch_add_relaxed) 2768 s64 ret; 2769 __atomic_pre_full_fence(); 2770 ret = arch_atomic64_fetch_add_relaxed(i, v); 2771 __atomic_post_full_fence(); 2772 return ret; 2773#else 2774#error "Unable to define raw_atomic64_fetch_add" 2775#endif 2776} 2777 2778/** 2779 * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering 2780 * @i: s64 value to add 2781 * @v: pointer to atomic64_t 2782 * 2783 * Atomically updates @v to (@v + @i) with acquire ordering. 2784 * 2785 * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere. 2786 * 2787 * Return: The original value of @v. 2788 */ 2789static __always_inline s64 2790raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) 2791{ 2792#if defined(arch_atomic64_fetch_add_acquire) 2793 return arch_atomic64_fetch_add_acquire(i, v); 2794#elif defined(arch_atomic64_fetch_add_relaxed) 2795 s64 ret = arch_atomic64_fetch_add_relaxed(i, v); 2796 __atomic_acquire_fence(); 2797 return ret; 2798#elif defined(arch_atomic64_fetch_add) 2799 return arch_atomic64_fetch_add(i, v); 2800#else 2801#error "Unable to define raw_atomic64_fetch_add_acquire" 2802#endif 2803} 2804 2805/** 2806 * raw_atomic64_fetch_add_release() - atomic add with release ordering 2807 * @i: s64 value to add 2808 * @v: pointer to atomic64_t 2809 * 2810 * Atomically updates @v to (@v + @i) with release ordering. 2811 * 2812 * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere. 2813 * 2814 * Return: The original value of @v. 2815 */ 2816static __always_inline s64 2817raw_atomic64_fetch_add_release(s64 i, atomic64_t *v) 2818{ 2819#if defined(arch_atomic64_fetch_add_release) 2820 return arch_atomic64_fetch_add_release(i, v); 2821#elif defined(arch_atomic64_fetch_add_relaxed) 2822 __atomic_release_fence(); 2823 return arch_atomic64_fetch_add_relaxed(i, v); 2824#elif defined(arch_atomic64_fetch_add) 2825 return arch_atomic64_fetch_add(i, v); 2826#else 2827#error "Unable to define raw_atomic64_fetch_add_release" 2828#endif 2829} 2830 2831/** 2832 * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering 2833 * @i: s64 value to add 2834 * @v: pointer to atomic64_t 2835 * 2836 * Atomically updates @v to (@v + @i) with relaxed ordering. 2837 * 2838 * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere. 2839 * 2840 * Return: The original value of @v. 2841 */ 2842static __always_inline s64 2843raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) 2844{ 2845#if defined(arch_atomic64_fetch_add_relaxed) 2846 return arch_atomic64_fetch_add_relaxed(i, v); 2847#elif defined(arch_atomic64_fetch_add) 2848 return arch_atomic64_fetch_add(i, v); 2849#else 2850#error "Unable to define raw_atomic64_fetch_add_relaxed" 2851#endif 2852} 2853 2854/** 2855 * raw_atomic64_sub() - atomic subtract with relaxed ordering 2856 * @i: s64 value to subtract 2857 * @v: pointer to atomic64_t 2858 * 2859 * Atomically updates @v to (@v - @i) with relaxed ordering. 2860 * 2861 * Safe to use in noinstr code; prefer atomic64_sub() elsewhere. 2862 * 2863 * Return: Nothing. 2864 */ 2865static __always_inline void 2866raw_atomic64_sub(s64 i, atomic64_t *v) 2867{ 2868 arch_atomic64_sub(i, v); 2869} 2870 2871/** 2872 * raw_atomic64_sub_return() - atomic subtract with full ordering 2873 * @i: s64 value to subtract 2874 * @v: pointer to atomic64_t 2875 * 2876 * Atomically updates @v to (@v - @i) with full ordering. 2877 * 2878 * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere. 2879 * 2880 * Return: The updated value of @v. 2881 */ 2882static __always_inline s64 2883raw_atomic64_sub_return(s64 i, atomic64_t *v) 2884{ 2885#if defined(arch_atomic64_sub_return) 2886 return arch_atomic64_sub_return(i, v); 2887#elif defined(arch_atomic64_sub_return_relaxed) 2888 s64 ret; 2889 __atomic_pre_full_fence(); 2890 ret = arch_atomic64_sub_return_relaxed(i, v); 2891 __atomic_post_full_fence(); 2892 return ret; 2893#else 2894#error "Unable to define raw_atomic64_sub_return" 2895#endif 2896} 2897 2898/** 2899 * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering 2900 * @i: s64 value to subtract 2901 * @v: pointer to atomic64_t 2902 * 2903 * Atomically updates @v to (@v - @i) with acquire ordering. 2904 * 2905 * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere. 2906 * 2907 * Return: The updated value of @v. 2908 */ 2909static __always_inline s64 2910raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v) 2911{ 2912#if defined(arch_atomic64_sub_return_acquire) 2913 return arch_atomic64_sub_return_acquire(i, v); 2914#elif defined(arch_atomic64_sub_return_relaxed) 2915 s64 ret = arch_atomic64_sub_return_relaxed(i, v); 2916 __atomic_acquire_fence(); 2917 return ret; 2918#elif defined(arch_atomic64_sub_return) 2919 return arch_atomic64_sub_return(i, v); 2920#else 2921#error "Unable to define raw_atomic64_sub_return_acquire" 2922#endif 2923} 2924 2925/** 2926 * raw_atomic64_sub_return_release() - atomic subtract with release ordering 2927 * @i: s64 value to subtract 2928 * @v: pointer to atomic64_t 2929 * 2930 * Atomically updates @v to (@v - @i) with release ordering. 2931 * 2932 * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere. 2933 * 2934 * Return: The updated value of @v. 2935 */ 2936static __always_inline s64 2937raw_atomic64_sub_return_release(s64 i, atomic64_t *v) 2938{ 2939#if defined(arch_atomic64_sub_return_release) 2940 return arch_atomic64_sub_return_release(i, v); 2941#elif defined(arch_atomic64_sub_return_relaxed) 2942 __atomic_release_fence(); 2943 return arch_atomic64_sub_return_relaxed(i, v); 2944#elif defined(arch_atomic64_sub_return) 2945 return arch_atomic64_sub_return(i, v); 2946#else 2947#error "Unable to define raw_atomic64_sub_return_release" 2948#endif 2949} 2950 2951/** 2952 * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering 2953 * @i: s64 value to subtract 2954 * @v: pointer to atomic64_t 2955 * 2956 * Atomically updates @v to (@v - @i) with relaxed ordering. 2957 * 2958 * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere. 2959 * 2960 * Return: The updated value of @v. 2961 */ 2962static __always_inline s64 2963raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v) 2964{ 2965#if defined(arch_atomic64_sub_return_relaxed) 2966 return arch_atomic64_sub_return_relaxed(i, v); 2967#elif defined(arch_atomic64_sub_return) 2968 return arch_atomic64_sub_return(i, v); 2969#else 2970#error "Unable to define raw_atomic64_sub_return_relaxed" 2971#endif 2972} 2973 2974/** 2975 * raw_atomic64_fetch_sub() - atomic subtract with full ordering 2976 * @i: s64 value to subtract 2977 * @v: pointer to atomic64_t 2978 * 2979 * Atomically updates @v to (@v - @i) with full ordering. 2980 * 2981 * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere. 2982 * 2983 * Return: The original value of @v. 2984 */ 2985static __always_inline s64 2986raw_atomic64_fetch_sub(s64 i, atomic64_t *v) 2987{ 2988#if defined(arch_atomic64_fetch_sub) 2989 return arch_atomic64_fetch_sub(i, v); 2990#elif defined(arch_atomic64_fetch_sub_relaxed) 2991 s64 ret; 2992 __atomic_pre_full_fence(); 2993 ret = arch_atomic64_fetch_sub_relaxed(i, v); 2994 __atomic_post_full_fence(); 2995 return ret; 2996#else 2997#error "Unable to define raw_atomic64_fetch_sub" 2998#endif 2999} 3000 3001/** 3002 * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering 3003 * @i: s64 value to subtract 3004 * @v: pointer to atomic64_t 3005 * 3006 * Atomically updates @v to (@v - @i) with acquire ordering. 3007 * 3008 * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere. 3009 * 3010 * Return: The original value of @v. 3011 */ 3012static __always_inline s64 3013raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) 3014{ 3015#if defined(arch_atomic64_fetch_sub_acquire) 3016 return arch_atomic64_fetch_sub_acquire(i, v); 3017#elif defined(arch_atomic64_fetch_sub_relaxed) 3018 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v); 3019 __atomic_acquire_fence(); 3020 return ret; 3021#elif defined(arch_atomic64_fetch_sub) 3022 return arch_atomic64_fetch_sub(i, v); 3023#else 3024#error "Unable to define raw_atomic64_fetch_sub_acquire" 3025#endif 3026} 3027 3028/** 3029 * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering 3030 * @i: s64 value to subtract 3031 * @v: pointer to atomic64_t 3032 * 3033 * Atomically updates @v to (@v - @i) with release ordering. 3034 * 3035 * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere. 3036 * 3037 * Return: The original value of @v. 3038 */ 3039static __always_inline s64 3040raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v) 3041{ 3042#if defined(arch_atomic64_fetch_sub_release) 3043 return arch_atomic64_fetch_sub_release(i, v); 3044#elif defined(arch_atomic64_fetch_sub_relaxed) 3045 __atomic_release_fence(); 3046 return arch_atomic64_fetch_sub_relaxed(i, v); 3047#elif defined(arch_atomic64_fetch_sub) 3048 return arch_atomic64_fetch_sub(i, v); 3049#else 3050#error "Unable to define raw_atomic64_fetch_sub_release" 3051#endif 3052} 3053 3054/** 3055 * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering 3056 * @i: s64 value to subtract 3057 * @v: pointer to atomic64_t 3058 * 3059 * Atomically updates @v to (@v - @i) with relaxed ordering. 3060 * 3061 * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere. 3062 * 3063 * Return: The original value of @v. 3064 */ 3065static __always_inline s64 3066raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) 3067{ 3068#if defined(arch_atomic64_fetch_sub_relaxed) 3069 return arch_atomic64_fetch_sub_relaxed(i, v); 3070#elif defined(arch_atomic64_fetch_sub) 3071 return arch_atomic64_fetch_sub(i, v); 3072#else 3073#error "Unable to define raw_atomic64_fetch_sub_relaxed" 3074#endif 3075} 3076 3077/** 3078 * raw_atomic64_inc() - atomic increment with relaxed ordering 3079 * @v: pointer to atomic64_t 3080 * 3081 * Atomically updates @v to (@v + 1) with relaxed ordering. 3082 * 3083 * Safe to use in noinstr code; prefer atomic64_inc() elsewhere. 3084 * 3085 * Return: Nothing. 3086 */ 3087static __always_inline void 3088raw_atomic64_inc(atomic64_t *v) 3089{ 3090#if defined(arch_atomic64_inc) 3091 arch_atomic64_inc(v); 3092#else 3093 raw_atomic64_add(1, v); 3094#endif 3095} 3096 3097/** 3098 * raw_atomic64_inc_return() - atomic increment with full ordering 3099 * @v: pointer to atomic64_t 3100 * 3101 * Atomically updates @v to (@v + 1) with full ordering. 3102 * 3103 * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere. 3104 * 3105 * Return: The updated value of @v. 3106 */ 3107static __always_inline s64 3108raw_atomic64_inc_return(atomic64_t *v) 3109{ 3110#if defined(arch_atomic64_inc_return) 3111 return arch_atomic64_inc_return(v); 3112#elif defined(arch_atomic64_inc_return_relaxed) 3113 s64 ret; 3114 __atomic_pre_full_fence(); 3115 ret = arch_atomic64_inc_return_relaxed(v); 3116 __atomic_post_full_fence(); 3117 return ret; 3118#else 3119 return raw_atomic64_add_return(1, v); 3120#endif 3121} 3122 3123/** 3124 * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering 3125 * @v: pointer to atomic64_t 3126 * 3127 * Atomically updates @v to (@v + 1) with acquire ordering. 3128 * 3129 * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere. 3130 * 3131 * Return: The updated value of @v. 3132 */ 3133static __always_inline s64 3134raw_atomic64_inc_return_acquire(atomic64_t *v) 3135{ 3136#if defined(arch_atomic64_inc_return_acquire) 3137 return arch_atomic64_inc_return_acquire(v); 3138#elif defined(arch_atomic64_inc_return_relaxed) 3139 s64 ret = arch_atomic64_inc_return_relaxed(v); 3140 __atomic_acquire_fence(); 3141 return ret; 3142#elif defined(arch_atomic64_inc_return) 3143 return arch_atomic64_inc_return(v); 3144#else 3145 return raw_atomic64_add_return_acquire(1, v); 3146#endif 3147} 3148 3149/** 3150 * raw_atomic64_inc_return_release() - atomic increment with release ordering 3151 * @v: pointer to atomic64_t 3152 * 3153 * Atomically updates @v to (@v + 1) with release ordering. 3154 * 3155 * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere. 3156 * 3157 * Return: The updated value of @v. 3158 */ 3159static __always_inline s64 3160raw_atomic64_inc_return_release(atomic64_t *v) 3161{ 3162#if defined(arch_atomic64_inc_return_release) 3163 return arch_atomic64_inc_return_release(v); 3164#elif defined(arch_atomic64_inc_return_relaxed) 3165 __atomic_release_fence(); 3166 return arch_atomic64_inc_return_relaxed(v); 3167#elif defined(arch_atomic64_inc_return) 3168 return arch_atomic64_inc_return(v); 3169#else 3170 return raw_atomic64_add_return_release(1, v); 3171#endif 3172} 3173 3174/** 3175 * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering 3176 * @v: pointer to atomic64_t 3177 * 3178 * Atomically updates @v to (@v + 1) with relaxed ordering. 3179 * 3180 * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere. 3181 * 3182 * Return: The updated value of @v. 3183 */ 3184static __always_inline s64 3185raw_atomic64_inc_return_relaxed(atomic64_t *v) 3186{ 3187#if defined(arch_atomic64_inc_return_relaxed) 3188 return arch_atomic64_inc_return_relaxed(v); 3189#elif defined(arch_atomic64_inc_return) 3190 return arch_atomic64_inc_return(v); 3191#else 3192 return raw_atomic64_add_return_relaxed(1, v); 3193#endif 3194} 3195 3196/** 3197 * raw_atomic64_fetch_inc() - atomic increment with full ordering 3198 * @v: pointer to atomic64_t 3199 * 3200 * Atomically updates @v to (@v + 1) with full ordering. 3201 * 3202 * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere. 3203 * 3204 * Return: The original value of @v. 3205 */ 3206static __always_inline s64 3207raw_atomic64_fetch_inc(atomic64_t *v) 3208{ 3209#if defined(arch_atomic64_fetch_inc) 3210 return arch_atomic64_fetch_inc(v); 3211#elif defined(arch_atomic64_fetch_inc_relaxed) 3212 s64 ret; 3213 __atomic_pre_full_fence(); 3214 ret = arch_atomic64_fetch_inc_relaxed(v); 3215 __atomic_post_full_fence(); 3216 return ret; 3217#else 3218 return raw_atomic64_fetch_add(1, v); 3219#endif 3220} 3221 3222/** 3223 * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering 3224 * @v: pointer to atomic64_t 3225 * 3226 * Atomically updates @v to (@v + 1) with acquire ordering. 3227 * 3228 * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere. 3229 * 3230 * Return: The original value of @v. 3231 */ 3232static __always_inline s64 3233raw_atomic64_fetch_inc_acquire(atomic64_t *v) 3234{ 3235#if defined(arch_atomic64_fetch_inc_acquire) 3236 return arch_atomic64_fetch_inc_acquire(v); 3237#elif defined(arch_atomic64_fetch_inc_relaxed) 3238 s64 ret = arch_atomic64_fetch_inc_relaxed(v); 3239 __atomic_acquire_fence(); 3240 return ret; 3241#elif defined(arch_atomic64_fetch_inc) 3242 return arch_atomic64_fetch_inc(v); 3243#else 3244 return raw_atomic64_fetch_add_acquire(1, v); 3245#endif 3246} 3247 3248/** 3249 * raw_atomic64_fetch_inc_release() - atomic increment with release ordering 3250 * @v: pointer to atomic64_t 3251 * 3252 * Atomically updates @v to (@v + 1) with release ordering. 3253 * 3254 * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere. 3255 * 3256 * Return: The original value of @v. 3257 */ 3258static __always_inline s64 3259raw_atomic64_fetch_inc_release(atomic64_t *v) 3260{ 3261#if defined(arch_atomic64_fetch_inc_release) 3262 return arch_atomic64_fetch_inc_release(v); 3263#elif defined(arch_atomic64_fetch_inc_relaxed) 3264 __atomic_release_fence(); 3265 return arch_atomic64_fetch_inc_relaxed(v); 3266#elif defined(arch_atomic64_fetch_inc) 3267 return arch_atomic64_fetch_inc(v); 3268#else 3269 return raw_atomic64_fetch_add_release(1, v); 3270#endif 3271} 3272 3273/** 3274 * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering 3275 * @v: pointer to atomic64_t 3276 * 3277 * Atomically updates @v to (@v + 1) with relaxed ordering. 3278 * 3279 * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere. 3280 * 3281 * Return: The original value of @v. 3282 */ 3283static __always_inline s64 3284raw_atomic64_fetch_inc_relaxed(atomic64_t *v) 3285{ 3286#if defined(arch_atomic64_fetch_inc_relaxed) 3287 return arch_atomic64_fetch_inc_relaxed(v); 3288#elif defined(arch_atomic64_fetch_inc) 3289 return arch_atomic64_fetch_inc(v); 3290#else 3291 return raw_atomic64_fetch_add_relaxed(1, v); 3292#endif 3293} 3294 3295/** 3296 * raw_atomic64_dec() - atomic decrement with relaxed ordering 3297 * @v: pointer to atomic64_t 3298 * 3299 * Atomically updates @v to (@v - 1) with relaxed ordering. 3300 * 3301 * Safe to use in noinstr code; prefer atomic64_dec() elsewhere. 3302 * 3303 * Return: Nothing. 3304 */ 3305static __always_inline void 3306raw_atomic64_dec(atomic64_t *v) 3307{ 3308#if defined(arch_atomic64_dec) 3309 arch_atomic64_dec(v); 3310#else 3311 raw_atomic64_sub(1, v); 3312#endif 3313} 3314 3315/** 3316 * raw_atomic64_dec_return() - atomic decrement with full ordering 3317 * @v: pointer to atomic64_t 3318 * 3319 * Atomically updates @v to (@v - 1) with full ordering. 3320 * 3321 * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere. 3322 * 3323 * Return: The updated value of @v. 3324 */ 3325static __always_inline s64 3326raw_atomic64_dec_return(atomic64_t *v) 3327{ 3328#if defined(arch_atomic64_dec_return) 3329 return arch_atomic64_dec_return(v); 3330#elif defined(arch_atomic64_dec_return_relaxed) 3331 s64 ret; 3332 __atomic_pre_full_fence(); 3333 ret = arch_atomic64_dec_return_relaxed(v); 3334 __atomic_post_full_fence(); 3335 return ret; 3336#else 3337 return raw_atomic64_sub_return(1, v); 3338#endif 3339} 3340 3341/** 3342 * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering 3343 * @v: pointer to atomic64_t 3344 * 3345 * Atomically updates @v to (@v - 1) with acquire ordering. 3346 * 3347 * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere. 3348 * 3349 * Return: The updated value of @v. 3350 */ 3351static __always_inline s64 3352raw_atomic64_dec_return_acquire(atomic64_t *v) 3353{ 3354#if defined(arch_atomic64_dec_return_acquire) 3355 return arch_atomic64_dec_return_acquire(v); 3356#elif defined(arch_atomic64_dec_return_relaxed) 3357 s64 ret = arch_atomic64_dec_return_relaxed(v); 3358 __atomic_acquire_fence(); 3359 return ret; 3360#elif defined(arch_atomic64_dec_return) 3361 return arch_atomic64_dec_return(v); 3362#else 3363 return raw_atomic64_sub_return_acquire(1, v); 3364#endif 3365} 3366 3367/** 3368 * raw_atomic64_dec_return_release() - atomic decrement with release ordering 3369 * @v: pointer to atomic64_t 3370 * 3371 * Atomically updates @v to (@v - 1) with release ordering. 3372 * 3373 * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere. 3374 * 3375 * Return: The updated value of @v. 3376 */ 3377static __always_inline s64 3378raw_atomic64_dec_return_release(atomic64_t *v) 3379{ 3380#if defined(arch_atomic64_dec_return_release) 3381 return arch_atomic64_dec_return_release(v); 3382#elif defined(arch_atomic64_dec_return_relaxed) 3383 __atomic_release_fence(); 3384 return arch_atomic64_dec_return_relaxed(v); 3385#elif defined(arch_atomic64_dec_return) 3386 return arch_atomic64_dec_return(v); 3387#else 3388 return raw_atomic64_sub_return_release(1, v); 3389#endif 3390} 3391 3392/** 3393 * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering 3394 * @v: pointer to atomic64_t 3395 * 3396 * Atomically updates @v to (@v - 1) with relaxed ordering. 3397 * 3398 * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere. 3399 * 3400 * Return: The updated value of @v. 3401 */ 3402static __always_inline s64 3403raw_atomic64_dec_return_relaxed(atomic64_t *v) 3404{ 3405#if defined(arch_atomic64_dec_return_relaxed) 3406 return arch_atomic64_dec_return_relaxed(v); 3407#elif defined(arch_atomic64_dec_return) 3408 return arch_atomic64_dec_return(v); 3409#else 3410 return raw_atomic64_sub_return_relaxed(1, v); 3411#endif 3412} 3413 3414/** 3415 * raw_atomic64_fetch_dec() - atomic decrement with full ordering 3416 * @v: pointer to atomic64_t 3417 * 3418 * Atomically updates @v to (@v - 1) with full ordering. 3419 * 3420 * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere. 3421 * 3422 * Return: The original value of @v. 3423 */ 3424static __always_inline s64 3425raw_atomic64_fetch_dec(atomic64_t *v) 3426{ 3427#if defined(arch_atomic64_fetch_dec) 3428 return arch_atomic64_fetch_dec(v); 3429#elif defined(arch_atomic64_fetch_dec_relaxed) 3430 s64 ret; 3431 __atomic_pre_full_fence(); 3432 ret = arch_atomic64_fetch_dec_relaxed(v); 3433 __atomic_post_full_fence(); 3434 return ret; 3435#else 3436 return raw_atomic64_fetch_sub(1, v); 3437#endif 3438} 3439 3440/** 3441 * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering 3442 * @v: pointer to atomic64_t 3443 * 3444 * Atomically updates @v to (@v - 1) with acquire ordering. 3445 * 3446 * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere. 3447 * 3448 * Return: The original value of @v. 3449 */ 3450static __always_inline s64 3451raw_atomic64_fetch_dec_acquire(atomic64_t *v) 3452{ 3453#if defined(arch_atomic64_fetch_dec_acquire) 3454 return arch_atomic64_fetch_dec_acquire(v); 3455#elif defined(arch_atomic64_fetch_dec_relaxed) 3456 s64 ret = arch_atomic64_fetch_dec_relaxed(v); 3457 __atomic_acquire_fence(); 3458 return ret; 3459#elif defined(arch_atomic64_fetch_dec) 3460 return arch_atomic64_fetch_dec(v); 3461#else 3462 return raw_atomic64_fetch_sub_acquire(1, v); 3463#endif 3464} 3465 3466/** 3467 * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering 3468 * @v: pointer to atomic64_t 3469 * 3470 * Atomically updates @v to (@v - 1) with release ordering. 3471 * 3472 * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere. 3473 * 3474 * Return: The original value of @v. 3475 */ 3476static __always_inline s64 3477raw_atomic64_fetch_dec_release(atomic64_t *v) 3478{ 3479#if defined(arch_atomic64_fetch_dec_release) 3480 return arch_atomic64_fetch_dec_release(v); 3481#elif defined(arch_atomic64_fetch_dec_relaxed) 3482 __atomic_release_fence(); 3483 return arch_atomic64_fetch_dec_relaxed(v); 3484#elif defined(arch_atomic64_fetch_dec) 3485 return arch_atomic64_fetch_dec(v); 3486#else 3487 return raw_atomic64_fetch_sub_release(1, v); 3488#endif 3489} 3490 3491/** 3492 * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering 3493 * @v: pointer to atomic64_t 3494 * 3495 * Atomically updates @v to (@v - 1) with relaxed ordering. 3496 * 3497 * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere. 3498 * 3499 * Return: The original value of @v. 3500 */ 3501static __always_inline s64 3502raw_atomic64_fetch_dec_relaxed(atomic64_t *v) 3503{ 3504#if defined(arch_atomic64_fetch_dec_relaxed) 3505 return arch_atomic64_fetch_dec_relaxed(v); 3506#elif defined(arch_atomic64_fetch_dec) 3507 return arch_atomic64_fetch_dec(v); 3508#else 3509 return raw_atomic64_fetch_sub_relaxed(1, v); 3510#endif 3511} 3512 3513/** 3514 * raw_atomic64_and() - atomic bitwise AND with relaxed ordering 3515 * @i: s64 value 3516 * @v: pointer to atomic64_t 3517 * 3518 * Atomically updates @v to (@v & @i) with relaxed ordering. 3519 * 3520 * Safe to use in noinstr code; prefer atomic64_and() elsewhere. 3521 * 3522 * Return: Nothing. 3523 */ 3524static __always_inline void 3525raw_atomic64_and(s64 i, atomic64_t *v) 3526{ 3527 arch_atomic64_and(i, v); 3528} 3529 3530/** 3531 * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering 3532 * @i: s64 value 3533 * @v: pointer to atomic64_t 3534 * 3535 * Atomically updates @v to (@v & @i) with full ordering. 3536 * 3537 * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere. 3538 * 3539 * Return: The original value of @v. 3540 */ 3541static __always_inline s64 3542raw_atomic64_fetch_and(s64 i, atomic64_t *v) 3543{ 3544#if defined(arch_atomic64_fetch_and) 3545 return arch_atomic64_fetch_and(i, v); 3546#elif defined(arch_atomic64_fetch_and_relaxed) 3547 s64 ret; 3548 __atomic_pre_full_fence(); 3549 ret = arch_atomic64_fetch_and_relaxed(i, v); 3550 __atomic_post_full_fence(); 3551 return ret; 3552#else 3553#error "Unable to define raw_atomic64_fetch_and" 3554#endif 3555} 3556 3557/** 3558 * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering 3559 * @i: s64 value 3560 * @v: pointer to atomic64_t 3561 * 3562 * Atomically updates @v to (@v & @i) with acquire ordering. 3563 * 3564 * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere. 3565 * 3566 * Return: The original value of @v. 3567 */ 3568static __always_inline s64 3569raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) 3570{ 3571#if defined(arch_atomic64_fetch_and_acquire) 3572 return arch_atomic64_fetch_and_acquire(i, v); 3573#elif defined(arch_atomic64_fetch_and_relaxed) 3574 s64 ret = arch_atomic64_fetch_and_relaxed(i, v); 3575 __atomic_acquire_fence(); 3576 return ret; 3577#elif defined(arch_atomic64_fetch_and) 3578 return arch_atomic64_fetch_and(i, v); 3579#else 3580#error "Unable to define raw_atomic64_fetch_and_acquire" 3581#endif 3582} 3583 3584/** 3585 * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering 3586 * @i: s64 value 3587 * @v: pointer to atomic64_t 3588 * 3589 * Atomically updates @v to (@v & @i) with release ordering. 3590 * 3591 * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere. 3592 * 3593 * Return: The original value of @v. 3594 */ 3595static __always_inline s64 3596raw_atomic64_fetch_and_release(s64 i, atomic64_t *v) 3597{ 3598#if defined(arch_atomic64_fetch_and_release) 3599 return arch_atomic64_fetch_and_release(i, v); 3600#elif defined(arch_atomic64_fetch_and_relaxed) 3601 __atomic_release_fence(); 3602 return arch_atomic64_fetch_and_relaxed(i, v); 3603#elif defined(arch_atomic64_fetch_and) 3604 return arch_atomic64_fetch_and(i, v); 3605#else 3606#error "Unable to define raw_atomic64_fetch_and_release" 3607#endif 3608} 3609 3610/** 3611 * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering 3612 * @i: s64 value 3613 * @v: pointer to atomic64_t 3614 * 3615 * Atomically updates @v to (@v & @i) with relaxed ordering. 3616 * 3617 * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere. 3618 * 3619 * Return: The original value of @v. 3620 */ 3621static __always_inline s64 3622raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) 3623{ 3624#if defined(arch_atomic64_fetch_and_relaxed) 3625 return arch_atomic64_fetch_and_relaxed(i, v); 3626#elif defined(arch_atomic64_fetch_and) 3627 return arch_atomic64_fetch_and(i, v); 3628#else 3629#error "Unable to define raw_atomic64_fetch_and_relaxed" 3630#endif 3631} 3632 3633/** 3634 * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering 3635 * @i: s64 value 3636 * @v: pointer to atomic64_t 3637 * 3638 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 3639 * 3640 * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere. 3641 * 3642 * Return: Nothing. 3643 */ 3644static __always_inline void 3645raw_atomic64_andnot(s64 i, atomic64_t *v) 3646{ 3647#if defined(arch_atomic64_andnot) 3648 arch_atomic64_andnot(i, v); 3649#else 3650 raw_atomic64_and(~i, v); 3651#endif 3652} 3653 3654/** 3655 * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering 3656 * @i: s64 value 3657 * @v: pointer to atomic64_t 3658 * 3659 * Atomically updates @v to (@v & ~@i) with full ordering. 3660 * 3661 * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere. 3662 * 3663 * Return: The original value of @v. 3664 */ 3665static __always_inline s64 3666raw_atomic64_fetch_andnot(s64 i, atomic64_t *v) 3667{ 3668#if defined(arch_atomic64_fetch_andnot) 3669 return arch_atomic64_fetch_andnot(i, v); 3670#elif defined(arch_atomic64_fetch_andnot_relaxed) 3671 s64 ret; 3672 __atomic_pre_full_fence(); 3673 ret = arch_atomic64_fetch_andnot_relaxed(i, v); 3674 __atomic_post_full_fence(); 3675 return ret; 3676#else 3677 return raw_atomic64_fetch_and(~i, v); 3678#endif 3679} 3680 3681/** 3682 * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering 3683 * @i: s64 value 3684 * @v: pointer to atomic64_t 3685 * 3686 * Atomically updates @v to (@v & ~@i) with acquire ordering. 3687 * 3688 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere. 3689 * 3690 * Return: The original value of @v. 3691 */ 3692static __always_inline s64 3693raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) 3694{ 3695#if defined(arch_atomic64_fetch_andnot_acquire) 3696 return arch_atomic64_fetch_andnot_acquire(i, v); 3697#elif defined(arch_atomic64_fetch_andnot_relaxed) 3698 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v); 3699 __atomic_acquire_fence(); 3700 return ret; 3701#elif defined(arch_atomic64_fetch_andnot) 3702 return arch_atomic64_fetch_andnot(i, v); 3703#else 3704 return raw_atomic64_fetch_and_acquire(~i, v); 3705#endif 3706} 3707 3708/** 3709 * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering 3710 * @i: s64 value 3711 * @v: pointer to atomic64_t 3712 * 3713 * Atomically updates @v to (@v & ~@i) with release ordering. 3714 * 3715 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere. 3716 * 3717 * Return: The original value of @v. 3718 */ 3719static __always_inline s64 3720raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) 3721{ 3722#if defined(arch_atomic64_fetch_andnot_release) 3723 return arch_atomic64_fetch_andnot_release(i, v); 3724#elif defined(arch_atomic64_fetch_andnot_relaxed) 3725 __atomic_release_fence(); 3726 return arch_atomic64_fetch_andnot_relaxed(i, v); 3727#elif defined(arch_atomic64_fetch_andnot) 3728 return arch_atomic64_fetch_andnot(i, v); 3729#else 3730 return raw_atomic64_fetch_and_release(~i, v); 3731#endif 3732} 3733 3734/** 3735 * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering 3736 * @i: s64 value 3737 * @v: pointer to atomic64_t 3738 * 3739 * Atomically updates @v to (@v & ~@i) with relaxed ordering. 3740 * 3741 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere. 3742 * 3743 * Return: The original value of @v. 3744 */ 3745static __always_inline s64 3746raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) 3747{ 3748#if defined(arch_atomic64_fetch_andnot_relaxed) 3749 return arch_atomic64_fetch_andnot_relaxed(i, v); 3750#elif defined(arch_atomic64_fetch_andnot) 3751 return arch_atomic64_fetch_andnot(i, v); 3752#else 3753 return raw_atomic64_fetch_and_relaxed(~i, v); 3754#endif 3755} 3756 3757/** 3758 * raw_atomic64_or() - atomic bitwise OR with relaxed ordering 3759 * @i: s64 value 3760 * @v: pointer to atomic64_t 3761 * 3762 * Atomically updates @v to (@v | @i) with relaxed ordering. 3763 * 3764 * Safe to use in noinstr code; prefer atomic64_or() elsewhere. 3765 * 3766 * Return: Nothing. 3767 */ 3768static __always_inline void 3769raw_atomic64_or(s64 i, atomic64_t *v) 3770{ 3771 arch_atomic64_or(i, v); 3772} 3773 3774/** 3775 * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering 3776 * @i: s64 value 3777 * @v: pointer to atomic64_t 3778 * 3779 * Atomically updates @v to (@v | @i) with full ordering. 3780 * 3781 * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere. 3782 * 3783 * Return: The original value of @v. 3784 */ 3785static __always_inline s64 3786raw_atomic64_fetch_or(s64 i, atomic64_t *v) 3787{ 3788#if defined(arch_atomic64_fetch_or) 3789 return arch_atomic64_fetch_or(i, v); 3790#elif defined(arch_atomic64_fetch_or_relaxed) 3791 s64 ret; 3792 __atomic_pre_full_fence(); 3793 ret = arch_atomic64_fetch_or_relaxed(i, v); 3794 __atomic_post_full_fence(); 3795 return ret; 3796#else 3797#error "Unable to define raw_atomic64_fetch_or" 3798#endif 3799} 3800 3801/** 3802 * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering 3803 * @i: s64 value 3804 * @v: pointer to atomic64_t 3805 * 3806 * Atomically updates @v to (@v | @i) with acquire ordering. 3807 * 3808 * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere. 3809 * 3810 * Return: The original value of @v. 3811 */ 3812static __always_inline s64 3813raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) 3814{ 3815#if defined(arch_atomic64_fetch_or_acquire) 3816 return arch_atomic64_fetch_or_acquire(i, v); 3817#elif defined(arch_atomic64_fetch_or_relaxed) 3818 s64 ret = arch_atomic64_fetch_or_relaxed(i, v); 3819 __atomic_acquire_fence(); 3820 return ret; 3821#elif defined(arch_atomic64_fetch_or) 3822 return arch_atomic64_fetch_or(i, v); 3823#else 3824#error "Unable to define raw_atomic64_fetch_or_acquire" 3825#endif 3826} 3827 3828/** 3829 * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering 3830 * @i: s64 value 3831 * @v: pointer to atomic64_t 3832 * 3833 * Atomically updates @v to (@v | @i) with release ordering. 3834 * 3835 * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere. 3836 * 3837 * Return: The original value of @v. 3838 */ 3839static __always_inline s64 3840raw_atomic64_fetch_or_release(s64 i, atomic64_t *v) 3841{ 3842#if defined(arch_atomic64_fetch_or_release) 3843 return arch_atomic64_fetch_or_release(i, v); 3844#elif defined(arch_atomic64_fetch_or_relaxed) 3845 __atomic_release_fence(); 3846 return arch_atomic64_fetch_or_relaxed(i, v); 3847#elif defined(arch_atomic64_fetch_or) 3848 return arch_atomic64_fetch_or(i, v); 3849#else 3850#error "Unable to define raw_atomic64_fetch_or_release" 3851#endif 3852} 3853 3854/** 3855 * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering 3856 * @i: s64 value 3857 * @v: pointer to atomic64_t 3858 * 3859 * Atomically updates @v to (@v | @i) with relaxed ordering. 3860 * 3861 * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere. 3862 * 3863 * Return: The original value of @v. 3864 */ 3865static __always_inline s64 3866raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) 3867{ 3868#if defined(arch_atomic64_fetch_or_relaxed) 3869 return arch_atomic64_fetch_or_relaxed(i, v); 3870#elif defined(arch_atomic64_fetch_or) 3871 return arch_atomic64_fetch_or(i, v); 3872#else 3873#error "Unable to define raw_atomic64_fetch_or_relaxed" 3874#endif 3875} 3876 3877/** 3878 * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering 3879 * @i: s64 value 3880 * @v: pointer to atomic64_t 3881 * 3882 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 3883 * 3884 * Safe to use in noinstr code; prefer atomic64_xor() elsewhere. 3885 * 3886 * Return: Nothing. 3887 */ 3888static __always_inline void 3889raw_atomic64_xor(s64 i, atomic64_t *v) 3890{ 3891 arch_atomic64_xor(i, v); 3892} 3893 3894/** 3895 * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering 3896 * @i: s64 value 3897 * @v: pointer to atomic64_t 3898 * 3899 * Atomically updates @v to (@v ^ @i) with full ordering. 3900 * 3901 * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere. 3902 * 3903 * Return: The original value of @v. 3904 */ 3905static __always_inline s64 3906raw_atomic64_fetch_xor(s64 i, atomic64_t *v) 3907{ 3908#if defined(arch_atomic64_fetch_xor) 3909 return arch_atomic64_fetch_xor(i, v); 3910#elif defined(arch_atomic64_fetch_xor_relaxed) 3911 s64 ret; 3912 __atomic_pre_full_fence(); 3913 ret = arch_atomic64_fetch_xor_relaxed(i, v); 3914 __atomic_post_full_fence(); 3915 return ret; 3916#else 3917#error "Unable to define raw_atomic64_fetch_xor" 3918#endif 3919} 3920 3921/** 3922 * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering 3923 * @i: s64 value 3924 * @v: pointer to atomic64_t 3925 * 3926 * Atomically updates @v to (@v ^ @i) with acquire ordering. 3927 * 3928 * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere. 3929 * 3930 * Return: The original value of @v. 3931 */ 3932static __always_inline s64 3933raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) 3934{ 3935#if defined(arch_atomic64_fetch_xor_acquire) 3936 return arch_atomic64_fetch_xor_acquire(i, v); 3937#elif defined(arch_atomic64_fetch_xor_relaxed) 3938 s64 ret = arch_atomic64_fetch_xor_relaxed(i, v); 3939 __atomic_acquire_fence(); 3940 return ret; 3941#elif defined(arch_atomic64_fetch_xor) 3942 return arch_atomic64_fetch_xor(i, v); 3943#else 3944#error "Unable to define raw_atomic64_fetch_xor_acquire" 3945#endif 3946} 3947 3948/** 3949 * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering 3950 * @i: s64 value 3951 * @v: pointer to atomic64_t 3952 * 3953 * Atomically updates @v to (@v ^ @i) with release ordering. 3954 * 3955 * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere. 3956 * 3957 * Return: The original value of @v. 3958 */ 3959static __always_inline s64 3960raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v) 3961{ 3962#if defined(arch_atomic64_fetch_xor_release) 3963 return arch_atomic64_fetch_xor_release(i, v); 3964#elif defined(arch_atomic64_fetch_xor_relaxed) 3965 __atomic_release_fence(); 3966 return arch_atomic64_fetch_xor_relaxed(i, v); 3967#elif defined(arch_atomic64_fetch_xor) 3968 return arch_atomic64_fetch_xor(i, v); 3969#else 3970#error "Unable to define raw_atomic64_fetch_xor_release" 3971#endif 3972} 3973 3974/** 3975 * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering 3976 * @i: s64 value 3977 * @v: pointer to atomic64_t 3978 * 3979 * Atomically updates @v to (@v ^ @i) with relaxed ordering. 3980 * 3981 * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere. 3982 * 3983 * Return: The original value of @v. 3984 */ 3985static __always_inline s64 3986raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) 3987{ 3988#if defined(arch_atomic64_fetch_xor_relaxed) 3989 return arch_atomic64_fetch_xor_relaxed(i, v); 3990#elif defined(arch_atomic64_fetch_xor) 3991 return arch_atomic64_fetch_xor(i, v); 3992#else 3993#error "Unable to define raw_atomic64_fetch_xor_relaxed" 3994#endif 3995} 3996 3997/** 3998 * raw_atomic64_xchg() - atomic exchange with full ordering 3999 * @v: pointer to atomic64_t 4000 * @new: s64 value to assign 4001 * 4002 * Atomically updates @v to @new with full ordering. 4003 * 4004 * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere. 4005 * 4006 * Return: The original value of @v. 4007 */ 4008static __always_inline s64 4009raw_atomic64_xchg(atomic64_t *v, s64 new) 4010{ 4011#if defined(arch_atomic64_xchg) 4012 return arch_atomic64_xchg(v, new); 4013#elif defined(arch_atomic64_xchg_relaxed) 4014 s64 ret; 4015 __atomic_pre_full_fence(); 4016 ret = arch_atomic64_xchg_relaxed(v, new); 4017 __atomic_post_full_fence(); 4018 return ret; 4019#else 4020 return raw_xchg(&v->counter, new); 4021#endif 4022} 4023 4024/** 4025 * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering 4026 * @v: pointer to atomic64_t 4027 * @new: s64 value to assign 4028 * 4029 * Atomically updates @v to @new with acquire ordering. 4030 * 4031 * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere. 4032 * 4033 * Return: The original value of @v. 4034 */ 4035static __always_inline s64 4036raw_atomic64_xchg_acquire(atomic64_t *v, s64 new) 4037{ 4038#if defined(arch_atomic64_xchg_acquire) 4039 return arch_atomic64_xchg_acquire(v, new); 4040#elif defined(arch_atomic64_xchg_relaxed) 4041 s64 ret = arch_atomic64_xchg_relaxed(v, new); 4042 __atomic_acquire_fence(); 4043 return ret; 4044#elif defined(arch_atomic64_xchg) 4045 return arch_atomic64_xchg(v, new); 4046#else 4047 return raw_xchg_acquire(&v->counter, new); 4048#endif 4049} 4050 4051/** 4052 * raw_atomic64_xchg_release() - atomic exchange with release ordering 4053 * @v: pointer to atomic64_t 4054 * @new: s64 value to assign 4055 * 4056 * Atomically updates @v to @new with release ordering. 4057 * 4058 * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere. 4059 * 4060 * Return: The original value of @v. 4061 */ 4062static __always_inline s64 4063raw_atomic64_xchg_release(atomic64_t *v, s64 new) 4064{ 4065#if defined(arch_atomic64_xchg_release) 4066 return arch_atomic64_xchg_release(v, new); 4067#elif defined(arch_atomic64_xchg_relaxed) 4068 __atomic_release_fence(); 4069 return arch_atomic64_xchg_relaxed(v, new); 4070#elif defined(arch_atomic64_xchg) 4071 return arch_atomic64_xchg(v, new); 4072#else 4073 return raw_xchg_release(&v->counter, new); 4074#endif 4075} 4076 4077/** 4078 * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering 4079 * @v: pointer to atomic64_t 4080 * @new: s64 value to assign 4081 * 4082 * Atomically updates @v to @new with relaxed ordering. 4083 * 4084 * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere. 4085 * 4086 * Return: The original value of @v. 4087 */ 4088static __always_inline s64 4089raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new) 4090{ 4091#if defined(arch_atomic64_xchg_relaxed) 4092 return arch_atomic64_xchg_relaxed(v, new); 4093#elif defined(arch_atomic64_xchg) 4094 return arch_atomic64_xchg(v, new); 4095#else 4096 return raw_xchg_relaxed(&v->counter, new); 4097#endif 4098} 4099 4100/** 4101 * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering 4102 * @v: pointer to atomic64_t 4103 * @old: s64 value to compare with 4104 * @new: s64 value to assign 4105 * 4106 * If (@v == @old), atomically updates @v to @new with full ordering. 4107 * 4108 * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere. 4109 * 4110 * Return: The original value of @v. 4111 */ 4112static __always_inline s64 4113raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) 4114{ 4115#if defined(arch_atomic64_cmpxchg) 4116 return arch_atomic64_cmpxchg(v, old, new); 4117#elif defined(arch_atomic64_cmpxchg_relaxed) 4118 s64 ret; 4119 __atomic_pre_full_fence(); 4120 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); 4121 __atomic_post_full_fence(); 4122 return ret; 4123#else 4124 return raw_cmpxchg(&v->counter, old, new); 4125#endif 4126} 4127 4128/** 4129 * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 4130 * @v: pointer to atomic64_t 4131 * @old: s64 value to compare with 4132 * @new: s64 value to assign 4133 * 4134 * If (@v == @old), atomically updates @v to @new with acquire ordering. 4135 * 4136 * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere. 4137 * 4138 * Return: The original value of @v. 4139 */ 4140static __always_inline s64 4141raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) 4142{ 4143#if defined(arch_atomic64_cmpxchg_acquire) 4144 return arch_atomic64_cmpxchg_acquire(v, old, new); 4145#elif defined(arch_atomic64_cmpxchg_relaxed) 4146 s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); 4147 __atomic_acquire_fence(); 4148 return ret; 4149#elif defined(arch_atomic64_cmpxchg) 4150 return arch_atomic64_cmpxchg(v, old, new); 4151#else 4152 return raw_cmpxchg_acquire(&v->counter, old, new); 4153#endif 4154} 4155 4156/** 4157 * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering 4158 * @v: pointer to atomic64_t 4159 * @old: s64 value to compare with 4160 * @new: s64 value to assign 4161 * 4162 * If (@v == @old), atomically updates @v to @new with release ordering. 4163 * 4164 * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere. 4165 * 4166 * Return: The original value of @v. 4167 */ 4168static __always_inline s64 4169raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) 4170{ 4171#if defined(arch_atomic64_cmpxchg_release) 4172 return arch_atomic64_cmpxchg_release(v, old, new); 4173#elif defined(arch_atomic64_cmpxchg_relaxed) 4174 __atomic_release_fence(); 4175 return arch_atomic64_cmpxchg_relaxed(v, old, new); 4176#elif defined(arch_atomic64_cmpxchg) 4177 return arch_atomic64_cmpxchg(v, old, new); 4178#else 4179 return raw_cmpxchg_release(&v->counter, old, new); 4180#endif 4181} 4182 4183/** 4184 * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 4185 * @v: pointer to atomic64_t 4186 * @old: s64 value to compare with 4187 * @new: s64 value to assign 4188 * 4189 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 4190 * 4191 * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere. 4192 * 4193 * Return: The original value of @v. 4194 */ 4195static __always_inline s64 4196raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) 4197{ 4198#if defined(arch_atomic64_cmpxchg_relaxed) 4199 return arch_atomic64_cmpxchg_relaxed(v, old, new); 4200#elif defined(arch_atomic64_cmpxchg) 4201 return arch_atomic64_cmpxchg(v, old, new); 4202#else 4203 return raw_cmpxchg_relaxed(&v->counter, old, new); 4204#endif 4205} 4206 4207/** 4208 * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering 4209 * @v: pointer to atomic64_t 4210 * @old: pointer to s64 value to compare with 4211 * @new: s64 value to assign 4212 * 4213 * If (@v == @old), atomically updates @v to @new with full ordering. 4214 * Otherwise, updates @old to the current value of @v. 4215 * 4216 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere. 4217 * 4218 * Return: @true if the exchange occured, @false otherwise. 4219 */ 4220static __always_inline bool 4221raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) 4222{ 4223#if defined(arch_atomic64_try_cmpxchg) 4224 return arch_atomic64_try_cmpxchg(v, old, new); 4225#elif defined(arch_atomic64_try_cmpxchg_relaxed) 4226 bool ret; 4227 __atomic_pre_full_fence(); 4228 ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); 4229 __atomic_post_full_fence(); 4230 return ret; 4231#else 4232 s64 r, o = *old; 4233 r = raw_atomic64_cmpxchg(v, o, new); 4234 if (unlikely(r != o)) 4235 *old = r; 4236 return likely(r == o); 4237#endif 4238} 4239 4240/** 4241 * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering 4242 * @v: pointer to atomic64_t 4243 * @old: pointer to s64 value to compare with 4244 * @new: s64 value to assign 4245 * 4246 * If (@v == @old), atomically updates @v to @new with acquire ordering. 4247 * Otherwise, updates @old to the current value of @v. 4248 * 4249 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere. 4250 * 4251 * Return: @true if the exchange occured, @false otherwise. 4252 */ 4253static __always_inline bool 4254raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) 4255{ 4256#if defined(arch_atomic64_try_cmpxchg_acquire) 4257 return arch_atomic64_try_cmpxchg_acquire(v, old, new); 4258#elif defined(arch_atomic64_try_cmpxchg_relaxed) 4259 bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); 4260 __atomic_acquire_fence(); 4261 return ret; 4262#elif defined(arch_atomic64_try_cmpxchg) 4263 return arch_atomic64_try_cmpxchg(v, old, new); 4264#else 4265 s64 r, o = *old; 4266 r = raw_atomic64_cmpxchg_acquire(v, o, new); 4267 if (unlikely(r != o)) 4268 *old = r; 4269 return likely(r == o); 4270#endif 4271} 4272 4273/** 4274 * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering 4275 * @v: pointer to atomic64_t 4276 * @old: pointer to s64 value to compare with 4277 * @new: s64 value to assign 4278 * 4279 * If (@v == @old), atomically updates @v to @new with release ordering. 4280 * Otherwise, updates @old to the current value of @v. 4281 * 4282 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere. 4283 * 4284 * Return: @true if the exchange occured, @false otherwise. 4285 */ 4286static __always_inline bool 4287raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) 4288{ 4289#if defined(arch_atomic64_try_cmpxchg_release) 4290 return arch_atomic64_try_cmpxchg_release(v, old, new); 4291#elif defined(arch_atomic64_try_cmpxchg_relaxed) 4292 __atomic_release_fence(); 4293 return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 4294#elif defined(arch_atomic64_try_cmpxchg) 4295 return arch_atomic64_try_cmpxchg(v, old, new); 4296#else 4297 s64 r, o = *old; 4298 r = raw_atomic64_cmpxchg_release(v, o, new); 4299 if (unlikely(r != o)) 4300 *old = r; 4301 return likely(r == o); 4302#endif 4303} 4304 4305/** 4306 * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering 4307 * @v: pointer to atomic64_t 4308 * @old: pointer to s64 value to compare with 4309 * @new: s64 value to assign 4310 * 4311 * If (@v == @old), atomically updates @v to @new with relaxed ordering. 4312 * Otherwise, updates @old to the current value of @v. 4313 * 4314 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere. 4315 * 4316 * Return: @true if the exchange occured, @false otherwise. 4317 */ 4318static __always_inline bool 4319raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) 4320{ 4321#if defined(arch_atomic64_try_cmpxchg_relaxed) 4322 return arch_atomic64_try_cmpxchg_relaxed(v, old, new); 4323#elif defined(arch_atomic64_try_cmpxchg) 4324 return arch_atomic64_try_cmpxchg(v, old, new); 4325#else 4326 s64 r, o = *old; 4327 r = raw_atomic64_cmpxchg_relaxed(v, o, new); 4328 if (unlikely(r != o)) 4329 *old = r; 4330 return likely(r == o); 4331#endif 4332} 4333 4334/** 4335 * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering 4336 * @i: s64 value to add 4337 * @v: pointer to atomic64_t 4338 * 4339 * Atomically updates @v to (@v - @i) with full ordering. 4340 * 4341 * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere. 4342 * 4343 * Return: @true if the resulting value of @v is zero, @false otherwise. 4344 */ 4345static __always_inline bool 4346raw_atomic64_sub_and_test(s64 i, atomic64_t *v) 4347{ 4348#if defined(arch_atomic64_sub_and_test) 4349 return arch_atomic64_sub_and_test(i, v); 4350#else 4351 return raw_atomic64_sub_return(i, v) == 0; 4352#endif 4353} 4354 4355/** 4356 * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering 4357 * @v: pointer to atomic64_t 4358 * 4359 * Atomically updates @v to (@v - 1) with full ordering. 4360 * 4361 * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere. 4362 * 4363 * Return: @true if the resulting value of @v is zero, @false otherwise. 4364 */ 4365static __always_inline bool 4366raw_atomic64_dec_and_test(atomic64_t *v) 4367{ 4368#if defined(arch_atomic64_dec_and_test) 4369 return arch_atomic64_dec_and_test(v); 4370#else 4371 return raw_atomic64_dec_return(v) == 0; 4372#endif 4373} 4374 4375/** 4376 * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering 4377 * @v: pointer to atomic64_t 4378 * 4379 * Atomically updates @v to (@v + 1) with full ordering. 4380 * 4381 * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere. 4382 * 4383 * Return: @true if the resulting value of @v is zero, @false otherwise. 4384 */ 4385static __always_inline bool 4386raw_atomic64_inc_and_test(atomic64_t *v) 4387{ 4388#if defined(arch_atomic64_inc_and_test) 4389 return arch_atomic64_inc_and_test(v); 4390#else 4391 return raw_atomic64_inc_return(v) == 0; 4392#endif 4393} 4394 4395/** 4396 * raw_atomic64_add_negative() - atomic add and test if negative with full ordering 4397 * @i: s64 value to add 4398 * @v: pointer to atomic64_t 4399 * 4400 * Atomically updates @v to (@v + @i) with full ordering. 4401 * 4402 * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere. 4403 * 4404 * Return: @true if the resulting value of @v is negative, @false otherwise. 4405 */ 4406static __always_inline bool 4407raw_atomic64_add_negative(s64 i, atomic64_t *v) 4408{ 4409#if defined(arch_atomic64_add_negative) 4410 return arch_atomic64_add_negative(i, v); 4411#elif defined(arch_atomic64_add_negative_relaxed) 4412 bool ret; 4413 __atomic_pre_full_fence(); 4414 ret = arch_atomic64_add_negative_relaxed(i, v); 4415 __atomic_post_full_fence(); 4416 return ret; 4417#else 4418 return raw_atomic64_add_return(i, v) < 0; 4419#endif 4420} 4421 4422/** 4423 * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering 4424 * @i: s64 value to add 4425 * @v: pointer to atomic64_t 4426 * 4427 * Atomically updates @v to (@v + @i) with acquire ordering. 4428 * 4429 * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere. 4430 * 4431 * Return: @true if the resulting value of @v is negative, @false otherwise. 4432 */ 4433static __always_inline bool 4434raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v) 4435{ 4436#if defined(arch_atomic64_add_negative_acquire) 4437 return arch_atomic64_add_negative_acquire(i, v); 4438#elif defined(arch_atomic64_add_negative_relaxed) 4439 bool ret = arch_atomic64_add_negative_relaxed(i, v); 4440 __atomic_acquire_fence(); 4441 return ret; 4442#elif defined(arch_atomic64_add_negative) 4443 return arch_atomic64_add_negative(i, v); 4444#else 4445 return raw_atomic64_add_return_acquire(i, v) < 0; 4446#endif 4447} 4448 4449/** 4450 * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering 4451 * @i: s64 value to add 4452 * @v: pointer to atomic64_t 4453 * 4454 * Atomically updates @v to (@v + @i) with release ordering. 4455 * 4456 * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere. 4457 * 4458 * Return: @true if the resulting value of @v is negative, @false otherwise. 4459 */ 4460static __always_inline bool 4461raw_atomic64_add_negative_release(s64 i, atomic64_t *v) 4462{ 4463#if defined(arch_atomic64_add_negative_release) 4464 return arch_atomic64_add_negative_release(i, v); 4465#elif defined(arch_atomic64_add_negative_relaxed) 4466 __atomic_release_fence(); 4467 return arch_atomic64_add_negative_relaxed(i, v); 4468#elif defined(arch_atomic64_add_negative) 4469 return arch_atomic64_add_negative(i, v); 4470#else 4471 return raw_atomic64_add_return_release(i, v) < 0; 4472#endif 4473} 4474 4475/** 4476 * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering 4477 * @i: s64 value to add 4478 * @v: pointer to atomic64_t 4479 * 4480 * Atomically updates @v to (@v + @i) with relaxed ordering. 4481 * 4482 * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere. 4483 * 4484 * Return: @true if the resulting value of @v is negative, @false otherwise. 4485 */ 4486static __always_inline bool 4487raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v) 4488{ 4489#if defined(arch_atomic64_add_negative_relaxed) 4490 return arch_atomic64_add_negative_relaxed(i, v); 4491#elif defined(arch_atomic64_add_negative) 4492 return arch_atomic64_add_negative(i, v); 4493#else 4494 return raw_atomic64_add_return_relaxed(i, v) < 0; 4495#endif 4496} 4497 4498/** 4499 * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering 4500 * @v: pointer to atomic64_t 4501 * @a: s64 value to add 4502 * @u: s64 value to compare with 4503 * 4504 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 4505 * 4506 * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere. 4507 * 4508 * Return: The original value of @v. 4509 */ 4510static __always_inline s64 4511raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 4512{ 4513#if defined(arch_atomic64_fetch_add_unless) 4514 return arch_atomic64_fetch_add_unless(v, a, u); 4515#else 4516 s64 c = raw_atomic64_read(v); 4517 4518 do { 4519 if (unlikely(c == u)) 4520 break; 4521 } while (!raw_atomic64_try_cmpxchg(v, &c, c + a)); 4522 4523 return c; 4524#endif 4525} 4526 4527/** 4528 * raw_atomic64_add_unless() - atomic add unless value with full ordering 4529 * @v: pointer to atomic64_t 4530 * @a: s64 value to add 4531 * @u: s64 value to compare with 4532 * 4533 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering. 4534 * 4535 * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere. 4536 * 4537 * Return: @true if @v was updated, @false otherwise. 4538 */ 4539static __always_inline bool 4540raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) 4541{ 4542#if defined(arch_atomic64_add_unless) 4543 return arch_atomic64_add_unless(v, a, u); 4544#else 4545 return raw_atomic64_fetch_add_unless(v, a, u) != u; 4546#endif 4547} 4548 4549/** 4550 * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering 4551 * @v: pointer to atomic64_t 4552 * 4553 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering. 4554 * 4555 * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere. 4556 * 4557 * Return: @true if @v was updated, @false otherwise. 4558 */ 4559static __always_inline bool 4560raw_atomic64_inc_not_zero(atomic64_t *v) 4561{ 4562#if defined(arch_atomic64_inc_not_zero) 4563 return arch_atomic64_inc_not_zero(v); 4564#else 4565 return raw_atomic64_add_unless(v, 1, 0); 4566#endif 4567} 4568 4569/** 4570 * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering 4571 * @v: pointer to atomic64_t 4572 * 4573 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering. 4574 * 4575 * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere. 4576 * 4577 * Return: @true if @v was updated, @false otherwise. 4578 */ 4579static __always_inline bool 4580raw_atomic64_inc_unless_negative(atomic64_t *v) 4581{ 4582#if defined(arch_atomic64_inc_unless_negative) 4583 return arch_atomic64_inc_unless_negative(v); 4584#else 4585 s64 c = raw_atomic64_read(v); 4586 4587 do { 4588 if (unlikely(c < 0)) 4589 return false; 4590 } while (!raw_atomic64_try_cmpxchg(v, &c, c + 1)); 4591 4592 return true; 4593#endif 4594} 4595 4596/** 4597 * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering 4598 * @v: pointer to atomic64_t 4599 * 4600 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering. 4601 * 4602 * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere. 4603 * 4604 * Return: @true if @v was updated, @false otherwise. 4605 */ 4606static __always_inline bool 4607raw_atomic64_dec_unless_positive(atomic64_t *v) 4608{ 4609#if defined(arch_atomic64_dec_unless_positive) 4610 return arch_atomic64_dec_unless_positive(v); 4611#else 4612 s64 c = raw_atomic64_read(v); 4613 4614 do { 4615 if (unlikely(c > 0)) 4616 return false; 4617 } while (!raw_atomic64_try_cmpxchg(v, &c, c - 1)); 4618 4619 return true; 4620#endif 4621} 4622 4623/** 4624 * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering 4625 * @v: pointer to atomic64_t 4626 * 4627 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering. 4628 * 4629 * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere. 4630 * 4631 * Return: The old value of (@v - 1), regardless of whether @v was updated. 4632 */ 4633static __always_inline s64 4634raw_atomic64_dec_if_positive(atomic64_t *v) 4635{ 4636#if defined(arch_atomic64_dec_if_positive) 4637 return arch_atomic64_dec_if_positive(v); 4638#else 4639 s64 dec, c = raw_atomic64_read(v); 4640 4641 do { 4642 dec = c - 1; 4643 if (unlikely(dec < 0)) 4644 break; 4645 } while (!raw_atomic64_try_cmpxchg(v, &c, dec)); 4646 4647 return dec; 4648#endif 4649} 4650 4651#endif /* _LINUX_ATOMIC_FALLBACK_H */ 4652// 2fdd6702823fa842f9cea57a002e6e4476ae780c