Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2
3// Generated by scripts/atomic/gen-atomic-fallback.sh
4// DO NOT MODIFY THIS FILE DIRECTLY
5
6#ifndef _LINUX_ATOMIC_FALLBACK_H
7#define _LINUX_ATOMIC_FALLBACK_H
8
9#include <linux/compiler.h>
10
11#if defined(arch_xchg)
12#define raw_xchg arch_xchg
13#elif defined(arch_xchg_relaxed)
14#define raw_xchg(...) \
15 __atomic_op_fence(arch_xchg, __VA_ARGS__)
16#else
17extern void raw_xchg_not_implemented(void);
18#define raw_xchg(...) raw_xchg_not_implemented()
19#endif
20
21#if defined(arch_xchg_acquire)
22#define raw_xchg_acquire arch_xchg_acquire
23#elif defined(arch_xchg_relaxed)
24#define raw_xchg_acquire(...) \
25 __atomic_op_acquire(arch_xchg, __VA_ARGS__)
26#elif defined(arch_xchg)
27#define raw_xchg_acquire arch_xchg
28#else
29extern void raw_xchg_acquire_not_implemented(void);
30#define raw_xchg_acquire(...) raw_xchg_acquire_not_implemented()
31#endif
32
33#if defined(arch_xchg_release)
34#define raw_xchg_release arch_xchg_release
35#elif defined(arch_xchg_relaxed)
36#define raw_xchg_release(...) \
37 __atomic_op_release(arch_xchg, __VA_ARGS__)
38#elif defined(arch_xchg)
39#define raw_xchg_release arch_xchg
40#else
41extern void raw_xchg_release_not_implemented(void);
42#define raw_xchg_release(...) raw_xchg_release_not_implemented()
43#endif
44
45#if defined(arch_xchg_relaxed)
46#define raw_xchg_relaxed arch_xchg_relaxed
47#elif defined(arch_xchg)
48#define raw_xchg_relaxed arch_xchg
49#else
50extern void raw_xchg_relaxed_not_implemented(void);
51#define raw_xchg_relaxed(...) raw_xchg_relaxed_not_implemented()
52#endif
53
54#if defined(arch_cmpxchg)
55#define raw_cmpxchg arch_cmpxchg
56#elif defined(arch_cmpxchg_relaxed)
57#define raw_cmpxchg(...) \
58 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
59#else
60extern void raw_cmpxchg_not_implemented(void);
61#define raw_cmpxchg(...) raw_cmpxchg_not_implemented()
62#endif
63
64#if defined(arch_cmpxchg_acquire)
65#define raw_cmpxchg_acquire arch_cmpxchg_acquire
66#elif defined(arch_cmpxchg_relaxed)
67#define raw_cmpxchg_acquire(...) \
68 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
69#elif defined(arch_cmpxchg)
70#define raw_cmpxchg_acquire arch_cmpxchg
71#else
72extern void raw_cmpxchg_acquire_not_implemented(void);
73#define raw_cmpxchg_acquire(...) raw_cmpxchg_acquire_not_implemented()
74#endif
75
76#if defined(arch_cmpxchg_release)
77#define raw_cmpxchg_release arch_cmpxchg_release
78#elif defined(arch_cmpxchg_relaxed)
79#define raw_cmpxchg_release(...) \
80 __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
81#elif defined(arch_cmpxchg)
82#define raw_cmpxchg_release arch_cmpxchg
83#else
84extern void raw_cmpxchg_release_not_implemented(void);
85#define raw_cmpxchg_release(...) raw_cmpxchg_release_not_implemented()
86#endif
87
88#if defined(arch_cmpxchg_relaxed)
89#define raw_cmpxchg_relaxed arch_cmpxchg_relaxed
90#elif defined(arch_cmpxchg)
91#define raw_cmpxchg_relaxed arch_cmpxchg
92#else
93extern void raw_cmpxchg_relaxed_not_implemented(void);
94#define raw_cmpxchg_relaxed(...) raw_cmpxchg_relaxed_not_implemented()
95#endif
96
97#if defined(arch_cmpxchg64)
98#define raw_cmpxchg64 arch_cmpxchg64
99#elif defined(arch_cmpxchg64_relaxed)
100#define raw_cmpxchg64(...) \
101 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
102#else
103extern void raw_cmpxchg64_not_implemented(void);
104#define raw_cmpxchg64(...) raw_cmpxchg64_not_implemented()
105#endif
106
107#if defined(arch_cmpxchg64_acquire)
108#define raw_cmpxchg64_acquire arch_cmpxchg64_acquire
109#elif defined(arch_cmpxchg64_relaxed)
110#define raw_cmpxchg64_acquire(...) \
111 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
112#elif defined(arch_cmpxchg64)
113#define raw_cmpxchg64_acquire arch_cmpxchg64
114#else
115extern void raw_cmpxchg64_acquire_not_implemented(void);
116#define raw_cmpxchg64_acquire(...) raw_cmpxchg64_acquire_not_implemented()
117#endif
118
119#if defined(arch_cmpxchg64_release)
120#define raw_cmpxchg64_release arch_cmpxchg64_release
121#elif defined(arch_cmpxchg64_relaxed)
122#define raw_cmpxchg64_release(...) \
123 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
124#elif defined(arch_cmpxchg64)
125#define raw_cmpxchg64_release arch_cmpxchg64
126#else
127extern void raw_cmpxchg64_release_not_implemented(void);
128#define raw_cmpxchg64_release(...) raw_cmpxchg64_release_not_implemented()
129#endif
130
131#if defined(arch_cmpxchg64_relaxed)
132#define raw_cmpxchg64_relaxed arch_cmpxchg64_relaxed
133#elif defined(arch_cmpxchg64)
134#define raw_cmpxchg64_relaxed arch_cmpxchg64
135#else
136extern void raw_cmpxchg64_relaxed_not_implemented(void);
137#define raw_cmpxchg64_relaxed(...) raw_cmpxchg64_relaxed_not_implemented()
138#endif
139
140#if defined(arch_cmpxchg128)
141#define raw_cmpxchg128 arch_cmpxchg128
142#elif defined(arch_cmpxchg128_relaxed)
143#define raw_cmpxchg128(...) \
144 __atomic_op_fence(arch_cmpxchg128, __VA_ARGS__)
145#else
146extern void raw_cmpxchg128_not_implemented(void);
147#define raw_cmpxchg128(...) raw_cmpxchg128_not_implemented()
148#endif
149
150#if defined(arch_cmpxchg128_acquire)
151#define raw_cmpxchg128_acquire arch_cmpxchg128_acquire
152#elif defined(arch_cmpxchg128_relaxed)
153#define raw_cmpxchg128_acquire(...) \
154 __atomic_op_acquire(arch_cmpxchg128, __VA_ARGS__)
155#elif defined(arch_cmpxchg128)
156#define raw_cmpxchg128_acquire arch_cmpxchg128
157#else
158extern void raw_cmpxchg128_acquire_not_implemented(void);
159#define raw_cmpxchg128_acquire(...) raw_cmpxchg128_acquire_not_implemented()
160#endif
161
162#if defined(arch_cmpxchg128_release)
163#define raw_cmpxchg128_release arch_cmpxchg128_release
164#elif defined(arch_cmpxchg128_relaxed)
165#define raw_cmpxchg128_release(...) \
166 __atomic_op_release(arch_cmpxchg128, __VA_ARGS__)
167#elif defined(arch_cmpxchg128)
168#define raw_cmpxchg128_release arch_cmpxchg128
169#else
170extern void raw_cmpxchg128_release_not_implemented(void);
171#define raw_cmpxchg128_release(...) raw_cmpxchg128_release_not_implemented()
172#endif
173
174#if defined(arch_cmpxchg128_relaxed)
175#define raw_cmpxchg128_relaxed arch_cmpxchg128_relaxed
176#elif defined(arch_cmpxchg128)
177#define raw_cmpxchg128_relaxed arch_cmpxchg128
178#else
179extern void raw_cmpxchg128_relaxed_not_implemented(void);
180#define raw_cmpxchg128_relaxed(...) raw_cmpxchg128_relaxed_not_implemented()
181#endif
182
183#if defined(arch_try_cmpxchg)
184#define raw_try_cmpxchg arch_try_cmpxchg
185#elif defined(arch_try_cmpxchg_relaxed)
186#define raw_try_cmpxchg(...) \
187 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
188#else
189#define raw_try_cmpxchg(_ptr, _oldp, _new) \
190({ \
191 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
192 ___r = raw_cmpxchg((_ptr), ___o, (_new)); \
193 if (unlikely(___r != ___o)) \
194 *___op = ___r; \
195 likely(___r == ___o); \
196})
197#endif
198
199#if defined(arch_try_cmpxchg_acquire)
200#define raw_try_cmpxchg_acquire arch_try_cmpxchg_acquire
201#elif defined(arch_try_cmpxchg_relaxed)
202#define raw_try_cmpxchg_acquire(...) \
203 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
204#elif defined(arch_try_cmpxchg)
205#define raw_try_cmpxchg_acquire arch_try_cmpxchg
206#else
207#define raw_try_cmpxchg_acquire(_ptr, _oldp, _new) \
208({ \
209 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
210 ___r = raw_cmpxchg_acquire((_ptr), ___o, (_new)); \
211 if (unlikely(___r != ___o)) \
212 *___op = ___r; \
213 likely(___r == ___o); \
214})
215#endif
216
217#if defined(arch_try_cmpxchg_release)
218#define raw_try_cmpxchg_release arch_try_cmpxchg_release
219#elif defined(arch_try_cmpxchg_relaxed)
220#define raw_try_cmpxchg_release(...) \
221 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
222#elif defined(arch_try_cmpxchg)
223#define raw_try_cmpxchg_release arch_try_cmpxchg
224#else
225#define raw_try_cmpxchg_release(_ptr, _oldp, _new) \
226({ \
227 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
228 ___r = raw_cmpxchg_release((_ptr), ___o, (_new)); \
229 if (unlikely(___r != ___o)) \
230 *___op = ___r; \
231 likely(___r == ___o); \
232})
233#endif
234
235#if defined(arch_try_cmpxchg_relaxed)
236#define raw_try_cmpxchg_relaxed arch_try_cmpxchg_relaxed
237#elif defined(arch_try_cmpxchg)
238#define raw_try_cmpxchg_relaxed arch_try_cmpxchg
239#else
240#define raw_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
241({ \
242 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
243 ___r = raw_cmpxchg_relaxed((_ptr), ___o, (_new)); \
244 if (unlikely(___r != ___o)) \
245 *___op = ___r; \
246 likely(___r == ___o); \
247})
248#endif
249
250#if defined(arch_try_cmpxchg64)
251#define raw_try_cmpxchg64 arch_try_cmpxchg64
252#elif defined(arch_try_cmpxchg64_relaxed)
253#define raw_try_cmpxchg64(...) \
254 __atomic_op_fence(arch_try_cmpxchg64, __VA_ARGS__)
255#else
256#define raw_try_cmpxchg64(_ptr, _oldp, _new) \
257({ \
258 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
259 ___r = raw_cmpxchg64((_ptr), ___o, (_new)); \
260 if (unlikely(___r != ___o)) \
261 *___op = ___r; \
262 likely(___r == ___o); \
263})
264#endif
265
266#if defined(arch_try_cmpxchg64_acquire)
267#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64_acquire
268#elif defined(arch_try_cmpxchg64_relaxed)
269#define raw_try_cmpxchg64_acquire(...) \
270 __atomic_op_acquire(arch_try_cmpxchg64, __VA_ARGS__)
271#elif defined(arch_try_cmpxchg64)
272#define raw_try_cmpxchg64_acquire arch_try_cmpxchg64
273#else
274#define raw_try_cmpxchg64_acquire(_ptr, _oldp, _new) \
275({ \
276 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
277 ___r = raw_cmpxchg64_acquire((_ptr), ___o, (_new)); \
278 if (unlikely(___r != ___o)) \
279 *___op = ___r; \
280 likely(___r == ___o); \
281})
282#endif
283
284#if defined(arch_try_cmpxchg64_release)
285#define raw_try_cmpxchg64_release arch_try_cmpxchg64_release
286#elif defined(arch_try_cmpxchg64_relaxed)
287#define raw_try_cmpxchg64_release(...) \
288 __atomic_op_release(arch_try_cmpxchg64, __VA_ARGS__)
289#elif defined(arch_try_cmpxchg64)
290#define raw_try_cmpxchg64_release arch_try_cmpxchg64
291#else
292#define raw_try_cmpxchg64_release(_ptr, _oldp, _new) \
293({ \
294 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
295 ___r = raw_cmpxchg64_release((_ptr), ___o, (_new)); \
296 if (unlikely(___r != ___o)) \
297 *___op = ___r; \
298 likely(___r == ___o); \
299})
300#endif
301
302#if defined(arch_try_cmpxchg64_relaxed)
303#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64_relaxed
304#elif defined(arch_try_cmpxchg64)
305#define raw_try_cmpxchg64_relaxed arch_try_cmpxchg64
306#else
307#define raw_try_cmpxchg64_relaxed(_ptr, _oldp, _new) \
308({ \
309 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
310 ___r = raw_cmpxchg64_relaxed((_ptr), ___o, (_new)); \
311 if (unlikely(___r != ___o)) \
312 *___op = ___r; \
313 likely(___r == ___o); \
314})
315#endif
316
317#if defined(arch_try_cmpxchg128)
318#define raw_try_cmpxchg128 arch_try_cmpxchg128
319#elif defined(arch_try_cmpxchg128_relaxed)
320#define raw_try_cmpxchg128(...) \
321 __atomic_op_fence(arch_try_cmpxchg128, __VA_ARGS__)
322#else
323#define raw_try_cmpxchg128(_ptr, _oldp, _new) \
324({ \
325 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
326 ___r = raw_cmpxchg128((_ptr), ___o, (_new)); \
327 if (unlikely(___r != ___o)) \
328 *___op = ___r; \
329 likely(___r == ___o); \
330})
331#endif
332
333#if defined(arch_try_cmpxchg128_acquire)
334#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128_acquire
335#elif defined(arch_try_cmpxchg128_relaxed)
336#define raw_try_cmpxchg128_acquire(...) \
337 __atomic_op_acquire(arch_try_cmpxchg128, __VA_ARGS__)
338#elif defined(arch_try_cmpxchg128)
339#define raw_try_cmpxchg128_acquire arch_try_cmpxchg128
340#else
341#define raw_try_cmpxchg128_acquire(_ptr, _oldp, _new) \
342({ \
343 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
344 ___r = raw_cmpxchg128_acquire((_ptr), ___o, (_new)); \
345 if (unlikely(___r != ___o)) \
346 *___op = ___r; \
347 likely(___r == ___o); \
348})
349#endif
350
351#if defined(arch_try_cmpxchg128_release)
352#define raw_try_cmpxchg128_release arch_try_cmpxchg128_release
353#elif defined(arch_try_cmpxchg128_relaxed)
354#define raw_try_cmpxchg128_release(...) \
355 __atomic_op_release(arch_try_cmpxchg128, __VA_ARGS__)
356#elif defined(arch_try_cmpxchg128)
357#define raw_try_cmpxchg128_release arch_try_cmpxchg128
358#else
359#define raw_try_cmpxchg128_release(_ptr, _oldp, _new) \
360({ \
361 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
362 ___r = raw_cmpxchg128_release((_ptr), ___o, (_new)); \
363 if (unlikely(___r != ___o)) \
364 *___op = ___r; \
365 likely(___r == ___o); \
366})
367#endif
368
369#if defined(arch_try_cmpxchg128_relaxed)
370#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128_relaxed
371#elif defined(arch_try_cmpxchg128)
372#define raw_try_cmpxchg128_relaxed arch_try_cmpxchg128
373#else
374#define raw_try_cmpxchg128_relaxed(_ptr, _oldp, _new) \
375({ \
376 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
377 ___r = raw_cmpxchg128_relaxed((_ptr), ___o, (_new)); \
378 if (unlikely(___r != ___o)) \
379 *___op = ___r; \
380 likely(___r == ___o); \
381})
382#endif
383
384#define raw_cmpxchg_local arch_cmpxchg_local
385
386#ifdef arch_try_cmpxchg_local
387#define raw_try_cmpxchg_local arch_try_cmpxchg_local
388#else
389#define raw_try_cmpxchg_local(_ptr, _oldp, _new) \
390({ \
391 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
392 ___r = raw_cmpxchg_local((_ptr), ___o, (_new)); \
393 if (unlikely(___r != ___o)) \
394 *___op = ___r; \
395 likely(___r == ___o); \
396})
397#endif
398
399#define raw_cmpxchg64_local arch_cmpxchg64_local
400
401#ifdef arch_try_cmpxchg64_local
402#define raw_try_cmpxchg64_local arch_try_cmpxchg64_local
403#else
404#define raw_try_cmpxchg64_local(_ptr, _oldp, _new) \
405({ \
406 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
407 ___r = raw_cmpxchg64_local((_ptr), ___o, (_new)); \
408 if (unlikely(___r != ___o)) \
409 *___op = ___r; \
410 likely(___r == ___o); \
411})
412#endif
413
414#define raw_cmpxchg128_local arch_cmpxchg128_local
415
416#ifdef arch_try_cmpxchg128_local
417#define raw_try_cmpxchg128_local arch_try_cmpxchg128_local
418#else
419#define raw_try_cmpxchg128_local(_ptr, _oldp, _new) \
420({ \
421 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
422 ___r = raw_cmpxchg128_local((_ptr), ___o, (_new)); \
423 if (unlikely(___r != ___o)) \
424 *___op = ___r; \
425 likely(___r == ___o); \
426})
427#endif
428
429#define raw_sync_cmpxchg arch_sync_cmpxchg
430
431#ifdef arch_sync_try_cmpxchg
432#define raw_sync_try_cmpxchg arch_sync_try_cmpxchg
433#else
434#define raw_sync_try_cmpxchg(_ptr, _oldp, _new) \
435({ \
436 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
437 ___r = raw_sync_cmpxchg((_ptr), ___o, (_new)); \
438 if (unlikely(___r != ___o)) \
439 *___op = ___r; \
440 likely(___r == ___o); \
441})
442#endif
443
444/**
445 * raw_atomic_read() - atomic load with relaxed ordering
446 * @v: pointer to atomic_t
447 *
448 * Atomically loads the value of @v with relaxed ordering.
449 *
450 * Safe to use in noinstr code; prefer atomic_read() elsewhere.
451 *
452 * Return: The value loaded from @v.
453 */
454static __always_inline int
455raw_atomic_read(const atomic_t *v)
456{
457 return arch_atomic_read(v);
458}
459
460/**
461 * raw_atomic_read_acquire() - atomic load with acquire ordering
462 * @v: pointer to atomic_t
463 *
464 * Atomically loads the value of @v with acquire ordering.
465 *
466 * Safe to use in noinstr code; prefer atomic_read_acquire() elsewhere.
467 *
468 * Return: The value loaded from @v.
469 */
470static __always_inline int
471raw_atomic_read_acquire(const atomic_t *v)
472{
473#if defined(arch_atomic_read_acquire)
474 return arch_atomic_read_acquire(v);
475#else
476 int ret;
477
478 if (__native_word(atomic_t)) {
479 ret = smp_load_acquire(&(v)->counter);
480 } else {
481 ret = raw_atomic_read(v);
482 __atomic_acquire_fence();
483 }
484
485 return ret;
486#endif
487}
488
489/**
490 * raw_atomic_set() - atomic set with relaxed ordering
491 * @v: pointer to atomic_t
492 * @i: int value to assign
493 *
494 * Atomically sets @v to @i with relaxed ordering.
495 *
496 * Safe to use in noinstr code; prefer atomic_set() elsewhere.
497 *
498 * Return: Nothing.
499 */
500static __always_inline void
501raw_atomic_set(atomic_t *v, int i)
502{
503 arch_atomic_set(v, i);
504}
505
506/**
507 * raw_atomic_set_release() - atomic set with release ordering
508 * @v: pointer to atomic_t
509 * @i: int value to assign
510 *
511 * Atomically sets @v to @i with release ordering.
512 *
513 * Safe to use in noinstr code; prefer atomic_set_release() elsewhere.
514 *
515 * Return: Nothing.
516 */
517static __always_inline void
518raw_atomic_set_release(atomic_t *v, int i)
519{
520#if defined(arch_atomic_set_release)
521 arch_atomic_set_release(v, i);
522#else
523 if (__native_word(atomic_t)) {
524 smp_store_release(&(v)->counter, i);
525 } else {
526 __atomic_release_fence();
527 raw_atomic_set(v, i);
528 }
529#endif
530}
531
532/**
533 * raw_atomic_add() - atomic add with relaxed ordering
534 * @i: int value to add
535 * @v: pointer to atomic_t
536 *
537 * Atomically updates @v to (@v + @i) with relaxed ordering.
538 *
539 * Safe to use in noinstr code; prefer atomic_add() elsewhere.
540 *
541 * Return: Nothing.
542 */
543static __always_inline void
544raw_atomic_add(int i, atomic_t *v)
545{
546 arch_atomic_add(i, v);
547}
548
549/**
550 * raw_atomic_add_return() - atomic add with full ordering
551 * @i: int value to add
552 * @v: pointer to atomic_t
553 *
554 * Atomically updates @v to (@v + @i) with full ordering.
555 *
556 * Safe to use in noinstr code; prefer atomic_add_return() elsewhere.
557 *
558 * Return: The updated value of @v.
559 */
560static __always_inline int
561raw_atomic_add_return(int i, atomic_t *v)
562{
563#if defined(arch_atomic_add_return)
564 return arch_atomic_add_return(i, v);
565#elif defined(arch_atomic_add_return_relaxed)
566 int ret;
567 __atomic_pre_full_fence();
568 ret = arch_atomic_add_return_relaxed(i, v);
569 __atomic_post_full_fence();
570 return ret;
571#else
572#error "Unable to define raw_atomic_add_return"
573#endif
574}
575
576/**
577 * raw_atomic_add_return_acquire() - atomic add with acquire ordering
578 * @i: int value to add
579 * @v: pointer to atomic_t
580 *
581 * Atomically updates @v to (@v + @i) with acquire ordering.
582 *
583 * Safe to use in noinstr code; prefer atomic_add_return_acquire() elsewhere.
584 *
585 * Return: The updated value of @v.
586 */
587static __always_inline int
588raw_atomic_add_return_acquire(int i, atomic_t *v)
589{
590#if defined(arch_atomic_add_return_acquire)
591 return arch_atomic_add_return_acquire(i, v);
592#elif defined(arch_atomic_add_return_relaxed)
593 int ret = arch_atomic_add_return_relaxed(i, v);
594 __atomic_acquire_fence();
595 return ret;
596#elif defined(arch_atomic_add_return)
597 return arch_atomic_add_return(i, v);
598#else
599#error "Unable to define raw_atomic_add_return_acquire"
600#endif
601}
602
603/**
604 * raw_atomic_add_return_release() - atomic add with release ordering
605 * @i: int value to add
606 * @v: pointer to atomic_t
607 *
608 * Atomically updates @v to (@v + @i) with release ordering.
609 *
610 * Safe to use in noinstr code; prefer atomic_add_return_release() elsewhere.
611 *
612 * Return: The updated value of @v.
613 */
614static __always_inline int
615raw_atomic_add_return_release(int i, atomic_t *v)
616{
617#if defined(arch_atomic_add_return_release)
618 return arch_atomic_add_return_release(i, v);
619#elif defined(arch_atomic_add_return_relaxed)
620 __atomic_release_fence();
621 return arch_atomic_add_return_relaxed(i, v);
622#elif defined(arch_atomic_add_return)
623 return arch_atomic_add_return(i, v);
624#else
625#error "Unable to define raw_atomic_add_return_release"
626#endif
627}
628
629/**
630 * raw_atomic_add_return_relaxed() - atomic add with relaxed ordering
631 * @i: int value to add
632 * @v: pointer to atomic_t
633 *
634 * Atomically updates @v to (@v + @i) with relaxed ordering.
635 *
636 * Safe to use in noinstr code; prefer atomic_add_return_relaxed() elsewhere.
637 *
638 * Return: The updated value of @v.
639 */
640static __always_inline int
641raw_atomic_add_return_relaxed(int i, atomic_t *v)
642{
643#if defined(arch_atomic_add_return_relaxed)
644 return arch_atomic_add_return_relaxed(i, v);
645#elif defined(arch_atomic_add_return)
646 return arch_atomic_add_return(i, v);
647#else
648#error "Unable to define raw_atomic_add_return_relaxed"
649#endif
650}
651
652/**
653 * raw_atomic_fetch_add() - atomic add with full ordering
654 * @i: int value to add
655 * @v: pointer to atomic_t
656 *
657 * Atomically updates @v to (@v + @i) with full ordering.
658 *
659 * Safe to use in noinstr code; prefer atomic_fetch_add() elsewhere.
660 *
661 * Return: The original value of @v.
662 */
663static __always_inline int
664raw_atomic_fetch_add(int i, atomic_t *v)
665{
666#if defined(arch_atomic_fetch_add)
667 return arch_atomic_fetch_add(i, v);
668#elif defined(arch_atomic_fetch_add_relaxed)
669 int ret;
670 __atomic_pre_full_fence();
671 ret = arch_atomic_fetch_add_relaxed(i, v);
672 __atomic_post_full_fence();
673 return ret;
674#else
675#error "Unable to define raw_atomic_fetch_add"
676#endif
677}
678
679/**
680 * raw_atomic_fetch_add_acquire() - atomic add with acquire ordering
681 * @i: int value to add
682 * @v: pointer to atomic_t
683 *
684 * Atomically updates @v to (@v + @i) with acquire ordering.
685 *
686 * Safe to use in noinstr code; prefer atomic_fetch_add_acquire() elsewhere.
687 *
688 * Return: The original value of @v.
689 */
690static __always_inline int
691raw_atomic_fetch_add_acquire(int i, atomic_t *v)
692{
693#if defined(arch_atomic_fetch_add_acquire)
694 return arch_atomic_fetch_add_acquire(i, v);
695#elif defined(arch_atomic_fetch_add_relaxed)
696 int ret = arch_atomic_fetch_add_relaxed(i, v);
697 __atomic_acquire_fence();
698 return ret;
699#elif defined(arch_atomic_fetch_add)
700 return arch_atomic_fetch_add(i, v);
701#else
702#error "Unable to define raw_atomic_fetch_add_acquire"
703#endif
704}
705
706/**
707 * raw_atomic_fetch_add_release() - atomic add with release ordering
708 * @i: int value to add
709 * @v: pointer to atomic_t
710 *
711 * Atomically updates @v to (@v + @i) with release ordering.
712 *
713 * Safe to use in noinstr code; prefer atomic_fetch_add_release() elsewhere.
714 *
715 * Return: The original value of @v.
716 */
717static __always_inline int
718raw_atomic_fetch_add_release(int i, atomic_t *v)
719{
720#if defined(arch_atomic_fetch_add_release)
721 return arch_atomic_fetch_add_release(i, v);
722#elif defined(arch_atomic_fetch_add_relaxed)
723 __atomic_release_fence();
724 return arch_atomic_fetch_add_relaxed(i, v);
725#elif defined(arch_atomic_fetch_add)
726 return arch_atomic_fetch_add(i, v);
727#else
728#error "Unable to define raw_atomic_fetch_add_release"
729#endif
730}
731
732/**
733 * raw_atomic_fetch_add_relaxed() - atomic add with relaxed ordering
734 * @i: int value to add
735 * @v: pointer to atomic_t
736 *
737 * Atomically updates @v to (@v + @i) with relaxed ordering.
738 *
739 * Safe to use in noinstr code; prefer atomic_fetch_add_relaxed() elsewhere.
740 *
741 * Return: The original value of @v.
742 */
743static __always_inline int
744raw_atomic_fetch_add_relaxed(int i, atomic_t *v)
745{
746#if defined(arch_atomic_fetch_add_relaxed)
747 return arch_atomic_fetch_add_relaxed(i, v);
748#elif defined(arch_atomic_fetch_add)
749 return arch_atomic_fetch_add(i, v);
750#else
751#error "Unable to define raw_atomic_fetch_add_relaxed"
752#endif
753}
754
755/**
756 * raw_atomic_sub() - atomic subtract with relaxed ordering
757 * @i: int value to subtract
758 * @v: pointer to atomic_t
759 *
760 * Atomically updates @v to (@v - @i) with relaxed ordering.
761 *
762 * Safe to use in noinstr code; prefer atomic_sub() elsewhere.
763 *
764 * Return: Nothing.
765 */
766static __always_inline void
767raw_atomic_sub(int i, atomic_t *v)
768{
769 arch_atomic_sub(i, v);
770}
771
772/**
773 * raw_atomic_sub_return() - atomic subtract with full ordering
774 * @i: int value to subtract
775 * @v: pointer to atomic_t
776 *
777 * Atomically updates @v to (@v - @i) with full ordering.
778 *
779 * Safe to use in noinstr code; prefer atomic_sub_return() elsewhere.
780 *
781 * Return: The updated value of @v.
782 */
783static __always_inline int
784raw_atomic_sub_return(int i, atomic_t *v)
785{
786#if defined(arch_atomic_sub_return)
787 return arch_atomic_sub_return(i, v);
788#elif defined(arch_atomic_sub_return_relaxed)
789 int ret;
790 __atomic_pre_full_fence();
791 ret = arch_atomic_sub_return_relaxed(i, v);
792 __atomic_post_full_fence();
793 return ret;
794#else
795#error "Unable to define raw_atomic_sub_return"
796#endif
797}
798
799/**
800 * raw_atomic_sub_return_acquire() - atomic subtract with acquire ordering
801 * @i: int value to subtract
802 * @v: pointer to atomic_t
803 *
804 * Atomically updates @v to (@v - @i) with acquire ordering.
805 *
806 * Safe to use in noinstr code; prefer atomic_sub_return_acquire() elsewhere.
807 *
808 * Return: The updated value of @v.
809 */
810static __always_inline int
811raw_atomic_sub_return_acquire(int i, atomic_t *v)
812{
813#if defined(arch_atomic_sub_return_acquire)
814 return arch_atomic_sub_return_acquire(i, v);
815#elif defined(arch_atomic_sub_return_relaxed)
816 int ret = arch_atomic_sub_return_relaxed(i, v);
817 __atomic_acquire_fence();
818 return ret;
819#elif defined(arch_atomic_sub_return)
820 return arch_atomic_sub_return(i, v);
821#else
822#error "Unable to define raw_atomic_sub_return_acquire"
823#endif
824}
825
826/**
827 * raw_atomic_sub_return_release() - atomic subtract with release ordering
828 * @i: int value to subtract
829 * @v: pointer to atomic_t
830 *
831 * Atomically updates @v to (@v - @i) with release ordering.
832 *
833 * Safe to use in noinstr code; prefer atomic_sub_return_release() elsewhere.
834 *
835 * Return: The updated value of @v.
836 */
837static __always_inline int
838raw_atomic_sub_return_release(int i, atomic_t *v)
839{
840#if defined(arch_atomic_sub_return_release)
841 return arch_atomic_sub_return_release(i, v);
842#elif defined(arch_atomic_sub_return_relaxed)
843 __atomic_release_fence();
844 return arch_atomic_sub_return_relaxed(i, v);
845#elif defined(arch_atomic_sub_return)
846 return arch_atomic_sub_return(i, v);
847#else
848#error "Unable to define raw_atomic_sub_return_release"
849#endif
850}
851
852/**
853 * raw_atomic_sub_return_relaxed() - atomic subtract with relaxed ordering
854 * @i: int value to subtract
855 * @v: pointer to atomic_t
856 *
857 * Atomically updates @v to (@v - @i) with relaxed ordering.
858 *
859 * Safe to use in noinstr code; prefer atomic_sub_return_relaxed() elsewhere.
860 *
861 * Return: The updated value of @v.
862 */
863static __always_inline int
864raw_atomic_sub_return_relaxed(int i, atomic_t *v)
865{
866#if defined(arch_atomic_sub_return_relaxed)
867 return arch_atomic_sub_return_relaxed(i, v);
868#elif defined(arch_atomic_sub_return)
869 return arch_atomic_sub_return(i, v);
870#else
871#error "Unable to define raw_atomic_sub_return_relaxed"
872#endif
873}
874
875/**
876 * raw_atomic_fetch_sub() - atomic subtract with full ordering
877 * @i: int value to subtract
878 * @v: pointer to atomic_t
879 *
880 * Atomically updates @v to (@v - @i) with full ordering.
881 *
882 * Safe to use in noinstr code; prefer atomic_fetch_sub() elsewhere.
883 *
884 * Return: The original value of @v.
885 */
886static __always_inline int
887raw_atomic_fetch_sub(int i, atomic_t *v)
888{
889#if defined(arch_atomic_fetch_sub)
890 return arch_atomic_fetch_sub(i, v);
891#elif defined(arch_atomic_fetch_sub_relaxed)
892 int ret;
893 __atomic_pre_full_fence();
894 ret = arch_atomic_fetch_sub_relaxed(i, v);
895 __atomic_post_full_fence();
896 return ret;
897#else
898#error "Unable to define raw_atomic_fetch_sub"
899#endif
900}
901
902/**
903 * raw_atomic_fetch_sub_acquire() - atomic subtract with acquire ordering
904 * @i: int value to subtract
905 * @v: pointer to atomic_t
906 *
907 * Atomically updates @v to (@v - @i) with acquire ordering.
908 *
909 * Safe to use in noinstr code; prefer atomic_fetch_sub_acquire() elsewhere.
910 *
911 * Return: The original value of @v.
912 */
913static __always_inline int
914raw_atomic_fetch_sub_acquire(int i, atomic_t *v)
915{
916#if defined(arch_atomic_fetch_sub_acquire)
917 return arch_atomic_fetch_sub_acquire(i, v);
918#elif defined(arch_atomic_fetch_sub_relaxed)
919 int ret = arch_atomic_fetch_sub_relaxed(i, v);
920 __atomic_acquire_fence();
921 return ret;
922#elif defined(arch_atomic_fetch_sub)
923 return arch_atomic_fetch_sub(i, v);
924#else
925#error "Unable to define raw_atomic_fetch_sub_acquire"
926#endif
927}
928
929/**
930 * raw_atomic_fetch_sub_release() - atomic subtract with release ordering
931 * @i: int value to subtract
932 * @v: pointer to atomic_t
933 *
934 * Atomically updates @v to (@v - @i) with release ordering.
935 *
936 * Safe to use in noinstr code; prefer atomic_fetch_sub_release() elsewhere.
937 *
938 * Return: The original value of @v.
939 */
940static __always_inline int
941raw_atomic_fetch_sub_release(int i, atomic_t *v)
942{
943#if defined(arch_atomic_fetch_sub_release)
944 return arch_atomic_fetch_sub_release(i, v);
945#elif defined(arch_atomic_fetch_sub_relaxed)
946 __atomic_release_fence();
947 return arch_atomic_fetch_sub_relaxed(i, v);
948#elif defined(arch_atomic_fetch_sub)
949 return arch_atomic_fetch_sub(i, v);
950#else
951#error "Unable to define raw_atomic_fetch_sub_release"
952#endif
953}
954
955/**
956 * raw_atomic_fetch_sub_relaxed() - atomic subtract with relaxed ordering
957 * @i: int value to subtract
958 * @v: pointer to atomic_t
959 *
960 * Atomically updates @v to (@v - @i) with relaxed ordering.
961 *
962 * Safe to use in noinstr code; prefer atomic_fetch_sub_relaxed() elsewhere.
963 *
964 * Return: The original value of @v.
965 */
966static __always_inline int
967raw_atomic_fetch_sub_relaxed(int i, atomic_t *v)
968{
969#if defined(arch_atomic_fetch_sub_relaxed)
970 return arch_atomic_fetch_sub_relaxed(i, v);
971#elif defined(arch_atomic_fetch_sub)
972 return arch_atomic_fetch_sub(i, v);
973#else
974#error "Unable to define raw_atomic_fetch_sub_relaxed"
975#endif
976}
977
978/**
979 * raw_atomic_inc() - atomic increment with relaxed ordering
980 * @v: pointer to atomic_t
981 *
982 * Atomically updates @v to (@v + 1) with relaxed ordering.
983 *
984 * Safe to use in noinstr code; prefer atomic_inc() elsewhere.
985 *
986 * Return: Nothing.
987 */
988static __always_inline void
989raw_atomic_inc(atomic_t *v)
990{
991#if defined(arch_atomic_inc)
992 arch_atomic_inc(v);
993#else
994 raw_atomic_add(1, v);
995#endif
996}
997
998/**
999 * raw_atomic_inc_return() - atomic increment with full ordering
1000 * @v: pointer to atomic_t
1001 *
1002 * Atomically updates @v to (@v + 1) with full ordering.
1003 *
1004 * Safe to use in noinstr code; prefer atomic_inc_return() elsewhere.
1005 *
1006 * Return: The updated value of @v.
1007 */
1008static __always_inline int
1009raw_atomic_inc_return(atomic_t *v)
1010{
1011#if defined(arch_atomic_inc_return)
1012 return arch_atomic_inc_return(v);
1013#elif defined(arch_atomic_inc_return_relaxed)
1014 int ret;
1015 __atomic_pre_full_fence();
1016 ret = arch_atomic_inc_return_relaxed(v);
1017 __atomic_post_full_fence();
1018 return ret;
1019#else
1020 return raw_atomic_add_return(1, v);
1021#endif
1022}
1023
1024/**
1025 * raw_atomic_inc_return_acquire() - atomic increment with acquire ordering
1026 * @v: pointer to atomic_t
1027 *
1028 * Atomically updates @v to (@v + 1) with acquire ordering.
1029 *
1030 * Safe to use in noinstr code; prefer atomic_inc_return_acquire() elsewhere.
1031 *
1032 * Return: The updated value of @v.
1033 */
1034static __always_inline int
1035raw_atomic_inc_return_acquire(atomic_t *v)
1036{
1037#if defined(arch_atomic_inc_return_acquire)
1038 return arch_atomic_inc_return_acquire(v);
1039#elif defined(arch_atomic_inc_return_relaxed)
1040 int ret = arch_atomic_inc_return_relaxed(v);
1041 __atomic_acquire_fence();
1042 return ret;
1043#elif defined(arch_atomic_inc_return)
1044 return arch_atomic_inc_return(v);
1045#else
1046 return raw_atomic_add_return_acquire(1, v);
1047#endif
1048}
1049
1050/**
1051 * raw_atomic_inc_return_release() - atomic increment with release ordering
1052 * @v: pointer to atomic_t
1053 *
1054 * Atomically updates @v to (@v + 1) with release ordering.
1055 *
1056 * Safe to use in noinstr code; prefer atomic_inc_return_release() elsewhere.
1057 *
1058 * Return: The updated value of @v.
1059 */
1060static __always_inline int
1061raw_atomic_inc_return_release(atomic_t *v)
1062{
1063#if defined(arch_atomic_inc_return_release)
1064 return arch_atomic_inc_return_release(v);
1065#elif defined(arch_atomic_inc_return_relaxed)
1066 __atomic_release_fence();
1067 return arch_atomic_inc_return_relaxed(v);
1068#elif defined(arch_atomic_inc_return)
1069 return arch_atomic_inc_return(v);
1070#else
1071 return raw_atomic_add_return_release(1, v);
1072#endif
1073}
1074
1075/**
1076 * raw_atomic_inc_return_relaxed() - atomic increment with relaxed ordering
1077 * @v: pointer to atomic_t
1078 *
1079 * Atomically updates @v to (@v + 1) with relaxed ordering.
1080 *
1081 * Safe to use in noinstr code; prefer atomic_inc_return_relaxed() elsewhere.
1082 *
1083 * Return: The updated value of @v.
1084 */
1085static __always_inline int
1086raw_atomic_inc_return_relaxed(atomic_t *v)
1087{
1088#if defined(arch_atomic_inc_return_relaxed)
1089 return arch_atomic_inc_return_relaxed(v);
1090#elif defined(arch_atomic_inc_return)
1091 return arch_atomic_inc_return(v);
1092#else
1093 return raw_atomic_add_return_relaxed(1, v);
1094#endif
1095}
1096
1097/**
1098 * raw_atomic_fetch_inc() - atomic increment with full ordering
1099 * @v: pointer to atomic_t
1100 *
1101 * Atomically updates @v to (@v + 1) with full ordering.
1102 *
1103 * Safe to use in noinstr code; prefer atomic_fetch_inc() elsewhere.
1104 *
1105 * Return: The original value of @v.
1106 */
1107static __always_inline int
1108raw_atomic_fetch_inc(atomic_t *v)
1109{
1110#if defined(arch_atomic_fetch_inc)
1111 return arch_atomic_fetch_inc(v);
1112#elif defined(arch_atomic_fetch_inc_relaxed)
1113 int ret;
1114 __atomic_pre_full_fence();
1115 ret = arch_atomic_fetch_inc_relaxed(v);
1116 __atomic_post_full_fence();
1117 return ret;
1118#else
1119 return raw_atomic_fetch_add(1, v);
1120#endif
1121}
1122
1123/**
1124 * raw_atomic_fetch_inc_acquire() - atomic increment with acquire ordering
1125 * @v: pointer to atomic_t
1126 *
1127 * Atomically updates @v to (@v + 1) with acquire ordering.
1128 *
1129 * Safe to use in noinstr code; prefer atomic_fetch_inc_acquire() elsewhere.
1130 *
1131 * Return: The original value of @v.
1132 */
1133static __always_inline int
1134raw_atomic_fetch_inc_acquire(atomic_t *v)
1135{
1136#if defined(arch_atomic_fetch_inc_acquire)
1137 return arch_atomic_fetch_inc_acquire(v);
1138#elif defined(arch_atomic_fetch_inc_relaxed)
1139 int ret = arch_atomic_fetch_inc_relaxed(v);
1140 __atomic_acquire_fence();
1141 return ret;
1142#elif defined(arch_atomic_fetch_inc)
1143 return arch_atomic_fetch_inc(v);
1144#else
1145 return raw_atomic_fetch_add_acquire(1, v);
1146#endif
1147}
1148
1149/**
1150 * raw_atomic_fetch_inc_release() - atomic increment with release ordering
1151 * @v: pointer to atomic_t
1152 *
1153 * Atomically updates @v to (@v + 1) with release ordering.
1154 *
1155 * Safe to use in noinstr code; prefer atomic_fetch_inc_release() elsewhere.
1156 *
1157 * Return: The original value of @v.
1158 */
1159static __always_inline int
1160raw_atomic_fetch_inc_release(atomic_t *v)
1161{
1162#if defined(arch_atomic_fetch_inc_release)
1163 return arch_atomic_fetch_inc_release(v);
1164#elif defined(arch_atomic_fetch_inc_relaxed)
1165 __atomic_release_fence();
1166 return arch_atomic_fetch_inc_relaxed(v);
1167#elif defined(arch_atomic_fetch_inc)
1168 return arch_atomic_fetch_inc(v);
1169#else
1170 return raw_atomic_fetch_add_release(1, v);
1171#endif
1172}
1173
1174/**
1175 * raw_atomic_fetch_inc_relaxed() - atomic increment with relaxed ordering
1176 * @v: pointer to atomic_t
1177 *
1178 * Atomically updates @v to (@v + 1) with relaxed ordering.
1179 *
1180 * Safe to use in noinstr code; prefer atomic_fetch_inc_relaxed() elsewhere.
1181 *
1182 * Return: The original value of @v.
1183 */
1184static __always_inline int
1185raw_atomic_fetch_inc_relaxed(atomic_t *v)
1186{
1187#if defined(arch_atomic_fetch_inc_relaxed)
1188 return arch_atomic_fetch_inc_relaxed(v);
1189#elif defined(arch_atomic_fetch_inc)
1190 return arch_atomic_fetch_inc(v);
1191#else
1192 return raw_atomic_fetch_add_relaxed(1, v);
1193#endif
1194}
1195
1196/**
1197 * raw_atomic_dec() - atomic decrement with relaxed ordering
1198 * @v: pointer to atomic_t
1199 *
1200 * Atomically updates @v to (@v - 1) with relaxed ordering.
1201 *
1202 * Safe to use in noinstr code; prefer atomic_dec() elsewhere.
1203 *
1204 * Return: Nothing.
1205 */
1206static __always_inline void
1207raw_atomic_dec(atomic_t *v)
1208{
1209#if defined(arch_atomic_dec)
1210 arch_atomic_dec(v);
1211#else
1212 raw_atomic_sub(1, v);
1213#endif
1214}
1215
1216/**
1217 * raw_atomic_dec_return() - atomic decrement with full ordering
1218 * @v: pointer to atomic_t
1219 *
1220 * Atomically updates @v to (@v - 1) with full ordering.
1221 *
1222 * Safe to use in noinstr code; prefer atomic_dec_return() elsewhere.
1223 *
1224 * Return: The updated value of @v.
1225 */
1226static __always_inline int
1227raw_atomic_dec_return(atomic_t *v)
1228{
1229#if defined(arch_atomic_dec_return)
1230 return arch_atomic_dec_return(v);
1231#elif defined(arch_atomic_dec_return_relaxed)
1232 int ret;
1233 __atomic_pre_full_fence();
1234 ret = arch_atomic_dec_return_relaxed(v);
1235 __atomic_post_full_fence();
1236 return ret;
1237#else
1238 return raw_atomic_sub_return(1, v);
1239#endif
1240}
1241
1242/**
1243 * raw_atomic_dec_return_acquire() - atomic decrement with acquire ordering
1244 * @v: pointer to atomic_t
1245 *
1246 * Atomically updates @v to (@v - 1) with acquire ordering.
1247 *
1248 * Safe to use in noinstr code; prefer atomic_dec_return_acquire() elsewhere.
1249 *
1250 * Return: The updated value of @v.
1251 */
1252static __always_inline int
1253raw_atomic_dec_return_acquire(atomic_t *v)
1254{
1255#if defined(arch_atomic_dec_return_acquire)
1256 return arch_atomic_dec_return_acquire(v);
1257#elif defined(arch_atomic_dec_return_relaxed)
1258 int ret = arch_atomic_dec_return_relaxed(v);
1259 __atomic_acquire_fence();
1260 return ret;
1261#elif defined(arch_atomic_dec_return)
1262 return arch_atomic_dec_return(v);
1263#else
1264 return raw_atomic_sub_return_acquire(1, v);
1265#endif
1266}
1267
1268/**
1269 * raw_atomic_dec_return_release() - atomic decrement with release ordering
1270 * @v: pointer to atomic_t
1271 *
1272 * Atomically updates @v to (@v - 1) with release ordering.
1273 *
1274 * Safe to use in noinstr code; prefer atomic_dec_return_release() elsewhere.
1275 *
1276 * Return: The updated value of @v.
1277 */
1278static __always_inline int
1279raw_atomic_dec_return_release(atomic_t *v)
1280{
1281#if defined(arch_atomic_dec_return_release)
1282 return arch_atomic_dec_return_release(v);
1283#elif defined(arch_atomic_dec_return_relaxed)
1284 __atomic_release_fence();
1285 return arch_atomic_dec_return_relaxed(v);
1286#elif defined(arch_atomic_dec_return)
1287 return arch_atomic_dec_return(v);
1288#else
1289 return raw_atomic_sub_return_release(1, v);
1290#endif
1291}
1292
1293/**
1294 * raw_atomic_dec_return_relaxed() - atomic decrement with relaxed ordering
1295 * @v: pointer to atomic_t
1296 *
1297 * Atomically updates @v to (@v - 1) with relaxed ordering.
1298 *
1299 * Safe to use in noinstr code; prefer atomic_dec_return_relaxed() elsewhere.
1300 *
1301 * Return: The updated value of @v.
1302 */
1303static __always_inline int
1304raw_atomic_dec_return_relaxed(atomic_t *v)
1305{
1306#if defined(arch_atomic_dec_return_relaxed)
1307 return arch_atomic_dec_return_relaxed(v);
1308#elif defined(arch_atomic_dec_return)
1309 return arch_atomic_dec_return(v);
1310#else
1311 return raw_atomic_sub_return_relaxed(1, v);
1312#endif
1313}
1314
1315/**
1316 * raw_atomic_fetch_dec() - atomic decrement with full ordering
1317 * @v: pointer to atomic_t
1318 *
1319 * Atomically updates @v to (@v - 1) with full ordering.
1320 *
1321 * Safe to use in noinstr code; prefer atomic_fetch_dec() elsewhere.
1322 *
1323 * Return: The original value of @v.
1324 */
1325static __always_inline int
1326raw_atomic_fetch_dec(atomic_t *v)
1327{
1328#if defined(arch_atomic_fetch_dec)
1329 return arch_atomic_fetch_dec(v);
1330#elif defined(arch_atomic_fetch_dec_relaxed)
1331 int ret;
1332 __atomic_pre_full_fence();
1333 ret = arch_atomic_fetch_dec_relaxed(v);
1334 __atomic_post_full_fence();
1335 return ret;
1336#else
1337 return raw_atomic_fetch_sub(1, v);
1338#endif
1339}
1340
1341/**
1342 * raw_atomic_fetch_dec_acquire() - atomic decrement with acquire ordering
1343 * @v: pointer to atomic_t
1344 *
1345 * Atomically updates @v to (@v - 1) with acquire ordering.
1346 *
1347 * Safe to use in noinstr code; prefer atomic_fetch_dec_acquire() elsewhere.
1348 *
1349 * Return: The original value of @v.
1350 */
1351static __always_inline int
1352raw_atomic_fetch_dec_acquire(atomic_t *v)
1353{
1354#if defined(arch_atomic_fetch_dec_acquire)
1355 return arch_atomic_fetch_dec_acquire(v);
1356#elif defined(arch_atomic_fetch_dec_relaxed)
1357 int ret = arch_atomic_fetch_dec_relaxed(v);
1358 __atomic_acquire_fence();
1359 return ret;
1360#elif defined(arch_atomic_fetch_dec)
1361 return arch_atomic_fetch_dec(v);
1362#else
1363 return raw_atomic_fetch_sub_acquire(1, v);
1364#endif
1365}
1366
1367/**
1368 * raw_atomic_fetch_dec_release() - atomic decrement with release ordering
1369 * @v: pointer to atomic_t
1370 *
1371 * Atomically updates @v to (@v - 1) with release ordering.
1372 *
1373 * Safe to use in noinstr code; prefer atomic_fetch_dec_release() elsewhere.
1374 *
1375 * Return: The original value of @v.
1376 */
1377static __always_inline int
1378raw_atomic_fetch_dec_release(atomic_t *v)
1379{
1380#if defined(arch_atomic_fetch_dec_release)
1381 return arch_atomic_fetch_dec_release(v);
1382#elif defined(arch_atomic_fetch_dec_relaxed)
1383 __atomic_release_fence();
1384 return arch_atomic_fetch_dec_relaxed(v);
1385#elif defined(arch_atomic_fetch_dec)
1386 return arch_atomic_fetch_dec(v);
1387#else
1388 return raw_atomic_fetch_sub_release(1, v);
1389#endif
1390}
1391
1392/**
1393 * raw_atomic_fetch_dec_relaxed() - atomic decrement with relaxed ordering
1394 * @v: pointer to atomic_t
1395 *
1396 * Atomically updates @v to (@v - 1) with relaxed ordering.
1397 *
1398 * Safe to use in noinstr code; prefer atomic_fetch_dec_relaxed() elsewhere.
1399 *
1400 * Return: The original value of @v.
1401 */
1402static __always_inline int
1403raw_atomic_fetch_dec_relaxed(atomic_t *v)
1404{
1405#if defined(arch_atomic_fetch_dec_relaxed)
1406 return arch_atomic_fetch_dec_relaxed(v);
1407#elif defined(arch_atomic_fetch_dec)
1408 return arch_atomic_fetch_dec(v);
1409#else
1410 return raw_atomic_fetch_sub_relaxed(1, v);
1411#endif
1412}
1413
1414/**
1415 * raw_atomic_and() - atomic bitwise AND with relaxed ordering
1416 * @i: int value
1417 * @v: pointer to atomic_t
1418 *
1419 * Atomically updates @v to (@v & @i) with relaxed ordering.
1420 *
1421 * Safe to use in noinstr code; prefer atomic_and() elsewhere.
1422 *
1423 * Return: Nothing.
1424 */
1425static __always_inline void
1426raw_atomic_and(int i, atomic_t *v)
1427{
1428 arch_atomic_and(i, v);
1429}
1430
1431/**
1432 * raw_atomic_fetch_and() - atomic bitwise AND with full ordering
1433 * @i: int value
1434 * @v: pointer to atomic_t
1435 *
1436 * Atomically updates @v to (@v & @i) with full ordering.
1437 *
1438 * Safe to use in noinstr code; prefer atomic_fetch_and() elsewhere.
1439 *
1440 * Return: The original value of @v.
1441 */
1442static __always_inline int
1443raw_atomic_fetch_and(int i, atomic_t *v)
1444{
1445#if defined(arch_atomic_fetch_and)
1446 return arch_atomic_fetch_and(i, v);
1447#elif defined(arch_atomic_fetch_and_relaxed)
1448 int ret;
1449 __atomic_pre_full_fence();
1450 ret = arch_atomic_fetch_and_relaxed(i, v);
1451 __atomic_post_full_fence();
1452 return ret;
1453#else
1454#error "Unable to define raw_atomic_fetch_and"
1455#endif
1456}
1457
1458/**
1459 * raw_atomic_fetch_and_acquire() - atomic bitwise AND with acquire ordering
1460 * @i: int value
1461 * @v: pointer to atomic_t
1462 *
1463 * Atomically updates @v to (@v & @i) with acquire ordering.
1464 *
1465 * Safe to use in noinstr code; prefer atomic_fetch_and_acquire() elsewhere.
1466 *
1467 * Return: The original value of @v.
1468 */
1469static __always_inline int
1470raw_atomic_fetch_and_acquire(int i, atomic_t *v)
1471{
1472#if defined(arch_atomic_fetch_and_acquire)
1473 return arch_atomic_fetch_and_acquire(i, v);
1474#elif defined(arch_atomic_fetch_and_relaxed)
1475 int ret = arch_atomic_fetch_and_relaxed(i, v);
1476 __atomic_acquire_fence();
1477 return ret;
1478#elif defined(arch_atomic_fetch_and)
1479 return arch_atomic_fetch_and(i, v);
1480#else
1481#error "Unable to define raw_atomic_fetch_and_acquire"
1482#endif
1483}
1484
1485/**
1486 * raw_atomic_fetch_and_release() - atomic bitwise AND with release ordering
1487 * @i: int value
1488 * @v: pointer to atomic_t
1489 *
1490 * Atomically updates @v to (@v & @i) with release ordering.
1491 *
1492 * Safe to use in noinstr code; prefer atomic_fetch_and_release() elsewhere.
1493 *
1494 * Return: The original value of @v.
1495 */
1496static __always_inline int
1497raw_atomic_fetch_and_release(int i, atomic_t *v)
1498{
1499#if defined(arch_atomic_fetch_and_release)
1500 return arch_atomic_fetch_and_release(i, v);
1501#elif defined(arch_atomic_fetch_and_relaxed)
1502 __atomic_release_fence();
1503 return arch_atomic_fetch_and_relaxed(i, v);
1504#elif defined(arch_atomic_fetch_and)
1505 return arch_atomic_fetch_and(i, v);
1506#else
1507#error "Unable to define raw_atomic_fetch_and_release"
1508#endif
1509}
1510
1511/**
1512 * raw_atomic_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
1513 * @i: int value
1514 * @v: pointer to atomic_t
1515 *
1516 * Atomically updates @v to (@v & @i) with relaxed ordering.
1517 *
1518 * Safe to use in noinstr code; prefer atomic_fetch_and_relaxed() elsewhere.
1519 *
1520 * Return: The original value of @v.
1521 */
1522static __always_inline int
1523raw_atomic_fetch_and_relaxed(int i, atomic_t *v)
1524{
1525#if defined(arch_atomic_fetch_and_relaxed)
1526 return arch_atomic_fetch_and_relaxed(i, v);
1527#elif defined(arch_atomic_fetch_and)
1528 return arch_atomic_fetch_and(i, v);
1529#else
1530#error "Unable to define raw_atomic_fetch_and_relaxed"
1531#endif
1532}
1533
1534/**
1535 * raw_atomic_andnot() - atomic bitwise AND NOT with relaxed ordering
1536 * @i: int value
1537 * @v: pointer to atomic_t
1538 *
1539 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1540 *
1541 * Safe to use in noinstr code; prefer atomic_andnot() elsewhere.
1542 *
1543 * Return: Nothing.
1544 */
1545static __always_inline void
1546raw_atomic_andnot(int i, atomic_t *v)
1547{
1548#if defined(arch_atomic_andnot)
1549 arch_atomic_andnot(i, v);
1550#else
1551 raw_atomic_and(~i, v);
1552#endif
1553}
1554
1555/**
1556 * raw_atomic_fetch_andnot() - atomic bitwise AND NOT with full ordering
1557 * @i: int value
1558 * @v: pointer to atomic_t
1559 *
1560 * Atomically updates @v to (@v & ~@i) with full ordering.
1561 *
1562 * Safe to use in noinstr code; prefer atomic_fetch_andnot() elsewhere.
1563 *
1564 * Return: The original value of @v.
1565 */
1566static __always_inline int
1567raw_atomic_fetch_andnot(int i, atomic_t *v)
1568{
1569#if defined(arch_atomic_fetch_andnot)
1570 return arch_atomic_fetch_andnot(i, v);
1571#elif defined(arch_atomic_fetch_andnot_relaxed)
1572 int ret;
1573 __atomic_pre_full_fence();
1574 ret = arch_atomic_fetch_andnot_relaxed(i, v);
1575 __atomic_post_full_fence();
1576 return ret;
1577#else
1578 return raw_atomic_fetch_and(~i, v);
1579#endif
1580}
1581
1582/**
1583 * raw_atomic_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
1584 * @i: int value
1585 * @v: pointer to atomic_t
1586 *
1587 * Atomically updates @v to (@v & ~@i) with acquire ordering.
1588 *
1589 * Safe to use in noinstr code; prefer atomic_fetch_andnot_acquire() elsewhere.
1590 *
1591 * Return: The original value of @v.
1592 */
1593static __always_inline int
1594raw_atomic_fetch_andnot_acquire(int i, atomic_t *v)
1595{
1596#if defined(arch_atomic_fetch_andnot_acquire)
1597 return arch_atomic_fetch_andnot_acquire(i, v);
1598#elif defined(arch_atomic_fetch_andnot_relaxed)
1599 int ret = arch_atomic_fetch_andnot_relaxed(i, v);
1600 __atomic_acquire_fence();
1601 return ret;
1602#elif defined(arch_atomic_fetch_andnot)
1603 return arch_atomic_fetch_andnot(i, v);
1604#else
1605 return raw_atomic_fetch_and_acquire(~i, v);
1606#endif
1607}
1608
1609/**
1610 * raw_atomic_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
1611 * @i: int value
1612 * @v: pointer to atomic_t
1613 *
1614 * Atomically updates @v to (@v & ~@i) with release ordering.
1615 *
1616 * Safe to use in noinstr code; prefer atomic_fetch_andnot_release() elsewhere.
1617 *
1618 * Return: The original value of @v.
1619 */
1620static __always_inline int
1621raw_atomic_fetch_andnot_release(int i, atomic_t *v)
1622{
1623#if defined(arch_atomic_fetch_andnot_release)
1624 return arch_atomic_fetch_andnot_release(i, v);
1625#elif defined(arch_atomic_fetch_andnot_relaxed)
1626 __atomic_release_fence();
1627 return arch_atomic_fetch_andnot_relaxed(i, v);
1628#elif defined(arch_atomic_fetch_andnot)
1629 return arch_atomic_fetch_andnot(i, v);
1630#else
1631 return raw_atomic_fetch_and_release(~i, v);
1632#endif
1633}
1634
1635/**
1636 * raw_atomic_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
1637 * @i: int value
1638 * @v: pointer to atomic_t
1639 *
1640 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
1641 *
1642 * Safe to use in noinstr code; prefer atomic_fetch_andnot_relaxed() elsewhere.
1643 *
1644 * Return: The original value of @v.
1645 */
1646static __always_inline int
1647raw_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
1648{
1649#if defined(arch_atomic_fetch_andnot_relaxed)
1650 return arch_atomic_fetch_andnot_relaxed(i, v);
1651#elif defined(arch_atomic_fetch_andnot)
1652 return arch_atomic_fetch_andnot(i, v);
1653#else
1654 return raw_atomic_fetch_and_relaxed(~i, v);
1655#endif
1656}
1657
1658/**
1659 * raw_atomic_or() - atomic bitwise OR with relaxed ordering
1660 * @i: int value
1661 * @v: pointer to atomic_t
1662 *
1663 * Atomically updates @v to (@v | @i) with relaxed ordering.
1664 *
1665 * Safe to use in noinstr code; prefer atomic_or() elsewhere.
1666 *
1667 * Return: Nothing.
1668 */
1669static __always_inline void
1670raw_atomic_or(int i, atomic_t *v)
1671{
1672 arch_atomic_or(i, v);
1673}
1674
1675/**
1676 * raw_atomic_fetch_or() - atomic bitwise OR with full ordering
1677 * @i: int value
1678 * @v: pointer to atomic_t
1679 *
1680 * Atomically updates @v to (@v | @i) with full ordering.
1681 *
1682 * Safe to use in noinstr code; prefer atomic_fetch_or() elsewhere.
1683 *
1684 * Return: The original value of @v.
1685 */
1686static __always_inline int
1687raw_atomic_fetch_or(int i, atomic_t *v)
1688{
1689#if defined(arch_atomic_fetch_or)
1690 return arch_atomic_fetch_or(i, v);
1691#elif defined(arch_atomic_fetch_or_relaxed)
1692 int ret;
1693 __atomic_pre_full_fence();
1694 ret = arch_atomic_fetch_or_relaxed(i, v);
1695 __atomic_post_full_fence();
1696 return ret;
1697#else
1698#error "Unable to define raw_atomic_fetch_or"
1699#endif
1700}
1701
1702/**
1703 * raw_atomic_fetch_or_acquire() - atomic bitwise OR with acquire ordering
1704 * @i: int value
1705 * @v: pointer to atomic_t
1706 *
1707 * Atomically updates @v to (@v | @i) with acquire ordering.
1708 *
1709 * Safe to use in noinstr code; prefer atomic_fetch_or_acquire() elsewhere.
1710 *
1711 * Return: The original value of @v.
1712 */
1713static __always_inline int
1714raw_atomic_fetch_or_acquire(int i, atomic_t *v)
1715{
1716#if defined(arch_atomic_fetch_or_acquire)
1717 return arch_atomic_fetch_or_acquire(i, v);
1718#elif defined(arch_atomic_fetch_or_relaxed)
1719 int ret = arch_atomic_fetch_or_relaxed(i, v);
1720 __atomic_acquire_fence();
1721 return ret;
1722#elif defined(arch_atomic_fetch_or)
1723 return arch_atomic_fetch_or(i, v);
1724#else
1725#error "Unable to define raw_atomic_fetch_or_acquire"
1726#endif
1727}
1728
1729/**
1730 * raw_atomic_fetch_or_release() - atomic bitwise OR with release ordering
1731 * @i: int value
1732 * @v: pointer to atomic_t
1733 *
1734 * Atomically updates @v to (@v | @i) with release ordering.
1735 *
1736 * Safe to use in noinstr code; prefer atomic_fetch_or_release() elsewhere.
1737 *
1738 * Return: The original value of @v.
1739 */
1740static __always_inline int
1741raw_atomic_fetch_or_release(int i, atomic_t *v)
1742{
1743#if defined(arch_atomic_fetch_or_release)
1744 return arch_atomic_fetch_or_release(i, v);
1745#elif defined(arch_atomic_fetch_or_relaxed)
1746 __atomic_release_fence();
1747 return arch_atomic_fetch_or_relaxed(i, v);
1748#elif defined(arch_atomic_fetch_or)
1749 return arch_atomic_fetch_or(i, v);
1750#else
1751#error "Unable to define raw_atomic_fetch_or_release"
1752#endif
1753}
1754
1755/**
1756 * raw_atomic_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
1757 * @i: int value
1758 * @v: pointer to atomic_t
1759 *
1760 * Atomically updates @v to (@v | @i) with relaxed ordering.
1761 *
1762 * Safe to use in noinstr code; prefer atomic_fetch_or_relaxed() elsewhere.
1763 *
1764 * Return: The original value of @v.
1765 */
1766static __always_inline int
1767raw_atomic_fetch_or_relaxed(int i, atomic_t *v)
1768{
1769#if defined(arch_atomic_fetch_or_relaxed)
1770 return arch_atomic_fetch_or_relaxed(i, v);
1771#elif defined(arch_atomic_fetch_or)
1772 return arch_atomic_fetch_or(i, v);
1773#else
1774#error "Unable to define raw_atomic_fetch_or_relaxed"
1775#endif
1776}
1777
1778/**
1779 * raw_atomic_xor() - atomic bitwise XOR with relaxed ordering
1780 * @i: int value
1781 * @v: pointer to atomic_t
1782 *
1783 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1784 *
1785 * Safe to use in noinstr code; prefer atomic_xor() elsewhere.
1786 *
1787 * Return: Nothing.
1788 */
1789static __always_inline void
1790raw_atomic_xor(int i, atomic_t *v)
1791{
1792 arch_atomic_xor(i, v);
1793}
1794
1795/**
1796 * raw_atomic_fetch_xor() - atomic bitwise XOR with full ordering
1797 * @i: int value
1798 * @v: pointer to atomic_t
1799 *
1800 * Atomically updates @v to (@v ^ @i) with full ordering.
1801 *
1802 * Safe to use in noinstr code; prefer atomic_fetch_xor() elsewhere.
1803 *
1804 * Return: The original value of @v.
1805 */
1806static __always_inline int
1807raw_atomic_fetch_xor(int i, atomic_t *v)
1808{
1809#if defined(arch_atomic_fetch_xor)
1810 return arch_atomic_fetch_xor(i, v);
1811#elif defined(arch_atomic_fetch_xor_relaxed)
1812 int ret;
1813 __atomic_pre_full_fence();
1814 ret = arch_atomic_fetch_xor_relaxed(i, v);
1815 __atomic_post_full_fence();
1816 return ret;
1817#else
1818#error "Unable to define raw_atomic_fetch_xor"
1819#endif
1820}
1821
1822/**
1823 * raw_atomic_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
1824 * @i: int value
1825 * @v: pointer to atomic_t
1826 *
1827 * Atomically updates @v to (@v ^ @i) with acquire ordering.
1828 *
1829 * Safe to use in noinstr code; prefer atomic_fetch_xor_acquire() elsewhere.
1830 *
1831 * Return: The original value of @v.
1832 */
1833static __always_inline int
1834raw_atomic_fetch_xor_acquire(int i, atomic_t *v)
1835{
1836#if defined(arch_atomic_fetch_xor_acquire)
1837 return arch_atomic_fetch_xor_acquire(i, v);
1838#elif defined(arch_atomic_fetch_xor_relaxed)
1839 int ret = arch_atomic_fetch_xor_relaxed(i, v);
1840 __atomic_acquire_fence();
1841 return ret;
1842#elif defined(arch_atomic_fetch_xor)
1843 return arch_atomic_fetch_xor(i, v);
1844#else
1845#error "Unable to define raw_atomic_fetch_xor_acquire"
1846#endif
1847}
1848
1849/**
1850 * raw_atomic_fetch_xor_release() - atomic bitwise XOR with release ordering
1851 * @i: int value
1852 * @v: pointer to atomic_t
1853 *
1854 * Atomically updates @v to (@v ^ @i) with release ordering.
1855 *
1856 * Safe to use in noinstr code; prefer atomic_fetch_xor_release() elsewhere.
1857 *
1858 * Return: The original value of @v.
1859 */
1860static __always_inline int
1861raw_atomic_fetch_xor_release(int i, atomic_t *v)
1862{
1863#if defined(arch_atomic_fetch_xor_release)
1864 return arch_atomic_fetch_xor_release(i, v);
1865#elif defined(arch_atomic_fetch_xor_relaxed)
1866 __atomic_release_fence();
1867 return arch_atomic_fetch_xor_relaxed(i, v);
1868#elif defined(arch_atomic_fetch_xor)
1869 return arch_atomic_fetch_xor(i, v);
1870#else
1871#error "Unable to define raw_atomic_fetch_xor_release"
1872#endif
1873}
1874
1875/**
1876 * raw_atomic_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
1877 * @i: int value
1878 * @v: pointer to atomic_t
1879 *
1880 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
1881 *
1882 * Safe to use in noinstr code; prefer atomic_fetch_xor_relaxed() elsewhere.
1883 *
1884 * Return: The original value of @v.
1885 */
1886static __always_inline int
1887raw_atomic_fetch_xor_relaxed(int i, atomic_t *v)
1888{
1889#if defined(arch_atomic_fetch_xor_relaxed)
1890 return arch_atomic_fetch_xor_relaxed(i, v);
1891#elif defined(arch_atomic_fetch_xor)
1892 return arch_atomic_fetch_xor(i, v);
1893#else
1894#error "Unable to define raw_atomic_fetch_xor_relaxed"
1895#endif
1896}
1897
1898/**
1899 * raw_atomic_xchg() - atomic exchange with full ordering
1900 * @v: pointer to atomic_t
1901 * @new: int value to assign
1902 *
1903 * Atomically updates @v to @new with full ordering.
1904 *
1905 * Safe to use in noinstr code; prefer atomic_xchg() elsewhere.
1906 *
1907 * Return: The original value of @v.
1908 */
1909static __always_inline int
1910raw_atomic_xchg(atomic_t *v, int new)
1911{
1912#if defined(arch_atomic_xchg)
1913 return arch_atomic_xchg(v, new);
1914#elif defined(arch_atomic_xchg_relaxed)
1915 int ret;
1916 __atomic_pre_full_fence();
1917 ret = arch_atomic_xchg_relaxed(v, new);
1918 __atomic_post_full_fence();
1919 return ret;
1920#else
1921 return raw_xchg(&v->counter, new);
1922#endif
1923}
1924
1925/**
1926 * raw_atomic_xchg_acquire() - atomic exchange with acquire ordering
1927 * @v: pointer to atomic_t
1928 * @new: int value to assign
1929 *
1930 * Atomically updates @v to @new with acquire ordering.
1931 *
1932 * Safe to use in noinstr code; prefer atomic_xchg_acquire() elsewhere.
1933 *
1934 * Return: The original value of @v.
1935 */
1936static __always_inline int
1937raw_atomic_xchg_acquire(atomic_t *v, int new)
1938{
1939#if defined(arch_atomic_xchg_acquire)
1940 return arch_atomic_xchg_acquire(v, new);
1941#elif defined(arch_atomic_xchg_relaxed)
1942 int ret = arch_atomic_xchg_relaxed(v, new);
1943 __atomic_acquire_fence();
1944 return ret;
1945#elif defined(arch_atomic_xchg)
1946 return arch_atomic_xchg(v, new);
1947#else
1948 return raw_xchg_acquire(&v->counter, new);
1949#endif
1950}
1951
1952/**
1953 * raw_atomic_xchg_release() - atomic exchange with release ordering
1954 * @v: pointer to atomic_t
1955 * @new: int value to assign
1956 *
1957 * Atomically updates @v to @new with release ordering.
1958 *
1959 * Safe to use in noinstr code; prefer atomic_xchg_release() elsewhere.
1960 *
1961 * Return: The original value of @v.
1962 */
1963static __always_inline int
1964raw_atomic_xchg_release(atomic_t *v, int new)
1965{
1966#if defined(arch_atomic_xchg_release)
1967 return arch_atomic_xchg_release(v, new);
1968#elif defined(arch_atomic_xchg_relaxed)
1969 __atomic_release_fence();
1970 return arch_atomic_xchg_relaxed(v, new);
1971#elif defined(arch_atomic_xchg)
1972 return arch_atomic_xchg(v, new);
1973#else
1974 return raw_xchg_release(&v->counter, new);
1975#endif
1976}
1977
1978/**
1979 * raw_atomic_xchg_relaxed() - atomic exchange with relaxed ordering
1980 * @v: pointer to atomic_t
1981 * @new: int value to assign
1982 *
1983 * Atomically updates @v to @new with relaxed ordering.
1984 *
1985 * Safe to use in noinstr code; prefer atomic_xchg_relaxed() elsewhere.
1986 *
1987 * Return: The original value of @v.
1988 */
1989static __always_inline int
1990raw_atomic_xchg_relaxed(atomic_t *v, int new)
1991{
1992#if defined(arch_atomic_xchg_relaxed)
1993 return arch_atomic_xchg_relaxed(v, new);
1994#elif defined(arch_atomic_xchg)
1995 return arch_atomic_xchg(v, new);
1996#else
1997 return raw_xchg_relaxed(&v->counter, new);
1998#endif
1999}
2000
2001/**
2002 * raw_atomic_cmpxchg() - atomic compare and exchange with full ordering
2003 * @v: pointer to atomic_t
2004 * @old: int value to compare with
2005 * @new: int value to assign
2006 *
2007 * If (@v == @old), atomically updates @v to @new with full ordering.
2008 *
2009 * Safe to use in noinstr code; prefer atomic_cmpxchg() elsewhere.
2010 *
2011 * Return: The original value of @v.
2012 */
2013static __always_inline int
2014raw_atomic_cmpxchg(atomic_t *v, int old, int new)
2015{
2016#if defined(arch_atomic_cmpxchg)
2017 return arch_atomic_cmpxchg(v, old, new);
2018#elif defined(arch_atomic_cmpxchg_relaxed)
2019 int ret;
2020 __atomic_pre_full_fence();
2021 ret = arch_atomic_cmpxchg_relaxed(v, old, new);
2022 __atomic_post_full_fence();
2023 return ret;
2024#else
2025 return raw_cmpxchg(&v->counter, old, new);
2026#endif
2027}
2028
2029/**
2030 * raw_atomic_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2031 * @v: pointer to atomic_t
2032 * @old: int value to compare with
2033 * @new: int value to assign
2034 *
2035 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2036 *
2037 * Safe to use in noinstr code; prefer atomic_cmpxchg_acquire() elsewhere.
2038 *
2039 * Return: The original value of @v.
2040 */
2041static __always_inline int
2042raw_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
2043{
2044#if defined(arch_atomic_cmpxchg_acquire)
2045 return arch_atomic_cmpxchg_acquire(v, old, new);
2046#elif defined(arch_atomic_cmpxchg_relaxed)
2047 int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
2048 __atomic_acquire_fence();
2049 return ret;
2050#elif defined(arch_atomic_cmpxchg)
2051 return arch_atomic_cmpxchg(v, old, new);
2052#else
2053 return raw_cmpxchg_acquire(&v->counter, old, new);
2054#endif
2055}
2056
2057/**
2058 * raw_atomic_cmpxchg_release() - atomic compare and exchange with release ordering
2059 * @v: pointer to atomic_t
2060 * @old: int value to compare with
2061 * @new: int value to assign
2062 *
2063 * If (@v == @old), atomically updates @v to @new with release ordering.
2064 *
2065 * Safe to use in noinstr code; prefer atomic_cmpxchg_release() elsewhere.
2066 *
2067 * Return: The original value of @v.
2068 */
2069static __always_inline int
2070raw_atomic_cmpxchg_release(atomic_t *v, int old, int new)
2071{
2072#if defined(arch_atomic_cmpxchg_release)
2073 return arch_atomic_cmpxchg_release(v, old, new);
2074#elif defined(arch_atomic_cmpxchg_relaxed)
2075 __atomic_release_fence();
2076 return arch_atomic_cmpxchg_relaxed(v, old, new);
2077#elif defined(arch_atomic_cmpxchg)
2078 return arch_atomic_cmpxchg(v, old, new);
2079#else
2080 return raw_cmpxchg_release(&v->counter, old, new);
2081#endif
2082}
2083
2084/**
2085 * raw_atomic_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2086 * @v: pointer to atomic_t
2087 * @old: int value to compare with
2088 * @new: int value to assign
2089 *
2090 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2091 *
2092 * Safe to use in noinstr code; prefer atomic_cmpxchg_relaxed() elsewhere.
2093 *
2094 * Return: The original value of @v.
2095 */
2096static __always_inline int
2097raw_atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
2098{
2099#if defined(arch_atomic_cmpxchg_relaxed)
2100 return arch_atomic_cmpxchg_relaxed(v, old, new);
2101#elif defined(arch_atomic_cmpxchg)
2102 return arch_atomic_cmpxchg(v, old, new);
2103#else
2104 return raw_cmpxchg_relaxed(&v->counter, old, new);
2105#endif
2106}
2107
2108/**
2109 * raw_atomic_try_cmpxchg() - atomic compare and exchange with full ordering
2110 * @v: pointer to atomic_t
2111 * @old: pointer to int value to compare with
2112 * @new: int value to assign
2113 *
2114 * If (@v == @old), atomically updates @v to @new with full ordering.
2115 * Otherwise, updates @old to the current value of @v.
2116 *
2117 * Safe to use in noinstr code; prefer atomic_try_cmpxchg() elsewhere.
2118 *
2119 * Return: @true if the exchange occured, @false otherwise.
2120 */
2121static __always_inline bool
2122raw_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
2123{
2124#if defined(arch_atomic_try_cmpxchg)
2125 return arch_atomic_try_cmpxchg(v, old, new);
2126#elif defined(arch_atomic_try_cmpxchg_relaxed)
2127 bool ret;
2128 __atomic_pre_full_fence();
2129 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2130 __atomic_post_full_fence();
2131 return ret;
2132#else
2133 int r, o = *old;
2134 r = raw_atomic_cmpxchg(v, o, new);
2135 if (unlikely(r != o))
2136 *old = r;
2137 return likely(r == o);
2138#endif
2139}
2140
2141/**
2142 * raw_atomic_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
2143 * @v: pointer to atomic_t
2144 * @old: pointer to int value to compare with
2145 * @new: int value to assign
2146 *
2147 * If (@v == @old), atomically updates @v to @new with acquire ordering.
2148 * Otherwise, updates @old to the current value of @v.
2149 *
2150 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_acquire() elsewhere.
2151 *
2152 * Return: @true if the exchange occured, @false otherwise.
2153 */
2154static __always_inline bool
2155raw_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
2156{
2157#if defined(arch_atomic_try_cmpxchg_acquire)
2158 return arch_atomic_try_cmpxchg_acquire(v, old, new);
2159#elif defined(arch_atomic_try_cmpxchg_relaxed)
2160 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
2161 __atomic_acquire_fence();
2162 return ret;
2163#elif defined(arch_atomic_try_cmpxchg)
2164 return arch_atomic_try_cmpxchg(v, old, new);
2165#else
2166 int r, o = *old;
2167 r = raw_atomic_cmpxchg_acquire(v, o, new);
2168 if (unlikely(r != o))
2169 *old = r;
2170 return likely(r == o);
2171#endif
2172}
2173
2174/**
2175 * raw_atomic_try_cmpxchg_release() - atomic compare and exchange with release ordering
2176 * @v: pointer to atomic_t
2177 * @old: pointer to int value to compare with
2178 * @new: int value to assign
2179 *
2180 * If (@v == @old), atomically updates @v to @new with release ordering.
2181 * Otherwise, updates @old to the current value of @v.
2182 *
2183 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_release() elsewhere.
2184 *
2185 * Return: @true if the exchange occured, @false otherwise.
2186 */
2187static __always_inline bool
2188raw_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
2189{
2190#if defined(arch_atomic_try_cmpxchg_release)
2191 return arch_atomic_try_cmpxchg_release(v, old, new);
2192#elif defined(arch_atomic_try_cmpxchg_relaxed)
2193 __atomic_release_fence();
2194 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2195#elif defined(arch_atomic_try_cmpxchg)
2196 return arch_atomic_try_cmpxchg(v, old, new);
2197#else
2198 int r, o = *old;
2199 r = raw_atomic_cmpxchg_release(v, o, new);
2200 if (unlikely(r != o))
2201 *old = r;
2202 return likely(r == o);
2203#endif
2204}
2205
2206/**
2207 * raw_atomic_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
2208 * @v: pointer to atomic_t
2209 * @old: pointer to int value to compare with
2210 * @new: int value to assign
2211 *
2212 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
2213 * Otherwise, updates @old to the current value of @v.
2214 *
2215 * Safe to use in noinstr code; prefer atomic_try_cmpxchg_relaxed() elsewhere.
2216 *
2217 * Return: @true if the exchange occured, @false otherwise.
2218 */
2219static __always_inline bool
2220raw_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
2221{
2222#if defined(arch_atomic_try_cmpxchg_relaxed)
2223 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
2224#elif defined(arch_atomic_try_cmpxchg)
2225 return arch_atomic_try_cmpxchg(v, old, new);
2226#else
2227 int r, o = *old;
2228 r = raw_atomic_cmpxchg_relaxed(v, o, new);
2229 if (unlikely(r != o))
2230 *old = r;
2231 return likely(r == o);
2232#endif
2233}
2234
2235/**
2236 * raw_atomic_sub_and_test() - atomic subtract and test if zero with full ordering
2237 * @i: int value to add
2238 * @v: pointer to atomic_t
2239 *
2240 * Atomically updates @v to (@v - @i) with full ordering.
2241 *
2242 * Safe to use in noinstr code; prefer atomic_sub_and_test() elsewhere.
2243 *
2244 * Return: @true if the resulting value of @v is zero, @false otherwise.
2245 */
2246static __always_inline bool
2247raw_atomic_sub_and_test(int i, atomic_t *v)
2248{
2249#if defined(arch_atomic_sub_and_test)
2250 return arch_atomic_sub_and_test(i, v);
2251#else
2252 return raw_atomic_sub_return(i, v) == 0;
2253#endif
2254}
2255
2256/**
2257 * raw_atomic_dec_and_test() - atomic decrement and test if zero with full ordering
2258 * @v: pointer to atomic_t
2259 *
2260 * Atomically updates @v to (@v - 1) with full ordering.
2261 *
2262 * Safe to use in noinstr code; prefer atomic_dec_and_test() elsewhere.
2263 *
2264 * Return: @true if the resulting value of @v is zero, @false otherwise.
2265 */
2266static __always_inline bool
2267raw_atomic_dec_and_test(atomic_t *v)
2268{
2269#if defined(arch_atomic_dec_and_test)
2270 return arch_atomic_dec_and_test(v);
2271#else
2272 return raw_atomic_dec_return(v) == 0;
2273#endif
2274}
2275
2276/**
2277 * raw_atomic_inc_and_test() - atomic increment and test if zero with full ordering
2278 * @v: pointer to atomic_t
2279 *
2280 * Atomically updates @v to (@v + 1) with full ordering.
2281 *
2282 * Safe to use in noinstr code; prefer atomic_inc_and_test() elsewhere.
2283 *
2284 * Return: @true if the resulting value of @v is zero, @false otherwise.
2285 */
2286static __always_inline bool
2287raw_atomic_inc_and_test(atomic_t *v)
2288{
2289#if defined(arch_atomic_inc_and_test)
2290 return arch_atomic_inc_and_test(v);
2291#else
2292 return raw_atomic_inc_return(v) == 0;
2293#endif
2294}
2295
2296/**
2297 * raw_atomic_add_negative() - atomic add and test if negative with full ordering
2298 * @i: int value to add
2299 * @v: pointer to atomic_t
2300 *
2301 * Atomically updates @v to (@v + @i) with full ordering.
2302 *
2303 * Safe to use in noinstr code; prefer atomic_add_negative() elsewhere.
2304 *
2305 * Return: @true if the resulting value of @v is negative, @false otherwise.
2306 */
2307static __always_inline bool
2308raw_atomic_add_negative(int i, atomic_t *v)
2309{
2310#if defined(arch_atomic_add_negative)
2311 return arch_atomic_add_negative(i, v);
2312#elif defined(arch_atomic_add_negative_relaxed)
2313 bool ret;
2314 __atomic_pre_full_fence();
2315 ret = arch_atomic_add_negative_relaxed(i, v);
2316 __atomic_post_full_fence();
2317 return ret;
2318#else
2319 return raw_atomic_add_return(i, v) < 0;
2320#endif
2321}
2322
2323/**
2324 * raw_atomic_add_negative_acquire() - atomic add and test if negative with acquire ordering
2325 * @i: int value to add
2326 * @v: pointer to atomic_t
2327 *
2328 * Atomically updates @v to (@v + @i) with acquire ordering.
2329 *
2330 * Safe to use in noinstr code; prefer atomic_add_negative_acquire() elsewhere.
2331 *
2332 * Return: @true if the resulting value of @v is negative, @false otherwise.
2333 */
2334static __always_inline bool
2335raw_atomic_add_negative_acquire(int i, atomic_t *v)
2336{
2337#if defined(arch_atomic_add_negative_acquire)
2338 return arch_atomic_add_negative_acquire(i, v);
2339#elif defined(arch_atomic_add_negative_relaxed)
2340 bool ret = arch_atomic_add_negative_relaxed(i, v);
2341 __atomic_acquire_fence();
2342 return ret;
2343#elif defined(arch_atomic_add_negative)
2344 return arch_atomic_add_negative(i, v);
2345#else
2346 return raw_atomic_add_return_acquire(i, v) < 0;
2347#endif
2348}
2349
2350/**
2351 * raw_atomic_add_negative_release() - atomic add and test if negative with release ordering
2352 * @i: int value to add
2353 * @v: pointer to atomic_t
2354 *
2355 * Atomically updates @v to (@v + @i) with release ordering.
2356 *
2357 * Safe to use in noinstr code; prefer atomic_add_negative_release() elsewhere.
2358 *
2359 * Return: @true if the resulting value of @v is negative, @false otherwise.
2360 */
2361static __always_inline bool
2362raw_atomic_add_negative_release(int i, atomic_t *v)
2363{
2364#if defined(arch_atomic_add_negative_release)
2365 return arch_atomic_add_negative_release(i, v);
2366#elif defined(arch_atomic_add_negative_relaxed)
2367 __atomic_release_fence();
2368 return arch_atomic_add_negative_relaxed(i, v);
2369#elif defined(arch_atomic_add_negative)
2370 return arch_atomic_add_negative(i, v);
2371#else
2372 return raw_atomic_add_return_release(i, v) < 0;
2373#endif
2374}
2375
2376/**
2377 * raw_atomic_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
2378 * @i: int value to add
2379 * @v: pointer to atomic_t
2380 *
2381 * Atomically updates @v to (@v + @i) with relaxed ordering.
2382 *
2383 * Safe to use in noinstr code; prefer atomic_add_negative_relaxed() elsewhere.
2384 *
2385 * Return: @true if the resulting value of @v is negative, @false otherwise.
2386 */
2387static __always_inline bool
2388raw_atomic_add_negative_relaxed(int i, atomic_t *v)
2389{
2390#if defined(arch_atomic_add_negative_relaxed)
2391 return arch_atomic_add_negative_relaxed(i, v);
2392#elif defined(arch_atomic_add_negative)
2393 return arch_atomic_add_negative(i, v);
2394#else
2395 return raw_atomic_add_return_relaxed(i, v) < 0;
2396#endif
2397}
2398
2399/**
2400 * raw_atomic_fetch_add_unless() - atomic add unless value with full ordering
2401 * @v: pointer to atomic_t
2402 * @a: int value to add
2403 * @u: int value to compare with
2404 *
2405 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2406 *
2407 * Safe to use in noinstr code; prefer atomic_fetch_add_unless() elsewhere.
2408 *
2409 * Return: The original value of @v.
2410 */
2411static __always_inline int
2412raw_atomic_fetch_add_unless(atomic_t *v, int a, int u)
2413{
2414#if defined(arch_atomic_fetch_add_unless)
2415 return arch_atomic_fetch_add_unless(v, a, u);
2416#else
2417 int c = raw_atomic_read(v);
2418
2419 do {
2420 if (unlikely(c == u))
2421 break;
2422 } while (!raw_atomic_try_cmpxchg(v, &c, c + a));
2423
2424 return c;
2425#endif
2426}
2427
2428/**
2429 * raw_atomic_add_unless() - atomic add unless value with full ordering
2430 * @v: pointer to atomic_t
2431 * @a: int value to add
2432 * @u: int value to compare with
2433 *
2434 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
2435 *
2436 * Safe to use in noinstr code; prefer atomic_add_unless() elsewhere.
2437 *
2438 * Return: @true if @v was updated, @false otherwise.
2439 */
2440static __always_inline bool
2441raw_atomic_add_unless(atomic_t *v, int a, int u)
2442{
2443#if defined(arch_atomic_add_unless)
2444 return arch_atomic_add_unless(v, a, u);
2445#else
2446 return raw_atomic_fetch_add_unless(v, a, u) != u;
2447#endif
2448}
2449
2450/**
2451 * raw_atomic_inc_not_zero() - atomic increment unless zero with full ordering
2452 * @v: pointer to atomic_t
2453 *
2454 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
2455 *
2456 * Safe to use in noinstr code; prefer atomic_inc_not_zero() elsewhere.
2457 *
2458 * Return: @true if @v was updated, @false otherwise.
2459 */
2460static __always_inline bool
2461raw_atomic_inc_not_zero(atomic_t *v)
2462{
2463#if defined(arch_atomic_inc_not_zero)
2464 return arch_atomic_inc_not_zero(v);
2465#else
2466 return raw_atomic_add_unless(v, 1, 0);
2467#endif
2468}
2469
2470/**
2471 * raw_atomic_inc_unless_negative() - atomic increment unless negative with full ordering
2472 * @v: pointer to atomic_t
2473 *
2474 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
2475 *
2476 * Safe to use in noinstr code; prefer atomic_inc_unless_negative() elsewhere.
2477 *
2478 * Return: @true if @v was updated, @false otherwise.
2479 */
2480static __always_inline bool
2481raw_atomic_inc_unless_negative(atomic_t *v)
2482{
2483#if defined(arch_atomic_inc_unless_negative)
2484 return arch_atomic_inc_unless_negative(v);
2485#else
2486 int c = raw_atomic_read(v);
2487
2488 do {
2489 if (unlikely(c < 0))
2490 return false;
2491 } while (!raw_atomic_try_cmpxchg(v, &c, c + 1));
2492
2493 return true;
2494#endif
2495}
2496
2497/**
2498 * raw_atomic_dec_unless_positive() - atomic decrement unless positive with full ordering
2499 * @v: pointer to atomic_t
2500 *
2501 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
2502 *
2503 * Safe to use in noinstr code; prefer atomic_dec_unless_positive() elsewhere.
2504 *
2505 * Return: @true if @v was updated, @false otherwise.
2506 */
2507static __always_inline bool
2508raw_atomic_dec_unless_positive(atomic_t *v)
2509{
2510#if defined(arch_atomic_dec_unless_positive)
2511 return arch_atomic_dec_unless_positive(v);
2512#else
2513 int c = raw_atomic_read(v);
2514
2515 do {
2516 if (unlikely(c > 0))
2517 return false;
2518 } while (!raw_atomic_try_cmpxchg(v, &c, c - 1));
2519
2520 return true;
2521#endif
2522}
2523
2524/**
2525 * raw_atomic_dec_if_positive() - atomic decrement if positive with full ordering
2526 * @v: pointer to atomic_t
2527 *
2528 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
2529 *
2530 * Safe to use in noinstr code; prefer atomic_dec_if_positive() elsewhere.
2531 *
2532 * Return: The old value of (@v - 1), regardless of whether @v was updated.
2533 */
2534static __always_inline int
2535raw_atomic_dec_if_positive(atomic_t *v)
2536{
2537#if defined(arch_atomic_dec_if_positive)
2538 return arch_atomic_dec_if_positive(v);
2539#else
2540 int dec, c = raw_atomic_read(v);
2541
2542 do {
2543 dec = c - 1;
2544 if (unlikely(dec < 0))
2545 break;
2546 } while (!raw_atomic_try_cmpxchg(v, &c, dec));
2547
2548 return dec;
2549#endif
2550}
2551
2552#ifdef CONFIG_GENERIC_ATOMIC64
2553#include <asm-generic/atomic64.h>
2554#endif
2555
2556/**
2557 * raw_atomic64_read() - atomic load with relaxed ordering
2558 * @v: pointer to atomic64_t
2559 *
2560 * Atomically loads the value of @v with relaxed ordering.
2561 *
2562 * Safe to use in noinstr code; prefer atomic64_read() elsewhere.
2563 *
2564 * Return: The value loaded from @v.
2565 */
2566static __always_inline s64
2567raw_atomic64_read(const atomic64_t *v)
2568{
2569 return arch_atomic64_read(v);
2570}
2571
2572/**
2573 * raw_atomic64_read_acquire() - atomic load with acquire ordering
2574 * @v: pointer to atomic64_t
2575 *
2576 * Atomically loads the value of @v with acquire ordering.
2577 *
2578 * Safe to use in noinstr code; prefer atomic64_read_acquire() elsewhere.
2579 *
2580 * Return: The value loaded from @v.
2581 */
2582static __always_inline s64
2583raw_atomic64_read_acquire(const atomic64_t *v)
2584{
2585#if defined(arch_atomic64_read_acquire)
2586 return arch_atomic64_read_acquire(v);
2587#else
2588 s64 ret;
2589
2590 if (__native_word(atomic64_t)) {
2591 ret = smp_load_acquire(&(v)->counter);
2592 } else {
2593 ret = raw_atomic64_read(v);
2594 __atomic_acquire_fence();
2595 }
2596
2597 return ret;
2598#endif
2599}
2600
2601/**
2602 * raw_atomic64_set() - atomic set with relaxed ordering
2603 * @v: pointer to atomic64_t
2604 * @i: s64 value to assign
2605 *
2606 * Atomically sets @v to @i with relaxed ordering.
2607 *
2608 * Safe to use in noinstr code; prefer atomic64_set() elsewhere.
2609 *
2610 * Return: Nothing.
2611 */
2612static __always_inline void
2613raw_atomic64_set(atomic64_t *v, s64 i)
2614{
2615 arch_atomic64_set(v, i);
2616}
2617
2618/**
2619 * raw_atomic64_set_release() - atomic set with release ordering
2620 * @v: pointer to atomic64_t
2621 * @i: s64 value to assign
2622 *
2623 * Atomically sets @v to @i with release ordering.
2624 *
2625 * Safe to use in noinstr code; prefer atomic64_set_release() elsewhere.
2626 *
2627 * Return: Nothing.
2628 */
2629static __always_inline void
2630raw_atomic64_set_release(atomic64_t *v, s64 i)
2631{
2632#if defined(arch_atomic64_set_release)
2633 arch_atomic64_set_release(v, i);
2634#else
2635 if (__native_word(atomic64_t)) {
2636 smp_store_release(&(v)->counter, i);
2637 } else {
2638 __atomic_release_fence();
2639 raw_atomic64_set(v, i);
2640 }
2641#endif
2642}
2643
2644/**
2645 * raw_atomic64_add() - atomic add with relaxed ordering
2646 * @i: s64 value to add
2647 * @v: pointer to atomic64_t
2648 *
2649 * Atomically updates @v to (@v + @i) with relaxed ordering.
2650 *
2651 * Safe to use in noinstr code; prefer atomic64_add() elsewhere.
2652 *
2653 * Return: Nothing.
2654 */
2655static __always_inline void
2656raw_atomic64_add(s64 i, atomic64_t *v)
2657{
2658 arch_atomic64_add(i, v);
2659}
2660
2661/**
2662 * raw_atomic64_add_return() - atomic add with full ordering
2663 * @i: s64 value to add
2664 * @v: pointer to atomic64_t
2665 *
2666 * Atomically updates @v to (@v + @i) with full ordering.
2667 *
2668 * Safe to use in noinstr code; prefer atomic64_add_return() elsewhere.
2669 *
2670 * Return: The updated value of @v.
2671 */
2672static __always_inline s64
2673raw_atomic64_add_return(s64 i, atomic64_t *v)
2674{
2675#if defined(arch_atomic64_add_return)
2676 return arch_atomic64_add_return(i, v);
2677#elif defined(arch_atomic64_add_return_relaxed)
2678 s64 ret;
2679 __atomic_pre_full_fence();
2680 ret = arch_atomic64_add_return_relaxed(i, v);
2681 __atomic_post_full_fence();
2682 return ret;
2683#else
2684#error "Unable to define raw_atomic64_add_return"
2685#endif
2686}
2687
2688/**
2689 * raw_atomic64_add_return_acquire() - atomic add with acquire ordering
2690 * @i: s64 value to add
2691 * @v: pointer to atomic64_t
2692 *
2693 * Atomically updates @v to (@v + @i) with acquire ordering.
2694 *
2695 * Safe to use in noinstr code; prefer atomic64_add_return_acquire() elsewhere.
2696 *
2697 * Return: The updated value of @v.
2698 */
2699static __always_inline s64
2700raw_atomic64_add_return_acquire(s64 i, atomic64_t *v)
2701{
2702#if defined(arch_atomic64_add_return_acquire)
2703 return arch_atomic64_add_return_acquire(i, v);
2704#elif defined(arch_atomic64_add_return_relaxed)
2705 s64 ret = arch_atomic64_add_return_relaxed(i, v);
2706 __atomic_acquire_fence();
2707 return ret;
2708#elif defined(arch_atomic64_add_return)
2709 return arch_atomic64_add_return(i, v);
2710#else
2711#error "Unable to define raw_atomic64_add_return_acquire"
2712#endif
2713}
2714
2715/**
2716 * raw_atomic64_add_return_release() - atomic add with release ordering
2717 * @i: s64 value to add
2718 * @v: pointer to atomic64_t
2719 *
2720 * Atomically updates @v to (@v + @i) with release ordering.
2721 *
2722 * Safe to use in noinstr code; prefer atomic64_add_return_release() elsewhere.
2723 *
2724 * Return: The updated value of @v.
2725 */
2726static __always_inline s64
2727raw_atomic64_add_return_release(s64 i, atomic64_t *v)
2728{
2729#if defined(arch_atomic64_add_return_release)
2730 return arch_atomic64_add_return_release(i, v);
2731#elif defined(arch_atomic64_add_return_relaxed)
2732 __atomic_release_fence();
2733 return arch_atomic64_add_return_relaxed(i, v);
2734#elif defined(arch_atomic64_add_return)
2735 return arch_atomic64_add_return(i, v);
2736#else
2737#error "Unable to define raw_atomic64_add_return_release"
2738#endif
2739}
2740
2741/**
2742 * raw_atomic64_add_return_relaxed() - atomic add with relaxed ordering
2743 * @i: s64 value to add
2744 * @v: pointer to atomic64_t
2745 *
2746 * Atomically updates @v to (@v + @i) with relaxed ordering.
2747 *
2748 * Safe to use in noinstr code; prefer atomic64_add_return_relaxed() elsewhere.
2749 *
2750 * Return: The updated value of @v.
2751 */
2752static __always_inline s64
2753raw_atomic64_add_return_relaxed(s64 i, atomic64_t *v)
2754{
2755#if defined(arch_atomic64_add_return_relaxed)
2756 return arch_atomic64_add_return_relaxed(i, v);
2757#elif defined(arch_atomic64_add_return)
2758 return arch_atomic64_add_return(i, v);
2759#else
2760#error "Unable to define raw_atomic64_add_return_relaxed"
2761#endif
2762}
2763
2764/**
2765 * raw_atomic64_fetch_add() - atomic add with full ordering
2766 * @i: s64 value to add
2767 * @v: pointer to atomic64_t
2768 *
2769 * Atomically updates @v to (@v + @i) with full ordering.
2770 *
2771 * Safe to use in noinstr code; prefer atomic64_fetch_add() elsewhere.
2772 *
2773 * Return: The original value of @v.
2774 */
2775static __always_inline s64
2776raw_atomic64_fetch_add(s64 i, atomic64_t *v)
2777{
2778#if defined(arch_atomic64_fetch_add)
2779 return arch_atomic64_fetch_add(i, v);
2780#elif defined(arch_atomic64_fetch_add_relaxed)
2781 s64 ret;
2782 __atomic_pre_full_fence();
2783 ret = arch_atomic64_fetch_add_relaxed(i, v);
2784 __atomic_post_full_fence();
2785 return ret;
2786#else
2787#error "Unable to define raw_atomic64_fetch_add"
2788#endif
2789}
2790
2791/**
2792 * raw_atomic64_fetch_add_acquire() - atomic add with acquire ordering
2793 * @i: s64 value to add
2794 * @v: pointer to atomic64_t
2795 *
2796 * Atomically updates @v to (@v + @i) with acquire ordering.
2797 *
2798 * Safe to use in noinstr code; prefer atomic64_fetch_add_acquire() elsewhere.
2799 *
2800 * Return: The original value of @v.
2801 */
2802static __always_inline s64
2803raw_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
2804{
2805#if defined(arch_atomic64_fetch_add_acquire)
2806 return arch_atomic64_fetch_add_acquire(i, v);
2807#elif defined(arch_atomic64_fetch_add_relaxed)
2808 s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
2809 __atomic_acquire_fence();
2810 return ret;
2811#elif defined(arch_atomic64_fetch_add)
2812 return arch_atomic64_fetch_add(i, v);
2813#else
2814#error "Unable to define raw_atomic64_fetch_add_acquire"
2815#endif
2816}
2817
2818/**
2819 * raw_atomic64_fetch_add_release() - atomic add with release ordering
2820 * @i: s64 value to add
2821 * @v: pointer to atomic64_t
2822 *
2823 * Atomically updates @v to (@v + @i) with release ordering.
2824 *
2825 * Safe to use in noinstr code; prefer atomic64_fetch_add_release() elsewhere.
2826 *
2827 * Return: The original value of @v.
2828 */
2829static __always_inline s64
2830raw_atomic64_fetch_add_release(s64 i, atomic64_t *v)
2831{
2832#if defined(arch_atomic64_fetch_add_release)
2833 return arch_atomic64_fetch_add_release(i, v);
2834#elif defined(arch_atomic64_fetch_add_relaxed)
2835 __atomic_release_fence();
2836 return arch_atomic64_fetch_add_relaxed(i, v);
2837#elif defined(arch_atomic64_fetch_add)
2838 return arch_atomic64_fetch_add(i, v);
2839#else
2840#error "Unable to define raw_atomic64_fetch_add_release"
2841#endif
2842}
2843
2844/**
2845 * raw_atomic64_fetch_add_relaxed() - atomic add with relaxed ordering
2846 * @i: s64 value to add
2847 * @v: pointer to atomic64_t
2848 *
2849 * Atomically updates @v to (@v + @i) with relaxed ordering.
2850 *
2851 * Safe to use in noinstr code; prefer atomic64_fetch_add_relaxed() elsewhere.
2852 *
2853 * Return: The original value of @v.
2854 */
2855static __always_inline s64
2856raw_atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
2857{
2858#if defined(arch_atomic64_fetch_add_relaxed)
2859 return arch_atomic64_fetch_add_relaxed(i, v);
2860#elif defined(arch_atomic64_fetch_add)
2861 return arch_atomic64_fetch_add(i, v);
2862#else
2863#error "Unable to define raw_atomic64_fetch_add_relaxed"
2864#endif
2865}
2866
2867/**
2868 * raw_atomic64_sub() - atomic subtract with relaxed ordering
2869 * @i: s64 value to subtract
2870 * @v: pointer to atomic64_t
2871 *
2872 * Atomically updates @v to (@v - @i) with relaxed ordering.
2873 *
2874 * Safe to use in noinstr code; prefer atomic64_sub() elsewhere.
2875 *
2876 * Return: Nothing.
2877 */
2878static __always_inline void
2879raw_atomic64_sub(s64 i, atomic64_t *v)
2880{
2881 arch_atomic64_sub(i, v);
2882}
2883
2884/**
2885 * raw_atomic64_sub_return() - atomic subtract with full ordering
2886 * @i: s64 value to subtract
2887 * @v: pointer to atomic64_t
2888 *
2889 * Atomically updates @v to (@v - @i) with full ordering.
2890 *
2891 * Safe to use in noinstr code; prefer atomic64_sub_return() elsewhere.
2892 *
2893 * Return: The updated value of @v.
2894 */
2895static __always_inline s64
2896raw_atomic64_sub_return(s64 i, atomic64_t *v)
2897{
2898#if defined(arch_atomic64_sub_return)
2899 return arch_atomic64_sub_return(i, v);
2900#elif defined(arch_atomic64_sub_return_relaxed)
2901 s64 ret;
2902 __atomic_pre_full_fence();
2903 ret = arch_atomic64_sub_return_relaxed(i, v);
2904 __atomic_post_full_fence();
2905 return ret;
2906#else
2907#error "Unable to define raw_atomic64_sub_return"
2908#endif
2909}
2910
2911/**
2912 * raw_atomic64_sub_return_acquire() - atomic subtract with acquire ordering
2913 * @i: s64 value to subtract
2914 * @v: pointer to atomic64_t
2915 *
2916 * Atomically updates @v to (@v - @i) with acquire ordering.
2917 *
2918 * Safe to use in noinstr code; prefer atomic64_sub_return_acquire() elsewhere.
2919 *
2920 * Return: The updated value of @v.
2921 */
2922static __always_inline s64
2923raw_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
2924{
2925#if defined(arch_atomic64_sub_return_acquire)
2926 return arch_atomic64_sub_return_acquire(i, v);
2927#elif defined(arch_atomic64_sub_return_relaxed)
2928 s64 ret = arch_atomic64_sub_return_relaxed(i, v);
2929 __atomic_acquire_fence();
2930 return ret;
2931#elif defined(arch_atomic64_sub_return)
2932 return arch_atomic64_sub_return(i, v);
2933#else
2934#error "Unable to define raw_atomic64_sub_return_acquire"
2935#endif
2936}
2937
2938/**
2939 * raw_atomic64_sub_return_release() - atomic subtract with release ordering
2940 * @i: s64 value to subtract
2941 * @v: pointer to atomic64_t
2942 *
2943 * Atomically updates @v to (@v - @i) with release ordering.
2944 *
2945 * Safe to use in noinstr code; prefer atomic64_sub_return_release() elsewhere.
2946 *
2947 * Return: The updated value of @v.
2948 */
2949static __always_inline s64
2950raw_atomic64_sub_return_release(s64 i, atomic64_t *v)
2951{
2952#if defined(arch_atomic64_sub_return_release)
2953 return arch_atomic64_sub_return_release(i, v);
2954#elif defined(arch_atomic64_sub_return_relaxed)
2955 __atomic_release_fence();
2956 return arch_atomic64_sub_return_relaxed(i, v);
2957#elif defined(arch_atomic64_sub_return)
2958 return arch_atomic64_sub_return(i, v);
2959#else
2960#error "Unable to define raw_atomic64_sub_return_release"
2961#endif
2962}
2963
2964/**
2965 * raw_atomic64_sub_return_relaxed() - atomic subtract with relaxed ordering
2966 * @i: s64 value to subtract
2967 * @v: pointer to atomic64_t
2968 *
2969 * Atomically updates @v to (@v - @i) with relaxed ordering.
2970 *
2971 * Safe to use in noinstr code; prefer atomic64_sub_return_relaxed() elsewhere.
2972 *
2973 * Return: The updated value of @v.
2974 */
2975static __always_inline s64
2976raw_atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
2977{
2978#if defined(arch_atomic64_sub_return_relaxed)
2979 return arch_atomic64_sub_return_relaxed(i, v);
2980#elif defined(arch_atomic64_sub_return)
2981 return arch_atomic64_sub_return(i, v);
2982#else
2983#error "Unable to define raw_atomic64_sub_return_relaxed"
2984#endif
2985}
2986
2987/**
2988 * raw_atomic64_fetch_sub() - atomic subtract with full ordering
2989 * @i: s64 value to subtract
2990 * @v: pointer to atomic64_t
2991 *
2992 * Atomically updates @v to (@v - @i) with full ordering.
2993 *
2994 * Safe to use in noinstr code; prefer atomic64_fetch_sub() elsewhere.
2995 *
2996 * Return: The original value of @v.
2997 */
2998static __always_inline s64
2999raw_atomic64_fetch_sub(s64 i, atomic64_t *v)
3000{
3001#if defined(arch_atomic64_fetch_sub)
3002 return arch_atomic64_fetch_sub(i, v);
3003#elif defined(arch_atomic64_fetch_sub_relaxed)
3004 s64 ret;
3005 __atomic_pre_full_fence();
3006 ret = arch_atomic64_fetch_sub_relaxed(i, v);
3007 __atomic_post_full_fence();
3008 return ret;
3009#else
3010#error "Unable to define raw_atomic64_fetch_sub"
3011#endif
3012}
3013
3014/**
3015 * raw_atomic64_fetch_sub_acquire() - atomic subtract with acquire ordering
3016 * @i: s64 value to subtract
3017 * @v: pointer to atomic64_t
3018 *
3019 * Atomically updates @v to (@v - @i) with acquire ordering.
3020 *
3021 * Safe to use in noinstr code; prefer atomic64_fetch_sub_acquire() elsewhere.
3022 *
3023 * Return: The original value of @v.
3024 */
3025static __always_inline s64
3026raw_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
3027{
3028#if defined(arch_atomic64_fetch_sub_acquire)
3029 return arch_atomic64_fetch_sub_acquire(i, v);
3030#elif defined(arch_atomic64_fetch_sub_relaxed)
3031 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
3032 __atomic_acquire_fence();
3033 return ret;
3034#elif defined(arch_atomic64_fetch_sub)
3035 return arch_atomic64_fetch_sub(i, v);
3036#else
3037#error "Unable to define raw_atomic64_fetch_sub_acquire"
3038#endif
3039}
3040
3041/**
3042 * raw_atomic64_fetch_sub_release() - atomic subtract with release ordering
3043 * @i: s64 value to subtract
3044 * @v: pointer to atomic64_t
3045 *
3046 * Atomically updates @v to (@v - @i) with release ordering.
3047 *
3048 * Safe to use in noinstr code; prefer atomic64_fetch_sub_release() elsewhere.
3049 *
3050 * Return: The original value of @v.
3051 */
3052static __always_inline s64
3053raw_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
3054{
3055#if defined(arch_atomic64_fetch_sub_release)
3056 return arch_atomic64_fetch_sub_release(i, v);
3057#elif defined(arch_atomic64_fetch_sub_relaxed)
3058 __atomic_release_fence();
3059 return arch_atomic64_fetch_sub_relaxed(i, v);
3060#elif defined(arch_atomic64_fetch_sub)
3061 return arch_atomic64_fetch_sub(i, v);
3062#else
3063#error "Unable to define raw_atomic64_fetch_sub_release"
3064#endif
3065}
3066
3067/**
3068 * raw_atomic64_fetch_sub_relaxed() - atomic subtract with relaxed ordering
3069 * @i: s64 value to subtract
3070 * @v: pointer to atomic64_t
3071 *
3072 * Atomically updates @v to (@v - @i) with relaxed ordering.
3073 *
3074 * Safe to use in noinstr code; prefer atomic64_fetch_sub_relaxed() elsewhere.
3075 *
3076 * Return: The original value of @v.
3077 */
3078static __always_inline s64
3079raw_atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
3080{
3081#if defined(arch_atomic64_fetch_sub_relaxed)
3082 return arch_atomic64_fetch_sub_relaxed(i, v);
3083#elif defined(arch_atomic64_fetch_sub)
3084 return arch_atomic64_fetch_sub(i, v);
3085#else
3086#error "Unable to define raw_atomic64_fetch_sub_relaxed"
3087#endif
3088}
3089
3090/**
3091 * raw_atomic64_inc() - atomic increment with relaxed ordering
3092 * @v: pointer to atomic64_t
3093 *
3094 * Atomically updates @v to (@v + 1) with relaxed ordering.
3095 *
3096 * Safe to use in noinstr code; prefer atomic64_inc() elsewhere.
3097 *
3098 * Return: Nothing.
3099 */
3100static __always_inline void
3101raw_atomic64_inc(atomic64_t *v)
3102{
3103#if defined(arch_atomic64_inc)
3104 arch_atomic64_inc(v);
3105#else
3106 raw_atomic64_add(1, v);
3107#endif
3108}
3109
3110/**
3111 * raw_atomic64_inc_return() - atomic increment with full ordering
3112 * @v: pointer to atomic64_t
3113 *
3114 * Atomically updates @v to (@v + 1) with full ordering.
3115 *
3116 * Safe to use in noinstr code; prefer atomic64_inc_return() elsewhere.
3117 *
3118 * Return: The updated value of @v.
3119 */
3120static __always_inline s64
3121raw_atomic64_inc_return(atomic64_t *v)
3122{
3123#if defined(arch_atomic64_inc_return)
3124 return arch_atomic64_inc_return(v);
3125#elif defined(arch_atomic64_inc_return_relaxed)
3126 s64 ret;
3127 __atomic_pre_full_fence();
3128 ret = arch_atomic64_inc_return_relaxed(v);
3129 __atomic_post_full_fence();
3130 return ret;
3131#else
3132 return raw_atomic64_add_return(1, v);
3133#endif
3134}
3135
3136/**
3137 * raw_atomic64_inc_return_acquire() - atomic increment with acquire ordering
3138 * @v: pointer to atomic64_t
3139 *
3140 * Atomically updates @v to (@v + 1) with acquire ordering.
3141 *
3142 * Safe to use in noinstr code; prefer atomic64_inc_return_acquire() elsewhere.
3143 *
3144 * Return: The updated value of @v.
3145 */
3146static __always_inline s64
3147raw_atomic64_inc_return_acquire(atomic64_t *v)
3148{
3149#if defined(arch_atomic64_inc_return_acquire)
3150 return arch_atomic64_inc_return_acquire(v);
3151#elif defined(arch_atomic64_inc_return_relaxed)
3152 s64 ret = arch_atomic64_inc_return_relaxed(v);
3153 __atomic_acquire_fence();
3154 return ret;
3155#elif defined(arch_atomic64_inc_return)
3156 return arch_atomic64_inc_return(v);
3157#else
3158 return raw_atomic64_add_return_acquire(1, v);
3159#endif
3160}
3161
3162/**
3163 * raw_atomic64_inc_return_release() - atomic increment with release ordering
3164 * @v: pointer to atomic64_t
3165 *
3166 * Atomically updates @v to (@v + 1) with release ordering.
3167 *
3168 * Safe to use in noinstr code; prefer atomic64_inc_return_release() elsewhere.
3169 *
3170 * Return: The updated value of @v.
3171 */
3172static __always_inline s64
3173raw_atomic64_inc_return_release(atomic64_t *v)
3174{
3175#if defined(arch_atomic64_inc_return_release)
3176 return arch_atomic64_inc_return_release(v);
3177#elif defined(arch_atomic64_inc_return_relaxed)
3178 __atomic_release_fence();
3179 return arch_atomic64_inc_return_relaxed(v);
3180#elif defined(arch_atomic64_inc_return)
3181 return arch_atomic64_inc_return(v);
3182#else
3183 return raw_atomic64_add_return_release(1, v);
3184#endif
3185}
3186
3187/**
3188 * raw_atomic64_inc_return_relaxed() - atomic increment with relaxed ordering
3189 * @v: pointer to atomic64_t
3190 *
3191 * Atomically updates @v to (@v + 1) with relaxed ordering.
3192 *
3193 * Safe to use in noinstr code; prefer atomic64_inc_return_relaxed() elsewhere.
3194 *
3195 * Return: The updated value of @v.
3196 */
3197static __always_inline s64
3198raw_atomic64_inc_return_relaxed(atomic64_t *v)
3199{
3200#if defined(arch_atomic64_inc_return_relaxed)
3201 return arch_atomic64_inc_return_relaxed(v);
3202#elif defined(arch_atomic64_inc_return)
3203 return arch_atomic64_inc_return(v);
3204#else
3205 return raw_atomic64_add_return_relaxed(1, v);
3206#endif
3207}
3208
3209/**
3210 * raw_atomic64_fetch_inc() - atomic increment with full ordering
3211 * @v: pointer to atomic64_t
3212 *
3213 * Atomically updates @v to (@v + 1) with full ordering.
3214 *
3215 * Safe to use in noinstr code; prefer atomic64_fetch_inc() elsewhere.
3216 *
3217 * Return: The original value of @v.
3218 */
3219static __always_inline s64
3220raw_atomic64_fetch_inc(atomic64_t *v)
3221{
3222#if defined(arch_atomic64_fetch_inc)
3223 return arch_atomic64_fetch_inc(v);
3224#elif defined(arch_atomic64_fetch_inc_relaxed)
3225 s64 ret;
3226 __atomic_pre_full_fence();
3227 ret = arch_atomic64_fetch_inc_relaxed(v);
3228 __atomic_post_full_fence();
3229 return ret;
3230#else
3231 return raw_atomic64_fetch_add(1, v);
3232#endif
3233}
3234
3235/**
3236 * raw_atomic64_fetch_inc_acquire() - atomic increment with acquire ordering
3237 * @v: pointer to atomic64_t
3238 *
3239 * Atomically updates @v to (@v + 1) with acquire ordering.
3240 *
3241 * Safe to use in noinstr code; prefer atomic64_fetch_inc_acquire() elsewhere.
3242 *
3243 * Return: The original value of @v.
3244 */
3245static __always_inline s64
3246raw_atomic64_fetch_inc_acquire(atomic64_t *v)
3247{
3248#if defined(arch_atomic64_fetch_inc_acquire)
3249 return arch_atomic64_fetch_inc_acquire(v);
3250#elif defined(arch_atomic64_fetch_inc_relaxed)
3251 s64 ret = arch_atomic64_fetch_inc_relaxed(v);
3252 __atomic_acquire_fence();
3253 return ret;
3254#elif defined(arch_atomic64_fetch_inc)
3255 return arch_atomic64_fetch_inc(v);
3256#else
3257 return raw_atomic64_fetch_add_acquire(1, v);
3258#endif
3259}
3260
3261/**
3262 * raw_atomic64_fetch_inc_release() - atomic increment with release ordering
3263 * @v: pointer to atomic64_t
3264 *
3265 * Atomically updates @v to (@v + 1) with release ordering.
3266 *
3267 * Safe to use in noinstr code; prefer atomic64_fetch_inc_release() elsewhere.
3268 *
3269 * Return: The original value of @v.
3270 */
3271static __always_inline s64
3272raw_atomic64_fetch_inc_release(atomic64_t *v)
3273{
3274#if defined(arch_atomic64_fetch_inc_release)
3275 return arch_atomic64_fetch_inc_release(v);
3276#elif defined(arch_atomic64_fetch_inc_relaxed)
3277 __atomic_release_fence();
3278 return arch_atomic64_fetch_inc_relaxed(v);
3279#elif defined(arch_atomic64_fetch_inc)
3280 return arch_atomic64_fetch_inc(v);
3281#else
3282 return raw_atomic64_fetch_add_release(1, v);
3283#endif
3284}
3285
3286/**
3287 * raw_atomic64_fetch_inc_relaxed() - atomic increment with relaxed ordering
3288 * @v: pointer to atomic64_t
3289 *
3290 * Atomically updates @v to (@v + 1) with relaxed ordering.
3291 *
3292 * Safe to use in noinstr code; prefer atomic64_fetch_inc_relaxed() elsewhere.
3293 *
3294 * Return: The original value of @v.
3295 */
3296static __always_inline s64
3297raw_atomic64_fetch_inc_relaxed(atomic64_t *v)
3298{
3299#if defined(arch_atomic64_fetch_inc_relaxed)
3300 return arch_atomic64_fetch_inc_relaxed(v);
3301#elif defined(arch_atomic64_fetch_inc)
3302 return arch_atomic64_fetch_inc(v);
3303#else
3304 return raw_atomic64_fetch_add_relaxed(1, v);
3305#endif
3306}
3307
3308/**
3309 * raw_atomic64_dec() - atomic decrement with relaxed ordering
3310 * @v: pointer to atomic64_t
3311 *
3312 * Atomically updates @v to (@v - 1) with relaxed ordering.
3313 *
3314 * Safe to use in noinstr code; prefer atomic64_dec() elsewhere.
3315 *
3316 * Return: Nothing.
3317 */
3318static __always_inline void
3319raw_atomic64_dec(atomic64_t *v)
3320{
3321#if defined(arch_atomic64_dec)
3322 arch_atomic64_dec(v);
3323#else
3324 raw_atomic64_sub(1, v);
3325#endif
3326}
3327
3328/**
3329 * raw_atomic64_dec_return() - atomic decrement with full ordering
3330 * @v: pointer to atomic64_t
3331 *
3332 * Atomically updates @v to (@v - 1) with full ordering.
3333 *
3334 * Safe to use in noinstr code; prefer atomic64_dec_return() elsewhere.
3335 *
3336 * Return: The updated value of @v.
3337 */
3338static __always_inline s64
3339raw_atomic64_dec_return(atomic64_t *v)
3340{
3341#if defined(arch_atomic64_dec_return)
3342 return arch_atomic64_dec_return(v);
3343#elif defined(arch_atomic64_dec_return_relaxed)
3344 s64 ret;
3345 __atomic_pre_full_fence();
3346 ret = arch_atomic64_dec_return_relaxed(v);
3347 __atomic_post_full_fence();
3348 return ret;
3349#else
3350 return raw_atomic64_sub_return(1, v);
3351#endif
3352}
3353
3354/**
3355 * raw_atomic64_dec_return_acquire() - atomic decrement with acquire ordering
3356 * @v: pointer to atomic64_t
3357 *
3358 * Atomically updates @v to (@v - 1) with acquire ordering.
3359 *
3360 * Safe to use in noinstr code; prefer atomic64_dec_return_acquire() elsewhere.
3361 *
3362 * Return: The updated value of @v.
3363 */
3364static __always_inline s64
3365raw_atomic64_dec_return_acquire(atomic64_t *v)
3366{
3367#if defined(arch_atomic64_dec_return_acquire)
3368 return arch_atomic64_dec_return_acquire(v);
3369#elif defined(arch_atomic64_dec_return_relaxed)
3370 s64 ret = arch_atomic64_dec_return_relaxed(v);
3371 __atomic_acquire_fence();
3372 return ret;
3373#elif defined(arch_atomic64_dec_return)
3374 return arch_atomic64_dec_return(v);
3375#else
3376 return raw_atomic64_sub_return_acquire(1, v);
3377#endif
3378}
3379
3380/**
3381 * raw_atomic64_dec_return_release() - atomic decrement with release ordering
3382 * @v: pointer to atomic64_t
3383 *
3384 * Atomically updates @v to (@v - 1) with release ordering.
3385 *
3386 * Safe to use in noinstr code; prefer atomic64_dec_return_release() elsewhere.
3387 *
3388 * Return: The updated value of @v.
3389 */
3390static __always_inline s64
3391raw_atomic64_dec_return_release(atomic64_t *v)
3392{
3393#if defined(arch_atomic64_dec_return_release)
3394 return arch_atomic64_dec_return_release(v);
3395#elif defined(arch_atomic64_dec_return_relaxed)
3396 __atomic_release_fence();
3397 return arch_atomic64_dec_return_relaxed(v);
3398#elif defined(arch_atomic64_dec_return)
3399 return arch_atomic64_dec_return(v);
3400#else
3401 return raw_atomic64_sub_return_release(1, v);
3402#endif
3403}
3404
3405/**
3406 * raw_atomic64_dec_return_relaxed() - atomic decrement with relaxed ordering
3407 * @v: pointer to atomic64_t
3408 *
3409 * Atomically updates @v to (@v - 1) with relaxed ordering.
3410 *
3411 * Safe to use in noinstr code; prefer atomic64_dec_return_relaxed() elsewhere.
3412 *
3413 * Return: The updated value of @v.
3414 */
3415static __always_inline s64
3416raw_atomic64_dec_return_relaxed(atomic64_t *v)
3417{
3418#if defined(arch_atomic64_dec_return_relaxed)
3419 return arch_atomic64_dec_return_relaxed(v);
3420#elif defined(arch_atomic64_dec_return)
3421 return arch_atomic64_dec_return(v);
3422#else
3423 return raw_atomic64_sub_return_relaxed(1, v);
3424#endif
3425}
3426
3427/**
3428 * raw_atomic64_fetch_dec() - atomic decrement with full ordering
3429 * @v: pointer to atomic64_t
3430 *
3431 * Atomically updates @v to (@v - 1) with full ordering.
3432 *
3433 * Safe to use in noinstr code; prefer atomic64_fetch_dec() elsewhere.
3434 *
3435 * Return: The original value of @v.
3436 */
3437static __always_inline s64
3438raw_atomic64_fetch_dec(atomic64_t *v)
3439{
3440#if defined(arch_atomic64_fetch_dec)
3441 return arch_atomic64_fetch_dec(v);
3442#elif defined(arch_atomic64_fetch_dec_relaxed)
3443 s64 ret;
3444 __atomic_pre_full_fence();
3445 ret = arch_atomic64_fetch_dec_relaxed(v);
3446 __atomic_post_full_fence();
3447 return ret;
3448#else
3449 return raw_atomic64_fetch_sub(1, v);
3450#endif
3451}
3452
3453/**
3454 * raw_atomic64_fetch_dec_acquire() - atomic decrement with acquire ordering
3455 * @v: pointer to atomic64_t
3456 *
3457 * Atomically updates @v to (@v - 1) with acquire ordering.
3458 *
3459 * Safe to use in noinstr code; prefer atomic64_fetch_dec_acquire() elsewhere.
3460 *
3461 * Return: The original value of @v.
3462 */
3463static __always_inline s64
3464raw_atomic64_fetch_dec_acquire(atomic64_t *v)
3465{
3466#if defined(arch_atomic64_fetch_dec_acquire)
3467 return arch_atomic64_fetch_dec_acquire(v);
3468#elif defined(arch_atomic64_fetch_dec_relaxed)
3469 s64 ret = arch_atomic64_fetch_dec_relaxed(v);
3470 __atomic_acquire_fence();
3471 return ret;
3472#elif defined(arch_atomic64_fetch_dec)
3473 return arch_atomic64_fetch_dec(v);
3474#else
3475 return raw_atomic64_fetch_sub_acquire(1, v);
3476#endif
3477}
3478
3479/**
3480 * raw_atomic64_fetch_dec_release() - atomic decrement with release ordering
3481 * @v: pointer to atomic64_t
3482 *
3483 * Atomically updates @v to (@v - 1) with release ordering.
3484 *
3485 * Safe to use in noinstr code; prefer atomic64_fetch_dec_release() elsewhere.
3486 *
3487 * Return: The original value of @v.
3488 */
3489static __always_inline s64
3490raw_atomic64_fetch_dec_release(atomic64_t *v)
3491{
3492#if defined(arch_atomic64_fetch_dec_release)
3493 return arch_atomic64_fetch_dec_release(v);
3494#elif defined(arch_atomic64_fetch_dec_relaxed)
3495 __atomic_release_fence();
3496 return arch_atomic64_fetch_dec_relaxed(v);
3497#elif defined(arch_atomic64_fetch_dec)
3498 return arch_atomic64_fetch_dec(v);
3499#else
3500 return raw_atomic64_fetch_sub_release(1, v);
3501#endif
3502}
3503
3504/**
3505 * raw_atomic64_fetch_dec_relaxed() - atomic decrement with relaxed ordering
3506 * @v: pointer to atomic64_t
3507 *
3508 * Atomically updates @v to (@v - 1) with relaxed ordering.
3509 *
3510 * Safe to use in noinstr code; prefer atomic64_fetch_dec_relaxed() elsewhere.
3511 *
3512 * Return: The original value of @v.
3513 */
3514static __always_inline s64
3515raw_atomic64_fetch_dec_relaxed(atomic64_t *v)
3516{
3517#if defined(arch_atomic64_fetch_dec_relaxed)
3518 return arch_atomic64_fetch_dec_relaxed(v);
3519#elif defined(arch_atomic64_fetch_dec)
3520 return arch_atomic64_fetch_dec(v);
3521#else
3522 return raw_atomic64_fetch_sub_relaxed(1, v);
3523#endif
3524}
3525
3526/**
3527 * raw_atomic64_and() - atomic bitwise AND with relaxed ordering
3528 * @i: s64 value
3529 * @v: pointer to atomic64_t
3530 *
3531 * Atomically updates @v to (@v & @i) with relaxed ordering.
3532 *
3533 * Safe to use in noinstr code; prefer atomic64_and() elsewhere.
3534 *
3535 * Return: Nothing.
3536 */
3537static __always_inline void
3538raw_atomic64_and(s64 i, atomic64_t *v)
3539{
3540 arch_atomic64_and(i, v);
3541}
3542
3543/**
3544 * raw_atomic64_fetch_and() - atomic bitwise AND with full ordering
3545 * @i: s64 value
3546 * @v: pointer to atomic64_t
3547 *
3548 * Atomically updates @v to (@v & @i) with full ordering.
3549 *
3550 * Safe to use in noinstr code; prefer atomic64_fetch_and() elsewhere.
3551 *
3552 * Return: The original value of @v.
3553 */
3554static __always_inline s64
3555raw_atomic64_fetch_and(s64 i, atomic64_t *v)
3556{
3557#if defined(arch_atomic64_fetch_and)
3558 return arch_atomic64_fetch_and(i, v);
3559#elif defined(arch_atomic64_fetch_and_relaxed)
3560 s64 ret;
3561 __atomic_pre_full_fence();
3562 ret = arch_atomic64_fetch_and_relaxed(i, v);
3563 __atomic_post_full_fence();
3564 return ret;
3565#else
3566#error "Unable to define raw_atomic64_fetch_and"
3567#endif
3568}
3569
3570/**
3571 * raw_atomic64_fetch_and_acquire() - atomic bitwise AND with acquire ordering
3572 * @i: s64 value
3573 * @v: pointer to atomic64_t
3574 *
3575 * Atomically updates @v to (@v & @i) with acquire ordering.
3576 *
3577 * Safe to use in noinstr code; prefer atomic64_fetch_and_acquire() elsewhere.
3578 *
3579 * Return: The original value of @v.
3580 */
3581static __always_inline s64
3582raw_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
3583{
3584#if defined(arch_atomic64_fetch_and_acquire)
3585 return arch_atomic64_fetch_and_acquire(i, v);
3586#elif defined(arch_atomic64_fetch_and_relaxed)
3587 s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
3588 __atomic_acquire_fence();
3589 return ret;
3590#elif defined(arch_atomic64_fetch_and)
3591 return arch_atomic64_fetch_and(i, v);
3592#else
3593#error "Unable to define raw_atomic64_fetch_and_acquire"
3594#endif
3595}
3596
3597/**
3598 * raw_atomic64_fetch_and_release() - atomic bitwise AND with release ordering
3599 * @i: s64 value
3600 * @v: pointer to atomic64_t
3601 *
3602 * Atomically updates @v to (@v & @i) with release ordering.
3603 *
3604 * Safe to use in noinstr code; prefer atomic64_fetch_and_release() elsewhere.
3605 *
3606 * Return: The original value of @v.
3607 */
3608static __always_inline s64
3609raw_atomic64_fetch_and_release(s64 i, atomic64_t *v)
3610{
3611#if defined(arch_atomic64_fetch_and_release)
3612 return arch_atomic64_fetch_and_release(i, v);
3613#elif defined(arch_atomic64_fetch_and_relaxed)
3614 __atomic_release_fence();
3615 return arch_atomic64_fetch_and_relaxed(i, v);
3616#elif defined(arch_atomic64_fetch_and)
3617 return arch_atomic64_fetch_and(i, v);
3618#else
3619#error "Unable to define raw_atomic64_fetch_and_release"
3620#endif
3621}
3622
3623/**
3624 * raw_atomic64_fetch_and_relaxed() - atomic bitwise AND with relaxed ordering
3625 * @i: s64 value
3626 * @v: pointer to atomic64_t
3627 *
3628 * Atomically updates @v to (@v & @i) with relaxed ordering.
3629 *
3630 * Safe to use in noinstr code; prefer atomic64_fetch_and_relaxed() elsewhere.
3631 *
3632 * Return: The original value of @v.
3633 */
3634static __always_inline s64
3635raw_atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
3636{
3637#if defined(arch_atomic64_fetch_and_relaxed)
3638 return arch_atomic64_fetch_and_relaxed(i, v);
3639#elif defined(arch_atomic64_fetch_and)
3640 return arch_atomic64_fetch_and(i, v);
3641#else
3642#error "Unable to define raw_atomic64_fetch_and_relaxed"
3643#endif
3644}
3645
3646/**
3647 * raw_atomic64_andnot() - atomic bitwise AND NOT with relaxed ordering
3648 * @i: s64 value
3649 * @v: pointer to atomic64_t
3650 *
3651 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3652 *
3653 * Safe to use in noinstr code; prefer atomic64_andnot() elsewhere.
3654 *
3655 * Return: Nothing.
3656 */
3657static __always_inline void
3658raw_atomic64_andnot(s64 i, atomic64_t *v)
3659{
3660#if defined(arch_atomic64_andnot)
3661 arch_atomic64_andnot(i, v);
3662#else
3663 raw_atomic64_and(~i, v);
3664#endif
3665}
3666
3667/**
3668 * raw_atomic64_fetch_andnot() - atomic bitwise AND NOT with full ordering
3669 * @i: s64 value
3670 * @v: pointer to atomic64_t
3671 *
3672 * Atomically updates @v to (@v & ~@i) with full ordering.
3673 *
3674 * Safe to use in noinstr code; prefer atomic64_fetch_andnot() elsewhere.
3675 *
3676 * Return: The original value of @v.
3677 */
3678static __always_inline s64
3679raw_atomic64_fetch_andnot(s64 i, atomic64_t *v)
3680{
3681#if defined(arch_atomic64_fetch_andnot)
3682 return arch_atomic64_fetch_andnot(i, v);
3683#elif defined(arch_atomic64_fetch_andnot_relaxed)
3684 s64 ret;
3685 __atomic_pre_full_fence();
3686 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3687 __atomic_post_full_fence();
3688 return ret;
3689#else
3690 return raw_atomic64_fetch_and(~i, v);
3691#endif
3692}
3693
3694/**
3695 * raw_atomic64_fetch_andnot_acquire() - atomic bitwise AND NOT with acquire ordering
3696 * @i: s64 value
3697 * @v: pointer to atomic64_t
3698 *
3699 * Atomically updates @v to (@v & ~@i) with acquire ordering.
3700 *
3701 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_acquire() elsewhere.
3702 *
3703 * Return: The original value of @v.
3704 */
3705static __always_inline s64
3706raw_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
3707{
3708#if defined(arch_atomic64_fetch_andnot_acquire)
3709 return arch_atomic64_fetch_andnot_acquire(i, v);
3710#elif defined(arch_atomic64_fetch_andnot_relaxed)
3711 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
3712 __atomic_acquire_fence();
3713 return ret;
3714#elif defined(arch_atomic64_fetch_andnot)
3715 return arch_atomic64_fetch_andnot(i, v);
3716#else
3717 return raw_atomic64_fetch_and_acquire(~i, v);
3718#endif
3719}
3720
3721/**
3722 * raw_atomic64_fetch_andnot_release() - atomic bitwise AND NOT with release ordering
3723 * @i: s64 value
3724 * @v: pointer to atomic64_t
3725 *
3726 * Atomically updates @v to (@v & ~@i) with release ordering.
3727 *
3728 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_release() elsewhere.
3729 *
3730 * Return: The original value of @v.
3731 */
3732static __always_inline s64
3733raw_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
3734{
3735#if defined(arch_atomic64_fetch_andnot_release)
3736 return arch_atomic64_fetch_andnot_release(i, v);
3737#elif defined(arch_atomic64_fetch_andnot_relaxed)
3738 __atomic_release_fence();
3739 return arch_atomic64_fetch_andnot_relaxed(i, v);
3740#elif defined(arch_atomic64_fetch_andnot)
3741 return arch_atomic64_fetch_andnot(i, v);
3742#else
3743 return raw_atomic64_fetch_and_release(~i, v);
3744#endif
3745}
3746
3747/**
3748 * raw_atomic64_fetch_andnot_relaxed() - atomic bitwise AND NOT with relaxed ordering
3749 * @i: s64 value
3750 * @v: pointer to atomic64_t
3751 *
3752 * Atomically updates @v to (@v & ~@i) with relaxed ordering.
3753 *
3754 * Safe to use in noinstr code; prefer atomic64_fetch_andnot_relaxed() elsewhere.
3755 *
3756 * Return: The original value of @v.
3757 */
3758static __always_inline s64
3759raw_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
3760{
3761#if defined(arch_atomic64_fetch_andnot_relaxed)
3762 return arch_atomic64_fetch_andnot_relaxed(i, v);
3763#elif defined(arch_atomic64_fetch_andnot)
3764 return arch_atomic64_fetch_andnot(i, v);
3765#else
3766 return raw_atomic64_fetch_and_relaxed(~i, v);
3767#endif
3768}
3769
3770/**
3771 * raw_atomic64_or() - atomic bitwise OR with relaxed ordering
3772 * @i: s64 value
3773 * @v: pointer to atomic64_t
3774 *
3775 * Atomically updates @v to (@v | @i) with relaxed ordering.
3776 *
3777 * Safe to use in noinstr code; prefer atomic64_or() elsewhere.
3778 *
3779 * Return: Nothing.
3780 */
3781static __always_inline void
3782raw_atomic64_or(s64 i, atomic64_t *v)
3783{
3784 arch_atomic64_or(i, v);
3785}
3786
3787/**
3788 * raw_atomic64_fetch_or() - atomic bitwise OR with full ordering
3789 * @i: s64 value
3790 * @v: pointer to atomic64_t
3791 *
3792 * Atomically updates @v to (@v | @i) with full ordering.
3793 *
3794 * Safe to use in noinstr code; prefer atomic64_fetch_or() elsewhere.
3795 *
3796 * Return: The original value of @v.
3797 */
3798static __always_inline s64
3799raw_atomic64_fetch_or(s64 i, atomic64_t *v)
3800{
3801#if defined(arch_atomic64_fetch_or)
3802 return arch_atomic64_fetch_or(i, v);
3803#elif defined(arch_atomic64_fetch_or_relaxed)
3804 s64 ret;
3805 __atomic_pre_full_fence();
3806 ret = arch_atomic64_fetch_or_relaxed(i, v);
3807 __atomic_post_full_fence();
3808 return ret;
3809#else
3810#error "Unable to define raw_atomic64_fetch_or"
3811#endif
3812}
3813
3814/**
3815 * raw_atomic64_fetch_or_acquire() - atomic bitwise OR with acquire ordering
3816 * @i: s64 value
3817 * @v: pointer to atomic64_t
3818 *
3819 * Atomically updates @v to (@v | @i) with acquire ordering.
3820 *
3821 * Safe to use in noinstr code; prefer atomic64_fetch_or_acquire() elsewhere.
3822 *
3823 * Return: The original value of @v.
3824 */
3825static __always_inline s64
3826raw_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
3827{
3828#if defined(arch_atomic64_fetch_or_acquire)
3829 return arch_atomic64_fetch_or_acquire(i, v);
3830#elif defined(arch_atomic64_fetch_or_relaxed)
3831 s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
3832 __atomic_acquire_fence();
3833 return ret;
3834#elif defined(arch_atomic64_fetch_or)
3835 return arch_atomic64_fetch_or(i, v);
3836#else
3837#error "Unable to define raw_atomic64_fetch_or_acquire"
3838#endif
3839}
3840
3841/**
3842 * raw_atomic64_fetch_or_release() - atomic bitwise OR with release ordering
3843 * @i: s64 value
3844 * @v: pointer to atomic64_t
3845 *
3846 * Atomically updates @v to (@v | @i) with release ordering.
3847 *
3848 * Safe to use in noinstr code; prefer atomic64_fetch_or_release() elsewhere.
3849 *
3850 * Return: The original value of @v.
3851 */
3852static __always_inline s64
3853raw_atomic64_fetch_or_release(s64 i, atomic64_t *v)
3854{
3855#if defined(arch_atomic64_fetch_or_release)
3856 return arch_atomic64_fetch_or_release(i, v);
3857#elif defined(arch_atomic64_fetch_or_relaxed)
3858 __atomic_release_fence();
3859 return arch_atomic64_fetch_or_relaxed(i, v);
3860#elif defined(arch_atomic64_fetch_or)
3861 return arch_atomic64_fetch_or(i, v);
3862#else
3863#error "Unable to define raw_atomic64_fetch_or_release"
3864#endif
3865}
3866
3867/**
3868 * raw_atomic64_fetch_or_relaxed() - atomic bitwise OR with relaxed ordering
3869 * @i: s64 value
3870 * @v: pointer to atomic64_t
3871 *
3872 * Atomically updates @v to (@v | @i) with relaxed ordering.
3873 *
3874 * Safe to use in noinstr code; prefer atomic64_fetch_or_relaxed() elsewhere.
3875 *
3876 * Return: The original value of @v.
3877 */
3878static __always_inline s64
3879raw_atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
3880{
3881#if defined(arch_atomic64_fetch_or_relaxed)
3882 return arch_atomic64_fetch_or_relaxed(i, v);
3883#elif defined(arch_atomic64_fetch_or)
3884 return arch_atomic64_fetch_or(i, v);
3885#else
3886#error "Unable to define raw_atomic64_fetch_or_relaxed"
3887#endif
3888}
3889
3890/**
3891 * raw_atomic64_xor() - atomic bitwise XOR with relaxed ordering
3892 * @i: s64 value
3893 * @v: pointer to atomic64_t
3894 *
3895 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
3896 *
3897 * Safe to use in noinstr code; prefer atomic64_xor() elsewhere.
3898 *
3899 * Return: Nothing.
3900 */
3901static __always_inline void
3902raw_atomic64_xor(s64 i, atomic64_t *v)
3903{
3904 arch_atomic64_xor(i, v);
3905}
3906
3907/**
3908 * raw_atomic64_fetch_xor() - atomic bitwise XOR with full ordering
3909 * @i: s64 value
3910 * @v: pointer to atomic64_t
3911 *
3912 * Atomically updates @v to (@v ^ @i) with full ordering.
3913 *
3914 * Safe to use in noinstr code; prefer atomic64_fetch_xor() elsewhere.
3915 *
3916 * Return: The original value of @v.
3917 */
3918static __always_inline s64
3919raw_atomic64_fetch_xor(s64 i, atomic64_t *v)
3920{
3921#if defined(arch_atomic64_fetch_xor)
3922 return arch_atomic64_fetch_xor(i, v);
3923#elif defined(arch_atomic64_fetch_xor_relaxed)
3924 s64 ret;
3925 __atomic_pre_full_fence();
3926 ret = arch_atomic64_fetch_xor_relaxed(i, v);
3927 __atomic_post_full_fence();
3928 return ret;
3929#else
3930#error "Unable to define raw_atomic64_fetch_xor"
3931#endif
3932}
3933
3934/**
3935 * raw_atomic64_fetch_xor_acquire() - atomic bitwise XOR with acquire ordering
3936 * @i: s64 value
3937 * @v: pointer to atomic64_t
3938 *
3939 * Atomically updates @v to (@v ^ @i) with acquire ordering.
3940 *
3941 * Safe to use in noinstr code; prefer atomic64_fetch_xor_acquire() elsewhere.
3942 *
3943 * Return: The original value of @v.
3944 */
3945static __always_inline s64
3946raw_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
3947{
3948#if defined(arch_atomic64_fetch_xor_acquire)
3949 return arch_atomic64_fetch_xor_acquire(i, v);
3950#elif defined(arch_atomic64_fetch_xor_relaxed)
3951 s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
3952 __atomic_acquire_fence();
3953 return ret;
3954#elif defined(arch_atomic64_fetch_xor)
3955 return arch_atomic64_fetch_xor(i, v);
3956#else
3957#error "Unable to define raw_atomic64_fetch_xor_acquire"
3958#endif
3959}
3960
3961/**
3962 * raw_atomic64_fetch_xor_release() - atomic bitwise XOR with release ordering
3963 * @i: s64 value
3964 * @v: pointer to atomic64_t
3965 *
3966 * Atomically updates @v to (@v ^ @i) with release ordering.
3967 *
3968 * Safe to use in noinstr code; prefer atomic64_fetch_xor_release() elsewhere.
3969 *
3970 * Return: The original value of @v.
3971 */
3972static __always_inline s64
3973raw_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
3974{
3975#if defined(arch_atomic64_fetch_xor_release)
3976 return arch_atomic64_fetch_xor_release(i, v);
3977#elif defined(arch_atomic64_fetch_xor_relaxed)
3978 __atomic_release_fence();
3979 return arch_atomic64_fetch_xor_relaxed(i, v);
3980#elif defined(arch_atomic64_fetch_xor)
3981 return arch_atomic64_fetch_xor(i, v);
3982#else
3983#error "Unable to define raw_atomic64_fetch_xor_release"
3984#endif
3985}
3986
3987/**
3988 * raw_atomic64_fetch_xor_relaxed() - atomic bitwise XOR with relaxed ordering
3989 * @i: s64 value
3990 * @v: pointer to atomic64_t
3991 *
3992 * Atomically updates @v to (@v ^ @i) with relaxed ordering.
3993 *
3994 * Safe to use in noinstr code; prefer atomic64_fetch_xor_relaxed() elsewhere.
3995 *
3996 * Return: The original value of @v.
3997 */
3998static __always_inline s64
3999raw_atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
4000{
4001#if defined(arch_atomic64_fetch_xor_relaxed)
4002 return arch_atomic64_fetch_xor_relaxed(i, v);
4003#elif defined(arch_atomic64_fetch_xor)
4004 return arch_atomic64_fetch_xor(i, v);
4005#else
4006#error "Unable to define raw_atomic64_fetch_xor_relaxed"
4007#endif
4008}
4009
4010/**
4011 * raw_atomic64_xchg() - atomic exchange with full ordering
4012 * @v: pointer to atomic64_t
4013 * @new: s64 value to assign
4014 *
4015 * Atomically updates @v to @new with full ordering.
4016 *
4017 * Safe to use in noinstr code; prefer atomic64_xchg() elsewhere.
4018 *
4019 * Return: The original value of @v.
4020 */
4021static __always_inline s64
4022raw_atomic64_xchg(atomic64_t *v, s64 new)
4023{
4024#if defined(arch_atomic64_xchg)
4025 return arch_atomic64_xchg(v, new);
4026#elif defined(arch_atomic64_xchg_relaxed)
4027 s64 ret;
4028 __atomic_pre_full_fence();
4029 ret = arch_atomic64_xchg_relaxed(v, new);
4030 __atomic_post_full_fence();
4031 return ret;
4032#else
4033 return raw_xchg(&v->counter, new);
4034#endif
4035}
4036
4037/**
4038 * raw_atomic64_xchg_acquire() - atomic exchange with acquire ordering
4039 * @v: pointer to atomic64_t
4040 * @new: s64 value to assign
4041 *
4042 * Atomically updates @v to @new with acquire ordering.
4043 *
4044 * Safe to use in noinstr code; prefer atomic64_xchg_acquire() elsewhere.
4045 *
4046 * Return: The original value of @v.
4047 */
4048static __always_inline s64
4049raw_atomic64_xchg_acquire(atomic64_t *v, s64 new)
4050{
4051#if defined(arch_atomic64_xchg_acquire)
4052 return arch_atomic64_xchg_acquire(v, new);
4053#elif defined(arch_atomic64_xchg_relaxed)
4054 s64 ret = arch_atomic64_xchg_relaxed(v, new);
4055 __atomic_acquire_fence();
4056 return ret;
4057#elif defined(arch_atomic64_xchg)
4058 return arch_atomic64_xchg(v, new);
4059#else
4060 return raw_xchg_acquire(&v->counter, new);
4061#endif
4062}
4063
4064/**
4065 * raw_atomic64_xchg_release() - atomic exchange with release ordering
4066 * @v: pointer to atomic64_t
4067 * @new: s64 value to assign
4068 *
4069 * Atomically updates @v to @new with release ordering.
4070 *
4071 * Safe to use in noinstr code; prefer atomic64_xchg_release() elsewhere.
4072 *
4073 * Return: The original value of @v.
4074 */
4075static __always_inline s64
4076raw_atomic64_xchg_release(atomic64_t *v, s64 new)
4077{
4078#if defined(arch_atomic64_xchg_release)
4079 return arch_atomic64_xchg_release(v, new);
4080#elif defined(arch_atomic64_xchg_relaxed)
4081 __atomic_release_fence();
4082 return arch_atomic64_xchg_relaxed(v, new);
4083#elif defined(arch_atomic64_xchg)
4084 return arch_atomic64_xchg(v, new);
4085#else
4086 return raw_xchg_release(&v->counter, new);
4087#endif
4088}
4089
4090/**
4091 * raw_atomic64_xchg_relaxed() - atomic exchange with relaxed ordering
4092 * @v: pointer to atomic64_t
4093 * @new: s64 value to assign
4094 *
4095 * Atomically updates @v to @new with relaxed ordering.
4096 *
4097 * Safe to use in noinstr code; prefer atomic64_xchg_relaxed() elsewhere.
4098 *
4099 * Return: The original value of @v.
4100 */
4101static __always_inline s64
4102raw_atomic64_xchg_relaxed(atomic64_t *v, s64 new)
4103{
4104#if defined(arch_atomic64_xchg_relaxed)
4105 return arch_atomic64_xchg_relaxed(v, new);
4106#elif defined(arch_atomic64_xchg)
4107 return arch_atomic64_xchg(v, new);
4108#else
4109 return raw_xchg_relaxed(&v->counter, new);
4110#endif
4111}
4112
4113/**
4114 * raw_atomic64_cmpxchg() - atomic compare and exchange with full ordering
4115 * @v: pointer to atomic64_t
4116 * @old: s64 value to compare with
4117 * @new: s64 value to assign
4118 *
4119 * If (@v == @old), atomically updates @v to @new with full ordering.
4120 *
4121 * Safe to use in noinstr code; prefer atomic64_cmpxchg() elsewhere.
4122 *
4123 * Return: The original value of @v.
4124 */
4125static __always_inline s64
4126raw_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
4127{
4128#if defined(arch_atomic64_cmpxchg)
4129 return arch_atomic64_cmpxchg(v, old, new);
4130#elif defined(arch_atomic64_cmpxchg_relaxed)
4131 s64 ret;
4132 __atomic_pre_full_fence();
4133 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
4134 __atomic_post_full_fence();
4135 return ret;
4136#else
4137 return raw_cmpxchg(&v->counter, old, new);
4138#endif
4139}
4140
4141/**
4142 * raw_atomic64_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4143 * @v: pointer to atomic64_t
4144 * @old: s64 value to compare with
4145 * @new: s64 value to assign
4146 *
4147 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4148 *
4149 * Safe to use in noinstr code; prefer atomic64_cmpxchg_acquire() elsewhere.
4150 *
4151 * Return: The original value of @v.
4152 */
4153static __always_inline s64
4154raw_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
4155{
4156#if defined(arch_atomic64_cmpxchg_acquire)
4157 return arch_atomic64_cmpxchg_acquire(v, old, new);
4158#elif defined(arch_atomic64_cmpxchg_relaxed)
4159 s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
4160 __atomic_acquire_fence();
4161 return ret;
4162#elif defined(arch_atomic64_cmpxchg)
4163 return arch_atomic64_cmpxchg(v, old, new);
4164#else
4165 return raw_cmpxchg_acquire(&v->counter, old, new);
4166#endif
4167}
4168
4169/**
4170 * raw_atomic64_cmpxchg_release() - atomic compare and exchange with release ordering
4171 * @v: pointer to atomic64_t
4172 * @old: s64 value to compare with
4173 * @new: s64 value to assign
4174 *
4175 * If (@v == @old), atomically updates @v to @new with release ordering.
4176 *
4177 * Safe to use in noinstr code; prefer atomic64_cmpxchg_release() elsewhere.
4178 *
4179 * Return: The original value of @v.
4180 */
4181static __always_inline s64
4182raw_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
4183{
4184#if defined(arch_atomic64_cmpxchg_release)
4185 return arch_atomic64_cmpxchg_release(v, old, new);
4186#elif defined(arch_atomic64_cmpxchg_relaxed)
4187 __atomic_release_fence();
4188 return arch_atomic64_cmpxchg_relaxed(v, old, new);
4189#elif defined(arch_atomic64_cmpxchg)
4190 return arch_atomic64_cmpxchg(v, old, new);
4191#else
4192 return raw_cmpxchg_release(&v->counter, old, new);
4193#endif
4194}
4195
4196/**
4197 * raw_atomic64_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4198 * @v: pointer to atomic64_t
4199 * @old: s64 value to compare with
4200 * @new: s64 value to assign
4201 *
4202 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4203 *
4204 * Safe to use in noinstr code; prefer atomic64_cmpxchg_relaxed() elsewhere.
4205 *
4206 * Return: The original value of @v.
4207 */
4208static __always_inline s64
4209raw_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
4210{
4211#if defined(arch_atomic64_cmpxchg_relaxed)
4212 return arch_atomic64_cmpxchg_relaxed(v, old, new);
4213#elif defined(arch_atomic64_cmpxchg)
4214 return arch_atomic64_cmpxchg(v, old, new);
4215#else
4216 return raw_cmpxchg_relaxed(&v->counter, old, new);
4217#endif
4218}
4219
4220/**
4221 * raw_atomic64_try_cmpxchg() - atomic compare and exchange with full ordering
4222 * @v: pointer to atomic64_t
4223 * @old: pointer to s64 value to compare with
4224 * @new: s64 value to assign
4225 *
4226 * If (@v == @old), atomically updates @v to @new with full ordering.
4227 * Otherwise, updates @old to the current value of @v.
4228 *
4229 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg() elsewhere.
4230 *
4231 * Return: @true if the exchange occured, @false otherwise.
4232 */
4233static __always_inline bool
4234raw_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
4235{
4236#if defined(arch_atomic64_try_cmpxchg)
4237 return arch_atomic64_try_cmpxchg(v, old, new);
4238#elif defined(arch_atomic64_try_cmpxchg_relaxed)
4239 bool ret;
4240 __atomic_pre_full_fence();
4241 ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4242 __atomic_post_full_fence();
4243 return ret;
4244#else
4245 s64 r, o = *old;
4246 r = raw_atomic64_cmpxchg(v, o, new);
4247 if (unlikely(r != o))
4248 *old = r;
4249 return likely(r == o);
4250#endif
4251}
4252
4253/**
4254 * raw_atomic64_try_cmpxchg_acquire() - atomic compare and exchange with acquire ordering
4255 * @v: pointer to atomic64_t
4256 * @old: pointer to s64 value to compare with
4257 * @new: s64 value to assign
4258 *
4259 * If (@v == @old), atomically updates @v to @new with acquire ordering.
4260 * Otherwise, updates @old to the current value of @v.
4261 *
4262 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_acquire() elsewhere.
4263 *
4264 * Return: @true if the exchange occured, @false otherwise.
4265 */
4266static __always_inline bool
4267raw_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
4268{
4269#if defined(arch_atomic64_try_cmpxchg_acquire)
4270 return arch_atomic64_try_cmpxchg_acquire(v, old, new);
4271#elif defined(arch_atomic64_try_cmpxchg_relaxed)
4272 bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4273 __atomic_acquire_fence();
4274 return ret;
4275#elif defined(arch_atomic64_try_cmpxchg)
4276 return arch_atomic64_try_cmpxchg(v, old, new);
4277#else
4278 s64 r, o = *old;
4279 r = raw_atomic64_cmpxchg_acquire(v, o, new);
4280 if (unlikely(r != o))
4281 *old = r;
4282 return likely(r == o);
4283#endif
4284}
4285
4286/**
4287 * raw_atomic64_try_cmpxchg_release() - atomic compare and exchange with release ordering
4288 * @v: pointer to atomic64_t
4289 * @old: pointer to s64 value to compare with
4290 * @new: s64 value to assign
4291 *
4292 * If (@v == @old), atomically updates @v to @new with release ordering.
4293 * Otherwise, updates @old to the current value of @v.
4294 *
4295 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_release() elsewhere.
4296 *
4297 * Return: @true if the exchange occured, @false otherwise.
4298 */
4299static __always_inline bool
4300raw_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
4301{
4302#if defined(arch_atomic64_try_cmpxchg_release)
4303 return arch_atomic64_try_cmpxchg_release(v, old, new);
4304#elif defined(arch_atomic64_try_cmpxchg_relaxed)
4305 __atomic_release_fence();
4306 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4307#elif defined(arch_atomic64_try_cmpxchg)
4308 return arch_atomic64_try_cmpxchg(v, old, new);
4309#else
4310 s64 r, o = *old;
4311 r = raw_atomic64_cmpxchg_release(v, o, new);
4312 if (unlikely(r != o))
4313 *old = r;
4314 return likely(r == o);
4315#endif
4316}
4317
4318/**
4319 * raw_atomic64_try_cmpxchg_relaxed() - atomic compare and exchange with relaxed ordering
4320 * @v: pointer to atomic64_t
4321 * @old: pointer to s64 value to compare with
4322 * @new: s64 value to assign
4323 *
4324 * If (@v == @old), atomically updates @v to @new with relaxed ordering.
4325 * Otherwise, updates @old to the current value of @v.
4326 *
4327 * Safe to use in noinstr code; prefer atomic64_try_cmpxchg_relaxed() elsewhere.
4328 *
4329 * Return: @true if the exchange occured, @false otherwise.
4330 */
4331static __always_inline bool
4332raw_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
4333{
4334#if defined(arch_atomic64_try_cmpxchg_relaxed)
4335 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
4336#elif defined(arch_atomic64_try_cmpxchg)
4337 return arch_atomic64_try_cmpxchg(v, old, new);
4338#else
4339 s64 r, o = *old;
4340 r = raw_atomic64_cmpxchg_relaxed(v, o, new);
4341 if (unlikely(r != o))
4342 *old = r;
4343 return likely(r == o);
4344#endif
4345}
4346
4347/**
4348 * raw_atomic64_sub_and_test() - atomic subtract and test if zero with full ordering
4349 * @i: s64 value to add
4350 * @v: pointer to atomic64_t
4351 *
4352 * Atomically updates @v to (@v - @i) with full ordering.
4353 *
4354 * Safe to use in noinstr code; prefer atomic64_sub_and_test() elsewhere.
4355 *
4356 * Return: @true if the resulting value of @v is zero, @false otherwise.
4357 */
4358static __always_inline bool
4359raw_atomic64_sub_and_test(s64 i, atomic64_t *v)
4360{
4361#if defined(arch_atomic64_sub_and_test)
4362 return arch_atomic64_sub_and_test(i, v);
4363#else
4364 return raw_atomic64_sub_return(i, v) == 0;
4365#endif
4366}
4367
4368/**
4369 * raw_atomic64_dec_and_test() - atomic decrement and test if zero with full ordering
4370 * @v: pointer to atomic64_t
4371 *
4372 * Atomically updates @v to (@v - 1) with full ordering.
4373 *
4374 * Safe to use in noinstr code; prefer atomic64_dec_and_test() elsewhere.
4375 *
4376 * Return: @true if the resulting value of @v is zero, @false otherwise.
4377 */
4378static __always_inline bool
4379raw_atomic64_dec_and_test(atomic64_t *v)
4380{
4381#if defined(arch_atomic64_dec_and_test)
4382 return arch_atomic64_dec_and_test(v);
4383#else
4384 return raw_atomic64_dec_return(v) == 0;
4385#endif
4386}
4387
4388/**
4389 * raw_atomic64_inc_and_test() - atomic increment and test if zero with full ordering
4390 * @v: pointer to atomic64_t
4391 *
4392 * Atomically updates @v to (@v + 1) with full ordering.
4393 *
4394 * Safe to use in noinstr code; prefer atomic64_inc_and_test() elsewhere.
4395 *
4396 * Return: @true if the resulting value of @v is zero, @false otherwise.
4397 */
4398static __always_inline bool
4399raw_atomic64_inc_and_test(atomic64_t *v)
4400{
4401#if defined(arch_atomic64_inc_and_test)
4402 return arch_atomic64_inc_and_test(v);
4403#else
4404 return raw_atomic64_inc_return(v) == 0;
4405#endif
4406}
4407
4408/**
4409 * raw_atomic64_add_negative() - atomic add and test if negative with full ordering
4410 * @i: s64 value to add
4411 * @v: pointer to atomic64_t
4412 *
4413 * Atomically updates @v to (@v + @i) with full ordering.
4414 *
4415 * Safe to use in noinstr code; prefer atomic64_add_negative() elsewhere.
4416 *
4417 * Return: @true if the resulting value of @v is negative, @false otherwise.
4418 */
4419static __always_inline bool
4420raw_atomic64_add_negative(s64 i, atomic64_t *v)
4421{
4422#if defined(arch_atomic64_add_negative)
4423 return arch_atomic64_add_negative(i, v);
4424#elif defined(arch_atomic64_add_negative_relaxed)
4425 bool ret;
4426 __atomic_pre_full_fence();
4427 ret = arch_atomic64_add_negative_relaxed(i, v);
4428 __atomic_post_full_fence();
4429 return ret;
4430#else
4431 return raw_atomic64_add_return(i, v) < 0;
4432#endif
4433}
4434
4435/**
4436 * raw_atomic64_add_negative_acquire() - atomic add and test if negative with acquire ordering
4437 * @i: s64 value to add
4438 * @v: pointer to atomic64_t
4439 *
4440 * Atomically updates @v to (@v + @i) with acquire ordering.
4441 *
4442 * Safe to use in noinstr code; prefer atomic64_add_negative_acquire() elsewhere.
4443 *
4444 * Return: @true if the resulting value of @v is negative, @false otherwise.
4445 */
4446static __always_inline bool
4447raw_atomic64_add_negative_acquire(s64 i, atomic64_t *v)
4448{
4449#if defined(arch_atomic64_add_negative_acquire)
4450 return arch_atomic64_add_negative_acquire(i, v);
4451#elif defined(arch_atomic64_add_negative_relaxed)
4452 bool ret = arch_atomic64_add_negative_relaxed(i, v);
4453 __atomic_acquire_fence();
4454 return ret;
4455#elif defined(arch_atomic64_add_negative)
4456 return arch_atomic64_add_negative(i, v);
4457#else
4458 return raw_atomic64_add_return_acquire(i, v) < 0;
4459#endif
4460}
4461
4462/**
4463 * raw_atomic64_add_negative_release() - atomic add and test if negative with release ordering
4464 * @i: s64 value to add
4465 * @v: pointer to atomic64_t
4466 *
4467 * Atomically updates @v to (@v + @i) with release ordering.
4468 *
4469 * Safe to use in noinstr code; prefer atomic64_add_negative_release() elsewhere.
4470 *
4471 * Return: @true if the resulting value of @v is negative, @false otherwise.
4472 */
4473static __always_inline bool
4474raw_atomic64_add_negative_release(s64 i, atomic64_t *v)
4475{
4476#if defined(arch_atomic64_add_negative_release)
4477 return arch_atomic64_add_negative_release(i, v);
4478#elif defined(arch_atomic64_add_negative_relaxed)
4479 __atomic_release_fence();
4480 return arch_atomic64_add_negative_relaxed(i, v);
4481#elif defined(arch_atomic64_add_negative)
4482 return arch_atomic64_add_negative(i, v);
4483#else
4484 return raw_atomic64_add_return_release(i, v) < 0;
4485#endif
4486}
4487
4488/**
4489 * raw_atomic64_add_negative_relaxed() - atomic add and test if negative with relaxed ordering
4490 * @i: s64 value to add
4491 * @v: pointer to atomic64_t
4492 *
4493 * Atomically updates @v to (@v + @i) with relaxed ordering.
4494 *
4495 * Safe to use in noinstr code; prefer atomic64_add_negative_relaxed() elsewhere.
4496 *
4497 * Return: @true if the resulting value of @v is negative, @false otherwise.
4498 */
4499static __always_inline bool
4500raw_atomic64_add_negative_relaxed(s64 i, atomic64_t *v)
4501{
4502#if defined(arch_atomic64_add_negative_relaxed)
4503 return arch_atomic64_add_negative_relaxed(i, v);
4504#elif defined(arch_atomic64_add_negative)
4505 return arch_atomic64_add_negative(i, v);
4506#else
4507 return raw_atomic64_add_return_relaxed(i, v) < 0;
4508#endif
4509}
4510
4511/**
4512 * raw_atomic64_fetch_add_unless() - atomic add unless value with full ordering
4513 * @v: pointer to atomic64_t
4514 * @a: s64 value to add
4515 * @u: s64 value to compare with
4516 *
4517 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4518 *
4519 * Safe to use in noinstr code; prefer atomic64_fetch_add_unless() elsewhere.
4520 *
4521 * Return: The original value of @v.
4522 */
4523static __always_inline s64
4524raw_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
4525{
4526#if defined(arch_atomic64_fetch_add_unless)
4527 return arch_atomic64_fetch_add_unless(v, a, u);
4528#else
4529 s64 c = raw_atomic64_read(v);
4530
4531 do {
4532 if (unlikely(c == u))
4533 break;
4534 } while (!raw_atomic64_try_cmpxchg(v, &c, c + a));
4535
4536 return c;
4537#endif
4538}
4539
4540/**
4541 * raw_atomic64_add_unless() - atomic add unless value with full ordering
4542 * @v: pointer to atomic64_t
4543 * @a: s64 value to add
4544 * @u: s64 value to compare with
4545 *
4546 * If (@v != @u), atomically updates @v to (@v + @a) with full ordering.
4547 *
4548 * Safe to use in noinstr code; prefer atomic64_add_unless() elsewhere.
4549 *
4550 * Return: @true if @v was updated, @false otherwise.
4551 */
4552static __always_inline bool
4553raw_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
4554{
4555#if defined(arch_atomic64_add_unless)
4556 return arch_atomic64_add_unless(v, a, u);
4557#else
4558 return raw_atomic64_fetch_add_unless(v, a, u) != u;
4559#endif
4560}
4561
4562/**
4563 * raw_atomic64_inc_not_zero() - atomic increment unless zero with full ordering
4564 * @v: pointer to atomic64_t
4565 *
4566 * If (@v != 0), atomically updates @v to (@v + 1) with full ordering.
4567 *
4568 * Safe to use in noinstr code; prefer atomic64_inc_not_zero() elsewhere.
4569 *
4570 * Return: @true if @v was updated, @false otherwise.
4571 */
4572static __always_inline bool
4573raw_atomic64_inc_not_zero(atomic64_t *v)
4574{
4575#if defined(arch_atomic64_inc_not_zero)
4576 return arch_atomic64_inc_not_zero(v);
4577#else
4578 return raw_atomic64_add_unless(v, 1, 0);
4579#endif
4580}
4581
4582/**
4583 * raw_atomic64_inc_unless_negative() - atomic increment unless negative with full ordering
4584 * @v: pointer to atomic64_t
4585 *
4586 * If (@v >= 0), atomically updates @v to (@v + 1) with full ordering.
4587 *
4588 * Safe to use in noinstr code; prefer atomic64_inc_unless_negative() elsewhere.
4589 *
4590 * Return: @true if @v was updated, @false otherwise.
4591 */
4592static __always_inline bool
4593raw_atomic64_inc_unless_negative(atomic64_t *v)
4594{
4595#if defined(arch_atomic64_inc_unless_negative)
4596 return arch_atomic64_inc_unless_negative(v);
4597#else
4598 s64 c = raw_atomic64_read(v);
4599
4600 do {
4601 if (unlikely(c < 0))
4602 return false;
4603 } while (!raw_atomic64_try_cmpxchg(v, &c, c + 1));
4604
4605 return true;
4606#endif
4607}
4608
4609/**
4610 * raw_atomic64_dec_unless_positive() - atomic decrement unless positive with full ordering
4611 * @v: pointer to atomic64_t
4612 *
4613 * If (@v <= 0), atomically updates @v to (@v - 1) with full ordering.
4614 *
4615 * Safe to use in noinstr code; prefer atomic64_dec_unless_positive() elsewhere.
4616 *
4617 * Return: @true if @v was updated, @false otherwise.
4618 */
4619static __always_inline bool
4620raw_atomic64_dec_unless_positive(atomic64_t *v)
4621{
4622#if defined(arch_atomic64_dec_unless_positive)
4623 return arch_atomic64_dec_unless_positive(v);
4624#else
4625 s64 c = raw_atomic64_read(v);
4626
4627 do {
4628 if (unlikely(c > 0))
4629 return false;
4630 } while (!raw_atomic64_try_cmpxchg(v, &c, c - 1));
4631
4632 return true;
4633#endif
4634}
4635
4636/**
4637 * raw_atomic64_dec_if_positive() - atomic decrement if positive with full ordering
4638 * @v: pointer to atomic64_t
4639 *
4640 * If (@v > 0), atomically updates @v to (@v - 1) with full ordering.
4641 *
4642 * Safe to use in noinstr code; prefer atomic64_dec_if_positive() elsewhere.
4643 *
4644 * Return: The old value of (@v - 1), regardless of whether @v was updated.
4645 */
4646static __always_inline s64
4647raw_atomic64_dec_if_positive(atomic64_t *v)
4648{
4649#if defined(arch_atomic64_dec_if_positive)
4650 return arch_atomic64_dec_if_positive(v);
4651#else
4652 s64 dec, c = raw_atomic64_read(v);
4653
4654 do {
4655 dec = c - 1;
4656 if (unlikely(dec < 0))
4657 break;
4658 } while (!raw_atomic64_try_cmpxchg(v, &c, dec));
4659
4660 return dec;
4661#endif
4662}
4663
4664#endif /* _LINUX_ATOMIC_FALLBACK_H */
4665// eec048affea735b8464f58e6d96992101f8f85f1