Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2
3// Generated by scripts/atomic/gen-atomic-fallback.sh
4// DO NOT MODIFY THIS FILE DIRECTLY
5
6#ifndef _LINUX_ATOMIC_FALLBACK_H
7#define _LINUX_ATOMIC_FALLBACK_H
8
9#include <linux/compiler.h>
10
11#ifndef arch_xchg_relaxed
12#define arch_xchg_acquire arch_xchg
13#define arch_xchg_release arch_xchg
14#define arch_xchg_relaxed arch_xchg
15#else /* arch_xchg_relaxed */
16
17#ifndef arch_xchg_acquire
18#define arch_xchg_acquire(...) \
19 __atomic_op_acquire(arch_xchg, __VA_ARGS__)
20#endif
21
22#ifndef arch_xchg_release
23#define arch_xchg_release(...) \
24 __atomic_op_release(arch_xchg, __VA_ARGS__)
25#endif
26
27#ifndef arch_xchg
28#define arch_xchg(...) \
29 __atomic_op_fence(arch_xchg, __VA_ARGS__)
30#endif
31
32#endif /* arch_xchg_relaxed */
33
34#ifndef arch_cmpxchg_relaxed
35#define arch_cmpxchg_acquire arch_cmpxchg
36#define arch_cmpxchg_release arch_cmpxchg
37#define arch_cmpxchg_relaxed arch_cmpxchg
38#else /* arch_cmpxchg_relaxed */
39
40#ifndef arch_cmpxchg_acquire
41#define arch_cmpxchg_acquire(...) \
42 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
43#endif
44
45#ifndef arch_cmpxchg_release
46#define arch_cmpxchg_release(...) \
47 __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
48#endif
49
50#ifndef arch_cmpxchg
51#define arch_cmpxchg(...) \
52 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
53#endif
54
55#endif /* arch_cmpxchg_relaxed */
56
57#ifndef arch_cmpxchg64_relaxed
58#define arch_cmpxchg64_acquire arch_cmpxchg64
59#define arch_cmpxchg64_release arch_cmpxchg64
60#define arch_cmpxchg64_relaxed arch_cmpxchg64
61#else /* arch_cmpxchg64_relaxed */
62
63#ifndef arch_cmpxchg64_acquire
64#define arch_cmpxchg64_acquire(...) \
65 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
66#endif
67
68#ifndef arch_cmpxchg64_release
69#define arch_cmpxchg64_release(...) \
70 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
71#endif
72
73#ifndef arch_cmpxchg64
74#define arch_cmpxchg64(...) \
75 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
76#endif
77
78#endif /* arch_cmpxchg64_relaxed */
79
80#ifndef arch_try_cmpxchg_relaxed
81#ifdef arch_try_cmpxchg
82#define arch_try_cmpxchg_acquire arch_try_cmpxchg
83#define arch_try_cmpxchg_release arch_try_cmpxchg
84#define arch_try_cmpxchg_relaxed arch_try_cmpxchg
85#endif /* arch_try_cmpxchg */
86
87#ifndef arch_try_cmpxchg
88#define arch_try_cmpxchg(_ptr, _oldp, _new) \
89({ \
90 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
91 ___r = arch_cmpxchg((_ptr), ___o, (_new)); \
92 if (unlikely(___r != ___o)) \
93 *___op = ___r; \
94 likely(___r == ___o); \
95})
96#endif /* arch_try_cmpxchg */
97
98#ifndef arch_try_cmpxchg_acquire
99#define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \
100({ \
101 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
102 ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \
103 if (unlikely(___r != ___o)) \
104 *___op = ___r; \
105 likely(___r == ___o); \
106})
107#endif /* arch_try_cmpxchg_acquire */
108
109#ifndef arch_try_cmpxchg_release
110#define arch_try_cmpxchg_release(_ptr, _oldp, _new) \
111({ \
112 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
113 ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \
114 if (unlikely(___r != ___o)) \
115 *___op = ___r; \
116 likely(___r == ___o); \
117})
118#endif /* arch_try_cmpxchg_release */
119
120#ifndef arch_try_cmpxchg_relaxed
121#define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
122({ \
123 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
124 ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \
125 if (unlikely(___r != ___o)) \
126 *___op = ___r; \
127 likely(___r == ___o); \
128})
129#endif /* arch_try_cmpxchg_relaxed */
130
131#else /* arch_try_cmpxchg_relaxed */
132
133#ifndef arch_try_cmpxchg_acquire
134#define arch_try_cmpxchg_acquire(...) \
135 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
136#endif
137
138#ifndef arch_try_cmpxchg_release
139#define arch_try_cmpxchg_release(...) \
140 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
141#endif
142
143#ifndef arch_try_cmpxchg
144#define arch_try_cmpxchg(...) \
145 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
146#endif
147
148#endif /* arch_try_cmpxchg_relaxed */
149
150#ifndef arch_atomic_read_acquire
151static __always_inline int
152arch_atomic_read_acquire(const atomic_t *v)
153{
154 return smp_load_acquire(&(v)->counter);
155}
156#define arch_atomic_read_acquire arch_atomic_read_acquire
157#endif
158
159#ifndef arch_atomic_set_release
160static __always_inline void
161arch_atomic_set_release(atomic_t *v, int i)
162{
163 smp_store_release(&(v)->counter, i);
164}
165#define arch_atomic_set_release arch_atomic_set_release
166#endif
167
168#ifndef arch_atomic_add_return_relaxed
169#define arch_atomic_add_return_acquire arch_atomic_add_return
170#define arch_atomic_add_return_release arch_atomic_add_return
171#define arch_atomic_add_return_relaxed arch_atomic_add_return
172#else /* arch_atomic_add_return_relaxed */
173
174#ifndef arch_atomic_add_return_acquire
175static __always_inline int
176arch_atomic_add_return_acquire(int i, atomic_t *v)
177{
178 int ret = arch_atomic_add_return_relaxed(i, v);
179 __atomic_acquire_fence();
180 return ret;
181}
182#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
183#endif
184
185#ifndef arch_atomic_add_return_release
186static __always_inline int
187arch_atomic_add_return_release(int i, atomic_t *v)
188{
189 __atomic_release_fence();
190 return arch_atomic_add_return_relaxed(i, v);
191}
192#define arch_atomic_add_return_release arch_atomic_add_return_release
193#endif
194
195#ifndef arch_atomic_add_return
196static __always_inline int
197arch_atomic_add_return(int i, atomic_t *v)
198{
199 int ret;
200 __atomic_pre_full_fence();
201 ret = arch_atomic_add_return_relaxed(i, v);
202 __atomic_post_full_fence();
203 return ret;
204}
205#define arch_atomic_add_return arch_atomic_add_return
206#endif
207
208#endif /* arch_atomic_add_return_relaxed */
209
210#ifndef arch_atomic_fetch_add_relaxed
211#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
212#define arch_atomic_fetch_add_release arch_atomic_fetch_add
213#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
214#else /* arch_atomic_fetch_add_relaxed */
215
216#ifndef arch_atomic_fetch_add_acquire
217static __always_inline int
218arch_atomic_fetch_add_acquire(int i, atomic_t *v)
219{
220 int ret = arch_atomic_fetch_add_relaxed(i, v);
221 __atomic_acquire_fence();
222 return ret;
223}
224#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
225#endif
226
227#ifndef arch_atomic_fetch_add_release
228static __always_inline int
229arch_atomic_fetch_add_release(int i, atomic_t *v)
230{
231 __atomic_release_fence();
232 return arch_atomic_fetch_add_relaxed(i, v);
233}
234#define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
235#endif
236
237#ifndef arch_atomic_fetch_add
238static __always_inline int
239arch_atomic_fetch_add(int i, atomic_t *v)
240{
241 int ret;
242 __atomic_pre_full_fence();
243 ret = arch_atomic_fetch_add_relaxed(i, v);
244 __atomic_post_full_fence();
245 return ret;
246}
247#define arch_atomic_fetch_add arch_atomic_fetch_add
248#endif
249
250#endif /* arch_atomic_fetch_add_relaxed */
251
252#ifndef arch_atomic_sub_return_relaxed
253#define arch_atomic_sub_return_acquire arch_atomic_sub_return
254#define arch_atomic_sub_return_release arch_atomic_sub_return
255#define arch_atomic_sub_return_relaxed arch_atomic_sub_return
256#else /* arch_atomic_sub_return_relaxed */
257
258#ifndef arch_atomic_sub_return_acquire
259static __always_inline int
260arch_atomic_sub_return_acquire(int i, atomic_t *v)
261{
262 int ret = arch_atomic_sub_return_relaxed(i, v);
263 __atomic_acquire_fence();
264 return ret;
265}
266#define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
267#endif
268
269#ifndef arch_atomic_sub_return_release
270static __always_inline int
271arch_atomic_sub_return_release(int i, atomic_t *v)
272{
273 __atomic_release_fence();
274 return arch_atomic_sub_return_relaxed(i, v);
275}
276#define arch_atomic_sub_return_release arch_atomic_sub_return_release
277#endif
278
279#ifndef arch_atomic_sub_return
280static __always_inline int
281arch_atomic_sub_return(int i, atomic_t *v)
282{
283 int ret;
284 __atomic_pre_full_fence();
285 ret = arch_atomic_sub_return_relaxed(i, v);
286 __atomic_post_full_fence();
287 return ret;
288}
289#define arch_atomic_sub_return arch_atomic_sub_return
290#endif
291
292#endif /* arch_atomic_sub_return_relaxed */
293
294#ifndef arch_atomic_fetch_sub_relaxed
295#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
296#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
297#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
298#else /* arch_atomic_fetch_sub_relaxed */
299
300#ifndef arch_atomic_fetch_sub_acquire
301static __always_inline int
302arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
303{
304 int ret = arch_atomic_fetch_sub_relaxed(i, v);
305 __atomic_acquire_fence();
306 return ret;
307}
308#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
309#endif
310
311#ifndef arch_atomic_fetch_sub_release
312static __always_inline int
313arch_atomic_fetch_sub_release(int i, atomic_t *v)
314{
315 __atomic_release_fence();
316 return arch_atomic_fetch_sub_relaxed(i, v);
317}
318#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
319#endif
320
321#ifndef arch_atomic_fetch_sub
322static __always_inline int
323arch_atomic_fetch_sub(int i, atomic_t *v)
324{
325 int ret;
326 __atomic_pre_full_fence();
327 ret = arch_atomic_fetch_sub_relaxed(i, v);
328 __atomic_post_full_fence();
329 return ret;
330}
331#define arch_atomic_fetch_sub arch_atomic_fetch_sub
332#endif
333
334#endif /* arch_atomic_fetch_sub_relaxed */
335
336#ifndef arch_atomic_inc
337static __always_inline void
338arch_atomic_inc(atomic_t *v)
339{
340 arch_atomic_add(1, v);
341}
342#define arch_atomic_inc arch_atomic_inc
343#endif
344
345#ifndef arch_atomic_inc_return_relaxed
346#ifdef arch_atomic_inc_return
347#define arch_atomic_inc_return_acquire arch_atomic_inc_return
348#define arch_atomic_inc_return_release arch_atomic_inc_return
349#define arch_atomic_inc_return_relaxed arch_atomic_inc_return
350#endif /* arch_atomic_inc_return */
351
352#ifndef arch_atomic_inc_return
353static __always_inline int
354arch_atomic_inc_return(atomic_t *v)
355{
356 return arch_atomic_add_return(1, v);
357}
358#define arch_atomic_inc_return arch_atomic_inc_return
359#endif
360
361#ifndef arch_atomic_inc_return_acquire
362static __always_inline int
363arch_atomic_inc_return_acquire(atomic_t *v)
364{
365 return arch_atomic_add_return_acquire(1, v);
366}
367#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
368#endif
369
370#ifndef arch_atomic_inc_return_release
371static __always_inline int
372arch_atomic_inc_return_release(atomic_t *v)
373{
374 return arch_atomic_add_return_release(1, v);
375}
376#define arch_atomic_inc_return_release arch_atomic_inc_return_release
377#endif
378
379#ifndef arch_atomic_inc_return_relaxed
380static __always_inline int
381arch_atomic_inc_return_relaxed(atomic_t *v)
382{
383 return arch_atomic_add_return_relaxed(1, v);
384}
385#define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
386#endif
387
388#else /* arch_atomic_inc_return_relaxed */
389
390#ifndef arch_atomic_inc_return_acquire
391static __always_inline int
392arch_atomic_inc_return_acquire(atomic_t *v)
393{
394 int ret = arch_atomic_inc_return_relaxed(v);
395 __atomic_acquire_fence();
396 return ret;
397}
398#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
399#endif
400
401#ifndef arch_atomic_inc_return_release
402static __always_inline int
403arch_atomic_inc_return_release(atomic_t *v)
404{
405 __atomic_release_fence();
406 return arch_atomic_inc_return_relaxed(v);
407}
408#define arch_atomic_inc_return_release arch_atomic_inc_return_release
409#endif
410
411#ifndef arch_atomic_inc_return
412static __always_inline int
413arch_atomic_inc_return(atomic_t *v)
414{
415 int ret;
416 __atomic_pre_full_fence();
417 ret = arch_atomic_inc_return_relaxed(v);
418 __atomic_post_full_fence();
419 return ret;
420}
421#define arch_atomic_inc_return arch_atomic_inc_return
422#endif
423
424#endif /* arch_atomic_inc_return_relaxed */
425
426#ifndef arch_atomic_fetch_inc_relaxed
427#ifdef arch_atomic_fetch_inc
428#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
429#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
430#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
431#endif /* arch_atomic_fetch_inc */
432
433#ifndef arch_atomic_fetch_inc
434static __always_inline int
435arch_atomic_fetch_inc(atomic_t *v)
436{
437 return arch_atomic_fetch_add(1, v);
438}
439#define arch_atomic_fetch_inc arch_atomic_fetch_inc
440#endif
441
442#ifndef arch_atomic_fetch_inc_acquire
443static __always_inline int
444arch_atomic_fetch_inc_acquire(atomic_t *v)
445{
446 return arch_atomic_fetch_add_acquire(1, v);
447}
448#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
449#endif
450
451#ifndef arch_atomic_fetch_inc_release
452static __always_inline int
453arch_atomic_fetch_inc_release(atomic_t *v)
454{
455 return arch_atomic_fetch_add_release(1, v);
456}
457#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
458#endif
459
460#ifndef arch_atomic_fetch_inc_relaxed
461static __always_inline int
462arch_atomic_fetch_inc_relaxed(atomic_t *v)
463{
464 return arch_atomic_fetch_add_relaxed(1, v);
465}
466#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
467#endif
468
469#else /* arch_atomic_fetch_inc_relaxed */
470
471#ifndef arch_atomic_fetch_inc_acquire
472static __always_inline int
473arch_atomic_fetch_inc_acquire(atomic_t *v)
474{
475 int ret = arch_atomic_fetch_inc_relaxed(v);
476 __atomic_acquire_fence();
477 return ret;
478}
479#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
480#endif
481
482#ifndef arch_atomic_fetch_inc_release
483static __always_inline int
484arch_atomic_fetch_inc_release(atomic_t *v)
485{
486 __atomic_release_fence();
487 return arch_atomic_fetch_inc_relaxed(v);
488}
489#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
490#endif
491
492#ifndef arch_atomic_fetch_inc
493static __always_inline int
494arch_atomic_fetch_inc(atomic_t *v)
495{
496 int ret;
497 __atomic_pre_full_fence();
498 ret = arch_atomic_fetch_inc_relaxed(v);
499 __atomic_post_full_fence();
500 return ret;
501}
502#define arch_atomic_fetch_inc arch_atomic_fetch_inc
503#endif
504
505#endif /* arch_atomic_fetch_inc_relaxed */
506
507#ifndef arch_atomic_dec
508static __always_inline void
509arch_atomic_dec(atomic_t *v)
510{
511 arch_atomic_sub(1, v);
512}
513#define arch_atomic_dec arch_atomic_dec
514#endif
515
516#ifndef arch_atomic_dec_return_relaxed
517#ifdef arch_atomic_dec_return
518#define arch_atomic_dec_return_acquire arch_atomic_dec_return
519#define arch_atomic_dec_return_release arch_atomic_dec_return
520#define arch_atomic_dec_return_relaxed arch_atomic_dec_return
521#endif /* arch_atomic_dec_return */
522
523#ifndef arch_atomic_dec_return
524static __always_inline int
525arch_atomic_dec_return(atomic_t *v)
526{
527 return arch_atomic_sub_return(1, v);
528}
529#define arch_atomic_dec_return arch_atomic_dec_return
530#endif
531
532#ifndef arch_atomic_dec_return_acquire
533static __always_inline int
534arch_atomic_dec_return_acquire(atomic_t *v)
535{
536 return arch_atomic_sub_return_acquire(1, v);
537}
538#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
539#endif
540
541#ifndef arch_atomic_dec_return_release
542static __always_inline int
543arch_atomic_dec_return_release(atomic_t *v)
544{
545 return arch_atomic_sub_return_release(1, v);
546}
547#define arch_atomic_dec_return_release arch_atomic_dec_return_release
548#endif
549
550#ifndef arch_atomic_dec_return_relaxed
551static __always_inline int
552arch_atomic_dec_return_relaxed(atomic_t *v)
553{
554 return arch_atomic_sub_return_relaxed(1, v);
555}
556#define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
557#endif
558
559#else /* arch_atomic_dec_return_relaxed */
560
561#ifndef arch_atomic_dec_return_acquire
562static __always_inline int
563arch_atomic_dec_return_acquire(atomic_t *v)
564{
565 int ret = arch_atomic_dec_return_relaxed(v);
566 __atomic_acquire_fence();
567 return ret;
568}
569#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
570#endif
571
572#ifndef arch_atomic_dec_return_release
573static __always_inline int
574arch_atomic_dec_return_release(atomic_t *v)
575{
576 __atomic_release_fence();
577 return arch_atomic_dec_return_relaxed(v);
578}
579#define arch_atomic_dec_return_release arch_atomic_dec_return_release
580#endif
581
582#ifndef arch_atomic_dec_return
583static __always_inline int
584arch_atomic_dec_return(atomic_t *v)
585{
586 int ret;
587 __atomic_pre_full_fence();
588 ret = arch_atomic_dec_return_relaxed(v);
589 __atomic_post_full_fence();
590 return ret;
591}
592#define arch_atomic_dec_return arch_atomic_dec_return
593#endif
594
595#endif /* arch_atomic_dec_return_relaxed */
596
597#ifndef arch_atomic_fetch_dec_relaxed
598#ifdef arch_atomic_fetch_dec
599#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
600#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
601#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
602#endif /* arch_atomic_fetch_dec */
603
604#ifndef arch_atomic_fetch_dec
605static __always_inline int
606arch_atomic_fetch_dec(atomic_t *v)
607{
608 return arch_atomic_fetch_sub(1, v);
609}
610#define arch_atomic_fetch_dec arch_atomic_fetch_dec
611#endif
612
613#ifndef arch_atomic_fetch_dec_acquire
614static __always_inline int
615arch_atomic_fetch_dec_acquire(atomic_t *v)
616{
617 return arch_atomic_fetch_sub_acquire(1, v);
618}
619#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
620#endif
621
622#ifndef arch_atomic_fetch_dec_release
623static __always_inline int
624arch_atomic_fetch_dec_release(atomic_t *v)
625{
626 return arch_atomic_fetch_sub_release(1, v);
627}
628#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
629#endif
630
631#ifndef arch_atomic_fetch_dec_relaxed
632static __always_inline int
633arch_atomic_fetch_dec_relaxed(atomic_t *v)
634{
635 return arch_atomic_fetch_sub_relaxed(1, v);
636}
637#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
638#endif
639
640#else /* arch_atomic_fetch_dec_relaxed */
641
642#ifndef arch_atomic_fetch_dec_acquire
643static __always_inline int
644arch_atomic_fetch_dec_acquire(atomic_t *v)
645{
646 int ret = arch_atomic_fetch_dec_relaxed(v);
647 __atomic_acquire_fence();
648 return ret;
649}
650#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
651#endif
652
653#ifndef arch_atomic_fetch_dec_release
654static __always_inline int
655arch_atomic_fetch_dec_release(atomic_t *v)
656{
657 __atomic_release_fence();
658 return arch_atomic_fetch_dec_relaxed(v);
659}
660#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
661#endif
662
663#ifndef arch_atomic_fetch_dec
664static __always_inline int
665arch_atomic_fetch_dec(atomic_t *v)
666{
667 int ret;
668 __atomic_pre_full_fence();
669 ret = arch_atomic_fetch_dec_relaxed(v);
670 __atomic_post_full_fence();
671 return ret;
672}
673#define arch_atomic_fetch_dec arch_atomic_fetch_dec
674#endif
675
676#endif /* arch_atomic_fetch_dec_relaxed */
677
678#ifndef arch_atomic_fetch_and_relaxed
679#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
680#define arch_atomic_fetch_and_release arch_atomic_fetch_and
681#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
682#else /* arch_atomic_fetch_and_relaxed */
683
684#ifndef arch_atomic_fetch_and_acquire
685static __always_inline int
686arch_atomic_fetch_and_acquire(int i, atomic_t *v)
687{
688 int ret = arch_atomic_fetch_and_relaxed(i, v);
689 __atomic_acquire_fence();
690 return ret;
691}
692#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
693#endif
694
695#ifndef arch_atomic_fetch_and_release
696static __always_inline int
697arch_atomic_fetch_and_release(int i, atomic_t *v)
698{
699 __atomic_release_fence();
700 return arch_atomic_fetch_and_relaxed(i, v);
701}
702#define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
703#endif
704
705#ifndef arch_atomic_fetch_and
706static __always_inline int
707arch_atomic_fetch_and(int i, atomic_t *v)
708{
709 int ret;
710 __atomic_pre_full_fence();
711 ret = arch_atomic_fetch_and_relaxed(i, v);
712 __atomic_post_full_fence();
713 return ret;
714}
715#define arch_atomic_fetch_and arch_atomic_fetch_and
716#endif
717
718#endif /* arch_atomic_fetch_and_relaxed */
719
720#ifndef arch_atomic_andnot
721static __always_inline void
722arch_atomic_andnot(int i, atomic_t *v)
723{
724 arch_atomic_and(~i, v);
725}
726#define arch_atomic_andnot arch_atomic_andnot
727#endif
728
729#ifndef arch_atomic_fetch_andnot_relaxed
730#ifdef arch_atomic_fetch_andnot
731#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
732#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
733#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
734#endif /* arch_atomic_fetch_andnot */
735
736#ifndef arch_atomic_fetch_andnot
737static __always_inline int
738arch_atomic_fetch_andnot(int i, atomic_t *v)
739{
740 return arch_atomic_fetch_and(~i, v);
741}
742#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
743#endif
744
745#ifndef arch_atomic_fetch_andnot_acquire
746static __always_inline int
747arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
748{
749 return arch_atomic_fetch_and_acquire(~i, v);
750}
751#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
752#endif
753
754#ifndef arch_atomic_fetch_andnot_release
755static __always_inline int
756arch_atomic_fetch_andnot_release(int i, atomic_t *v)
757{
758 return arch_atomic_fetch_and_release(~i, v);
759}
760#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
761#endif
762
763#ifndef arch_atomic_fetch_andnot_relaxed
764static __always_inline int
765arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
766{
767 return arch_atomic_fetch_and_relaxed(~i, v);
768}
769#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
770#endif
771
772#else /* arch_atomic_fetch_andnot_relaxed */
773
774#ifndef arch_atomic_fetch_andnot_acquire
775static __always_inline int
776arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
777{
778 int ret = arch_atomic_fetch_andnot_relaxed(i, v);
779 __atomic_acquire_fence();
780 return ret;
781}
782#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
783#endif
784
785#ifndef arch_atomic_fetch_andnot_release
786static __always_inline int
787arch_atomic_fetch_andnot_release(int i, atomic_t *v)
788{
789 __atomic_release_fence();
790 return arch_atomic_fetch_andnot_relaxed(i, v);
791}
792#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
793#endif
794
795#ifndef arch_atomic_fetch_andnot
796static __always_inline int
797arch_atomic_fetch_andnot(int i, atomic_t *v)
798{
799 int ret;
800 __atomic_pre_full_fence();
801 ret = arch_atomic_fetch_andnot_relaxed(i, v);
802 __atomic_post_full_fence();
803 return ret;
804}
805#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
806#endif
807
808#endif /* arch_atomic_fetch_andnot_relaxed */
809
810#ifndef arch_atomic_fetch_or_relaxed
811#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
812#define arch_atomic_fetch_or_release arch_atomic_fetch_or
813#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
814#else /* arch_atomic_fetch_or_relaxed */
815
816#ifndef arch_atomic_fetch_or_acquire
817static __always_inline int
818arch_atomic_fetch_or_acquire(int i, atomic_t *v)
819{
820 int ret = arch_atomic_fetch_or_relaxed(i, v);
821 __atomic_acquire_fence();
822 return ret;
823}
824#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
825#endif
826
827#ifndef arch_atomic_fetch_or_release
828static __always_inline int
829arch_atomic_fetch_or_release(int i, atomic_t *v)
830{
831 __atomic_release_fence();
832 return arch_atomic_fetch_or_relaxed(i, v);
833}
834#define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
835#endif
836
837#ifndef arch_atomic_fetch_or
838static __always_inline int
839arch_atomic_fetch_or(int i, atomic_t *v)
840{
841 int ret;
842 __atomic_pre_full_fence();
843 ret = arch_atomic_fetch_or_relaxed(i, v);
844 __atomic_post_full_fence();
845 return ret;
846}
847#define arch_atomic_fetch_or arch_atomic_fetch_or
848#endif
849
850#endif /* arch_atomic_fetch_or_relaxed */
851
852#ifndef arch_atomic_fetch_xor_relaxed
853#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
854#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
855#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
856#else /* arch_atomic_fetch_xor_relaxed */
857
858#ifndef arch_atomic_fetch_xor_acquire
859static __always_inline int
860arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
861{
862 int ret = arch_atomic_fetch_xor_relaxed(i, v);
863 __atomic_acquire_fence();
864 return ret;
865}
866#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
867#endif
868
869#ifndef arch_atomic_fetch_xor_release
870static __always_inline int
871arch_atomic_fetch_xor_release(int i, atomic_t *v)
872{
873 __atomic_release_fence();
874 return arch_atomic_fetch_xor_relaxed(i, v);
875}
876#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
877#endif
878
879#ifndef arch_atomic_fetch_xor
880static __always_inline int
881arch_atomic_fetch_xor(int i, atomic_t *v)
882{
883 int ret;
884 __atomic_pre_full_fence();
885 ret = arch_atomic_fetch_xor_relaxed(i, v);
886 __atomic_post_full_fence();
887 return ret;
888}
889#define arch_atomic_fetch_xor arch_atomic_fetch_xor
890#endif
891
892#endif /* arch_atomic_fetch_xor_relaxed */
893
894#ifndef arch_atomic_xchg_relaxed
895#define arch_atomic_xchg_acquire arch_atomic_xchg
896#define arch_atomic_xchg_release arch_atomic_xchg
897#define arch_atomic_xchg_relaxed arch_atomic_xchg
898#else /* arch_atomic_xchg_relaxed */
899
900#ifndef arch_atomic_xchg_acquire
901static __always_inline int
902arch_atomic_xchg_acquire(atomic_t *v, int i)
903{
904 int ret = arch_atomic_xchg_relaxed(v, i);
905 __atomic_acquire_fence();
906 return ret;
907}
908#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
909#endif
910
911#ifndef arch_atomic_xchg_release
912static __always_inline int
913arch_atomic_xchg_release(atomic_t *v, int i)
914{
915 __atomic_release_fence();
916 return arch_atomic_xchg_relaxed(v, i);
917}
918#define arch_atomic_xchg_release arch_atomic_xchg_release
919#endif
920
921#ifndef arch_atomic_xchg
922static __always_inline int
923arch_atomic_xchg(atomic_t *v, int i)
924{
925 int ret;
926 __atomic_pre_full_fence();
927 ret = arch_atomic_xchg_relaxed(v, i);
928 __atomic_post_full_fence();
929 return ret;
930}
931#define arch_atomic_xchg arch_atomic_xchg
932#endif
933
934#endif /* arch_atomic_xchg_relaxed */
935
936#ifndef arch_atomic_cmpxchg_relaxed
937#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
938#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
939#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
940#else /* arch_atomic_cmpxchg_relaxed */
941
942#ifndef arch_atomic_cmpxchg_acquire
943static __always_inline int
944arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
945{
946 int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
947 __atomic_acquire_fence();
948 return ret;
949}
950#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
951#endif
952
953#ifndef arch_atomic_cmpxchg_release
954static __always_inline int
955arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
956{
957 __atomic_release_fence();
958 return arch_atomic_cmpxchg_relaxed(v, old, new);
959}
960#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
961#endif
962
963#ifndef arch_atomic_cmpxchg
964static __always_inline int
965arch_atomic_cmpxchg(atomic_t *v, int old, int new)
966{
967 int ret;
968 __atomic_pre_full_fence();
969 ret = arch_atomic_cmpxchg_relaxed(v, old, new);
970 __atomic_post_full_fence();
971 return ret;
972}
973#define arch_atomic_cmpxchg arch_atomic_cmpxchg
974#endif
975
976#endif /* arch_atomic_cmpxchg_relaxed */
977
978#ifndef arch_atomic_try_cmpxchg_relaxed
979#ifdef arch_atomic_try_cmpxchg
980#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
981#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
982#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
983#endif /* arch_atomic_try_cmpxchg */
984
985#ifndef arch_atomic_try_cmpxchg
986static __always_inline bool
987arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
988{
989 int r, o = *old;
990 r = arch_atomic_cmpxchg(v, o, new);
991 if (unlikely(r != o))
992 *old = r;
993 return likely(r == o);
994}
995#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
996#endif
997
998#ifndef arch_atomic_try_cmpxchg_acquire
999static __always_inline bool
1000arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1001{
1002 int r, o = *old;
1003 r = arch_atomic_cmpxchg_acquire(v, o, new);
1004 if (unlikely(r != o))
1005 *old = r;
1006 return likely(r == o);
1007}
1008#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1009#endif
1010
1011#ifndef arch_atomic_try_cmpxchg_release
1012static __always_inline bool
1013arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1014{
1015 int r, o = *old;
1016 r = arch_atomic_cmpxchg_release(v, o, new);
1017 if (unlikely(r != o))
1018 *old = r;
1019 return likely(r == o);
1020}
1021#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1022#endif
1023
1024#ifndef arch_atomic_try_cmpxchg_relaxed
1025static __always_inline bool
1026arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1027{
1028 int r, o = *old;
1029 r = arch_atomic_cmpxchg_relaxed(v, o, new);
1030 if (unlikely(r != o))
1031 *old = r;
1032 return likely(r == o);
1033}
1034#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
1035#endif
1036
1037#else /* arch_atomic_try_cmpxchg_relaxed */
1038
1039#ifndef arch_atomic_try_cmpxchg_acquire
1040static __always_inline bool
1041arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1042{
1043 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1044 __atomic_acquire_fence();
1045 return ret;
1046}
1047#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1048#endif
1049
1050#ifndef arch_atomic_try_cmpxchg_release
1051static __always_inline bool
1052arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1053{
1054 __atomic_release_fence();
1055 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
1056}
1057#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1058#endif
1059
1060#ifndef arch_atomic_try_cmpxchg
1061static __always_inline bool
1062arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1063{
1064 bool ret;
1065 __atomic_pre_full_fence();
1066 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1067 __atomic_post_full_fence();
1068 return ret;
1069}
1070#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1071#endif
1072
1073#endif /* arch_atomic_try_cmpxchg_relaxed */
1074
1075#ifndef arch_atomic_sub_and_test
1076/**
1077 * arch_atomic_sub_and_test - subtract value from variable and test result
1078 * @i: integer value to subtract
1079 * @v: pointer of type atomic_t
1080 *
1081 * Atomically subtracts @i from @v and returns
1082 * true if the result is zero, or false for all
1083 * other cases.
1084 */
1085static __always_inline bool
1086arch_atomic_sub_and_test(int i, atomic_t *v)
1087{
1088 return arch_atomic_sub_return(i, v) == 0;
1089}
1090#define arch_atomic_sub_and_test arch_atomic_sub_and_test
1091#endif
1092
1093#ifndef arch_atomic_dec_and_test
1094/**
1095 * arch_atomic_dec_and_test - decrement and test
1096 * @v: pointer of type atomic_t
1097 *
1098 * Atomically decrements @v by 1 and
1099 * returns true if the result is 0, or false for all other
1100 * cases.
1101 */
1102static __always_inline bool
1103arch_atomic_dec_and_test(atomic_t *v)
1104{
1105 return arch_atomic_dec_return(v) == 0;
1106}
1107#define arch_atomic_dec_and_test arch_atomic_dec_and_test
1108#endif
1109
1110#ifndef arch_atomic_inc_and_test
1111/**
1112 * arch_atomic_inc_and_test - increment and test
1113 * @v: pointer of type atomic_t
1114 *
1115 * Atomically increments @v by 1
1116 * and returns true if the result is zero, or false for all
1117 * other cases.
1118 */
1119static __always_inline bool
1120arch_atomic_inc_and_test(atomic_t *v)
1121{
1122 return arch_atomic_inc_return(v) == 0;
1123}
1124#define arch_atomic_inc_and_test arch_atomic_inc_and_test
1125#endif
1126
1127#ifndef arch_atomic_add_negative
1128/**
1129 * arch_atomic_add_negative - add and test if negative
1130 * @i: integer value to add
1131 * @v: pointer of type atomic_t
1132 *
1133 * Atomically adds @i to @v and returns true
1134 * if the result is negative, or false when
1135 * result is greater than or equal to zero.
1136 */
1137static __always_inline bool
1138arch_atomic_add_negative(int i, atomic_t *v)
1139{
1140 return arch_atomic_add_return(i, v) < 0;
1141}
1142#define arch_atomic_add_negative arch_atomic_add_negative
1143#endif
1144
1145#ifndef arch_atomic_fetch_add_unless
1146/**
1147 * arch_atomic_fetch_add_unless - add unless the number is already a given value
1148 * @v: pointer of type atomic_t
1149 * @a: the amount to add to v...
1150 * @u: ...unless v is equal to u.
1151 *
1152 * Atomically adds @a to @v, so long as @v was not already @u.
1153 * Returns original value of @v
1154 */
1155static __always_inline int
1156arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
1157{
1158 int c = arch_atomic_read(v);
1159
1160 do {
1161 if (unlikely(c == u))
1162 break;
1163 } while (!arch_atomic_try_cmpxchg(v, &c, c + a));
1164
1165 return c;
1166}
1167#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
1168#endif
1169
1170#ifndef arch_atomic_add_unless
1171/**
1172 * arch_atomic_add_unless - add unless the number is already a given value
1173 * @v: pointer of type atomic_t
1174 * @a: the amount to add to v...
1175 * @u: ...unless v is equal to u.
1176 *
1177 * Atomically adds @a to @v, if @v was not already @u.
1178 * Returns true if the addition was done.
1179 */
1180static __always_inline bool
1181arch_atomic_add_unless(atomic_t *v, int a, int u)
1182{
1183 return arch_atomic_fetch_add_unless(v, a, u) != u;
1184}
1185#define arch_atomic_add_unless arch_atomic_add_unless
1186#endif
1187
1188#ifndef arch_atomic_inc_not_zero
1189/**
1190 * arch_atomic_inc_not_zero - increment unless the number is zero
1191 * @v: pointer of type atomic_t
1192 *
1193 * Atomically increments @v by 1, if @v is non-zero.
1194 * Returns true if the increment was done.
1195 */
1196static __always_inline bool
1197arch_atomic_inc_not_zero(atomic_t *v)
1198{
1199 return arch_atomic_add_unless(v, 1, 0);
1200}
1201#define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
1202#endif
1203
1204#ifndef arch_atomic_inc_unless_negative
1205static __always_inline bool
1206arch_atomic_inc_unless_negative(atomic_t *v)
1207{
1208 int c = arch_atomic_read(v);
1209
1210 do {
1211 if (unlikely(c < 0))
1212 return false;
1213 } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
1214
1215 return true;
1216}
1217#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
1218#endif
1219
1220#ifndef arch_atomic_dec_unless_positive
1221static __always_inline bool
1222arch_atomic_dec_unless_positive(atomic_t *v)
1223{
1224 int c = arch_atomic_read(v);
1225
1226 do {
1227 if (unlikely(c > 0))
1228 return false;
1229 } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
1230
1231 return true;
1232}
1233#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
1234#endif
1235
1236#ifndef arch_atomic_dec_if_positive
1237static __always_inline int
1238arch_atomic_dec_if_positive(atomic_t *v)
1239{
1240 int dec, c = arch_atomic_read(v);
1241
1242 do {
1243 dec = c - 1;
1244 if (unlikely(dec < 0))
1245 break;
1246 } while (!arch_atomic_try_cmpxchg(v, &c, dec));
1247
1248 return dec;
1249}
1250#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
1251#endif
1252
1253#ifdef CONFIG_GENERIC_ATOMIC64
1254#include <asm-generic/atomic64.h>
1255#endif
1256
1257#ifndef arch_atomic64_read_acquire
1258static __always_inline s64
1259arch_atomic64_read_acquire(const atomic64_t *v)
1260{
1261 return smp_load_acquire(&(v)->counter);
1262}
1263#define arch_atomic64_read_acquire arch_atomic64_read_acquire
1264#endif
1265
1266#ifndef arch_atomic64_set_release
1267static __always_inline void
1268arch_atomic64_set_release(atomic64_t *v, s64 i)
1269{
1270 smp_store_release(&(v)->counter, i);
1271}
1272#define arch_atomic64_set_release arch_atomic64_set_release
1273#endif
1274
1275#ifndef arch_atomic64_add_return_relaxed
1276#define arch_atomic64_add_return_acquire arch_atomic64_add_return
1277#define arch_atomic64_add_return_release arch_atomic64_add_return
1278#define arch_atomic64_add_return_relaxed arch_atomic64_add_return
1279#else /* arch_atomic64_add_return_relaxed */
1280
1281#ifndef arch_atomic64_add_return_acquire
1282static __always_inline s64
1283arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
1284{
1285 s64 ret = arch_atomic64_add_return_relaxed(i, v);
1286 __atomic_acquire_fence();
1287 return ret;
1288}
1289#define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
1290#endif
1291
1292#ifndef arch_atomic64_add_return_release
1293static __always_inline s64
1294arch_atomic64_add_return_release(s64 i, atomic64_t *v)
1295{
1296 __atomic_release_fence();
1297 return arch_atomic64_add_return_relaxed(i, v);
1298}
1299#define arch_atomic64_add_return_release arch_atomic64_add_return_release
1300#endif
1301
1302#ifndef arch_atomic64_add_return
1303static __always_inline s64
1304arch_atomic64_add_return(s64 i, atomic64_t *v)
1305{
1306 s64 ret;
1307 __atomic_pre_full_fence();
1308 ret = arch_atomic64_add_return_relaxed(i, v);
1309 __atomic_post_full_fence();
1310 return ret;
1311}
1312#define arch_atomic64_add_return arch_atomic64_add_return
1313#endif
1314
1315#endif /* arch_atomic64_add_return_relaxed */
1316
1317#ifndef arch_atomic64_fetch_add_relaxed
1318#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
1319#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
1320#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
1321#else /* arch_atomic64_fetch_add_relaxed */
1322
1323#ifndef arch_atomic64_fetch_add_acquire
1324static __always_inline s64
1325arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1326{
1327 s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
1328 __atomic_acquire_fence();
1329 return ret;
1330}
1331#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
1332#endif
1333
1334#ifndef arch_atomic64_fetch_add_release
1335static __always_inline s64
1336arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
1337{
1338 __atomic_release_fence();
1339 return arch_atomic64_fetch_add_relaxed(i, v);
1340}
1341#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
1342#endif
1343
1344#ifndef arch_atomic64_fetch_add
1345static __always_inline s64
1346arch_atomic64_fetch_add(s64 i, atomic64_t *v)
1347{
1348 s64 ret;
1349 __atomic_pre_full_fence();
1350 ret = arch_atomic64_fetch_add_relaxed(i, v);
1351 __atomic_post_full_fence();
1352 return ret;
1353}
1354#define arch_atomic64_fetch_add arch_atomic64_fetch_add
1355#endif
1356
1357#endif /* arch_atomic64_fetch_add_relaxed */
1358
1359#ifndef arch_atomic64_sub_return_relaxed
1360#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
1361#define arch_atomic64_sub_return_release arch_atomic64_sub_return
1362#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
1363#else /* arch_atomic64_sub_return_relaxed */
1364
1365#ifndef arch_atomic64_sub_return_acquire
1366static __always_inline s64
1367arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1368{
1369 s64 ret = arch_atomic64_sub_return_relaxed(i, v);
1370 __atomic_acquire_fence();
1371 return ret;
1372}
1373#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
1374#endif
1375
1376#ifndef arch_atomic64_sub_return_release
1377static __always_inline s64
1378arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
1379{
1380 __atomic_release_fence();
1381 return arch_atomic64_sub_return_relaxed(i, v);
1382}
1383#define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
1384#endif
1385
1386#ifndef arch_atomic64_sub_return
1387static __always_inline s64
1388arch_atomic64_sub_return(s64 i, atomic64_t *v)
1389{
1390 s64 ret;
1391 __atomic_pre_full_fence();
1392 ret = arch_atomic64_sub_return_relaxed(i, v);
1393 __atomic_post_full_fence();
1394 return ret;
1395}
1396#define arch_atomic64_sub_return arch_atomic64_sub_return
1397#endif
1398
1399#endif /* arch_atomic64_sub_return_relaxed */
1400
1401#ifndef arch_atomic64_fetch_sub_relaxed
1402#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
1403#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
1404#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
1405#else /* arch_atomic64_fetch_sub_relaxed */
1406
1407#ifndef arch_atomic64_fetch_sub_acquire
1408static __always_inline s64
1409arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1410{
1411 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1412 __atomic_acquire_fence();
1413 return ret;
1414}
1415#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
1416#endif
1417
1418#ifndef arch_atomic64_fetch_sub_release
1419static __always_inline s64
1420arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1421{
1422 __atomic_release_fence();
1423 return arch_atomic64_fetch_sub_relaxed(i, v);
1424}
1425#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
1426#endif
1427
1428#ifndef arch_atomic64_fetch_sub
1429static __always_inline s64
1430arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
1431{
1432 s64 ret;
1433 __atomic_pre_full_fence();
1434 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1435 __atomic_post_full_fence();
1436 return ret;
1437}
1438#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
1439#endif
1440
1441#endif /* arch_atomic64_fetch_sub_relaxed */
1442
1443#ifndef arch_atomic64_inc
1444static __always_inline void
1445arch_atomic64_inc(atomic64_t *v)
1446{
1447 arch_atomic64_add(1, v);
1448}
1449#define arch_atomic64_inc arch_atomic64_inc
1450#endif
1451
1452#ifndef arch_atomic64_inc_return_relaxed
1453#ifdef arch_atomic64_inc_return
1454#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
1455#define arch_atomic64_inc_return_release arch_atomic64_inc_return
1456#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
1457#endif /* arch_atomic64_inc_return */
1458
1459#ifndef arch_atomic64_inc_return
1460static __always_inline s64
1461arch_atomic64_inc_return(atomic64_t *v)
1462{
1463 return arch_atomic64_add_return(1, v);
1464}
1465#define arch_atomic64_inc_return arch_atomic64_inc_return
1466#endif
1467
1468#ifndef arch_atomic64_inc_return_acquire
1469static __always_inline s64
1470arch_atomic64_inc_return_acquire(atomic64_t *v)
1471{
1472 return arch_atomic64_add_return_acquire(1, v);
1473}
1474#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1475#endif
1476
1477#ifndef arch_atomic64_inc_return_release
1478static __always_inline s64
1479arch_atomic64_inc_return_release(atomic64_t *v)
1480{
1481 return arch_atomic64_add_return_release(1, v);
1482}
1483#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1484#endif
1485
1486#ifndef arch_atomic64_inc_return_relaxed
1487static __always_inline s64
1488arch_atomic64_inc_return_relaxed(atomic64_t *v)
1489{
1490 return arch_atomic64_add_return_relaxed(1, v);
1491}
1492#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
1493#endif
1494
1495#else /* arch_atomic64_inc_return_relaxed */
1496
1497#ifndef arch_atomic64_inc_return_acquire
1498static __always_inline s64
1499arch_atomic64_inc_return_acquire(atomic64_t *v)
1500{
1501 s64 ret = arch_atomic64_inc_return_relaxed(v);
1502 __atomic_acquire_fence();
1503 return ret;
1504}
1505#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1506#endif
1507
1508#ifndef arch_atomic64_inc_return_release
1509static __always_inline s64
1510arch_atomic64_inc_return_release(atomic64_t *v)
1511{
1512 __atomic_release_fence();
1513 return arch_atomic64_inc_return_relaxed(v);
1514}
1515#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1516#endif
1517
1518#ifndef arch_atomic64_inc_return
1519static __always_inline s64
1520arch_atomic64_inc_return(atomic64_t *v)
1521{
1522 s64 ret;
1523 __atomic_pre_full_fence();
1524 ret = arch_atomic64_inc_return_relaxed(v);
1525 __atomic_post_full_fence();
1526 return ret;
1527}
1528#define arch_atomic64_inc_return arch_atomic64_inc_return
1529#endif
1530
1531#endif /* arch_atomic64_inc_return_relaxed */
1532
1533#ifndef arch_atomic64_fetch_inc_relaxed
1534#ifdef arch_atomic64_fetch_inc
1535#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
1536#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
1537#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
1538#endif /* arch_atomic64_fetch_inc */
1539
1540#ifndef arch_atomic64_fetch_inc
1541static __always_inline s64
1542arch_atomic64_fetch_inc(atomic64_t *v)
1543{
1544 return arch_atomic64_fetch_add(1, v);
1545}
1546#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1547#endif
1548
1549#ifndef arch_atomic64_fetch_inc_acquire
1550static __always_inline s64
1551arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1552{
1553 return arch_atomic64_fetch_add_acquire(1, v);
1554}
1555#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1556#endif
1557
1558#ifndef arch_atomic64_fetch_inc_release
1559static __always_inline s64
1560arch_atomic64_fetch_inc_release(atomic64_t *v)
1561{
1562 return arch_atomic64_fetch_add_release(1, v);
1563}
1564#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1565#endif
1566
1567#ifndef arch_atomic64_fetch_inc_relaxed
1568static __always_inline s64
1569arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
1570{
1571 return arch_atomic64_fetch_add_relaxed(1, v);
1572}
1573#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
1574#endif
1575
1576#else /* arch_atomic64_fetch_inc_relaxed */
1577
1578#ifndef arch_atomic64_fetch_inc_acquire
1579static __always_inline s64
1580arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1581{
1582 s64 ret = arch_atomic64_fetch_inc_relaxed(v);
1583 __atomic_acquire_fence();
1584 return ret;
1585}
1586#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1587#endif
1588
1589#ifndef arch_atomic64_fetch_inc_release
1590static __always_inline s64
1591arch_atomic64_fetch_inc_release(atomic64_t *v)
1592{
1593 __atomic_release_fence();
1594 return arch_atomic64_fetch_inc_relaxed(v);
1595}
1596#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1597#endif
1598
1599#ifndef arch_atomic64_fetch_inc
1600static __always_inline s64
1601arch_atomic64_fetch_inc(atomic64_t *v)
1602{
1603 s64 ret;
1604 __atomic_pre_full_fence();
1605 ret = arch_atomic64_fetch_inc_relaxed(v);
1606 __atomic_post_full_fence();
1607 return ret;
1608}
1609#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1610#endif
1611
1612#endif /* arch_atomic64_fetch_inc_relaxed */
1613
1614#ifndef arch_atomic64_dec
1615static __always_inline void
1616arch_atomic64_dec(atomic64_t *v)
1617{
1618 arch_atomic64_sub(1, v);
1619}
1620#define arch_atomic64_dec arch_atomic64_dec
1621#endif
1622
1623#ifndef arch_atomic64_dec_return_relaxed
1624#ifdef arch_atomic64_dec_return
1625#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
1626#define arch_atomic64_dec_return_release arch_atomic64_dec_return
1627#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
1628#endif /* arch_atomic64_dec_return */
1629
1630#ifndef arch_atomic64_dec_return
1631static __always_inline s64
1632arch_atomic64_dec_return(atomic64_t *v)
1633{
1634 return arch_atomic64_sub_return(1, v);
1635}
1636#define arch_atomic64_dec_return arch_atomic64_dec_return
1637#endif
1638
1639#ifndef arch_atomic64_dec_return_acquire
1640static __always_inline s64
1641arch_atomic64_dec_return_acquire(atomic64_t *v)
1642{
1643 return arch_atomic64_sub_return_acquire(1, v);
1644}
1645#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1646#endif
1647
1648#ifndef arch_atomic64_dec_return_release
1649static __always_inline s64
1650arch_atomic64_dec_return_release(atomic64_t *v)
1651{
1652 return arch_atomic64_sub_return_release(1, v);
1653}
1654#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1655#endif
1656
1657#ifndef arch_atomic64_dec_return_relaxed
1658static __always_inline s64
1659arch_atomic64_dec_return_relaxed(atomic64_t *v)
1660{
1661 return arch_atomic64_sub_return_relaxed(1, v);
1662}
1663#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
1664#endif
1665
1666#else /* arch_atomic64_dec_return_relaxed */
1667
1668#ifndef arch_atomic64_dec_return_acquire
1669static __always_inline s64
1670arch_atomic64_dec_return_acquire(atomic64_t *v)
1671{
1672 s64 ret = arch_atomic64_dec_return_relaxed(v);
1673 __atomic_acquire_fence();
1674 return ret;
1675}
1676#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1677#endif
1678
1679#ifndef arch_atomic64_dec_return_release
1680static __always_inline s64
1681arch_atomic64_dec_return_release(atomic64_t *v)
1682{
1683 __atomic_release_fence();
1684 return arch_atomic64_dec_return_relaxed(v);
1685}
1686#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1687#endif
1688
1689#ifndef arch_atomic64_dec_return
1690static __always_inline s64
1691arch_atomic64_dec_return(atomic64_t *v)
1692{
1693 s64 ret;
1694 __atomic_pre_full_fence();
1695 ret = arch_atomic64_dec_return_relaxed(v);
1696 __atomic_post_full_fence();
1697 return ret;
1698}
1699#define arch_atomic64_dec_return arch_atomic64_dec_return
1700#endif
1701
1702#endif /* arch_atomic64_dec_return_relaxed */
1703
1704#ifndef arch_atomic64_fetch_dec_relaxed
1705#ifdef arch_atomic64_fetch_dec
1706#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
1707#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
1708#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
1709#endif /* arch_atomic64_fetch_dec */
1710
1711#ifndef arch_atomic64_fetch_dec
1712static __always_inline s64
1713arch_atomic64_fetch_dec(atomic64_t *v)
1714{
1715 return arch_atomic64_fetch_sub(1, v);
1716}
1717#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1718#endif
1719
1720#ifndef arch_atomic64_fetch_dec_acquire
1721static __always_inline s64
1722arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1723{
1724 return arch_atomic64_fetch_sub_acquire(1, v);
1725}
1726#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1727#endif
1728
1729#ifndef arch_atomic64_fetch_dec_release
1730static __always_inline s64
1731arch_atomic64_fetch_dec_release(atomic64_t *v)
1732{
1733 return arch_atomic64_fetch_sub_release(1, v);
1734}
1735#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1736#endif
1737
1738#ifndef arch_atomic64_fetch_dec_relaxed
1739static __always_inline s64
1740arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
1741{
1742 return arch_atomic64_fetch_sub_relaxed(1, v);
1743}
1744#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
1745#endif
1746
1747#else /* arch_atomic64_fetch_dec_relaxed */
1748
1749#ifndef arch_atomic64_fetch_dec_acquire
1750static __always_inline s64
1751arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1752{
1753 s64 ret = arch_atomic64_fetch_dec_relaxed(v);
1754 __atomic_acquire_fence();
1755 return ret;
1756}
1757#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1758#endif
1759
1760#ifndef arch_atomic64_fetch_dec_release
1761static __always_inline s64
1762arch_atomic64_fetch_dec_release(atomic64_t *v)
1763{
1764 __atomic_release_fence();
1765 return arch_atomic64_fetch_dec_relaxed(v);
1766}
1767#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1768#endif
1769
1770#ifndef arch_atomic64_fetch_dec
1771static __always_inline s64
1772arch_atomic64_fetch_dec(atomic64_t *v)
1773{
1774 s64 ret;
1775 __atomic_pre_full_fence();
1776 ret = arch_atomic64_fetch_dec_relaxed(v);
1777 __atomic_post_full_fence();
1778 return ret;
1779}
1780#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1781#endif
1782
1783#endif /* arch_atomic64_fetch_dec_relaxed */
1784
1785#ifndef arch_atomic64_fetch_and_relaxed
1786#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
1787#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
1788#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
1789#else /* arch_atomic64_fetch_and_relaxed */
1790
1791#ifndef arch_atomic64_fetch_and_acquire
1792static __always_inline s64
1793arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1794{
1795 s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
1796 __atomic_acquire_fence();
1797 return ret;
1798}
1799#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
1800#endif
1801
1802#ifndef arch_atomic64_fetch_and_release
1803static __always_inline s64
1804arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
1805{
1806 __atomic_release_fence();
1807 return arch_atomic64_fetch_and_relaxed(i, v);
1808}
1809#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
1810#endif
1811
1812#ifndef arch_atomic64_fetch_and
1813static __always_inline s64
1814arch_atomic64_fetch_and(s64 i, atomic64_t *v)
1815{
1816 s64 ret;
1817 __atomic_pre_full_fence();
1818 ret = arch_atomic64_fetch_and_relaxed(i, v);
1819 __atomic_post_full_fence();
1820 return ret;
1821}
1822#define arch_atomic64_fetch_and arch_atomic64_fetch_and
1823#endif
1824
1825#endif /* arch_atomic64_fetch_and_relaxed */
1826
1827#ifndef arch_atomic64_andnot
1828static __always_inline void
1829arch_atomic64_andnot(s64 i, atomic64_t *v)
1830{
1831 arch_atomic64_and(~i, v);
1832}
1833#define arch_atomic64_andnot arch_atomic64_andnot
1834#endif
1835
1836#ifndef arch_atomic64_fetch_andnot_relaxed
1837#ifdef arch_atomic64_fetch_andnot
1838#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
1839#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
1840#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
1841#endif /* arch_atomic64_fetch_andnot */
1842
1843#ifndef arch_atomic64_fetch_andnot
1844static __always_inline s64
1845arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1846{
1847 return arch_atomic64_fetch_and(~i, v);
1848}
1849#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1850#endif
1851
1852#ifndef arch_atomic64_fetch_andnot_acquire
1853static __always_inline s64
1854arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1855{
1856 return arch_atomic64_fetch_and_acquire(~i, v);
1857}
1858#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1859#endif
1860
1861#ifndef arch_atomic64_fetch_andnot_release
1862static __always_inline s64
1863arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1864{
1865 return arch_atomic64_fetch_and_release(~i, v);
1866}
1867#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1868#endif
1869
1870#ifndef arch_atomic64_fetch_andnot_relaxed
1871static __always_inline s64
1872arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1873{
1874 return arch_atomic64_fetch_and_relaxed(~i, v);
1875}
1876#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
1877#endif
1878
1879#else /* arch_atomic64_fetch_andnot_relaxed */
1880
1881#ifndef arch_atomic64_fetch_andnot_acquire
1882static __always_inline s64
1883arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1884{
1885 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1886 __atomic_acquire_fence();
1887 return ret;
1888}
1889#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1890#endif
1891
1892#ifndef arch_atomic64_fetch_andnot_release
1893static __always_inline s64
1894arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1895{
1896 __atomic_release_fence();
1897 return arch_atomic64_fetch_andnot_relaxed(i, v);
1898}
1899#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1900#endif
1901
1902#ifndef arch_atomic64_fetch_andnot
1903static __always_inline s64
1904arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1905{
1906 s64 ret;
1907 __atomic_pre_full_fence();
1908 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1909 __atomic_post_full_fence();
1910 return ret;
1911}
1912#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1913#endif
1914
1915#endif /* arch_atomic64_fetch_andnot_relaxed */
1916
1917#ifndef arch_atomic64_fetch_or_relaxed
1918#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
1919#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
1920#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or
1921#else /* arch_atomic64_fetch_or_relaxed */
1922
1923#ifndef arch_atomic64_fetch_or_acquire
1924static __always_inline s64
1925arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1926{
1927 s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
1928 __atomic_acquire_fence();
1929 return ret;
1930}
1931#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
1932#endif
1933
1934#ifndef arch_atomic64_fetch_or_release
1935static __always_inline s64
1936arch_atomic64_fetch_or_release(s64 i, atomic64_t *v)
1937{
1938 __atomic_release_fence();
1939 return arch_atomic64_fetch_or_relaxed(i, v);
1940}
1941#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
1942#endif
1943
1944#ifndef arch_atomic64_fetch_or
1945static __always_inline s64
1946arch_atomic64_fetch_or(s64 i, atomic64_t *v)
1947{
1948 s64 ret;
1949 __atomic_pre_full_fence();
1950 ret = arch_atomic64_fetch_or_relaxed(i, v);
1951 __atomic_post_full_fence();
1952 return ret;
1953}
1954#define arch_atomic64_fetch_or arch_atomic64_fetch_or
1955#endif
1956
1957#endif /* arch_atomic64_fetch_or_relaxed */
1958
1959#ifndef arch_atomic64_fetch_xor_relaxed
1960#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
1961#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
1962#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor
1963#else /* arch_atomic64_fetch_xor_relaxed */
1964
1965#ifndef arch_atomic64_fetch_xor_acquire
1966static __always_inline s64
1967arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1968{
1969 s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
1970 __atomic_acquire_fence();
1971 return ret;
1972}
1973#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
1974#endif
1975
1976#ifndef arch_atomic64_fetch_xor_release
1977static __always_inline s64
1978arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1979{
1980 __atomic_release_fence();
1981 return arch_atomic64_fetch_xor_relaxed(i, v);
1982}
1983#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
1984#endif
1985
1986#ifndef arch_atomic64_fetch_xor
1987static __always_inline s64
1988arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
1989{
1990 s64 ret;
1991 __atomic_pre_full_fence();
1992 ret = arch_atomic64_fetch_xor_relaxed(i, v);
1993 __atomic_post_full_fence();
1994 return ret;
1995}
1996#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
1997#endif
1998
1999#endif /* arch_atomic64_fetch_xor_relaxed */
2000
2001#ifndef arch_atomic64_xchg_relaxed
2002#define arch_atomic64_xchg_acquire arch_atomic64_xchg
2003#define arch_atomic64_xchg_release arch_atomic64_xchg
2004#define arch_atomic64_xchg_relaxed arch_atomic64_xchg
2005#else /* arch_atomic64_xchg_relaxed */
2006
2007#ifndef arch_atomic64_xchg_acquire
2008static __always_inline s64
2009arch_atomic64_xchg_acquire(atomic64_t *v, s64 i)
2010{
2011 s64 ret = arch_atomic64_xchg_relaxed(v, i);
2012 __atomic_acquire_fence();
2013 return ret;
2014}
2015#define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire
2016#endif
2017
2018#ifndef arch_atomic64_xchg_release
2019static __always_inline s64
2020arch_atomic64_xchg_release(atomic64_t *v, s64 i)
2021{
2022 __atomic_release_fence();
2023 return arch_atomic64_xchg_relaxed(v, i);
2024}
2025#define arch_atomic64_xchg_release arch_atomic64_xchg_release
2026#endif
2027
2028#ifndef arch_atomic64_xchg
2029static __always_inline s64
2030arch_atomic64_xchg(atomic64_t *v, s64 i)
2031{
2032 s64 ret;
2033 __atomic_pre_full_fence();
2034 ret = arch_atomic64_xchg_relaxed(v, i);
2035 __atomic_post_full_fence();
2036 return ret;
2037}
2038#define arch_atomic64_xchg arch_atomic64_xchg
2039#endif
2040
2041#endif /* arch_atomic64_xchg_relaxed */
2042
2043#ifndef arch_atomic64_cmpxchg_relaxed
2044#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg
2045#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg
2046#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg
2047#else /* arch_atomic64_cmpxchg_relaxed */
2048
2049#ifndef arch_atomic64_cmpxchg_acquire
2050static __always_inline s64
2051arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
2052{
2053 s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2054 __atomic_acquire_fence();
2055 return ret;
2056}
2057#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire
2058#endif
2059
2060#ifndef arch_atomic64_cmpxchg_release
2061static __always_inline s64
2062arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
2063{
2064 __atomic_release_fence();
2065 return arch_atomic64_cmpxchg_relaxed(v, old, new);
2066}
2067#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release
2068#endif
2069
2070#ifndef arch_atomic64_cmpxchg
2071static __always_inline s64
2072arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2073{
2074 s64 ret;
2075 __atomic_pre_full_fence();
2076 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2077 __atomic_post_full_fence();
2078 return ret;
2079}
2080#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
2081#endif
2082
2083#endif /* arch_atomic64_cmpxchg_relaxed */
2084
2085#ifndef arch_atomic64_try_cmpxchg_relaxed
2086#ifdef arch_atomic64_try_cmpxchg
2087#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg
2088#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg
2089#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg
2090#endif /* arch_atomic64_try_cmpxchg */
2091
2092#ifndef arch_atomic64_try_cmpxchg
2093static __always_inline bool
2094arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2095{
2096 s64 r, o = *old;
2097 r = arch_atomic64_cmpxchg(v, o, new);
2098 if (unlikely(r != o))
2099 *old = r;
2100 return likely(r == o);
2101}
2102#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2103#endif
2104
2105#ifndef arch_atomic64_try_cmpxchg_acquire
2106static __always_inline bool
2107arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2108{
2109 s64 r, o = *old;
2110 r = arch_atomic64_cmpxchg_acquire(v, o, new);
2111 if (unlikely(r != o))
2112 *old = r;
2113 return likely(r == o);
2114}
2115#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2116#endif
2117
2118#ifndef arch_atomic64_try_cmpxchg_release
2119static __always_inline bool
2120arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2121{
2122 s64 r, o = *old;
2123 r = arch_atomic64_cmpxchg_release(v, o, new);
2124 if (unlikely(r != o))
2125 *old = r;
2126 return likely(r == o);
2127}
2128#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2129#endif
2130
2131#ifndef arch_atomic64_try_cmpxchg_relaxed
2132static __always_inline bool
2133arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2134{
2135 s64 r, o = *old;
2136 r = arch_atomic64_cmpxchg_relaxed(v, o, new);
2137 if (unlikely(r != o))
2138 *old = r;
2139 return likely(r == o);
2140}
2141#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed
2142#endif
2143
2144#else /* arch_atomic64_try_cmpxchg_relaxed */
2145
2146#ifndef arch_atomic64_try_cmpxchg_acquire
2147static __always_inline bool
2148arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2149{
2150 bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2151 __atomic_acquire_fence();
2152 return ret;
2153}
2154#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2155#endif
2156
2157#ifndef arch_atomic64_try_cmpxchg_release
2158static __always_inline bool
2159arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2160{
2161 __atomic_release_fence();
2162 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2163}
2164#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2165#endif
2166
2167#ifndef arch_atomic64_try_cmpxchg
2168static __always_inline bool
2169arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2170{
2171 bool ret;
2172 __atomic_pre_full_fence();
2173 ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2174 __atomic_post_full_fence();
2175 return ret;
2176}
2177#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2178#endif
2179
2180#endif /* arch_atomic64_try_cmpxchg_relaxed */
2181
2182#ifndef arch_atomic64_sub_and_test
2183/**
2184 * arch_atomic64_sub_and_test - subtract value from variable and test result
2185 * @i: integer value to subtract
2186 * @v: pointer of type atomic64_t
2187 *
2188 * Atomically subtracts @i from @v and returns
2189 * true if the result is zero, or false for all
2190 * other cases.
2191 */
2192static __always_inline bool
2193arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
2194{
2195 return arch_atomic64_sub_return(i, v) == 0;
2196}
2197#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
2198#endif
2199
2200#ifndef arch_atomic64_dec_and_test
2201/**
2202 * arch_atomic64_dec_and_test - decrement and test
2203 * @v: pointer of type atomic64_t
2204 *
2205 * Atomically decrements @v by 1 and
2206 * returns true if the result is 0, or false for all other
2207 * cases.
2208 */
2209static __always_inline bool
2210arch_atomic64_dec_and_test(atomic64_t *v)
2211{
2212 return arch_atomic64_dec_return(v) == 0;
2213}
2214#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
2215#endif
2216
2217#ifndef arch_atomic64_inc_and_test
2218/**
2219 * arch_atomic64_inc_and_test - increment and test
2220 * @v: pointer of type atomic64_t
2221 *
2222 * Atomically increments @v by 1
2223 * and returns true if the result is zero, or false for all
2224 * other cases.
2225 */
2226static __always_inline bool
2227arch_atomic64_inc_and_test(atomic64_t *v)
2228{
2229 return arch_atomic64_inc_return(v) == 0;
2230}
2231#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
2232#endif
2233
2234#ifndef arch_atomic64_add_negative
2235/**
2236 * arch_atomic64_add_negative - add and test if negative
2237 * @i: integer value to add
2238 * @v: pointer of type atomic64_t
2239 *
2240 * Atomically adds @i to @v and returns true
2241 * if the result is negative, or false when
2242 * result is greater than or equal to zero.
2243 */
2244static __always_inline bool
2245arch_atomic64_add_negative(s64 i, atomic64_t *v)
2246{
2247 return arch_atomic64_add_return(i, v) < 0;
2248}
2249#define arch_atomic64_add_negative arch_atomic64_add_negative
2250#endif
2251
2252#ifndef arch_atomic64_fetch_add_unless
2253/**
2254 * arch_atomic64_fetch_add_unless - add unless the number is already a given value
2255 * @v: pointer of type atomic64_t
2256 * @a: the amount to add to v...
2257 * @u: ...unless v is equal to u.
2258 *
2259 * Atomically adds @a to @v, so long as @v was not already @u.
2260 * Returns original value of @v
2261 */
2262static __always_inline s64
2263arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2264{
2265 s64 c = arch_atomic64_read(v);
2266
2267 do {
2268 if (unlikely(c == u))
2269 break;
2270 } while (!arch_atomic64_try_cmpxchg(v, &c, c + a));
2271
2272 return c;
2273}
2274#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
2275#endif
2276
2277#ifndef arch_atomic64_add_unless
2278/**
2279 * arch_atomic64_add_unless - add unless the number is already a given value
2280 * @v: pointer of type atomic64_t
2281 * @a: the amount to add to v...
2282 * @u: ...unless v is equal to u.
2283 *
2284 * Atomically adds @a to @v, if @v was not already @u.
2285 * Returns true if the addition was done.
2286 */
2287static __always_inline bool
2288arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2289{
2290 return arch_atomic64_fetch_add_unless(v, a, u) != u;
2291}
2292#define arch_atomic64_add_unless arch_atomic64_add_unless
2293#endif
2294
2295#ifndef arch_atomic64_inc_not_zero
2296/**
2297 * arch_atomic64_inc_not_zero - increment unless the number is zero
2298 * @v: pointer of type atomic64_t
2299 *
2300 * Atomically increments @v by 1, if @v is non-zero.
2301 * Returns true if the increment was done.
2302 */
2303static __always_inline bool
2304arch_atomic64_inc_not_zero(atomic64_t *v)
2305{
2306 return arch_atomic64_add_unless(v, 1, 0);
2307}
2308#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
2309#endif
2310
2311#ifndef arch_atomic64_inc_unless_negative
2312static __always_inline bool
2313arch_atomic64_inc_unless_negative(atomic64_t *v)
2314{
2315 s64 c = arch_atomic64_read(v);
2316
2317 do {
2318 if (unlikely(c < 0))
2319 return false;
2320 } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1));
2321
2322 return true;
2323}
2324#define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
2325#endif
2326
2327#ifndef arch_atomic64_dec_unless_positive
2328static __always_inline bool
2329arch_atomic64_dec_unless_positive(atomic64_t *v)
2330{
2331 s64 c = arch_atomic64_read(v);
2332
2333 do {
2334 if (unlikely(c > 0))
2335 return false;
2336 } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1));
2337
2338 return true;
2339}
2340#define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
2341#endif
2342
2343#ifndef arch_atomic64_dec_if_positive
2344static __always_inline s64
2345arch_atomic64_dec_if_positive(atomic64_t *v)
2346{
2347 s64 dec, c = arch_atomic64_read(v);
2348
2349 do {
2350 dec = c - 1;
2351 if (unlikely(dec < 0))
2352 break;
2353 } while (!arch_atomic64_try_cmpxchg(v, &c, dec));
2354
2355 return dec;
2356}
2357#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
2358#endif
2359
2360#endif /* _LINUX_ATOMIC_FALLBACK_H */
2361// cca554917d7ea73d5e3e7397dd70c484cad9b2c4