Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2
3// Generated by scripts/atomic/gen-atomic-fallback.sh
4// DO NOT MODIFY THIS FILE DIRECTLY
5
6#ifndef _LINUX_ATOMIC_FALLBACK_H
7#define _LINUX_ATOMIC_FALLBACK_H
8
9#include <linux/compiler.h>
10
11#ifndef arch_xchg_relaxed
12#define arch_xchg_acquire arch_xchg
13#define arch_xchg_release arch_xchg
14#define arch_xchg_relaxed arch_xchg
15#else /* arch_xchg_relaxed */
16
17#ifndef arch_xchg_acquire
18#define arch_xchg_acquire(...) \
19 __atomic_op_acquire(arch_xchg, __VA_ARGS__)
20#endif
21
22#ifndef arch_xchg_release
23#define arch_xchg_release(...) \
24 __atomic_op_release(arch_xchg, __VA_ARGS__)
25#endif
26
27#ifndef arch_xchg
28#define arch_xchg(...) \
29 __atomic_op_fence(arch_xchg, __VA_ARGS__)
30#endif
31
32#endif /* arch_xchg_relaxed */
33
34#ifndef arch_cmpxchg_relaxed
35#define arch_cmpxchg_acquire arch_cmpxchg
36#define arch_cmpxchg_release arch_cmpxchg
37#define arch_cmpxchg_relaxed arch_cmpxchg
38#else /* arch_cmpxchg_relaxed */
39
40#ifndef arch_cmpxchg_acquire
41#define arch_cmpxchg_acquire(...) \
42 __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__)
43#endif
44
45#ifndef arch_cmpxchg_release
46#define arch_cmpxchg_release(...) \
47 __atomic_op_release(arch_cmpxchg, __VA_ARGS__)
48#endif
49
50#ifndef arch_cmpxchg
51#define arch_cmpxchg(...) \
52 __atomic_op_fence(arch_cmpxchg, __VA_ARGS__)
53#endif
54
55#endif /* arch_cmpxchg_relaxed */
56
57#ifndef arch_cmpxchg64_relaxed
58#define arch_cmpxchg64_acquire arch_cmpxchg64
59#define arch_cmpxchg64_release arch_cmpxchg64
60#define arch_cmpxchg64_relaxed arch_cmpxchg64
61#else /* arch_cmpxchg64_relaxed */
62
63#ifndef arch_cmpxchg64_acquire
64#define arch_cmpxchg64_acquire(...) \
65 __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__)
66#endif
67
68#ifndef arch_cmpxchg64_release
69#define arch_cmpxchg64_release(...) \
70 __atomic_op_release(arch_cmpxchg64, __VA_ARGS__)
71#endif
72
73#ifndef arch_cmpxchg64
74#define arch_cmpxchg64(...) \
75 __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__)
76#endif
77
78#endif /* arch_cmpxchg64_relaxed */
79
80#ifndef arch_try_cmpxchg_relaxed
81#ifdef arch_try_cmpxchg
82#define arch_try_cmpxchg_acquire arch_try_cmpxchg
83#define arch_try_cmpxchg_release arch_try_cmpxchg
84#define arch_try_cmpxchg_relaxed arch_try_cmpxchg
85#endif /* arch_try_cmpxchg */
86
87#ifndef arch_try_cmpxchg
88#define arch_try_cmpxchg(_ptr, _oldp, _new) \
89({ \
90 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
91 ___r = arch_cmpxchg((_ptr), ___o, (_new)); \
92 if (unlikely(___r != ___o)) \
93 *___op = ___r; \
94 likely(___r == ___o); \
95})
96#endif /* arch_try_cmpxchg */
97
98#ifndef arch_try_cmpxchg_acquire
99#define arch_try_cmpxchg_acquire(_ptr, _oldp, _new) \
100({ \
101 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
102 ___r = arch_cmpxchg_acquire((_ptr), ___o, (_new)); \
103 if (unlikely(___r != ___o)) \
104 *___op = ___r; \
105 likely(___r == ___o); \
106})
107#endif /* arch_try_cmpxchg_acquire */
108
109#ifndef arch_try_cmpxchg_release
110#define arch_try_cmpxchg_release(_ptr, _oldp, _new) \
111({ \
112 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
113 ___r = arch_cmpxchg_release((_ptr), ___o, (_new)); \
114 if (unlikely(___r != ___o)) \
115 *___op = ___r; \
116 likely(___r == ___o); \
117})
118#endif /* arch_try_cmpxchg_release */
119
120#ifndef arch_try_cmpxchg_relaxed
121#define arch_try_cmpxchg_relaxed(_ptr, _oldp, _new) \
122({ \
123 typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \
124 ___r = arch_cmpxchg_relaxed((_ptr), ___o, (_new)); \
125 if (unlikely(___r != ___o)) \
126 *___op = ___r; \
127 likely(___r == ___o); \
128})
129#endif /* arch_try_cmpxchg_relaxed */
130
131#else /* arch_try_cmpxchg_relaxed */
132
133#ifndef arch_try_cmpxchg_acquire
134#define arch_try_cmpxchg_acquire(...) \
135 __atomic_op_acquire(arch_try_cmpxchg, __VA_ARGS__)
136#endif
137
138#ifndef arch_try_cmpxchg_release
139#define arch_try_cmpxchg_release(...) \
140 __atomic_op_release(arch_try_cmpxchg, __VA_ARGS__)
141#endif
142
143#ifndef arch_try_cmpxchg
144#define arch_try_cmpxchg(...) \
145 __atomic_op_fence(arch_try_cmpxchg, __VA_ARGS__)
146#endif
147
148#endif /* arch_try_cmpxchg_relaxed */
149
150#ifndef arch_atomic_read_acquire
151static __always_inline int
152arch_atomic_read_acquire(const atomic_t *v)
153{
154 int ret;
155
156 if (__native_word(atomic_t)) {
157 ret = smp_load_acquire(&(v)->counter);
158 } else {
159 ret = arch_atomic_read(v);
160 __atomic_acquire_fence();
161 }
162
163 return ret;
164}
165#define arch_atomic_read_acquire arch_atomic_read_acquire
166#endif
167
168#ifndef arch_atomic_set_release
169static __always_inline void
170arch_atomic_set_release(atomic_t *v, int i)
171{
172 if (__native_word(atomic_t)) {
173 smp_store_release(&(v)->counter, i);
174 } else {
175 __atomic_release_fence();
176 arch_atomic_set(v, i);
177 }
178}
179#define arch_atomic_set_release arch_atomic_set_release
180#endif
181
182#ifndef arch_atomic_add_return_relaxed
183#define arch_atomic_add_return_acquire arch_atomic_add_return
184#define arch_atomic_add_return_release arch_atomic_add_return
185#define arch_atomic_add_return_relaxed arch_atomic_add_return
186#else /* arch_atomic_add_return_relaxed */
187
188#ifndef arch_atomic_add_return_acquire
189static __always_inline int
190arch_atomic_add_return_acquire(int i, atomic_t *v)
191{
192 int ret = arch_atomic_add_return_relaxed(i, v);
193 __atomic_acquire_fence();
194 return ret;
195}
196#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
197#endif
198
199#ifndef arch_atomic_add_return_release
200static __always_inline int
201arch_atomic_add_return_release(int i, atomic_t *v)
202{
203 __atomic_release_fence();
204 return arch_atomic_add_return_relaxed(i, v);
205}
206#define arch_atomic_add_return_release arch_atomic_add_return_release
207#endif
208
209#ifndef arch_atomic_add_return
210static __always_inline int
211arch_atomic_add_return(int i, atomic_t *v)
212{
213 int ret;
214 __atomic_pre_full_fence();
215 ret = arch_atomic_add_return_relaxed(i, v);
216 __atomic_post_full_fence();
217 return ret;
218}
219#define arch_atomic_add_return arch_atomic_add_return
220#endif
221
222#endif /* arch_atomic_add_return_relaxed */
223
224#ifndef arch_atomic_fetch_add_relaxed
225#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
226#define arch_atomic_fetch_add_release arch_atomic_fetch_add
227#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add
228#else /* arch_atomic_fetch_add_relaxed */
229
230#ifndef arch_atomic_fetch_add_acquire
231static __always_inline int
232arch_atomic_fetch_add_acquire(int i, atomic_t *v)
233{
234 int ret = arch_atomic_fetch_add_relaxed(i, v);
235 __atomic_acquire_fence();
236 return ret;
237}
238#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
239#endif
240
241#ifndef arch_atomic_fetch_add_release
242static __always_inline int
243arch_atomic_fetch_add_release(int i, atomic_t *v)
244{
245 __atomic_release_fence();
246 return arch_atomic_fetch_add_relaxed(i, v);
247}
248#define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
249#endif
250
251#ifndef arch_atomic_fetch_add
252static __always_inline int
253arch_atomic_fetch_add(int i, atomic_t *v)
254{
255 int ret;
256 __atomic_pre_full_fence();
257 ret = arch_atomic_fetch_add_relaxed(i, v);
258 __atomic_post_full_fence();
259 return ret;
260}
261#define arch_atomic_fetch_add arch_atomic_fetch_add
262#endif
263
264#endif /* arch_atomic_fetch_add_relaxed */
265
266#ifndef arch_atomic_sub_return_relaxed
267#define arch_atomic_sub_return_acquire arch_atomic_sub_return
268#define arch_atomic_sub_return_release arch_atomic_sub_return
269#define arch_atomic_sub_return_relaxed arch_atomic_sub_return
270#else /* arch_atomic_sub_return_relaxed */
271
272#ifndef arch_atomic_sub_return_acquire
273static __always_inline int
274arch_atomic_sub_return_acquire(int i, atomic_t *v)
275{
276 int ret = arch_atomic_sub_return_relaxed(i, v);
277 __atomic_acquire_fence();
278 return ret;
279}
280#define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
281#endif
282
283#ifndef arch_atomic_sub_return_release
284static __always_inline int
285arch_atomic_sub_return_release(int i, atomic_t *v)
286{
287 __atomic_release_fence();
288 return arch_atomic_sub_return_relaxed(i, v);
289}
290#define arch_atomic_sub_return_release arch_atomic_sub_return_release
291#endif
292
293#ifndef arch_atomic_sub_return
294static __always_inline int
295arch_atomic_sub_return(int i, atomic_t *v)
296{
297 int ret;
298 __atomic_pre_full_fence();
299 ret = arch_atomic_sub_return_relaxed(i, v);
300 __atomic_post_full_fence();
301 return ret;
302}
303#define arch_atomic_sub_return arch_atomic_sub_return
304#endif
305
306#endif /* arch_atomic_sub_return_relaxed */
307
308#ifndef arch_atomic_fetch_sub_relaxed
309#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
310#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
311#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub
312#else /* arch_atomic_fetch_sub_relaxed */
313
314#ifndef arch_atomic_fetch_sub_acquire
315static __always_inline int
316arch_atomic_fetch_sub_acquire(int i, atomic_t *v)
317{
318 int ret = arch_atomic_fetch_sub_relaxed(i, v);
319 __atomic_acquire_fence();
320 return ret;
321}
322#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
323#endif
324
325#ifndef arch_atomic_fetch_sub_release
326static __always_inline int
327arch_atomic_fetch_sub_release(int i, atomic_t *v)
328{
329 __atomic_release_fence();
330 return arch_atomic_fetch_sub_relaxed(i, v);
331}
332#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
333#endif
334
335#ifndef arch_atomic_fetch_sub
336static __always_inline int
337arch_atomic_fetch_sub(int i, atomic_t *v)
338{
339 int ret;
340 __atomic_pre_full_fence();
341 ret = arch_atomic_fetch_sub_relaxed(i, v);
342 __atomic_post_full_fence();
343 return ret;
344}
345#define arch_atomic_fetch_sub arch_atomic_fetch_sub
346#endif
347
348#endif /* arch_atomic_fetch_sub_relaxed */
349
350#ifndef arch_atomic_inc
351static __always_inline void
352arch_atomic_inc(atomic_t *v)
353{
354 arch_atomic_add(1, v);
355}
356#define arch_atomic_inc arch_atomic_inc
357#endif
358
359#ifndef arch_atomic_inc_return_relaxed
360#ifdef arch_atomic_inc_return
361#define arch_atomic_inc_return_acquire arch_atomic_inc_return
362#define arch_atomic_inc_return_release arch_atomic_inc_return
363#define arch_atomic_inc_return_relaxed arch_atomic_inc_return
364#endif /* arch_atomic_inc_return */
365
366#ifndef arch_atomic_inc_return
367static __always_inline int
368arch_atomic_inc_return(atomic_t *v)
369{
370 return arch_atomic_add_return(1, v);
371}
372#define arch_atomic_inc_return arch_atomic_inc_return
373#endif
374
375#ifndef arch_atomic_inc_return_acquire
376static __always_inline int
377arch_atomic_inc_return_acquire(atomic_t *v)
378{
379 return arch_atomic_add_return_acquire(1, v);
380}
381#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
382#endif
383
384#ifndef arch_atomic_inc_return_release
385static __always_inline int
386arch_atomic_inc_return_release(atomic_t *v)
387{
388 return arch_atomic_add_return_release(1, v);
389}
390#define arch_atomic_inc_return_release arch_atomic_inc_return_release
391#endif
392
393#ifndef arch_atomic_inc_return_relaxed
394static __always_inline int
395arch_atomic_inc_return_relaxed(atomic_t *v)
396{
397 return arch_atomic_add_return_relaxed(1, v);
398}
399#define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed
400#endif
401
402#else /* arch_atomic_inc_return_relaxed */
403
404#ifndef arch_atomic_inc_return_acquire
405static __always_inline int
406arch_atomic_inc_return_acquire(atomic_t *v)
407{
408 int ret = arch_atomic_inc_return_relaxed(v);
409 __atomic_acquire_fence();
410 return ret;
411}
412#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire
413#endif
414
415#ifndef arch_atomic_inc_return_release
416static __always_inline int
417arch_atomic_inc_return_release(atomic_t *v)
418{
419 __atomic_release_fence();
420 return arch_atomic_inc_return_relaxed(v);
421}
422#define arch_atomic_inc_return_release arch_atomic_inc_return_release
423#endif
424
425#ifndef arch_atomic_inc_return
426static __always_inline int
427arch_atomic_inc_return(atomic_t *v)
428{
429 int ret;
430 __atomic_pre_full_fence();
431 ret = arch_atomic_inc_return_relaxed(v);
432 __atomic_post_full_fence();
433 return ret;
434}
435#define arch_atomic_inc_return arch_atomic_inc_return
436#endif
437
438#endif /* arch_atomic_inc_return_relaxed */
439
440#ifndef arch_atomic_fetch_inc_relaxed
441#ifdef arch_atomic_fetch_inc
442#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc
443#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc
444#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc
445#endif /* arch_atomic_fetch_inc */
446
447#ifndef arch_atomic_fetch_inc
448static __always_inline int
449arch_atomic_fetch_inc(atomic_t *v)
450{
451 return arch_atomic_fetch_add(1, v);
452}
453#define arch_atomic_fetch_inc arch_atomic_fetch_inc
454#endif
455
456#ifndef arch_atomic_fetch_inc_acquire
457static __always_inline int
458arch_atomic_fetch_inc_acquire(atomic_t *v)
459{
460 return arch_atomic_fetch_add_acquire(1, v);
461}
462#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
463#endif
464
465#ifndef arch_atomic_fetch_inc_release
466static __always_inline int
467arch_atomic_fetch_inc_release(atomic_t *v)
468{
469 return arch_atomic_fetch_add_release(1, v);
470}
471#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
472#endif
473
474#ifndef arch_atomic_fetch_inc_relaxed
475static __always_inline int
476arch_atomic_fetch_inc_relaxed(atomic_t *v)
477{
478 return arch_atomic_fetch_add_relaxed(1, v);
479}
480#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed
481#endif
482
483#else /* arch_atomic_fetch_inc_relaxed */
484
485#ifndef arch_atomic_fetch_inc_acquire
486static __always_inline int
487arch_atomic_fetch_inc_acquire(atomic_t *v)
488{
489 int ret = arch_atomic_fetch_inc_relaxed(v);
490 __atomic_acquire_fence();
491 return ret;
492}
493#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire
494#endif
495
496#ifndef arch_atomic_fetch_inc_release
497static __always_inline int
498arch_atomic_fetch_inc_release(atomic_t *v)
499{
500 __atomic_release_fence();
501 return arch_atomic_fetch_inc_relaxed(v);
502}
503#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release
504#endif
505
506#ifndef arch_atomic_fetch_inc
507static __always_inline int
508arch_atomic_fetch_inc(atomic_t *v)
509{
510 int ret;
511 __atomic_pre_full_fence();
512 ret = arch_atomic_fetch_inc_relaxed(v);
513 __atomic_post_full_fence();
514 return ret;
515}
516#define arch_atomic_fetch_inc arch_atomic_fetch_inc
517#endif
518
519#endif /* arch_atomic_fetch_inc_relaxed */
520
521#ifndef arch_atomic_dec
522static __always_inline void
523arch_atomic_dec(atomic_t *v)
524{
525 arch_atomic_sub(1, v);
526}
527#define arch_atomic_dec arch_atomic_dec
528#endif
529
530#ifndef arch_atomic_dec_return_relaxed
531#ifdef arch_atomic_dec_return
532#define arch_atomic_dec_return_acquire arch_atomic_dec_return
533#define arch_atomic_dec_return_release arch_atomic_dec_return
534#define arch_atomic_dec_return_relaxed arch_atomic_dec_return
535#endif /* arch_atomic_dec_return */
536
537#ifndef arch_atomic_dec_return
538static __always_inline int
539arch_atomic_dec_return(atomic_t *v)
540{
541 return arch_atomic_sub_return(1, v);
542}
543#define arch_atomic_dec_return arch_atomic_dec_return
544#endif
545
546#ifndef arch_atomic_dec_return_acquire
547static __always_inline int
548arch_atomic_dec_return_acquire(atomic_t *v)
549{
550 return arch_atomic_sub_return_acquire(1, v);
551}
552#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
553#endif
554
555#ifndef arch_atomic_dec_return_release
556static __always_inline int
557arch_atomic_dec_return_release(atomic_t *v)
558{
559 return arch_atomic_sub_return_release(1, v);
560}
561#define arch_atomic_dec_return_release arch_atomic_dec_return_release
562#endif
563
564#ifndef arch_atomic_dec_return_relaxed
565static __always_inline int
566arch_atomic_dec_return_relaxed(atomic_t *v)
567{
568 return arch_atomic_sub_return_relaxed(1, v);
569}
570#define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed
571#endif
572
573#else /* arch_atomic_dec_return_relaxed */
574
575#ifndef arch_atomic_dec_return_acquire
576static __always_inline int
577arch_atomic_dec_return_acquire(atomic_t *v)
578{
579 int ret = arch_atomic_dec_return_relaxed(v);
580 __atomic_acquire_fence();
581 return ret;
582}
583#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire
584#endif
585
586#ifndef arch_atomic_dec_return_release
587static __always_inline int
588arch_atomic_dec_return_release(atomic_t *v)
589{
590 __atomic_release_fence();
591 return arch_atomic_dec_return_relaxed(v);
592}
593#define arch_atomic_dec_return_release arch_atomic_dec_return_release
594#endif
595
596#ifndef arch_atomic_dec_return
597static __always_inline int
598arch_atomic_dec_return(atomic_t *v)
599{
600 int ret;
601 __atomic_pre_full_fence();
602 ret = arch_atomic_dec_return_relaxed(v);
603 __atomic_post_full_fence();
604 return ret;
605}
606#define arch_atomic_dec_return arch_atomic_dec_return
607#endif
608
609#endif /* arch_atomic_dec_return_relaxed */
610
611#ifndef arch_atomic_fetch_dec_relaxed
612#ifdef arch_atomic_fetch_dec
613#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec
614#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec
615#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec
616#endif /* arch_atomic_fetch_dec */
617
618#ifndef arch_atomic_fetch_dec
619static __always_inline int
620arch_atomic_fetch_dec(atomic_t *v)
621{
622 return arch_atomic_fetch_sub(1, v);
623}
624#define arch_atomic_fetch_dec arch_atomic_fetch_dec
625#endif
626
627#ifndef arch_atomic_fetch_dec_acquire
628static __always_inline int
629arch_atomic_fetch_dec_acquire(atomic_t *v)
630{
631 return arch_atomic_fetch_sub_acquire(1, v);
632}
633#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
634#endif
635
636#ifndef arch_atomic_fetch_dec_release
637static __always_inline int
638arch_atomic_fetch_dec_release(atomic_t *v)
639{
640 return arch_atomic_fetch_sub_release(1, v);
641}
642#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
643#endif
644
645#ifndef arch_atomic_fetch_dec_relaxed
646static __always_inline int
647arch_atomic_fetch_dec_relaxed(atomic_t *v)
648{
649 return arch_atomic_fetch_sub_relaxed(1, v);
650}
651#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed
652#endif
653
654#else /* arch_atomic_fetch_dec_relaxed */
655
656#ifndef arch_atomic_fetch_dec_acquire
657static __always_inline int
658arch_atomic_fetch_dec_acquire(atomic_t *v)
659{
660 int ret = arch_atomic_fetch_dec_relaxed(v);
661 __atomic_acquire_fence();
662 return ret;
663}
664#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire
665#endif
666
667#ifndef arch_atomic_fetch_dec_release
668static __always_inline int
669arch_atomic_fetch_dec_release(atomic_t *v)
670{
671 __atomic_release_fence();
672 return arch_atomic_fetch_dec_relaxed(v);
673}
674#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release
675#endif
676
677#ifndef arch_atomic_fetch_dec
678static __always_inline int
679arch_atomic_fetch_dec(atomic_t *v)
680{
681 int ret;
682 __atomic_pre_full_fence();
683 ret = arch_atomic_fetch_dec_relaxed(v);
684 __atomic_post_full_fence();
685 return ret;
686}
687#define arch_atomic_fetch_dec arch_atomic_fetch_dec
688#endif
689
690#endif /* arch_atomic_fetch_dec_relaxed */
691
692#ifndef arch_atomic_fetch_and_relaxed
693#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
694#define arch_atomic_fetch_and_release arch_atomic_fetch_and
695#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and
696#else /* arch_atomic_fetch_and_relaxed */
697
698#ifndef arch_atomic_fetch_and_acquire
699static __always_inline int
700arch_atomic_fetch_and_acquire(int i, atomic_t *v)
701{
702 int ret = arch_atomic_fetch_and_relaxed(i, v);
703 __atomic_acquire_fence();
704 return ret;
705}
706#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
707#endif
708
709#ifndef arch_atomic_fetch_and_release
710static __always_inline int
711arch_atomic_fetch_and_release(int i, atomic_t *v)
712{
713 __atomic_release_fence();
714 return arch_atomic_fetch_and_relaxed(i, v);
715}
716#define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
717#endif
718
719#ifndef arch_atomic_fetch_and
720static __always_inline int
721arch_atomic_fetch_and(int i, atomic_t *v)
722{
723 int ret;
724 __atomic_pre_full_fence();
725 ret = arch_atomic_fetch_and_relaxed(i, v);
726 __atomic_post_full_fence();
727 return ret;
728}
729#define arch_atomic_fetch_and arch_atomic_fetch_and
730#endif
731
732#endif /* arch_atomic_fetch_and_relaxed */
733
734#ifndef arch_atomic_andnot
735static __always_inline void
736arch_atomic_andnot(int i, atomic_t *v)
737{
738 arch_atomic_and(~i, v);
739}
740#define arch_atomic_andnot arch_atomic_andnot
741#endif
742
743#ifndef arch_atomic_fetch_andnot_relaxed
744#ifdef arch_atomic_fetch_andnot
745#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot
746#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot
747#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot
748#endif /* arch_atomic_fetch_andnot */
749
750#ifndef arch_atomic_fetch_andnot
751static __always_inline int
752arch_atomic_fetch_andnot(int i, atomic_t *v)
753{
754 return arch_atomic_fetch_and(~i, v);
755}
756#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
757#endif
758
759#ifndef arch_atomic_fetch_andnot_acquire
760static __always_inline int
761arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
762{
763 return arch_atomic_fetch_and_acquire(~i, v);
764}
765#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
766#endif
767
768#ifndef arch_atomic_fetch_andnot_release
769static __always_inline int
770arch_atomic_fetch_andnot_release(int i, atomic_t *v)
771{
772 return arch_atomic_fetch_and_release(~i, v);
773}
774#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
775#endif
776
777#ifndef arch_atomic_fetch_andnot_relaxed
778static __always_inline int
779arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v)
780{
781 return arch_atomic_fetch_and_relaxed(~i, v);
782}
783#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
784#endif
785
786#else /* arch_atomic_fetch_andnot_relaxed */
787
788#ifndef arch_atomic_fetch_andnot_acquire
789static __always_inline int
790arch_atomic_fetch_andnot_acquire(int i, atomic_t *v)
791{
792 int ret = arch_atomic_fetch_andnot_relaxed(i, v);
793 __atomic_acquire_fence();
794 return ret;
795}
796#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
797#endif
798
799#ifndef arch_atomic_fetch_andnot_release
800static __always_inline int
801arch_atomic_fetch_andnot_release(int i, atomic_t *v)
802{
803 __atomic_release_fence();
804 return arch_atomic_fetch_andnot_relaxed(i, v);
805}
806#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
807#endif
808
809#ifndef arch_atomic_fetch_andnot
810static __always_inline int
811arch_atomic_fetch_andnot(int i, atomic_t *v)
812{
813 int ret;
814 __atomic_pre_full_fence();
815 ret = arch_atomic_fetch_andnot_relaxed(i, v);
816 __atomic_post_full_fence();
817 return ret;
818}
819#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
820#endif
821
822#endif /* arch_atomic_fetch_andnot_relaxed */
823
824#ifndef arch_atomic_fetch_or_relaxed
825#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
826#define arch_atomic_fetch_or_release arch_atomic_fetch_or
827#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or
828#else /* arch_atomic_fetch_or_relaxed */
829
830#ifndef arch_atomic_fetch_or_acquire
831static __always_inline int
832arch_atomic_fetch_or_acquire(int i, atomic_t *v)
833{
834 int ret = arch_atomic_fetch_or_relaxed(i, v);
835 __atomic_acquire_fence();
836 return ret;
837}
838#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
839#endif
840
841#ifndef arch_atomic_fetch_or_release
842static __always_inline int
843arch_atomic_fetch_or_release(int i, atomic_t *v)
844{
845 __atomic_release_fence();
846 return arch_atomic_fetch_or_relaxed(i, v);
847}
848#define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
849#endif
850
851#ifndef arch_atomic_fetch_or
852static __always_inline int
853arch_atomic_fetch_or(int i, atomic_t *v)
854{
855 int ret;
856 __atomic_pre_full_fence();
857 ret = arch_atomic_fetch_or_relaxed(i, v);
858 __atomic_post_full_fence();
859 return ret;
860}
861#define arch_atomic_fetch_or arch_atomic_fetch_or
862#endif
863
864#endif /* arch_atomic_fetch_or_relaxed */
865
866#ifndef arch_atomic_fetch_xor_relaxed
867#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
868#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
869#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor
870#else /* arch_atomic_fetch_xor_relaxed */
871
872#ifndef arch_atomic_fetch_xor_acquire
873static __always_inline int
874arch_atomic_fetch_xor_acquire(int i, atomic_t *v)
875{
876 int ret = arch_atomic_fetch_xor_relaxed(i, v);
877 __atomic_acquire_fence();
878 return ret;
879}
880#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
881#endif
882
883#ifndef arch_atomic_fetch_xor_release
884static __always_inline int
885arch_atomic_fetch_xor_release(int i, atomic_t *v)
886{
887 __atomic_release_fence();
888 return arch_atomic_fetch_xor_relaxed(i, v);
889}
890#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
891#endif
892
893#ifndef arch_atomic_fetch_xor
894static __always_inline int
895arch_atomic_fetch_xor(int i, atomic_t *v)
896{
897 int ret;
898 __atomic_pre_full_fence();
899 ret = arch_atomic_fetch_xor_relaxed(i, v);
900 __atomic_post_full_fence();
901 return ret;
902}
903#define arch_atomic_fetch_xor arch_atomic_fetch_xor
904#endif
905
906#endif /* arch_atomic_fetch_xor_relaxed */
907
908#ifndef arch_atomic_xchg_relaxed
909#define arch_atomic_xchg_acquire arch_atomic_xchg
910#define arch_atomic_xchg_release arch_atomic_xchg
911#define arch_atomic_xchg_relaxed arch_atomic_xchg
912#else /* arch_atomic_xchg_relaxed */
913
914#ifndef arch_atomic_xchg_acquire
915static __always_inline int
916arch_atomic_xchg_acquire(atomic_t *v, int i)
917{
918 int ret = arch_atomic_xchg_relaxed(v, i);
919 __atomic_acquire_fence();
920 return ret;
921}
922#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire
923#endif
924
925#ifndef arch_atomic_xchg_release
926static __always_inline int
927arch_atomic_xchg_release(atomic_t *v, int i)
928{
929 __atomic_release_fence();
930 return arch_atomic_xchg_relaxed(v, i);
931}
932#define arch_atomic_xchg_release arch_atomic_xchg_release
933#endif
934
935#ifndef arch_atomic_xchg
936static __always_inline int
937arch_atomic_xchg(atomic_t *v, int i)
938{
939 int ret;
940 __atomic_pre_full_fence();
941 ret = arch_atomic_xchg_relaxed(v, i);
942 __atomic_post_full_fence();
943 return ret;
944}
945#define arch_atomic_xchg arch_atomic_xchg
946#endif
947
948#endif /* arch_atomic_xchg_relaxed */
949
950#ifndef arch_atomic_cmpxchg_relaxed
951#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg
952#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg
953#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg
954#else /* arch_atomic_cmpxchg_relaxed */
955
956#ifndef arch_atomic_cmpxchg_acquire
957static __always_inline int
958arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
959{
960 int ret = arch_atomic_cmpxchg_relaxed(v, old, new);
961 __atomic_acquire_fence();
962 return ret;
963}
964#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire
965#endif
966
967#ifndef arch_atomic_cmpxchg_release
968static __always_inline int
969arch_atomic_cmpxchg_release(atomic_t *v, int old, int new)
970{
971 __atomic_release_fence();
972 return arch_atomic_cmpxchg_relaxed(v, old, new);
973}
974#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release
975#endif
976
977#ifndef arch_atomic_cmpxchg
978static __always_inline int
979arch_atomic_cmpxchg(atomic_t *v, int old, int new)
980{
981 int ret;
982 __atomic_pre_full_fence();
983 ret = arch_atomic_cmpxchg_relaxed(v, old, new);
984 __atomic_post_full_fence();
985 return ret;
986}
987#define arch_atomic_cmpxchg arch_atomic_cmpxchg
988#endif
989
990#endif /* arch_atomic_cmpxchg_relaxed */
991
992#ifndef arch_atomic_try_cmpxchg_relaxed
993#ifdef arch_atomic_try_cmpxchg
994#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg
995#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg
996#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg
997#endif /* arch_atomic_try_cmpxchg */
998
999#ifndef arch_atomic_try_cmpxchg
1000static __always_inline bool
1001arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1002{
1003 int r, o = *old;
1004 r = arch_atomic_cmpxchg(v, o, new);
1005 if (unlikely(r != o))
1006 *old = r;
1007 return likely(r == o);
1008}
1009#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1010#endif
1011
1012#ifndef arch_atomic_try_cmpxchg_acquire
1013static __always_inline bool
1014arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1015{
1016 int r, o = *old;
1017 r = arch_atomic_cmpxchg_acquire(v, o, new);
1018 if (unlikely(r != o))
1019 *old = r;
1020 return likely(r == o);
1021}
1022#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1023#endif
1024
1025#ifndef arch_atomic_try_cmpxchg_release
1026static __always_inline bool
1027arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1028{
1029 int r, o = *old;
1030 r = arch_atomic_cmpxchg_release(v, o, new);
1031 if (unlikely(r != o))
1032 *old = r;
1033 return likely(r == o);
1034}
1035#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1036#endif
1037
1038#ifndef arch_atomic_try_cmpxchg_relaxed
1039static __always_inline bool
1040arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
1041{
1042 int r, o = *old;
1043 r = arch_atomic_cmpxchg_relaxed(v, o, new);
1044 if (unlikely(r != o))
1045 *old = r;
1046 return likely(r == o);
1047}
1048#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed
1049#endif
1050
1051#else /* arch_atomic_try_cmpxchg_relaxed */
1052
1053#ifndef arch_atomic_try_cmpxchg_acquire
1054static __always_inline bool
1055arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
1056{
1057 bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1058 __atomic_acquire_fence();
1059 return ret;
1060}
1061#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire
1062#endif
1063
1064#ifndef arch_atomic_try_cmpxchg_release
1065static __always_inline bool
1066arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
1067{
1068 __atomic_release_fence();
1069 return arch_atomic_try_cmpxchg_relaxed(v, old, new);
1070}
1071#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release
1072#endif
1073
1074#ifndef arch_atomic_try_cmpxchg
1075static __always_inline bool
1076arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new)
1077{
1078 bool ret;
1079 __atomic_pre_full_fence();
1080 ret = arch_atomic_try_cmpxchg_relaxed(v, old, new);
1081 __atomic_post_full_fence();
1082 return ret;
1083}
1084#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg
1085#endif
1086
1087#endif /* arch_atomic_try_cmpxchg_relaxed */
1088
1089#ifndef arch_atomic_sub_and_test
1090/**
1091 * arch_atomic_sub_and_test - subtract value from variable and test result
1092 * @i: integer value to subtract
1093 * @v: pointer of type atomic_t
1094 *
1095 * Atomically subtracts @i from @v and returns
1096 * true if the result is zero, or false for all
1097 * other cases.
1098 */
1099static __always_inline bool
1100arch_atomic_sub_and_test(int i, atomic_t *v)
1101{
1102 return arch_atomic_sub_return(i, v) == 0;
1103}
1104#define arch_atomic_sub_and_test arch_atomic_sub_and_test
1105#endif
1106
1107#ifndef arch_atomic_dec_and_test
1108/**
1109 * arch_atomic_dec_and_test - decrement and test
1110 * @v: pointer of type atomic_t
1111 *
1112 * Atomically decrements @v by 1 and
1113 * returns true if the result is 0, or false for all other
1114 * cases.
1115 */
1116static __always_inline bool
1117arch_atomic_dec_and_test(atomic_t *v)
1118{
1119 return arch_atomic_dec_return(v) == 0;
1120}
1121#define arch_atomic_dec_and_test arch_atomic_dec_and_test
1122#endif
1123
1124#ifndef arch_atomic_inc_and_test
1125/**
1126 * arch_atomic_inc_and_test - increment and test
1127 * @v: pointer of type atomic_t
1128 *
1129 * Atomically increments @v by 1
1130 * and returns true if the result is zero, or false for all
1131 * other cases.
1132 */
1133static __always_inline bool
1134arch_atomic_inc_and_test(atomic_t *v)
1135{
1136 return arch_atomic_inc_return(v) == 0;
1137}
1138#define arch_atomic_inc_and_test arch_atomic_inc_and_test
1139#endif
1140
1141#ifndef arch_atomic_add_negative
1142/**
1143 * arch_atomic_add_negative - add and test if negative
1144 * @i: integer value to add
1145 * @v: pointer of type atomic_t
1146 *
1147 * Atomically adds @i to @v and returns true
1148 * if the result is negative, or false when
1149 * result is greater than or equal to zero.
1150 */
1151static __always_inline bool
1152arch_atomic_add_negative(int i, atomic_t *v)
1153{
1154 return arch_atomic_add_return(i, v) < 0;
1155}
1156#define arch_atomic_add_negative arch_atomic_add_negative
1157#endif
1158
1159#ifndef arch_atomic_fetch_add_unless
1160/**
1161 * arch_atomic_fetch_add_unless - add unless the number is already a given value
1162 * @v: pointer of type atomic_t
1163 * @a: the amount to add to v...
1164 * @u: ...unless v is equal to u.
1165 *
1166 * Atomically adds @a to @v, so long as @v was not already @u.
1167 * Returns original value of @v
1168 */
1169static __always_inline int
1170arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
1171{
1172 int c = arch_atomic_read(v);
1173
1174 do {
1175 if (unlikely(c == u))
1176 break;
1177 } while (!arch_atomic_try_cmpxchg(v, &c, c + a));
1178
1179 return c;
1180}
1181#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless
1182#endif
1183
1184#ifndef arch_atomic_add_unless
1185/**
1186 * arch_atomic_add_unless - add unless the number is already a given value
1187 * @v: pointer of type atomic_t
1188 * @a: the amount to add to v...
1189 * @u: ...unless v is equal to u.
1190 *
1191 * Atomically adds @a to @v, if @v was not already @u.
1192 * Returns true if the addition was done.
1193 */
1194static __always_inline bool
1195arch_atomic_add_unless(atomic_t *v, int a, int u)
1196{
1197 return arch_atomic_fetch_add_unless(v, a, u) != u;
1198}
1199#define arch_atomic_add_unless arch_atomic_add_unless
1200#endif
1201
1202#ifndef arch_atomic_inc_not_zero
1203/**
1204 * arch_atomic_inc_not_zero - increment unless the number is zero
1205 * @v: pointer of type atomic_t
1206 *
1207 * Atomically increments @v by 1, if @v is non-zero.
1208 * Returns true if the increment was done.
1209 */
1210static __always_inline bool
1211arch_atomic_inc_not_zero(atomic_t *v)
1212{
1213 return arch_atomic_add_unless(v, 1, 0);
1214}
1215#define arch_atomic_inc_not_zero arch_atomic_inc_not_zero
1216#endif
1217
1218#ifndef arch_atomic_inc_unless_negative
1219static __always_inline bool
1220arch_atomic_inc_unless_negative(atomic_t *v)
1221{
1222 int c = arch_atomic_read(v);
1223
1224 do {
1225 if (unlikely(c < 0))
1226 return false;
1227 } while (!arch_atomic_try_cmpxchg(v, &c, c + 1));
1228
1229 return true;
1230}
1231#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative
1232#endif
1233
1234#ifndef arch_atomic_dec_unless_positive
1235static __always_inline bool
1236arch_atomic_dec_unless_positive(atomic_t *v)
1237{
1238 int c = arch_atomic_read(v);
1239
1240 do {
1241 if (unlikely(c > 0))
1242 return false;
1243 } while (!arch_atomic_try_cmpxchg(v, &c, c - 1));
1244
1245 return true;
1246}
1247#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive
1248#endif
1249
1250#ifndef arch_atomic_dec_if_positive
1251static __always_inline int
1252arch_atomic_dec_if_positive(atomic_t *v)
1253{
1254 int dec, c = arch_atomic_read(v);
1255
1256 do {
1257 dec = c - 1;
1258 if (unlikely(dec < 0))
1259 break;
1260 } while (!arch_atomic_try_cmpxchg(v, &c, dec));
1261
1262 return dec;
1263}
1264#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive
1265#endif
1266
1267#ifdef CONFIG_GENERIC_ATOMIC64
1268#include <asm-generic/atomic64.h>
1269#endif
1270
1271#ifndef arch_atomic64_read_acquire
1272static __always_inline s64
1273arch_atomic64_read_acquire(const atomic64_t *v)
1274{
1275 s64 ret;
1276
1277 if (__native_word(atomic64_t)) {
1278 ret = smp_load_acquire(&(v)->counter);
1279 } else {
1280 ret = arch_atomic64_read(v);
1281 __atomic_acquire_fence();
1282 }
1283
1284 return ret;
1285}
1286#define arch_atomic64_read_acquire arch_atomic64_read_acquire
1287#endif
1288
1289#ifndef arch_atomic64_set_release
1290static __always_inline void
1291arch_atomic64_set_release(atomic64_t *v, s64 i)
1292{
1293 if (__native_word(atomic64_t)) {
1294 smp_store_release(&(v)->counter, i);
1295 } else {
1296 __atomic_release_fence();
1297 arch_atomic64_set(v, i);
1298 }
1299}
1300#define arch_atomic64_set_release arch_atomic64_set_release
1301#endif
1302
1303#ifndef arch_atomic64_add_return_relaxed
1304#define arch_atomic64_add_return_acquire arch_atomic64_add_return
1305#define arch_atomic64_add_return_release arch_atomic64_add_return
1306#define arch_atomic64_add_return_relaxed arch_atomic64_add_return
1307#else /* arch_atomic64_add_return_relaxed */
1308
1309#ifndef arch_atomic64_add_return_acquire
1310static __always_inline s64
1311arch_atomic64_add_return_acquire(s64 i, atomic64_t *v)
1312{
1313 s64 ret = arch_atomic64_add_return_relaxed(i, v);
1314 __atomic_acquire_fence();
1315 return ret;
1316}
1317#define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
1318#endif
1319
1320#ifndef arch_atomic64_add_return_release
1321static __always_inline s64
1322arch_atomic64_add_return_release(s64 i, atomic64_t *v)
1323{
1324 __atomic_release_fence();
1325 return arch_atomic64_add_return_relaxed(i, v);
1326}
1327#define arch_atomic64_add_return_release arch_atomic64_add_return_release
1328#endif
1329
1330#ifndef arch_atomic64_add_return
1331static __always_inline s64
1332arch_atomic64_add_return(s64 i, atomic64_t *v)
1333{
1334 s64 ret;
1335 __atomic_pre_full_fence();
1336 ret = arch_atomic64_add_return_relaxed(i, v);
1337 __atomic_post_full_fence();
1338 return ret;
1339}
1340#define arch_atomic64_add_return arch_atomic64_add_return
1341#endif
1342
1343#endif /* arch_atomic64_add_return_relaxed */
1344
1345#ifndef arch_atomic64_fetch_add_relaxed
1346#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
1347#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
1348#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add
1349#else /* arch_atomic64_fetch_add_relaxed */
1350
1351#ifndef arch_atomic64_fetch_add_acquire
1352static __always_inline s64
1353arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1354{
1355 s64 ret = arch_atomic64_fetch_add_relaxed(i, v);
1356 __atomic_acquire_fence();
1357 return ret;
1358}
1359#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
1360#endif
1361
1362#ifndef arch_atomic64_fetch_add_release
1363static __always_inline s64
1364arch_atomic64_fetch_add_release(s64 i, atomic64_t *v)
1365{
1366 __atomic_release_fence();
1367 return arch_atomic64_fetch_add_relaxed(i, v);
1368}
1369#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
1370#endif
1371
1372#ifndef arch_atomic64_fetch_add
1373static __always_inline s64
1374arch_atomic64_fetch_add(s64 i, atomic64_t *v)
1375{
1376 s64 ret;
1377 __atomic_pre_full_fence();
1378 ret = arch_atomic64_fetch_add_relaxed(i, v);
1379 __atomic_post_full_fence();
1380 return ret;
1381}
1382#define arch_atomic64_fetch_add arch_atomic64_fetch_add
1383#endif
1384
1385#endif /* arch_atomic64_fetch_add_relaxed */
1386
1387#ifndef arch_atomic64_sub_return_relaxed
1388#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
1389#define arch_atomic64_sub_return_release arch_atomic64_sub_return
1390#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return
1391#else /* arch_atomic64_sub_return_relaxed */
1392
1393#ifndef arch_atomic64_sub_return_acquire
1394static __always_inline s64
1395arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1396{
1397 s64 ret = arch_atomic64_sub_return_relaxed(i, v);
1398 __atomic_acquire_fence();
1399 return ret;
1400}
1401#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
1402#endif
1403
1404#ifndef arch_atomic64_sub_return_release
1405static __always_inline s64
1406arch_atomic64_sub_return_release(s64 i, atomic64_t *v)
1407{
1408 __atomic_release_fence();
1409 return arch_atomic64_sub_return_relaxed(i, v);
1410}
1411#define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
1412#endif
1413
1414#ifndef arch_atomic64_sub_return
1415static __always_inline s64
1416arch_atomic64_sub_return(s64 i, atomic64_t *v)
1417{
1418 s64 ret;
1419 __atomic_pre_full_fence();
1420 ret = arch_atomic64_sub_return_relaxed(i, v);
1421 __atomic_post_full_fence();
1422 return ret;
1423}
1424#define arch_atomic64_sub_return arch_atomic64_sub_return
1425#endif
1426
1427#endif /* arch_atomic64_sub_return_relaxed */
1428
1429#ifndef arch_atomic64_fetch_sub_relaxed
1430#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
1431#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
1432#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub
1433#else /* arch_atomic64_fetch_sub_relaxed */
1434
1435#ifndef arch_atomic64_fetch_sub_acquire
1436static __always_inline s64
1437arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1438{
1439 s64 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1440 __atomic_acquire_fence();
1441 return ret;
1442}
1443#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
1444#endif
1445
1446#ifndef arch_atomic64_fetch_sub_release
1447static __always_inline s64
1448arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1449{
1450 __atomic_release_fence();
1451 return arch_atomic64_fetch_sub_relaxed(i, v);
1452}
1453#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
1454#endif
1455
1456#ifndef arch_atomic64_fetch_sub
1457static __always_inline s64
1458arch_atomic64_fetch_sub(s64 i, atomic64_t *v)
1459{
1460 s64 ret;
1461 __atomic_pre_full_fence();
1462 ret = arch_atomic64_fetch_sub_relaxed(i, v);
1463 __atomic_post_full_fence();
1464 return ret;
1465}
1466#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
1467#endif
1468
1469#endif /* arch_atomic64_fetch_sub_relaxed */
1470
1471#ifndef arch_atomic64_inc
1472static __always_inline void
1473arch_atomic64_inc(atomic64_t *v)
1474{
1475 arch_atomic64_add(1, v);
1476}
1477#define arch_atomic64_inc arch_atomic64_inc
1478#endif
1479
1480#ifndef arch_atomic64_inc_return_relaxed
1481#ifdef arch_atomic64_inc_return
1482#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return
1483#define arch_atomic64_inc_return_release arch_atomic64_inc_return
1484#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return
1485#endif /* arch_atomic64_inc_return */
1486
1487#ifndef arch_atomic64_inc_return
1488static __always_inline s64
1489arch_atomic64_inc_return(atomic64_t *v)
1490{
1491 return arch_atomic64_add_return(1, v);
1492}
1493#define arch_atomic64_inc_return arch_atomic64_inc_return
1494#endif
1495
1496#ifndef arch_atomic64_inc_return_acquire
1497static __always_inline s64
1498arch_atomic64_inc_return_acquire(atomic64_t *v)
1499{
1500 return arch_atomic64_add_return_acquire(1, v);
1501}
1502#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1503#endif
1504
1505#ifndef arch_atomic64_inc_return_release
1506static __always_inline s64
1507arch_atomic64_inc_return_release(atomic64_t *v)
1508{
1509 return arch_atomic64_add_return_release(1, v);
1510}
1511#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1512#endif
1513
1514#ifndef arch_atomic64_inc_return_relaxed
1515static __always_inline s64
1516arch_atomic64_inc_return_relaxed(atomic64_t *v)
1517{
1518 return arch_atomic64_add_return_relaxed(1, v);
1519}
1520#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed
1521#endif
1522
1523#else /* arch_atomic64_inc_return_relaxed */
1524
1525#ifndef arch_atomic64_inc_return_acquire
1526static __always_inline s64
1527arch_atomic64_inc_return_acquire(atomic64_t *v)
1528{
1529 s64 ret = arch_atomic64_inc_return_relaxed(v);
1530 __atomic_acquire_fence();
1531 return ret;
1532}
1533#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire
1534#endif
1535
1536#ifndef arch_atomic64_inc_return_release
1537static __always_inline s64
1538arch_atomic64_inc_return_release(atomic64_t *v)
1539{
1540 __atomic_release_fence();
1541 return arch_atomic64_inc_return_relaxed(v);
1542}
1543#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release
1544#endif
1545
1546#ifndef arch_atomic64_inc_return
1547static __always_inline s64
1548arch_atomic64_inc_return(atomic64_t *v)
1549{
1550 s64 ret;
1551 __atomic_pre_full_fence();
1552 ret = arch_atomic64_inc_return_relaxed(v);
1553 __atomic_post_full_fence();
1554 return ret;
1555}
1556#define arch_atomic64_inc_return arch_atomic64_inc_return
1557#endif
1558
1559#endif /* arch_atomic64_inc_return_relaxed */
1560
1561#ifndef arch_atomic64_fetch_inc_relaxed
1562#ifdef arch_atomic64_fetch_inc
1563#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc
1564#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc
1565#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc
1566#endif /* arch_atomic64_fetch_inc */
1567
1568#ifndef arch_atomic64_fetch_inc
1569static __always_inline s64
1570arch_atomic64_fetch_inc(atomic64_t *v)
1571{
1572 return arch_atomic64_fetch_add(1, v);
1573}
1574#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1575#endif
1576
1577#ifndef arch_atomic64_fetch_inc_acquire
1578static __always_inline s64
1579arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1580{
1581 return arch_atomic64_fetch_add_acquire(1, v);
1582}
1583#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1584#endif
1585
1586#ifndef arch_atomic64_fetch_inc_release
1587static __always_inline s64
1588arch_atomic64_fetch_inc_release(atomic64_t *v)
1589{
1590 return arch_atomic64_fetch_add_release(1, v);
1591}
1592#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1593#endif
1594
1595#ifndef arch_atomic64_fetch_inc_relaxed
1596static __always_inline s64
1597arch_atomic64_fetch_inc_relaxed(atomic64_t *v)
1598{
1599 return arch_atomic64_fetch_add_relaxed(1, v);
1600}
1601#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed
1602#endif
1603
1604#else /* arch_atomic64_fetch_inc_relaxed */
1605
1606#ifndef arch_atomic64_fetch_inc_acquire
1607static __always_inline s64
1608arch_atomic64_fetch_inc_acquire(atomic64_t *v)
1609{
1610 s64 ret = arch_atomic64_fetch_inc_relaxed(v);
1611 __atomic_acquire_fence();
1612 return ret;
1613}
1614#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire
1615#endif
1616
1617#ifndef arch_atomic64_fetch_inc_release
1618static __always_inline s64
1619arch_atomic64_fetch_inc_release(atomic64_t *v)
1620{
1621 __atomic_release_fence();
1622 return arch_atomic64_fetch_inc_relaxed(v);
1623}
1624#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release
1625#endif
1626
1627#ifndef arch_atomic64_fetch_inc
1628static __always_inline s64
1629arch_atomic64_fetch_inc(atomic64_t *v)
1630{
1631 s64 ret;
1632 __atomic_pre_full_fence();
1633 ret = arch_atomic64_fetch_inc_relaxed(v);
1634 __atomic_post_full_fence();
1635 return ret;
1636}
1637#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc
1638#endif
1639
1640#endif /* arch_atomic64_fetch_inc_relaxed */
1641
1642#ifndef arch_atomic64_dec
1643static __always_inline void
1644arch_atomic64_dec(atomic64_t *v)
1645{
1646 arch_atomic64_sub(1, v);
1647}
1648#define arch_atomic64_dec arch_atomic64_dec
1649#endif
1650
1651#ifndef arch_atomic64_dec_return_relaxed
1652#ifdef arch_atomic64_dec_return
1653#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return
1654#define arch_atomic64_dec_return_release arch_atomic64_dec_return
1655#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return
1656#endif /* arch_atomic64_dec_return */
1657
1658#ifndef arch_atomic64_dec_return
1659static __always_inline s64
1660arch_atomic64_dec_return(atomic64_t *v)
1661{
1662 return arch_atomic64_sub_return(1, v);
1663}
1664#define arch_atomic64_dec_return arch_atomic64_dec_return
1665#endif
1666
1667#ifndef arch_atomic64_dec_return_acquire
1668static __always_inline s64
1669arch_atomic64_dec_return_acquire(atomic64_t *v)
1670{
1671 return arch_atomic64_sub_return_acquire(1, v);
1672}
1673#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1674#endif
1675
1676#ifndef arch_atomic64_dec_return_release
1677static __always_inline s64
1678arch_atomic64_dec_return_release(atomic64_t *v)
1679{
1680 return arch_atomic64_sub_return_release(1, v);
1681}
1682#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1683#endif
1684
1685#ifndef arch_atomic64_dec_return_relaxed
1686static __always_inline s64
1687arch_atomic64_dec_return_relaxed(atomic64_t *v)
1688{
1689 return arch_atomic64_sub_return_relaxed(1, v);
1690}
1691#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed
1692#endif
1693
1694#else /* arch_atomic64_dec_return_relaxed */
1695
1696#ifndef arch_atomic64_dec_return_acquire
1697static __always_inline s64
1698arch_atomic64_dec_return_acquire(atomic64_t *v)
1699{
1700 s64 ret = arch_atomic64_dec_return_relaxed(v);
1701 __atomic_acquire_fence();
1702 return ret;
1703}
1704#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire
1705#endif
1706
1707#ifndef arch_atomic64_dec_return_release
1708static __always_inline s64
1709arch_atomic64_dec_return_release(atomic64_t *v)
1710{
1711 __atomic_release_fence();
1712 return arch_atomic64_dec_return_relaxed(v);
1713}
1714#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release
1715#endif
1716
1717#ifndef arch_atomic64_dec_return
1718static __always_inline s64
1719arch_atomic64_dec_return(atomic64_t *v)
1720{
1721 s64 ret;
1722 __atomic_pre_full_fence();
1723 ret = arch_atomic64_dec_return_relaxed(v);
1724 __atomic_post_full_fence();
1725 return ret;
1726}
1727#define arch_atomic64_dec_return arch_atomic64_dec_return
1728#endif
1729
1730#endif /* arch_atomic64_dec_return_relaxed */
1731
1732#ifndef arch_atomic64_fetch_dec_relaxed
1733#ifdef arch_atomic64_fetch_dec
1734#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec
1735#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec
1736#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec
1737#endif /* arch_atomic64_fetch_dec */
1738
1739#ifndef arch_atomic64_fetch_dec
1740static __always_inline s64
1741arch_atomic64_fetch_dec(atomic64_t *v)
1742{
1743 return arch_atomic64_fetch_sub(1, v);
1744}
1745#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1746#endif
1747
1748#ifndef arch_atomic64_fetch_dec_acquire
1749static __always_inline s64
1750arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1751{
1752 return arch_atomic64_fetch_sub_acquire(1, v);
1753}
1754#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1755#endif
1756
1757#ifndef arch_atomic64_fetch_dec_release
1758static __always_inline s64
1759arch_atomic64_fetch_dec_release(atomic64_t *v)
1760{
1761 return arch_atomic64_fetch_sub_release(1, v);
1762}
1763#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1764#endif
1765
1766#ifndef arch_atomic64_fetch_dec_relaxed
1767static __always_inline s64
1768arch_atomic64_fetch_dec_relaxed(atomic64_t *v)
1769{
1770 return arch_atomic64_fetch_sub_relaxed(1, v);
1771}
1772#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed
1773#endif
1774
1775#else /* arch_atomic64_fetch_dec_relaxed */
1776
1777#ifndef arch_atomic64_fetch_dec_acquire
1778static __always_inline s64
1779arch_atomic64_fetch_dec_acquire(atomic64_t *v)
1780{
1781 s64 ret = arch_atomic64_fetch_dec_relaxed(v);
1782 __atomic_acquire_fence();
1783 return ret;
1784}
1785#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire
1786#endif
1787
1788#ifndef arch_atomic64_fetch_dec_release
1789static __always_inline s64
1790arch_atomic64_fetch_dec_release(atomic64_t *v)
1791{
1792 __atomic_release_fence();
1793 return arch_atomic64_fetch_dec_relaxed(v);
1794}
1795#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release
1796#endif
1797
1798#ifndef arch_atomic64_fetch_dec
1799static __always_inline s64
1800arch_atomic64_fetch_dec(atomic64_t *v)
1801{
1802 s64 ret;
1803 __atomic_pre_full_fence();
1804 ret = arch_atomic64_fetch_dec_relaxed(v);
1805 __atomic_post_full_fence();
1806 return ret;
1807}
1808#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec
1809#endif
1810
1811#endif /* arch_atomic64_fetch_dec_relaxed */
1812
1813#ifndef arch_atomic64_fetch_and_relaxed
1814#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
1815#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
1816#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and
1817#else /* arch_atomic64_fetch_and_relaxed */
1818
1819#ifndef arch_atomic64_fetch_and_acquire
1820static __always_inline s64
1821arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1822{
1823 s64 ret = arch_atomic64_fetch_and_relaxed(i, v);
1824 __atomic_acquire_fence();
1825 return ret;
1826}
1827#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
1828#endif
1829
1830#ifndef arch_atomic64_fetch_and_release
1831static __always_inline s64
1832arch_atomic64_fetch_and_release(s64 i, atomic64_t *v)
1833{
1834 __atomic_release_fence();
1835 return arch_atomic64_fetch_and_relaxed(i, v);
1836}
1837#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
1838#endif
1839
1840#ifndef arch_atomic64_fetch_and
1841static __always_inline s64
1842arch_atomic64_fetch_and(s64 i, atomic64_t *v)
1843{
1844 s64 ret;
1845 __atomic_pre_full_fence();
1846 ret = arch_atomic64_fetch_and_relaxed(i, v);
1847 __atomic_post_full_fence();
1848 return ret;
1849}
1850#define arch_atomic64_fetch_and arch_atomic64_fetch_and
1851#endif
1852
1853#endif /* arch_atomic64_fetch_and_relaxed */
1854
1855#ifndef arch_atomic64_andnot
1856static __always_inline void
1857arch_atomic64_andnot(s64 i, atomic64_t *v)
1858{
1859 arch_atomic64_and(~i, v);
1860}
1861#define arch_atomic64_andnot arch_atomic64_andnot
1862#endif
1863
1864#ifndef arch_atomic64_fetch_andnot_relaxed
1865#ifdef arch_atomic64_fetch_andnot
1866#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot
1867#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot
1868#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot
1869#endif /* arch_atomic64_fetch_andnot */
1870
1871#ifndef arch_atomic64_fetch_andnot
1872static __always_inline s64
1873arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1874{
1875 return arch_atomic64_fetch_and(~i, v);
1876}
1877#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1878#endif
1879
1880#ifndef arch_atomic64_fetch_andnot_acquire
1881static __always_inline s64
1882arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1883{
1884 return arch_atomic64_fetch_and_acquire(~i, v);
1885}
1886#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1887#endif
1888
1889#ifndef arch_atomic64_fetch_andnot_release
1890static __always_inline s64
1891arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1892{
1893 return arch_atomic64_fetch_and_release(~i, v);
1894}
1895#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1896#endif
1897
1898#ifndef arch_atomic64_fetch_andnot_relaxed
1899static __always_inline s64
1900arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1901{
1902 return arch_atomic64_fetch_and_relaxed(~i, v);
1903}
1904#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
1905#endif
1906
1907#else /* arch_atomic64_fetch_andnot_relaxed */
1908
1909#ifndef arch_atomic64_fetch_andnot_acquire
1910static __always_inline s64
1911arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1912{
1913 s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1914 __atomic_acquire_fence();
1915 return ret;
1916}
1917#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
1918#endif
1919
1920#ifndef arch_atomic64_fetch_andnot_release
1921static __always_inline s64
1922arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1923{
1924 __atomic_release_fence();
1925 return arch_atomic64_fetch_andnot_relaxed(i, v);
1926}
1927#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
1928#endif
1929
1930#ifndef arch_atomic64_fetch_andnot
1931static __always_inline s64
1932arch_atomic64_fetch_andnot(s64 i, atomic64_t *v)
1933{
1934 s64 ret;
1935 __atomic_pre_full_fence();
1936 ret = arch_atomic64_fetch_andnot_relaxed(i, v);
1937 __atomic_post_full_fence();
1938 return ret;
1939}
1940#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
1941#endif
1942
1943#endif /* arch_atomic64_fetch_andnot_relaxed */
1944
1945#ifndef arch_atomic64_fetch_or_relaxed
1946#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
1947#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
1948#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or
1949#else /* arch_atomic64_fetch_or_relaxed */
1950
1951#ifndef arch_atomic64_fetch_or_acquire
1952static __always_inline s64
1953arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1954{
1955 s64 ret = arch_atomic64_fetch_or_relaxed(i, v);
1956 __atomic_acquire_fence();
1957 return ret;
1958}
1959#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
1960#endif
1961
1962#ifndef arch_atomic64_fetch_or_release
1963static __always_inline s64
1964arch_atomic64_fetch_or_release(s64 i, atomic64_t *v)
1965{
1966 __atomic_release_fence();
1967 return arch_atomic64_fetch_or_relaxed(i, v);
1968}
1969#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
1970#endif
1971
1972#ifndef arch_atomic64_fetch_or
1973static __always_inline s64
1974arch_atomic64_fetch_or(s64 i, atomic64_t *v)
1975{
1976 s64 ret;
1977 __atomic_pre_full_fence();
1978 ret = arch_atomic64_fetch_or_relaxed(i, v);
1979 __atomic_post_full_fence();
1980 return ret;
1981}
1982#define arch_atomic64_fetch_or arch_atomic64_fetch_or
1983#endif
1984
1985#endif /* arch_atomic64_fetch_or_relaxed */
1986
1987#ifndef arch_atomic64_fetch_xor_relaxed
1988#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
1989#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
1990#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor
1991#else /* arch_atomic64_fetch_xor_relaxed */
1992
1993#ifndef arch_atomic64_fetch_xor_acquire
1994static __always_inline s64
1995arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1996{
1997 s64 ret = arch_atomic64_fetch_xor_relaxed(i, v);
1998 __atomic_acquire_fence();
1999 return ret;
2000}
2001#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
2002#endif
2003
2004#ifndef arch_atomic64_fetch_xor_release
2005static __always_inline s64
2006arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v)
2007{
2008 __atomic_release_fence();
2009 return arch_atomic64_fetch_xor_relaxed(i, v);
2010}
2011#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
2012#endif
2013
2014#ifndef arch_atomic64_fetch_xor
2015static __always_inline s64
2016arch_atomic64_fetch_xor(s64 i, atomic64_t *v)
2017{
2018 s64 ret;
2019 __atomic_pre_full_fence();
2020 ret = arch_atomic64_fetch_xor_relaxed(i, v);
2021 __atomic_post_full_fence();
2022 return ret;
2023}
2024#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
2025#endif
2026
2027#endif /* arch_atomic64_fetch_xor_relaxed */
2028
2029#ifndef arch_atomic64_xchg_relaxed
2030#define arch_atomic64_xchg_acquire arch_atomic64_xchg
2031#define arch_atomic64_xchg_release arch_atomic64_xchg
2032#define arch_atomic64_xchg_relaxed arch_atomic64_xchg
2033#else /* arch_atomic64_xchg_relaxed */
2034
2035#ifndef arch_atomic64_xchg_acquire
2036static __always_inline s64
2037arch_atomic64_xchg_acquire(atomic64_t *v, s64 i)
2038{
2039 s64 ret = arch_atomic64_xchg_relaxed(v, i);
2040 __atomic_acquire_fence();
2041 return ret;
2042}
2043#define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire
2044#endif
2045
2046#ifndef arch_atomic64_xchg_release
2047static __always_inline s64
2048arch_atomic64_xchg_release(atomic64_t *v, s64 i)
2049{
2050 __atomic_release_fence();
2051 return arch_atomic64_xchg_relaxed(v, i);
2052}
2053#define arch_atomic64_xchg_release arch_atomic64_xchg_release
2054#endif
2055
2056#ifndef arch_atomic64_xchg
2057static __always_inline s64
2058arch_atomic64_xchg(atomic64_t *v, s64 i)
2059{
2060 s64 ret;
2061 __atomic_pre_full_fence();
2062 ret = arch_atomic64_xchg_relaxed(v, i);
2063 __atomic_post_full_fence();
2064 return ret;
2065}
2066#define arch_atomic64_xchg arch_atomic64_xchg
2067#endif
2068
2069#endif /* arch_atomic64_xchg_relaxed */
2070
2071#ifndef arch_atomic64_cmpxchg_relaxed
2072#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg
2073#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg
2074#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg
2075#else /* arch_atomic64_cmpxchg_relaxed */
2076
2077#ifndef arch_atomic64_cmpxchg_acquire
2078static __always_inline s64
2079arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
2080{
2081 s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2082 __atomic_acquire_fence();
2083 return ret;
2084}
2085#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire
2086#endif
2087
2088#ifndef arch_atomic64_cmpxchg_release
2089static __always_inline s64
2090arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
2091{
2092 __atomic_release_fence();
2093 return arch_atomic64_cmpxchg_relaxed(v, old, new);
2094}
2095#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release
2096#endif
2097
2098#ifndef arch_atomic64_cmpxchg
2099static __always_inline s64
2100arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2101{
2102 s64 ret;
2103 __atomic_pre_full_fence();
2104 ret = arch_atomic64_cmpxchg_relaxed(v, old, new);
2105 __atomic_post_full_fence();
2106 return ret;
2107}
2108#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg
2109#endif
2110
2111#endif /* arch_atomic64_cmpxchg_relaxed */
2112
2113#ifndef arch_atomic64_try_cmpxchg_relaxed
2114#ifdef arch_atomic64_try_cmpxchg
2115#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg
2116#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg
2117#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg
2118#endif /* arch_atomic64_try_cmpxchg */
2119
2120#ifndef arch_atomic64_try_cmpxchg
2121static __always_inline bool
2122arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2123{
2124 s64 r, o = *old;
2125 r = arch_atomic64_cmpxchg(v, o, new);
2126 if (unlikely(r != o))
2127 *old = r;
2128 return likely(r == o);
2129}
2130#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2131#endif
2132
2133#ifndef arch_atomic64_try_cmpxchg_acquire
2134static __always_inline bool
2135arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2136{
2137 s64 r, o = *old;
2138 r = arch_atomic64_cmpxchg_acquire(v, o, new);
2139 if (unlikely(r != o))
2140 *old = r;
2141 return likely(r == o);
2142}
2143#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2144#endif
2145
2146#ifndef arch_atomic64_try_cmpxchg_release
2147static __always_inline bool
2148arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2149{
2150 s64 r, o = *old;
2151 r = arch_atomic64_cmpxchg_release(v, o, new);
2152 if (unlikely(r != o))
2153 *old = r;
2154 return likely(r == o);
2155}
2156#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2157#endif
2158
2159#ifndef arch_atomic64_try_cmpxchg_relaxed
2160static __always_inline bool
2161arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2162{
2163 s64 r, o = *old;
2164 r = arch_atomic64_cmpxchg_relaxed(v, o, new);
2165 if (unlikely(r != o))
2166 *old = r;
2167 return likely(r == o);
2168}
2169#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed
2170#endif
2171
2172#else /* arch_atomic64_try_cmpxchg_relaxed */
2173
2174#ifndef arch_atomic64_try_cmpxchg_acquire
2175static __always_inline bool
2176arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2177{
2178 bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2179 __atomic_acquire_fence();
2180 return ret;
2181}
2182#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire
2183#endif
2184
2185#ifndef arch_atomic64_try_cmpxchg_release
2186static __always_inline bool
2187arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2188{
2189 __atomic_release_fence();
2190 return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2191}
2192#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release
2193#endif
2194
2195#ifndef arch_atomic64_try_cmpxchg
2196static __always_inline bool
2197arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2198{
2199 bool ret;
2200 __atomic_pre_full_fence();
2201 ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new);
2202 __atomic_post_full_fence();
2203 return ret;
2204}
2205#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg
2206#endif
2207
2208#endif /* arch_atomic64_try_cmpxchg_relaxed */
2209
2210#ifndef arch_atomic64_sub_and_test
2211/**
2212 * arch_atomic64_sub_and_test - subtract value from variable and test result
2213 * @i: integer value to subtract
2214 * @v: pointer of type atomic64_t
2215 *
2216 * Atomically subtracts @i from @v and returns
2217 * true if the result is zero, or false for all
2218 * other cases.
2219 */
2220static __always_inline bool
2221arch_atomic64_sub_and_test(s64 i, atomic64_t *v)
2222{
2223 return arch_atomic64_sub_return(i, v) == 0;
2224}
2225#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test
2226#endif
2227
2228#ifndef arch_atomic64_dec_and_test
2229/**
2230 * arch_atomic64_dec_and_test - decrement and test
2231 * @v: pointer of type atomic64_t
2232 *
2233 * Atomically decrements @v by 1 and
2234 * returns true if the result is 0, or false for all other
2235 * cases.
2236 */
2237static __always_inline bool
2238arch_atomic64_dec_and_test(atomic64_t *v)
2239{
2240 return arch_atomic64_dec_return(v) == 0;
2241}
2242#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test
2243#endif
2244
2245#ifndef arch_atomic64_inc_and_test
2246/**
2247 * arch_atomic64_inc_and_test - increment and test
2248 * @v: pointer of type atomic64_t
2249 *
2250 * Atomically increments @v by 1
2251 * and returns true if the result is zero, or false for all
2252 * other cases.
2253 */
2254static __always_inline bool
2255arch_atomic64_inc_and_test(atomic64_t *v)
2256{
2257 return arch_atomic64_inc_return(v) == 0;
2258}
2259#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test
2260#endif
2261
2262#ifndef arch_atomic64_add_negative
2263/**
2264 * arch_atomic64_add_negative - add and test if negative
2265 * @i: integer value to add
2266 * @v: pointer of type atomic64_t
2267 *
2268 * Atomically adds @i to @v and returns true
2269 * if the result is negative, or false when
2270 * result is greater than or equal to zero.
2271 */
2272static __always_inline bool
2273arch_atomic64_add_negative(s64 i, atomic64_t *v)
2274{
2275 return arch_atomic64_add_return(i, v) < 0;
2276}
2277#define arch_atomic64_add_negative arch_atomic64_add_negative
2278#endif
2279
2280#ifndef arch_atomic64_fetch_add_unless
2281/**
2282 * arch_atomic64_fetch_add_unless - add unless the number is already a given value
2283 * @v: pointer of type atomic64_t
2284 * @a: the amount to add to v...
2285 * @u: ...unless v is equal to u.
2286 *
2287 * Atomically adds @a to @v, so long as @v was not already @u.
2288 * Returns original value of @v
2289 */
2290static __always_inline s64
2291arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2292{
2293 s64 c = arch_atomic64_read(v);
2294
2295 do {
2296 if (unlikely(c == u))
2297 break;
2298 } while (!arch_atomic64_try_cmpxchg(v, &c, c + a));
2299
2300 return c;
2301}
2302#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless
2303#endif
2304
2305#ifndef arch_atomic64_add_unless
2306/**
2307 * arch_atomic64_add_unless - add unless the number is already a given value
2308 * @v: pointer of type atomic64_t
2309 * @a: the amount to add to v...
2310 * @u: ...unless v is equal to u.
2311 *
2312 * Atomically adds @a to @v, if @v was not already @u.
2313 * Returns true if the addition was done.
2314 */
2315static __always_inline bool
2316arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2317{
2318 return arch_atomic64_fetch_add_unless(v, a, u) != u;
2319}
2320#define arch_atomic64_add_unless arch_atomic64_add_unless
2321#endif
2322
2323#ifndef arch_atomic64_inc_not_zero
2324/**
2325 * arch_atomic64_inc_not_zero - increment unless the number is zero
2326 * @v: pointer of type atomic64_t
2327 *
2328 * Atomically increments @v by 1, if @v is non-zero.
2329 * Returns true if the increment was done.
2330 */
2331static __always_inline bool
2332arch_atomic64_inc_not_zero(atomic64_t *v)
2333{
2334 return arch_atomic64_add_unless(v, 1, 0);
2335}
2336#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero
2337#endif
2338
2339#ifndef arch_atomic64_inc_unless_negative
2340static __always_inline bool
2341arch_atomic64_inc_unless_negative(atomic64_t *v)
2342{
2343 s64 c = arch_atomic64_read(v);
2344
2345 do {
2346 if (unlikely(c < 0))
2347 return false;
2348 } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1));
2349
2350 return true;
2351}
2352#define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative
2353#endif
2354
2355#ifndef arch_atomic64_dec_unless_positive
2356static __always_inline bool
2357arch_atomic64_dec_unless_positive(atomic64_t *v)
2358{
2359 s64 c = arch_atomic64_read(v);
2360
2361 do {
2362 if (unlikely(c > 0))
2363 return false;
2364 } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1));
2365
2366 return true;
2367}
2368#define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive
2369#endif
2370
2371#ifndef arch_atomic64_dec_if_positive
2372static __always_inline s64
2373arch_atomic64_dec_if_positive(atomic64_t *v)
2374{
2375 s64 dec, c = arch_atomic64_read(v);
2376
2377 do {
2378 dec = c - 1;
2379 if (unlikely(dec < 0))
2380 break;
2381 } while (!arch_atomic64_try_cmpxchg(v, &c, dec));
2382
2383 return dec;
2384}
2385#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
2386#endif
2387
2388#endif /* _LINUX_ATOMIC_FALLBACK_H */
2389// 8e2cc06bc0d2c0967d2f8424762bd48555ee40ae