Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* Atomic operations usable in machine independent code */
2#ifndef _LINUX_ATOMIC_H
3#define _LINUX_ATOMIC_H
4#include <asm/atomic.h>
5#include <asm/barrier.h>
6
7/*
8 * Relaxed variants of xchg, cmpxchg and some atomic operations.
9 *
10 * We support four variants:
11 *
12 * - Fully ordered: The default implementation, no suffix required.
13 * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
14 * - Release: Provides RELEASE semantics, _release suffix.
15 * - Relaxed: No ordering guarantees, _relaxed suffix.
16 *
17 * For compound atomics performing both a load and a store, ACQUIRE
18 * semantics apply only to the load and RELEASE semantics only to the
19 * store portion of the operation. Note that a failed cmpxchg_acquire
20 * does -not- imply any memory ordering constraints.
21 *
22 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
23 */
24
25#ifndef atomic_read_acquire
26#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
27#endif
28
29#ifndef atomic_set_release
30#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
31#endif
32
33/*
34 * The idea here is to build acquire/release variants by adding explicit
35 * barriers on top of the relaxed variant. In the case where the relaxed
36 * variant is already fully ordered, no additional barriers are needed.
37 *
38 * Besides, if an arch has a special barrier for acquire/release, it could
39 * implement its own __atomic_op_* and use the same framework for building
40 * variants
41 */
42#ifndef __atomic_op_acquire
43#define __atomic_op_acquire(op, args...) \
44({ \
45 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
46 smp_mb__after_atomic(); \
47 __ret; \
48})
49#endif
50
51#ifndef __atomic_op_release
52#define __atomic_op_release(op, args...) \
53({ \
54 smp_mb__before_atomic(); \
55 op##_relaxed(args); \
56})
57#endif
58
59#ifndef __atomic_op_fence
60#define __atomic_op_fence(op, args...) \
61({ \
62 typeof(op##_relaxed(args)) __ret; \
63 smp_mb__before_atomic(); \
64 __ret = op##_relaxed(args); \
65 smp_mb__after_atomic(); \
66 __ret; \
67})
68#endif
69
70/* atomic_add_return_relaxed */
71#ifndef atomic_add_return_relaxed
72#define atomic_add_return_relaxed atomic_add_return
73#define atomic_add_return_acquire atomic_add_return
74#define atomic_add_return_release atomic_add_return
75
76#else /* atomic_add_return_relaxed */
77
78#ifndef atomic_add_return_acquire
79#define atomic_add_return_acquire(...) \
80 __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
81#endif
82
83#ifndef atomic_add_return_release
84#define atomic_add_return_release(...) \
85 __atomic_op_release(atomic_add_return, __VA_ARGS__)
86#endif
87
88#ifndef atomic_add_return
89#define atomic_add_return(...) \
90 __atomic_op_fence(atomic_add_return, __VA_ARGS__)
91#endif
92#endif /* atomic_add_return_relaxed */
93
94/* atomic_inc_return_relaxed */
95#ifndef atomic_inc_return_relaxed
96#define atomic_inc_return_relaxed atomic_inc_return
97#define atomic_inc_return_acquire atomic_inc_return
98#define atomic_inc_return_release atomic_inc_return
99
100#else /* atomic_inc_return_relaxed */
101
102#ifndef atomic_inc_return_acquire
103#define atomic_inc_return_acquire(...) \
104 __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
105#endif
106
107#ifndef atomic_inc_return_release
108#define atomic_inc_return_release(...) \
109 __atomic_op_release(atomic_inc_return, __VA_ARGS__)
110#endif
111
112#ifndef atomic_inc_return
113#define atomic_inc_return(...) \
114 __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
115#endif
116#endif /* atomic_inc_return_relaxed */
117
118/* atomic_sub_return_relaxed */
119#ifndef atomic_sub_return_relaxed
120#define atomic_sub_return_relaxed atomic_sub_return
121#define atomic_sub_return_acquire atomic_sub_return
122#define atomic_sub_return_release atomic_sub_return
123
124#else /* atomic_sub_return_relaxed */
125
126#ifndef atomic_sub_return_acquire
127#define atomic_sub_return_acquire(...) \
128 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
129#endif
130
131#ifndef atomic_sub_return_release
132#define atomic_sub_return_release(...) \
133 __atomic_op_release(atomic_sub_return, __VA_ARGS__)
134#endif
135
136#ifndef atomic_sub_return
137#define atomic_sub_return(...) \
138 __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
139#endif
140#endif /* atomic_sub_return_relaxed */
141
142/* atomic_dec_return_relaxed */
143#ifndef atomic_dec_return_relaxed
144#define atomic_dec_return_relaxed atomic_dec_return
145#define atomic_dec_return_acquire atomic_dec_return
146#define atomic_dec_return_release atomic_dec_return
147
148#else /* atomic_dec_return_relaxed */
149
150#ifndef atomic_dec_return_acquire
151#define atomic_dec_return_acquire(...) \
152 __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
153#endif
154
155#ifndef atomic_dec_return_release
156#define atomic_dec_return_release(...) \
157 __atomic_op_release(atomic_dec_return, __VA_ARGS__)
158#endif
159
160#ifndef atomic_dec_return
161#define atomic_dec_return(...) \
162 __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
163#endif
164#endif /* atomic_dec_return_relaxed */
165
166
167/* atomic_fetch_add_relaxed */
168#ifndef atomic_fetch_add_relaxed
169#define atomic_fetch_add_relaxed atomic_fetch_add
170#define atomic_fetch_add_acquire atomic_fetch_add
171#define atomic_fetch_add_release atomic_fetch_add
172
173#else /* atomic_fetch_add_relaxed */
174
175#ifndef atomic_fetch_add_acquire
176#define atomic_fetch_add_acquire(...) \
177 __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
178#endif
179
180#ifndef atomic_fetch_add_release
181#define atomic_fetch_add_release(...) \
182 __atomic_op_release(atomic_fetch_add, __VA_ARGS__)
183#endif
184
185#ifndef atomic_fetch_add
186#define atomic_fetch_add(...) \
187 __atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
188#endif
189#endif /* atomic_fetch_add_relaxed */
190
191/* atomic_fetch_inc_relaxed */
192#ifndef atomic_fetch_inc_relaxed
193
194#ifndef atomic_fetch_inc
195#define atomic_fetch_inc(v) atomic_fetch_add(1, (v))
196#define atomic_fetch_inc_relaxed(v) atomic_fetch_add_relaxed(1, (v))
197#define atomic_fetch_inc_acquire(v) atomic_fetch_add_acquire(1, (v))
198#define atomic_fetch_inc_release(v) atomic_fetch_add_release(1, (v))
199#else /* atomic_fetch_inc */
200#define atomic_fetch_inc_relaxed atomic_fetch_inc
201#define atomic_fetch_inc_acquire atomic_fetch_inc
202#define atomic_fetch_inc_release atomic_fetch_inc
203#endif /* atomic_fetch_inc */
204
205#else /* atomic_fetch_inc_relaxed */
206
207#ifndef atomic_fetch_inc_acquire
208#define atomic_fetch_inc_acquire(...) \
209 __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
210#endif
211
212#ifndef atomic_fetch_inc_release
213#define atomic_fetch_inc_release(...) \
214 __atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
215#endif
216
217#ifndef atomic_fetch_inc
218#define atomic_fetch_inc(...) \
219 __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
220#endif
221#endif /* atomic_fetch_inc_relaxed */
222
223/* atomic_fetch_sub_relaxed */
224#ifndef atomic_fetch_sub_relaxed
225#define atomic_fetch_sub_relaxed atomic_fetch_sub
226#define atomic_fetch_sub_acquire atomic_fetch_sub
227#define atomic_fetch_sub_release atomic_fetch_sub
228
229#else /* atomic_fetch_sub_relaxed */
230
231#ifndef atomic_fetch_sub_acquire
232#define atomic_fetch_sub_acquire(...) \
233 __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
234#endif
235
236#ifndef atomic_fetch_sub_release
237#define atomic_fetch_sub_release(...) \
238 __atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
239#endif
240
241#ifndef atomic_fetch_sub
242#define atomic_fetch_sub(...) \
243 __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
244#endif
245#endif /* atomic_fetch_sub_relaxed */
246
247/* atomic_fetch_dec_relaxed */
248#ifndef atomic_fetch_dec_relaxed
249
250#ifndef atomic_fetch_dec
251#define atomic_fetch_dec(v) atomic_fetch_sub(1, (v))
252#define atomic_fetch_dec_relaxed(v) atomic_fetch_sub_relaxed(1, (v))
253#define atomic_fetch_dec_acquire(v) atomic_fetch_sub_acquire(1, (v))
254#define atomic_fetch_dec_release(v) atomic_fetch_sub_release(1, (v))
255#else /* atomic_fetch_dec */
256#define atomic_fetch_dec_relaxed atomic_fetch_dec
257#define atomic_fetch_dec_acquire atomic_fetch_dec
258#define atomic_fetch_dec_release atomic_fetch_dec
259#endif /* atomic_fetch_dec */
260
261#else /* atomic_fetch_dec_relaxed */
262
263#ifndef atomic_fetch_dec_acquire
264#define atomic_fetch_dec_acquire(...) \
265 __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
266#endif
267
268#ifndef atomic_fetch_dec_release
269#define atomic_fetch_dec_release(...) \
270 __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
271#endif
272
273#ifndef atomic_fetch_dec
274#define atomic_fetch_dec(...) \
275 __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
276#endif
277#endif /* atomic_fetch_dec_relaxed */
278
279/* atomic_fetch_or_relaxed */
280#ifndef atomic_fetch_or_relaxed
281#define atomic_fetch_or_relaxed atomic_fetch_or
282#define atomic_fetch_or_acquire atomic_fetch_or
283#define atomic_fetch_or_release atomic_fetch_or
284
285#else /* atomic_fetch_or_relaxed */
286
287#ifndef atomic_fetch_or_acquire
288#define atomic_fetch_or_acquire(...) \
289 __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
290#endif
291
292#ifndef atomic_fetch_or_release
293#define atomic_fetch_or_release(...) \
294 __atomic_op_release(atomic_fetch_or, __VA_ARGS__)
295#endif
296
297#ifndef atomic_fetch_or
298#define atomic_fetch_or(...) \
299 __atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
300#endif
301#endif /* atomic_fetch_or_relaxed */
302
303/* atomic_fetch_and_relaxed */
304#ifndef atomic_fetch_and_relaxed
305#define atomic_fetch_and_relaxed atomic_fetch_and
306#define atomic_fetch_and_acquire atomic_fetch_and
307#define atomic_fetch_and_release atomic_fetch_and
308
309#else /* atomic_fetch_and_relaxed */
310
311#ifndef atomic_fetch_and_acquire
312#define atomic_fetch_and_acquire(...) \
313 __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
314#endif
315
316#ifndef atomic_fetch_and_release
317#define atomic_fetch_and_release(...) \
318 __atomic_op_release(atomic_fetch_and, __VA_ARGS__)
319#endif
320
321#ifndef atomic_fetch_and
322#define atomic_fetch_and(...) \
323 __atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
324#endif
325#endif /* atomic_fetch_and_relaxed */
326
327#ifdef atomic_andnot
328/* atomic_fetch_andnot_relaxed */
329#ifndef atomic_fetch_andnot_relaxed
330#define atomic_fetch_andnot_relaxed atomic_fetch_andnot
331#define atomic_fetch_andnot_acquire atomic_fetch_andnot
332#define atomic_fetch_andnot_release atomic_fetch_andnot
333
334#else /* atomic_fetch_andnot_relaxed */
335
336#ifndef atomic_fetch_andnot_acquire
337#define atomic_fetch_andnot_acquire(...) \
338 __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
339#endif
340
341#ifndef atomic_fetch_andnot_release
342#define atomic_fetch_andnot_release(...) \
343 __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
344#endif
345
346#ifndef atomic_fetch_andnot
347#define atomic_fetch_andnot(...) \
348 __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
349#endif
350#endif /* atomic_fetch_andnot_relaxed */
351#endif /* atomic_andnot */
352
353/* atomic_fetch_xor_relaxed */
354#ifndef atomic_fetch_xor_relaxed
355#define atomic_fetch_xor_relaxed atomic_fetch_xor
356#define atomic_fetch_xor_acquire atomic_fetch_xor
357#define atomic_fetch_xor_release atomic_fetch_xor
358
359#else /* atomic_fetch_xor_relaxed */
360
361#ifndef atomic_fetch_xor_acquire
362#define atomic_fetch_xor_acquire(...) \
363 __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
364#endif
365
366#ifndef atomic_fetch_xor_release
367#define atomic_fetch_xor_release(...) \
368 __atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
369#endif
370
371#ifndef atomic_fetch_xor
372#define atomic_fetch_xor(...) \
373 __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
374#endif
375#endif /* atomic_fetch_xor_relaxed */
376
377
378/* atomic_xchg_relaxed */
379#ifndef atomic_xchg_relaxed
380#define atomic_xchg_relaxed atomic_xchg
381#define atomic_xchg_acquire atomic_xchg
382#define atomic_xchg_release atomic_xchg
383
384#else /* atomic_xchg_relaxed */
385
386#ifndef atomic_xchg_acquire
387#define atomic_xchg_acquire(...) \
388 __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
389#endif
390
391#ifndef atomic_xchg_release
392#define atomic_xchg_release(...) \
393 __atomic_op_release(atomic_xchg, __VA_ARGS__)
394#endif
395
396#ifndef atomic_xchg
397#define atomic_xchg(...) \
398 __atomic_op_fence(atomic_xchg, __VA_ARGS__)
399#endif
400#endif /* atomic_xchg_relaxed */
401
402/* atomic_cmpxchg_relaxed */
403#ifndef atomic_cmpxchg_relaxed
404#define atomic_cmpxchg_relaxed atomic_cmpxchg
405#define atomic_cmpxchg_acquire atomic_cmpxchg
406#define atomic_cmpxchg_release atomic_cmpxchg
407
408#else /* atomic_cmpxchg_relaxed */
409
410#ifndef atomic_cmpxchg_acquire
411#define atomic_cmpxchg_acquire(...) \
412 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
413#endif
414
415#ifndef atomic_cmpxchg_release
416#define atomic_cmpxchg_release(...) \
417 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
418#endif
419
420#ifndef atomic_cmpxchg
421#define atomic_cmpxchg(...) \
422 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
423#endif
424#endif /* atomic_cmpxchg_relaxed */
425
426#ifndef atomic_try_cmpxchg
427
428#define __atomic_try_cmpxchg(type, _p, _po, _n) \
429({ \
430 typeof(_po) __po = (_po); \
431 typeof(*(_po)) __r, __o = *__po; \
432 __r = atomic_cmpxchg##type((_p), __o, (_n)); \
433 if (unlikely(__r != __o)) \
434 *__po = __r; \
435 likely(__r == __o); \
436})
437
438#define atomic_try_cmpxchg(_p, _po, _n) __atomic_try_cmpxchg(, _p, _po, _n)
439#define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n)
440#define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n)
441#define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n)
442
443#else /* atomic_try_cmpxchg */
444#define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg
445#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg
446#define atomic_try_cmpxchg_release atomic_try_cmpxchg
447#endif /* atomic_try_cmpxchg */
448
449/* cmpxchg_relaxed */
450#ifndef cmpxchg_relaxed
451#define cmpxchg_relaxed cmpxchg
452#define cmpxchg_acquire cmpxchg
453#define cmpxchg_release cmpxchg
454
455#else /* cmpxchg_relaxed */
456
457#ifndef cmpxchg_acquire
458#define cmpxchg_acquire(...) \
459 __atomic_op_acquire(cmpxchg, __VA_ARGS__)
460#endif
461
462#ifndef cmpxchg_release
463#define cmpxchg_release(...) \
464 __atomic_op_release(cmpxchg, __VA_ARGS__)
465#endif
466
467#ifndef cmpxchg
468#define cmpxchg(...) \
469 __atomic_op_fence(cmpxchg, __VA_ARGS__)
470#endif
471#endif /* cmpxchg_relaxed */
472
473/* cmpxchg64_relaxed */
474#ifndef cmpxchg64_relaxed
475#define cmpxchg64_relaxed cmpxchg64
476#define cmpxchg64_acquire cmpxchg64
477#define cmpxchg64_release cmpxchg64
478
479#else /* cmpxchg64_relaxed */
480
481#ifndef cmpxchg64_acquire
482#define cmpxchg64_acquire(...) \
483 __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
484#endif
485
486#ifndef cmpxchg64_release
487#define cmpxchg64_release(...) \
488 __atomic_op_release(cmpxchg64, __VA_ARGS__)
489#endif
490
491#ifndef cmpxchg64
492#define cmpxchg64(...) \
493 __atomic_op_fence(cmpxchg64, __VA_ARGS__)
494#endif
495#endif /* cmpxchg64_relaxed */
496
497/* xchg_relaxed */
498#ifndef xchg_relaxed
499#define xchg_relaxed xchg
500#define xchg_acquire xchg
501#define xchg_release xchg
502
503#else /* xchg_relaxed */
504
505#ifndef xchg_acquire
506#define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__)
507#endif
508
509#ifndef xchg_release
510#define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__)
511#endif
512
513#ifndef xchg
514#define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__)
515#endif
516#endif /* xchg_relaxed */
517
518/**
519 * atomic_add_unless - add unless the number is already a given value
520 * @v: pointer of type atomic_t
521 * @a: the amount to add to v...
522 * @u: ...unless v is equal to u.
523 *
524 * Atomically adds @a to @v, so long as @v was not already @u.
525 * Returns non-zero if @v was not @u, and zero otherwise.
526 */
527static inline int atomic_add_unless(atomic_t *v, int a, int u)
528{
529 return __atomic_add_unless(v, a, u) != u;
530}
531
532/**
533 * atomic_inc_not_zero - increment unless the number is zero
534 * @v: pointer of type atomic_t
535 *
536 * Atomically increments @v by 1, so long as @v is non-zero.
537 * Returns non-zero if @v was non-zero, and zero otherwise.
538 */
539#ifndef atomic_inc_not_zero
540#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
541#endif
542
543#ifndef atomic_andnot
544static inline void atomic_andnot(int i, atomic_t *v)
545{
546 atomic_and(~i, v);
547}
548
549static inline int atomic_fetch_andnot(int i, atomic_t *v)
550{
551 return atomic_fetch_and(~i, v);
552}
553
554static inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v)
555{
556 return atomic_fetch_and_relaxed(~i, v);
557}
558
559static inline int atomic_fetch_andnot_acquire(int i, atomic_t *v)
560{
561 return atomic_fetch_and_acquire(~i, v);
562}
563
564static inline int atomic_fetch_andnot_release(int i, atomic_t *v)
565{
566 return atomic_fetch_and_release(~i, v);
567}
568#endif
569
570/**
571 * atomic_inc_not_zero_hint - increment if not null
572 * @v: pointer of type atomic_t
573 * @hint: probable value of the atomic before the increment
574 *
575 * This version of atomic_inc_not_zero() gives a hint of probable
576 * value of the atomic. This helps processor to not read the memory
577 * before doing the atomic read/modify/write cycle, lowering
578 * number of bus transactions on some arches.
579 *
580 * Returns: 0 if increment was not done, 1 otherwise.
581 */
582#ifndef atomic_inc_not_zero_hint
583static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
584{
585 int val, c = hint;
586
587 /* sanity test, should be removed by compiler if hint is a constant */
588 if (!hint)
589 return atomic_inc_not_zero(v);
590
591 do {
592 val = atomic_cmpxchg(v, c, c + 1);
593 if (val == c)
594 return 1;
595 c = val;
596 } while (c);
597
598 return 0;
599}
600#endif
601
602#ifndef atomic_inc_unless_negative
603static inline int atomic_inc_unless_negative(atomic_t *p)
604{
605 int v, v1;
606 for (v = 0; v >= 0; v = v1) {
607 v1 = atomic_cmpxchg(p, v, v + 1);
608 if (likely(v1 == v))
609 return 1;
610 }
611 return 0;
612}
613#endif
614
615#ifndef atomic_dec_unless_positive
616static inline int atomic_dec_unless_positive(atomic_t *p)
617{
618 int v, v1;
619 for (v = 0; v <= 0; v = v1) {
620 v1 = atomic_cmpxchg(p, v, v - 1);
621 if (likely(v1 == v))
622 return 1;
623 }
624 return 0;
625}
626#endif
627
628/*
629 * atomic_dec_if_positive - decrement by 1 if old value positive
630 * @v: pointer of type atomic_t
631 *
632 * The function returns the old value of *v minus 1, even if
633 * the atomic variable, v, was not decremented.
634 */
635#ifndef atomic_dec_if_positive
636static inline int atomic_dec_if_positive(atomic_t *v)
637{
638 int c, old, dec;
639 c = atomic_read(v);
640 for (;;) {
641 dec = c - 1;
642 if (unlikely(dec < 0))
643 break;
644 old = atomic_cmpxchg((v), c, dec);
645 if (likely(old == c))
646 break;
647 c = old;
648 }
649 return dec;
650}
651#endif
652
653#ifdef CONFIG_GENERIC_ATOMIC64
654#include <asm-generic/atomic64.h>
655#endif
656
657#ifndef atomic64_read_acquire
658#define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter)
659#endif
660
661#ifndef atomic64_set_release
662#define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i))
663#endif
664
665/* atomic64_add_return_relaxed */
666#ifndef atomic64_add_return_relaxed
667#define atomic64_add_return_relaxed atomic64_add_return
668#define atomic64_add_return_acquire atomic64_add_return
669#define atomic64_add_return_release atomic64_add_return
670
671#else /* atomic64_add_return_relaxed */
672
673#ifndef atomic64_add_return_acquire
674#define atomic64_add_return_acquire(...) \
675 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
676#endif
677
678#ifndef atomic64_add_return_release
679#define atomic64_add_return_release(...) \
680 __atomic_op_release(atomic64_add_return, __VA_ARGS__)
681#endif
682
683#ifndef atomic64_add_return
684#define atomic64_add_return(...) \
685 __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
686#endif
687#endif /* atomic64_add_return_relaxed */
688
689/* atomic64_inc_return_relaxed */
690#ifndef atomic64_inc_return_relaxed
691#define atomic64_inc_return_relaxed atomic64_inc_return
692#define atomic64_inc_return_acquire atomic64_inc_return
693#define atomic64_inc_return_release atomic64_inc_return
694
695#else /* atomic64_inc_return_relaxed */
696
697#ifndef atomic64_inc_return_acquire
698#define atomic64_inc_return_acquire(...) \
699 __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
700#endif
701
702#ifndef atomic64_inc_return_release
703#define atomic64_inc_return_release(...) \
704 __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
705#endif
706
707#ifndef atomic64_inc_return
708#define atomic64_inc_return(...) \
709 __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
710#endif
711#endif /* atomic64_inc_return_relaxed */
712
713
714/* atomic64_sub_return_relaxed */
715#ifndef atomic64_sub_return_relaxed
716#define atomic64_sub_return_relaxed atomic64_sub_return
717#define atomic64_sub_return_acquire atomic64_sub_return
718#define atomic64_sub_return_release atomic64_sub_return
719
720#else /* atomic64_sub_return_relaxed */
721
722#ifndef atomic64_sub_return_acquire
723#define atomic64_sub_return_acquire(...) \
724 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
725#endif
726
727#ifndef atomic64_sub_return_release
728#define atomic64_sub_return_release(...) \
729 __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
730#endif
731
732#ifndef atomic64_sub_return
733#define atomic64_sub_return(...) \
734 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
735#endif
736#endif /* atomic64_sub_return_relaxed */
737
738/* atomic64_dec_return_relaxed */
739#ifndef atomic64_dec_return_relaxed
740#define atomic64_dec_return_relaxed atomic64_dec_return
741#define atomic64_dec_return_acquire atomic64_dec_return
742#define atomic64_dec_return_release atomic64_dec_return
743
744#else /* atomic64_dec_return_relaxed */
745
746#ifndef atomic64_dec_return_acquire
747#define atomic64_dec_return_acquire(...) \
748 __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
749#endif
750
751#ifndef atomic64_dec_return_release
752#define atomic64_dec_return_release(...) \
753 __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
754#endif
755
756#ifndef atomic64_dec_return
757#define atomic64_dec_return(...) \
758 __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
759#endif
760#endif /* atomic64_dec_return_relaxed */
761
762
763/* atomic64_fetch_add_relaxed */
764#ifndef atomic64_fetch_add_relaxed
765#define atomic64_fetch_add_relaxed atomic64_fetch_add
766#define atomic64_fetch_add_acquire atomic64_fetch_add
767#define atomic64_fetch_add_release atomic64_fetch_add
768
769#else /* atomic64_fetch_add_relaxed */
770
771#ifndef atomic64_fetch_add_acquire
772#define atomic64_fetch_add_acquire(...) \
773 __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
774#endif
775
776#ifndef atomic64_fetch_add_release
777#define atomic64_fetch_add_release(...) \
778 __atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
779#endif
780
781#ifndef atomic64_fetch_add
782#define atomic64_fetch_add(...) \
783 __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
784#endif
785#endif /* atomic64_fetch_add_relaxed */
786
787/* atomic64_fetch_inc_relaxed */
788#ifndef atomic64_fetch_inc_relaxed
789
790#ifndef atomic64_fetch_inc
791#define atomic64_fetch_inc(v) atomic64_fetch_add(1, (v))
792#define atomic64_fetch_inc_relaxed(v) atomic64_fetch_add_relaxed(1, (v))
793#define atomic64_fetch_inc_acquire(v) atomic64_fetch_add_acquire(1, (v))
794#define atomic64_fetch_inc_release(v) atomic64_fetch_add_release(1, (v))
795#else /* atomic64_fetch_inc */
796#define atomic64_fetch_inc_relaxed atomic64_fetch_inc
797#define atomic64_fetch_inc_acquire atomic64_fetch_inc
798#define atomic64_fetch_inc_release atomic64_fetch_inc
799#endif /* atomic64_fetch_inc */
800
801#else /* atomic64_fetch_inc_relaxed */
802
803#ifndef atomic64_fetch_inc_acquire
804#define atomic64_fetch_inc_acquire(...) \
805 __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
806#endif
807
808#ifndef atomic64_fetch_inc_release
809#define atomic64_fetch_inc_release(...) \
810 __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
811#endif
812
813#ifndef atomic64_fetch_inc
814#define atomic64_fetch_inc(...) \
815 __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
816#endif
817#endif /* atomic64_fetch_inc_relaxed */
818
819/* atomic64_fetch_sub_relaxed */
820#ifndef atomic64_fetch_sub_relaxed
821#define atomic64_fetch_sub_relaxed atomic64_fetch_sub
822#define atomic64_fetch_sub_acquire atomic64_fetch_sub
823#define atomic64_fetch_sub_release atomic64_fetch_sub
824
825#else /* atomic64_fetch_sub_relaxed */
826
827#ifndef atomic64_fetch_sub_acquire
828#define atomic64_fetch_sub_acquire(...) \
829 __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
830#endif
831
832#ifndef atomic64_fetch_sub_release
833#define atomic64_fetch_sub_release(...) \
834 __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
835#endif
836
837#ifndef atomic64_fetch_sub
838#define atomic64_fetch_sub(...) \
839 __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
840#endif
841#endif /* atomic64_fetch_sub_relaxed */
842
843/* atomic64_fetch_dec_relaxed */
844#ifndef atomic64_fetch_dec_relaxed
845
846#ifndef atomic64_fetch_dec
847#define atomic64_fetch_dec(v) atomic64_fetch_sub(1, (v))
848#define atomic64_fetch_dec_relaxed(v) atomic64_fetch_sub_relaxed(1, (v))
849#define atomic64_fetch_dec_acquire(v) atomic64_fetch_sub_acquire(1, (v))
850#define atomic64_fetch_dec_release(v) atomic64_fetch_sub_release(1, (v))
851#else /* atomic64_fetch_dec */
852#define atomic64_fetch_dec_relaxed atomic64_fetch_dec
853#define atomic64_fetch_dec_acquire atomic64_fetch_dec
854#define atomic64_fetch_dec_release atomic64_fetch_dec
855#endif /* atomic64_fetch_dec */
856
857#else /* atomic64_fetch_dec_relaxed */
858
859#ifndef atomic64_fetch_dec_acquire
860#define atomic64_fetch_dec_acquire(...) \
861 __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
862#endif
863
864#ifndef atomic64_fetch_dec_release
865#define atomic64_fetch_dec_release(...) \
866 __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
867#endif
868
869#ifndef atomic64_fetch_dec
870#define atomic64_fetch_dec(...) \
871 __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
872#endif
873#endif /* atomic64_fetch_dec_relaxed */
874
875/* atomic64_fetch_or_relaxed */
876#ifndef atomic64_fetch_or_relaxed
877#define atomic64_fetch_or_relaxed atomic64_fetch_or
878#define atomic64_fetch_or_acquire atomic64_fetch_or
879#define atomic64_fetch_or_release atomic64_fetch_or
880
881#else /* atomic64_fetch_or_relaxed */
882
883#ifndef atomic64_fetch_or_acquire
884#define atomic64_fetch_or_acquire(...) \
885 __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
886#endif
887
888#ifndef atomic64_fetch_or_release
889#define atomic64_fetch_or_release(...) \
890 __atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
891#endif
892
893#ifndef atomic64_fetch_or
894#define atomic64_fetch_or(...) \
895 __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
896#endif
897#endif /* atomic64_fetch_or_relaxed */
898
899/* atomic64_fetch_and_relaxed */
900#ifndef atomic64_fetch_and_relaxed
901#define atomic64_fetch_and_relaxed atomic64_fetch_and
902#define atomic64_fetch_and_acquire atomic64_fetch_and
903#define atomic64_fetch_and_release atomic64_fetch_and
904
905#else /* atomic64_fetch_and_relaxed */
906
907#ifndef atomic64_fetch_and_acquire
908#define atomic64_fetch_and_acquire(...) \
909 __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
910#endif
911
912#ifndef atomic64_fetch_and_release
913#define atomic64_fetch_and_release(...) \
914 __atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
915#endif
916
917#ifndef atomic64_fetch_and
918#define atomic64_fetch_and(...) \
919 __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
920#endif
921#endif /* atomic64_fetch_and_relaxed */
922
923#ifdef atomic64_andnot
924/* atomic64_fetch_andnot_relaxed */
925#ifndef atomic64_fetch_andnot_relaxed
926#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot
927#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot
928#define atomic64_fetch_andnot_release atomic64_fetch_andnot
929
930#else /* atomic64_fetch_andnot_relaxed */
931
932#ifndef atomic64_fetch_andnot_acquire
933#define atomic64_fetch_andnot_acquire(...) \
934 __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
935#endif
936
937#ifndef atomic64_fetch_andnot_release
938#define atomic64_fetch_andnot_release(...) \
939 __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
940#endif
941
942#ifndef atomic64_fetch_andnot
943#define atomic64_fetch_andnot(...) \
944 __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
945#endif
946#endif /* atomic64_fetch_andnot_relaxed */
947#endif /* atomic64_andnot */
948
949/* atomic64_fetch_xor_relaxed */
950#ifndef atomic64_fetch_xor_relaxed
951#define atomic64_fetch_xor_relaxed atomic64_fetch_xor
952#define atomic64_fetch_xor_acquire atomic64_fetch_xor
953#define atomic64_fetch_xor_release atomic64_fetch_xor
954
955#else /* atomic64_fetch_xor_relaxed */
956
957#ifndef atomic64_fetch_xor_acquire
958#define atomic64_fetch_xor_acquire(...) \
959 __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
960#endif
961
962#ifndef atomic64_fetch_xor_release
963#define atomic64_fetch_xor_release(...) \
964 __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
965#endif
966
967#ifndef atomic64_fetch_xor
968#define atomic64_fetch_xor(...) \
969 __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
970#endif
971#endif /* atomic64_fetch_xor_relaxed */
972
973
974/* atomic64_xchg_relaxed */
975#ifndef atomic64_xchg_relaxed
976#define atomic64_xchg_relaxed atomic64_xchg
977#define atomic64_xchg_acquire atomic64_xchg
978#define atomic64_xchg_release atomic64_xchg
979
980#else /* atomic64_xchg_relaxed */
981
982#ifndef atomic64_xchg_acquire
983#define atomic64_xchg_acquire(...) \
984 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
985#endif
986
987#ifndef atomic64_xchg_release
988#define atomic64_xchg_release(...) \
989 __atomic_op_release(atomic64_xchg, __VA_ARGS__)
990#endif
991
992#ifndef atomic64_xchg
993#define atomic64_xchg(...) \
994 __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
995#endif
996#endif /* atomic64_xchg_relaxed */
997
998/* atomic64_cmpxchg_relaxed */
999#ifndef atomic64_cmpxchg_relaxed
1000#define atomic64_cmpxchg_relaxed atomic64_cmpxchg
1001#define atomic64_cmpxchg_acquire atomic64_cmpxchg
1002#define atomic64_cmpxchg_release atomic64_cmpxchg
1003
1004#else /* atomic64_cmpxchg_relaxed */
1005
1006#ifndef atomic64_cmpxchg_acquire
1007#define atomic64_cmpxchg_acquire(...) \
1008 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
1009#endif
1010
1011#ifndef atomic64_cmpxchg_release
1012#define atomic64_cmpxchg_release(...) \
1013 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
1014#endif
1015
1016#ifndef atomic64_cmpxchg
1017#define atomic64_cmpxchg(...) \
1018 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
1019#endif
1020#endif /* atomic64_cmpxchg_relaxed */
1021
1022#ifndef atomic64_try_cmpxchg
1023
1024#define __atomic64_try_cmpxchg(type, _p, _po, _n) \
1025({ \
1026 typeof(_po) __po = (_po); \
1027 typeof(*(_po)) __r, __o = *__po; \
1028 __r = atomic64_cmpxchg##type((_p), __o, (_n)); \
1029 if (unlikely(__r != __o)) \
1030 *__po = __r; \
1031 likely(__r == __o); \
1032})
1033
1034#define atomic64_try_cmpxchg(_p, _po, _n) __atomic64_try_cmpxchg(, _p, _po, _n)
1035#define atomic64_try_cmpxchg_relaxed(_p, _po, _n) __atomic64_try_cmpxchg(_relaxed, _p, _po, _n)
1036#define atomic64_try_cmpxchg_acquire(_p, _po, _n) __atomic64_try_cmpxchg(_acquire, _p, _po, _n)
1037#define atomic64_try_cmpxchg_release(_p, _po, _n) __atomic64_try_cmpxchg(_release, _p, _po, _n)
1038
1039#else /* atomic64_try_cmpxchg */
1040#define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg
1041#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg
1042#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg
1043#endif /* atomic64_try_cmpxchg */
1044
1045#ifndef atomic64_andnot
1046static inline void atomic64_andnot(long long i, atomic64_t *v)
1047{
1048 atomic64_and(~i, v);
1049}
1050
1051static inline long long atomic64_fetch_andnot(long long i, atomic64_t *v)
1052{
1053 return atomic64_fetch_and(~i, v);
1054}
1055
1056static inline long long atomic64_fetch_andnot_relaxed(long long i, atomic64_t *v)
1057{
1058 return atomic64_fetch_and_relaxed(~i, v);
1059}
1060
1061static inline long long atomic64_fetch_andnot_acquire(long long i, atomic64_t *v)
1062{
1063 return atomic64_fetch_and_acquire(~i, v);
1064}
1065
1066static inline long long atomic64_fetch_andnot_release(long long i, atomic64_t *v)
1067{
1068 return atomic64_fetch_and_release(~i, v);
1069}
1070#endif
1071
1072#include <asm-generic/atomic-long.h>
1073
1074#endif /* _LINUX_ATOMIC_H */