Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/* Atomic operations usable in machine independent code */
3#ifndef _LINUX_ATOMIC_H
4#define _LINUX_ATOMIC_H
5#include <linux/types.h>
6
7#include <asm/atomic.h>
8#include <asm/barrier.h>
9
10/*
11 * Relaxed variants of xchg, cmpxchg and some atomic operations.
12 *
13 * We support four variants:
14 *
15 * - Fully ordered: The default implementation, no suffix required.
16 * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
17 * - Release: Provides RELEASE semantics, _release suffix.
18 * - Relaxed: No ordering guarantees, _relaxed suffix.
19 *
20 * For compound atomics performing both a load and a store, ACQUIRE
21 * semantics apply only to the load and RELEASE semantics only to the
22 * store portion of the operation. Note that a failed cmpxchg_acquire
23 * does -not- imply any memory ordering constraints.
24 *
25 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
26 */
27
28#ifndef atomic_read_acquire
29#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
30#endif
31
32#ifndef atomic_set_release
33#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
34#endif
35
36/*
37 * The idea here is to build acquire/release variants by adding explicit
38 * barriers on top of the relaxed variant. In the case where the relaxed
39 * variant is already fully ordered, no additional barriers are needed.
40 *
41 * If an architecture overrides __atomic_acquire_fence() it will probably
42 * want to define smp_mb__after_spinlock().
43 */
44#ifndef __atomic_acquire_fence
45#define __atomic_acquire_fence smp_mb__after_atomic
46#endif
47
48#ifndef __atomic_release_fence
49#define __atomic_release_fence smp_mb__before_atomic
50#endif
51
52#ifndef __atomic_pre_full_fence
53#define __atomic_pre_full_fence smp_mb__before_atomic
54#endif
55
56#ifndef __atomic_post_full_fence
57#define __atomic_post_full_fence smp_mb__after_atomic
58#endif
59
60#define __atomic_op_acquire(op, args...) \
61({ \
62 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
63 __atomic_acquire_fence(); \
64 __ret; \
65})
66
67#define __atomic_op_release(op, args...) \
68({ \
69 __atomic_release_fence(); \
70 op##_relaxed(args); \
71})
72
73#define __atomic_op_fence(op, args...) \
74({ \
75 typeof(op##_relaxed(args)) __ret; \
76 __atomic_pre_full_fence(); \
77 __ret = op##_relaxed(args); \
78 __atomic_post_full_fence(); \
79 __ret; \
80})
81
82/* atomic_add_return_relaxed */
83#ifndef atomic_add_return_relaxed
84#define atomic_add_return_relaxed atomic_add_return
85#define atomic_add_return_acquire atomic_add_return
86#define atomic_add_return_release atomic_add_return
87
88#else /* atomic_add_return_relaxed */
89
90#ifndef atomic_add_return_acquire
91#define atomic_add_return_acquire(...) \
92 __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
93#endif
94
95#ifndef atomic_add_return_release
96#define atomic_add_return_release(...) \
97 __atomic_op_release(atomic_add_return, __VA_ARGS__)
98#endif
99
100#ifndef atomic_add_return
101#define atomic_add_return(...) \
102 __atomic_op_fence(atomic_add_return, __VA_ARGS__)
103#endif
104#endif /* atomic_add_return_relaxed */
105
106#ifndef atomic_inc
107#define atomic_inc(v) atomic_add(1, (v))
108#endif
109
110/* atomic_inc_return_relaxed */
111#ifndef atomic_inc_return_relaxed
112
113#ifndef atomic_inc_return
114#define atomic_inc_return(v) atomic_add_return(1, (v))
115#define atomic_inc_return_relaxed(v) atomic_add_return_relaxed(1, (v))
116#define atomic_inc_return_acquire(v) atomic_add_return_acquire(1, (v))
117#define atomic_inc_return_release(v) atomic_add_return_release(1, (v))
118#else /* atomic_inc_return */
119#define atomic_inc_return_relaxed atomic_inc_return
120#define atomic_inc_return_acquire atomic_inc_return
121#define atomic_inc_return_release atomic_inc_return
122#endif /* atomic_inc_return */
123
124#else /* atomic_inc_return_relaxed */
125
126#ifndef atomic_inc_return_acquire
127#define atomic_inc_return_acquire(...) \
128 __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
129#endif
130
131#ifndef atomic_inc_return_release
132#define atomic_inc_return_release(...) \
133 __atomic_op_release(atomic_inc_return, __VA_ARGS__)
134#endif
135
136#ifndef atomic_inc_return
137#define atomic_inc_return(...) \
138 __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
139#endif
140#endif /* atomic_inc_return_relaxed */
141
142/* atomic_sub_return_relaxed */
143#ifndef atomic_sub_return_relaxed
144#define atomic_sub_return_relaxed atomic_sub_return
145#define atomic_sub_return_acquire atomic_sub_return
146#define atomic_sub_return_release atomic_sub_return
147
148#else /* atomic_sub_return_relaxed */
149
150#ifndef atomic_sub_return_acquire
151#define atomic_sub_return_acquire(...) \
152 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
153#endif
154
155#ifndef atomic_sub_return_release
156#define atomic_sub_return_release(...) \
157 __atomic_op_release(atomic_sub_return, __VA_ARGS__)
158#endif
159
160#ifndef atomic_sub_return
161#define atomic_sub_return(...) \
162 __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
163#endif
164#endif /* atomic_sub_return_relaxed */
165
166#ifndef atomic_dec
167#define atomic_dec(v) atomic_sub(1, (v))
168#endif
169
170/* atomic_dec_return_relaxed */
171#ifndef atomic_dec_return_relaxed
172
173#ifndef atomic_dec_return
174#define atomic_dec_return(v) atomic_sub_return(1, (v))
175#define atomic_dec_return_relaxed(v) atomic_sub_return_relaxed(1, (v))
176#define atomic_dec_return_acquire(v) atomic_sub_return_acquire(1, (v))
177#define atomic_dec_return_release(v) atomic_sub_return_release(1, (v))
178#else /* atomic_dec_return */
179#define atomic_dec_return_relaxed atomic_dec_return
180#define atomic_dec_return_acquire atomic_dec_return
181#define atomic_dec_return_release atomic_dec_return
182#endif /* atomic_dec_return */
183
184#else /* atomic_dec_return_relaxed */
185
186#ifndef atomic_dec_return_acquire
187#define atomic_dec_return_acquire(...) \
188 __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
189#endif
190
191#ifndef atomic_dec_return_release
192#define atomic_dec_return_release(...) \
193 __atomic_op_release(atomic_dec_return, __VA_ARGS__)
194#endif
195
196#ifndef atomic_dec_return
197#define atomic_dec_return(...) \
198 __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
199#endif
200#endif /* atomic_dec_return_relaxed */
201
202
203/* atomic_fetch_add_relaxed */
204#ifndef atomic_fetch_add_relaxed
205#define atomic_fetch_add_relaxed atomic_fetch_add
206#define atomic_fetch_add_acquire atomic_fetch_add
207#define atomic_fetch_add_release atomic_fetch_add
208
209#else /* atomic_fetch_add_relaxed */
210
211#ifndef atomic_fetch_add_acquire
212#define atomic_fetch_add_acquire(...) \
213 __atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
214#endif
215
216#ifndef atomic_fetch_add_release
217#define atomic_fetch_add_release(...) \
218 __atomic_op_release(atomic_fetch_add, __VA_ARGS__)
219#endif
220
221#ifndef atomic_fetch_add
222#define atomic_fetch_add(...) \
223 __atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
224#endif
225#endif /* atomic_fetch_add_relaxed */
226
227/* atomic_fetch_inc_relaxed */
228#ifndef atomic_fetch_inc_relaxed
229
230#ifndef atomic_fetch_inc
231#define atomic_fetch_inc(v) atomic_fetch_add(1, (v))
232#define atomic_fetch_inc_relaxed(v) atomic_fetch_add_relaxed(1, (v))
233#define atomic_fetch_inc_acquire(v) atomic_fetch_add_acquire(1, (v))
234#define atomic_fetch_inc_release(v) atomic_fetch_add_release(1, (v))
235#else /* atomic_fetch_inc */
236#define atomic_fetch_inc_relaxed atomic_fetch_inc
237#define atomic_fetch_inc_acquire atomic_fetch_inc
238#define atomic_fetch_inc_release atomic_fetch_inc
239#endif /* atomic_fetch_inc */
240
241#else /* atomic_fetch_inc_relaxed */
242
243#ifndef atomic_fetch_inc_acquire
244#define atomic_fetch_inc_acquire(...) \
245 __atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
246#endif
247
248#ifndef atomic_fetch_inc_release
249#define atomic_fetch_inc_release(...) \
250 __atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
251#endif
252
253#ifndef atomic_fetch_inc
254#define atomic_fetch_inc(...) \
255 __atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
256#endif
257#endif /* atomic_fetch_inc_relaxed */
258
259/* atomic_fetch_sub_relaxed */
260#ifndef atomic_fetch_sub_relaxed
261#define atomic_fetch_sub_relaxed atomic_fetch_sub
262#define atomic_fetch_sub_acquire atomic_fetch_sub
263#define atomic_fetch_sub_release atomic_fetch_sub
264
265#else /* atomic_fetch_sub_relaxed */
266
267#ifndef atomic_fetch_sub_acquire
268#define atomic_fetch_sub_acquire(...) \
269 __atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
270#endif
271
272#ifndef atomic_fetch_sub_release
273#define atomic_fetch_sub_release(...) \
274 __atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
275#endif
276
277#ifndef atomic_fetch_sub
278#define atomic_fetch_sub(...) \
279 __atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
280#endif
281#endif /* atomic_fetch_sub_relaxed */
282
283/* atomic_fetch_dec_relaxed */
284#ifndef atomic_fetch_dec_relaxed
285
286#ifndef atomic_fetch_dec
287#define atomic_fetch_dec(v) atomic_fetch_sub(1, (v))
288#define atomic_fetch_dec_relaxed(v) atomic_fetch_sub_relaxed(1, (v))
289#define atomic_fetch_dec_acquire(v) atomic_fetch_sub_acquire(1, (v))
290#define atomic_fetch_dec_release(v) atomic_fetch_sub_release(1, (v))
291#else /* atomic_fetch_dec */
292#define atomic_fetch_dec_relaxed atomic_fetch_dec
293#define atomic_fetch_dec_acquire atomic_fetch_dec
294#define atomic_fetch_dec_release atomic_fetch_dec
295#endif /* atomic_fetch_dec */
296
297#else /* atomic_fetch_dec_relaxed */
298
299#ifndef atomic_fetch_dec_acquire
300#define atomic_fetch_dec_acquire(...) \
301 __atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
302#endif
303
304#ifndef atomic_fetch_dec_release
305#define atomic_fetch_dec_release(...) \
306 __atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
307#endif
308
309#ifndef atomic_fetch_dec
310#define atomic_fetch_dec(...) \
311 __atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
312#endif
313#endif /* atomic_fetch_dec_relaxed */
314
315/* atomic_fetch_or_relaxed */
316#ifndef atomic_fetch_or_relaxed
317#define atomic_fetch_or_relaxed atomic_fetch_or
318#define atomic_fetch_or_acquire atomic_fetch_or
319#define atomic_fetch_or_release atomic_fetch_or
320
321#else /* atomic_fetch_or_relaxed */
322
323#ifndef atomic_fetch_or_acquire
324#define atomic_fetch_or_acquire(...) \
325 __atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
326#endif
327
328#ifndef atomic_fetch_or_release
329#define atomic_fetch_or_release(...) \
330 __atomic_op_release(atomic_fetch_or, __VA_ARGS__)
331#endif
332
333#ifndef atomic_fetch_or
334#define atomic_fetch_or(...) \
335 __atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
336#endif
337#endif /* atomic_fetch_or_relaxed */
338
339/* atomic_fetch_and_relaxed */
340#ifndef atomic_fetch_and_relaxed
341#define atomic_fetch_and_relaxed atomic_fetch_and
342#define atomic_fetch_and_acquire atomic_fetch_and
343#define atomic_fetch_and_release atomic_fetch_and
344
345#else /* atomic_fetch_and_relaxed */
346
347#ifndef atomic_fetch_and_acquire
348#define atomic_fetch_and_acquire(...) \
349 __atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
350#endif
351
352#ifndef atomic_fetch_and_release
353#define atomic_fetch_and_release(...) \
354 __atomic_op_release(atomic_fetch_and, __VA_ARGS__)
355#endif
356
357#ifndef atomic_fetch_and
358#define atomic_fetch_and(...) \
359 __atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
360#endif
361#endif /* atomic_fetch_and_relaxed */
362
363#ifndef atomic_andnot
364#define atomic_andnot(i, v) atomic_and(~(int)(i), (v))
365#endif
366
367#ifndef atomic_fetch_andnot_relaxed
368
369#ifndef atomic_fetch_andnot
370#define atomic_fetch_andnot(i, v) atomic_fetch_and(~(int)(i), (v))
371#define atomic_fetch_andnot_relaxed(i, v) atomic_fetch_and_relaxed(~(int)(i), (v))
372#define atomic_fetch_andnot_acquire(i, v) atomic_fetch_and_acquire(~(int)(i), (v))
373#define atomic_fetch_andnot_release(i, v) atomic_fetch_and_release(~(int)(i), (v))
374#else /* atomic_fetch_andnot */
375#define atomic_fetch_andnot_relaxed atomic_fetch_andnot
376#define atomic_fetch_andnot_acquire atomic_fetch_andnot
377#define atomic_fetch_andnot_release atomic_fetch_andnot
378#endif /* atomic_fetch_andnot */
379
380#else /* atomic_fetch_andnot_relaxed */
381
382#ifndef atomic_fetch_andnot_acquire
383#define atomic_fetch_andnot_acquire(...) \
384 __atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
385#endif
386
387#ifndef atomic_fetch_andnot_release
388#define atomic_fetch_andnot_release(...) \
389 __atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
390#endif
391
392#ifndef atomic_fetch_andnot
393#define atomic_fetch_andnot(...) \
394 __atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
395#endif
396#endif /* atomic_fetch_andnot_relaxed */
397
398/* atomic_fetch_xor_relaxed */
399#ifndef atomic_fetch_xor_relaxed
400#define atomic_fetch_xor_relaxed atomic_fetch_xor
401#define atomic_fetch_xor_acquire atomic_fetch_xor
402#define atomic_fetch_xor_release atomic_fetch_xor
403
404#else /* atomic_fetch_xor_relaxed */
405
406#ifndef atomic_fetch_xor_acquire
407#define atomic_fetch_xor_acquire(...) \
408 __atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
409#endif
410
411#ifndef atomic_fetch_xor_release
412#define atomic_fetch_xor_release(...) \
413 __atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
414#endif
415
416#ifndef atomic_fetch_xor
417#define atomic_fetch_xor(...) \
418 __atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
419#endif
420#endif /* atomic_fetch_xor_relaxed */
421
422
423/* atomic_xchg_relaxed */
424#ifndef atomic_xchg_relaxed
425#define atomic_xchg_relaxed atomic_xchg
426#define atomic_xchg_acquire atomic_xchg
427#define atomic_xchg_release atomic_xchg
428
429#else /* atomic_xchg_relaxed */
430
431#ifndef atomic_xchg_acquire
432#define atomic_xchg_acquire(...) \
433 __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
434#endif
435
436#ifndef atomic_xchg_release
437#define atomic_xchg_release(...) \
438 __atomic_op_release(atomic_xchg, __VA_ARGS__)
439#endif
440
441#ifndef atomic_xchg
442#define atomic_xchg(...) \
443 __atomic_op_fence(atomic_xchg, __VA_ARGS__)
444#endif
445#endif /* atomic_xchg_relaxed */
446
447/* atomic_cmpxchg_relaxed */
448#ifndef atomic_cmpxchg_relaxed
449#define atomic_cmpxchg_relaxed atomic_cmpxchg
450#define atomic_cmpxchg_acquire atomic_cmpxchg
451#define atomic_cmpxchg_release atomic_cmpxchg
452
453#else /* atomic_cmpxchg_relaxed */
454
455#ifndef atomic_cmpxchg_acquire
456#define atomic_cmpxchg_acquire(...) \
457 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
458#endif
459
460#ifndef atomic_cmpxchg_release
461#define atomic_cmpxchg_release(...) \
462 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
463#endif
464
465#ifndef atomic_cmpxchg
466#define atomic_cmpxchg(...) \
467 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
468#endif
469#endif /* atomic_cmpxchg_relaxed */
470
471#ifndef atomic_try_cmpxchg
472
473#define __atomic_try_cmpxchg(type, _p, _po, _n) \
474({ \
475 typeof(_po) __po = (_po); \
476 typeof(*(_po)) __r, __o = *__po; \
477 __r = atomic_cmpxchg##type((_p), __o, (_n)); \
478 if (unlikely(__r != __o)) \
479 *__po = __r; \
480 likely(__r == __o); \
481})
482
483#define atomic_try_cmpxchg(_p, _po, _n) __atomic_try_cmpxchg(, _p, _po, _n)
484#define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n)
485#define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n)
486#define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n)
487
488#else /* atomic_try_cmpxchg */
489#define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg
490#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg
491#define atomic_try_cmpxchg_release atomic_try_cmpxchg
492#endif /* atomic_try_cmpxchg */
493
494/* cmpxchg_relaxed */
495#ifndef cmpxchg_relaxed
496#define cmpxchg_relaxed cmpxchg
497#define cmpxchg_acquire cmpxchg
498#define cmpxchg_release cmpxchg
499
500#else /* cmpxchg_relaxed */
501
502#ifndef cmpxchg_acquire
503#define cmpxchg_acquire(...) \
504 __atomic_op_acquire(cmpxchg, __VA_ARGS__)
505#endif
506
507#ifndef cmpxchg_release
508#define cmpxchg_release(...) \
509 __atomic_op_release(cmpxchg, __VA_ARGS__)
510#endif
511
512#ifndef cmpxchg
513#define cmpxchg(...) \
514 __atomic_op_fence(cmpxchg, __VA_ARGS__)
515#endif
516#endif /* cmpxchg_relaxed */
517
518/* cmpxchg64_relaxed */
519#ifndef cmpxchg64_relaxed
520#define cmpxchg64_relaxed cmpxchg64
521#define cmpxchg64_acquire cmpxchg64
522#define cmpxchg64_release cmpxchg64
523
524#else /* cmpxchg64_relaxed */
525
526#ifndef cmpxchg64_acquire
527#define cmpxchg64_acquire(...) \
528 __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
529#endif
530
531#ifndef cmpxchg64_release
532#define cmpxchg64_release(...) \
533 __atomic_op_release(cmpxchg64, __VA_ARGS__)
534#endif
535
536#ifndef cmpxchg64
537#define cmpxchg64(...) \
538 __atomic_op_fence(cmpxchg64, __VA_ARGS__)
539#endif
540#endif /* cmpxchg64_relaxed */
541
542/* xchg_relaxed */
543#ifndef xchg_relaxed
544#define xchg_relaxed xchg
545#define xchg_acquire xchg
546#define xchg_release xchg
547
548#else /* xchg_relaxed */
549
550#ifndef xchg_acquire
551#define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__)
552#endif
553
554#ifndef xchg_release
555#define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__)
556#endif
557
558#ifndef xchg
559#define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__)
560#endif
561#endif /* xchg_relaxed */
562
563/**
564 * atomic_fetch_add_unless - add unless the number is already a given value
565 * @v: pointer of type atomic_t
566 * @a: the amount to add to v...
567 * @u: ...unless v is equal to u.
568 *
569 * Atomically adds @a to @v, if @v was not already @u.
570 * Returns the original value of @v.
571 */
572#ifndef atomic_fetch_add_unless
573static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)
574{
575 int c = atomic_read(v);
576
577 do {
578 if (unlikely(c == u))
579 break;
580 } while (!atomic_try_cmpxchg(v, &c, c + a));
581
582 return c;
583}
584#endif
585
586/**
587 * atomic_add_unless - add unless the number is already a given value
588 * @v: pointer of type atomic_t
589 * @a: the amount to add to v...
590 * @u: ...unless v is equal to u.
591 *
592 * Atomically adds @a to @v, if @v was not already @u.
593 * Returns true if the addition was done.
594 */
595static inline bool atomic_add_unless(atomic_t *v, int a, int u)
596{
597 return atomic_fetch_add_unless(v, a, u) != u;
598}
599
600/**
601 * atomic_inc_not_zero - increment unless the number is zero
602 * @v: pointer of type atomic_t
603 *
604 * Atomically increments @v by 1, if @v is non-zero.
605 * Returns true if the increment was done.
606 */
607#ifndef atomic_inc_not_zero
608#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
609#endif
610
611/**
612 * atomic_inc_and_test - increment and test
613 * @v: pointer of type atomic_t
614 *
615 * Atomically increments @v by 1
616 * and returns true if the result is zero, or false for all
617 * other cases.
618 */
619#ifndef atomic_inc_and_test
620static inline bool atomic_inc_and_test(atomic_t *v)
621{
622 return atomic_inc_return(v) == 0;
623}
624#endif
625
626/**
627 * atomic_dec_and_test - decrement and test
628 * @v: pointer of type atomic_t
629 *
630 * Atomically decrements @v by 1 and
631 * returns true if the result is 0, or false for all other
632 * cases.
633 */
634#ifndef atomic_dec_and_test
635static inline bool atomic_dec_and_test(atomic_t *v)
636{
637 return atomic_dec_return(v) == 0;
638}
639#endif
640
641/**
642 * atomic_sub_and_test - subtract value from variable and test result
643 * @i: integer value to subtract
644 * @v: pointer of type atomic_t
645 *
646 * Atomically subtracts @i from @v and returns
647 * true if the result is zero, or false for all
648 * other cases.
649 */
650#ifndef atomic_sub_and_test
651static inline bool atomic_sub_and_test(int i, atomic_t *v)
652{
653 return atomic_sub_return(i, v) == 0;
654}
655#endif
656
657/**
658 * atomic_add_negative - add and test if negative
659 * @i: integer value to add
660 * @v: pointer of type atomic_t
661 *
662 * Atomically adds @i to @v and returns true
663 * if the result is negative, or false when
664 * result is greater than or equal to zero.
665 */
666#ifndef atomic_add_negative
667static inline bool atomic_add_negative(int i, atomic_t *v)
668{
669 return atomic_add_return(i, v) < 0;
670}
671#endif
672
673#ifndef atomic_inc_unless_negative
674static inline bool atomic_inc_unless_negative(atomic_t *v)
675{
676 int c = atomic_read(v);
677
678 do {
679 if (unlikely(c < 0))
680 return false;
681 } while (!atomic_try_cmpxchg(v, &c, c + 1));
682
683 return true;
684}
685#endif
686
687#ifndef atomic_dec_unless_positive
688static inline bool atomic_dec_unless_positive(atomic_t *v)
689{
690 int c = atomic_read(v);
691
692 do {
693 if (unlikely(c > 0))
694 return false;
695 } while (!atomic_try_cmpxchg(v, &c, c - 1));
696
697 return true;
698}
699#endif
700
701/*
702 * atomic_dec_if_positive - decrement by 1 if old value positive
703 * @v: pointer of type atomic_t
704 *
705 * The function returns the old value of *v minus 1, even if
706 * the atomic variable, v, was not decremented.
707 */
708#ifndef atomic_dec_if_positive
709static inline int atomic_dec_if_positive(atomic_t *v)
710{
711 int dec, c = atomic_read(v);
712
713 do {
714 dec = c - 1;
715 if (unlikely(dec < 0))
716 break;
717 } while (!atomic_try_cmpxchg(v, &c, dec));
718
719 return dec;
720}
721#endif
722
723#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
724#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
725
726#ifdef CONFIG_GENERIC_ATOMIC64
727#include <asm-generic/atomic64.h>
728#endif
729
730#ifndef atomic64_read_acquire
731#define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter)
732#endif
733
734#ifndef atomic64_set_release
735#define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i))
736#endif
737
738/* atomic64_add_return_relaxed */
739#ifndef atomic64_add_return_relaxed
740#define atomic64_add_return_relaxed atomic64_add_return
741#define atomic64_add_return_acquire atomic64_add_return
742#define atomic64_add_return_release atomic64_add_return
743
744#else /* atomic64_add_return_relaxed */
745
746#ifndef atomic64_add_return_acquire
747#define atomic64_add_return_acquire(...) \
748 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
749#endif
750
751#ifndef atomic64_add_return_release
752#define atomic64_add_return_release(...) \
753 __atomic_op_release(atomic64_add_return, __VA_ARGS__)
754#endif
755
756#ifndef atomic64_add_return
757#define atomic64_add_return(...) \
758 __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
759#endif
760#endif /* atomic64_add_return_relaxed */
761
762#ifndef atomic64_inc
763#define atomic64_inc(v) atomic64_add(1, (v))
764#endif
765
766/* atomic64_inc_return_relaxed */
767#ifndef atomic64_inc_return_relaxed
768
769#ifndef atomic64_inc_return
770#define atomic64_inc_return(v) atomic64_add_return(1, (v))
771#define atomic64_inc_return_relaxed(v) atomic64_add_return_relaxed(1, (v))
772#define atomic64_inc_return_acquire(v) atomic64_add_return_acquire(1, (v))
773#define atomic64_inc_return_release(v) atomic64_add_return_release(1, (v))
774#else /* atomic64_inc_return */
775#define atomic64_inc_return_relaxed atomic64_inc_return
776#define atomic64_inc_return_acquire atomic64_inc_return
777#define atomic64_inc_return_release atomic64_inc_return
778#endif /* atomic64_inc_return */
779
780#else /* atomic64_inc_return_relaxed */
781
782#ifndef atomic64_inc_return_acquire
783#define atomic64_inc_return_acquire(...) \
784 __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
785#endif
786
787#ifndef atomic64_inc_return_release
788#define atomic64_inc_return_release(...) \
789 __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
790#endif
791
792#ifndef atomic64_inc_return
793#define atomic64_inc_return(...) \
794 __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
795#endif
796#endif /* atomic64_inc_return_relaxed */
797
798
799/* atomic64_sub_return_relaxed */
800#ifndef atomic64_sub_return_relaxed
801#define atomic64_sub_return_relaxed atomic64_sub_return
802#define atomic64_sub_return_acquire atomic64_sub_return
803#define atomic64_sub_return_release atomic64_sub_return
804
805#else /* atomic64_sub_return_relaxed */
806
807#ifndef atomic64_sub_return_acquire
808#define atomic64_sub_return_acquire(...) \
809 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
810#endif
811
812#ifndef atomic64_sub_return_release
813#define atomic64_sub_return_release(...) \
814 __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
815#endif
816
817#ifndef atomic64_sub_return
818#define atomic64_sub_return(...) \
819 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
820#endif
821#endif /* atomic64_sub_return_relaxed */
822
823#ifndef atomic64_dec
824#define atomic64_dec(v) atomic64_sub(1, (v))
825#endif
826
827/* atomic64_dec_return_relaxed */
828#ifndef atomic64_dec_return_relaxed
829
830#ifndef atomic64_dec_return
831#define atomic64_dec_return(v) atomic64_sub_return(1, (v))
832#define atomic64_dec_return_relaxed(v) atomic64_sub_return_relaxed(1, (v))
833#define atomic64_dec_return_acquire(v) atomic64_sub_return_acquire(1, (v))
834#define atomic64_dec_return_release(v) atomic64_sub_return_release(1, (v))
835#else /* atomic64_dec_return */
836#define atomic64_dec_return_relaxed atomic64_dec_return
837#define atomic64_dec_return_acquire atomic64_dec_return
838#define atomic64_dec_return_release atomic64_dec_return
839#endif /* atomic64_dec_return */
840
841#else /* atomic64_dec_return_relaxed */
842
843#ifndef atomic64_dec_return_acquire
844#define atomic64_dec_return_acquire(...) \
845 __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
846#endif
847
848#ifndef atomic64_dec_return_release
849#define atomic64_dec_return_release(...) \
850 __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
851#endif
852
853#ifndef atomic64_dec_return
854#define atomic64_dec_return(...) \
855 __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
856#endif
857#endif /* atomic64_dec_return_relaxed */
858
859
860/* atomic64_fetch_add_relaxed */
861#ifndef atomic64_fetch_add_relaxed
862#define atomic64_fetch_add_relaxed atomic64_fetch_add
863#define atomic64_fetch_add_acquire atomic64_fetch_add
864#define atomic64_fetch_add_release atomic64_fetch_add
865
866#else /* atomic64_fetch_add_relaxed */
867
868#ifndef atomic64_fetch_add_acquire
869#define atomic64_fetch_add_acquire(...) \
870 __atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
871#endif
872
873#ifndef atomic64_fetch_add_release
874#define atomic64_fetch_add_release(...) \
875 __atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
876#endif
877
878#ifndef atomic64_fetch_add
879#define atomic64_fetch_add(...) \
880 __atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
881#endif
882#endif /* atomic64_fetch_add_relaxed */
883
884/* atomic64_fetch_inc_relaxed */
885#ifndef atomic64_fetch_inc_relaxed
886
887#ifndef atomic64_fetch_inc
888#define atomic64_fetch_inc(v) atomic64_fetch_add(1, (v))
889#define atomic64_fetch_inc_relaxed(v) atomic64_fetch_add_relaxed(1, (v))
890#define atomic64_fetch_inc_acquire(v) atomic64_fetch_add_acquire(1, (v))
891#define atomic64_fetch_inc_release(v) atomic64_fetch_add_release(1, (v))
892#else /* atomic64_fetch_inc */
893#define atomic64_fetch_inc_relaxed atomic64_fetch_inc
894#define atomic64_fetch_inc_acquire atomic64_fetch_inc
895#define atomic64_fetch_inc_release atomic64_fetch_inc
896#endif /* atomic64_fetch_inc */
897
898#else /* atomic64_fetch_inc_relaxed */
899
900#ifndef atomic64_fetch_inc_acquire
901#define atomic64_fetch_inc_acquire(...) \
902 __atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
903#endif
904
905#ifndef atomic64_fetch_inc_release
906#define atomic64_fetch_inc_release(...) \
907 __atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
908#endif
909
910#ifndef atomic64_fetch_inc
911#define atomic64_fetch_inc(...) \
912 __atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
913#endif
914#endif /* atomic64_fetch_inc_relaxed */
915
916/* atomic64_fetch_sub_relaxed */
917#ifndef atomic64_fetch_sub_relaxed
918#define atomic64_fetch_sub_relaxed atomic64_fetch_sub
919#define atomic64_fetch_sub_acquire atomic64_fetch_sub
920#define atomic64_fetch_sub_release atomic64_fetch_sub
921
922#else /* atomic64_fetch_sub_relaxed */
923
924#ifndef atomic64_fetch_sub_acquire
925#define atomic64_fetch_sub_acquire(...) \
926 __atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
927#endif
928
929#ifndef atomic64_fetch_sub_release
930#define atomic64_fetch_sub_release(...) \
931 __atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
932#endif
933
934#ifndef atomic64_fetch_sub
935#define atomic64_fetch_sub(...) \
936 __atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
937#endif
938#endif /* atomic64_fetch_sub_relaxed */
939
940/* atomic64_fetch_dec_relaxed */
941#ifndef atomic64_fetch_dec_relaxed
942
943#ifndef atomic64_fetch_dec
944#define atomic64_fetch_dec(v) atomic64_fetch_sub(1, (v))
945#define atomic64_fetch_dec_relaxed(v) atomic64_fetch_sub_relaxed(1, (v))
946#define atomic64_fetch_dec_acquire(v) atomic64_fetch_sub_acquire(1, (v))
947#define atomic64_fetch_dec_release(v) atomic64_fetch_sub_release(1, (v))
948#else /* atomic64_fetch_dec */
949#define atomic64_fetch_dec_relaxed atomic64_fetch_dec
950#define atomic64_fetch_dec_acquire atomic64_fetch_dec
951#define atomic64_fetch_dec_release atomic64_fetch_dec
952#endif /* atomic64_fetch_dec */
953
954#else /* atomic64_fetch_dec_relaxed */
955
956#ifndef atomic64_fetch_dec_acquire
957#define atomic64_fetch_dec_acquire(...) \
958 __atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
959#endif
960
961#ifndef atomic64_fetch_dec_release
962#define atomic64_fetch_dec_release(...) \
963 __atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
964#endif
965
966#ifndef atomic64_fetch_dec
967#define atomic64_fetch_dec(...) \
968 __atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
969#endif
970#endif /* atomic64_fetch_dec_relaxed */
971
972/* atomic64_fetch_or_relaxed */
973#ifndef atomic64_fetch_or_relaxed
974#define atomic64_fetch_or_relaxed atomic64_fetch_or
975#define atomic64_fetch_or_acquire atomic64_fetch_or
976#define atomic64_fetch_or_release atomic64_fetch_or
977
978#else /* atomic64_fetch_or_relaxed */
979
980#ifndef atomic64_fetch_or_acquire
981#define atomic64_fetch_or_acquire(...) \
982 __atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
983#endif
984
985#ifndef atomic64_fetch_or_release
986#define atomic64_fetch_or_release(...) \
987 __atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
988#endif
989
990#ifndef atomic64_fetch_or
991#define atomic64_fetch_or(...) \
992 __atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
993#endif
994#endif /* atomic64_fetch_or_relaxed */
995
996/* atomic64_fetch_and_relaxed */
997#ifndef atomic64_fetch_and_relaxed
998#define atomic64_fetch_and_relaxed atomic64_fetch_and
999#define atomic64_fetch_and_acquire atomic64_fetch_and
1000#define atomic64_fetch_and_release atomic64_fetch_and
1001
1002#else /* atomic64_fetch_and_relaxed */
1003
1004#ifndef atomic64_fetch_and_acquire
1005#define atomic64_fetch_and_acquire(...) \
1006 __atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
1007#endif
1008
1009#ifndef atomic64_fetch_and_release
1010#define atomic64_fetch_and_release(...) \
1011 __atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
1012#endif
1013
1014#ifndef atomic64_fetch_and
1015#define atomic64_fetch_and(...) \
1016 __atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
1017#endif
1018#endif /* atomic64_fetch_and_relaxed */
1019
1020#ifndef atomic64_andnot
1021#define atomic64_andnot(i, v) atomic64_and(~(long long)(i), (v))
1022#endif
1023
1024#ifndef atomic64_fetch_andnot_relaxed
1025
1026#ifndef atomic64_fetch_andnot
1027#define atomic64_fetch_andnot(i, v) atomic64_fetch_and(~(long long)(i), (v))
1028#define atomic64_fetch_andnot_relaxed(i, v) atomic64_fetch_and_relaxed(~(long long)(i), (v))
1029#define atomic64_fetch_andnot_acquire(i, v) atomic64_fetch_and_acquire(~(long long)(i), (v))
1030#define atomic64_fetch_andnot_release(i, v) atomic64_fetch_and_release(~(long long)(i), (v))
1031#else /* atomic64_fetch_andnot */
1032#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot
1033#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot
1034#define atomic64_fetch_andnot_release atomic64_fetch_andnot
1035#endif /* atomic64_fetch_andnot */
1036
1037#else /* atomic64_fetch_andnot_relaxed */
1038
1039#ifndef atomic64_fetch_andnot_acquire
1040#define atomic64_fetch_andnot_acquire(...) \
1041 __atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
1042#endif
1043
1044#ifndef atomic64_fetch_andnot_release
1045#define atomic64_fetch_andnot_release(...) \
1046 __atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
1047#endif
1048
1049#ifndef atomic64_fetch_andnot
1050#define atomic64_fetch_andnot(...) \
1051 __atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
1052#endif
1053#endif /* atomic64_fetch_andnot_relaxed */
1054
1055/* atomic64_fetch_xor_relaxed */
1056#ifndef atomic64_fetch_xor_relaxed
1057#define atomic64_fetch_xor_relaxed atomic64_fetch_xor
1058#define atomic64_fetch_xor_acquire atomic64_fetch_xor
1059#define atomic64_fetch_xor_release atomic64_fetch_xor
1060
1061#else /* atomic64_fetch_xor_relaxed */
1062
1063#ifndef atomic64_fetch_xor_acquire
1064#define atomic64_fetch_xor_acquire(...) \
1065 __atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
1066#endif
1067
1068#ifndef atomic64_fetch_xor_release
1069#define atomic64_fetch_xor_release(...) \
1070 __atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
1071#endif
1072
1073#ifndef atomic64_fetch_xor
1074#define atomic64_fetch_xor(...) \
1075 __atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
1076#endif
1077#endif /* atomic64_fetch_xor_relaxed */
1078
1079
1080/* atomic64_xchg_relaxed */
1081#ifndef atomic64_xchg_relaxed
1082#define atomic64_xchg_relaxed atomic64_xchg
1083#define atomic64_xchg_acquire atomic64_xchg
1084#define atomic64_xchg_release atomic64_xchg
1085
1086#else /* atomic64_xchg_relaxed */
1087
1088#ifndef atomic64_xchg_acquire
1089#define atomic64_xchg_acquire(...) \
1090 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
1091#endif
1092
1093#ifndef atomic64_xchg_release
1094#define atomic64_xchg_release(...) \
1095 __atomic_op_release(atomic64_xchg, __VA_ARGS__)
1096#endif
1097
1098#ifndef atomic64_xchg
1099#define atomic64_xchg(...) \
1100 __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
1101#endif
1102#endif /* atomic64_xchg_relaxed */
1103
1104/* atomic64_cmpxchg_relaxed */
1105#ifndef atomic64_cmpxchg_relaxed
1106#define atomic64_cmpxchg_relaxed atomic64_cmpxchg
1107#define atomic64_cmpxchg_acquire atomic64_cmpxchg
1108#define atomic64_cmpxchg_release atomic64_cmpxchg
1109
1110#else /* atomic64_cmpxchg_relaxed */
1111
1112#ifndef atomic64_cmpxchg_acquire
1113#define atomic64_cmpxchg_acquire(...) \
1114 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
1115#endif
1116
1117#ifndef atomic64_cmpxchg_release
1118#define atomic64_cmpxchg_release(...) \
1119 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
1120#endif
1121
1122#ifndef atomic64_cmpxchg
1123#define atomic64_cmpxchg(...) \
1124 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
1125#endif
1126#endif /* atomic64_cmpxchg_relaxed */
1127
1128#ifndef atomic64_try_cmpxchg
1129
1130#define __atomic64_try_cmpxchg(type, _p, _po, _n) \
1131({ \
1132 typeof(_po) __po = (_po); \
1133 typeof(*(_po)) __r, __o = *__po; \
1134 __r = atomic64_cmpxchg##type((_p), __o, (_n)); \
1135 if (unlikely(__r != __o)) \
1136 *__po = __r; \
1137 likely(__r == __o); \
1138})
1139
1140#define atomic64_try_cmpxchg(_p, _po, _n) __atomic64_try_cmpxchg(, _p, _po, _n)
1141#define atomic64_try_cmpxchg_relaxed(_p, _po, _n) __atomic64_try_cmpxchg(_relaxed, _p, _po, _n)
1142#define atomic64_try_cmpxchg_acquire(_p, _po, _n) __atomic64_try_cmpxchg(_acquire, _p, _po, _n)
1143#define atomic64_try_cmpxchg_release(_p, _po, _n) __atomic64_try_cmpxchg(_release, _p, _po, _n)
1144
1145#else /* atomic64_try_cmpxchg */
1146#define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg
1147#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg
1148#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg
1149#endif /* atomic64_try_cmpxchg */
1150
1151/**
1152 * atomic64_fetch_add_unless - add unless the number is already a given value
1153 * @v: pointer of type atomic64_t
1154 * @a: the amount to add to v...
1155 * @u: ...unless v is equal to u.
1156 *
1157 * Atomically adds @a to @v, if @v was not already @u.
1158 * Returns the original value of @v.
1159 */
1160#ifndef atomic64_fetch_add_unless
1161static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a,
1162 long long u)
1163{
1164 long long c = atomic64_read(v);
1165
1166 do {
1167 if (unlikely(c == u))
1168 break;
1169 } while (!atomic64_try_cmpxchg(v, &c, c + a));
1170
1171 return c;
1172}
1173#endif
1174
1175/**
1176 * atomic64_add_unless - add unless the number is already a given value
1177 * @v: pointer of type atomic_t
1178 * @a: the amount to add to v...
1179 * @u: ...unless v is equal to u.
1180 *
1181 * Atomically adds @a to @v, if @v was not already @u.
1182 * Returns true if the addition was done.
1183 */
1184static inline bool atomic64_add_unless(atomic64_t *v, long long a, long long u)
1185{
1186 return atomic64_fetch_add_unless(v, a, u) != u;
1187}
1188
1189/**
1190 * atomic64_inc_not_zero - increment unless the number is zero
1191 * @v: pointer of type atomic64_t
1192 *
1193 * Atomically increments @v by 1, if @v is non-zero.
1194 * Returns true if the increment was done.
1195 */
1196#ifndef atomic64_inc_not_zero
1197#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
1198#endif
1199
1200/**
1201 * atomic64_inc_and_test - increment and test
1202 * @v: pointer of type atomic64_t
1203 *
1204 * Atomically increments @v by 1
1205 * and returns true if the result is zero, or false for all
1206 * other cases.
1207 */
1208#ifndef atomic64_inc_and_test
1209static inline bool atomic64_inc_and_test(atomic64_t *v)
1210{
1211 return atomic64_inc_return(v) == 0;
1212}
1213#endif
1214
1215/**
1216 * atomic64_dec_and_test - decrement and test
1217 * @v: pointer of type atomic64_t
1218 *
1219 * Atomically decrements @v by 1 and
1220 * returns true if the result is 0, or false for all other
1221 * cases.
1222 */
1223#ifndef atomic64_dec_and_test
1224static inline bool atomic64_dec_and_test(atomic64_t *v)
1225{
1226 return atomic64_dec_return(v) == 0;
1227}
1228#endif
1229
1230/**
1231 * atomic64_sub_and_test - subtract value from variable and test result
1232 * @i: integer value to subtract
1233 * @v: pointer of type atomic64_t
1234 *
1235 * Atomically subtracts @i from @v and returns
1236 * true if the result is zero, or false for all
1237 * other cases.
1238 */
1239#ifndef atomic64_sub_and_test
1240static inline bool atomic64_sub_and_test(long long i, atomic64_t *v)
1241{
1242 return atomic64_sub_return(i, v) == 0;
1243}
1244#endif
1245
1246/**
1247 * atomic64_add_negative - add and test if negative
1248 * @i: integer value to add
1249 * @v: pointer of type atomic64_t
1250 *
1251 * Atomically adds @i to @v and returns true
1252 * if the result is negative, or false when
1253 * result is greater than or equal to zero.
1254 */
1255#ifndef atomic64_add_negative
1256static inline bool atomic64_add_negative(long long i, atomic64_t *v)
1257{
1258 return atomic64_add_return(i, v) < 0;
1259}
1260#endif
1261
1262#ifndef atomic64_inc_unless_negative
1263static inline bool atomic64_inc_unless_negative(atomic64_t *v)
1264{
1265 long long c = atomic64_read(v);
1266
1267 do {
1268 if (unlikely(c < 0))
1269 return false;
1270 } while (!atomic64_try_cmpxchg(v, &c, c + 1));
1271
1272 return true;
1273}
1274#endif
1275
1276#ifndef atomic64_dec_unless_positive
1277static inline bool atomic64_dec_unless_positive(atomic64_t *v)
1278{
1279 long long c = atomic64_read(v);
1280
1281 do {
1282 if (unlikely(c > 0))
1283 return false;
1284 } while (!atomic64_try_cmpxchg(v, &c, c - 1));
1285
1286 return true;
1287}
1288#endif
1289
1290/*
1291 * atomic64_dec_if_positive - decrement by 1 if old value positive
1292 * @v: pointer of type atomic64_t
1293 *
1294 * The function returns the old value of *v minus 1, even if
1295 * the atomic64 variable, v, was not decremented.
1296 */
1297#ifndef atomic64_dec_if_positive
1298static inline long long atomic64_dec_if_positive(atomic64_t *v)
1299{
1300 long long dec, c = atomic64_read(v);
1301
1302 do {
1303 dec = c - 1;
1304 if (unlikely(dec < 0))
1305 break;
1306 } while (!atomic64_try_cmpxchg(v, &c, dec));
1307
1308 return dec;
1309}
1310#endif
1311
1312#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
1313#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
1314
1315#include <asm-generic/atomic-long.h>
1316
1317#endif /* _LINUX_ATOMIC_H */