Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* Atomic operations usable in machine independent code */
2#ifndef _LINUX_ATOMIC_H
3#define _LINUX_ATOMIC_H
4#include <asm/atomic.h>
5#include <asm/barrier.h>
6
7/*
8 * Relaxed variants of xchg, cmpxchg and some atomic operations.
9 *
10 * We support four variants:
11 *
12 * - Fully ordered: The default implementation, no suffix required.
13 * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
14 * - Release: Provides RELEASE semantics, _release suffix.
15 * - Relaxed: No ordering guarantees, _relaxed suffix.
16 *
17 * For compound atomics performing both a load and a store, ACQUIRE
18 * semantics apply only to the load and RELEASE semantics only to the
19 * store portion of the operation. Note that a failed cmpxchg_acquire
20 * does -not- imply any memory ordering constraints.
21 *
22 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
23 */
24
25#ifndef atomic_read_acquire
26#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
27#endif
28
29#ifndef atomic_set_release
30#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
31#endif
32
33/*
34 * The idea here is to build acquire/release variants by adding explicit
35 * barriers on top of the relaxed variant. In the case where the relaxed
36 * variant is already fully ordered, no additional barriers are needed.
37 *
38 * Besides, if an arch has a special barrier for acquire/release, it could
39 * implement its own __atomic_op_* and use the same framework for building
40 * variants
41 */
42#ifndef __atomic_op_acquire
43#define __atomic_op_acquire(op, args...) \
44({ \
45 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
46 smp_mb__after_atomic(); \
47 __ret; \
48})
49#endif
50
51#ifndef __atomic_op_release
52#define __atomic_op_release(op, args...) \
53({ \
54 smp_mb__before_atomic(); \
55 op##_relaxed(args); \
56})
57#endif
58
59#ifndef __atomic_op_fence
60#define __atomic_op_fence(op, args...) \
61({ \
62 typeof(op##_relaxed(args)) __ret; \
63 smp_mb__before_atomic(); \
64 __ret = op##_relaxed(args); \
65 smp_mb__after_atomic(); \
66 __ret; \
67})
68#endif
69
70/* atomic_add_return_relaxed */
71#ifndef atomic_add_return_relaxed
72#define atomic_add_return_relaxed atomic_add_return
73#define atomic_add_return_acquire atomic_add_return
74#define atomic_add_return_release atomic_add_return
75
76#else /* atomic_add_return_relaxed */
77
78#ifndef atomic_add_return_acquire
79#define atomic_add_return_acquire(...) \
80 __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
81#endif
82
83#ifndef atomic_add_return_release
84#define atomic_add_return_release(...) \
85 __atomic_op_release(atomic_add_return, __VA_ARGS__)
86#endif
87
88#ifndef atomic_add_return
89#define atomic_add_return(...) \
90 __atomic_op_fence(atomic_add_return, __VA_ARGS__)
91#endif
92#endif /* atomic_add_return_relaxed */
93
94/* atomic_inc_return_relaxed */
95#ifndef atomic_inc_return_relaxed
96#define atomic_inc_return_relaxed atomic_inc_return
97#define atomic_inc_return_acquire atomic_inc_return
98#define atomic_inc_return_release atomic_inc_return
99
100#else /* atomic_inc_return_relaxed */
101
102#ifndef atomic_inc_return_acquire
103#define atomic_inc_return_acquire(...) \
104 __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
105#endif
106
107#ifndef atomic_inc_return_release
108#define atomic_inc_return_release(...) \
109 __atomic_op_release(atomic_inc_return, __VA_ARGS__)
110#endif
111
112#ifndef atomic_inc_return
113#define atomic_inc_return(...) \
114 __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
115#endif
116#endif /* atomic_inc_return_relaxed */
117
118/* atomic_sub_return_relaxed */
119#ifndef atomic_sub_return_relaxed
120#define atomic_sub_return_relaxed atomic_sub_return
121#define atomic_sub_return_acquire atomic_sub_return
122#define atomic_sub_return_release atomic_sub_return
123
124#else /* atomic_sub_return_relaxed */
125
126#ifndef atomic_sub_return_acquire
127#define atomic_sub_return_acquire(...) \
128 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
129#endif
130
131#ifndef atomic_sub_return_release
132#define atomic_sub_return_release(...) \
133 __atomic_op_release(atomic_sub_return, __VA_ARGS__)
134#endif
135
136#ifndef atomic_sub_return
137#define atomic_sub_return(...) \
138 __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
139#endif
140#endif /* atomic_sub_return_relaxed */
141
142/* atomic_dec_return_relaxed */
143#ifndef atomic_dec_return_relaxed
144#define atomic_dec_return_relaxed atomic_dec_return
145#define atomic_dec_return_acquire atomic_dec_return
146#define atomic_dec_return_release atomic_dec_return
147
148#else /* atomic_dec_return_relaxed */
149
150#ifndef atomic_dec_return_acquire
151#define atomic_dec_return_acquire(...) \
152 __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
153#endif
154
155#ifndef atomic_dec_return_release
156#define atomic_dec_return_release(...) \
157 __atomic_op_release(atomic_dec_return, __VA_ARGS__)
158#endif
159
160#ifndef atomic_dec_return
161#define atomic_dec_return(...) \
162 __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
163#endif
164#endif /* atomic_dec_return_relaxed */
165
166/* atomic_xchg_relaxed */
167#ifndef atomic_xchg_relaxed
168#define atomic_xchg_relaxed atomic_xchg
169#define atomic_xchg_acquire atomic_xchg
170#define atomic_xchg_release atomic_xchg
171
172#else /* atomic_xchg_relaxed */
173
174#ifndef atomic_xchg_acquire
175#define atomic_xchg_acquire(...) \
176 __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
177#endif
178
179#ifndef atomic_xchg_release
180#define atomic_xchg_release(...) \
181 __atomic_op_release(atomic_xchg, __VA_ARGS__)
182#endif
183
184#ifndef atomic_xchg
185#define atomic_xchg(...) \
186 __atomic_op_fence(atomic_xchg, __VA_ARGS__)
187#endif
188#endif /* atomic_xchg_relaxed */
189
190/* atomic_cmpxchg_relaxed */
191#ifndef atomic_cmpxchg_relaxed
192#define atomic_cmpxchg_relaxed atomic_cmpxchg
193#define atomic_cmpxchg_acquire atomic_cmpxchg
194#define atomic_cmpxchg_release atomic_cmpxchg
195
196#else /* atomic_cmpxchg_relaxed */
197
198#ifndef atomic_cmpxchg_acquire
199#define atomic_cmpxchg_acquire(...) \
200 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
201#endif
202
203#ifndef atomic_cmpxchg_release
204#define atomic_cmpxchg_release(...) \
205 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
206#endif
207
208#ifndef atomic_cmpxchg
209#define atomic_cmpxchg(...) \
210 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
211#endif
212#endif /* atomic_cmpxchg_relaxed */
213
214#ifndef atomic64_read_acquire
215#define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter)
216#endif
217
218#ifndef atomic64_set_release
219#define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i))
220#endif
221
222/* atomic64_add_return_relaxed */
223#ifndef atomic64_add_return_relaxed
224#define atomic64_add_return_relaxed atomic64_add_return
225#define atomic64_add_return_acquire atomic64_add_return
226#define atomic64_add_return_release atomic64_add_return
227
228#else /* atomic64_add_return_relaxed */
229
230#ifndef atomic64_add_return_acquire
231#define atomic64_add_return_acquire(...) \
232 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
233#endif
234
235#ifndef atomic64_add_return_release
236#define atomic64_add_return_release(...) \
237 __atomic_op_release(atomic64_add_return, __VA_ARGS__)
238#endif
239
240#ifndef atomic64_add_return
241#define atomic64_add_return(...) \
242 __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
243#endif
244#endif /* atomic64_add_return_relaxed */
245
246/* atomic64_inc_return_relaxed */
247#ifndef atomic64_inc_return_relaxed
248#define atomic64_inc_return_relaxed atomic64_inc_return
249#define atomic64_inc_return_acquire atomic64_inc_return
250#define atomic64_inc_return_release atomic64_inc_return
251
252#else /* atomic64_inc_return_relaxed */
253
254#ifndef atomic64_inc_return_acquire
255#define atomic64_inc_return_acquire(...) \
256 __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
257#endif
258
259#ifndef atomic64_inc_return_release
260#define atomic64_inc_return_release(...) \
261 __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
262#endif
263
264#ifndef atomic64_inc_return
265#define atomic64_inc_return(...) \
266 __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
267#endif
268#endif /* atomic64_inc_return_relaxed */
269
270
271/* atomic64_sub_return_relaxed */
272#ifndef atomic64_sub_return_relaxed
273#define atomic64_sub_return_relaxed atomic64_sub_return
274#define atomic64_sub_return_acquire atomic64_sub_return
275#define atomic64_sub_return_release atomic64_sub_return
276
277#else /* atomic64_sub_return_relaxed */
278
279#ifndef atomic64_sub_return_acquire
280#define atomic64_sub_return_acquire(...) \
281 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
282#endif
283
284#ifndef atomic64_sub_return_release
285#define atomic64_sub_return_release(...) \
286 __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
287#endif
288
289#ifndef atomic64_sub_return
290#define atomic64_sub_return(...) \
291 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
292#endif
293#endif /* atomic64_sub_return_relaxed */
294
295/* atomic64_dec_return_relaxed */
296#ifndef atomic64_dec_return_relaxed
297#define atomic64_dec_return_relaxed atomic64_dec_return
298#define atomic64_dec_return_acquire atomic64_dec_return
299#define atomic64_dec_return_release atomic64_dec_return
300
301#else /* atomic64_dec_return_relaxed */
302
303#ifndef atomic64_dec_return_acquire
304#define atomic64_dec_return_acquire(...) \
305 __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
306#endif
307
308#ifndef atomic64_dec_return_release
309#define atomic64_dec_return_release(...) \
310 __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
311#endif
312
313#ifndef atomic64_dec_return
314#define atomic64_dec_return(...) \
315 __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
316#endif
317#endif /* atomic64_dec_return_relaxed */
318
319/* atomic64_xchg_relaxed */
320#ifndef atomic64_xchg_relaxed
321#define atomic64_xchg_relaxed atomic64_xchg
322#define atomic64_xchg_acquire atomic64_xchg
323#define atomic64_xchg_release atomic64_xchg
324
325#else /* atomic64_xchg_relaxed */
326
327#ifndef atomic64_xchg_acquire
328#define atomic64_xchg_acquire(...) \
329 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
330#endif
331
332#ifndef atomic64_xchg_release
333#define atomic64_xchg_release(...) \
334 __atomic_op_release(atomic64_xchg, __VA_ARGS__)
335#endif
336
337#ifndef atomic64_xchg
338#define atomic64_xchg(...) \
339 __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
340#endif
341#endif /* atomic64_xchg_relaxed */
342
343/* atomic64_cmpxchg_relaxed */
344#ifndef atomic64_cmpxchg_relaxed
345#define atomic64_cmpxchg_relaxed atomic64_cmpxchg
346#define atomic64_cmpxchg_acquire atomic64_cmpxchg
347#define atomic64_cmpxchg_release atomic64_cmpxchg
348
349#else /* atomic64_cmpxchg_relaxed */
350
351#ifndef atomic64_cmpxchg_acquire
352#define atomic64_cmpxchg_acquire(...) \
353 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
354#endif
355
356#ifndef atomic64_cmpxchg_release
357#define atomic64_cmpxchg_release(...) \
358 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
359#endif
360
361#ifndef atomic64_cmpxchg
362#define atomic64_cmpxchg(...) \
363 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
364#endif
365#endif /* atomic64_cmpxchg_relaxed */
366
367/* cmpxchg_relaxed */
368#ifndef cmpxchg_relaxed
369#define cmpxchg_relaxed cmpxchg
370#define cmpxchg_acquire cmpxchg
371#define cmpxchg_release cmpxchg
372
373#else /* cmpxchg_relaxed */
374
375#ifndef cmpxchg_acquire
376#define cmpxchg_acquire(...) \
377 __atomic_op_acquire(cmpxchg, __VA_ARGS__)
378#endif
379
380#ifndef cmpxchg_release
381#define cmpxchg_release(...) \
382 __atomic_op_release(cmpxchg, __VA_ARGS__)
383#endif
384
385#ifndef cmpxchg
386#define cmpxchg(...) \
387 __atomic_op_fence(cmpxchg, __VA_ARGS__)
388#endif
389#endif /* cmpxchg_relaxed */
390
391/* cmpxchg64_relaxed */
392#ifndef cmpxchg64_relaxed
393#define cmpxchg64_relaxed cmpxchg64
394#define cmpxchg64_acquire cmpxchg64
395#define cmpxchg64_release cmpxchg64
396
397#else /* cmpxchg64_relaxed */
398
399#ifndef cmpxchg64_acquire
400#define cmpxchg64_acquire(...) \
401 __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
402#endif
403
404#ifndef cmpxchg64_release
405#define cmpxchg64_release(...) \
406 __atomic_op_release(cmpxchg64, __VA_ARGS__)
407#endif
408
409#ifndef cmpxchg64
410#define cmpxchg64(...) \
411 __atomic_op_fence(cmpxchg64, __VA_ARGS__)
412#endif
413#endif /* cmpxchg64_relaxed */
414
415/* xchg_relaxed */
416#ifndef xchg_relaxed
417#define xchg_relaxed xchg
418#define xchg_acquire xchg
419#define xchg_release xchg
420
421#else /* xchg_relaxed */
422
423#ifndef xchg_acquire
424#define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__)
425#endif
426
427#ifndef xchg_release
428#define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__)
429#endif
430
431#ifndef xchg
432#define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__)
433#endif
434#endif /* xchg_relaxed */
435
436/**
437 * atomic_add_unless - add unless the number is already a given value
438 * @v: pointer of type atomic_t
439 * @a: the amount to add to v...
440 * @u: ...unless v is equal to u.
441 *
442 * Atomically adds @a to @v, so long as @v was not already @u.
443 * Returns non-zero if @v was not @u, and zero otherwise.
444 */
445static inline int atomic_add_unless(atomic_t *v, int a, int u)
446{
447 return __atomic_add_unless(v, a, u) != u;
448}
449
450/**
451 * atomic_inc_not_zero - increment unless the number is zero
452 * @v: pointer of type atomic_t
453 *
454 * Atomically increments @v by 1, so long as @v is non-zero.
455 * Returns non-zero if @v was non-zero, and zero otherwise.
456 */
457#ifndef atomic_inc_not_zero
458#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
459#endif
460
461#ifndef atomic_andnot
462static inline void atomic_andnot(int i, atomic_t *v)
463{
464 atomic_and(~i, v);
465}
466#endif
467
468static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
469{
470 atomic_andnot(mask, v);
471}
472
473static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v)
474{
475 atomic_or(mask, v);
476}
477
478/**
479 * atomic_inc_not_zero_hint - increment if not null
480 * @v: pointer of type atomic_t
481 * @hint: probable value of the atomic before the increment
482 *
483 * This version of atomic_inc_not_zero() gives a hint of probable
484 * value of the atomic. This helps processor to not read the memory
485 * before doing the atomic read/modify/write cycle, lowering
486 * number of bus transactions on some arches.
487 *
488 * Returns: 0 if increment was not done, 1 otherwise.
489 */
490#ifndef atomic_inc_not_zero_hint
491static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
492{
493 int val, c = hint;
494
495 /* sanity test, should be removed by compiler if hint is a constant */
496 if (!hint)
497 return atomic_inc_not_zero(v);
498
499 do {
500 val = atomic_cmpxchg(v, c, c + 1);
501 if (val == c)
502 return 1;
503 c = val;
504 } while (c);
505
506 return 0;
507}
508#endif
509
510#ifndef atomic_inc_unless_negative
511static inline int atomic_inc_unless_negative(atomic_t *p)
512{
513 int v, v1;
514 for (v = 0; v >= 0; v = v1) {
515 v1 = atomic_cmpxchg(p, v, v + 1);
516 if (likely(v1 == v))
517 return 1;
518 }
519 return 0;
520}
521#endif
522
523#ifndef atomic_dec_unless_positive
524static inline int atomic_dec_unless_positive(atomic_t *p)
525{
526 int v, v1;
527 for (v = 0; v <= 0; v = v1) {
528 v1 = atomic_cmpxchg(p, v, v - 1);
529 if (likely(v1 == v))
530 return 1;
531 }
532 return 0;
533}
534#endif
535
536/*
537 * atomic_dec_if_positive - decrement by 1 if old value positive
538 * @v: pointer of type atomic_t
539 *
540 * The function returns the old value of *v minus 1, even if
541 * the atomic variable, v, was not decremented.
542 */
543#ifndef atomic_dec_if_positive
544static inline int atomic_dec_if_positive(atomic_t *v)
545{
546 int c, old, dec;
547 c = atomic_read(v);
548 for (;;) {
549 dec = c - 1;
550 if (unlikely(dec < 0))
551 break;
552 old = atomic_cmpxchg((v), c, dec);
553 if (likely(old == c))
554 break;
555 c = old;
556 }
557 return dec;
558}
559#endif
560
561/**
562 * atomic_fetch_or - perform *p |= mask and return old value of *p
563 * @mask: mask to OR on the atomic_t
564 * @p: pointer to atomic_t
565 */
566#ifndef atomic_fetch_or
567static inline int atomic_fetch_or(int mask, atomic_t *p)
568{
569 int old, val = atomic_read(p);
570
571 for (;;) {
572 old = atomic_cmpxchg(p, val, val | mask);
573 if (old == val)
574 break;
575 val = old;
576 }
577
578 return old;
579}
580#endif
581
582#ifdef CONFIG_GENERIC_ATOMIC64
583#include <asm-generic/atomic64.h>
584#endif
585
586#ifndef atomic64_andnot
587static inline void atomic64_andnot(long long i, atomic64_t *v)
588{
589 atomic64_and(~i, v);
590}
591#endif
592
593#include <asm-generic/atomic-long.h>
594
595#endif /* _LINUX_ATOMIC_H */