Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* Atomic operations usable in machine independent code */
2#ifndef _LINUX_ATOMIC_H
3#define _LINUX_ATOMIC_H
4#include <asm/atomic.h>
5#include <asm/barrier.h>
6
7/*
8 * Relaxed variants of xchg, cmpxchg and some atomic operations.
9 *
10 * We support four variants:
11 *
12 * - Fully ordered: The default implementation, no suffix required.
13 * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
14 * - Release: Provides RELEASE semantics, _release suffix.
15 * - Relaxed: No ordering guarantees, _relaxed suffix.
16 *
17 * For compound atomics performing both a load and a store, ACQUIRE
18 * semantics apply only to the load and RELEASE semantics only to the
19 * store portion of the operation. Note that a failed cmpxchg_acquire
20 * does -not- imply any memory ordering constraints.
21 *
22 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
23 */
24
25#ifndef atomic_read_acquire
26#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
27#endif
28
29#ifndef atomic_set_release
30#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
31#endif
32
33/*
34 * The idea here is to build acquire/release variants by adding explicit
35 * barriers on top of the relaxed variant. In the case where the relaxed
36 * variant is already fully ordered, no additional barriers are needed.
37 */
38#define __atomic_op_acquire(op, args...) \
39({ \
40 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
41 smp_mb__after_atomic(); \
42 __ret; \
43})
44
45#define __atomic_op_release(op, args...) \
46({ \
47 smp_mb__before_atomic(); \
48 op##_relaxed(args); \
49})
50
51#define __atomic_op_fence(op, args...) \
52({ \
53 typeof(op##_relaxed(args)) __ret; \
54 smp_mb__before_atomic(); \
55 __ret = op##_relaxed(args); \
56 smp_mb__after_atomic(); \
57 __ret; \
58})
59
60/* atomic_add_return_relaxed */
61#ifndef atomic_add_return_relaxed
62#define atomic_add_return_relaxed atomic_add_return
63#define atomic_add_return_acquire atomic_add_return
64#define atomic_add_return_release atomic_add_return
65
66#else /* atomic_add_return_relaxed */
67
68#ifndef atomic_add_return_acquire
69#define atomic_add_return_acquire(...) \
70 __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
71#endif
72
73#ifndef atomic_add_return_release
74#define atomic_add_return_release(...) \
75 __atomic_op_release(atomic_add_return, __VA_ARGS__)
76#endif
77
78#ifndef atomic_add_return
79#define atomic_add_return(...) \
80 __atomic_op_fence(atomic_add_return, __VA_ARGS__)
81#endif
82#endif /* atomic_add_return_relaxed */
83
84/* atomic_inc_return_relaxed */
85#ifndef atomic_inc_return_relaxed
86#define atomic_inc_return_relaxed atomic_inc_return
87#define atomic_inc_return_acquire atomic_inc_return
88#define atomic_inc_return_release atomic_inc_return
89
90#else /* atomic_inc_return_relaxed */
91
92#ifndef atomic_inc_return_acquire
93#define atomic_inc_return_acquire(...) \
94 __atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
95#endif
96
97#ifndef atomic_inc_return_release
98#define atomic_inc_return_release(...) \
99 __atomic_op_release(atomic_inc_return, __VA_ARGS__)
100#endif
101
102#ifndef atomic_inc_return
103#define atomic_inc_return(...) \
104 __atomic_op_fence(atomic_inc_return, __VA_ARGS__)
105#endif
106#endif /* atomic_inc_return_relaxed */
107
108/* atomic_sub_return_relaxed */
109#ifndef atomic_sub_return_relaxed
110#define atomic_sub_return_relaxed atomic_sub_return
111#define atomic_sub_return_acquire atomic_sub_return
112#define atomic_sub_return_release atomic_sub_return
113
114#else /* atomic_sub_return_relaxed */
115
116#ifndef atomic_sub_return_acquire
117#define atomic_sub_return_acquire(...) \
118 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
119#endif
120
121#ifndef atomic_sub_return_release
122#define atomic_sub_return_release(...) \
123 __atomic_op_release(atomic_sub_return, __VA_ARGS__)
124#endif
125
126#ifndef atomic_sub_return
127#define atomic_sub_return(...) \
128 __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
129#endif
130#endif /* atomic_sub_return_relaxed */
131
132/* atomic_dec_return_relaxed */
133#ifndef atomic_dec_return_relaxed
134#define atomic_dec_return_relaxed atomic_dec_return
135#define atomic_dec_return_acquire atomic_dec_return
136#define atomic_dec_return_release atomic_dec_return
137
138#else /* atomic_dec_return_relaxed */
139
140#ifndef atomic_dec_return_acquire
141#define atomic_dec_return_acquire(...) \
142 __atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
143#endif
144
145#ifndef atomic_dec_return_release
146#define atomic_dec_return_release(...) \
147 __atomic_op_release(atomic_dec_return, __VA_ARGS__)
148#endif
149
150#ifndef atomic_dec_return
151#define atomic_dec_return(...) \
152 __atomic_op_fence(atomic_dec_return, __VA_ARGS__)
153#endif
154#endif /* atomic_dec_return_relaxed */
155
156/* atomic_xchg_relaxed */
157#ifndef atomic_xchg_relaxed
158#define atomic_xchg_relaxed atomic_xchg
159#define atomic_xchg_acquire atomic_xchg
160#define atomic_xchg_release atomic_xchg
161
162#else /* atomic_xchg_relaxed */
163
164#ifndef atomic_xchg_acquire
165#define atomic_xchg_acquire(...) \
166 __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
167#endif
168
169#ifndef atomic_xchg_release
170#define atomic_xchg_release(...) \
171 __atomic_op_release(atomic_xchg, __VA_ARGS__)
172#endif
173
174#ifndef atomic_xchg
175#define atomic_xchg(...) \
176 __atomic_op_fence(atomic_xchg, __VA_ARGS__)
177#endif
178#endif /* atomic_xchg_relaxed */
179
180/* atomic_cmpxchg_relaxed */
181#ifndef atomic_cmpxchg_relaxed
182#define atomic_cmpxchg_relaxed atomic_cmpxchg
183#define atomic_cmpxchg_acquire atomic_cmpxchg
184#define atomic_cmpxchg_release atomic_cmpxchg
185
186#else /* atomic_cmpxchg_relaxed */
187
188#ifndef atomic_cmpxchg_acquire
189#define atomic_cmpxchg_acquire(...) \
190 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
191#endif
192
193#ifndef atomic_cmpxchg_release
194#define atomic_cmpxchg_release(...) \
195 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
196#endif
197
198#ifndef atomic_cmpxchg
199#define atomic_cmpxchg(...) \
200 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
201#endif
202#endif /* atomic_cmpxchg_relaxed */
203
204#ifndef atomic64_read_acquire
205#define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter)
206#endif
207
208#ifndef atomic64_set_release
209#define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i))
210#endif
211
212/* atomic64_add_return_relaxed */
213#ifndef atomic64_add_return_relaxed
214#define atomic64_add_return_relaxed atomic64_add_return
215#define atomic64_add_return_acquire atomic64_add_return
216#define atomic64_add_return_release atomic64_add_return
217
218#else /* atomic64_add_return_relaxed */
219
220#ifndef atomic64_add_return_acquire
221#define atomic64_add_return_acquire(...) \
222 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
223#endif
224
225#ifndef atomic64_add_return_release
226#define atomic64_add_return_release(...) \
227 __atomic_op_release(atomic64_add_return, __VA_ARGS__)
228#endif
229
230#ifndef atomic64_add_return
231#define atomic64_add_return(...) \
232 __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
233#endif
234#endif /* atomic64_add_return_relaxed */
235
236/* atomic64_inc_return_relaxed */
237#ifndef atomic64_inc_return_relaxed
238#define atomic64_inc_return_relaxed atomic64_inc_return
239#define atomic64_inc_return_acquire atomic64_inc_return
240#define atomic64_inc_return_release atomic64_inc_return
241
242#else /* atomic64_inc_return_relaxed */
243
244#ifndef atomic64_inc_return_acquire
245#define atomic64_inc_return_acquire(...) \
246 __atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
247#endif
248
249#ifndef atomic64_inc_return_release
250#define atomic64_inc_return_release(...) \
251 __atomic_op_release(atomic64_inc_return, __VA_ARGS__)
252#endif
253
254#ifndef atomic64_inc_return
255#define atomic64_inc_return(...) \
256 __atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
257#endif
258#endif /* atomic64_inc_return_relaxed */
259
260
261/* atomic64_sub_return_relaxed */
262#ifndef atomic64_sub_return_relaxed
263#define atomic64_sub_return_relaxed atomic64_sub_return
264#define atomic64_sub_return_acquire atomic64_sub_return
265#define atomic64_sub_return_release atomic64_sub_return
266
267#else /* atomic64_sub_return_relaxed */
268
269#ifndef atomic64_sub_return_acquire
270#define atomic64_sub_return_acquire(...) \
271 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
272#endif
273
274#ifndef atomic64_sub_return_release
275#define atomic64_sub_return_release(...) \
276 __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
277#endif
278
279#ifndef atomic64_sub_return
280#define atomic64_sub_return(...) \
281 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
282#endif
283#endif /* atomic64_sub_return_relaxed */
284
285/* atomic64_dec_return_relaxed */
286#ifndef atomic64_dec_return_relaxed
287#define atomic64_dec_return_relaxed atomic64_dec_return
288#define atomic64_dec_return_acquire atomic64_dec_return
289#define atomic64_dec_return_release atomic64_dec_return
290
291#else /* atomic64_dec_return_relaxed */
292
293#ifndef atomic64_dec_return_acquire
294#define atomic64_dec_return_acquire(...) \
295 __atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
296#endif
297
298#ifndef atomic64_dec_return_release
299#define atomic64_dec_return_release(...) \
300 __atomic_op_release(atomic64_dec_return, __VA_ARGS__)
301#endif
302
303#ifndef atomic64_dec_return
304#define atomic64_dec_return(...) \
305 __atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
306#endif
307#endif /* atomic64_dec_return_relaxed */
308
309/* atomic64_xchg_relaxed */
310#ifndef atomic64_xchg_relaxed
311#define atomic64_xchg_relaxed atomic64_xchg
312#define atomic64_xchg_acquire atomic64_xchg
313#define atomic64_xchg_release atomic64_xchg
314
315#else /* atomic64_xchg_relaxed */
316
317#ifndef atomic64_xchg_acquire
318#define atomic64_xchg_acquire(...) \
319 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
320#endif
321
322#ifndef atomic64_xchg_release
323#define atomic64_xchg_release(...) \
324 __atomic_op_release(atomic64_xchg, __VA_ARGS__)
325#endif
326
327#ifndef atomic64_xchg
328#define atomic64_xchg(...) \
329 __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
330#endif
331#endif /* atomic64_xchg_relaxed */
332
333/* atomic64_cmpxchg_relaxed */
334#ifndef atomic64_cmpxchg_relaxed
335#define atomic64_cmpxchg_relaxed atomic64_cmpxchg
336#define atomic64_cmpxchg_acquire atomic64_cmpxchg
337#define atomic64_cmpxchg_release atomic64_cmpxchg
338
339#else /* atomic64_cmpxchg_relaxed */
340
341#ifndef atomic64_cmpxchg_acquire
342#define atomic64_cmpxchg_acquire(...) \
343 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
344#endif
345
346#ifndef atomic64_cmpxchg_release
347#define atomic64_cmpxchg_release(...) \
348 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
349#endif
350
351#ifndef atomic64_cmpxchg
352#define atomic64_cmpxchg(...) \
353 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
354#endif
355#endif /* atomic64_cmpxchg_relaxed */
356
357/* cmpxchg_relaxed */
358#ifndef cmpxchg_relaxed
359#define cmpxchg_relaxed cmpxchg
360#define cmpxchg_acquire cmpxchg
361#define cmpxchg_release cmpxchg
362
363#else /* cmpxchg_relaxed */
364
365#ifndef cmpxchg_acquire
366#define cmpxchg_acquire(...) \
367 __atomic_op_acquire(cmpxchg, __VA_ARGS__)
368#endif
369
370#ifndef cmpxchg_release
371#define cmpxchg_release(...) \
372 __atomic_op_release(cmpxchg, __VA_ARGS__)
373#endif
374
375#ifndef cmpxchg
376#define cmpxchg(...) \
377 __atomic_op_fence(cmpxchg, __VA_ARGS__)
378#endif
379#endif /* cmpxchg_relaxed */
380
381/* cmpxchg64_relaxed */
382#ifndef cmpxchg64_relaxed
383#define cmpxchg64_relaxed cmpxchg64
384#define cmpxchg64_acquire cmpxchg64
385#define cmpxchg64_release cmpxchg64
386
387#else /* cmpxchg64_relaxed */
388
389#ifndef cmpxchg64_acquire
390#define cmpxchg64_acquire(...) \
391 __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
392#endif
393
394#ifndef cmpxchg64_release
395#define cmpxchg64_release(...) \
396 __atomic_op_release(cmpxchg64, __VA_ARGS__)
397#endif
398
399#ifndef cmpxchg64
400#define cmpxchg64(...) \
401 __atomic_op_fence(cmpxchg64, __VA_ARGS__)
402#endif
403#endif /* cmpxchg64_relaxed */
404
405/* xchg_relaxed */
406#ifndef xchg_relaxed
407#define xchg_relaxed xchg
408#define xchg_acquire xchg
409#define xchg_release xchg
410
411#else /* xchg_relaxed */
412
413#ifndef xchg_acquire
414#define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__)
415#endif
416
417#ifndef xchg_release
418#define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__)
419#endif
420
421#ifndef xchg
422#define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__)
423#endif
424#endif /* xchg_relaxed */
425
426/**
427 * atomic_add_unless - add unless the number is already a given value
428 * @v: pointer of type atomic_t
429 * @a: the amount to add to v...
430 * @u: ...unless v is equal to u.
431 *
432 * Atomically adds @a to @v, so long as @v was not already @u.
433 * Returns non-zero if @v was not @u, and zero otherwise.
434 */
435static inline int atomic_add_unless(atomic_t *v, int a, int u)
436{
437 return __atomic_add_unless(v, a, u) != u;
438}
439
440/**
441 * atomic_inc_not_zero - increment unless the number is zero
442 * @v: pointer of type atomic_t
443 *
444 * Atomically increments @v by 1, so long as @v is non-zero.
445 * Returns non-zero if @v was non-zero, and zero otherwise.
446 */
447#ifndef atomic_inc_not_zero
448#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
449#endif
450
451#ifndef atomic_andnot
452static inline void atomic_andnot(int i, atomic_t *v)
453{
454 atomic_and(~i, v);
455}
456#endif
457
458static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
459{
460 atomic_andnot(mask, v);
461}
462
463static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v)
464{
465 atomic_or(mask, v);
466}
467
468/**
469 * atomic_inc_not_zero_hint - increment if not null
470 * @v: pointer of type atomic_t
471 * @hint: probable value of the atomic before the increment
472 *
473 * This version of atomic_inc_not_zero() gives a hint of probable
474 * value of the atomic. This helps processor to not read the memory
475 * before doing the atomic read/modify/write cycle, lowering
476 * number of bus transactions on some arches.
477 *
478 * Returns: 0 if increment was not done, 1 otherwise.
479 */
480#ifndef atomic_inc_not_zero_hint
481static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
482{
483 int val, c = hint;
484
485 /* sanity test, should be removed by compiler if hint is a constant */
486 if (!hint)
487 return atomic_inc_not_zero(v);
488
489 do {
490 val = atomic_cmpxchg(v, c, c + 1);
491 if (val == c)
492 return 1;
493 c = val;
494 } while (c);
495
496 return 0;
497}
498#endif
499
500#ifndef atomic_inc_unless_negative
501static inline int atomic_inc_unless_negative(atomic_t *p)
502{
503 int v, v1;
504 for (v = 0; v >= 0; v = v1) {
505 v1 = atomic_cmpxchg(p, v, v + 1);
506 if (likely(v1 == v))
507 return 1;
508 }
509 return 0;
510}
511#endif
512
513#ifndef atomic_dec_unless_positive
514static inline int atomic_dec_unless_positive(atomic_t *p)
515{
516 int v, v1;
517 for (v = 0; v <= 0; v = v1) {
518 v1 = atomic_cmpxchg(p, v, v - 1);
519 if (likely(v1 == v))
520 return 1;
521 }
522 return 0;
523}
524#endif
525
526/*
527 * atomic_dec_if_positive - decrement by 1 if old value positive
528 * @v: pointer of type atomic_t
529 *
530 * The function returns the old value of *v minus 1, even if
531 * the atomic variable, v, was not decremented.
532 */
533#ifndef atomic_dec_if_positive
534static inline int atomic_dec_if_positive(atomic_t *v)
535{
536 int c, old, dec;
537 c = atomic_read(v);
538 for (;;) {
539 dec = c - 1;
540 if (unlikely(dec < 0))
541 break;
542 old = atomic_cmpxchg((v), c, dec);
543 if (likely(old == c))
544 break;
545 c = old;
546 }
547 return dec;
548}
549#endif
550
551#ifdef CONFIG_GENERIC_ATOMIC64
552#include <asm-generic/atomic64.h>
553#endif
554
555#ifndef atomic64_andnot
556static inline void atomic64_andnot(long long i, atomic64_t *v)
557{
558 atomic64_and(~i, v);
559}
560#endif
561
562#include <asm-generic/atomic-long.h>
563
564#endif /* _LINUX_ATOMIC_H */