Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* Atomic operations usable in machine independent code */
2#ifndef _LINUX_ATOMIC_H
3#define _LINUX_ATOMIC_H
4#include <asm/atomic.h>
5#include <asm/barrier.h>
6
7/*
8 * Relaxed variants of xchg, cmpxchg and some atomic operations.
9 *
10 * We support four variants:
11 *
12 * - Fully ordered: The default implementation, no suffix required.
13 * - Acquire: Provides ACQUIRE semantics, _acquire suffix.
14 * - Release: Provides RELEASE semantics, _release suffix.
15 * - Relaxed: No ordering guarantees, _relaxed suffix.
16 *
17 * For compound atomics performing both a load and a store, ACQUIRE
18 * semantics apply only to the load and RELEASE semantics only to the
19 * store portion of the operation. Note that a failed cmpxchg_acquire
20 * does -not- imply any memory ordering constraints.
21 *
22 * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
23 */
24
25#ifndef atomic_read_acquire
26#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
27#endif
28
29#ifndef atomic_set_release
30#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
31#endif
32
33/*
34 * The idea here is to build acquire/release variants by adding explicit
35 * barriers on top of the relaxed variant. In the case where the relaxed
36 * variant is already fully ordered, no additional barriers are needed.
37 */
38#define __atomic_op_acquire(op, args...) \
39({ \
40 typeof(op##_relaxed(args)) __ret = op##_relaxed(args); \
41 smp_mb__after_atomic(); \
42 __ret; \
43})
44
45#define __atomic_op_release(op, args...) \
46({ \
47 smp_mb__before_atomic(); \
48 op##_relaxed(args); \
49})
50
51#define __atomic_op_fence(op, args...) \
52({ \
53 typeof(op##_relaxed(args)) __ret; \
54 smp_mb__before_atomic(); \
55 __ret = op##_relaxed(args); \
56 smp_mb__after_atomic(); \
57 __ret; \
58})
59
60/* atomic_add_return_relaxed */
61#ifndef atomic_add_return_relaxed
62#define atomic_add_return_relaxed atomic_add_return
63#define atomic_add_return_acquire atomic_add_return
64#define atomic_add_return_release atomic_add_return
65
66#else /* atomic_add_return_relaxed */
67
68#ifndef atomic_add_return_acquire
69#define atomic_add_return_acquire(...) \
70 __atomic_op_acquire(atomic_add_return, __VA_ARGS__)
71#endif
72
73#ifndef atomic_add_return_release
74#define atomic_add_return_release(...) \
75 __atomic_op_release(atomic_add_return, __VA_ARGS__)
76#endif
77
78#ifndef atomic_add_return
79#define atomic_add_return(...) \
80 __atomic_op_fence(atomic_add_return, __VA_ARGS__)
81#endif
82#endif /* atomic_add_return_relaxed */
83
84/* atomic_sub_return_relaxed */
85#ifndef atomic_sub_return_relaxed
86#define atomic_sub_return_relaxed atomic_sub_return
87#define atomic_sub_return_acquire atomic_sub_return
88#define atomic_sub_return_release atomic_sub_return
89
90#else /* atomic_sub_return_relaxed */
91
92#ifndef atomic_sub_return_acquire
93#define atomic_sub_return_acquire(...) \
94 __atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
95#endif
96
97#ifndef atomic_sub_return_release
98#define atomic_sub_return_release(...) \
99 __atomic_op_release(atomic_sub_return, __VA_ARGS__)
100#endif
101
102#ifndef atomic_sub_return
103#define atomic_sub_return(...) \
104 __atomic_op_fence(atomic_sub_return, __VA_ARGS__)
105#endif
106#endif /* atomic_sub_return_relaxed */
107
108/* atomic_xchg_relaxed */
109#ifndef atomic_xchg_relaxed
110#define atomic_xchg_relaxed atomic_xchg
111#define atomic_xchg_acquire atomic_xchg
112#define atomic_xchg_release atomic_xchg
113
114#else /* atomic_xchg_relaxed */
115
116#ifndef atomic_xchg_acquire
117#define atomic_xchg_acquire(...) \
118 __atomic_op_acquire(atomic_xchg, __VA_ARGS__)
119#endif
120
121#ifndef atomic_xchg_release
122#define atomic_xchg_release(...) \
123 __atomic_op_release(atomic_xchg, __VA_ARGS__)
124#endif
125
126#ifndef atomic_xchg
127#define atomic_xchg(...) \
128 __atomic_op_fence(atomic_xchg, __VA_ARGS__)
129#endif
130#endif /* atomic_xchg_relaxed */
131
132/* atomic_cmpxchg_relaxed */
133#ifndef atomic_cmpxchg_relaxed
134#define atomic_cmpxchg_relaxed atomic_cmpxchg
135#define atomic_cmpxchg_acquire atomic_cmpxchg
136#define atomic_cmpxchg_release atomic_cmpxchg
137
138#else /* atomic_cmpxchg_relaxed */
139
140#ifndef atomic_cmpxchg_acquire
141#define atomic_cmpxchg_acquire(...) \
142 __atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
143#endif
144
145#ifndef atomic_cmpxchg_release
146#define atomic_cmpxchg_release(...) \
147 __atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
148#endif
149
150#ifndef atomic_cmpxchg
151#define atomic_cmpxchg(...) \
152 __atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
153#endif
154#endif /* atomic_cmpxchg_relaxed */
155
156#ifndef atomic64_read_acquire
157#define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter)
158#endif
159
160#ifndef atomic64_set_release
161#define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i))
162#endif
163
164/* atomic64_add_return_relaxed */
165#ifndef atomic64_add_return_relaxed
166#define atomic64_add_return_relaxed atomic64_add_return
167#define atomic64_add_return_acquire atomic64_add_return
168#define atomic64_add_return_release atomic64_add_return
169
170#else /* atomic64_add_return_relaxed */
171
172#ifndef atomic64_add_return_acquire
173#define atomic64_add_return_acquire(...) \
174 __atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
175#endif
176
177#ifndef atomic64_add_return_release
178#define atomic64_add_return_release(...) \
179 __atomic_op_release(atomic64_add_return, __VA_ARGS__)
180#endif
181
182#ifndef atomic64_add_return
183#define atomic64_add_return(...) \
184 __atomic_op_fence(atomic64_add_return, __VA_ARGS__)
185#endif
186#endif /* atomic64_add_return_relaxed */
187
188/* atomic64_sub_return_relaxed */
189#ifndef atomic64_sub_return_relaxed
190#define atomic64_sub_return_relaxed atomic64_sub_return
191#define atomic64_sub_return_acquire atomic64_sub_return
192#define atomic64_sub_return_release atomic64_sub_return
193
194#else /* atomic64_sub_return_relaxed */
195
196#ifndef atomic64_sub_return_acquire
197#define atomic64_sub_return_acquire(...) \
198 __atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
199#endif
200
201#ifndef atomic64_sub_return_release
202#define atomic64_sub_return_release(...) \
203 __atomic_op_release(atomic64_sub_return, __VA_ARGS__)
204#endif
205
206#ifndef atomic64_sub_return
207#define atomic64_sub_return(...) \
208 __atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
209#endif
210#endif /* atomic64_sub_return_relaxed */
211
212/* atomic64_xchg_relaxed */
213#ifndef atomic64_xchg_relaxed
214#define atomic64_xchg_relaxed atomic64_xchg
215#define atomic64_xchg_acquire atomic64_xchg
216#define atomic64_xchg_release atomic64_xchg
217
218#else /* atomic64_xchg_relaxed */
219
220#ifndef atomic64_xchg_acquire
221#define atomic64_xchg_acquire(...) \
222 __atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
223#endif
224
225#ifndef atomic64_xchg_release
226#define atomic64_xchg_release(...) \
227 __atomic_op_release(atomic64_xchg, __VA_ARGS__)
228#endif
229
230#ifndef atomic64_xchg
231#define atomic64_xchg(...) \
232 __atomic_op_fence(atomic64_xchg, __VA_ARGS__)
233#endif
234#endif /* atomic64_xchg_relaxed */
235
236/* atomic64_cmpxchg_relaxed */
237#ifndef atomic64_cmpxchg_relaxed
238#define atomic64_cmpxchg_relaxed atomic64_cmpxchg
239#define atomic64_cmpxchg_acquire atomic64_cmpxchg
240#define atomic64_cmpxchg_release atomic64_cmpxchg
241
242#else /* atomic64_cmpxchg_relaxed */
243
244#ifndef atomic64_cmpxchg_acquire
245#define atomic64_cmpxchg_acquire(...) \
246 __atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
247#endif
248
249#ifndef atomic64_cmpxchg_release
250#define atomic64_cmpxchg_release(...) \
251 __atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
252#endif
253
254#ifndef atomic64_cmpxchg
255#define atomic64_cmpxchg(...) \
256 __atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
257#endif
258#endif /* atomic64_cmpxchg_relaxed */
259
260/* cmpxchg_relaxed */
261#ifndef cmpxchg_relaxed
262#define cmpxchg_relaxed cmpxchg
263#define cmpxchg_acquire cmpxchg
264#define cmpxchg_release cmpxchg
265
266#else /* cmpxchg_relaxed */
267
268#ifndef cmpxchg_acquire
269#define cmpxchg_acquire(...) \
270 __atomic_op_acquire(cmpxchg, __VA_ARGS__)
271#endif
272
273#ifndef cmpxchg_release
274#define cmpxchg_release(...) \
275 __atomic_op_release(cmpxchg, __VA_ARGS__)
276#endif
277
278#ifndef cmpxchg
279#define cmpxchg(...) \
280 __atomic_op_fence(cmpxchg, __VA_ARGS__)
281#endif
282#endif /* cmpxchg_relaxed */
283
284/* cmpxchg64_relaxed */
285#ifndef cmpxchg64_relaxed
286#define cmpxchg64_relaxed cmpxchg64
287#define cmpxchg64_acquire cmpxchg64
288#define cmpxchg64_release cmpxchg64
289
290#else /* cmpxchg64_relaxed */
291
292#ifndef cmpxchg64_acquire
293#define cmpxchg64_acquire(...) \
294 __atomic_op_acquire(cmpxchg64, __VA_ARGS__)
295#endif
296
297#ifndef cmpxchg64_release
298#define cmpxchg64_release(...) \
299 __atomic_op_release(cmpxchg64, __VA_ARGS__)
300#endif
301
302#ifndef cmpxchg64
303#define cmpxchg64(...) \
304 __atomic_op_fence(cmpxchg64, __VA_ARGS__)
305#endif
306#endif /* cmpxchg64_relaxed */
307
308/* xchg_relaxed */
309#ifndef xchg_relaxed
310#define xchg_relaxed xchg
311#define xchg_acquire xchg
312#define xchg_release xchg
313
314#else /* xchg_relaxed */
315
316#ifndef xchg_acquire
317#define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__)
318#endif
319
320#ifndef xchg_release
321#define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__)
322#endif
323
324#ifndef xchg
325#define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__)
326#endif
327#endif /* xchg_relaxed */
328
329/**
330 * atomic_add_unless - add unless the number is already a given value
331 * @v: pointer of type atomic_t
332 * @a: the amount to add to v...
333 * @u: ...unless v is equal to u.
334 *
335 * Atomically adds @a to @v, so long as @v was not already @u.
336 * Returns non-zero if @v was not @u, and zero otherwise.
337 */
338static inline int atomic_add_unless(atomic_t *v, int a, int u)
339{
340 return __atomic_add_unless(v, a, u) != u;
341}
342
343/**
344 * atomic_inc_not_zero - increment unless the number is zero
345 * @v: pointer of type atomic_t
346 *
347 * Atomically increments @v by 1, so long as @v is non-zero.
348 * Returns non-zero if @v was non-zero, and zero otherwise.
349 */
350#ifndef atomic_inc_not_zero
351#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
352#endif
353
354#ifndef atomic_andnot
355static inline void atomic_andnot(int i, atomic_t *v)
356{
357 atomic_and(~i, v);
358}
359#endif
360
361static inline __deprecated void atomic_clear_mask(unsigned int mask, atomic_t *v)
362{
363 atomic_andnot(mask, v);
364}
365
366static inline __deprecated void atomic_set_mask(unsigned int mask, atomic_t *v)
367{
368 atomic_or(mask, v);
369}
370
371/**
372 * atomic_inc_not_zero_hint - increment if not null
373 * @v: pointer of type atomic_t
374 * @hint: probable value of the atomic before the increment
375 *
376 * This version of atomic_inc_not_zero() gives a hint of probable
377 * value of the atomic. This helps processor to not read the memory
378 * before doing the atomic read/modify/write cycle, lowering
379 * number of bus transactions on some arches.
380 *
381 * Returns: 0 if increment was not done, 1 otherwise.
382 */
383#ifndef atomic_inc_not_zero_hint
384static inline int atomic_inc_not_zero_hint(atomic_t *v, int hint)
385{
386 int val, c = hint;
387
388 /* sanity test, should be removed by compiler if hint is a constant */
389 if (!hint)
390 return atomic_inc_not_zero(v);
391
392 do {
393 val = atomic_cmpxchg(v, c, c + 1);
394 if (val == c)
395 return 1;
396 c = val;
397 } while (c);
398
399 return 0;
400}
401#endif
402
403#ifndef atomic_inc_unless_negative
404static inline int atomic_inc_unless_negative(atomic_t *p)
405{
406 int v, v1;
407 for (v = 0; v >= 0; v = v1) {
408 v1 = atomic_cmpxchg(p, v, v + 1);
409 if (likely(v1 == v))
410 return 1;
411 }
412 return 0;
413}
414#endif
415
416#ifndef atomic_dec_unless_positive
417static inline int atomic_dec_unless_positive(atomic_t *p)
418{
419 int v, v1;
420 for (v = 0; v <= 0; v = v1) {
421 v1 = atomic_cmpxchg(p, v, v - 1);
422 if (likely(v1 == v))
423 return 1;
424 }
425 return 0;
426}
427#endif
428
429/*
430 * atomic_dec_if_positive - decrement by 1 if old value positive
431 * @v: pointer of type atomic_t
432 *
433 * The function returns the old value of *v minus 1, even if
434 * the atomic variable, v, was not decremented.
435 */
436#ifndef atomic_dec_if_positive
437static inline int atomic_dec_if_positive(atomic_t *v)
438{
439 int c, old, dec;
440 c = atomic_read(v);
441 for (;;) {
442 dec = c - 1;
443 if (unlikely(dec < 0))
444 break;
445 old = atomic_cmpxchg((v), c, dec);
446 if (likely(old == c))
447 break;
448 c = old;
449 }
450 return dec;
451}
452#endif
453
454#include <asm-generic/atomic-long.h>
455#ifdef CONFIG_GENERIC_ATOMIC64
456#include <asm-generic/atomic64.h>
457#endif
458
459#ifndef atomic64_andnot
460static inline void atomic64_andnot(long long i, atomic64_t *v)
461{
462 atomic64_and(~i, v);
463}
464#endif
465
466#endif /* _LINUX_ATOMIC_H */