tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
Merge branch 'locking/atomics' into locking/core, to pick up WIP commits
Signed-off-by: Ingo Molnar <mingo@kernel.org>
+6036
-1954
+16
-2
Kbuild
+16
-2
Kbuild
···
6
6
# 2) Generate timeconst.h
7
7
# 3) Generate asm-offsets.h (may need bounds.h and timeconst.h)
8
8
# 4) Check for missing system calls
9
-
# 5) Generate constants.py (may need bounds.h)
9
+
# 5) check atomics headers are up-to-date
10
+
# 6) Generate constants.py (may need bounds.h)
10
11
11
12
#####
12
13
# 1) Generate bounds.h
···
60
59
$(call cmd,syscalls)
61
60
62
61
#####
63
-
# 5) Generate constants for Python GDB integration
62
+
# 5) Check atomic headers are up-to-date
63
+
#
64
+
65
+
always += old-atomics
66
+
targets += old-atomics
67
+
68
+
quiet_cmd_atomics = CALL $<
69
+
cmd_atomics = $(CONFIG_SHELL) $<
70
+
71
+
old-atomics: scripts/atomic/check-atomics.sh FORCE
72
+
$(call cmd,atomics)
73
+
74
+
#####
75
+
# 6) Generate constants for Python GDB integration
64
76
#
65
77
66
78
extra-$(CONFIG_GDB_SCRIPTS) += build_constants_py
+1
MAINTAINERS
+1
MAINTAINERS
+99
-92
arch/arm64/include/asm/atomic.h
+99
-92
arch/arm64/include/asm/atomic.h
···
42
42
43
43
#define ATOMIC_INIT(i) { (i) }
44
44
45
-
#define atomic_read(v) READ_ONCE((v)->counter)
46
-
#define atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
45
+
#define arch_atomic_read(v) READ_ONCE((v)->counter)
46
+
#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i))
47
47
48
-
#define atomic_add_return_relaxed atomic_add_return_relaxed
49
-
#define atomic_add_return_acquire atomic_add_return_acquire
50
-
#define atomic_add_return_release atomic_add_return_release
51
-
#define atomic_add_return atomic_add_return
48
+
#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
49
+
#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire
50
+
#define arch_atomic_add_return_release arch_atomic_add_return_release
51
+
#define arch_atomic_add_return arch_atomic_add_return
52
52
53
-
#define atomic_sub_return_relaxed atomic_sub_return_relaxed
54
-
#define atomic_sub_return_acquire atomic_sub_return_acquire
55
-
#define atomic_sub_return_release atomic_sub_return_release
56
-
#define atomic_sub_return atomic_sub_return
53
+
#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
54
+
#define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire
55
+
#define arch_atomic_sub_return_release arch_atomic_sub_return_release
56
+
#define arch_atomic_sub_return arch_atomic_sub_return
57
57
58
-
#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
59
-
#define atomic_fetch_add_acquire atomic_fetch_add_acquire
60
-
#define atomic_fetch_add_release atomic_fetch_add_release
61
-
#define atomic_fetch_add atomic_fetch_add
58
+
#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
59
+
#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire
60
+
#define arch_atomic_fetch_add_release arch_atomic_fetch_add_release
61
+
#define arch_atomic_fetch_add arch_atomic_fetch_add
62
62
63
-
#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
64
-
#define atomic_fetch_sub_acquire atomic_fetch_sub_acquire
65
-
#define atomic_fetch_sub_release atomic_fetch_sub_release
66
-
#define atomic_fetch_sub atomic_fetch_sub
63
+
#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
64
+
#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire
65
+
#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release
66
+
#define arch_atomic_fetch_sub arch_atomic_fetch_sub
67
67
68
-
#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
69
-
#define atomic_fetch_and_acquire atomic_fetch_and_acquire
70
-
#define atomic_fetch_and_release atomic_fetch_and_release
71
-
#define atomic_fetch_and atomic_fetch_and
68
+
#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
69
+
#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire
70
+
#define arch_atomic_fetch_and_release arch_atomic_fetch_and_release
71
+
#define arch_atomic_fetch_and arch_atomic_fetch_and
72
72
73
-
#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed
74
-
#define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire
75
-
#define atomic_fetch_andnot_release atomic_fetch_andnot_release
76
-
#define atomic_fetch_andnot atomic_fetch_andnot
73
+
#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed
74
+
#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire
75
+
#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release
76
+
#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot
77
77
78
-
#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed
79
-
#define atomic_fetch_or_acquire atomic_fetch_or_acquire
80
-
#define atomic_fetch_or_release atomic_fetch_or_release
81
-
#define atomic_fetch_or atomic_fetch_or
78
+
#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
79
+
#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire
80
+
#define arch_atomic_fetch_or_release arch_atomic_fetch_or_release
81
+
#define arch_atomic_fetch_or arch_atomic_fetch_or
82
82
83
-
#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
84
-
#define atomic_fetch_xor_acquire atomic_fetch_xor_acquire
85
-
#define atomic_fetch_xor_release atomic_fetch_xor_release
86
-
#define atomic_fetch_xor atomic_fetch_xor
83
+
#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
84
+
#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire
85
+
#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release
86
+
#define arch_atomic_fetch_xor arch_atomic_fetch_xor
87
87
88
-
#define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
89
-
#define atomic_xchg_acquire(v, new) xchg_acquire(&((v)->counter), (new))
90
-
#define atomic_xchg_release(v, new) xchg_release(&((v)->counter), (new))
91
-
#define atomic_xchg(v, new) xchg(&((v)->counter), (new))
88
+
#define arch_atomic_xchg_relaxed(v, new) \
89
+
arch_xchg_relaxed(&((v)->counter), (new))
90
+
#define arch_atomic_xchg_acquire(v, new) \
91
+
arch_xchg_acquire(&((v)->counter), (new))
92
+
#define arch_atomic_xchg_release(v, new) \
93
+
arch_xchg_release(&((v)->counter), (new))
94
+
#define arch_atomic_xchg(v, new) \
95
+
arch_xchg(&((v)->counter), (new))
92
96
93
-
#define atomic_cmpxchg_relaxed(v, old, new) \
94
-
cmpxchg_relaxed(&((v)->counter), (old), (new))
95
-
#define atomic_cmpxchg_acquire(v, old, new) \
96
-
cmpxchg_acquire(&((v)->counter), (old), (new))
97
-
#define atomic_cmpxchg_release(v, old, new) \
98
-
cmpxchg_release(&((v)->counter), (old), (new))
99
-
#define atomic_cmpxchg(v, old, new) cmpxchg(&((v)->counter), (old), (new))
97
+
#define arch_atomic_cmpxchg_relaxed(v, old, new) \
98
+
arch_cmpxchg_relaxed(&((v)->counter), (old), (new))
99
+
#define arch_atomic_cmpxchg_acquire(v, old, new) \
100
+
arch_cmpxchg_acquire(&((v)->counter), (old), (new))
101
+
#define arch_atomic_cmpxchg_release(v, old, new) \
102
+
arch_cmpxchg_release(&((v)->counter), (old), (new))
103
+
#define arch_atomic_cmpxchg(v, old, new) \
104
+
arch_cmpxchg(&((v)->counter), (old), (new))
100
105
101
-
#define atomic_andnot atomic_andnot
106
+
#define arch_atomic_andnot arch_atomic_andnot
102
107
103
108
/*
104
-
* 64-bit atomic operations.
109
+
* 64-bit arch_atomic operations.
105
110
*/
106
-
#define ATOMIC64_INIT ATOMIC_INIT
107
-
#define atomic64_read atomic_read
108
-
#define atomic64_set atomic_set
111
+
#define ATOMIC64_INIT ATOMIC_INIT
112
+
#define arch_atomic64_read arch_atomic_read
113
+
#define arch_atomic64_set arch_atomic_set
109
114
110
-
#define atomic64_add_return_relaxed atomic64_add_return_relaxed
111
-
#define atomic64_add_return_acquire atomic64_add_return_acquire
112
-
#define atomic64_add_return_release atomic64_add_return_release
113
-
#define atomic64_add_return atomic64_add_return
115
+
#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
116
+
#define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire
117
+
#define arch_atomic64_add_return_release arch_atomic64_add_return_release
118
+
#define arch_atomic64_add_return arch_atomic64_add_return
114
119
115
-
#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
116
-
#define atomic64_sub_return_acquire atomic64_sub_return_acquire
117
-
#define atomic64_sub_return_release atomic64_sub_return_release
118
-
#define atomic64_sub_return atomic64_sub_return
120
+
#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
121
+
#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire
122
+
#define arch_atomic64_sub_return_release arch_atomic64_sub_return_release
123
+
#define arch_atomic64_sub_return arch_atomic64_sub_return
119
124
120
-
#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
121
-
#define atomic64_fetch_add_acquire atomic64_fetch_add_acquire
122
-
#define atomic64_fetch_add_release atomic64_fetch_add_release
123
-
#define atomic64_fetch_add atomic64_fetch_add
125
+
#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
126
+
#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire
127
+
#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release
128
+
#define arch_atomic64_fetch_add arch_atomic64_fetch_add
124
129
125
-
#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
126
-
#define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire
127
-
#define atomic64_fetch_sub_release atomic64_fetch_sub_release
128
-
#define atomic64_fetch_sub atomic64_fetch_sub
130
+
#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
131
+
#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire
132
+
#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release
133
+
#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
129
134
130
-
#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
131
-
#define atomic64_fetch_and_acquire atomic64_fetch_and_acquire
132
-
#define atomic64_fetch_and_release atomic64_fetch_and_release
133
-
#define atomic64_fetch_and atomic64_fetch_and
135
+
#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
136
+
#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire
137
+
#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release
138
+
#define arch_atomic64_fetch_and arch_atomic64_fetch_and
134
139
135
-
#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed
136
-
#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire
137
-
#define atomic64_fetch_andnot_release atomic64_fetch_andnot_release
138
-
#define atomic64_fetch_andnot atomic64_fetch_andnot
140
+
#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed
141
+
#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire
142
+
#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release
143
+
#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot
139
144
140
-
#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed
141
-
#define atomic64_fetch_or_acquire atomic64_fetch_or_acquire
142
-
#define atomic64_fetch_or_release atomic64_fetch_or_release
143
-
#define atomic64_fetch_or atomic64_fetch_or
145
+
#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed
146
+
#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire
147
+
#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release
148
+
#define arch_atomic64_fetch_or arch_atomic64_fetch_or
144
149
145
-
#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
146
-
#define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire
147
-
#define atomic64_fetch_xor_release atomic64_fetch_xor_release
148
-
#define atomic64_fetch_xor atomic64_fetch_xor
150
+
#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
151
+
#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire
152
+
#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release
153
+
#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
149
154
150
-
#define atomic64_xchg_relaxed atomic_xchg_relaxed
151
-
#define atomic64_xchg_acquire atomic_xchg_acquire
152
-
#define atomic64_xchg_release atomic_xchg_release
153
-
#define atomic64_xchg atomic_xchg
155
+
#define arch_atomic64_xchg_relaxed arch_atomic_xchg_relaxed
156
+
#define arch_atomic64_xchg_acquire arch_atomic_xchg_acquire
157
+
#define arch_atomic64_xchg_release arch_atomic_xchg_release
158
+
#define arch_atomic64_xchg arch_atomic_xchg
154
159
155
-
#define atomic64_cmpxchg_relaxed atomic_cmpxchg_relaxed
156
-
#define atomic64_cmpxchg_acquire atomic_cmpxchg_acquire
157
-
#define atomic64_cmpxchg_release atomic_cmpxchg_release
158
-
#define atomic64_cmpxchg atomic_cmpxchg
160
+
#define arch_atomic64_cmpxchg_relaxed arch_atomic_cmpxchg_relaxed
161
+
#define arch_atomic64_cmpxchg_acquire arch_atomic_cmpxchg_acquire
162
+
#define arch_atomic64_cmpxchg_release arch_atomic_cmpxchg_release
163
+
#define arch_atomic64_cmpxchg arch_atomic_cmpxchg
159
164
160
-
#define atomic64_andnot atomic64_andnot
165
+
#define arch_atomic64_andnot arch_atomic64_andnot
161
166
162
-
#define atomic64_dec_if_positive atomic64_dec_if_positive
167
+
#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive
168
+
169
+
#include <asm-generic/atomic-instrumented.h>
163
170
164
171
#endif
165
172
#endif
+14
-14
arch/arm64/include/asm/atomic_ll_sc.h
+14
-14
arch/arm64/include/asm/atomic_ll_sc.h
···
39
39
40
40
#define ATOMIC_OP(op, asm_op) \
41
41
__LL_SC_INLINE void \
42
-
__LL_SC_PREFIX(atomic_##op(int i, atomic_t *v)) \
42
+
__LL_SC_PREFIX(arch_atomic_##op(int i, atomic_t *v)) \
43
43
{ \
44
44
unsigned long tmp; \
45
45
int result; \
···
53
53
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
54
54
: "Ir" (i)); \
55
55
} \
56
-
__LL_SC_EXPORT(atomic_##op);
56
+
__LL_SC_EXPORT(arch_atomic_##op);
57
57
58
58
#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \
59
59
__LL_SC_INLINE int \
60
-
__LL_SC_PREFIX(atomic_##op##_return##name(int i, atomic_t *v)) \
60
+
__LL_SC_PREFIX(arch_atomic_##op##_return##name(int i, atomic_t *v)) \
61
61
{ \
62
62
unsigned long tmp; \
63
63
int result; \
···
75
75
\
76
76
return result; \
77
77
} \
78
-
__LL_SC_EXPORT(atomic_##op##_return##name);
78
+
__LL_SC_EXPORT(arch_atomic_##op##_return##name);
79
79
80
80
#define ATOMIC_FETCH_OP(name, mb, acq, rel, cl, op, asm_op) \
81
81
__LL_SC_INLINE int \
82
-
__LL_SC_PREFIX(atomic_fetch_##op##name(int i, atomic_t *v)) \
82
+
__LL_SC_PREFIX(arch_atomic_fetch_##op##name(int i, atomic_t *v)) \
83
83
{ \
84
84
unsigned long tmp; \
85
85
int val, result; \
···
97
97
\
98
98
return result; \
99
99
} \
100
-
__LL_SC_EXPORT(atomic_fetch_##op##name);
100
+
__LL_SC_EXPORT(arch_atomic_fetch_##op##name);
101
101
102
102
#define ATOMIC_OPS(...) \
103
103
ATOMIC_OP(__VA_ARGS__) \
···
133
133
134
134
#define ATOMIC64_OP(op, asm_op) \
135
135
__LL_SC_INLINE void \
136
-
__LL_SC_PREFIX(atomic64_##op(long i, atomic64_t *v)) \
136
+
__LL_SC_PREFIX(arch_atomic64_##op(long i, atomic64_t *v)) \
137
137
{ \
138
138
long result; \
139
139
unsigned long tmp; \
···
147
147
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
148
148
: "Ir" (i)); \
149
149
} \
150
-
__LL_SC_EXPORT(atomic64_##op);
150
+
__LL_SC_EXPORT(arch_atomic64_##op);
151
151
152
152
#define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \
153
153
__LL_SC_INLINE long \
154
-
__LL_SC_PREFIX(atomic64_##op##_return##name(long i, atomic64_t *v)) \
154
+
__LL_SC_PREFIX(arch_atomic64_##op##_return##name(long i, atomic64_t *v))\
155
155
{ \
156
156
long result; \
157
157
unsigned long tmp; \
···
169
169
\
170
170
return result; \
171
171
} \
172
-
__LL_SC_EXPORT(atomic64_##op##_return##name);
172
+
__LL_SC_EXPORT(arch_atomic64_##op##_return##name);
173
173
174
174
#define ATOMIC64_FETCH_OP(name, mb, acq, rel, cl, op, asm_op) \
175
175
__LL_SC_INLINE long \
176
-
__LL_SC_PREFIX(atomic64_fetch_##op##name(long i, atomic64_t *v)) \
176
+
__LL_SC_PREFIX(arch_atomic64_fetch_##op##name(long i, atomic64_t *v)) \
177
177
{ \
178
178
long result, val; \
179
179
unsigned long tmp; \
···
191
191
\
192
192
return result; \
193
193
} \
194
-
__LL_SC_EXPORT(atomic64_fetch_##op##name);
194
+
__LL_SC_EXPORT(arch_atomic64_fetch_##op##name);
195
195
196
196
#define ATOMIC64_OPS(...) \
197
197
ATOMIC64_OP(__VA_ARGS__) \
···
226
226
#undef ATOMIC64_OP
227
227
228
228
__LL_SC_INLINE long
229
-
__LL_SC_PREFIX(atomic64_dec_if_positive(atomic64_t *v))
229
+
__LL_SC_PREFIX(arch_atomic64_dec_if_positive(atomic64_t *v))
230
230
{
231
231
long result;
232
232
unsigned long tmp;
···
246
246
247
247
return result;
248
248
}
249
-
__LL_SC_EXPORT(atomic64_dec_if_positive);
249
+
__LL_SC_EXPORT(arch_atomic64_dec_if_positive);
250
250
251
251
#define __CMPXCHG_CASE(w, sfx, name, sz, mb, acq, rel, cl) \
252
252
__LL_SC_INLINE u##sz \
+19
-19
arch/arm64/include/asm/atomic_lse.h
+19
-19
arch/arm64/include/asm/atomic_lse.h
···
25
25
#error "please don't include this file directly"
26
26
#endif
27
27
28
-
#define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op)
28
+
#define __LL_SC_ATOMIC(op) __LL_SC_CALL(arch_atomic_##op)
29
29
#define ATOMIC_OP(op, asm_op) \
30
-
static inline void atomic_##op(int i, atomic_t *v) \
30
+
static inline void arch_atomic_##op(int i, atomic_t *v) \
31
31
{ \
32
32
register int w0 asm ("w0") = i; \
33
33
register atomic_t *x1 asm ("x1") = v; \
···
47
47
#undef ATOMIC_OP
48
48
49
49
#define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...) \
50
-
static inline int atomic_fetch_##op##name(int i, atomic_t *v) \
50
+
static inline int arch_atomic_fetch_##op##name(int i, atomic_t *v) \
51
51
{ \
52
52
register int w0 asm ("w0") = i; \
53
53
register atomic_t *x1 asm ("x1") = v; \
···
79
79
#undef ATOMIC_FETCH_OPS
80
80
81
81
#define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \
82
-
static inline int atomic_add_return##name(int i, atomic_t *v) \
82
+
static inline int arch_atomic_add_return##name(int i, atomic_t *v) \
83
83
{ \
84
84
register int w0 asm ("w0") = i; \
85
85
register atomic_t *x1 asm ("x1") = v; \
···
105
105
106
106
#undef ATOMIC_OP_ADD_RETURN
107
107
108
-
static inline void atomic_and(int i, atomic_t *v)
108
+
static inline void arch_atomic_and(int i, atomic_t *v)
109
109
{
110
110
register int w0 asm ("w0") = i;
111
111
register atomic_t *x1 asm ("x1") = v;
···
123
123
}
124
124
125
125
#define ATOMIC_FETCH_OP_AND(name, mb, cl...) \
126
-
static inline int atomic_fetch_and##name(int i, atomic_t *v) \
126
+
static inline int arch_atomic_fetch_and##name(int i, atomic_t *v) \
127
127
{ \
128
128
register int w0 asm ("w0") = i; \
129
129
register atomic_t *x1 asm ("x1") = v; \
···
149
149
150
150
#undef ATOMIC_FETCH_OP_AND
151
151
152
-
static inline void atomic_sub(int i, atomic_t *v)
152
+
static inline void arch_atomic_sub(int i, atomic_t *v)
153
153
{
154
154
register int w0 asm ("w0") = i;
155
155
register atomic_t *x1 asm ("x1") = v;
···
167
167
}
168
168
169
169
#define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \
170
-
static inline int atomic_sub_return##name(int i, atomic_t *v) \
170
+
static inline int arch_atomic_sub_return##name(int i, atomic_t *v) \
171
171
{ \
172
172
register int w0 asm ("w0") = i; \
173
173
register atomic_t *x1 asm ("x1") = v; \
···
195
195
#undef ATOMIC_OP_SUB_RETURN
196
196
197
197
#define ATOMIC_FETCH_OP_SUB(name, mb, cl...) \
198
-
static inline int atomic_fetch_sub##name(int i, atomic_t *v) \
198
+
static inline int arch_atomic_fetch_sub##name(int i, atomic_t *v) \
199
199
{ \
200
200
register int w0 asm ("w0") = i; \
201
201
register atomic_t *x1 asm ("x1") = v; \
···
222
222
#undef ATOMIC_FETCH_OP_SUB
223
223
#undef __LL_SC_ATOMIC
224
224
225
-
#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op)
225
+
#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(arch_atomic64_##op)
226
226
#define ATOMIC64_OP(op, asm_op) \
227
-
static inline void atomic64_##op(long i, atomic64_t *v) \
227
+
static inline void arch_atomic64_##op(long i, atomic64_t *v) \
228
228
{ \
229
229
register long x0 asm ("x0") = i; \
230
230
register atomic64_t *x1 asm ("x1") = v; \
···
244
244
#undef ATOMIC64_OP
245
245
246
246
#define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...) \
247
-
static inline long atomic64_fetch_##op##name(long i, atomic64_t *v) \
247
+
static inline long arch_atomic64_fetch_##op##name(long i, atomic64_t *v)\
248
248
{ \
249
249
register long x0 asm ("x0") = i; \
250
250
register atomic64_t *x1 asm ("x1") = v; \
···
276
276
#undef ATOMIC64_FETCH_OPS
277
277
278
278
#define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \
279
-
static inline long atomic64_add_return##name(long i, atomic64_t *v) \
279
+
static inline long arch_atomic64_add_return##name(long i, atomic64_t *v)\
280
280
{ \
281
281
register long x0 asm ("x0") = i; \
282
282
register atomic64_t *x1 asm ("x1") = v; \
···
302
302
303
303
#undef ATOMIC64_OP_ADD_RETURN
304
304
305
-
static inline void atomic64_and(long i, atomic64_t *v)
305
+
static inline void arch_atomic64_and(long i, atomic64_t *v)
306
306
{
307
307
register long x0 asm ("x0") = i;
308
308
register atomic64_t *x1 asm ("x1") = v;
···
320
320
}
321
321
322
322
#define ATOMIC64_FETCH_OP_AND(name, mb, cl...) \
323
-
static inline long atomic64_fetch_and##name(long i, atomic64_t *v) \
323
+
static inline long arch_atomic64_fetch_and##name(long i, atomic64_t *v) \
324
324
{ \
325
325
register long x0 asm ("x0") = i; \
326
326
register atomic64_t *x1 asm ("x1") = v; \
···
346
346
347
347
#undef ATOMIC64_FETCH_OP_AND
348
348
349
-
static inline void atomic64_sub(long i, atomic64_t *v)
349
+
static inline void arch_atomic64_sub(long i, atomic64_t *v)
350
350
{
351
351
register long x0 asm ("x0") = i;
352
352
register atomic64_t *x1 asm ("x1") = v;
···
364
364
}
365
365
366
366
#define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \
367
-
static inline long atomic64_sub_return##name(long i, atomic64_t *v) \
367
+
static inline long arch_atomic64_sub_return##name(long i, atomic64_t *v)\
368
368
{ \
369
369
register long x0 asm ("x0") = i; \
370
370
register atomic64_t *x1 asm ("x1") = v; \
···
392
392
#undef ATOMIC64_OP_SUB_RETURN
393
393
394
394
#define ATOMIC64_FETCH_OP_SUB(name, mb, cl...) \
395
-
static inline long atomic64_fetch_sub##name(long i, atomic64_t *v) \
395
+
static inline long arch_atomic64_fetch_sub##name(long i, atomic64_t *v) \
396
396
{ \
397
397
register long x0 asm ("x0") = i; \
398
398
register atomic64_t *x1 asm ("x1") = v; \
···
418
418
419
419
#undef ATOMIC64_FETCH_OP_SUB
420
420
421
-
static inline long atomic64_dec_if_positive(atomic64_t *v)
421
+
static inline long arch_atomic64_dec_if_positive(atomic64_t *v)
422
422
{
423
423
register long x0 asm ("x0") = (long)v;
424
424
+30
-30
arch/arm64/include/asm/cmpxchg.h
+30
-30
arch/arm64/include/asm/cmpxchg.h
···
110
110
})
111
111
112
112
/* xchg */
113
-
#define xchg_relaxed(...) __xchg_wrapper( , __VA_ARGS__)
114
-
#define xchg_acquire(...) __xchg_wrapper(_acq, __VA_ARGS__)
115
-
#define xchg_release(...) __xchg_wrapper(_rel, __VA_ARGS__)
116
-
#define xchg(...) __xchg_wrapper( _mb, __VA_ARGS__)
113
+
#define arch_xchg_relaxed(...) __xchg_wrapper( , __VA_ARGS__)
114
+
#define arch_xchg_acquire(...) __xchg_wrapper(_acq, __VA_ARGS__)
115
+
#define arch_xchg_release(...) __xchg_wrapper(_rel, __VA_ARGS__)
116
+
#define arch_xchg(...) __xchg_wrapper( _mb, __VA_ARGS__)
117
117
118
118
#define __CMPXCHG_GEN(sfx) \
119
119
static inline unsigned long __cmpxchg##sfx(volatile void *ptr, \
···
154
154
})
155
155
156
156
/* cmpxchg */
157
-
#define cmpxchg_relaxed(...) __cmpxchg_wrapper( , __VA_ARGS__)
158
-
#define cmpxchg_acquire(...) __cmpxchg_wrapper(_acq, __VA_ARGS__)
159
-
#define cmpxchg_release(...) __cmpxchg_wrapper(_rel, __VA_ARGS__)
160
-
#define cmpxchg(...) __cmpxchg_wrapper( _mb, __VA_ARGS__)
161
-
#define cmpxchg_local cmpxchg_relaxed
157
+
#define arch_cmpxchg_relaxed(...) __cmpxchg_wrapper( , __VA_ARGS__)
158
+
#define arch_cmpxchg_acquire(...) __cmpxchg_wrapper(_acq, __VA_ARGS__)
159
+
#define arch_cmpxchg_release(...) __cmpxchg_wrapper(_rel, __VA_ARGS__)
160
+
#define arch_cmpxchg(...) __cmpxchg_wrapper( _mb, __VA_ARGS__)
161
+
#define arch_cmpxchg_local arch_cmpxchg_relaxed
162
162
163
163
/* cmpxchg64 */
164
-
#define cmpxchg64_relaxed cmpxchg_relaxed
165
-
#define cmpxchg64_acquire cmpxchg_acquire
166
-
#define cmpxchg64_release cmpxchg_release
167
-
#define cmpxchg64 cmpxchg
168
-
#define cmpxchg64_local cmpxchg_local
164
+
#define arch_cmpxchg64_relaxed arch_cmpxchg_relaxed
165
+
#define arch_cmpxchg64_acquire arch_cmpxchg_acquire
166
+
#define arch_cmpxchg64_release arch_cmpxchg_release
167
+
#define arch_cmpxchg64 arch_cmpxchg
168
+
#define arch_cmpxchg64_local arch_cmpxchg_local
169
169
170
170
/* cmpxchg_double */
171
171
#define system_has_cmpxchg_double() 1
···
177
177
VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1); \
178
178
})
179
179
180
-
#define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
181
-
({\
182
-
int __ret;\
183
-
__cmpxchg_double_check(ptr1, ptr2); \
184
-
__ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
185
-
(unsigned long)(n1), (unsigned long)(n2), \
186
-
ptr1); \
187
-
__ret; \
180
+
#define arch_cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
181
+
({ \
182
+
int __ret; \
183
+
__cmpxchg_double_check(ptr1, ptr2); \
184
+
__ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
185
+
(unsigned long)(n1), (unsigned long)(n2), \
186
+
ptr1); \
187
+
__ret; \
188
188
})
189
189
190
-
#define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
191
-
({\
192
-
int __ret;\
193
-
__cmpxchg_double_check(ptr1, ptr2); \
194
-
__ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
195
-
(unsigned long)(n1), (unsigned long)(n2), \
196
-
ptr1); \
197
-
__ret; \
190
+
#define arch_cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
191
+
({ \
192
+
int __ret; \
193
+
__cmpxchg_double_check(ptr1, ptr2); \
194
+
__ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
195
+
(unsigned long)(n1), (unsigned long)(n2), \
196
+
ptr1); \
197
+
__ret; \
198
198
})
199
199
200
200
#define __CMPWAIT_CASE(w, sfx, sz) \
+8
-8
arch/arm64/include/asm/sync_bitops.h
+8
-8
arch/arm64/include/asm/sync_bitops.h
···
15
15
* ops which are SMP safe even on a UP kernel.
16
16
*/
17
17
18
-
#define sync_set_bit(nr, p) set_bit(nr, p)
19
-
#define sync_clear_bit(nr, p) clear_bit(nr, p)
20
-
#define sync_change_bit(nr, p) change_bit(nr, p)
21
-
#define sync_test_and_set_bit(nr, p) test_and_set_bit(nr, p)
22
-
#define sync_test_and_clear_bit(nr, p) test_and_clear_bit(nr, p)
23
-
#define sync_test_and_change_bit(nr, p) test_and_change_bit(nr, p)
24
-
#define sync_test_bit(nr, addr) test_bit(nr, addr)
25
-
#define sync_cmpxchg cmpxchg
18
+
#define sync_set_bit(nr, p) set_bit(nr, p)
19
+
#define sync_clear_bit(nr, p) clear_bit(nr, p)
20
+
#define sync_change_bit(nr, p) change_bit(nr, p)
21
+
#define sync_test_and_set_bit(nr, p) test_and_set_bit(nr, p)
22
+
#define sync_test_and_clear_bit(nr, p) test_and_clear_bit(nr, p)
23
+
#define sync_test_and_change_bit(nr, p) test_and_change_bit(nr, p)
24
+
#define sync_test_bit(nr, addr) test_bit(nr, addr)
25
+
#define arch_sync_cmpxchg arch_cmpxchg
26
26
27
27
#endif
+1634
-314
include/asm-generic/atomic-instrumented.h
+1634
-314
include/asm-generic/atomic-instrumented.h
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
3
+
// Generated by scripts/atomic/gen-atomic-instrumented.sh
4
+
// DO NOT MODIFY THIS FILE DIRECTLY
5
+
1
6
/*
2
7
* This file provides wrappers with KASAN instrumentation for atomic operations.
3
8
* To use this functionality an arch's atomic.h file needs to define all
···
14
9
* arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
15
10
* double instrumentation.
16
11
*/
17
-
18
-
#ifndef _LINUX_ATOMIC_INSTRUMENTED_H
19
-
#define _LINUX_ATOMIC_INSTRUMENTED_H
12
+
#ifndef _ASM_GENERIC_ATOMIC_INSTRUMENTED_H
13
+
#define _ASM_GENERIC_ATOMIC_INSTRUMENTED_H
20
14
21
15
#include <linux/build_bug.h>
22
16
#include <linux/kasan-checks.h>
23
17
24
-
static __always_inline int atomic_read(const atomic_t *v)
18
+
static inline int
19
+
atomic_read(const atomic_t *v)
25
20
{
26
21
kasan_check_read(v, sizeof(*v));
27
22
return arch_atomic_read(v);
28
23
}
24
+
#define atomic_read atomic_read
29
25
30
-
static __always_inline s64 atomic64_read(const atomic64_t *v)
26
+
#if defined(arch_atomic_read_acquire)
27
+
static inline int
28
+
atomic_read_acquire(const atomic_t *v)
31
29
{
32
30
kasan_check_read(v, sizeof(*v));
33
-
return arch_atomic64_read(v);
31
+
return arch_atomic_read_acquire(v);
34
32
}
33
+
#define atomic_read_acquire atomic_read_acquire
34
+
#endif
35
35
36
-
static __always_inline void atomic_set(atomic_t *v, int i)
36
+
static inline void
37
+
atomic_set(atomic_t *v, int i)
37
38
{
38
39
kasan_check_write(v, sizeof(*v));
39
40
arch_atomic_set(v, i);
40
41
}
42
+
#define atomic_set atomic_set
41
43
42
-
static __always_inline void atomic64_set(atomic64_t *v, s64 i)
44
+
#if defined(arch_atomic_set_release)
45
+
static inline void
46
+
atomic_set_release(atomic_t *v, int i)
43
47
{
44
48
kasan_check_write(v, sizeof(*v));
45
-
arch_atomic64_set(v, i);
49
+
arch_atomic_set_release(v, i);
46
50
}
47
-
48
-
static __always_inline int atomic_xchg(atomic_t *v, int i)
49
-
{
50
-
kasan_check_write(v, sizeof(*v));
51
-
return arch_atomic_xchg(v, i);
52
-
}
53
-
54
-
static __always_inline s64 atomic64_xchg(atomic64_t *v, s64 i)
55
-
{
56
-
kasan_check_write(v, sizeof(*v));
57
-
return arch_atomic64_xchg(v, i);
58
-
}
59
-
60
-
static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new)
61
-
{
62
-
kasan_check_write(v, sizeof(*v));
63
-
return arch_atomic_cmpxchg(v, old, new);
64
-
}
65
-
66
-
static __always_inline s64 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
67
-
{
68
-
kasan_check_write(v, sizeof(*v));
69
-
return arch_atomic64_cmpxchg(v, old, new);
70
-
}
71
-
72
-
#ifdef arch_atomic_try_cmpxchg
73
-
#define atomic_try_cmpxchg atomic_try_cmpxchg
74
-
static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new)
75
-
{
76
-
kasan_check_write(v, sizeof(*v));
77
-
kasan_check_read(old, sizeof(*old));
78
-
return arch_atomic_try_cmpxchg(v, old, new);
79
-
}
51
+
#define atomic_set_release atomic_set_release
80
52
#endif
81
53
82
-
#ifdef arch_atomic64_try_cmpxchg
83
-
#define atomic64_try_cmpxchg atomic64_try_cmpxchg
84
-
static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
85
-
{
86
-
kasan_check_write(v, sizeof(*v));
87
-
kasan_check_read(old, sizeof(*old));
88
-
return arch_atomic64_try_cmpxchg(v, old, new);
89
-
}
90
-
#endif
91
-
92
-
#ifdef arch_atomic_fetch_add_unless
93
-
#define atomic_fetch_add_unless atomic_fetch_add_unless
94
-
static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)
95
-
{
96
-
kasan_check_write(v, sizeof(*v));
97
-
return arch_atomic_fetch_add_unless(v, a, u);
98
-
}
99
-
#endif
100
-
101
-
#ifdef arch_atomic64_fetch_add_unless
102
-
#define atomic64_fetch_add_unless atomic64_fetch_add_unless
103
-
static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
104
-
{
105
-
kasan_check_write(v, sizeof(*v));
106
-
return arch_atomic64_fetch_add_unless(v, a, u);
107
-
}
108
-
#endif
109
-
110
-
#ifdef arch_atomic_inc
111
-
#define atomic_inc atomic_inc
112
-
static __always_inline void atomic_inc(atomic_t *v)
113
-
{
114
-
kasan_check_write(v, sizeof(*v));
115
-
arch_atomic_inc(v);
116
-
}
117
-
#endif
118
-
119
-
#ifdef arch_atomic64_inc
120
-
#define atomic64_inc atomic64_inc
121
-
static __always_inline void atomic64_inc(atomic64_t *v)
122
-
{
123
-
kasan_check_write(v, sizeof(*v));
124
-
arch_atomic64_inc(v);
125
-
}
126
-
#endif
127
-
128
-
#ifdef arch_atomic_dec
129
-
#define atomic_dec atomic_dec
130
-
static __always_inline void atomic_dec(atomic_t *v)
131
-
{
132
-
kasan_check_write(v, sizeof(*v));
133
-
arch_atomic_dec(v);
134
-
}
135
-
#endif
136
-
137
-
#ifdef atch_atomic64_dec
138
-
#define atomic64_dec
139
-
static __always_inline void atomic64_dec(atomic64_t *v)
140
-
{
141
-
kasan_check_write(v, sizeof(*v));
142
-
arch_atomic64_dec(v);
143
-
}
144
-
#endif
145
-
146
-
static __always_inline void atomic_add(int i, atomic_t *v)
54
+
static inline void
55
+
atomic_add(int i, atomic_t *v)
147
56
{
148
57
kasan_check_write(v, sizeof(*v));
149
58
arch_atomic_add(i, v);
150
59
}
60
+
#define atomic_add atomic_add
151
61
152
-
static __always_inline void atomic64_add(s64 i, atomic64_t *v)
153
-
{
154
-
kasan_check_write(v, sizeof(*v));
155
-
arch_atomic64_add(i, v);
156
-
}
157
-
158
-
static __always_inline void atomic_sub(int i, atomic_t *v)
159
-
{
160
-
kasan_check_write(v, sizeof(*v));
161
-
arch_atomic_sub(i, v);
162
-
}
163
-
164
-
static __always_inline void atomic64_sub(s64 i, atomic64_t *v)
165
-
{
166
-
kasan_check_write(v, sizeof(*v));
167
-
arch_atomic64_sub(i, v);
168
-
}
169
-
170
-
static __always_inline void atomic_and(int i, atomic_t *v)
171
-
{
172
-
kasan_check_write(v, sizeof(*v));
173
-
arch_atomic_and(i, v);
174
-
}
175
-
176
-
static __always_inline void atomic64_and(s64 i, atomic64_t *v)
177
-
{
178
-
kasan_check_write(v, sizeof(*v));
179
-
arch_atomic64_and(i, v);
180
-
}
181
-
182
-
static __always_inline void atomic_or(int i, atomic_t *v)
183
-
{
184
-
kasan_check_write(v, sizeof(*v));
185
-
arch_atomic_or(i, v);
186
-
}
187
-
188
-
static __always_inline void atomic64_or(s64 i, atomic64_t *v)
189
-
{
190
-
kasan_check_write(v, sizeof(*v));
191
-
arch_atomic64_or(i, v);
192
-
}
193
-
194
-
static __always_inline void atomic_xor(int i, atomic_t *v)
195
-
{
196
-
kasan_check_write(v, sizeof(*v));
197
-
arch_atomic_xor(i, v);
198
-
}
199
-
200
-
static __always_inline void atomic64_xor(s64 i, atomic64_t *v)
201
-
{
202
-
kasan_check_write(v, sizeof(*v));
203
-
arch_atomic64_xor(i, v);
204
-
}
205
-
206
-
#ifdef arch_atomic_inc_return
207
-
#define atomic_inc_return atomic_inc_return
208
-
static __always_inline int atomic_inc_return(atomic_t *v)
209
-
{
210
-
kasan_check_write(v, sizeof(*v));
211
-
return arch_atomic_inc_return(v);
212
-
}
213
-
#endif
214
-
215
-
#ifdef arch_atomic64_in_return
216
-
#define atomic64_inc_return atomic64_inc_return
217
-
static __always_inline s64 atomic64_inc_return(atomic64_t *v)
218
-
{
219
-
kasan_check_write(v, sizeof(*v));
220
-
return arch_atomic64_inc_return(v);
221
-
}
222
-
#endif
223
-
224
-
#ifdef arch_atomic_dec_return
225
-
#define atomic_dec_return atomic_dec_return
226
-
static __always_inline int atomic_dec_return(atomic_t *v)
227
-
{
228
-
kasan_check_write(v, sizeof(*v));
229
-
return arch_atomic_dec_return(v);
230
-
}
231
-
#endif
232
-
233
-
#ifdef arch_atomic64_dec_return
234
-
#define atomic64_dec_return atomic64_dec_return
235
-
static __always_inline s64 atomic64_dec_return(atomic64_t *v)
236
-
{
237
-
kasan_check_write(v, sizeof(*v));
238
-
return arch_atomic64_dec_return(v);
239
-
}
240
-
#endif
241
-
242
-
#ifdef arch_atomic64_inc_not_zero
243
-
#define atomic64_inc_not_zero atomic64_inc_not_zero
244
-
static __always_inline bool atomic64_inc_not_zero(atomic64_t *v)
245
-
{
246
-
kasan_check_write(v, sizeof(*v));
247
-
return arch_atomic64_inc_not_zero(v);
248
-
}
249
-
#endif
250
-
251
-
#ifdef arch_atomic64_dec_if_positive
252
-
#define atomic64_dec_if_positive atomic64_dec_if_positive
253
-
static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v)
254
-
{
255
-
kasan_check_write(v, sizeof(*v));
256
-
return arch_atomic64_dec_if_positive(v);
257
-
}
258
-
#endif
259
-
260
-
#ifdef arch_atomic_dec_and_test
261
-
#define atomic_dec_and_test atomic_dec_and_test
262
-
static __always_inline bool atomic_dec_and_test(atomic_t *v)
263
-
{
264
-
kasan_check_write(v, sizeof(*v));
265
-
return arch_atomic_dec_and_test(v);
266
-
}
267
-
#endif
268
-
269
-
#ifdef arch_atomic64_dec_and_test
270
-
#define atomic64_dec_and_test atomic64_dec_and_test
271
-
static __always_inline bool atomic64_dec_and_test(atomic64_t *v)
272
-
{
273
-
kasan_check_write(v, sizeof(*v));
274
-
return arch_atomic64_dec_and_test(v);
275
-
}
276
-
#endif
277
-
278
-
#ifdef arch_atomic_inc_and_test
279
-
#define atomic_inc_and_test atomic_inc_and_test
280
-
static __always_inline bool atomic_inc_and_test(atomic_t *v)
281
-
{
282
-
kasan_check_write(v, sizeof(*v));
283
-
return arch_atomic_inc_and_test(v);
284
-
}
285
-
#endif
286
-
287
-
#ifdef arch_atomic64_inc_and_test
288
-
#define atomic64_inc_and_test atomic64_inc_and_test
289
-
static __always_inline bool atomic64_inc_and_test(atomic64_t *v)
290
-
{
291
-
kasan_check_write(v, sizeof(*v));
292
-
return arch_atomic64_inc_and_test(v);
293
-
}
294
-
#endif
295
-
296
-
static __always_inline int atomic_add_return(int i, atomic_t *v)
62
+
#if !defined(arch_atomic_add_return_relaxed) || defined(arch_atomic_add_return)
63
+
static inline int
64
+
atomic_add_return(int i, atomic_t *v)
297
65
{
298
66
kasan_check_write(v, sizeof(*v));
299
67
return arch_atomic_add_return(i, v);
300
68
}
69
+
#define atomic_add_return atomic_add_return
70
+
#endif
301
71
302
-
static __always_inline s64 atomic64_add_return(s64 i, atomic64_t *v)
72
+
#if defined(arch_atomic_add_return_acquire)
73
+
static inline int
74
+
atomic_add_return_acquire(int i, atomic_t *v)
303
75
{
304
76
kasan_check_write(v, sizeof(*v));
305
-
return arch_atomic64_add_return(i, v);
77
+
return arch_atomic_add_return_acquire(i, v);
306
78
}
79
+
#define atomic_add_return_acquire atomic_add_return_acquire
80
+
#endif
307
81
308
-
static __always_inline int atomic_sub_return(int i, atomic_t *v)
82
+
#if defined(arch_atomic_add_return_release)
83
+
static inline int
84
+
atomic_add_return_release(int i, atomic_t *v)
309
85
{
310
86
kasan_check_write(v, sizeof(*v));
311
-
return arch_atomic_sub_return(i, v);
87
+
return arch_atomic_add_return_release(i, v);
312
88
}
89
+
#define atomic_add_return_release atomic_add_return_release
90
+
#endif
313
91
314
-
static __always_inline s64 atomic64_sub_return(s64 i, atomic64_t *v)
92
+
#if defined(arch_atomic_add_return_relaxed)
93
+
static inline int
94
+
atomic_add_return_relaxed(int i, atomic_t *v)
315
95
{
316
96
kasan_check_write(v, sizeof(*v));
317
-
return arch_atomic64_sub_return(i, v);
97
+
return arch_atomic_add_return_relaxed(i, v);
318
98
}
99
+
#define atomic_add_return_relaxed atomic_add_return_relaxed
100
+
#endif
319
101
320
-
static __always_inline int atomic_fetch_add(int i, atomic_t *v)
102
+
#if !defined(arch_atomic_fetch_add_relaxed) || defined(arch_atomic_fetch_add)
103
+
static inline int
104
+
atomic_fetch_add(int i, atomic_t *v)
321
105
{
322
106
kasan_check_write(v, sizeof(*v));
323
107
return arch_atomic_fetch_add(i, v);
324
108
}
109
+
#define atomic_fetch_add atomic_fetch_add
110
+
#endif
325
111
326
-
static __always_inline s64 atomic64_fetch_add(s64 i, atomic64_t *v)
112
+
#if defined(arch_atomic_fetch_add_acquire)
113
+
static inline int
114
+
atomic_fetch_add_acquire(int i, atomic_t *v)
327
115
{
328
116
kasan_check_write(v, sizeof(*v));
329
-
return arch_atomic64_fetch_add(i, v);
117
+
return arch_atomic_fetch_add_acquire(i, v);
330
118
}
119
+
#define atomic_fetch_add_acquire atomic_fetch_add_acquire
120
+
#endif
331
121
332
-
static __always_inline int atomic_fetch_sub(int i, atomic_t *v)
122
+
#if defined(arch_atomic_fetch_add_release)
123
+
static inline int
124
+
atomic_fetch_add_release(int i, atomic_t *v)
125
+
{
126
+
kasan_check_write(v, sizeof(*v));
127
+
return arch_atomic_fetch_add_release(i, v);
128
+
}
129
+
#define atomic_fetch_add_release atomic_fetch_add_release
130
+
#endif
131
+
132
+
#if defined(arch_atomic_fetch_add_relaxed)
133
+
static inline int
134
+
atomic_fetch_add_relaxed(int i, atomic_t *v)
135
+
{
136
+
kasan_check_write(v, sizeof(*v));
137
+
return arch_atomic_fetch_add_relaxed(i, v);
138
+
}
139
+
#define atomic_fetch_add_relaxed atomic_fetch_add_relaxed
140
+
#endif
141
+
142
+
static inline void
143
+
atomic_sub(int i, atomic_t *v)
144
+
{
145
+
kasan_check_write(v, sizeof(*v));
146
+
arch_atomic_sub(i, v);
147
+
}
148
+
#define atomic_sub atomic_sub
149
+
150
+
#if !defined(arch_atomic_sub_return_relaxed) || defined(arch_atomic_sub_return)
151
+
static inline int
152
+
atomic_sub_return(int i, atomic_t *v)
153
+
{
154
+
kasan_check_write(v, sizeof(*v));
155
+
return arch_atomic_sub_return(i, v);
156
+
}
157
+
#define atomic_sub_return atomic_sub_return
158
+
#endif
159
+
160
+
#if defined(arch_atomic_sub_return_acquire)
161
+
static inline int
162
+
atomic_sub_return_acquire(int i, atomic_t *v)
163
+
{
164
+
kasan_check_write(v, sizeof(*v));
165
+
return arch_atomic_sub_return_acquire(i, v);
166
+
}
167
+
#define atomic_sub_return_acquire atomic_sub_return_acquire
168
+
#endif
169
+
170
+
#if defined(arch_atomic_sub_return_release)
171
+
static inline int
172
+
atomic_sub_return_release(int i, atomic_t *v)
173
+
{
174
+
kasan_check_write(v, sizeof(*v));
175
+
return arch_atomic_sub_return_release(i, v);
176
+
}
177
+
#define atomic_sub_return_release atomic_sub_return_release
178
+
#endif
179
+
180
+
#if defined(arch_atomic_sub_return_relaxed)
181
+
static inline int
182
+
atomic_sub_return_relaxed(int i, atomic_t *v)
183
+
{
184
+
kasan_check_write(v, sizeof(*v));
185
+
return arch_atomic_sub_return_relaxed(i, v);
186
+
}
187
+
#define atomic_sub_return_relaxed atomic_sub_return_relaxed
188
+
#endif
189
+
190
+
#if !defined(arch_atomic_fetch_sub_relaxed) || defined(arch_atomic_fetch_sub)
191
+
static inline int
192
+
atomic_fetch_sub(int i, atomic_t *v)
333
193
{
334
194
kasan_check_write(v, sizeof(*v));
335
195
return arch_atomic_fetch_sub(i, v);
336
196
}
197
+
#define atomic_fetch_sub atomic_fetch_sub
198
+
#endif
337
199
338
-
static __always_inline s64 atomic64_fetch_sub(s64 i, atomic64_t *v)
200
+
#if defined(arch_atomic_fetch_sub_acquire)
201
+
static inline int
202
+
atomic_fetch_sub_acquire(int i, atomic_t *v)
339
203
{
340
204
kasan_check_write(v, sizeof(*v));
341
-
return arch_atomic64_fetch_sub(i, v);
205
+
return arch_atomic_fetch_sub_acquire(i, v);
342
206
}
207
+
#define atomic_fetch_sub_acquire atomic_fetch_sub_acquire
208
+
#endif
343
209
344
-
static __always_inline int atomic_fetch_and(int i, atomic_t *v)
210
+
#if defined(arch_atomic_fetch_sub_release)
211
+
static inline int
212
+
atomic_fetch_sub_release(int i, atomic_t *v)
213
+
{
214
+
kasan_check_write(v, sizeof(*v));
215
+
return arch_atomic_fetch_sub_release(i, v);
216
+
}
217
+
#define atomic_fetch_sub_release atomic_fetch_sub_release
218
+
#endif
219
+
220
+
#if defined(arch_atomic_fetch_sub_relaxed)
221
+
static inline int
222
+
atomic_fetch_sub_relaxed(int i, atomic_t *v)
223
+
{
224
+
kasan_check_write(v, sizeof(*v));
225
+
return arch_atomic_fetch_sub_relaxed(i, v);
226
+
}
227
+
#define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed
228
+
#endif
229
+
230
+
#if defined(arch_atomic_inc)
231
+
static inline void
232
+
atomic_inc(atomic_t *v)
233
+
{
234
+
kasan_check_write(v, sizeof(*v));
235
+
arch_atomic_inc(v);
236
+
}
237
+
#define atomic_inc atomic_inc
238
+
#endif
239
+
240
+
#if defined(arch_atomic_inc_return)
241
+
static inline int
242
+
atomic_inc_return(atomic_t *v)
243
+
{
244
+
kasan_check_write(v, sizeof(*v));
245
+
return arch_atomic_inc_return(v);
246
+
}
247
+
#define atomic_inc_return atomic_inc_return
248
+
#endif
249
+
250
+
#if defined(arch_atomic_inc_return_acquire)
251
+
static inline int
252
+
atomic_inc_return_acquire(atomic_t *v)
253
+
{
254
+
kasan_check_write(v, sizeof(*v));
255
+
return arch_atomic_inc_return_acquire(v);
256
+
}
257
+
#define atomic_inc_return_acquire atomic_inc_return_acquire
258
+
#endif
259
+
260
+
#if defined(arch_atomic_inc_return_release)
261
+
static inline int
262
+
atomic_inc_return_release(atomic_t *v)
263
+
{
264
+
kasan_check_write(v, sizeof(*v));
265
+
return arch_atomic_inc_return_release(v);
266
+
}
267
+
#define atomic_inc_return_release atomic_inc_return_release
268
+
#endif
269
+
270
+
#if defined(arch_atomic_inc_return_relaxed)
271
+
static inline int
272
+
atomic_inc_return_relaxed(atomic_t *v)
273
+
{
274
+
kasan_check_write(v, sizeof(*v));
275
+
return arch_atomic_inc_return_relaxed(v);
276
+
}
277
+
#define atomic_inc_return_relaxed atomic_inc_return_relaxed
278
+
#endif
279
+
280
+
#if defined(arch_atomic_fetch_inc)
281
+
static inline int
282
+
atomic_fetch_inc(atomic_t *v)
283
+
{
284
+
kasan_check_write(v, sizeof(*v));
285
+
return arch_atomic_fetch_inc(v);
286
+
}
287
+
#define atomic_fetch_inc atomic_fetch_inc
288
+
#endif
289
+
290
+
#if defined(arch_atomic_fetch_inc_acquire)
291
+
static inline int
292
+
atomic_fetch_inc_acquire(atomic_t *v)
293
+
{
294
+
kasan_check_write(v, sizeof(*v));
295
+
return arch_atomic_fetch_inc_acquire(v);
296
+
}
297
+
#define atomic_fetch_inc_acquire atomic_fetch_inc_acquire
298
+
#endif
299
+
300
+
#if defined(arch_atomic_fetch_inc_release)
301
+
static inline int
302
+
atomic_fetch_inc_release(atomic_t *v)
303
+
{
304
+
kasan_check_write(v, sizeof(*v));
305
+
return arch_atomic_fetch_inc_release(v);
306
+
}
307
+
#define atomic_fetch_inc_release atomic_fetch_inc_release
308
+
#endif
309
+
310
+
#if defined(arch_atomic_fetch_inc_relaxed)
311
+
static inline int
312
+
atomic_fetch_inc_relaxed(atomic_t *v)
313
+
{
314
+
kasan_check_write(v, sizeof(*v));
315
+
return arch_atomic_fetch_inc_relaxed(v);
316
+
}
317
+
#define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed
318
+
#endif
319
+
320
+
#if defined(arch_atomic_dec)
321
+
static inline void
322
+
atomic_dec(atomic_t *v)
323
+
{
324
+
kasan_check_write(v, sizeof(*v));
325
+
arch_atomic_dec(v);
326
+
}
327
+
#define atomic_dec atomic_dec
328
+
#endif
329
+
330
+
#if defined(arch_atomic_dec_return)
331
+
static inline int
332
+
atomic_dec_return(atomic_t *v)
333
+
{
334
+
kasan_check_write(v, sizeof(*v));
335
+
return arch_atomic_dec_return(v);
336
+
}
337
+
#define atomic_dec_return atomic_dec_return
338
+
#endif
339
+
340
+
#if defined(arch_atomic_dec_return_acquire)
341
+
static inline int
342
+
atomic_dec_return_acquire(atomic_t *v)
343
+
{
344
+
kasan_check_write(v, sizeof(*v));
345
+
return arch_atomic_dec_return_acquire(v);
346
+
}
347
+
#define atomic_dec_return_acquire atomic_dec_return_acquire
348
+
#endif
349
+
350
+
#if defined(arch_atomic_dec_return_release)
351
+
static inline int
352
+
atomic_dec_return_release(atomic_t *v)
353
+
{
354
+
kasan_check_write(v, sizeof(*v));
355
+
return arch_atomic_dec_return_release(v);
356
+
}
357
+
#define atomic_dec_return_release atomic_dec_return_release
358
+
#endif
359
+
360
+
#if defined(arch_atomic_dec_return_relaxed)
361
+
static inline int
362
+
atomic_dec_return_relaxed(atomic_t *v)
363
+
{
364
+
kasan_check_write(v, sizeof(*v));
365
+
return arch_atomic_dec_return_relaxed(v);
366
+
}
367
+
#define atomic_dec_return_relaxed atomic_dec_return_relaxed
368
+
#endif
369
+
370
+
#if defined(arch_atomic_fetch_dec)
371
+
static inline int
372
+
atomic_fetch_dec(atomic_t *v)
373
+
{
374
+
kasan_check_write(v, sizeof(*v));
375
+
return arch_atomic_fetch_dec(v);
376
+
}
377
+
#define atomic_fetch_dec atomic_fetch_dec
378
+
#endif
379
+
380
+
#if defined(arch_atomic_fetch_dec_acquire)
381
+
static inline int
382
+
atomic_fetch_dec_acquire(atomic_t *v)
383
+
{
384
+
kasan_check_write(v, sizeof(*v));
385
+
return arch_atomic_fetch_dec_acquire(v);
386
+
}
387
+
#define atomic_fetch_dec_acquire atomic_fetch_dec_acquire
388
+
#endif
389
+
390
+
#if defined(arch_atomic_fetch_dec_release)
391
+
static inline int
392
+
atomic_fetch_dec_release(atomic_t *v)
393
+
{
394
+
kasan_check_write(v, sizeof(*v));
395
+
return arch_atomic_fetch_dec_release(v);
396
+
}
397
+
#define atomic_fetch_dec_release atomic_fetch_dec_release
398
+
#endif
399
+
400
+
#if defined(arch_atomic_fetch_dec_relaxed)
401
+
static inline int
402
+
atomic_fetch_dec_relaxed(atomic_t *v)
403
+
{
404
+
kasan_check_write(v, sizeof(*v));
405
+
return arch_atomic_fetch_dec_relaxed(v);
406
+
}
407
+
#define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed
408
+
#endif
409
+
410
+
static inline void
411
+
atomic_and(int i, atomic_t *v)
412
+
{
413
+
kasan_check_write(v, sizeof(*v));
414
+
arch_atomic_and(i, v);
415
+
}
416
+
#define atomic_and atomic_and
417
+
418
+
#if !defined(arch_atomic_fetch_and_relaxed) || defined(arch_atomic_fetch_and)
419
+
static inline int
420
+
atomic_fetch_and(int i, atomic_t *v)
345
421
{
346
422
kasan_check_write(v, sizeof(*v));
347
423
return arch_atomic_fetch_and(i, v);
348
424
}
425
+
#define atomic_fetch_and atomic_fetch_and
426
+
#endif
349
427
350
-
static __always_inline s64 atomic64_fetch_and(s64 i, atomic64_t *v)
428
+
#if defined(arch_atomic_fetch_and_acquire)
429
+
static inline int
430
+
atomic_fetch_and_acquire(int i, atomic_t *v)
351
431
{
352
432
kasan_check_write(v, sizeof(*v));
353
-
return arch_atomic64_fetch_and(i, v);
433
+
return arch_atomic_fetch_and_acquire(i, v);
354
434
}
435
+
#define atomic_fetch_and_acquire atomic_fetch_and_acquire
436
+
#endif
355
437
356
-
static __always_inline int atomic_fetch_or(int i, atomic_t *v)
438
+
#if defined(arch_atomic_fetch_and_release)
439
+
static inline int
440
+
atomic_fetch_and_release(int i, atomic_t *v)
441
+
{
442
+
kasan_check_write(v, sizeof(*v));
443
+
return arch_atomic_fetch_and_release(i, v);
444
+
}
445
+
#define atomic_fetch_and_release atomic_fetch_and_release
446
+
#endif
447
+
448
+
#if defined(arch_atomic_fetch_and_relaxed)
449
+
static inline int
450
+
atomic_fetch_and_relaxed(int i, atomic_t *v)
451
+
{
452
+
kasan_check_write(v, sizeof(*v));
453
+
return arch_atomic_fetch_and_relaxed(i, v);
454
+
}
455
+
#define atomic_fetch_and_relaxed atomic_fetch_and_relaxed
456
+
#endif
457
+
458
+
#if defined(arch_atomic_andnot)
459
+
static inline void
460
+
atomic_andnot(int i, atomic_t *v)
461
+
{
462
+
kasan_check_write(v, sizeof(*v));
463
+
arch_atomic_andnot(i, v);
464
+
}
465
+
#define atomic_andnot atomic_andnot
466
+
#endif
467
+
468
+
#if defined(arch_atomic_fetch_andnot)
469
+
static inline int
470
+
atomic_fetch_andnot(int i, atomic_t *v)
471
+
{
472
+
kasan_check_write(v, sizeof(*v));
473
+
return arch_atomic_fetch_andnot(i, v);
474
+
}
475
+
#define atomic_fetch_andnot atomic_fetch_andnot
476
+
#endif
477
+
478
+
#if defined(arch_atomic_fetch_andnot_acquire)
479
+
static inline int
480
+
atomic_fetch_andnot_acquire(int i, atomic_t *v)
481
+
{
482
+
kasan_check_write(v, sizeof(*v));
483
+
return arch_atomic_fetch_andnot_acquire(i, v);
484
+
}
485
+
#define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire
486
+
#endif
487
+
488
+
#if defined(arch_atomic_fetch_andnot_release)
489
+
static inline int
490
+
atomic_fetch_andnot_release(int i, atomic_t *v)
491
+
{
492
+
kasan_check_write(v, sizeof(*v));
493
+
return arch_atomic_fetch_andnot_release(i, v);
494
+
}
495
+
#define atomic_fetch_andnot_release atomic_fetch_andnot_release
496
+
#endif
497
+
498
+
#if defined(arch_atomic_fetch_andnot_relaxed)
499
+
static inline int
500
+
atomic_fetch_andnot_relaxed(int i, atomic_t *v)
501
+
{
502
+
kasan_check_write(v, sizeof(*v));
503
+
return arch_atomic_fetch_andnot_relaxed(i, v);
504
+
}
505
+
#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed
506
+
#endif
507
+
508
+
static inline void
509
+
atomic_or(int i, atomic_t *v)
510
+
{
511
+
kasan_check_write(v, sizeof(*v));
512
+
arch_atomic_or(i, v);
513
+
}
514
+
#define atomic_or atomic_or
515
+
516
+
#if !defined(arch_atomic_fetch_or_relaxed) || defined(arch_atomic_fetch_or)
517
+
static inline int
518
+
atomic_fetch_or(int i, atomic_t *v)
357
519
{
358
520
kasan_check_write(v, sizeof(*v));
359
521
return arch_atomic_fetch_or(i, v);
360
522
}
523
+
#define atomic_fetch_or atomic_fetch_or
524
+
#endif
361
525
362
-
static __always_inline s64 atomic64_fetch_or(s64 i, atomic64_t *v)
526
+
#if defined(arch_atomic_fetch_or_acquire)
527
+
static inline int
528
+
atomic_fetch_or_acquire(int i, atomic_t *v)
363
529
{
364
530
kasan_check_write(v, sizeof(*v));
365
-
return arch_atomic64_fetch_or(i, v);
531
+
return arch_atomic_fetch_or_acquire(i, v);
366
532
}
533
+
#define atomic_fetch_or_acquire atomic_fetch_or_acquire
534
+
#endif
367
535
368
-
static __always_inline int atomic_fetch_xor(int i, atomic_t *v)
536
+
#if defined(arch_atomic_fetch_or_release)
537
+
static inline int
538
+
atomic_fetch_or_release(int i, atomic_t *v)
539
+
{
540
+
kasan_check_write(v, sizeof(*v));
541
+
return arch_atomic_fetch_or_release(i, v);
542
+
}
543
+
#define atomic_fetch_or_release atomic_fetch_or_release
544
+
#endif
545
+
546
+
#if defined(arch_atomic_fetch_or_relaxed)
547
+
static inline int
548
+
atomic_fetch_or_relaxed(int i, atomic_t *v)
549
+
{
550
+
kasan_check_write(v, sizeof(*v));
551
+
return arch_atomic_fetch_or_relaxed(i, v);
552
+
}
553
+
#define atomic_fetch_or_relaxed atomic_fetch_or_relaxed
554
+
#endif
555
+
556
+
static inline void
557
+
atomic_xor(int i, atomic_t *v)
558
+
{
559
+
kasan_check_write(v, sizeof(*v));
560
+
arch_atomic_xor(i, v);
561
+
}
562
+
#define atomic_xor atomic_xor
563
+
564
+
#if !defined(arch_atomic_fetch_xor_relaxed) || defined(arch_atomic_fetch_xor)
565
+
static inline int
566
+
atomic_fetch_xor(int i, atomic_t *v)
369
567
{
370
568
kasan_check_write(v, sizeof(*v));
371
569
return arch_atomic_fetch_xor(i, v);
372
570
}
571
+
#define atomic_fetch_xor atomic_fetch_xor
572
+
#endif
373
573
374
-
static __always_inline s64 atomic64_fetch_xor(s64 i, atomic64_t *v)
574
+
#if defined(arch_atomic_fetch_xor_acquire)
575
+
static inline int
576
+
atomic_fetch_xor_acquire(int i, atomic_t *v)
375
577
{
376
578
kasan_check_write(v, sizeof(*v));
377
-
return arch_atomic64_fetch_xor(i, v);
579
+
return arch_atomic_fetch_xor_acquire(i, v);
378
580
}
581
+
#define atomic_fetch_xor_acquire atomic_fetch_xor_acquire
582
+
#endif
379
583
380
-
#ifdef arch_atomic_sub_and_test
381
-
#define atomic_sub_and_test atomic_sub_and_test
382
-
static __always_inline bool atomic_sub_and_test(int i, atomic_t *v)
584
+
#if defined(arch_atomic_fetch_xor_release)
585
+
static inline int
586
+
atomic_fetch_xor_release(int i, atomic_t *v)
587
+
{
588
+
kasan_check_write(v, sizeof(*v));
589
+
return arch_atomic_fetch_xor_release(i, v);
590
+
}
591
+
#define atomic_fetch_xor_release atomic_fetch_xor_release
592
+
#endif
593
+
594
+
#if defined(arch_atomic_fetch_xor_relaxed)
595
+
static inline int
596
+
atomic_fetch_xor_relaxed(int i, atomic_t *v)
597
+
{
598
+
kasan_check_write(v, sizeof(*v));
599
+
return arch_atomic_fetch_xor_relaxed(i, v);
600
+
}
601
+
#define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed
602
+
#endif
603
+
604
+
#if !defined(arch_atomic_xchg_relaxed) || defined(arch_atomic_xchg)
605
+
static inline int
606
+
atomic_xchg(atomic_t *v, int i)
607
+
{
608
+
kasan_check_write(v, sizeof(*v));
609
+
return arch_atomic_xchg(v, i);
610
+
}
611
+
#define atomic_xchg atomic_xchg
612
+
#endif
613
+
614
+
#if defined(arch_atomic_xchg_acquire)
615
+
static inline int
616
+
atomic_xchg_acquire(atomic_t *v, int i)
617
+
{
618
+
kasan_check_write(v, sizeof(*v));
619
+
return arch_atomic_xchg_acquire(v, i);
620
+
}
621
+
#define atomic_xchg_acquire atomic_xchg_acquire
622
+
#endif
623
+
624
+
#if defined(arch_atomic_xchg_release)
625
+
static inline int
626
+
atomic_xchg_release(atomic_t *v, int i)
627
+
{
628
+
kasan_check_write(v, sizeof(*v));
629
+
return arch_atomic_xchg_release(v, i);
630
+
}
631
+
#define atomic_xchg_release atomic_xchg_release
632
+
#endif
633
+
634
+
#if defined(arch_atomic_xchg_relaxed)
635
+
static inline int
636
+
atomic_xchg_relaxed(atomic_t *v, int i)
637
+
{
638
+
kasan_check_write(v, sizeof(*v));
639
+
return arch_atomic_xchg_relaxed(v, i);
640
+
}
641
+
#define atomic_xchg_relaxed atomic_xchg_relaxed
642
+
#endif
643
+
644
+
#if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg)
645
+
static inline int
646
+
atomic_cmpxchg(atomic_t *v, int old, int new)
647
+
{
648
+
kasan_check_write(v, sizeof(*v));
649
+
return arch_atomic_cmpxchg(v, old, new);
650
+
}
651
+
#define atomic_cmpxchg atomic_cmpxchg
652
+
#endif
653
+
654
+
#if defined(arch_atomic_cmpxchg_acquire)
655
+
static inline int
656
+
atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
657
+
{
658
+
kasan_check_write(v, sizeof(*v));
659
+
return arch_atomic_cmpxchg_acquire(v, old, new);
660
+
}
661
+
#define atomic_cmpxchg_acquire atomic_cmpxchg_acquire
662
+
#endif
663
+
664
+
#if defined(arch_atomic_cmpxchg_release)
665
+
static inline int
666
+
atomic_cmpxchg_release(atomic_t *v, int old, int new)
667
+
{
668
+
kasan_check_write(v, sizeof(*v));
669
+
return arch_atomic_cmpxchg_release(v, old, new);
670
+
}
671
+
#define atomic_cmpxchg_release atomic_cmpxchg_release
672
+
#endif
673
+
674
+
#if defined(arch_atomic_cmpxchg_relaxed)
675
+
static inline int
676
+
atomic_cmpxchg_relaxed(atomic_t *v, int old, int new)
677
+
{
678
+
kasan_check_write(v, sizeof(*v));
679
+
return arch_atomic_cmpxchg_relaxed(v, old, new);
680
+
}
681
+
#define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed
682
+
#endif
683
+
684
+
#if defined(arch_atomic_try_cmpxchg)
685
+
static inline bool
686
+
atomic_try_cmpxchg(atomic_t *v, int *old, int new)
687
+
{
688
+
kasan_check_write(v, sizeof(*v));
689
+
kasan_check_write(old, sizeof(*old));
690
+
return arch_atomic_try_cmpxchg(v, old, new);
691
+
}
692
+
#define atomic_try_cmpxchg atomic_try_cmpxchg
693
+
#endif
694
+
695
+
#if defined(arch_atomic_try_cmpxchg_acquire)
696
+
static inline bool
697
+
atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
698
+
{
699
+
kasan_check_write(v, sizeof(*v));
700
+
kasan_check_write(old, sizeof(*old));
701
+
return arch_atomic_try_cmpxchg_acquire(v, old, new);
702
+
}
703
+
#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire
704
+
#endif
705
+
706
+
#if defined(arch_atomic_try_cmpxchg_release)
707
+
static inline bool
708
+
atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
709
+
{
710
+
kasan_check_write(v, sizeof(*v));
711
+
kasan_check_write(old, sizeof(*old));
712
+
return arch_atomic_try_cmpxchg_release(v, old, new);
713
+
}
714
+
#define atomic_try_cmpxchg_release atomic_try_cmpxchg_release
715
+
#endif
716
+
717
+
#if defined(arch_atomic_try_cmpxchg_relaxed)
718
+
static inline bool
719
+
atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
720
+
{
721
+
kasan_check_write(v, sizeof(*v));
722
+
kasan_check_write(old, sizeof(*old));
723
+
return arch_atomic_try_cmpxchg_relaxed(v, old, new);
724
+
}
725
+
#define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed
726
+
#endif
727
+
728
+
#if defined(arch_atomic_sub_and_test)
729
+
static inline bool
730
+
atomic_sub_and_test(int i, atomic_t *v)
383
731
{
384
732
kasan_check_write(v, sizeof(*v));
385
733
return arch_atomic_sub_and_test(i, v);
386
734
}
735
+
#define atomic_sub_and_test atomic_sub_and_test
387
736
#endif
388
737
389
-
#ifdef arch_atomic64_sub_and_test
390
-
#define atomic64_sub_and_test atomic64_sub_and_test
391
-
static __always_inline bool atomic64_sub_and_test(s64 i, atomic64_t *v)
738
+
#if defined(arch_atomic_dec_and_test)
739
+
static inline bool
740
+
atomic_dec_and_test(atomic_t *v)
392
741
{
393
742
kasan_check_write(v, sizeof(*v));
394
-
return arch_atomic64_sub_and_test(i, v);
743
+
return arch_atomic_dec_and_test(v);
395
744
}
745
+
#define atomic_dec_and_test atomic_dec_and_test
396
746
#endif
397
747
398
-
#ifdef arch_atomic_add_negative
399
-
#define atomic_add_negative atomic_add_negative
400
-
static __always_inline bool atomic_add_negative(int i, atomic_t *v)
748
+
#if defined(arch_atomic_inc_and_test)
749
+
static inline bool
750
+
atomic_inc_and_test(atomic_t *v)
751
+
{
752
+
kasan_check_write(v, sizeof(*v));
753
+
return arch_atomic_inc_and_test(v);
754
+
}
755
+
#define atomic_inc_and_test atomic_inc_and_test
756
+
#endif
757
+
758
+
#if defined(arch_atomic_add_negative)
759
+
static inline bool
760
+
atomic_add_negative(int i, atomic_t *v)
401
761
{
402
762
kasan_check_write(v, sizeof(*v));
403
763
return arch_atomic_add_negative(i, v);
404
764
}
765
+
#define atomic_add_negative atomic_add_negative
405
766
#endif
406
767
407
-
#ifdef arch_atomic64_add_negative
408
-
#define atomic64_add_negative atomic64_add_negative
409
-
static __always_inline bool atomic64_add_negative(s64 i, atomic64_t *v)
768
+
#if defined(arch_atomic_fetch_add_unless)
769
+
static inline int
770
+
atomic_fetch_add_unless(atomic_t *v, int a, int u)
771
+
{
772
+
kasan_check_write(v, sizeof(*v));
773
+
return arch_atomic_fetch_add_unless(v, a, u);
774
+
}
775
+
#define atomic_fetch_add_unless atomic_fetch_add_unless
776
+
#endif
777
+
778
+
#if defined(arch_atomic_add_unless)
779
+
static inline bool
780
+
atomic_add_unless(atomic_t *v, int a, int u)
781
+
{
782
+
kasan_check_write(v, sizeof(*v));
783
+
return arch_atomic_add_unless(v, a, u);
784
+
}
785
+
#define atomic_add_unless atomic_add_unless
786
+
#endif
787
+
788
+
#if defined(arch_atomic_inc_not_zero)
789
+
static inline bool
790
+
atomic_inc_not_zero(atomic_t *v)
791
+
{
792
+
kasan_check_write(v, sizeof(*v));
793
+
return arch_atomic_inc_not_zero(v);
794
+
}
795
+
#define atomic_inc_not_zero atomic_inc_not_zero
796
+
#endif
797
+
798
+
#if defined(arch_atomic_inc_unless_negative)
799
+
static inline bool
800
+
atomic_inc_unless_negative(atomic_t *v)
801
+
{
802
+
kasan_check_write(v, sizeof(*v));
803
+
return arch_atomic_inc_unless_negative(v);
804
+
}
805
+
#define atomic_inc_unless_negative atomic_inc_unless_negative
806
+
#endif
807
+
808
+
#if defined(arch_atomic_dec_unless_positive)
809
+
static inline bool
810
+
atomic_dec_unless_positive(atomic_t *v)
811
+
{
812
+
kasan_check_write(v, sizeof(*v));
813
+
return arch_atomic_dec_unless_positive(v);
814
+
}
815
+
#define atomic_dec_unless_positive atomic_dec_unless_positive
816
+
#endif
817
+
818
+
#if defined(arch_atomic_dec_if_positive)
819
+
static inline int
820
+
atomic_dec_if_positive(atomic_t *v)
821
+
{
822
+
kasan_check_write(v, sizeof(*v));
823
+
return arch_atomic_dec_if_positive(v);
824
+
}
825
+
#define atomic_dec_if_positive atomic_dec_if_positive
826
+
#endif
827
+
828
+
static inline s64
829
+
atomic64_read(const atomic64_t *v)
830
+
{
831
+
kasan_check_read(v, sizeof(*v));
832
+
return arch_atomic64_read(v);
833
+
}
834
+
#define atomic64_read atomic64_read
835
+
836
+
#if defined(arch_atomic64_read_acquire)
837
+
static inline s64
838
+
atomic64_read_acquire(const atomic64_t *v)
839
+
{
840
+
kasan_check_read(v, sizeof(*v));
841
+
return arch_atomic64_read_acquire(v);
842
+
}
843
+
#define atomic64_read_acquire atomic64_read_acquire
844
+
#endif
845
+
846
+
static inline void
847
+
atomic64_set(atomic64_t *v, s64 i)
848
+
{
849
+
kasan_check_write(v, sizeof(*v));
850
+
arch_atomic64_set(v, i);
851
+
}
852
+
#define atomic64_set atomic64_set
853
+
854
+
#if defined(arch_atomic64_set_release)
855
+
static inline void
856
+
atomic64_set_release(atomic64_t *v, s64 i)
857
+
{
858
+
kasan_check_write(v, sizeof(*v));
859
+
arch_atomic64_set_release(v, i);
860
+
}
861
+
#define atomic64_set_release atomic64_set_release
862
+
#endif
863
+
864
+
static inline void
865
+
atomic64_add(s64 i, atomic64_t *v)
866
+
{
867
+
kasan_check_write(v, sizeof(*v));
868
+
arch_atomic64_add(i, v);
869
+
}
870
+
#define atomic64_add atomic64_add
871
+
872
+
#if !defined(arch_atomic64_add_return_relaxed) || defined(arch_atomic64_add_return)
873
+
static inline s64
874
+
atomic64_add_return(s64 i, atomic64_t *v)
875
+
{
876
+
kasan_check_write(v, sizeof(*v));
877
+
return arch_atomic64_add_return(i, v);
878
+
}
879
+
#define atomic64_add_return atomic64_add_return
880
+
#endif
881
+
882
+
#if defined(arch_atomic64_add_return_acquire)
883
+
static inline s64
884
+
atomic64_add_return_acquire(s64 i, atomic64_t *v)
885
+
{
886
+
kasan_check_write(v, sizeof(*v));
887
+
return arch_atomic64_add_return_acquire(i, v);
888
+
}
889
+
#define atomic64_add_return_acquire atomic64_add_return_acquire
890
+
#endif
891
+
892
+
#if defined(arch_atomic64_add_return_release)
893
+
static inline s64
894
+
atomic64_add_return_release(s64 i, atomic64_t *v)
895
+
{
896
+
kasan_check_write(v, sizeof(*v));
897
+
return arch_atomic64_add_return_release(i, v);
898
+
}
899
+
#define atomic64_add_return_release atomic64_add_return_release
900
+
#endif
901
+
902
+
#if defined(arch_atomic64_add_return_relaxed)
903
+
static inline s64
904
+
atomic64_add_return_relaxed(s64 i, atomic64_t *v)
905
+
{
906
+
kasan_check_write(v, sizeof(*v));
907
+
return arch_atomic64_add_return_relaxed(i, v);
908
+
}
909
+
#define atomic64_add_return_relaxed atomic64_add_return_relaxed
910
+
#endif
911
+
912
+
#if !defined(arch_atomic64_fetch_add_relaxed) || defined(arch_atomic64_fetch_add)
913
+
static inline s64
914
+
atomic64_fetch_add(s64 i, atomic64_t *v)
915
+
{
916
+
kasan_check_write(v, sizeof(*v));
917
+
return arch_atomic64_fetch_add(i, v);
918
+
}
919
+
#define atomic64_fetch_add atomic64_fetch_add
920
+
#endif
921
+
922
+
#if defined(arch_atomic64_fetch_add_acquire)
923
+
static inline s64
924
+
atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
925
+
{
926
+
kasan_check_write(v, sizeof(*v));
927
+
return arch_atomic64_fetch_add_acquire(i, v);
928
+
}
929
+
#define atomic64_fetch_add_acquire atomic64_fetch_add_acquire
930
+
#endif
931
+
932
+
#if defined(arch_atomic64_fetch_add_release)
933
+
static inline s64
934
+
atomic64_fetch_add_release(s64 i, atomic64_t *v)
935
+
{
936
+
kasan_check_write(v, sizeof(*v));
937
+
return arch_atomic64_fetch_add_release(i, v);
938
+
}
939
+
#define atomic64_fetch_add_release atomic64_fetch_add_release
940
+
#endif
941
+
942
+
#if defined(arch_atomic64_fetch_add_relaxed)
943
+
static inline s64
944
+
atomic64_fetch_add_relaxed(s64 i, atomic64_t *v)
945
+
{
946
+
kasan_check_write(v, sizeof(*v));
947
+
return arch_atomic64_fetch_add_relaxed(i, v);
948
+
}
949
+
#define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed
950
+
#endif
951
+
952
+
static inline void
953
+
atomic64_sub(s64 i, atomic64_t *v)
954
+
{
955
+
kasan_check_write(v, sizeof(*v));
956
+
arch_atomic64_sub(i, v);
957
+
}
958
+
#define atomic64_sub atomic64_sub
959
+
960
+
#if !defined(arch_atomic64_sub_return_relaxed) || defined(arch_atomic64_sub_return)
961
+
static inline s64
962
+
atomic64_sub_return(s64 i, atomic64_t *v)
963
+
{
964
+
kasan_check_write(v, sizeof(*v));
965
+
return arch_atomic64_sub_return(i, v);
966
+
}
967
+
#define atomic64_sub_return atomic64_sub_return
968
+
#endif
969
+
970
+
#if defined(arch_atomic64_sub_return_acquire)
971
+
static inline s64
972
+
atomic64_sub_return_acquire(s64 i, atomic64_t *v)
973
+
{
974
+
kasan_check_write(v, sizeof(*v));
975
+
return arch_atomic64_sub_return_acquire(i, v);
976
+
}
977
+
#define atomic64_sub_return_acquire atomic64_sub_return_acquire
978
+
#endif
979
+
980
+
#if defined(arch_atomic64_sub_return_release)
981
+
static inline s64
982
+
atomic64_sub_return_release(s64 i, atomic64_t *v)
983
+
{
984
+
kasan_check_write(v, sizeof(*v));
985
+
return arch_atomic64_sub_return_release(i, v);
986
+
}
987
+
#define atomic64_sub_return_release atomic64_sub_return_release
988
+
#endif
989
+
990
+
#if defined(arch_atomic64_sub_return_relaxed)
991
+
static inline s64
992
+
atomic64_sub_return_relaxed(s64 i, atomic64_t *v)
993
+
{
994
+
kasan_check_write(v, sizeof(*v));
995
+
return arch_atomic64_sub_return_relaxed(i, v);
996
+
}
997
+
#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
998
+
#endif
999
+
1000
+
#if !defined(arch_atomic64_fetch_sub_relaxed) || defined(arch_atomic64_fetch_sub)
1001
+
static inline s64
1002
+
atomic64_fetch_sub(s64 i, atomic64_t *v)
1003
+
{
1004
+
kasan_check_write(v, sizeof(*v));
1005
+
return arch_atomic64_fetch_sub(i, v);
1006
+
}
1007
+
#define atomic64_fetch_sub atomic64_fetch_sub
1008
+
#endif
1009
+
1010
+
#if defined(arch_atomic64_fetch_sub_acquire)
1011
+
static inline s64
1012
+
atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1013
+
{
1014
+
kasan_check_write(v, sizeof(*v));
1015
+
return arch_atomic64_fetch_sub_acquire(i, v);
1016
+
}
1017
+
#define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire
1018
+
#endif
1019
+
1020
+
#if defined(arch_atomic64_fetch_sub_release)
1021
+
static inline s64
1022
+
atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1023
+
{
1024
+
kasan_check_write(v, sizeof(*v));
1025
+
return arch_atomic64_fetch_sub_release(i, v);
1026
+
}
1027
+
#define atomic64_fetch_sub_release atomic64_fetch_sub_release
1028
+
#endif
1029
+
1030
+
#if defined(arch_atomic64_fetch_sub_relaxed)
1031
+
static inline s64
1032
+
atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v)
1033
+
{
1034
+
kasan_check_write(v, sizeof(*v));
1035
+
return arch_atomic64_fetch_sub_relaxed(i, v);
1036
+
}
1037
+
#define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed
1038
+
#endif
1039
+
1040
+
#if defined(arch_atomic64_inc)
1041
+
static inline void
1042
+
atomic64_inc(atomic64_t *v)
1043
+
{
1044
+
kasan_check_write(v, sizeof(*v));
1045
+
arch_atomic64_inc(v);
1046
+
}
1047
+
#define atomic64_inc atomic64_inc
1048
+
#endif
1049
+
1050
+
#if defined(arch_atomic64_inc_return)
1051
+
static inline s64
1052
+
atomic64_inc_return(atomic64_t *v)
1053
+
{
1054
+
kasan_check_write(v, sizeof(*v));
1055
+
return arch_atomic64_inc_return(v);
1056
+
}
1057
+
#define atomic64_inc_return atomic64_inc_return
1058
+
#endif
1059
+
1060
+
#if defined(arch_atomic64_inc_return_acquire)
1061
+
static inline s64
1062
+
atomic64_inc_return_acquire(atomic64_t *v)
1063
+
{
1064
+
kasan_check_write(v, sizeof(*v));
1065
+
return arch_atomic64_inc_return_acquire(v);
1066
+
}
1067
+
#define atomic64_inc_return_acquire atomic64_inc_return_acquire
1068
+
#endif
1069
+
1070
+
#if defined(arch_atomic64_inc_return_release)
1071
+
static inline s64
1072
+
atomic64_inc_return_release(atomic64_t *v)
1073
+
{
1074
+
kasan_check_write(v, sizeof(*v));
1075
+
return arch_atomic64_inc_return_release(v);
1076
+
}
1077
+
#define atomic64_inc_return_release atomic64_inc_return_release
1078
+
#endif
1079
+
1080
+
#if defined(arch_atomic64_inc_return_relaxed)
1081
+
static inline s64
1082
+
atomic64_inc_return_relaxed(atomic64_t *v)
1083
+
{
1084
+
kasan_check_write(v, sizeof(*v));
1085
+
return arch_atomic64_inc_return_relaxed(v);
1086
+
}
1087
+
#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
1088
+
#endif
1089
+
1090
+
#if defined(arch_atomic64_fetch_inc)
1091
+
static inline s64
1092
+
atomic64_fetch_inc(atomic64_t *v)
1093
+
{
1094
+
kasan_check_write(v, sizeof(*v));
1095
+
return arch_atomic64_fetch_inc(v);
1096
+
}
1097
+
#define atomic64_fetch_inc atomic64_fetch_inc
1098
+
#endif
1099
+
1100
+
#if defined(arch_atomic64_fetch_inc_acquire)
1101
+
static inline s64
1102
+
atomic64_fetch_inc_acquire(atomic64_t *v)
1103
+
{
1104
+
kasan_check_write(v, sizeof(*v));
1105
+
return arch_atomic64_fetch_inc_acquire(v);
1106
+
}
1107
+
#define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire
1108
+
#endif
1109
+
1110
+
#if defined(arch_atomic64_fetch_inc_release)
1111
+
static inline s64
1112
+
atomic64_fetch_inc_release(atomic64_t *v)
1113
+
{
1114
+
kasan_check_write(v, sizeof(*v));
1115
+
return arch_atomic64_fetch_inc_release(v);
1116
+
}
1117
+
#define atomic64_fetch_inc_release atomic64_fetch_inc_release
1118
+
#endif
1119
+
1120
+
#if defined(arch_atomic64_fetch_inc_relaxed)
1121
+
static inline s64
1122
+
atomic64_fetch_inc_relaxed(atomic64_t *v)
1123
+
{
1124
+
kasan_check_write(v, sizeof(*v));
1125
+
return arch_atomic64_fetch_inc_relaxed(v);
1126
+
}
1127
+
#define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed
1128
+
#endif
1129
+
1130
+
#if defined(arch_atomic64_dec)
1131
+
static inline void
1132
+
atomic64_dec(atomic64_t *v)
1133
+
{
1134
+
kasan_check_write(v, sizeof(*v));
1135
+
arch_atomic64_dec(v);
1136
+
}
1137
+
#define atomic64_dec atomic64_dec
1138
+
#endif
1139
+
1140
+
#if defined(arch_atomic64_dec_return)
1141
+
static inline s64
1142
+
atomic64_dec_return(atomic64_t *v)
1143
+
{
1144
+
kasan_check_write(v, sizeof(*v));
1145
+
return arch_atomic64_dec_return(v);
1146
+
}
1147
+
#define atomic64_dec_return atomic64_dec_return
1148
+
#endif
1149
+
1150
+
#if defined(arch_atomic64_dec_return_acquire)
1151
+
static inline s64
1152
+
atomic64_dec_return_acquire(atomic64_t *v)
1153
+
{
1154
+
kasan_check_write(v, sizeof(*v));
1155
+
return arch_atomic64_dec_return_acquire(v);
1156
+
}
1157
+
#define atomic64_dec_return_acquire atomic64_dec_return_acquire
1158
+
#endif
1159
+
1160
+
#if defined(arch_atomic64_dec_return_release)
1161
+
static inline s64
1162
+
atomic64_dec_return_release(atomic64_t *v)
1163
+
{
1164
+
kasan_check_write(v, sizeof(*v));
1165
+
return arch_atomic64_dec_return_release(v);
1166
+
}
1167
+
#define atomic64_dec_return_release atomic64_dec_return_release
1168
+
#endif
1169
+
1170
+
#if defined(arch_atomic64_dec_return_relaxed)
1171
+
static inline s64
1172
+
atomic64_dec_return_relaxed(atomic64_t *v)
1173
+
{
1174
+
kasan_check_write(v, sizeof(*v));
1175
+
return arch_atomic64_dec_return_relaxed(v);
1176
+
}
1177
+
#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
1178
+
#endif
1179
+
1180
+
#if defined(arch_atomic64_fetch_dec)
1181
+
static inline s64
1182
+
atomic64_fetch_dec(atomic64_t *v)
1183
+
{
1184
+
kasan_check_write(v, sizeof(*v));
1185
+
return arch_atomic64_fetch_dec(v);
1186
+
}
1187
+
#define atomic64_fetch_dec atomic64_fetch_dec
1188
+
#endif
1189
+
1190
+
#if defined(arch_atomic64_fetch_dec_acquire)
1191
+
static inline s64
1192
+
atomic64_fetch_dec_acquire(atomic64_t *v)
1193
+
{
1194
+
kasan_check_write(v, sizeof(*v));
1195
+
return arch_atomic64_fetch_dec_acquire(v);
1196
+
}
1197
+
#define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire
1198
+
#endif
1199
+
1200
+
#if defined(arch_atomic64_fetch_dec_release)
1201
+
static inline s64
1202
+
atomic64_fetch_dec_release(atomic64_t *v)
1203
+
{
1204
+
kasan_check_write(v, sizeof(*v));
1205
+
return arch_atomic64_fetch_dec_release(v);
1206
+
}
1207
+
#define atomic64_fetch_dec_release atomic64_fetch_dec_release
1208
+
#endif
1209
+
1210
+
#if defined(arch_atomic64_fetch_dec_relaxed)
1211
+
static inline s64
1212
+
atomic64_fetch_dec_relaxed(atomic64_t *v)
1213
+
{
1214
+
kasan_check_write(v, sizeof(*v));
1215
+
return arch_atomic64_fetch_dec_relaxed(v);
1216
+
}
1217
+
#define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed
1218
+
#endif
1219
+
1220
+
static inline void
1221
+
atomic64_and(s64 i, atomic64_t *v)
1222
+
{
1223
+
kasan_check_write(v, sizeof(*v));
1224
+
arch_atomic64_and(i, v);
1225
+
}
1226
+
#define atomic64_and atomic64_and
1227
+
1228
+
#if !defined(arch_atomic64_fetch_and_relaxed) || defined(arch_atomic64_fetch_and)
1229
+
static inline s64
1230
+
atomic64_fetch_and(s64 i, atomic64_t *v)
1231
+
{
1232
+
kasan_check_write(v, sizeof(*v));
1233
+
return arch_atomic64_fetch_and(i, v);
1234
+
}
1235
+
#define atomic64_fetch_and atomic64_fetch_and
1236
+
#endif
1237
+
1238
+
#if defined(arch_atomic64_fetch_and_acquire)
1239
+
static inline s64
1240
+
atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1241
+
{
1242
+
kasan_check_write(v, sizeof(*v));
1243
+
return arch_atomic64_fetch_and_acquire(i, v);
1244
+
}
1245
+
#define atomic64_fetch_and_acquire atomic64_fetch_and_acquire
1246
+
#endif
1247
+
1248
+
#if defined(arch_atomic64_fetch_and_release)
1249
+
static inline s64
1250
+
atomic64_fetch_and_release(s64 i, atomic64_t *v)
1251
+
{
1252
+
kasan_check_write(v, sizeof(*v));
1253
+
return arch_atomic64_fetch_and_release(i, v);
1254
+
}
1255
+
#define atomic64_fetch_and_release atomic64_fetch_and_release
1256
+
#endif
1257
+
1258
+
#if defined(arch_atomic64_fetch_and_relaxed)
1259
+
static inline s64
1260
+
atomic64_fetch_and_relaxed(s64 i, atomic64_t *v)
1261
+
{
1262
+
kasan_check_write(v, sizeof(*v));
1263
+
return arch_atomic64_fetch_and_relaxed(i, v);
1264
+
}
1265
+
#define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed
1266
+
#endif
1267
+
1268
+
#if defined(arch_atomic64_andnot)
1269
+
static inline void
1270
+
atomic64_andnot(s64 i, atomic64_t *v)
1271
+
{
1272
+
kasan_check_write(v, sizeof(*v));
1273
+
arch_atomic64_andnot(i, v);
1274
+
}
1275
+
#define atomic64_andnot atomic64_andnot
1276
+
#endif
1277
+
1278
+
#if defined(arch_atomic64_fetch_andnot)
1279
+
static inline s64
1280
+
atomic64_fetch_andnot(s64 i, atomic64_t *v)
1281
+
{
1282
+
kasan_check_write(v, sizeof(*v));
1283
+
return arch_atomic64_fetch_andnot(i, v);
1284
+
}
1285
+
#define atomic64_fetch_andnot atomic64_fetch_andnot
1286
+
#endif
1287
+
1288
+
#if defined(arch_atomic64_fetch_andnot_acquire)
1289
+
static inline s64
1290
+
atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1291
+
{
1292
+
kasan_check_write(v, sizeof(*v));
1293
+
return arch_atomic64_fetch_andnot_acquire(i, v);
1294
+
}
1295
+
#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire
1296
+
#endif
1297
+
1298
+
#if defined(arch_atomic64_fetch_andnot_release)
1299
+
static inline s64
1300
+
atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1301
+
{
1302
+
kasan_check_write(v, sizeof(*v));
1303
+
return arch_atomic64_fetch_andnot_release(i, v);
1304
+
}
1305
+
#define atomic64_fetch_andnot_release atomic64_fetch_andnot_release
1306
+
#endif
1307
+
1308
+
#if defined(arch_atomic64_fetch_andnot_relaxed)
1309
+
static inline s64
1310
+
atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1311
+
{
1312
+
kasan_check_write(v, sizeof(*v));
1313
+
return arch_atomic64_fetch_andnot_relaxed(i, v);
1314
+
}
1315
+
#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed
1316
+
#endif
1317
+
1318
+
static inline void
1319
+
atomic64_or(s64 i, atomic64_t *v)
1320
+
{
1321
+
kasan_check_write(v, sizeof(*v));
1322
+
arch_atomic64_or(i, v);
1323
+
}
1324
+
#define atomic64_or atomic64_or
1325
+
1326
+
#if !defined(arch_atomic64_fetch_or_relaxed) || defined(arch_atomic64_fetch_or)
1327
+
static inline s64
1328
+
atomic64_fetch_or(s64 i, atomic64_t *v)
1329
+
{
1330
+
kasan_check_write(v, sizeof(*v));
1331
+
return arch_atomic64_fetch_or(i, v);
1332
+
}
1333
+
#define atomic64_fetch_or atomic64_fetch_or
1334
+
#endif
1335
+
1336
+
#if defined(arch_atomic64_fetch_or_acquire)
1337
+
static inline s64
1338
+
atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1339
+
{
1340
+
kasan_check_write(v, sizeof(*v));
1341
+
return arch_atomic64_fetch_or_acquire(i, v);
1342
+
}
1343
+
#define atomic64_fetch_or_acquire atomic64_fetch_or_acquire
1344
+
#endif
1345
+
1346
+
#if defined(arch_atomic64_fetch_or_release)
1347
+
static inline s64
1348
+
atomic64_fetch_or_release(s64 i, atomic64_t *v)
1349
+
{
1350
+
kasan_check_write(v, sizeof(*v));
1351
+
return arch_atomic64_fetch_or_release(i, v);
1352
+
}
1353
+
#define atomic64_fetch_or_release atomic64_fetch_or_release
1354
+
#endif
1355
+
1356
+
#if defined(arch_atomic64_fetch_or_relaxed)
1357
+
static inline s64
1358
+
atomic64_fetch_or_relaxed(s64 i, atomic64_t *v)
1359
+
{
1360
+
kasan_check_write(v, sizeof(*v));
1361
+
return arch_atomic64_fetch_or_relaxed(i, v);
1362
+
}
1363
+
#define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed
1364
+
#endif
1365
+
1366
+
static inline void
1367
+
atomic64_xor(s64 i, atomic64_t *v)
1368
+
{
1369
+
kasan_check_write(v, sizeof(*v));
1370
+
arch_atomic64_xor(i, v);
1371
+
}
1372
+
#define atomic64_xor atomic64_xor
1373
+
1374
+
#if !defined(arch_atomic64_fetch_xor_relaxed) || defined(arch_atomic64_fetch_xor)
1375
+
static inline s64
1376
+
atomic64_fetch_xor(s64 i, atomic64_t *v)
1377
+
{
1378
+
kasan_check_write(v, sizeof(*v));
1379
+
return arch_atomic64_fetch_xor(i, v);
1380
+
}
1381
+
#define atomic64_fetch_xor atomic64_fetch_xor
1382
+
#endif
1383
+
1384
+
#if defined(arch_atomic64_fetch_xor_acquire)
1385
+
static inline s64
1386
+
atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1387
+
{
1388
+
kasan_check_write(v, sizeof(*v));
1389
+
return arch_atomic64_fetch_xor_acquire(i, v);
1390
+
}
1391
+
#define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire
1392
+
#endif
1393
+
1394
+
#if defined(arch_atomic64_fetch_xor_release)
1395
+
static inline s64
1396
+
atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1397
+
{
1398
+
kasan_check_write(v, sizeof(*v));
1399
+
return arch_atomic64_fetch_xor_release(i, v);
1400
+
}
1401
+
#define atomic64_fetch_xor_release atomic64_fetch_xor_release
1402
+
#endif
1403
+
1404
+
#if defined(arch_atomic64_fetch_xor_relaxed)
1405
+
static inline s64
1406
+
atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v)
1407
+
{
1408
+
kasan_check_write(v, sizeof(*v));
1409
+
return arch_atomic64_fetch_xor_relaxed(i, v);
1410
+
}
1411
+
#define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed
1412
+
#endif
1413
+
1414
+
#if !defined(arch_atomic64_xchg_relaxed) || defined(arch_atomic64_xchg)
1415
+
static inline s64
1416
+
atomic64_xchg(atomic64_t *v, s64 i)
1417
+
{
1418
+
kasan_check_write(v, sizeof(*v));
1419
+
return arch_atomic64_xchg(v, i);
1420
+
}
1421
+
#define atomic64_xchg atomic64_xchg
1422
+
#endif
1423
+
1424
+
#if defined(arch_atomic64_xchg_acquire)
1425
+
static inline s64
1426
+
atomic64_xchg_acquire(atomic64_t *v, s64 i)
1427
+
{
1428
+
kasan_check_write(v, sizeof(*v));
1429
+
return arch_atomic64_xchg_acquire(v, i);
1430
+
}
1431
+
#define atomic64_xchg_acquire atomic64_xchg_acquire
1432
+
#endif
1433
+
1434
+
#if defined(arch_atomic64_xchg_release)
1435
+
static inline s64
1436
+
atomic64_xchg_release(atomic64_t *v, s64 i)
1437
+
{
1438
+
kasan_check_write(v, sizeof(*v));
1439
+
return arch_atomic64_xchg_release(v, i);
1440
+
}
1441
+
#define atomic64_xchg_release atomic64_xchg_release
1442
+
#endif
1443
+
1444
+
#if defined(arch_atomic64_xchg_relaxed)
1445
+
static inline s64
1446
+
atomic64_xchg_relaxed(atomic64_t *v, s64 i)
1447
+
{
1448
+
kasan_check_write(v, sizeof(*v));
1449
+
return arch_atomic64_xchg_relaxed(v, i);
1450
+
}
1451
+
#define atomic64_xchg_relaxed atomic64_xchg_relaxed
1452
+
#endif
1453
+
1454
+
#if !defined(arch_atomic64_cmpxchg_relaxed) || defined(arch_atomic64_cmpxchg)
1455
+
static inline s64
1456
+
atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
1457
+
{
1458
+
kasan_check_write(v, sizeof(*v));
1459
+
return arch_atomic64_cmpxchg(v, old, new);
1460
+
}
1461
+
#define atomic64_cmpxchg atomic64_cmpxchg
1462
+
#endif
1463
+
1464
+
#if defined(arch_atomic64_cmpxchg_acquire)
1465
+
static inline s64
1466
+
atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1467
+
{
1468
+
kasan_check_write(v, sizeof(*v));
1469
+
return arch_atomic64_cmpxchg_acquire(v, old, new);
1470
+
}
1471
+
#define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire
1472
+
#endif
1473
+
1474
+
#if defined(arch_atomic64_cmpxchg_release)
1475
+
static inline s64
1476
+
atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1477
+
{
1478
+
kasan_check_write(v, sizeof(*v));
1479
+
return arch_atomic64_cmpxchg_release(v, old, new);
1480
+
}
1481
+
#define atomic64_cmpxchg_release atomic64_cmpxchg_release
1482
+
#endif
1483
+
1484
+
#if defined(arch_atomic64_cmpxchg_relaxed)
1485
+
static inline s64
1486
+
atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new)
1487
+
{
1488
+
kasan_check_write(v, sizeof(*v));
1489
+
return arch_atomic64_cmpxchg_relaxed(v, old, new);
1490
+
}
1491
+
#define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed
1492
+
#endif
1493
+
1494
+
#if defined(arch_atomic64_try_cmpxchg)
1495
+
static inline bool
1496
+
atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
1497
+
{
1498
+
kasan_check_write(v, sizeof(*v));
1499
+
kasan_check_write(old, sizeof(*old));
1500
+
return arch_atomic64_try_cmpxchg(v, old, new);
1501
+
}
1502
+
#define atomic64_try_cmpxchg atomic64_try_cmpxchg
1503
+
#endif
1504
+
1505
+
#if defined(arch_atomic64_try_cmpxchg_acquire)
1506
+
static inline bool
1507
+
atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
1508
+
{
1509
+
kasan_check_write(v, sizeof(*v));
1510
+
kasan_check_write(old, sizeof(*old));
1511
+
return arch_atomic64_try_cmpxchg_acquire(v, old, new);
1512
+
}
1513
+
#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire
1514
+
#endif
1515
+
1516
+
#if defined(arch_atomic64_try_cmpxchg_release)
1517
+
static inline bool
1518
+
atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
1519
+
{
1520
+
kasan_check_write(v, sizeof(*v));
1521
+
kasan_check_write(old, sizeof(*old));
1522
+
return arch_atomic64_try_cmpxchg_release(v, old, new);
1523
+
}
1524
+
#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release
1525
+
#endif
1526
+
1527
+
#if defined(arch_atomic64_try_cmpxchg_relaxed)
1528
+
static inline bool
1529
+
atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
1530
+
{
1531
+
kasan_check_write(v, sizeof(*v));
1532
+
kasan_check_write(old, sizeof(*old));
1533
+
return arch_atomic64_try_cmpxchg_relaxed(v, old, new);
1534
+
}
1535
+
#define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed
1536
+
#endif
1537
+
1538
+
#if defined(arch_atomic64_sub_and_test)
1539
+
static inline bool
1540
+
atomic64_sub_and_test(s64 i, atomic64_t *v)
1541
+
{
1542
+
kasan_check_write(v, sizeof(*v));
1543
+
return arch_atomic64_sub_and_test(i, v);
1544
+
}
1545
+
#define atomic64_sub_and_test atomic64_sub_and_test
1546
+
#endif
1547
+
1548
+
#if defined(arch_atomic64_dec_and_test)
1549
+
static inline bool
1550
+
atomic64_dec_and_test(atomic64_t *v)
1551
+
{
1552
+
kasan_check_write(v, sizeof(*v));
1553
+
return arch_atomic64_dec_and_test(v);
1554
+
}
1555
+
#define atomic64_dec_and_test atomic64_dec_and_test
1556
+
#endif
1557
+
1558
+
#if defined(arch_atomic64_inc_and_test)
1559
+
static inline bool
1560
+
atomic64_inc_and_test(atomic64_t *v)
1561
+
{
1562
+
kasan_check_write(v, sizeof(*v));
1563
+
return arch_atomic64_inc_and_test(v);
1564
+
}
1565
+
#define atomic64_inc_and_test atomic64_inc_and_test
1566
+
#endif
1567
+
1568
+
#if defined(arch_atomic64_add_negative)
1569
+
static inline bool
1570
+
atomic64_add_negative(s64 i, atomic64_t *v)
410
1571
{
411
1572
kasan_check_write(v, sizeof(*v));
412
1573
return arch_atomic64_add_negative(i, v);
413
1574
}
1575
+
#define atomic64_add_negative atomic64_add_negative
414
1576
#endif
415
1577
416
-
#define xchg(ptr, new) \
1578
+
#if defined(arch_atomic64_fetch_add_unless)
1579
+
static inline s64
1580
+
atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
1581
+
{
1582
+
kasan_check_write(v, sizeof(*v));
1583
+
return arch_atomic64_fetch_add_unless(v, a, u);
1584
+
}
1585
+
#define atomic64_fetch_add_unless atomic64_fetch_add_unless
1586
+
#endif
1587
+
1588
+
#if defined(arch_atomic64_add_unless)
1589
+
static inline bool
1590
+
atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
1591
+
{
1592
+
kasan_check_write(v, sizeof(*v));
1593
+
return arch_atomic64_add_unless(v, a, u);
1594
+
}
1595
+
#define atomic64_add_unless atomic64_add_unless
1596
+
#endif
1597
+
1598
+
#if defined(arch_atomic64_inc_not_zero)
1599
+
static inline bool
1600
+
atomic64_inc_not_zero(atomic64_t *v)
1601
+
{
1602
+
kasan_check_write(v, sizeof(*v));
1603
+
return arch_atomic64_inc_not_zero(v);
1604
+
}
1605
+
#define atomic64_inc_not_zero atomic64_inc_not_zero
1606
+
#endif
1607
+
1608
+
#if defined(arch_atomic64_inc_unless_negative)
1609
+
static inline bool
1610
+
atomic64_inc_unless_negative(atomic64_t *v)
1611
+
{
1612
+
kasan_check_write(v, sizeof(*v));
1613
+
return arch_atomic64_inc_unless_negative(v);
1614
+
}
1615
+
#define atomic64_inc_unless_negative atomic64_inc_unless_negative
1616
+
#endif
1617
+
1618
+
#if defined(arch_atomic64_dec_unless_positive)
1619
+
static inline bool
1620
+
atomic64_dec_unless_positive(atomic64_t *v)
1621
+
{
1622
+
kasan_check_write(v, sizeof(*v));
1623
+
return arch_atomic64_dec_unless_positive(v);
1624
+
}
1625
+
#define atomic64_dec_unless_positive atomic64_dec_unless_positive
1626
+
#endif
1627
+
1628
+
#if defined(arch_atomic64_dec_if_positive)
1629
+
static inline s64
1630
+
atomic64_dec_if_positive(atomic64_t *v)
1631
+
{
1632
+
kasan_check_write(v, sizeof(*v));
1633
+
return arch_atomic64_dec_if_positive(v);
1634
+
}
1635
+
#define atomic64_dec_if_positive atomic64_dec_if_positive
1636
+
#endif
1637
+
1638
+
#if !defined(arch_xchg_relaxed) || defined(arch_xchg)
1639
+
#define xchg(ptr, ...) \
417
1640
({ \
418
1641
typeof(ptr) __ai_ptr = (ptr); \
419
-
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
420
-
arch_xchg(__ai_ptr, (new)); \
1642
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1643
+
arch_xchg(__ai_ptr, __VA_ARGS__); \
421
1644
})
1645
+
#endif
422
1646
423
-
#define cmpxchg(ptr, old, new) \
1647
+
#if defined(arch_xchg_acquire)
1648
+
#define xchg_acquire(ptr, ...) \
424
1649
({ \
425
1650
typeof(ptr) __ai_ptr = (ptr); \
426
-
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
427
-
arch_cmpxchg(__ai_ptr, (old), (new)); \
1651
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1652
+
arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \
428
1653
})
1654
+
#endif
429
1655
430
-
#define sync_cmpxchg(ptr, old, new) \
1656
+
#if defined(arch_xchg_release)
1657
+
#define xchg_release(ptr, ...) \
431
1658
({ \
432
1659
typeof(ptr) __ai_ptr = (ptr); \
433
-
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
434
-
arch_sync_cmpxchg(__ai_ptr, (old), (new)); \
1660
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1661
+
arch_xchg_release(__ai_ptr, __VA_ARGS__); \
435
1662
})
1663
+
#endif
436
1664
437
-
#define cmpxchg_local(ptr, old, new) \
1665
+
#if defined(arch_xchg_relaxed)
1666
+
#define xchg_relaxed(ptr, ...) \
438
1667
({ \
439
1668
typeof(ptr) __ai_ptr = (ptr); \
440
-
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
441
-
arch_cmpxchg_local(__ai_ptr, (old), (new)); \
1669
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1670
+
arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \
442
1671
})
1672
+
#endif
443
1673
444
-
#define cmpxchg64(ptr, old, new) \
1674
+
#if !defined(arch_cmpxchg_relaxed) || defined(arch_cmpxchg)
1675
+
#define cmpxchg(ptr, ...) \
445
1676
({ \
446
1677
typeof(ptr) __ai_ptr = (ptr); \
447
-
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
448
-
arch_cmpxchg64(__ai_ptr, (old), (new)); \
1678
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1679
+
arch_cmpxchg(__ai_ptr, __VA_ARGS__); \
449
1680
})
1681
+
#endif
450
1682
451
-
#define cmpxchg64_local(ptr, old, new) \
1683
+
#if defined(arch_cmpxchg_acquire)
1684
+
#define cmpxchg_acquire(ptr, ...) \
452
1685
({ \
453
1686
typeof(ptr) __ai_ptr = (ptr); \
454
-
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
455
-
arch_cmpxchg64_local(__ai_ptr, (old), (new)); \
1687
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1688
+
arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \
456
1689
})
1690
+
#endif
457
1691
458
-
#define cmpxchg_double(p1, p2, o1, o2, n1, n2) \
1692
+
#if defined(arch_cmpxchg_release)
1693
+
#define cmpxchg_release(ptr, ...) \
459
1694
({ \
460
-
typeof(p1) __ai_p1 = (p1); \
461
-
kasan_check_write(__ai_p1, 2 * sizeof(*__ai_p1)); \
462
-
arch_cmpxchg_double(__ai_p1, (p2), (o1), (o2), (n1), (n2)); \
1695
+
typeof(ptr) __ai_ptr = (ptr); \
1696
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1697
+
arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \
1698
+
})
1699
+
#endif
1700
+
1701
+
#if defined(arch_cmpxchg_relaxed)
1702
+
#define cmpxchg_relaxed(ptr, ...) \
1703
+
({ \
1704
+
typeof(ptr) __ai_ptr = (ptr); \
1705
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1706
+
arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \
1707
+
})
1708
+
#endif
1709
+
1710
+
#if !defined(arch_cmpxchg64_relaxed) || defined(arch_cmpxchg64)
1711
+
#define cmpxchg64(ptr, ...) \
1712
+
({ \
1713
+
typeof(ptr) __ai_ptr = (ptr); \
1714
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1715
+
arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \
1716
+
})
1717
+
#endif
1718
+
1719
+
#if defined(arch_cmpxchg64_acquire)
1720
+
#define cmpxchg64_acquire(ptr, ...) \
1721
+
({ \
1722
+
typeof(ptr) __ai_ptr = (ptr); \
1723
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1724
+
arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \
1725
+
})
1726
+
#endif
1727
+
1728
+
#if defined(arch_cmpxchg64_release)
1729
+
#define cmpxchg64_release(ptr, ...) \
1730
+
({ \
1731
+
typeof(ptr) __ai_ptr = (ptr); \
1732
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1733
+
arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \
1734
+
})
1735
+
#endif
1736
+
1737
+
#if defined(arch_cmpxchg64_relaxed)
1738
+
#define cmpxchg64_relaxed(ptr, ...) \
1739
+
({ \
1740
+
typeof(ptr) __ai_ptr = (ptr); \
1741
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1742
+
arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \
1743
+
})
1744
+
#endif
1745
+
1746
+
#define cmpxchg_local(ptr, ...) \
1747
+
({ \
1748
+
typeof(ptr) __ai_ptr = (ptr); \
1749
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1750
+
arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \
463
1751
})
464
1752
465
-
#define cmpxchg_double_local(p1, p2, o1, o2, n1, n2) \
466
-
({ \
467
-
typeof(p1) __ai_p1 = (p1); \
468
-
kasan_check_write(__ai_p1, 2 * sizeof(*__ai_p1)); \
469
-
arch_cmpxchg_double_local(__ai_p1, (p2), (o1), (o2), (n1), (n2)); \
1753
+
#define cmpxchg64_local(ptr, ...) \
1754
+
({ \
1755
+
typeof(ptr) __ai_ptr = (ptr); \
1756
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1757
+
arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \
470
1758
})
471
1759
472
-
#endif /* _LINUX_ATOMIC_INSTRUMENTED_H */
1760
+
#define sync_cmpxchg(ptr, ...) \
1761
+
({ \
1762
+
typeof(ptr) __ai_ptr = (ptr); \
1763
+
kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \
1764
+
arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \
1765
+
})
1766
+
1767
+
#define cmpxchg_double(ptr, ...) \
1768
+
({ \
1769
+
typeof(ptr) __ai_ptr = (ptr); \
1770
+
kasan_check_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1771
+
arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \
1772
+
})
1773
+
1774
+
1775
+
#define cmpxchg_double_local(ptr, ...) \
1776
+
({ \
1777
+
typeof(ptr) __ai_ptr = (ptr); \
1778
+
kasan_check_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \
1779
+
arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \
1780
+
})
1781
+
1782
+
#endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */
+978
-235
include/asm-generic/atomic-long.h
+978
-235
include/asm-generic/atomic-long.h
···
1
-
/* SPDX-License-Identifier: GPL-2.0 */
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
3
+
// Generated by scripts/atomic/gen-atomic-long.sh
4
+
// DO NOT MODIFY THIS FILE DIRECTLY
5
+
2
6
#ifndef _ASM_GENERIC_ATOMIC_LONG_H
3
7
#define _ASM_GENERIC_ATOMIC_LONG_H
4
-
/*
5
-
* Copyright (C) 2005 Silicon Graphics, Inc.
6
-
* Christoph Lameter
7
-
*
8
-
* Allows to provide arch independent atomic definitions without the need to
9
-
* edit all arch specific atomic.h files.
10
-
*/
11
8
12
9
#include <asm/types.h>
13
10
14
-
/*
15
-
* Suppport for atomic_long_t
16
-
*
17
-
* Casts for parameters are avoided for existing atomic functions in order to
18
-
* avoid issues with cast-as-lval under gcc 4.x and other limitations that the
19
-
* macros of a platform may have.
20
-
*/
21
-
22
-
#if BITS_PER_LONG == 64
23
-
11
+
#ifdef CONFIG_64BIT
24
12
typedef atomic64_t atomic_long_t;
25
-
26
-
#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i)
27
-
#define ATOMIC_LONG_PFX(x) atomic64 ## x
28
-
#define ATOMIC_LONG_TYPE s64
29
-
13
+
#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i)
14
+
#define atomic_long_cond_read_acquire atomic64_cond_read_acquire
15
+
#define atomic_long_cond_read_relaxed atomic64_cond_read_relaxed
30
16
#else
31
-
32
17
typedef atomic_t atomic_long_t;
33
-
34
-
#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i)
35
-
#define ATOMIC_LONG_PFX(x) atomic ## x
36
-
#define ATOMIC_LONG_TYPE int
37
-
18
+
#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i)
19
+
#define atomic_long_cond_read_acquire atomic_cond_read_acquire
20
+
#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed
38
21
#endif
39
22
40
-
#define ATOMIC_LONG_READ_OP(mo) \
41
-
static inline long atomic_long_read##mo(const atomic_long_t *l) \
42
-
{ \
43
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
44
-
\
45
-
return (long)ATOMIC_LONG_PFX(_read##mo)(v); \
46
-
}
47
-
ATOMIC_LONG_READ_OP()
48
-
ATOMIC_LONG_READ_OP(_acquire)
23
+
#ifdef CONFIG_64BIT
49
24
50
-
#undef ATOMIC_LONG_READ_OP
51
-
52
-
#define ATOMIC_LONG_SET_OP(mo) \
53
-
static inline void atomic_long_set##mo(atomic_long_t *l, long i) \
54
-
{ \
55
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
56
-
\
57
-
ATOMIC_LONG_PFX(_set##mo)(v, i); \
58
-
}
59
-
ATOMIC_LONG_SET_OP()
60
-
ATOMIC_LONG_SET_OP(_release)
61
-
62
-
#undef ATOMIC_LONG_SET_OP
63
-
64
-
#define ATOMIC_LONG_ADD_SUB_OP(op, mo) \
65
-
static inline long \
66
-
atomic_long_##op##_return##mo(long i, atomic_long_t *l) \
67
-
{ \
68
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
69
-
\
70
-
return (long)ATOMIC_LONG_PFX(_##op##_return##mo)(i, v); \
71
-
}
72
-
ATOMIC_LONG_ADD_SUB_OP(add,)
73
-
ATOMIC_LONG_ADD_SUB_OP(add, _relaxed)
74
-
ATOMIC_LONG_ADD_SUB_OP(add, _acquire)
75
-
ATOMIC_LONG_ADD_SUB_OP(add, _release)
76
-
ATOMIC_LONG_ADD_SUB_OP(sub,)
77
-
ATOMIC_LONG_ADD_SUB_OP(sub, _relaxed)
78
-
ATOMIC_LONG_ADD_SUB_OP(sub, _acquire)
79
-
ATOMIC_LONG_ADD_SUB_OP(sub, _release)
80
-
81
-
#undef ATOMIC_LONG_ADD_SUB_OP
82
-
83
-
#define atomic_long_cmpxchg_relaxed(l, old, new) \
84
-
(ATOMIC_LONG_PFX(_cmpxchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(l), \
85
-
(old), (new)))
86
-
#define atomic_long_cmpxchg_acquire(l, old, new) \
87
-
(ATOMIC_LONG_PFX(_cmpxchg_acquire)((ATOMIC_LONG_PFX(_t) *)(l), \
88
-
(old), (new)))
89
-
#define atomic_long_cmpxchg_release(l, old, new) \
90
-
(ATOMIC_LONG_PFX(_cmpxchg_release)((ATOMIC_LONG_PFX(_t) *)(l), \
91
-
(old), (new)))
92
-
#define atomic_long_cmpxchg(l, old, new) \
93
-
(ATOMIC_LONG_PFX(_cmpxchg)((ATOMIC_LONG_PFX(_t) *)(l), (old), (new)))
94
-
95
-
96
-
#define atomic_long_try_cmpxchg_relaxed(l, old, new) \
97
-
(ATOMIC_LONG_PFX(_try_cmpxchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(l), \
98
-
(ATOMIC_LONG_TYPE *)(old), (ATOMIC_LONG_TYPE)(new)))
99
-
#define atomic_long_try_cmpxchg_acquire(l, old, new) \
100
-
(ATOMIC_LONG_PFX(_try_cmpxchg_acquire)((ATOMIC_LONG_PFX(_t) *)(l), \
101
-
(ATOMIC_LONG_TYPE *)(old), (ATOMIC_LONG_TYPE)(new)))
102
-
#define atomic_long_try_cmpxchg_release(l, old, new) \
103
-
(ATOMIC_LONG_PFX(_try_cmpxchg_release)((ATOMIC_LONG_PFX(_t) *)(l), \
104
-
(ATOMIC_LONG_TYPE *)(old), (ATOMIC_LONG_TYPE)(new)))
105
-
#define atomic_long_try_cmpxchg(l, old, new) \
106
-
(ATOMIC_LONG_PFX(_try_cmpxchg)((ATOMIC_LONG_PFX(_t) *)(l), \
107
-
(ATOMIC_LONG_TYPE *)(old), (ATOMIC_LONG_TYPE)(new)))
108
-
109
-
110
-
#define atomic_long_xchg_relaxed(v, new) \
111
-
(ATOMIC_LONG_PFX(_xchg_relaxed)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
112
-
#define atomic_long_xchg_acquire(v, new) \
113
-
(ATOMIC_LONG_PFX(_xchg_acquire)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
114
-
#define atomic_long_xchg_release(v, new) \
115
-
(ATOMIC_LONG_PFX(_xchg_release)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
116
-
#define atomic_long_xchg(v, new) \
117
-
(ATOMIC_LONG_PFX(_xchg)((ATOMIC_LONG_PFX(_t) *)(v), (new)))
118
-
119
-
static __always_inline void atomic_long_inc(atomic_long_t *l)
25
+
static inline long
26
+
atomic_long_read(const atomic_long_t *v)
120
27
{
121
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
122
-
123
-
ATOMIC_LONG_PFX(_inc)(v);
28
+
return atomic64_read(v);
124
29
}
125
30
126
-
static __always_inline void atomic_long_dec(atomic_long_t *l)
31
+
static inline long
32
+
atomic_long_read_acquire(const atomic_long_t *v)
127
33
{
128
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
129
-
130
-
ATOMIC_LONG_PFX(_dec)(v);
34
+
return atomic64_read_acquire(v);
131
35
}
132
36
133
-
#define ATOMIC_LONG_FETCH_OP(op, mo) \
134
-
static inline long \
135
-
atomic_long_fetch_##op##mo(long i, atomic_long_t *l) \
136
-
{ \
137
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
138
-
\
139
-
return (long)ATOMIC_LONG_PFX(_fetch_##op##mo)(i, v); \
140
-
}
141
-
142
-
ATOMIC_LONG_FETCH_OP(add, )
143
-
ATOMIC_LONG_FETCH_OP(add, _relaxed)
144
-
ATOMIC_LONG_FETCH_OP(add, _acquire)
145
-
ATOMIC_LONG_FETCH_OP(add, _release)
146
-
ATOMIC_LONG_FETCH_OP(sub, )
147
-
ATOMIC_LONG_FETCH_OP(sub, _relaxed)
148
-
ATOMIC_LONG_FETCH_OP(sub, _acquire)
149
-
ATOMIC_LONG_FETCH_OP(sub, _release)
150
-
ATOMIC_LONG_FETCH_OP(and, )
151
-
ATOMIC_LONG_FETCH_OP(and, _relaxed)
152
-
ATOMIC_LONG_FETCH_OP(and, _acquire)
153
-
ATOMIC_LONG_FETCH_OP(and, _release)
154
-
ATOMIC_LONG_FETCH_OP(andnot, )
155
-
ATOMIC_LONG_FETCH_OP(andnot, _relaxed)
156
-
ATOMIC_LONG_FETCH_OP(andnot, _acquire)
157
-
ATOMIC_LONG_FETCH_OP(andnot, _release)
158
-
ATOMIC_LONG_FETCH_OP(or, )
159
-
ATOMIC_LONG_FETCH_OP(or, _relaxed)
160
-
ATOMIC_LONG_FETCH_OP(or, _acquire)
161
-
ATOMIC_LONG_FETCH_OP(or, _release)
162
-
ATOMIC_LONG_FETCH_OP(xor, )
163
-
ATOMIC_LONG_FETCH_OP(xor, _relaxed)
164
-
ATOMIC_LONG_FETCH_OP(xor, _acquire)
165
-
ATOMIC_LONG_FETCH_OP(xor, _release)
166
-
167
-
#undef ATOMIC_LONG_FETCH_OP
168
-
169
-
#define ATOMIC_LONG_FETCH_INC_DEC_OP(op, mo) \
170
-
static inline long \
171
-
atomic_long_fetch_##op##mo(atomic_long_t *l) \
172
-
{ \
173
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
174
-
\
175
-
return (long)ATOMIC_LONG_PFX(_fetch_##op##mo)(v); \
176
-
}
177
-
178
-
ATOMIC_LONG_FETCH_INC_DEC_OP(inc,)
179
-
ATOMIC_LONG_FETCH_INC_DEC_OP(inc, _relaxed)
180
-
ATOMIC_LONG_FETCH_INC_DEC_OP(inc, _acquire)
181
-
ATOMIC_LONG_FETCH_INC_DEC_OP(inc, _release)
182
-
ATOMIC_LONG_FETCH_INC_DEC_OP(dec,)
183
-
ATOMIC_LONG_FETCH_INC_DEC_OP(dec, _relaxed)
184
-
ATOMIC_LONG_FETCH_INC_DEC_OP(dec, _acquire)
185
-
ATOMIC_LONG_FETCH_INC_DEC_OP(dec, _release)
186
-
187
-
#undef ATOMIC_LONG_FETCH_INC_DEC_OP
188
-
189
-
#define ATOMIC_LONG_OP(op) \
190
-
static __always_inline void \
191
-
atomic_long_##op(long i, atomic_long_t *l) \
192
-
{ \
193
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
194
-
\
195
-
ATOMIC_LONG_PFX(_##op)(i, v); \
196
-
}
197
-
198
-
ATOMIC_LONG_OP(add)
199
-
ATOMIC_LONG_OP(sub)
200
-
ATOMIC_LONG_OP(and)
201
-
ATOMIC_LONG_OP(andnot)
202
-
ATOMIC_LONG_OP(or)
203
-
ATOMIC_LONG_OP(xor)
204
-
205
-
#undef ATOMIC_LONG_OP
206
-
207
-
static inline int atomic_long_sub_and_test(long i, atomic_long_t *l)
37
+
static inline void
38
+
atomic_long_set(atomic_long_t *v, long i)
208
39
{
209
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
210
-
211
-
return ATOMIC_LONG_PFX(_sub_and_test)(i, v);
40
+
atomic64_set(v, i);
212
41
}
213
42
214
-
static inline int atomic_long_dec_and_test(atomic_long_t *l)
43
+
static inline void
44
+
atomic_long_set_release(atomic_long_t *v, long i)
215
45
{
216
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
217
-
218
-
return ATOMIC_LONG_PFX(_dec_and_test)(v);
46
+
atomic64_set_release(v, i);
219
47
}
220
48
221
-
static inline int atomic_long_inc_and_test(atomic_long_t *l)
49
+
static inline void
50
+
atomic_long_add(long i, atomic_long_t *v)
222
51
{
223
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
224
-
225
-
return ATOMIC_LONG_PFX(_inc_and_test)(v);
52
+
atomic64_add(i, v);
226
53
}
227
54
228
-
static inline int atomic_long_add_negative(long i, atomic_long_t *l)
55
+
static inline long
56
+
atomic_long_add_return(long i, atomic_long_t *v)
229
57
{
230
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
231
-
232
-
return ATOMIC_LONG_PFX(_add_negative)(i, v);
58
+
return atomic64_add_return(i, v);
233
59
}
234
60
235
-
#define ATOMIC_LONG_INC_DEC_OP(op, mo) \
236
-
static inline long \
237
-
atomic_long_##op##_return##mo(atomic_long_t *l) \
238
-
{ \
239
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l; \
240
-
\
241
-
return (long)ATOMIC_LONG_PFX(_##op##_return##mo)(v); \
242
-
}
243
-
ATOMIC_LONG_INC_DEC_OP(inc,)
244
-
ATOMIC_LONG_INC_DEC_OP(inc, _relaxed)
245
-
ATOMIC_LONG_INC_DEC_OP(inc, _acquire)
246
-
ATOMIC_LONG_INC_DEC_OP(inc, _release)
247
-
ATOMIC_LONG_INC_DEC_OP(dec,)
248
-
ATOMIC_LONG_INC_DEC_OP(dec, _relaxed)
249
-
ATOMIC_LONG_INC_DEC_OP(dec, _acquire)
250
-
ATOMIC_LONG_INC_DEC_OP(dec, _release)
251
-
252
-
#undef ATOMIC_LONG_INC_DEC_OP
253
-
254
-
static inline long atomic_long_add_unless(atomic_long_t *l, long a, long u)
61
+
static inline long
62
+
atomic_long_add_return_acquire(long i, atomic_long_t *v)
255
63
{
256
-
ATOMIC_LONG_PFX(_t) *v = (ATOMIC_LONG_PFX(_t) *)l;
257
-
258
-
return (long)ATOMIC_LONG_PFX(_add_unless)(v, a, u);
64
+
return atomic64_add_return_acquire(i, v);
259
65
}
260
66
261
-
#define atomic_long_inc_not_zero(l) \
262
-
ATOMIC_LONG_PFX(_inc_not_zero)((ATOMIC_LONG_PFX(_t) *)(l))
67
+
static inline long
68
+
atomic_long_add_return_release(long i, atomic_long_t *v)
69
+
{
70
+
return atomic64_add_return_release(i, v);
71
+
}
263
72
264
-
#define atomic_long_cond_read_relaxed(v, c) \
265
-
ATOMIC_LONG_PFX(_cond_read_relaxed)((ATOMIC_LONG_PFX(_t) *)(v), (c))
266
-
#define atomic_long_cond_read_acquire(v, c) \
267
-
ATOMIC_LONG_PFX(_cond_read_acquire)((ATOMIC_LONG_PFX(_t) *)(v), (c))
73
+
static inline long
74
+
atomic_long_add_return_relaxed(long i, atomic_long_t *v)
75
+
{
76
+
return atomic64_add_return_relaxed(i, v);
77
+
}
268
78
269
-
#endif /* _ASM_GENERIC_ATOMIC_LONG_H */
79
+
static inline long
80
+
atomic_long_fetch_add(long i, atomic_long_t *v)
81
+
{
82
+
return atomic64_fetch_add(i, v);
83
+
}
84
+
85
+
static inline long
86
+
atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
87
+
{
88
+
return atomic64_fetch_add_acquire(i, v);
89
+
}
90
+
91
+
static inline long
92
+
atomic_long_fetch_add_release(long i, atomic_long_t *v)
93
+
{
94
+
return atomic64_fetch_add_release(i, v);
95
+
}
96
+
97
+
static inline long
98
+
atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
99
+
{
100
+
return atomic64_fetch_add_relaxed(i, v);
101
+
}
102
+
103
+
static inline void
104
+
atomic_long_sub(long i, atomic_long_t *v)
105
+
{
106
+
atomic64_sub(i, v);
107
+
}
108
+
109
+
static inline long
110
+
atomic_long_sub_return(long i, atomic_long_t *v)
111
+
{
112
+
return atomic64_sub_return(i, v);
113
+
}
114
+
115
+
static inline long
116
+
atomic_long_sub_return_acquire(long i, atomic_long_t *v)
117
+
{
118
+
return atomic64_sub_return_acquire(i, v);
119
+
}
120
+
121
+
static inline long
122
+
atomic_long_sub_return_release(long i, atomic_long_t *v)
123
+
{
124
+
return atomic64_sub_return_release(i, v);
125
+
}
126
+
127
+
static inline long
128
+
atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
129
+
{
130
+
return atomic64_sub_return_relaxed(i, v);
131
+
}
132
+
133
+
static inline long
134
+
atomic_long_fetch_sub(long i, atomic_long_t *v)
135
+
{
136
+
return atomic64_fetch_sub(i, v);
137
+
}
138
+
139
+
static inline long
140
+
atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
141
+
{
142
+
return atomic64_fetch_sub_acquire(i, v);
143
+
}
144
+
145
+
static inline long
146
+
atomic_long_fetch_sub_release(long i, atomic_long_t *v)
147
+
{
148
+
return atomic64_fetch_sub_release(i, v);
149
+
}
150
+
151
+
static inline long
152
+
atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
153
+
{
154
+
return atomic64_fetch_sub_relaxed(i, v);
155
+
}
156
+
157
+
static inline void
158
+
atomic_long_inc(atomic_long_t *v)
159
+
{
160
+
atomic64_inc(v);
161
+
}
162
+
163
+
static inline long
164
+
atomic_long_inc_return(atomic_long_t *v)
165
+
{
166
+
return atomic64_inc_return(v);
167
+
}
168
+
169
+
static inline long
170
+
atomic_long_inc_return_acquire(atomic_long_t *v)
171
+
{
172
+
return atomic64_inc_return_acquire(v);
173
+
}
174
+
175
+
static inline long
176
+
atomic_long_inc_return_release(atomic_long_t *v)
177
+
{
178
+
return atomic64_inc_return_release(v);
179
+
}
180
+
181
+
static inline long
182
+
atomic_long_inc_return_relaxed(atomic_long_t *v)
183
+
{
184
+
return atomic64_inc_return_relaxed(v);
185
+
}
186
+
187
+
static inline long
188
+
atomic_long_fetch_inc(atomic_long_t *v)
189
+
{
190
+
return atomic64_fetch_inc(v);
191
+
}
192
+
193
+
static inline long
194
+
atomic_long_fetch_inc_acquire(atomic_long_t *v)
195
+
{
196
+
return atomic64_fetch_inc_acquire(v);
197
+
}
198
+
199
+
static inline long
200
+
atomic_long_fetch_inc_release(atomic_long_t *v)
201
+
{
202
+
return atomic64_fetch_inc_release(v);
203
+
}
204
+
205
+
static inline long
206
+
atomic_long_fetch_inc_relaxed(atomic_long_t *v)
207
+
{
208
+
return atomic64_fetch_inc_relaxed(v);
209
+
}
210
+
211
+
static inline void
212
+
atomic_long_dec(atomic_long_t *v)
213
+
{
214
+
atomic64_dec(v);
215
+
}
216
+
217
+
static inline long
218
+
atomic_long_dec_return(atomic_long_t *v)
219
+
{
220
+
return atomic64_dec_return(v);
221
+
}
222
+
223
+
static inline long
224
+
atomic_long_dec_return_acquire(atomic_long_t *v)
225
+
{
226
+
return atomic64_dec_return_acquire(v);
227
+
}
228
+
229
+
static inline long
230
+
atomic_long_dec_return_release(atomic_long_t *v)
231
+
{
232
+
return atomic64_dec_return_release(v);
233
+
}
234
+
235
+
static inline long
236
+
atomic_long_dec_return_relaxed(atomic_long_t *v)
237
+
{
238
+
return atomic64_dec_return_relaxed(v);
239
+
}
240
+
241
+
static inline long
242
+
atomic_long_fetch_dec(atomic_long_t *v)
243
+
{
244
+
return atomic64_fetch_dec(v);
245
+
}
246
+
247
+
static inline long
248
+
atomic_long_fetch_dec_acquire(atomic_long_t *v)
249
+
{
250
+
return atomic64_fetch_dec_acquire(v);
251
+
}
252
+
253
+
static inline long
254
+
atomic_long_fetch_dec_release(atomic_long_t *v)
255
+
{
256
+
return atomic64_fetch_dec_release(v);
257
+
}
258
+
259
+
static inline long
260
+
atomic_long_fetch_dec_relaxed(atomic_long_t *v)
261
+
{
262
+
return atomic64_fetch_dec_relaxed(v);
263
+
}
264
+
265
+
static inline void
266
+
atomic_long_and(long i, atomic_long_t *v)
267
+
{
268
+
atomic64_and(i, v);
269
+
}
270
+
271
+
static inline long
272
+
atomic_long_fetch_and(long i, atomic_long_t *v)
273
+
{
274
+
return atomic64_fetch_and(i, v);
275
+
}
276
+
277
+
static inline long
278
+
atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
279
+
{
280
+
return atomic64_fetch_and_acquire(i, v);
281
+
}
282
+
283
+
static inline long
284
+
atomic_long_fetch_and_release(long i, atomic_long_t *v)
285
+
{
286
+
return atomic64_fetch_and_release(i, v);
287
+
}
288
+
289
+
static inline long
290
+
atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
291
+
{
292
+
return atomic64_fetch_and_relaxed(i, v);
293
+
}
294
+
295
+
static inline void
296
+
atomic_long_andnot(long i, atomic_long_t *v)
297
+
{
298
+
atomic64_andnot(i, v);
299
+
}
300
+
301
+
static inline long
302
+
atomic_long_fetch_andnot(long i, atomic_long_t *v)
303
+
{
304
+
return atomic64_fetch_andnot(i, v);
305
+
}
306
+
307
+
static inline long
308
+
atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
309
+
{
310
+
return atomic64_fetch_andnot_acquire(i, v);
311
+
}
312
+
313
+
static inline long
314
+
atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
315
+
{
316
+
return atomic64_fetch_andnot_release(i, v);
317
+
}
318
+
319
+
static inline long
320
+
atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
321
+
{
322
+
return atomic64_fetch_andnot_relaxed(i, v);
323
+
}
324
+
325
+
static inline void
326
+
atomic_long_or(long i, atomic_long_t *v)
327
+
{
328
+
atomic64_or(i, v);
329
+
}
330
+
331
+
static inline long
332
+
atomic_long_fetch_or(long i, atomic_long_t *v)
333
+
{
334
+
return atomic64_fetch_or(i, v);
335
+
}
336
+
337
+
static inline long
338
+
atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
339
+
{
340
+
return atomic64_fetch_or_acquire(i, v);
341
+
}
342
+
343
+
static inline long
344
+
atomic_long_fetch_or_release(long i, atomic_long_t *v)
345
+
{
346
+
return atomic64_fetch_or_release(i, v);
347
+
}
348
+
349
+
static inline long
350
+
atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
351
+
{
352
+
return atomic64_fetch_or_relaxed(i, v);
353
+
}
354
+
355
+
static inline void
356
+
atomic_long_xor(long i, atomic_long_t *v)
357
+
{
358
+
atomic64_xor(i, v);
359
+
}
360
+
361
+
static inline long
362
+
atomic_long_fetch_xor(long i, atomic_long_t *v)
363
+
{
364
+
return atomic64_fetch_xor(i, v);
365
+
}
366
+
367
+
static inline long
368
+
atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
369
+
{
370
+
return atomic64_fetch_xor_acquire(i, v);
371
+
}
372
+
373
+
static inline long
374
+
atomic_long_fetch_xor_release(long i, atomic_long_t *v)
375
+
{
376
+
return atomic64_fetch_xor_release(i, v);
377
+
}
378
+
379
+
static inline long
380
+
atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
381
+
{
382
+
return atomic64_fetch_xor_relaxed(i, v);
383
+
}
384
+
385
+
static inline long
386
+
atomic_long_xchg(atomic_long_t *v, long i)
387
+
{
388
+
return atomic64_xchg(v, i);
389
+
}
390
+
391
+
static inline long
392
+
atomic_long_xchg_acquire(atomic_long_t *v, long i)
393
+
{
394
+
return atomic64_xchg_acquire(v, i);
395
+
}
396
+
397
+
static inline long
398
+
atomic_long_xchg_release(atomic_long_t *v, long i)
399
+
{
400
+
return atomic64_xchg_release(v, i);
401
+
}
402
+
403
+
static inline long
404
+
atomic_long_xchg_relaxed(atomic_long_t *v, long i)
405
+
{
406
+
return atomic64_xchg_relaxed(v, i);
407
+
}
408
+
409
+
static inline long
410
+
atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
411
+
{
412
+
return atomic64_cmpxchg(v, old, new);
413
+
}
414
+
415
+
static inline long
416
+
atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
417
+
{
418
+
return atomic64_cmpxchg_acquire(v, old, new);
419
+
}
420
+
421
+
static inline long
422
+
atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
423
+
{
424
+
return atomic64_cmpxchg_release(v, old, new);
425
+
}
426
+
427
+
static inline long
428
+
atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
429
+
{
430
+
return atomic64_cmpxchg_relaxed(v, old, new);
431
+
}
432
+
433
+
static inline bool
434
+
atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
435
+
{
436
+
return atomic64_try_cmpxchg(v, (s64 *)old, new);
437
+
}
438
+
439
+
static inline bool
440
+
atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
441
+
{
442
+
return atomic64_try_cmpxchg_acquire(v, (s64 *)old, new);
443
+
}
444
+
445
+
static inline bool
446
+
atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
447
+
{
448
+
return atomic64_try_cmpxchg_release(v, (s64 *)old, new);
449
+
}
450
+
451
+
static inline bool
452
+
atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
453
+
{
454
+
return atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new);
455
+
}
456
+
457
+
static inline bool
458
+
atomic_long_sub_and_test(long i, atomic_long_t *v)
459
+
{
460
+
return atomic64_sub_and_test(i, v);
461
+
}
462
+
463
+
static inline bool
464
+
atomic_long_dec_and_test(atomic_long_t *v)
465
+
{
466
+
return atomic64_dec_and_test(v);
467
+
}
468
+
469
+
static inline bool
470
+
atomic_long_inc_and_test(atomic_long_t *v)
471
+
{
472
+
return atomic64_inc_and_test(v);
473
+
}
474
+
475
+
static inline bool
476
+
atomic_long_add_negative(long i, atomic_long_t *v)
477
+
{
478
+
return atomic64_add_negative(i, v);
479
+
}
480
+
481
+
static inline long
482
+
atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
483
+
{
484
+
return atomic64_fetch_add_unless(v, a, u);
485
+
}
486
+
487
+
static inline bool
488
+
atomic_long_add_unless(atomic_long_t *v, long a, long u)
489
+
{
490
+
return atomic64_add_unless(v, a, u);
491
+
}
492
+
493
+
static inline bool
494
+
atomic_long_inc_not_zero(atomic_long_t *v)
495
+
{
496
+
return atomic64_inc_not_zero(v);
497
+
}
498
+
499
+
static inline bool
500
+
atomic_long_inc_unless_negative(atomic_long_t *v)
501
+
{
502
+
return atomic64_inc_unless_negative(v);
503
+
}
504
+
505
+
static inline bool
506
+
atomic_long_dec_unless_positive(atomic_long_t *v)
507
+
{
508
+
return atomic64_dec_unless_positive(v);
509
+
}
510
+
511
+
static inline long
512
+
atomic_long_dec_if_positive(atomic_long_t *v)
513
+
{
514
+
return atomic64_dec_if_positive(v);
515
+
}
516
+
517
+
#else /* CONFIG_64BIT */
518
+
519
+
static inline long
520
+
atomic_long_read(const atomic_long_t *v)
521
+
{
522
+
return atomic_read(v);
523
+
}
524
+
525
+
static inline long
526
+
atomic_long_read_acquire(const atomic_long_t *v)
527
+
{
528
+
return atomic_read_acquire(v);
529
+
}
530
+
531
+
static inline void
532
+
atomic_long_set(atomic_long_t *v, long i)
533
+
{
534
+
atomic_set(v, i);
535
+
}
536
+
537
+
static inline void
538
+
atomic_long_set_release(atomic_long_t *v, long i)
539
+
{
540
+
atomic_set_release(v, i);
541
+
}
542
+
543
+
static inline void
544
+
atomic_long_add(long i, atomic_long_t *v)
545
+
{
546
+
atomic_add(i, v);
547
+
}
548
+
549
+
static inline long
550
+
atomic_long_add_return(long i, atomic_long_t *v)
551
+
{
552
+
return atomic_add_return(i, v);
553
+
}
554
+
555
+
static inline long
556
+
atomic_long_add_return_acquire(long i, atomic_long_t *v)
557
+
{
558
+
return atomic_add_return_acquire(i, v);
559
+
}
560
+
561
+
static inline long
562
+
atomic_long_add_return_release(long i, atomic_long_t *v)
563
+
{
564
+
return atomic_add_return_release(i, v);
565
+
}
566
+
567
+
static inline long
568
+
atomic_long_add_return_relaxed(long i, atomic_long_t *v)
569
+
{
570
+
return atomic_add_return_relaxed(i, v);
571
+
}
572
+
573
+
static inline long
574
+
atomic_long_fetch_add(long i, atomic_long_t *v)
575
+
{
576
+
return atomic_fetch_add(i, v);
577
+
}
578
+
579
+
static inline long
580
+
atomic_long_fetch_add_acquire(long i, atomic_long_t *v)
581
+
{
582
+
return atomic_fetch_add_acquire(i, v);
583
+
}
584
+
585
+
static inline long
586
+
atomic_long_fetch_add_release(long i, atomic_long_t *v)
587
+
{
588
+
return atomic_fetch_add_release(i, v);
589
+
}
590
+
591
+
static inline long
592
+
atomic_long_fetch_add_relaxed(long i, atomic_long_t *v)
593
+
{
594
+
return atomic_fetch_add_relaxed(i, v);
595
+
}
596
+
597
+
static inline void
598
+
atomic_long_sub(long i, atomic_long_t *v)
599
+
{
600
+
atomic_sub(i, v);
601
+
}
602
+
603
+
static inline long
604
+
atomic_long_sub_return(long i, atomic_long_t *v)
605
+
{
606
+
return atomic_sub_return(i, v);
607
+
}
608
+
609
+
static inline long
610
+
atomic_long_sub_return_acquire(long i, atomic_long_t *v)
611
+
{
612
+
return atomic_sub_return_acquire(i, v);
613
+
}
614
+
615
+
static inline long
616
+
atomic_long_sub_return_release(long i, atomic_long_t *v)
617
+
{
618
+
return atomic_sub_return_release(i, v);
619
+
}
620
+
621
+
static inline long
622
+
atomic_long_sub_return_relaxed(long i, atomic_long_t *v)
623
+
{
624
+
return atomic_sub_return_relaxed(i, v);
625
+
}
626
+
627
+
static inline long
628
+
atomic_long_fetch_sub(long i, atomic_long_t *v)
629
+
{
630
+
return atomic_fetch_sub(i, v);
631
+
}
632
+
633
+
static inline long
634
+
atomic_long_fetch_sub_acquire(long i, atomic_long_t *v)
635
+
{
636
+
return atomic_fetch_sub_acquire(i, v);
637
+
}
638
+
639
+
static inline long
640
+
atomic_long_fetch_sub_release(long i, atomic_long_t *v)
641
+
{
642
+
return atomic_fetch_sub_release(i, v);
643
+
}
644
+
645
+
static inline long
646
+
atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v)
647
+
{
648
+
return atomic_fetch_sub_relaxed(i, v);
649
+
}
650
+
651
+
static inline void
652
+
atomic_long_inc(atomic_long_t *v)
653
+
{
654
+
atomic_inc(v);
655
+
}
656
+
657
+
static inline long
658
+
atomic_long_inc_return(atomic_long_t *v)
659
+
{
660
+
return atomic_inc_return(v);
661
+
}
662
+
663
+
static inline long
664
+
atomic_long_inc_return_acquire(atomic_long_t *v)
665
+
{
666
+
return atomic_inc_return_acquire(v);
667
+
}
668
+
669
+
static inline long
670
+
atomic_long_inc_return_release(atomic_long_t *v)
671
+
{
672
+
return atomic_inc_return_release(v);
673
+
}
674
+
675
+
static inline long
676
+
atomic_long_inc_return_relaxed(atomic_long_t *v)
677
+
{
678
+
return atomic_inc_return_relaxed(v);
679
+
}
680
+
681
+
static inline long
682
+
atomic_long_fetch_inc(atomic_long_t *v)
683
+
{
684
+
return atomic_fetch_inc(v);
685
+
}
686
+
687
+
static inline long
688
+
atomic_long_fetch_inc_acquire(atomic_long_t *v)
689
+
{
690
+
return atomic_fetch_inc_acquire(v);
691
+
}
692
+
693
+
static inline long
694
+
atomic_long_fetch_inc_release(atomic_long_t *v)
695
+
{
696
+
return atomic_fetch_inc_release(v);
697
+
}
698
+
699
+
static inline long
700
+
atomic_long_fetch_inc_relaxed(atomic_long_t *v)
701
+
{
702
+
return atomic_fetch_inc_relaxed(v);
703
+
}
704
+
705
+
static inline void
706
+
atomic_long_dec(atomic_long_t *v)
707
+
{
708
+
atomic_dec(v);
709
+
}
710
+
711
+
static inline long
712
+
atomic_long_dec_return(atomic_long_t *v)
713
+
{
714
+
return atomic_dec_return(v);
715
+
}
716
+
717
+
static inline long
718
+
atomic_long_dec_return_acquire(atomic_long_t *v)
719
+
{
720
+
return atomic_dec_return_acquire(v);
721
+
}
722
+
723
+
static inline long
724
+
atomic_long_dec_return_release(atomic_long_t *v)
725
+
{
726
+
return atomic_dec_return_release(v);
727
+
}
728
+
729
+
static inline long
730
+
atomic_long_dec_return_relaxed(atomic_long_t *v)
731
+
{
732
+
return atomic_dec_return_relaxed(v);
733
+
}
734
+
735
+
static inline long
736
+
atomic_long_fetch_dec(atomic_long_t *v)
737
+
{
738
+
return atomic_fetch_dec(v);
739
+
}
740
+
741
+
static inline long
742
+
atomic_long_fetch_dec_acquire(atomic_long_t *v)
743
+
{
744
+
return atomic_fetch_dec_acquire(v);
745
+
}
746
+
747
+
static inline long
748
+
atomic_long_fetch_dec_release(atomic_long_t *v)
749
+
{
750
+
return atomic_fetch_dec_release(v);
751
+
}
752
+
753
+
static inline long
754
+
atomic_long_fetch_dec_relaxed(atomic_long_t *v)
755
+
{
756
+
return atomic_fetch_dec_relaxed(v);
757
+
}
758
+
759
+
static inline void
760
+
atomic_long_and(long i, atomic_long_t *v)
761
+
{
762
+
atomic_and(i, v);
763
+
}
764
+
765
+
static inline long
766
+
atomic_long_fetch_and(long i, atomic_long_t *v)
767
+
{
768
+
return atomic_fetch_and(i, v);
769
+
}
770
+
771
+
static inline long
772
+
atomic_long_fetch_and_acquire(long i, atomic_long_t *v)
773
+
{
774
+
return atomic_fetch_and_acquire(i, v);
775
+
}
776
+
777
+
static inline long
778
+
atomic_long_fetch_and_release(long i, atomic_long_t *v)
779
+
{
780
+
return atomic_fetch_and_release(i, v);
781
+
}
782
+
783
+
static inline long
784
+
atomic_long_fetch_and_relaxed(long i, atomic_long_t *v)
785
+
{
786
+
return atomic_fetch_and_relaxed(i, v);
787
+
}
788
+
789
+
static inline void
790
+
atomic_long_andnot(long i, atomic_long_t *v)
791
+
{
792
+
atomic_andnot(i, v);
793
+
}
794
+
795
+
static inline long
796
+
atomic_long_fetch_andnot(long i, atomic_long_t *v)
797
+
{
798
+
return atomic_fetch_andnot(i, v);
799
+
}
800
+
801
+
static inline long
802
+
atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v)
803
+
{
804
+
return atomic_fetch_andnot_acquire(i, v);
805
+
}
806
+
807
+
static inline long
808
+
atomic_long_fetch_andnot_release(long i, atomic_long_t *v)
809
+
{
810
+
return atomic_fetch_andnot_release(i, v);
811
+
}
812
+
813
+
static inline long
814
+
atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v)
815
+
{
816
+
return atomic_fetch_andnot_relaxed(i, v);
817
+
}
818
+
819
+
static inline void
820
+
atomic_long_or(long i, atomic_long_t *v)
821
+
{
822
+
atomic_or(i, v);
823
+
}
824
+
825
+
static inline long
826
+
atomic_long_fetch_or(long i, atomic_long_t *v)
827
+
{
828
+
return atomic_fetch_or(i, v);
829
+
}
830
+
831
+
static inline long
832
+
atomic_long_fetch_or_acquire(long i, atomic_long_t *v)
833
+
{
834
+
return atomic_fetch_or_acquire(i, v);
835
+
}
836
+
837
+
static inline long
838
+
atomic_long_fetch_or_release(long i, atomic_long_t *v)
839
+
{
840
+
return atomic_fetch_or_release(i, v);
841
+
}
842
+
843
+
static inline long
844
+
atomic_long_fetch_or_relaxed(long i, atomic_long_t *v)
845
+
{
846
+
return atomic_fetch_or_relaxed(i, v);
847
+
}
848
+
849
+
static inline void
850
+
atomic_long_xor(long i, atomic_long_t *v)
851
+
{
852
+
atomic_xor(i, v);
853
+
}
854
+
855
+
static inline long
856
+
atomic_long_fetch_xor(long i, atomic_long_t *v)
857
+
{
858
+
return atomic_fetch_xor(i, v);
859
+
}
860
+
861
+
static inline long
862
+
atomic_long_fetch_xor_acquire(long i, atomic_long_t *v)
863
+
{
864
+
return atomic_fetch_xor_acquire(i, v);
865
+
}
866
+
867
+
static inline long
868
+
atomic_long_fetch_xor_release(long i, atomic_long_t *v)
869
+
{
870
+
return atomic_fetch_xor_release(i, v);
871
+
}
872
+
873
+
static inline long
874
+
atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v)
875
+
{
876
+
return atomic_fetch_xor_relaxed(i, v);
877
+
}
878
+
879
+
static inline long
880
+
atomic_long_xchg(atomic_long_t *v, long i)
881
+
{
882
+
return atomic_xchg(v, i);
883
+
}
884
+
885
+
static inline long
886
+
atomic_long_xchg_acquire(atomic_long_t *v, long i)
887
+
{
888
+
return atomic_xchg_acquire(v, i);
889
+
}
890
+
891
+
static inline long
892
+
atomic_long_xchg_release(atomic_long_t *v, long i)
893
+
{
894
+
return atomic_xchg_release(v, i);
895
+
}
896
+
897
+
static inline long
898
+
atomic_long_xchg_relaxed(atomic_long_t *v, long i)
899
+
{
900
+
return atomic_xchg_relaxed(v, i);
901
+
}
902
+
903
+
static inline long
904
+
atomic_long_cmpxchg(atomic_long_t *v, long old, long new)
905
+
{
906
+
return atomic_cmpxchg(v, old, new);
907
+
}
908
+
909
+
static inline long
910
+
atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new)
911
+
{
912
+
return atomic_cmpxchg_acquire(v, old, new);
913
+
}
914
+
915
+
static inline long
916
+
atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new)
917
+
{
918
+
return atomic_cmpxchg_release(v, old, new);
919
+
}
920
+
921
+
static inline long
922
+
atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new)
923
+
{
924
+
return atomic_cmpxchg_relaxed(v, old, new);
925
+
}
926
+
927
+
static inline bool
928
+
atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new)
929
+
{
930
+
return atomic_try_cmpxchg(v, (int *)old, new);
931
+
}
932
+
933
+
static inline bool
934
+
atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new)
935
+
{
936
+
return atomic_try_cmpxchg_acquire(v, (int *)old, new);
937
+
}
938
+
939
+
static inline bool
940
+
atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new)
941
+
{
942
+
return atomic_try_cmpxchg_release(v, (int *)old, new);
943
+
}
944
+
945
+
static inline bool
946
+
atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new)
947
+
{
948
+
return atomic_try_cmpxchg_relaxed(v, (int *)old, new);
949
+
}
950
+
951
+
static inline bool
952
+
atomic_long_sub_and_test(long i, atomic_long_t *v)
953
+
{
954
+
return atomic_sub_and_test(i, v);
955
+
}
956
+
957
+
static inline bool
958
+
atomic_long_dec_and_test(atomic_long_t *v)
959
+
{
960
+
return atomic_dec_and_test(v);
961
+
}
962
+
963
+
static inline bool
964
+
atomic_long_inc_and_test(atomic_long_t *v)
965
+
{
966
+
return atomic_inc_and_test(v);
967
+
}
968
+
969
+
static inline bool
970
+
atomic_long_add_negative(long i, atomic_long_t *v)
971
+
{
972
+
return atomic_add_negative(i, v);
973
+
}
974
+
975
+
static inline long
976
+
atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u)
977
+
{
978
+
return atomic_fetch_add_unless(v, a, u);
979
+
}
980
+
981
+
static inline bool
982
+
atomic_long_add_unless(atomic_long_t *v, long a, long u)
983
+
{
984
+
return atomic_add_unless(v, a, u);
985
+
}
986
+
987
+
static inline bool
988
+
atomic_long_inc_not_zero(atomic_long_t *v)
989
+
{
990
+
return atomic_inc_not_zero(v);
991
+
}
992
+
993
+
static inline bool
994
+
atomic_long_inc_unless_negative(atomic_long_t *v)
995
+
{
996
+
return atomic_inc_unless_negative(v);
997
+
}
998
+
999
+
static inline bool
1000
+
atomic_long_dec_unless_positive(atomic_long_t *v)
1001
+
{
1002
+
return atomic_dec_unless_positive(v);
1003
+
}
1004
+
1005
+
static inline long
1006
+
atomic_long_dec_if_positive(atomic_long_t *v)
1007
+
{
1008
+
return atomic_dec_if_positive(v);
1009
+
}
1010
+
1011
+
#endif /* CONFIG_64BIT */
1012
+
#endif /* _ASM_GENERIC_ATOMIC_LONG_H */
+2294
include/linux/atomic-fallback.h
+2294
include/linux/atomic-fallback.h
···
1
+
// SPDX-License-Identifier: GPL-2.0
2
+
3
+
// Generated by scripts/atomic/gen-atomic-fallback.sh
4
+
// DO NOT MODIFY THIS FILE DIRECTLY
5
+
6
+
#ifndef _LINUX_ATOMIC_FALLBACK_H
7
+
#define _LINUX_ATOMIC_FALLBACK_H
8
+
9
+
#ifndef xchg_relaxed
10
+
#define xchg_relaxed xchg
11
+
#define xchg_acquire xchg
12
+
#define xchg_release xchg
13
+
#else /* xchg_relaxed */
14
+
15
+
#ifndef xchg_acquire
16
+
#define xchg_acquire(...) \
17
+
__atomic_op_acquire(xchg, __VA_ARGS__)
18
+
#endif
19
+
20
+
#ifndef xchg_release
21
+
#define xchg_release(...) \
22
+
__atomic_op_release(xchg, __VA_ARGS__)
23
+
#endif
24
+
25
+
#ifndef xchg
26
+
#define xchg(...) \
27
+
__atomic_op_fence(xchg, __VA_ARGS__)
28
+
#endif
29
+
30
+
#endif /* xchg_relaxed */
31
+
32
+
#ifndef cmpxchg_relaxed
33
+
#define cmpxchg_relaxed cmpxchg
34
+
#define cmpxchg_acquire cmpxchg
35
+
#define cmpxchg_release cmpxchg
36
+
#else /* cmpxchg_relaxed */
37
+
38
+
#ifndef cmpxchg_acquire
39
+
#define cmpxchg_acquire(...) \
40
+
__atomic_op_acquire(cmpxchg, __VA_ARGS__)
41
+
#endif
42
+
43
+
#ifndef cmpxchg_release
44
+
#define cmpxchg_release(...) \
45
+
__atomic_op_release(cmpxchg, __VA_ARGS__)
46
+
#endif
47
+
48
+
#ifndef cmpxchg
49
+
#define cmpxchg(...) \
50
+
__atomic_op_fence(cmpxchg, __VA_ARGS__)
51
+
#endif
52
+
53
+
#endif /* cmpxchg_relaxed */
54
+
55
+
#ifndef cmpxchg64_relaxed
56
+
#define cmpxchg64_relaxed cmpxchg64
57
+
#define cmpxchg64_acquire cmpxchg64
58
+
#define cmpxchg64_release cmpxchg64
59
+
#else /* cmpxchg64_relaxed */
60
+
61
+
#ifndef cmpxchg64_acquire
62
+
#define cmpxchg64_acquire(...) \
63
+
__atomic_op_acquire(cmpxchg64, __VA_ARGS__)
64
+
#endif
65
+
66
+
#ifndef cmpxchg64_release
67
+
#define cmpxchg64_release(...) \
68
+
__atomic_op_release(cmpxchg64, __VA_ARGS__)
69
+
#endif
70
+
71
+
#ifndef cmpxchg64
72
+
#define cmpxchg64(...) \
73
+
__atomic_op_fence(cmpxchg64, __VA_ARGS__)
74
+
#endif
75
+
76
+
#endif /* cmpxchg64_relaxed */
77
+
78
+
#ifndef atomic_read_acquire
79
+
static inline int
80
+
atomic_read_acquire(const atomic_t *v)
81
+
{
82
+
return smp_load_acquire(&(v)->counter);
83
+
}
84
+
#define atomic_read_acquire atomic_read_acquire
85
+
#endif
86
+
87
+
#ifndef atomic_set_release
88
+
static inline void
89
+
atomic_set_release(atomic_t *v, int i)
90
+
{
91
+
smp_store_release(&(v)->counter, i);
92
+
}
93
+
#define atomic_set_release atomic_set_release
94
+
#endif
95
+
96
+
#ifndef atomic_add_return_relaxed
97
+
#define atomic_add_return_acquire atomic_add_return
98
+
#define atomic_add_return_release atomic_add_return
99
+
#define atomic_add_return_relaxed atomic_add_return
100
+
#else /* atomic_add_return_relaxed */
101
+
102
+
#ifndef atomic_add_return_acquire
103
+
static inline int
104
+
atomic_add_return_acquire(int i, atomic_t *v)
105
+
{
106
+
int ret = atomic_add_return_relaxed(i, v);
107
+
__atomic_acquire_fence();
108
+
return ret;
109
+
}
110
+
#define atomic_add_return_acquire atomic_add_return_acquire
111
+
#endif
112
+
113
+
#ifndef atomic_add_return_release
114
+
static inline int
115
+
atomic_add_return_release(int i, atomic_t *v)
116
+
{
117
+
__atomic_release_fence();
118
+
return atomic_add_return_relaxed(i, v);
119
+
}
120
+
#define atomic_add_return_release atomic_add_return_release
121
+
#endif
122
+
123
+
#ifndef atomic_add_return
124
+
static inline int
125
+
atomic_add_return(int i, atomic_t *v)
126
+
{
127
+
int ret;
128
+
__atomic_pre_full_fence();
129
+
ret = atomic_add_return_relaxed(i, v);
130
+
__atomic_post_full_fence();
131
+
return ret;
132
+
}
133
+
#define atomic_add_return atomic_add_return
134
+
#endif
135
+
136
+
#endif /* atomic_add_return_relaxed */
137
+
138
+
#ifndef atomic_fetch_add_relaxed
139
+
#define atomic_fetch_add_acquire atomic_fetch_add
140
+
#define atomic_fetch_add_release atomic_fetch_add
141
+
#define atomic_fetch_add_relaxed atomic_fetch_add
142
+
#else /* atomic_fetch_add_relaxed */
143
+
144
+
#ifndef atomic_fetch_add_acquire
145
+
static inline int
146
+
atomic_fetch_add_acquire(int i, atomic_t *v)
147
+
{
148
+
int ret = atomic_fetch_add_relaxed(i, v);
149
+
__atomic_acquire_fence();
150
+
return ret;
151
+
}
152
+
#define atomic_fetch_add_acquire atomic_fetch_add_acquire
153
+
#endif
154
+
155
+
#ifndef atomic_fetch_add_release
156
+
static inline int
157
+
atomic_fetch_add_release(int i, atomic_t *v)
158
+
{
159
+
__atomic_release_fence();
160
+
return atomic_fetch_add_relaxed(i, v);
161
+
}
162
+
#define atomic_fetch_add_release atomic_fetch_add_release
163
+
#endif
164
+
165
+
#ifndef atomic_fetch_add
166
+
static inline int
167
+
atomic_fetch_add(int i, atomic_t *v)
168
+
{
169
+
int ret;
170
+
__atomic_pre_full_fence();
171
+
ret = atomic_fetch_add_relaxed(i, v);
172
+
__atomic_post_full_fence();
173
+
return ret;
174
+
}
175
+
#define atomic_fetch_add atomic_fetch_add
176
+
#endif
177
+
178
+
#endif /* atomic_fetch_add_relaxed */
179
+
180
+
#ifndef atomic_sub_return_relaxed
181
+
#define atomic_sub_return_acquire atomic_sub_return
182
+
#define atomic_sub_return_release atomic_sub_return
183
+
#define atomic_sub_return_relaxed atomic_sub_return
184
+
#else /* atomic_sub_return_relaxed */
185
+
186
+
#ifndef atomic_sub_return_acquire
187
+
static inline int
188
+
atomic_sub_return_acquire(int i, atomic_t *v)
189
+
{
190
+
int ret = atomic_sub_return_relaxed(i, v);
191
+
__atomic_acquire_fence();
192
+
return ret;
193
+
}
194
+
#define atomic_sub_return_acquire atomic_sub_return_acquire
195
+
#endif
196
+
197
+
#ifndef atomic_sub_return_release
198
+
static inline int
199
+
atomic_sub_return_release(int i, atomic_t *v)
200
+
{
201
+
__atomic_release_fence();
202
+
return atomic_sub_return_relaxed(i, v);
203
+
}
204
+
#define atomic_sub_return_release atomic_sub_return_release
205
+
#endif
206
+
207
+
#ifndef atomic_sub_return
208
+
static inline int
209
+
atomic_sub_return(int i, atomic_t *v)
210
+
{
211
+
int ret;
212
+
__atomic_pre_full_fence();
213
+
ret = atomic_sub_return_relaxed(i, v);
214
+
__atomic_post_full_fence();
215
+
return ret;
216
+
}
217
+
#define atomic_sub_return atomic_sub_return
218
+
#endif
219
+
220
+
#endif /* atomic_sub_return_relaxed */
221
+
222
+
#ifndef atomic_fetch_sub_relaxed
223
+
#define atomic_fetch_sub_acquire atomic_fetch_sub
224
+
#define atomic_fetch_sub_release atomic_fetch_sub
225
+
#define atomic_fetch_sub_relaxed atomic_fetch_sub
226
+
#else /* atomic_fetch_sub_relaxed */
227
+
228
+
#ifndef atomic_fetch_sub_acquire
229
+
static inline int
230
+
atomic_fetch_sub_acquire(int i, atomic_t *v)
231
+
{
232
+
int ret = atomic_fetch_sub_relaxed(i, v);
233
+
__atomic_acquire_fence();
234
+
return ret;
235
+
}
236
+
#define atomic_fetch_sub_acquire atomic_fetch_sub_acquire
237
+
#endif
238
+
239
+
#ifndef atomic_fetch_sub_release
240
+
static inline int
241
+
atomic_fetch_sub_release(int i, atomic_t *v)
242
+
{
243
+
__atomic_release_fence();
244
+
return atomic_fetch_sub_relaxed(i, v);
245
+
}
246
+
#define atomic_fetch_sub_release atomic_fetch_sub_release
247
+
#endif
248
+
249
+
#ifndef atomic_fetch_sub
250
+
static inline int
251
+
atomic_fetch_sub(int i, atomic_t *v)
252
+
{
253
+
int ret;
254
+
__atomic_pre_full_fence();
255
+
ret = atomic_fetch_sub_relaxed(i, v);
256
+
__atomic_post_full_fence();
257
+
return ret;
258
+
}
259
+
#define atomic_fetch_sub atomic_fetch_sub
260
+
#endif
261
+
262
+
#endif /* atomic_fetch_sub_relaxed */
263
+
264
+
#ifndef atomic_inc
265
+
static inline void
266
+
atomic_inc(atomic_t *v)
267
+
{
268
+
atomic_add(1, v);
269
+
}
270
+
#define atomic_inc atomic_inc
271
+
#endif
272
+
273
+
#ifndef atomic_inc_return_relaxed
274
+
#ifdef atomic_inc_return
275
+
#define atomic_inc_return_acquire atomic_inc_return
276
+
#define atomic_inc_return_release atomic_inc_return
277
+
#define atomic_inc_return_relaxed atomic_inc_return
278
+
#endif /* atomic_inc_return */
279
+
280
+
#ifndef atomic_inc_return
281
+
static inline int
282
+
atomic_inc_return(atomic_t *v)
283
+
{
284
+
return atomic_add_return(1, v);
285
+
}
286
+
#define atomic_inc_return atomic_inc_return
287
+
#endif
288
+
289
+
#ifndef atomic_inc_return_acquire
290
+
static inline int
291
+
atomic_inc_return_acquire(atomic_t *v)
292
+
{
293
+
return atomic_add_return_acquire(1, v);
294
+
}
295
+
#define atomic_inc_return_acquire atomic_inc_return_acquire
296
+
#endif
297
+
298
+
#ifndef atomic_inc_return_release
299
+
static inline int
300
+
atomic_inc_return_release(atomic_t *v)
301
+
{
302
+
return atomic_add_return_release(1, v);
303
+
}
304
+
#define atomic_inc_return_release atomic_inc_return_release
305
+
#endif
306
+
307
+
#ifndef atomic_inc_return_relaxed
308
+
static inline int
309
+
atomic_inc_return_relaxed(atomic_t *v)
310
+
{
311
+
return atomic_add_return_relaxed(1, v);
312
+
}
313
+
#define atomic_inc_return_relaxed atomic_inc_return_relaxed
314
+
#endif
315
+
316
+
#else /* atomic_inc_return_relaxed */
317
+
318
+
#ifndef atomic_inc_return_acquire
319
+
static inline int
320
+
atomic_inc_return_acquire(atomic_t *v)
321
+
{
322
+
int ret = atomic_inc_return_relaxed(v);
323
+
__atomic_acquire_fence();
324
+
return ret;
325
+
}
326
+
#define atomic_inc_return_acquire atomic_inc_return_acquire
327
+
#endif
328
+
329
+
#ifndef atomic_inc_return_release
330
+
static inline int
331
+
atomic_inc_return_release(atomic_t *v)
332
+
{
333
+
__atomic_release_fence();
334
+
return atomic_inc_return_relaxed(v);
335
+
}
336
+
#define atomic_inc_return_release atomic_inc_return_release
337
+
#endif
338
+
339
+
#ifndef atomic_inc_return
340
+
static inline int
341
+
atomic_inc_return(atomic_t *v)
342
+
{
343
+
int ret;
344
+
__atomic_pre_full_fence();
345
+
ret = atomic_inc_return_relaxed(v);
346
+
__atomic_post_full_fence();
347
+
return ret;
348
+
}
349
+
#define atomic_inc_return atomic_inc_return
350
+
#endif
351
+
352
+
#endif /* atomic_inc_return_relaxed */
353
+
354
+
#ifndef atomic_fetch_inc_relaxed
355
+
#ifdef atomic_fetch_inc
356
+
#define atomic_fetch_inc_acquire atomic_fetch_inc
357
+
#define atomic_fetch_inc_release atomic_fetch_inc
358
+
#define atomic_fetch_inc_relaxed atomic_fetch_inc
359
+
#endif /* atomic_fetch_inc */
360
+
361
+
#ifndef atomic_fetch_inc
362
+
static inline int
363
+
atomic_fetch_inc(atomic_t *v)
364
+
{
365
+
return atomic_fetch_add(1, v);
366
+
}
367
+
#define atomic_fetch_inc atomic_fetch_inc
368
+
#endif
369
+
370
+
#ifndef atomic_fetch_inc_acquire
371
+
static inline int
372
+
atomic_fetch_inc_acquire(atomic_t *v)
373
+
{
374
+
return atomic_fetch_add_acquire(1, v);
375
+
}
376
+
#define atomic_fetch_inc_acquire atomic_fetch_inc_acquire
377
+
#endif
378
+
379
+
#ifndef atomic_fetch_inc_release
380
+
static inline int
381
+
atomic_fetch_inc_release(atomic_t *v)
382
+
{
383
+
return atomic_fetch_add_release(1, v);
384
+
}
385
+
#define atomic_fetch_inc_release atomic_fetch_inc_release
386
+
#endif
387
+
388
+
#ifndef atomic_fetch_inc_relaxed
389
+
static inline int
390
+
atomic_fetch_inc_relaxed(atomic_t *v)
391
+
{
392
+
return atomic_fetch_add_relaxed(1, v);
393
+
}
394
+
#define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed
395
+
#endif
396
+
397
+
#else /* atomic_fetch_inc_relaxed */
398
+
399
+
#ifndef atomic_fetch_inc_acquire
400
+
static inline int
401
+
atomic_fetch_inc_acquire(atomic_t *v)
402
+
{
403
+
int ret = atomic_fetch_inc_relaxed(v);
404
+
__atomic_acquire_fence();
405
+
return ret;
406
+
}
407
+
#define atomic_fetch_inc_acquire atomic_fetch_inc_acquire
408
+
#endif
409
+
410
+
#ifndef atomic_fetch_inc_release
411
+
static inline int
412
+
atomic_fetch_inc_release(atomic_t *v)
413
+
{
414
+
__atomic_release_fence();
415
+
return atomic_fetch_inc_relaxed(v);
416
+
}
417
+
#define atomic_fetch_inc_release atomic_fetch_inc_release
418
+
#endif
419
+
420
+
#ifndef atomic_fetch_inc
421
+
static inline int
422
+
atomic_fetch_inc(atomic_t *v)
423
+
{
424
+
int ret;
425
+
__atomic_pre_full_fence();
426
+
ret = atomic_fetch_inc_relaxed(v);
427
+
__atomic_post_full_fence();
428
+
return ret;
429
+
}
430
+
#define atomic_fetch_inc atomic_fetch_inc
431
+
#endif
432
+
433
+
#endif /* atomic_fetch_inc_relaxed */
434
+
435
+
#ifndef atomic_dec
436
+
static inline void
437
+
atomic_dec(atomic_t *v)
438
+
{
439
+
atomic_sub(1, v);
440
+
}
441
+
#define atomic_dec atomic_dec
442
+
#endif
443
+
444
+
#ifndef atomic_dec_return_relaxed
445
+
#ifdef atomic_dec_return
446
+
#define atomic_dec_return_acquire atomic_dec_return
447
+
#define atomic_dec_return_release atomic_dec_return
448
+
#define atomic_dec_return_relaxed atomic_dec_return
449
+
#endif /* atomic_dec_return */
450
+
451
+
#ifndef atomic_dec_return
452
+
static inline int
453
+
atomic_dec_return(atomic_t *v)
454
+
{
455
+
return atomic_sub_return(1, v);
456
+
}
457
+
#define atomic_dec_return atomic_dec_return
458
+
#endif
459
+
460
+
#ifndef atomic_dec_return_acquire
461
+
static inline int
462
+
atomic_dec_return_acquire(atomic_t *v)
463
+
{
464
+
return atomic_sub_return_acquire(1, v);
465
+
}
466
+
#define atomic_dec_return_acquire atomic_dec_return_acquire
467
+
#endif
468
+
469
+
#ifndef atomic_dec_return_release
470
+
static inline int
471
+
atomic_dec_return_release(atomic_t *v)
472
+
{
473
+
return atomic_sub_return_release(1, v);
474
+
}
475
+
#define atomic_dec_return_release atomic_dec_return_release
476
+
#endif
477
+
478
+
#ifndef atomic_dec_return_relaxed
479
+
static inline int
480
+
atomic_dec_return_relaxed(atomic_t *v)
481
+
{
482
+
return atomic_sub_return_relaxed(1, v);
483
+
}
484
+
#define atomic_dec_return_relaxed atomic_dec_return_relaxed
485
+
#endif
486
+
487
+
#else /* atomic_dec_return_relaxed */
488
+
489
+
#ifndef atomic_dec_return_acquire
490
+
static inline int
491
+
atomic_dec_return_acquire(atomic_t *v)
492
+
{
493
+
int ret = atomic_dec_return_relaxed(v);
494
+
__atomic_acquire_fence();
495
+
return ret;
496
+
}
497
+
#define atomic_dec_return_acquire atomic_dec_return_acquire
498
+
#endif
499
+
500
+
#ifndef atomic_dec_return_release
501
+
static inline int
502
+
atomic_dec_return_release(atomic_t *v)
503
+
{
504
+
__atomic_release_fence();
505
+
return atomic_dec_return_relaxed(v);
506
+
}
507
+
#define atomic_dec_return_release atomic_dec_return_release
508
+
#endif
509
+
510
+
#ifndef atomic_dec_return
511
+
static inline int
512
+
atomic_dec_return(atomic_t *v)
513
+
{
514
+
int ret;
515
+
__atomic_pre_full_fence();
516
+
ret = atomic_dec_return_relaxed(v);
517
+
__atomic_post_full_fence();
518
+
return ret;
519
+
}
520
+
#define atomic_dec_return atomic_dec_return
521
+
#endif
522
+
523
+
#endif /* atomic_dec_return_relaxed */
524
+
525
+
#ifndef atomic_fetch_dec_relaxed
526
+
#ifdef atomic_fetch_dec
527
+
#define atomic_fetch_dec_acquire atomic_fetch_dec
528
+
#define atomic_fetch_dec_release atomic_fetch_dec
529
+
#define atomic_fetch_dec_relaxed atomic_fetch_dec
530
+
#endif /* atomic_fetch_dec */
531
+
532
+
#ifndef atomic_fetch_dec
533
+
static inline int
534
+
atomic_fetch_dec(atomic_t *v)
535
+
{
536
+
return atomic_fetch_sub(1, v);
537
+
}
538
+
#define atomic_fetch_dec atomic_fetch_dec
539
+
#endif
540
+
541
+
#ifndef atomic_fetch_dec_acquire
542
+
static inline int
543
+
atomic_fetch_dec_acquire(atomic_t *v)
544
+
{
545
+
return atomic_fetch_sub_acquire(1, v);
546
+
}
547
+
#define atomic_fetch_dec_acquire atomic_fetch_dec_acquire
548
+
#endif
549
+
550
+
#ifndef atomic_fetch_dec_release
551
+
static inline int
552
+
atomic_fetch_dec_release(atomic_t *v)
553
+
{
554
+
return atomic_fetch_sub_release(1, v);
555
+
}
556
+
#define atomic_fetch_dec_release atomic_fetch_dec_release
557
+
#endif
558
+
559
+
#ifndef atomic_fetch_dec_relaxed
560
+
static inline int
561
+
atomic_fetch_dec_relaxed(atomic_t *v)
562
+
{
563
+
return atomic_fetch_sub_relaxed(1, v);
564
+
}
565
+
#define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed
566
+
#endif
567
+
568
+
#else /* atomic_fetch_dec_relaxed */
569
+
570
+
#ifndef atomic_fetch_dec_acquire
571
+
static inline int
572
+
atomic_fetch_dec_acquire(atomic_t *v)
573
+
{
574
+
int ret = atomic_fetch_dec_relaxed(v);
575
+
__atomic_acquire_fence();
576
+
return ret;
577
+
}
578
+
#define atomic_fetch_dec_acquire atomic_fetch_dec_acquire
579
+
#endif
580
+
581
+
#ifndef atomic_fetch_dec_release
582
+
static inline int
583
+
atomic_fetch_dec_release(atomic_t *v)
584
+
{
585
+
__atomic_release_fence();
586
+
return atomic_fetch_dec_relaxed(v);
587
+
}
588
+
#define atomic_fetch_dec_release atomic_fetch_dec_release
589
+
#endif
590
+
591
+
#ifndef atomic_fetch_dec
592
+
static inline int
593
+
atomic_fetch_dec(atomic_t *v)
594
+
{
595
+
int ret;
596
+
__atomic_pre_full_fence();
597
+
ret = atomic_fetch_dec_relaxed(v);
598
+
__atomic_post_full_fence();
599
+
return ret;
600
+
}
601
+
#define atomic_fetch_dec atomic_fetch_dec
602
+
#endif
603
+
604
+
#endif /* atomic_fetch_dec_relaxed */
605
+
606
+
#ifndef atomic_fetch_and_relaxed
607
+
#define atomic_fetch_and_acquire atomic_fetch_and
608
+
#define atomic_fetch_and_release atomic_fetch_and
609
+
#define atomic_fetch_and_relaxed atomic_fetch_and
610
+
#else /* atomic_fetch_and_relaxed */
611
+
612
+
#ifndef atomic_fetch_and_acquire
613
+
static inline int
614
+
atomic_fetch_and_acquire(int i, atomic_t *v)
615
+
{
616
+
int ret = atomic_fetch_and_relaxed(i, v);
617
+
__atomic_acquire_fence();
618
+
return ret;
619
+
}
620
+
#define atomic_fetch_and_acquire atomic_fetch_and_acquire
621
+
#endif
622
+
623
+
#ifndef atomic_fetch_and_release
624
+
static inline int
625
+
atomic_fetch_and_release(int i, atomic_t *v)
626
+
{
627
+
__atomic_release_fence();
628
+
return atomic_fetch_and_relaxed(i, v);
629
+
}
630
+
#define atomic_fetch_and_release atomic_fetch_and_release
631
+
#endif
632
+
633
+
#ifndef atomic_fetch_and
634
+
static inline int
635
+
atomic_fetch_and(int i, atomic_t *v)
636
+
{
637
+
int ret;
638
+
__atomic_pre_full_fence();
639
+
ret = atomic_fetch_and_relaxed(i, v);
640
+
__atomic_post_full_fence();
641
+
return ret;
642
+
}
643
+
#define atomic_fetch_and atomic_fetch_and
644
+
#endif
645
+
646
+
#endif /* atomic_fetch_and_relaxed */
647
+
648
+
#ifndef atomic_andnot
649
+
static inline void
650
+
atomic_andnot(int i, atomic_t *v)
651
+
{
652
+
atomic_and(~i, v);
653
+
}
654
+
#define atomic_andnot atomic_andnot
655
+
#endif
656
+
657
+
#ifndef atomic_fetch_andnot_relaxed
658
+
#ifdef atomic_fetch_andnot
659
+
#define atomic_fetch_andnot_acquire atomic_fetch_andnot
660
+
#define atomic_fetch_andnot_release atomic_fetch_andnot
661
+
#define atomic_fetch_andnot_relaxed atomic_fetch_andnot
662
+
#endif /* atomic_fetch_andnot */
663
+
664
+
#ifndef atomic_fetch_andnot
665
+
static inline int
666
+
atomic_fetch_andnot(int i, atomic_t *v)
667
+
{
668
+
return atomic_fetch_and(~i, v);
669
+
}
670
+
#define atomic_fetch_andnot atomic_fetch_andnot
671
+
#endif
672
+
673
+
#ifndef atomic_fetch_andnot_acquire
674
+
static inline int
675
+
atomic_fetch_andnot_acquire(int i, atomic_t *v)
676
+
{
677
+
return atomic_fetch_and_acquire(~i, v);
678
+
}
679
+
#define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire
680
+
#endif
681
+
682
+
#ifndef atomic_fetch_andnot_release
683
+
static inline int
684
+
atomic_fetch_andnot_release(int i, atomic_t *v)
685
+
{
686
+
return atomic_fetch_and_release(~i, v);
687
+
}
688
+
#define atomic_fetch_andnot_release atomic_fetch_andnot_release
689
+
#endif
690
+
691
+
#ifndef atomic_fetch_andnot_relaxed
692
+
static inline int
693
+
atomic_fetch_andnot_relaxed(int i, atomic_t *v)
694
+
{
695
+
return atomic_fetch_and_relaxed(~i, v);
696
+
}
697
+
#define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed
698
+
#endif
699
+
700
+
#else /* atomic_fetch_andnot_relaxed */
701
+
702
+
#ifndef atomic_fetch_andnot_acquire
703
+
static inline int
704
+
atomic_fetch_andnot_acquire(int i, atomic_t *v)
705
+
{
706
+
int ret = atomic_fetch_andnot_relaxed(i, v);
707
+
__atomic_acquire_fence();
708
+
return ret;
709
+
}
710
+
#define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire
711
+
#endif
712
+
713
+
#ifndef atomic_fetch_andnot_release
714
+
static inline int
715
+
atomic_fetch_andnot_release(int i, atomic_t *v)
716
+
{
717
+
__atomic_release_fence();
718
+
return atomic_fetch_andnot_relaxed(i, v);
719
+
}
720
+
#define atomic_fetch_andnot_release atomic_fetch_andnot_release
721
+
#endif
722
+
723
+
#ifndef atomic_fetch_andnot
724
+
static inline int
725
+
atomic_fetch_andnot(int i, atomic_t *v)
726
+
{
727
+
int ret;
728
+
__atomic_pre_full_fence();
729
+
ret = atomic_fetch_andnot_relaxed(i, v);
730
+
__atomic_post_full_fence();
731
+
return ret;
732
+
}
733
+
#define atomic_fetch_andnot atomic_fetch_andnot
734
+
#endif
735
+
736
+
#endif /* atomic_fetch_andnot_relaxed */
737
+
738
+
#ifndef atomic_fetch_or_relaxed
739
+
#define atomic_fetch_or_acquire atomic_fetch_or
740
+
#define atomic_fetch_or_release atomic_fetch_or
741
+
#define atomic_fetch_or_relaxed atomic_fetch_or
742
+
#else /* atomic_fetch_or_relaxed */
743
+
744
+
#ifndef atomic_fetch_or_acquire
745
+
static inline int
746
+
atomic_fetch_or_acquire(int i, atomic_t *v)
747
+
{
748
+
int ret = atomic_fetch_or_relaxed(i, v);
749
+
__atomic_acquire_fence();
750
+
return ret;
751
+
}
752
+
#define atomic_fetch_or_acquire atomic_fetch_or_acquire
753
+
#endif
754
+
755
+
#ifndef atomic_fetch_or_release
756
+
static inline int
757
+
atomic_fetch_or_release(int i, atomic_t *v)
758
+
{
759
+
__atomic_release_fence();
760
+
return atomic_fetch_or_relaxed(i, v);
761
+
}
762
+
#define atomic_fetch_or_release atomic_fetch_or_release
763
+
#endif
764
+
765
+
#ifndef atomic_fetch_or
766
+
static inline int
767
+
atomic_fetch_or(int i, atomic_t *v)
768
+
{
769
+
int ret;
770
+
__atomic_pre_full_fence();
771
+
ret = atomic_fetch_or_relaxed(i, v);
772
+
__atomic_post_full_fence();
773
+
return ret;
774
+
}
775
+
#define atomic_fetch_or atomic_fetch_or
776
+
#endif
777
+
778
+
#endif /* atomic_fetch_or_relaxed */
779
+
780
+
#ifndef atomic_fetch_xor_relaxed
781
+
#define atomic_fetch_xor_acquire atomic_fetch_xor
782
+
#define atomic_fetch_xor_release atomic_fetch_xor
783
+
#define atomic_fetch_xor_relaxed atomic_fetch_xor
784
+
#else /* atomic_fetch_xor_relaxed */
785
+
786
+
#ifndef atomic_fetch_xor_acquire
787
+
static inline int
788
+
atomic_fetch_xor_acquire(int i, atomic_t *v)
789
+
{
790
+
int ret = atomic_fetch_xor_relaxed(i, v);
791
+
__atomic_acquire_fence();
792
+
return ret;
793
+
}
794
+
#define atomic_fetch_xor_acquire atomic_fetch_xor_acquire
795
+
#endif
796
+
797
+
#ifndef atomic_fetch_xor_release
798
+
static inline int
799
+
atomic_fetch_xor_release(int i, atomic_t *v)
800
+
{
801
+
__atomic_release_fence();
802
+
return atomic_fetch_xor_relaxed(i, v);
803
+
}
804
+
#define atomic_fetch_xor_release atomic_fetch_xor_release
805
+
#endif
806
+
807
+
#ifndef atomic_fetch_xor
808
+
static inline int
809
+
atomic_fetch_xor(int i, atomic_t *v)
810
+
{
811
+
int ret;
812
+
__atomic_pre_full_fence();
813
+
ret = atomic_fetch_xor_relaxed(i, v);
814
+
__atomic_post_full_fence();
815
+
return ret;
816
+
}
817
+
#define atomic_fetch_xor atomic_fetch_xor
818
+
#endif
819
+
820
+
#endif /* atomic_fetch_xor_relaxed */
821
+
822
+
#ifndef atomic_xchg_relaxed
823
+
#define atomic_xchg_acquire atomic_xchg
824
+
#define atomic_xchg_release atomic_xchg
825
+
#define atomic_xchg_relaxed atomic_xchg
826
+
#else /* atomic_xchg_relaxed */
827
+
828
+
#ifndef atomic_xchg_acquire
829
+
static inline int
830
+
atomic_xchg_acquire(atomic_t *v, int i)
831
+
{
832
+
int ret = atomic_xchg_relaxed(v, i);
833
+
__atomic_acquire_fence();
834
+
return ret;
835
+
}
836
+
#define atomic_xchg_acquire atomic_xchg_acquire
837
+
#endif
838
+
839
+
#ifndef atomic_xchg_release
840
+
static inline int
841
+
atomic_xchg_release(atomic_t *v, int i)
842
+
{
843
+
__atomic_release_fence();
844
+
return atomic_xchg_relaxed(v, i);
845
+
}
846
+
#define atomic_xchg_release atomic_xchg_release
847
+
#endif
848
+
849
+
#ifndef atomic_xchg
850
+
static inline int
851
+
atomic_xchg(atomic_t *v, int i)
852
+
{
853
+
int ret;
854
+
__atomic_pre_full_fence();
855
+
ret = atomic_xchg_relaxed(v, i);
856
+
__atomic_post_full_fence();
857
+
return ret;
858
+
}
859
+
#define atomic_xchg atomic_xchg
860
+
#endif
861
+
862
+
#endif /* atomic_xchg_relaxed */
863
+
864
+
#ifndef atomic_cmpxchg_relaxed
865
+
#define atomic_cmpxchg_acquire atomic_cmpxchg
866
+
#define atomic_cmpxchg_release atomic_cmpxchg
867
+
#define atomic_cmpxchg_relaxed atomic_cmpxchg
868
+
#else /* atomic_cmpxchg_relaxed */
869
+
870
+
#ifndef atomic_cmpxchg_acquire
871
+
static inline int
872
+
atomic_cmpxchg_acquire(atomic_t *v, int old, int new)
873
+
{
874
+
int ret = atomic_cmpxchg_relaxed(v, old, new);
875
+
__atomic_acquire_fence();
876
+
return ret;
877
+
}
878
+
#define atomic_cmpxchg_acquire atomic_cmpxchg_acquire
879
+
#endif
880
+
881
+
#ifndef atomic_cmpxchg_release
882
+
static inline int
883
+
atomic_cmpxchg_release(atomic_t *v, int old, int new)
884
+
{
885
+
__atomic_release_fence();
886
+
return atomic_cmpxchg_relaxed(v, old, new);
887
+
}
888
+
#define atomic_cmpxchg_release atomic_cmpxchg_release
889
+
#endif
890
+
891
+
#ifndef atomic_cmpxchg
892
+
static inline int
893
+
atomic_cmpxchg(atomic_t *v, int old, int new)
894
+
{
895
+
int ret;
896
+
__atomic_pre_full_fence();
897
+
ret = atomic_cmpxchg_relaxed(v, old, new);
898
+
__atomic_post_full_fence();
899
+
return ret;
900
+
}
901
+
#define atomic_cmpxchg atomic_cmpxchg
902
+
#endif
903
+
904
+
#endif /* atomic_cmpxchg_relaxed */
905
+
906
+
#ifndef atomic_try_cmpxchg_relaxed
907
+
#ifdef atomic_try_cmpxchg
908
+
#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg
909
+
#define atomic_try_cmpxchg_release atomic_try_cmpxchg
910
+
#define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg
911
+
#endif /* atomic_try_cmpxchg */
912
+
913
+
#ifndef atomic_try_cmpxchg
914
+
static inline bool
915
+
atomic_try_cmpxchg(atomic_t *v, int *old, int new)
916
+
{
917
+
int r, o = *old;
918
+
r = atomic_cmpxchg(v, o, new);
919
+
if (unlikely(r != o))
920
+
*old = r;
921
+
return likely(r == o);
922
+
}
923
+
#define atomic_try_cmpxchg atomic_try_cmpxchg
924
+
#endif
925
+
926
+
#ifndef atomic_try_cmpxchg_acquire
927
+
static inline bool
928
+
atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
929
+
{
930
+
int r, o = *old;
931
+
r = atomic_cmpxchg_acquire(v, o, new);
932
+
if (unlikely(r != o))
933
+
*old = r;
934
+
return likely(r == o);
935
+
}
936
+
#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire
937
+
#endif
938
+
939
+
#ifndef atomic_try_cmpxchg_release
940
+
static inline bool
941
+
atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
942
+
{
943
+
int r, o = *old;
944
+
r = atomic_cmpxchg_release(v, o, new);
945
+
if (unlikely(r != o))
946
+
*old = r;
947
+
return likely(r == o);
948
+
}
949
+
#define atomic_try_cmpxchg_release atomic_try_cmpxchg_release
950
+
#endif
951
+
952
+
#ifndef atomic_try_cmpxchg_relaxed
953
+
static inline bool
954
+
atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new)
955
+
{
956
+
int r, o = *old;
957
+
r = atomic_cmpxchg_relaxed(v, o, new);
958
+
if (unlikely(r != o))
959
+
*old = r;
960
+
return likely(r == o);
961
+
}
962
+
#define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed
963
+
#endif
964
+
965
+
#else /* atomic_try_cmpxchg_relaxed */
966
+
967
+
#ifndef atomic_try_cmpxchg_acquire
968
+
static inline bool
969
+
atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new)
970
+
{
971
+
bool ret = atomic_try_cmpxchg_relaxed(v, old, new);
972
+
__atomic_acquire_fence();
973
+
return ret;
974
+
}
975
+
#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire
976
+
#endif
977
+
978
+
#ifndef atomic_try_cmpxchg_release
979
+
static inline bool
980
+
atomic_try_cmpxchg_release(atomic_t *v, int *old, int new)
981
+
{
982
+
__atomic_release_fence();
983
+
return atomic_try_cmpxchg_relaxed(v, old, new);
984
+
}
985
+
#define atomic_try_cmpxchg_release atomic_try_cmpxchg_release
986
+
#endif
987
+
988
+
#ifndef atomic_try_cmpxchg
989
+
static inline bool
990
+
atomic_try_cmpxchg(atomic_t *v, int *old, int new)
991
+
{
992
+
bool ret;
993
+
__atomic_pre_full_fence();
994
+
ret = atomic_try_cmpxchg_relaxed(v, old, new);
995
+
__atomic_post_full_fence();
996
+
return ret;
997
+
}
998
+
#define atomic_try_cmpxchg atomic_try_cmpxchg
999
+
#endif
1000
+
1001
+
#endif /* atomic_try_cmpxchg_relaxed */
1002
+
1003
+
#ifndef atomic_sub_and_test
1004
+
/**
1005
+
* atomic_sub_and_test - subtract value from variable and test result
1006
+
* @i: integer value to subtract
1007
+
* @v: pointer of type atomic_t
1008
+
*
1009
+
* Atomically subtracts @i from @v and returns
1010
+
* true if the result is zero, or false for all
1011
+
* other cases.
1012
+
*/
1013
+
static inline bool
1014
+
atomic_sub_and_test(int i, atomic_t *v)
1015
+
{
1016
+
return atomic_sub_return(i, v) == 0;
1017
+
}
1018
+
#define atomic_sub_and_test atomic_sub_and_test
1019
+
#endif
1020
+
1021
+
#ifndef atomic_dec_and_test
1022
+
/**
1023
+
* atomic_dec_and_test - decrement and test
1024
+
* @v: pointer of type atomic_t
1025
+
*
1026
+
* Atomically decrements @v by 1 and
1027
+
* returns true if the result is 0, or false for all other
1028
+
* cases.
1029
+
*/
1030
+
static inline bool
1031
+
atomic_dec_and_test(atomic_t *v)
1032
+
{
1033
+
return atomic_dec_return(v) == 0;
1034
+
}
1035
+
#define atomic_dec_and_test atomic_dec_and_test
1036
+
#endif
1037
+
1038
+
#ifndef atomic_inc_and_test
1039
+
/**
1040
+
* atomic_inc_and_test - increment and test
1041
+
* @v: pointer of type atomic_t
1042
+
*
1043
+
* Atomically increments @v by 1
1044
+
* and returns true if the result is zero, or false for all
1045
+
* other cases.
1046
+
*/
1047
+
static inline bool
1048
+
atomic_inc_and_test(atomic_t *v)
1049
+
{
1050
+
return atomic_inc_return(v) == 0;
1051
+
}
1052
+
#define atomic_inc_and_test atomic_inc_and_test
1053
+
#endif
1054
+
1055
+
#ifndef atomic_add_negative
1056
+
/**
1057
+
* atomic_add_negative - add and test if negative
1058
+
* @i: integer value to add
1059
+
* @v: pointer of type atomic_t
1060
+
*
1061
+
* Atomically adds @i to @v and returns true
1062
+
* if the result is negative, or false when
1063
+
* result is greater than or equal to zero.
1064
+
*/
1065
+
static inline bool
1066
+
atomic_add_negative(int i, atomic_t *v)
1067
+
{
1068
+
return atomic_add_return(i, v) < 0;
1069
+
}
1070
+
#define atomic_add_negative atomic_add_negative
1071
+
#endif
1072
+
1073
+
#ifndef atomic_fetch_add_unless
1074
+
/**
1075
+
* atomic_fetch_add_unless - add unless the number is already a given value
1076
+
* @v: pointer of type atomic_t
1077
+
* @a: the amount to add to v...
1078
+
* @u: ...unless v is equal to u.
1079
+
*
1080
+
* Atomically adds @a to @v, so long as @v was not already @u.
1081
+
* Returns original value of @v
1082
+
*/
1083
+
static inline int
1084
+
atomic_fetch_add_unless(atomic_t *v, int a, int u)
1085
+
{
1086
+
int c = atomic_read(v);
1087
+
1088
+
do {
1089
+
if (unlikely(c == u))
1090
+
break;
1091
+
} while (!atomic_try_cmpxchg(v, &c, c + a));
1092
+
1093
+
return c;
1094
+
}
1095
+
#define atomic_fetch_add_unless atomic_fetch_add_unless
1096
+
#endif
1097
+
1098
+
#ifndef atomic_add_unless
1099
+
/**
1100
+
* atomic_add_unless - add unless the number is already a given value
1101
+
* @v: pointer of type atomic_t
1102
+
* @a: the amount to add to v...
1103
+
* @u: ...unless v is equal to u.
1104
+
*
1105
+
* Atomically adds @a to @v, if @v was not already @u.
1106
+
* Returns true if the addition was done.
1107
+
*/
1108
+
static inline bool
1109
+
atomic_add_unless(atomic_t *v, int a, int u)
1110
+
{
1111
+
return atomic_fetch_add_unless(v, a, u) != u;
1112
+
}
1113
+
#define atomic_add_unless atomic_add_unless
1114
+
#endif
1115
+
1116
+
#ifndef atomic_inc_not_zero
1117
+
/**
1118
+
* atomic_inc_not_zero - increment unless the number is zero
1119
+
* @v: pointer of type atomic_t
1120
+
*
1121
+
* Atomically increments @v by 1, if @v is non-zero.
1122
+
* Returns true if the increment was done.
1123
+
*/
1124
+
static inline bool
1125
+
atomic_inc_not_zero(atomic_t *v)
1126
+
{
1127
+
return atomic_add_unless(v, 1, 0);
1128
+
}
1129
+
#define atomic_inc_not_zero atomic_inc_not_zero
1130
+
#endif
1131
+
1132
+
#ifndef atomic_inc_unless_negative
1133
+
static inline bool
1134
+
atomic_inc_unless_negative(atomic_t *v)
1135
+
{
1136
+
int c = atomic_read(v);
1137
+
1138
+
do {
1139
+
if (unlikely(c < 0))
1140
+
return false;
1141
+
} while (!atomic_try_cmpxchg(v, &c, c + 1));
1142
+
1143
+
return true;
1144
+
}
1145
+
#define atomic_inc_unless_negative atomic_inc_unless_negative
1146
+
#endif
1147
+
1148
+
#ifndef atomic_dec_unless_positive
1149
+
static inline bool
1150
+
atomic_dec_unless_positive(atomic_t *v)
1151
+
{
1152
+
int c = atomic_read(v);
1153
+
1154
+
do {
1155
+
if (unlikely(c > 0))
1156
+
return false;
1157
+
} while (!atomic_try_cmpxchg(v, &c, c - 1));
1158
+
1159
+
return true;
1160
+
}
1161
+
#define atomic_dec_unless_positive atomic_dec_unless_positive
1162
+
#endif
1163
+
1164
+
#ifndef atomic_dec_if_positive
1165
+
static inline int
1166
+
atomic_dec_if_positive(atomic_t *v)
1167
+
{
1168
+
int dec, c = atomic_read(v);
1169
+
1170
+
do {
1171
+
dec = c - 1;
1172
+
if (unlikely(dec < 0))
1173
+
break;
1174
+
} while (!atomic_try_cmpxchg(v, &c, dec));
1175
+
1176
+
return dec;
1177
+
}
1178
+
#define atomic_dec_if_positive atomic_dec_if_positive
1179
+
#endif
1180
+
1181
+
#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
1182
+
#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
1183
+
1184
+
#ifdef CONFIG_GENERIC_ATOMIC64
1185
+
#include <asm-generic/atomic64.h>
1186
+
#endif
1187
+
1188
+
#ifndef atomic64_read_acquire
1189
+
static inline s64
1190
+
atomic64_read_acquire(const atomic64_t *v)
1191
+
{
1192
+
return smp_load_acquire(&(v)->counter);
1193
+
}
1194
+
#define atomic64_read_acquire atomic64_read_acquire
1195
+
#endif
1196
+
1197
+
#ifndef atomic64_set_release
1198
+
static inline void
1199
+
atomic64_set_release(atomic64_t *v, s64 i)
1200
+
{
1201
+
smp_store_release(&(v)->counter, i);
1202
+
}
1203
+
#define atomic64_set_release atomic64_set_release
1204
+
#endif
1205
+
1206
+
#ifndef atomic64_add_return_relaxed
1207
+
#define atomic64_add_return_acquire atomic64_add_return
1208
+
#define atomic64_add_return_release atomic64_add_return
1209
+
#define atomic64_add_return_relaxed atomic64_add_return
1210
+
#else /* atomic64_add_return_relaxed */
1211
+
1212
+
#ifndef atomic64_add_return_acquire
1213
+
static inline s64
1214
+
atomic64_add_return_acquire(s64 i, atomic64_t *v)
1215
+
{
1216
+
s64 ret = atomic64_add_return_relaxed(i, v);
1217
+
__atomic_acquire_fence();
1218
+
return ret;
1219
+
}
1220
+
#define atomic64_add_return_acquire atomic64_add_return_acquire
1221
+
#endif
1222
+
1223
+
#ifndef atomic64_add_return_release
1224
+
static inline s64
1225
+
atomic64_add_return_release(s64 i, atomic64_t *v)
1226
+
{
1227
+
__atomic_release_fence();
1228
+
return atomic64_add_return_relaxed(i, v);
1229
+
}
1230
+
#define atomic64_add_return_release atomic64_add_return_release
1231
+
#endif
1232
+
1233
+
#ifndef atomic64_add_return
1234
+
static inline s64
1235
+
atomic64_add_return(s64 i, atomic64_t *v)
1236
+
{
1237
+
s64 ret;
1238
+
__atomic_pre_full_fence();
1239
+
ret = atomic64_add_return_relaxed(i, v);
1240
+
__atomic_post_full_fence();
1241
+
return ret;
1242
+
}
1243
+
#define atomic64_add_return atomic64_add_return
1244
+
#endif
1245
+
1246
+
#endif /* atomic64_add_return_relaxed */
1247
+
1248
+
#ifndef atomic64_fetch_add_relaxed
1249
+
#define atomic64_fetch_add_acquire atomic64_fetch_add
1250
+
#define atomic64_fetch_add_release atomic64_fetch_add
1251
+
#define atomic64_fetch_add_relaxed atomic64_fetch_add
1252
+
#else /* atomic64_fetch_add_relaxed */
1253
+
1254
+
#ifndef atomic64_fetch_add_acquire
1255
+
static inline s64
1256
+
atomic64_fetch_add_acquire(s64 i, atomic64_t *v)
1257
+
{
1258
+
s64 ret = atomic64_fetch_add_relaxed(i, v);
1259
+
__atomic_acquire_fence();
1260
+
return ret;
1261
+
}
1262
+
#define atomic64_fetch_add_acquire atomic64_fetch_add_acquire
1263
+
#endif
1264
+
1265
+
#ifndef atomic64_fetch_add_release
1266
+
static inline s64
1267
+
atomic64_fetch_add_release(s64 i, atomic64_t *v)
1268
+
{
1269
+
__atomic_release_fence();
1270
+
return atomic64_fetch_add_relaxed(i, v);
1271
+
}
1272
+
#define atomic64_fetch_add_release atomic64_fetch_add_release
1273
+
#endif
1274
+
1275
+
#ifndef atomic64_fetch_add
1276
+
static inline s64
1277
+
atomic64_fetch_add(s64 i, atomic64_t *v)
1278
+
{
1279
+
s64 ret;
1280
+
__atomic_pre_full_fence();
1281
+
ret = atomic64_fetch_add_relaxed(i, v);
1282
+
__atomic_post_full_fence();
1283
+
return ret;
1284
+
}
1285
+
#define atomic64_fetch_add atomic64_fetch_add
1286
+
#endif
1287
+
1288
+
#endif /* atomic64_fetch_add_relaxed */
1289
+
1290
+
#ifndef atomic64_sub_return_relaxed
1291
+
#define atomic64_sub_return_acquire atomic64_sub_return
1292
+
#define atomic64_sub_return_release atomic64_sub_return
1293
+
#define atomic64_sub_return_relaxed atomic64_sub_return
1294
+
#else /* atomic64_sub_return_relaxed */
1295
+
1296
+
#ifndef atomic64_sub_return_acquire
1297
+
static inline s64
1298
+
atomic64_sub_return_acquire(s64 i, atomic64_t *v)
1299
+
{
1300
+
s64 ret = atomic64_sub_return_relaxed(i, v);
1301
+
__atomic_acquire_fence();
1302
+
return ret;
1303
+
}
1304
+
#define atomic64_sub_return_acquire atomic64_sub_return_acquire
1305
+
#endif
1306
+
1307
+
#ifndef atomic64_sub_return_release
1308
+
static inline s64
1309
+
atomic64_sub_return_release(s64 i, atomic64_t *v)
1310
+
{
1311
+
__atomic_release_fence();
1312
+
return atomic64_sub_return_relaxed(i, v);
1313
+
}
1314
+
#define atomic64_sub_return_release atomic64_sub_return_release
1315
+
#endif
1316
+
1317
+
#ifndef atomic64_sub_return
1318
+
static inline s64
1319
+
atomic64_sub_return(s64 i, atomic64_t *v)
1320
+
{
1321
+
s64 ret;
1322
+
__atomic_pre_full_fence();
1323
+
ret = atomic64_sub_return_relaxed(i, v);
1324
+
__atomic_post_full_fence();
1325
+
return ret;
1326
+
}
1327
+
#define atomic64_sub_return atomic64_sub_return
1328
+
#endif
1329
+
1330
+
#endif /* atomic64_sub_return_relaxed */
1331
+
1332
+
#ifndef atomic64_fetch_sub_relaxed
1333
+
#define atomic64_fetch_sub_acquire atomic64_fetch_sub
1334
+
#define atomic64_fetch_sub_release atomic64_fetch_sub
1335
+
#define atomic64_fetch_sub_relaxed atomic64_fetch_sub
1336
+
#else /* atomic64_fetch_sub_relaxed */
1337
+
1338
+
#ifndef atomic64_fetch_sub_acquire
1339
+
static inline s64
1340
+
atomic64_fetch_sub_acquire(s64 i, atomic64_t *v)
1341
+
{
1342
+
s64 ret = atomic64_fetch_sub_relaxed(i, v);
1343
+
__atomic_acquire_fence();
1344
+
return ret;
1345
+
}
1346
+
#define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire
1347
+
#endif
1348
+
1349
+
#ifndef atomic64_fetch_sub_release
1350
+
static inline s64
1351
+
atomic64_fetch_sub_release(s64 i, atomic64_t *v)
1352
+
{
1353
+
__atomic_release_fence();
1354
+
return atomic64_fetch_sub_relaxed(i, v);
1355
+
}
1356
+
#define atomic64_fetch_sub_release atomic64_fetch_sub_release
1357
+
#endif
1358
+
1359
+
#ifndef atomic64_fetch_sub
1360
+
static inline s64
1361
+
atomic64_fetch_sub(s64 i, atomic64_t *v)
1362
+
{
1363
+
s64 ret;
1364
+
__atomic_pre_full_fence();
1365
+
ret = atomic64_fetch_sub_relaxed(i, v);
1366
+
__atomic_post_full_fence();
1367
+
return ret;
1368
+
}
1369
+
#define atomic64_fetch_sub atomic64_fetch_sub
1370
+
#endif
1371
+
1372
+
#endif /* atomic64_fetch_sub_relaxed */
1373
+
1374
+
#ifndef atomic64_inc
1375
+
static inline void
1376
+
atomic64_inc(atomic64_t *v)
1377
+
{
1378
+
atomic64_add(1, v);
1379
+
}
1380
+
#define atomic64_inc atomic64_inc
1381
+
#endif
1382
+
1383
+
#ifndef atomic64_inc_return_relaxed
1384
+
#ifdef atomic64_inc_return
1385
+
#define atomic64_inc_return_acquire atomic64_inc_return
1386
+
#define atomic64_inc_return_release atomic64_inc_return
1387
+
#define atomic64_inc_return_relaxed atomic64_inc_return
1388
+
#endif /* atomic64_inc_return */
1389
+
1390
+
#ifndef atomic64_inc_return
1391
+
static inline s64
1392
+
atomic64_inc_return(atomic64_t *v)
1393
+
{
1394
+
return atomic64_add_return(1, v);
1395
+
}
1396
+
#define atomic64_inc_return atomic64_inc_return
1397
+
#endif
1398
+
1399
+
#ifndef atomic64_inc_return_acquire
1400
+
static inline s64
1401
+
atomic64_inc_return_acquire(atomic64_t *v)
1402
+
{
1403
+
return atomic64_add_return_acquire(1, v);
1404
+
}
1405
+
#define atomic64_inc_return_acquire atomic64_inc_return_acquire
1406
+
#endif
1407
+
1408
+
#ifndef atomic64_inc_return_release
1409
+
static inline s64
1410
+
atomic64_inc_return_release(atomic64_t *v)
1411
+
{
1412
+
return atomic64_add_return_release(1, v);
1413
+
}
1414
+
#define atomic64_inc_return_release atomic64_inc_return_release
1415
+
#endif
1416
+
1417
+
#ifndef atomic64_inc_return_relaxed
1418
+
static inline s64
1419
+
atomic64_inc_return_relaxed(atomic64_t *v)
1420
+
{
1421
+
return atomic64_add_return_relaxed(1, v);
1422
+
}
1423
+
#define atomic64_inc_return_relaxed atomic64_inc_return_relaxed
1424
+
#endif
1425
+
1426
+
#else /* atomic64_inc_return_relaxed */
1427
+
1428
+
#ifndef atomic64_inc_return_acquire
1429
+
static inline s64
1430
+
atomic64_inc_return_acquire(atomic64_t *v)
1431
+
{
1432
+
s64 ret = atomic64_inc_return_relaxed(v);
1433
+
__atomic_acquire_fence();
1434
+
return ret;
1435
+
}
1436
+
#define atomic64_inc_return_acquire atomic64_inc_return_acquire
1437
+
#endif
1438
+
1439
+
#ifndef atomic64_inc_return_release
1440
+
static inline s64
1441
+
atomic64_inc_return_release(atomic64_t *v)
1442
+
{
1443
+
__atomic_release_fence();
1444
+
return atomic64_inc_return_relaxed(v);
1445
+
}
1446
+
#define atomic64_inc_return_release atomic64_inc_return_release
1447
+
#endif
1448
+
1449
+
#ifndef atomic64_inc_return
1450
+
static inline s64
1451
+
atomic64_inc_return(atomic64_t *v)
1452
+
{
1453
+
s64 ret;
1454
+
__atomic_pre_full_fence();
1455
+
ret = atomic64_inc_return_relaxed(v);
1456
+
__atomic_post_full_fence();
1457
+
return ret;
1458
+
}
1459
+
#define atomic64_inc_return atomic64_inc_return
1460
+
#endif
1461
+
1462
+
#endif /* atomic64_inc_return_relaxed */
1463
+
1464
+
#ifndef atomic64_fetch_inc_relaxed
1465
+
#ifdef atomic64_fetch_inc
1466
+
#define atomic64_fetch_inc_acquire atomic64_fetch_inc
1467
+
#define atomic64_fetch_inc_release atomic64_fetch_inc
1468
+
#define atomic64_fetch_inc_relaxed atomic64_fetch_inc
1469
+
#endif /* atomic64_fetch_inc */
1470
+
1471
+
#ifndef atomic64_fetch_inc
1472
+
static inline s64
1473
+
atomic64_fetch_inc(atomic64_t *v)
1474
+
{
1475
+
return atomic64_fetch_add(1, v);
1476
+
}
1477
+
#define atomic64_fetch_inc atomic64_fetch_inc
1478
+
#endif
1479
+
1480
+
#ifndef atomic64_fetch_inc_acquire
1481
+
static inline s64
1482
+
atomic64_fetch_inc_acquire(atomic64_t *v)
1483
+
{
1484
+
return atomic64_fetch_add_acquire(1, v);
1485
+
}
1486
+
#define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire
1487
+
#endif
1488
+
1489
+
#ifndef atomic64_fetch_inc_release
1490
+
static inline s64
1491
+
atomic64_fetch_inc_release(atomic64_t *v)
1492
+
{
1493
+
return atomic64_fetch_add_release(1, v);
1494
+
}
1495
+
#define atomic64_fetch_inc_release atomic64_fetch_inc_release
1496
+
#endif
1497
+
1498
+
#ifndef atomic64_fetch_inc_relaxed
1499
+
static inline s64
1500
+
atomic64_fetch_inc_relaxed(atomic64_t *v)
1501
+
{
1502
+
return atomic64_fetch_add_relaxed(1, v);
1503
+
}
1504
+
#define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed
1505
+
#endif
1506
+
1507
+
#else /* atomic64_fetch_inc_relaxed */
1508
+
1509
+
#ifndef atomic64_fetch_inc_acquire
1510
+
static inline s64
1511
+
atomic64_fetch_inc_acquire(atomic64_t *v)
1512
+
{
1513
+
s64 ret = atomic64_fetch_inc_relaxed(v);
1514
+
__atomic_acquire_fence();
1515
+
return ret;
1516
+
}
1517
+
#define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire
1518
+
#endif
1519
+
1520
+
#ifndef atomic64_fetch_inc_release
1521
+
static inline s64
1522
+
atomic64_fetch_inc_release(atomic64_t *v)
1523
+
{
1524
+
__atomic_release_fence();
1525
+
return atomic64_fetch_inc_relaxed(v);
1526
+
}
1527
+
#define atomic64_fetch_inc_release atomic64_fetch_inc_release
1528
+
#endif
1529
+
1530
+
#ifndef atomic64_fetch_inc
1531
+
static inline s64
1532
+
atomic64_fetch_inc(atomic64_t *v)
1533
+
{
1534
+
s64 ret;
1535
+
__atomic_pre_full_fence();
1536
+
ret = atomic64_fetch_inc_relaxed(v);
1537
+
__atomic_post_full_fence();
1538
+
return ret;
1539
+
}
1540
+
#define atomic64_fetch_inc atomic64_fetch_inc
1541
+
#endif
1542
+
1543
+
#endif /* atomic64_fetch_inc_relaxed */
1544
+
1545
+
#ifndef atomic64_dec
1546
+
static inline void
1547
+
atomic64_dec(atomic64_t *v)
1548
+
{
1549
+
atomic64_sub(1, v);
1550
+
}
1551
+
#define atomic64_dec atomic64_dec
1552
+
#endif
1553
+
1554
+
#ifndef atomic64_dec_return_relaxed
1555
+
#ifdef atomic64_dec_return
1556
+
#define atomic64_dec_return_acquire atomic64_dec_return
1557
+
#define atomic64_dec_return_release atomic64_dec_return
1558
+
#define atomic64_dec_return_relaxed atomic64_dec_return
1559
+
#endif /* atomic64_dec_return */
1560
+
1561
+
#ifndef atomic64_dec_return
1562
+
static inline s64
1563
+
atomic64_dec_return(atomic64_t *v)
1564
+
{
1565
+
return atomic64_sub_return(1, v);
1566
+
}
1567
+
#define atomic64_dec_return atomic64_dec_return
1568
+
#endif
1569
+
1570
+
#ifndef atomic64_dec_return_acquire
1571
+
static inline s64
1572
+
atomic64_dec_return_acquire(atomic64_t *v)
1573
+
{
1574
+
return atomic64_sub_return_acquire(1, v);
1575
+
}
1576
+
#define atomic64_dec_return_acquire atomic64_dec_return_acquire
1577
+
#endif
1578
+
1579
+
#ifndef atomic64_dec_return_release
1580
+
static inline s64
1581
+
atomic64_dec_return_release(atomic64_t *v)
1582
+
{
1583
+
return atomic64_sub_return_release(1, v);
1584
+
}
1585
+
#define atomic64_dec_return_release atomic64_dec_return_release
1586
+
#endif
1587
+
1588
+
#ifndef atomic64_dec_return_relaxed
1589
+
static inline s64
1590
+
atomic64_dec_return_relaxed(atomic64_t *v)
1591
+
{
1592
+
return atomic64_sub_return_relaxed(1, v);
1593
+
}
1594
+
#define atomic64_dec_return_relaxed atomic64_dec_return_relaxed
1595
+
#endif
1596
+
1597
+
#else /* atomic64_dec_return_relaxed */
1598
+
1599
+
#ifndef atomic64_dec_return_acquire
1600
+
static inline s64
1601
+
atomic64_dec_return_acquire(atomic64_t *v)
1602
+
{
1603
+
s64 ret = atomic64_dec_return_relaxed(v);
1604
+
__atomic_acquire_fence();
1605
+
return ret;
1606
+
}
1607
+
#define atomic64_dec_return_acquire atomic64_dec_return_acquire
1608
+
#endif
1609
+
1610
+
#ifndef atomic64_dec_return_release
1611
+
static inline s64
1612
+
atomic64_dec_return_release(atomic64_t *v)
1613
+
{
1614
+
__atomic_release_fence();
1615
+
return atomic64_dec_return_relaxed(v);
1616
+
}
1617
+
#define atomic64_dec_return_release atomic64_dec_return_release
1618
+
#endif
1619
+
1620
+
#ifndef atomic64_dec_return
1621
+
static inline s64
1622
+
atomic64_dec_return(atomic64_t *v)
1623
+
{
1624
+
s64 ret;
1625
+
__atomic_pre_full_fence();
1626
+
ret = atomic64_dec_return_relaxed(v);
1627
+
__atomic_post_full_fence();
1628
+
return ret;
1629
+
}
1630
+
#define atomic64_dec_return atomic64_dec_return
1631
+
#endif
1632
+
1633
+
#endif /* atomic64_dec_return_relaxed */
1634
+
1635
+
#ifndef atomic64_fetch_dec_relaxed
1636
+
#ifdef atomic64_fetch_dec
1637
+
#define atomic64_fetch_dec_acquire atomic64_fetch_dec
1638
+
#define atomic64_fetch_dec_release atomic64_fetch_dec
1639
+
#define atomic64_fetch_dec_relaxed atomic64_fetch_dec
1640
+
#endif /* atomic64_fetch_dec */
1641
+
1642
+
#ifndef atomic64_fetch_dec
1643
+
static inline s64
1644
+
atomic64_fetch_dec(atomic64_t *v)
1645
+
{
1646
+
return atomic64_fetch_sub(1, v);
1647
+
}
1648
+
#define atomic64_fetch_dec atomic64_fetch_dec
1649
+
#endif
1650
+
1651
+
#ifndef atomic64_fetch_dec_acquire
1652
+
static inline s64
1653
+
atomic64_fetch_dec_acquire(atomic64_t *v)
1654
+
{
1655
+
return atomic64_fetch_sub_acquire(1, v);
1656
+
}
1657
+
#define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire
1658
+
#endif
1659
+
1660
+
#ifndef atomic64_fetch_dec_release
1661
+
static inline s64
1662
+
atomic64_fetch_dec_release(atomic64_t *v)
1663
+
{
1664
+
return atomic64_fetch_sub_release(1, v);
1665
+
}
1666
+
#define atomic64_fetch_dec_release atomic64_fetch_dec_release
1667
+
#endif
1668
+
1669
+
#ifndef atomic64_fetch_dec_relaxed
1670
+
static inline s64
1671
+
atomic64_fetch_dec_relaxed(atomic64_t *v)
1672
+
{
1673
+
return atomic64_fetch_sub_relaxed(1, v);
1674
+
}
1675
+
#define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed
1676
+
#endif
1677
+
1678
+
#else /* atomic64_fetch_dec_relaxed */
1679
+
1680
+
#ifndef atomic64_fetch_dec_acquire
1681
+
static inline s64
1682
+
atomic64_fetch_dec_acquire(atomic64_t *v)
1683
+
{
1684
+
s64 ret = atomic64_fetch_dec_relaxed(v);
1685
+
__atomic_acquire_fence();
1686
+
return ret;
1687
+
}
1688
+
#define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire
1689
+
#endif
1690
+
1691
+
#ifndef atomic64_fetch_dec_release
1692
+
static inline s64
1693
+
atomic64_fetch_dec_release(atomic64_t *v)
1694
+
{
1695
+
__atomic_release_fence();
1696
+
return atomic64_fetch_dec_relaxed(v);
1697
+
}
1698
+
#define atomic64_fetch_dec_release atomic64_fetch_dec_release
1699
+
#endif
1700
+
1701
+
#ifndef atomic64_fetch_dec
1702
+
static inline s64
1703
+
atomic64_fetch_dec(atomic64_t *v)
1704
+
{
1705
+
s64 ret;
1706
+
__atomic_pre_full_fence();
1707
+
ret = atomic64_fetch_dec_relaxed(v);
1708
+
__atomic_post_full_fence();
1709
+
return ret;
1710
+
}
1711
+
#define atomic64_fetch_dec atomic64_fetch_dec
1712
+
#endif
1713
+
1714
+
#endif /* atomic64_fetch_dec_relaxed */
1715
+
1716
+
#ifndef atomic64_fetch_and_relaxed
1717
+
#define atomic64_fetch_and_acquire atomic64_fetch_and
1718
+
#define atomic64_fetch_and_release atomic64_fetch_and
1719
+
#define atomic64_fetch_and_relaxed atomic64_fetch_and
1720
+
#else /* atomic64_fetch_and_relaxed */
1721
+
1722
+
#ifndef atomic64_fetch_and_acquire
1723
+
static inline s64
1724
+
atomic64_fetch_and_acquire(s64 i, atomic64_t *v)
1725
+
{
1726
+
s64 ret = atomic64_fetch_and_relaxed(i, v);
1727
+
__atomic_acquire_fence();
1728
+
return ret;
1729
+
}
1730
+
#define atomic64_fetch_and_acquire atomic64_fetch_and_acquire
1731
+
#endif
1732
+
1733
+
#ifndef atomic64_fetch_and_release
1734
+
static inline s64
1735
+
atomic64_fetch_and_release(s64 i, atomic64_t *v)
1736
+
{
1737
+
__atomic_release_fence();
1738
+
return atomic64_fetch_and_relaxed(i, v);
1739
+
}
1740
+
#define atomic64_fetch_and_release atomic64_fetch_and_release
1741
+
#endif
1742
+
1743
+
#ifndef atomic64_fetch_and
1744
+
static inline s64
1745
+
atomic64_fetch_and(s64 i, atomic64_t *v)
1746
+
{
1747
+
s64 ret;
1748
+
__atomic_pre_full_fence();
1749
+
ret = atomic64_fetch_and_relaxed(i, v);
1750
+
__atomic_post_full_fence();
1751
+
return ret;
1752
+
}
1753
+
#define atomic64_fetch_and atomic64_fetch_and
1754
+
#endif
1755
+
1756
+
#endif /* atomic64_fetch_and_relaxed */
1757
+
1758
+
#ifndef atomic64_andnot
1759
+
static inline void
1760
+
atomic64_andnot(s64 i, atomic64_t *v)
1761
+
{
1762
+
atomic64_and(~i, v);
1763
+
}
1764
+
#define atomic64_andnot atomic64_andnot
1765
+
#endif
1766
+
1767
+
#ifndef atomic64_fetch_andnot_relaxed
1768
+
#ifdef atomic64_fetch_andnot
1769
+
#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot
1770
+
#define atomic64_fetch_andnot_release atomic64_fetch_andnot
1771
+
#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot
1772
+
#endif /* atomic64_fetch_andnot */
1773
+
1774
+
#ifndef atomic64_fetch_andnot
1775
+
static inline s64
1776
+
atomic64_fetch_andnot(s64 i, atomic64_t *v)
1777
+
{
1778
+
return atomic64_fetch_and(~i, v);
1779
+
}
1780
+
#define atomic64_fetch_andnot atomic64_fetch_andnot
1781
+
#endif
1782
+
1783
+
#ifndef atomic64_fetch_andnot_acquire
1784
+
static inline s64
1785
+
atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1786
+
{
1787
+
return atomic64_fetch_and_acquire(~i, v);
1788
+
}
1789
+
#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire
1790
+
#endif
1791
+
1792
+
#ifndef atomic64_fetch_andnot_release
1793
+
static inline s64
1794
+
atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1795
+
{
1796
+
return atomic64_fetch_and_release(~i, v);
1797
+
}
1798
+
#define atomic64_fetch_andnot_release atomic64_fetch_andnot_release
1799
+
#endif
1800
+
1801
+
#ifndef atomic64_fetch_andnot_relaxed
1802
+
static inline s64
1803
+
atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v)
1804
+
{
1805
+
return atomic64_fetch_and_relaxed(~i, v);
1806
+
}
1807
+
#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed
1808
+
#endif
1809
+
1810
+
#else /* atomic64_fetch_andnot_relaxed */
1811
+
1812
+
#ifndef atomic64_fetch_andnot_acquire
1813
+
static inline s64
1814
+
atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v)
1815
+
{
1816
+
s64 ret = atomic64_fetch_andnot_relaxed(i, v);
1817
+
__atomic_acquire_fence();
1818
+
return ret;
1819
+
}
1820
+
#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire
1821
+
#endif
1822
+
1823
+
#ifndef atomic64_fetch_andnot_release
1824
+
static inline s64
1825
+
atomic64_fetch_andnot_release(s64 i, atomic64_t *v)
1826
+
{
1827
+
__atomic_release_fence();
1828
+
return atomic64_fetch_andnot_relaxed(i, v);
1829
+
}
1830
+
#define atomic64_fetch_andnot_release atomic64_fetch_andnot_release
1831
+
#endif
1832
+
1833
+
#ifndef atomic64_fetch_andnot
1834
+
static inline s64
1835
+
atomic64_fetch_andnot(s64 i, atomic64_t *v)
1836
+
{
1837
+
s64 ret;
1838
+
__atomic_pre_full_fence();
1839
+
ret = atomic64_fetch_andnot_relaxed(i, v);
1840
+
__atomic_post_full_fence();
1841
+
return ret;
1842
+
}
1843
+
#define atomic64_fetch_andnot atomic64_fetch_andnot
1844
+
#endif
1845
+
1846
+
#endif /* atomic64_fetch_andnot_relaxed */
1847
+
1848
+
#ifndef atomic64_fetch_or_relaxed
1849
+
#define atomic64_fetch_or_acquire atomic64_fetch_or
1850
+
#define atomic64_fetch_or_release atomic64_fetch_or
1851
+
#define atomic64_fetch_or_relaxed atomic64_fetch_or
1852
+
#else /* atomic64_fetch_or_relaxed */
1853
+
1854
+
#ifndef atomic64_fetch_or_acquire
1855
+
static inline s64
1856
+
atomic64_fetch_or_acquire(s64 i, atomic64_t *v)
1857
+
{
1858
+
s64 ret = atomic64_fetch_or_relaxed(i, v);
1859
+
__atomic_acquire_fence();
1860
+
return ret;
1861
+
}
1862
+
#define atomic64_fetch_or_acquire atomic64_fetch_or_acquire
1863
+
#endif
1864
+
1865
+
#ifndef atomic64_fetch_or_release
1866
+
static inline s64
1867
+
atomic64_fetch_or_release(s64 i, atomic64_t *v)
1868
+
{
1869
+
__atomic_release_fence();
1870
+
return atomic64_fetch_or_relaxed(i, v);
1871
+
}
1872
+
#define atomic64_fetch_or_release atomic64_fetch_or_release
1873
+
#endif
1874
+
1875
+
#ifndef atomic64_fetch_or
1876
+
static inline s64
1877
+
atomic64_fetch_or(s64 i, atomic64_t *v)
1878
+
{
1879
+
s64 ret;
1880
+
__atomic_pre_full_fence();
1881
+
ret = atomic64_fetch_or_relaxed(i, v);
1882
+
__atomic_post_full_fence();
1883
+
return ret;
1884
+
}
1885
+
#define atomic64_fetch_or atomic64_fetch_or
1886
+
#endif
1887
+
1888
+
#endif /* atomic64_fetch_or_relaxed */
1889
+
1890
+
#ifndef atomic64_fetch_xor_relaxed
1891
+
#define atomic64_fetch_xor_acquire atomic64_fetch_xor
1892
+
#define atomic64_fetch_xor_release atomic64_fetch_xor
1893
+
#define atomic64_fetch_xor_relaxed atomic64_fetch_xor
1894
+
#else /* atomic64_fetch_xor_relaxed */
1895
+
1896
+
#ifndef atomic64_fetch_xor_acquire
1897
+
static inline s64
1898
+
atomic64_fetch_xor_acquire(s64 i, atomic64_t *v)
1899
+
{
1900
+
s64 ret = atomic64_fetch_xor_relaxed(i, v);
1901
+
__atomic_acquire_fence();
1902
+
return ret;
1903
+
}
1904
+
#define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire
1905
+
#endif
1906
+
1907
+
#ifndef atomic64_fetch_xor_release
1908
+
static inline s64
1909
+
atomic64_fetch_xor_release(s64 i, atomic64_t *v)
1910
+
{
1911
+
__atomic_release_fence();
1912
+
return atomic64_fetch_xor_relaxed(i, v);
1913
+
}
1914
+
#define atomic64_fetch_xor_release atomic64_fetch_xor_release
1915
+
#endif
1916
+
1917
+
#ifndef atomic64_fetch_xor
1918
+
static inline s64
1919
+
atomic64_fetch_xor(s64 i, atomic64_t *v)
1920
+
{
1921
+
s64 ret;
1922
+
__atomic_pre_full_fence();
1923
+
ret = atomic64_fetch_xor_relaxed(i, v);
1924
+
__atomic_post_full_fence();
1925
+
return ret;
1926
+
}
1927
+
#define atomic64_fetch_xor atomic64_fetch_xor
1928
+
#endif
1929
+
1930
+
#endif /* atomic64_fetch_xor_relaxed */
1931
+
1932
+
#ifndef atomic64_xchg_relaxed
1933
+
#define atomic64_xchg_acquire atomic64_xchg
1934
+
#define atomic64_xchg_release atomic64_xchg
1935
+
#define atomic64_xchg_relaxed atomic64_xchg
1936
+
#else /* atomic64_xchg_relaxed */
1937
+
1938
+
#ifndef atomic64_xchg_acquire
1939
+
static inline s64
1940
+
atomic64_xchg_acquire(atomic64_t *v, s64 i)
1941
+
{
1942
+
s64 ret = atomic64_xchg_relaxed(v, i);
1943
+
__atomic_acquire_fence();
1944
+
return ret;
1945
+
}
1946
+
#define atomic64_xchg_acquire atomic64_xchg_acquire
1947
+
#endif
1948
+
1949
+
#ifndef atomic64_xchg_release
1950
+
static inline s64
1951
+
atomic64_xchg_release(atomic64_t *v, s64 i)
1952
+
{
1953
+
__atomic_release_fence();
1954
+
return atomic64_xchg_relaxed(v, i);
1955
+
}
1956
+
#define atomic64_xchg_release atomic64_xchg_release
1957
+
#endif
1958
+
1959
+
#ifndef atomic64_xchg
1960
+
static inline s64
1961
+
atomic64_xchg(atomic64_t *v, s64 i)
1962
+
{
1963
+
s64 ret;
1964
+
__atomic_pre_full_fence();
1965
+
ret = atomic64_xchg_relaxed(v, i);
1966
+
__atomic_post_full_fence();
1967
+
return ret;
1968
+
}
1969
+
#define atomic64_xchg atomic64_xchg
1970
+
#endif
1971
+
1972
+
#endif /* atomic64_xchg_relaxed */
1973
+
1974
+
#ifndef atomic64_cmpxchg_relaxed
1975
+
#define atomic64_cmpxchg_acquire atomic64_cmpxchg
1976
+
#define atomic64_cmpxchg_release atomic64_cmpxchg
1977
+
#define atomic64_cmpxchg_relaxed atomic64_cmpxchg
1978
+
#else /* atomic64_cmpxchg_relaxed */
1979
+
1980
+
#ifndef atomic64_cmpxchg_acquire
1981
+
static inline s64
1982
+
atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new)
1983
+
{
1984
+
s64 ret = atomic64_cmpxchg_relaxed(v, old, new);
1985
+
__atomic_acquire_fence();
1986
+
return ret;
1987
+
}
1988
+
#define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire
1989
+
#endif
1990
+
1991
+
#ifndef atomic64_cmpxchg_release
1992
+
static inline s64
1993
+
atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new)
1994
+
{
1995
+
__atomic_release_fence();
1996
+
return atomic64_cmpxchg_relaxed(v, old, new);
1997
+
}
1998
+
#define atomic64_cmpxchg_release atomic64_cmpxchg_release
1999
+
#endif
2000
+
2001
+
#ifndef atomic64_cmpxchg
2002
+
static inline s64
2003
+
atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new)
2004
+
{
2005
+
s64 ret;
2006
+
__atomic_pre_full_fence();
2007
+
ret = atomic64_cmpxchg_relaxed(v, old, new);
2008
+
__atomic_post_full_fence();
2009
+
return ret;
2010
+
}
2011
+
#define atomic64_cmpxchg atomic64_cmpxchg
2012
+
#endif
2013
+
2014
+
#endif /* atomic64_cmpxchg_relaxed */
2015
+
2016
+
#ifndef atomic64_try_cmpxchg_relaxed
2017
+
#ifdef atomic64_try_cmpxchg
2018
+
#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg
2019
+
#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg
2020
+
#define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg
2021
+
#endif /* atomic64_try_cmpxchg */
2022
+
2023
+
#ifndef atomic64_try_cmpxchg
2024
+
static inline bool
2025
+
atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2026
+
{
2027
+
s64 r, o = *old;
2028
+
r = atomic64_cmpxchg(v, o, new);
2029
+
if (unlikely(r != o))
2030
+
*old = r;
2031
+
return likely(r == o);
2032
+
}
2033
+
#define atomic64_try_cmpxchg atomic64_try_cmpxchg
2034
+
#endif
2035
+
2036
+
#ifndef atomic64_try_cmpxchg_acquire
2037
+
static inline bool
2038
+
atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2039
+
{
2040
+
s64 r, o = *old;
2041
+
r = atomic64_cmpxchg_acquire(v, o, new);
2042
+
if (unlikely(r != o))
2043
+
*old = r;
2044
+
return likely(r == o);
2045
+
}
2046
+
#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire
2047
+
#endif
2048
+
2049
+
#ifndef atomic64_try_cmpxchg_release
2050
+
static inline bool
2051
+
atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2052
+
{
2053
+
s64 r, o = *old;
2054
+
r = atomic64_cmpxchg_release(v, o, new);
2055
+
if (unlikely(r != o))
2056
+
*old = r;
2057
+
return likely(r == o);
2058
+
}
2059
+
#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release
2060
+
#endif
2061
+
2062
+
#ifndef atomic64_try_cmpxchg_relaxed
2063
+
static inline bool
2064
+
atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new)
2065
+
{
2066
+
s64 r, o = *old;
2067
+
r = atomic64_cmpxchg_relaxed(v, o, new);
2068
+
if (unlikely(r != o))
2069
+
*old = r;
2070
+
return likely(r == o);
2071
+
}
2072
+
#define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed
2073
+
#endif
2074
+
2075
+
#else /* atomic64_try_cmpxchg_relaxed */
2076
+
2077
+
#ifndef atomic64_try_cmpxchg_acquire
2078
+
static inline bool
2079
+
atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new)
2080
+
{
2081
+
bool ret = atomic64_try_cmpxchg_relaxed(v, old, new);
2082
+
__atomic_acquire_fence();
2083
+
return ret;
2084
+
}
2085
+
#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire
2086
+
#endif
2087
+
2088
+
#ifndef atomic64_try_cmpxchg_release
2089
+
static inline bool
2090
+
atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new)
2091
+
{
2092
+
__atomic_release_fence();
2093
+
return atomic64_try_cmpxchg_relaxed(v, old, new);
2094
+
}
2095
+
#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release
2096
+
#endif
2097
+
2098
+
#ifndef atomic64_try_cmpxchg
2099
+
static inline bool
2100
+
atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new)
2101
+
{
2102
+
bool ret;
2103
+
__atomic_pre_full_fence();
2104
+
ret = atomic64_try_cmpxchg_relaxed(v, old, new);
2105
+
__atomic_post_full_fence();
2106
+
return ret;
2107
+
}
2108
+
#define atomic64_try_cmpxchg atomic64_try_cmpxchg
2109
+
#endif
2110
+
2111
+
#endif /* atomic64_try_cmpxchg_relaxed */
2112
+
2113
+
#ifndef atomic64_sub_and_test
2114
+
/**
2115
+
* atomic64_sub_and_test - subtract value from variable and test result
2116
+
* @i: integer value to subtract
2117
+
* @v: pointer of type atomic64_t
2118
+
*
2119
+
* Atomically subtracts @i from @v and returns
2120
+
* true if the result is zero, or false for all
2121
+
* other cases.
2122
+
*/
2123
+
static inline bool
2124
+
atomic64_sub_and_test(s64 i, atomic64_t *v)
2125
+
{
2126
+
return atomic64_sub_return(i, v) == 0;
2127
+
}
2128
+
#define atomic64_sub_and_test atomic64_sub_and_test
2129
+
#endif
2130
+
2131
+
#ifndef atomic64_dec_and_test
2132
+
/**
2133
+
* atomic64_dec_and_test - decrement and test
2134
+
* @v: pointer of type atomic64_t
2135
+
*
2136
+
* Atomically decrements @v by 1 and
2137
+
* returns true if the result is 0, or false for all other
2138
+
* cases.
2139
+
*/
2140
+
static inline bool
2141
+
atomic64_dec_and_test(atomic64_t *v)
2142
+
{
2143
+
return atomic64_dec_return(v) == 0;
2144
+
}
2145
+
#define atomic64_dec_and_test atomic64_dec_and_test
2146
+
#endif
2147
+
2148
+
#ifndef atomic64_inc_and_test
2149
+
/**
2150
+
* atomic64_inc_and_test - increment and test
2151
+
* @v: pointer of type atomic64_t
2152
+
*
2153
+
* Atomically increments @v by 1
2154
+
* and returns true if the result is zero, or false for all
2155
+
* other cases.
2156
+
*/
2157
+
static inline bool
2158
+
atomic64_inc_and_test(atomic64_t *v)
2159
+
{
2160
+
return atomic64_inc_return(v) == 0;
2161
+
}
2162
+
#define atomic64_inc_and_test atomic64_inc_and_test
2163
+
#endif
2164
+
2165
+
#ifndef atomic64_add_negative
2166
+
/**
2167
+
* atomic64_add_negative - add and test if negative
2168
+
* @i: integer value to add
2169
+
* @v: pointer of type atomic64_t
2170
+
*
2171
+
* Atomically adds @i to @v and returns true
2172
+
* if the result is negative, or false when
2173
+
* result is greater than or equal to zero.
2174
+
*/
2175
+
static inline bool
2176
+
atomic64_add_negative(s64 i, atomic64_t *v)
2177
+
{
2178
+
return atomic64_add_return(i, v) < 0;
2179
+
}
2180
+
#define atomic64_add_negative atomic64_add_negative
2181
+
#endif
2182
+
2183
+
#ifndef atomic64_fetch_add_unless
2184
+
/**
2185
+
* atomic64_fetch_add_unless - add unless the number is already a given value
2186
+
* @v: pointer of type atomic64_t
2187
+
* @a: the amount to add to v...
2188
+
* @u: ...unless v is equal to u.
2189
+
*
2190
+
* Atomically adds @a to @v, so long as @v was not already @u.
2191
+
* Returns original value of @v
2192
+
*/
2193
+
static inline s64
2194
+
atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u)
2195
+
{
2196
+
s64 c = atomic64_read(v);
2197
+
2198
+
do {
2199
+
if (unlikely(c == u))
2200
+
break;
2201
+
} while (!atomic64_try_cmpxchg(v, &c, c + a));
2202
+
2203
+
return c;
2204
+
}
2205
+
#define atomic64_fetch_add_unless atomic64_fetch_add_unless
2206
+
#endif
2207
+
2208
+
#ifndef atomic64_add_unless
2209
+
/**
2210
+
* atomic64_add_unless - add unless the number is already a given value
2211
+
* @v: pointer of type atomic64_t
2212
+
* @a: the amount to add to v...
2213
+
* @u: ...unless v is equal to u.
2214
+
*
2215
+
* Atomically adds @a to @v, if @v was not already @u.
2216
+
* Returns true if the addition was done.
2217
+
*/
2218
+
static inline bool
2219
+
atomic64_add_unless(atomic64_t *v, s64 a, s64 u)
2220
+
{
2221
+
return atomic64_fetch_add_unless(v, a, u) != u;
2222
+
}
2223
+
#define atomic64_add_unless atomic64_add_unless
2224
+
#endif
2225
+
2226
+
#ifndef atomic64_inc_not_zero
2227
+
/**
2228
+
* atomic64_inc_not_zero - increment unless the number is zero
2229
+
* @v: pointer of type atomic64_t
2230
+
*
2231
+
* Atomically increments @v by 1, if @v is non-zero.
2232
+
* Returns true if the increment was done.
2233
+
*/
2234
+
static inline bool
2235
+
atomic64_inc_not_zero(atomic64_t *v)
2236
+
{
2237
+
return atomic64_add_unless(v, 1, 0);
2238
+
}
2239
+
#define atomic64_inc_not_zero atomic64_inc_not_zero
2240
+
#endif
2241
+
2242
+
#ifndef atomic64_inc_unless_negative
2243
+
static inline bool
2244
+
atomic64_inc_unless_negative(atomic64_t *v)
2245
+
{
2246
+
s64 c = atomic64_read(v);
2247
+
2248
+
do {
2249
+
if (unlikely(c < 0))
2250
+
return false;
2251
+
} while (!atomic64_try_cmpxchg(v, &c, c + 1));
2252
+
2253
+
return true;
2254
+
}
2255
+
#define atomic64_inc_unless_negative atomic64_inc_unless_negative
2256
+
#endif
2257
+
2258
+
#ifndef atomic64_dec_unless_positive
2259
+
static inline bool
2260
+
atomic64_dec_unless_positive(atomic64_t *v)
2261
+
{
2262
+
s64 c = atomic64_read(v);
2263
+
2264
+
do {
2265
+
if (unlikely(c > 0))
2266
+
return false;
2267
+
} while (!atomic64_try_cmpxchg(v, &c, c - 1));
2268
+
2269
+
return true;
2270
+
}
2271
+
#define atomic64_dec_unless_positive atomic64_dec_unless_positive
2272
+
#endif
2273
+
2274
+
#ifndef atomic64_dec_if_positive
2275
+
static inline s64
2276
+
atomic64_dec_if_positive(atomic64_t *v)
2277
+
{
2278
+
s64 dec, c = atomic64_read(v);
2279
+
2280
+
do {
2281
+
dec = c - 1;
2282
+
if (unlikely(dec < 0))
2283
+
break;
2284
+
} while (!atomic64_try_cmpxchg(v, &c, dec));
2285
+
2286
+
return dec;
2287
+
}
2288
+
#define atomic64_dec_if_positive atomic64_dec_if_positive
2289
+
#endif
2290
+
2291
+
#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
2292
+
#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
2293
+
2294
+
#endif /* _LINUX_ATOMIC_FALLBACK_H */
+1
-1240
include/linux/atomic.h
+1
-1240
include/linux/atomic.h
···
25
25
* See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions.
26
26
*/
27
27
28
-
#ifndef atomic_read_acquire
29
-
#define atomic_read_acquire(v) smp_load_acquire(&(v)->counter)
30
-
#endif
31
-
32
-
#ifndef atomic_set_release
33
-
#define atomic_set_release(v, i) smp_store_release(&(v)->counter, (i))
34
-
#endif
35
-
36
28
/*
37
29
* The idea here is to build acquire/release variants by adding explicit
38
30
* barriers on top of the relaxed variant. In the case where the relaxed
···
71
79
__ret; \
72
80
})
73
81
74
-
/* atomic_add_return_relaxed */
75
-
#ifndef atomic_add_return_relaxed
76
-
#define atomic_add_return_relaxed atomic_add_return
77
-
#define atomic_add_return_acquire atomic_add_return
78
-
#define atomic_add_return_release atomic_add_return
79
-
80
-
#else /* atomic_add_return_relaxed */
81
-
82
-
#ifndef atomic_add_return_acquire
83
-
#define atomic_add_return_acquire(...) \
84
-
__atomic_op_acquire(atomic_add_return, __VA_ARGS__)
85
-
#endif
86
-
87
-
#ifndef atomic_add_return_release
88
-
#define atomic_add_return_release(...) \
89
-
__atomic_op_release(atomic_add_return, __VA_ARGS__)
90
-
#endif
91
-
92
-
#ifndef atomic_add_return
93
-
#define atomic_add_return(...) \
94
-
__atomic_op_fence(atomic_add_return, __VA_ARGS__)
95
-
#endif
96
-
#endif /* atomic_add_return_relaxed */
97
-
98
-
#ifndef atomic_inc
99
-
#define atomic_inc(v) atomic_add(1, (v))
100
-
#endif
101
-
102
-
/* atomic_inc_return_relaxed */
103
-
#ifndef atomic_inc_return_relaxed
104
-
105
-
#ifndef atomic_inc_return
106
-
#define atomic_inc_return(v) atomic_add_return(1, (v))
107
-
#define atomic_inc_return_relaxed(v) atomic_add_return_relaxed(1, (v))
108
-
#define atomic_inc_return_acquire(v) atomic_add_return_acquire(1, (v))
109
-
#define atomic_inc_return_release(v) atomic_add_return_release(1, (v))
110
-
#else /* atomic_inc_return */
111
-
#define atomic_inc_return_relaxed atomic_inc_return
112
-
#define atomic_inc_return_acquire atomic_inc_return
113
-
#define atomic_inc_return_release atomic_inc_return
114
-
#endif /* atomic_inc_return */
115
-
116
-
#else /* atomic_inc_return_relaxed */
117
-
118
-
#ifndef atomic_inc_return_acquire
119
-
#define atomic_inc_return_acquire(...) \
120
-
__atomic_op_acquire(atomic_inc_return, __VA_ARGS__)
121
-
#endif
122
-
123
-
#ifndef atomic_inc_return_release
124
-
#define atomic_inc_return_release(...) \
125
-
__atomic_op_release(atomic_inc_return, __VA_ARGS__)
126
-
#endif
127
-
128
-
#ifndef atomic_inc_return
129
-
#define atomic_inc_return(...) \
130
-
__atomic_op_fence(atomic_inc_return, __VA_ARGS__)
131
-
#endif
132
-
#endif /* atomic_inc_return_relaxed */
133
-
134
-
/* atomic_sub_return_relaxed */
135
-
#ifndef atomic_sub_return_relaxed
136
-
#define atomic_sub_return_relaxed atomic_sub_return
137
-
#define atomic_sub_return_acquire atomic_sub_return
138
-
#define atomic_sub_return_release atomic_sub_return
139
-
140
-
#else /* atomic_sub_return_relaxed */
141
-
142
-
#ifndef atomic_sub_return_acquire
143
-
#define atomic_sub_return_acquire(...) \
144
-
__atomic_op_acquire(atomic_sub_return, __VA_ARGS__)
145
-
#endif
146
-
147
-
#ifndef atomic_sub_return_release
148
-
#define atomic_sub_return_release(...) \
149
-
__atomic_op_release(atomic_sub_return, __VA_ARGS__)
150
-
#endif
151
-
152
-
#ifndef atomic_sub_return
153
-
#define atomic_sub_return(...) \
154
-
__atomic_op_fence(atomic_sub_return, __VA_ARGS__)
155
-
#endif
156
-
#endif /* atomic_sub_return_relaxed */
157
-
158
-
#ifndef atomic_dec
159
-
#define atomic_dec(v) atomic_sub(1, (v))
160
-
#endif
161
-
162
-
/* atomic_dec_return_relaxed */
163
-
#ifndef atomic_dec_return_relaxed
164
-
165
-
#ifndef atomic_dec_return
166
-
#define atomic_dec_return(v) atomic_sub_return(1, (v))
167
-
#define atomic_dec_return_relaxed(v) atomic_sub_return_relaxed(1, (v))
168
-
#define atomic_dec_return_acquire(v) atomic_sub_return_acquire(1, (v))
169
-
#define atomic_dec_return_release(v) atomic_sub_return_release(1, (v))
170
-
#else /* atomic_dec_return */
171
-
#define atomic_dec_return_relaxed atomic_dec_return
172
-
#define atomic_dec_return_acquire atomic_dec_return
173
-
#define atomic_dec_return_release atomic_dec_return
174
-
#endif /* atomic_dec_return */
175
-
176
-
#else /* atomic_dec_return_relaxed */
177
-
178
-
#ifndef atomic_dec_return_acquire
179
-
#define atomic_dec_return_acquire(...) \
180
-
__atomic_op_acquire(atomic_dec_return, __VA_ARGS__)
181
-
#endif
182
-
183
-
#ifndef atomic_dec_return_release
184
-
#define atomic_dec_return_release(...) \
185
-
__atomic_op_release(atomic_dec_return, __VA_ARGS__)
186
-
#endif
187
-
188
-
#ifndef atomic_dec_return
189
-
#define atomic_dec_return(...) \
190
-
__atomic_op_fence(atomic_dec_return, __VA_ARGS__)
191
-
#endif
192
-
#endif /* atomic_dec_return_relaxed */
193
-
194
-
195
-
/* atomic_fetch_add_relaxed */
196
-
#ifndef atomic_fetch_add_relaxed
197
-
#define atomic_fetch_add_relaxed atomic_fetch_add
198
-
#define atomic_fetch_add_acquire atomic_fetch_add
199
-
#define atomic_fetch_add_release atomic_fetch_add
200
-
201
-
#else /* atomic_fetch_add_relaxed */
202
-
203
-
#ifndef atomic_fetch_add_acquire
204
-
#define atomic_fetch_add_acquire(...) \
205
-
__atomic_op_acquire(atomic_fetch_add, __VA_ARGS__)
206
-
#endif
207
-
208
-
#ifndef atomic_fetch_add_release
209
-
#define atomic_fetch_add_release(...) \
210
-
__atomic_op_release(atomic_fetch_add, __VA_ARGS__)
211
-
#endif
212
-
213
-
#ifndef atomic_fetch_add
214
-
#define atomic_fetch_add(...) \
215
-
__atomic_op_fence(atomic_fetch_add, __VA_ARGS__)
216
-
#endif
217
-
#endif /* atomic_fetch_add_relaxed */
218
-
219
-
/* atomic_fetch_inc_relaxed */
220
-
#ifndef atomic_fetch_inc_relaxed
221
-
222
-
#ifndef atomic_fetch_inc
223
-
#define atomic_fetch_inc(v) atomic_fetch_add(1, (v))
224
-
#define atomic_fetch_inc_relaxed(v) atomic_fetch_add_relaxed(1, (v))
225
-
#define atomic_fetch_inc_acquire(v) atomic_fetch_add_acquire(1, (v))
226
-
#define atomic_fetch_inc_release(v) atomic_fetch_add_release(1, (v))
227
-
#else /* atomic_fetch_inc */
228
-
#define atomic_fetch_inc_relaxed atomic_fetch_inc
229
-
#define atomic_fetch_inc_acquire atomic_fetch_inc
230
-
#define atomic_fetch_inc_release atomic_fetch_inc
231
-
#endif /* atomic_fetch_inc */
232
-
233
-
#else /* atomic_fetch_inc_relaxed */
234
-
235
-
#ifndef atomic_fetch_inc_acquire
236
-
#define atomic_fetch_inc_acquire(...) \
237
-
__atomic_op_acquire(atomic_fetch_inc, __VA_ARGS__)
238
-
#endif
239
-
240
-
#ifndef atomic_fetch_inc_release
241
-
#define atomic_fetch_inc_release(...) \
242
-
__atomic_op_release(atomic_fetch_inc, __VA_ARGS__)
243
-
#endif
244
-
245
-
#ifndef atomic_fetch_inc
246
-
#define atomic_fetch_inc(...) \
247
-
__atomic_op_fence(atomic_fetch_inc, __VA_ARGS__)
248
-
#endif
249
-
#endif /* atomic_fetch_inc_relaxed */
250
-
251
-
/* atomic_fetch_sub_relaxed */
252
-
#ifndef atomic_fetch_sub_relaxed
253
-
#define atomic_fetch_sub_relaxed atomic_fetch_sub
254
-
#define atomic_fetch_sub_acquire atomic_fetch_sub
255
-
#define atomic_fetch_sub_release atomic_fetch_sub
256
-
257
-
#else /* atomic_fetch_sub_relaxed */
258
-
259
-
#ifndef atomic_fetch_sub_acquire
260
-
#define atomic_fetch_sub_acquire(...) \
261
-
__atomic_op_acquire(atomic_fetch_sub, __VA_ARGS__)
262
-
#endif
263
-
264
-
#ifndef atomic_fetch_sub_release
265
-
#define atomic_fetch_sub_release(...) \
266
-
__atomic_op_release(atomic_fetch_sub, __VA_ARGS__)
267
-
#endif
268
-
269
-
#ifndef atomic_fetch_sub
270
-
#define atomic_fetch_sub(...) \
271
-
__atomic_op_fence(atomic_fetch_sub, __VA_ARGS__)
272
-
#endif
273
-
#endif /* atomic_fetch_sub_relaxed */
274
-
275
-
/* atomic_fetch_dec_relaxed */
276
-
#ifndef atomic_fetch_dec_relaxed
277
-
278
-
#ifndef atomic_fetch_dec
279
-
#define atomic_fetch_dec(v) atomic_fetch_sub(1, (v))
280
-
#define atomic_fetch_dec_relaxed(v) atomic_fetch_sub_relaxed(1, (v))
281
-
#define atomic_fetch_dec_acquire(v) atomic_fetch_sub_acquire(1, (v))
282
-
#define atomic_fetch_dec_release(v) atomic_fetch_sub_release(1, (v))
283
-
#else /* atomic_fetch_dec */
284
-
#define atomic_fetch_dec_relaxed atomic_fetch_dec
285
-
#define atomic_fetch_dec_acquire atomic_fetch_dec
286
-
#define atomic_fetch_dec_release atomic_fetch_dec
287
-
#endif /* atomic_fetch_dec */
288
-
289
-
#else /* atomic_fetch_dec_relaxed */
290
-
291
-
#ifndef atomic_fetch_dec_acquire
292
-
#define atomic_fetch_dec_acquire(...) \
293
-
__atomic_op_acquire(atomic_fetch_dec, __VA_ARGS__)
294
-
#endif
295
-
296
-
#ifndef atomic_fetch_dec_release
297
-
#define atomic_fetch_dec_release(...) \
298
-
__atomic_op_release(atomic_fetch_dec, __VA_ARGS__)
299
-
#endif
300
-
301
-
#ifndef atomic_fetch_dec
302
-
#define atomic_fetch_dec(...) \
303
-
__atomic_op_fence(atomic_fetch_dec, __VA_ARGS__)
304
-
#endif
305
-
#endif /* atomic_fetch_dec_relaxed */
306
-
307
-
/* atomic_fetch_or_relaxed */
308
-
#ifndef atomic_fetch_or_relaxed
309
-
#define atomic_fetch_or_relaxed atomic_fetch_or
310
-
#define atomic_fetch_or_acquire atomic_fetch_or
311
-
#define atomic_fetch_or_release atomic_fetch_or
312
-
313
-
#else /* atomic_fetch_or_relaxed */
314
-
315
-
#ifndef atomic_fetch_or_acquire
316
-
#define atomic_fetch_or_acquire(...) \
317
-
__atomic_op_acquire(atomic_fetch_or, __VA_ARGS__)
318
-
#endif
319
-
320
-
#ifndef atomic_fetch_or_release
321
-
#define atomic_fetch_or_release(...) \
322
-
__atomic_op_release(atomic_fetch_or, __VA_ARGS__)
323
-
#endif
324
-
325
-
#ifndef atomic_fetch_or
326
-
#define atomic_fetch_or(...) \
327
-
__atomic_op_fence(atomic_fetch_or, __VA_ARGS__)
328
-
#endif
329
-
#endif /* atomic_fetch_or_relaxed */
330
-
331
-
/* atomic_fetch_and_relaxed */
332
-
#ifndef atomic_fetch_and_relaxed
333
-
#define atomic_fetch_and_relaxed atomic_fetch_and
334
-
#define atomic_fetch_and_acquire atomic_fetch_and
335
-
#define atomic_fetch_and_release atomic_fetch_and
336
-
337
-
#else /* atomic_fetch_and_relaxed */
338
-
339
-
#ifndef atomic_fetch_and_acquire
340
-
#define atomic_fetch_and_acquire(...) \
341
-
__atomic_op_acquire(atomic_fetch_and, __VA_ARGS__)
342
-
#endif
343
-
344
-
#ifndef atomic_fetch_and_release
345
-
#define atomic_fetch_and_release(...) \
346
-
__atomic_op_release(atomic_fetch_and, __VA_ARGS__)
347
-
#endif
348
-
349
-
#ifndef atomic_fetch_and
350
-
#define atomic_fetch_and(...) \
351
-
__atomic_op_fence(atomic_fetch_and, __VA_ARGS__)
352
-
#endif
353
-
#endif /* atomic_fetch_and_relaxed */
354
-
355
-
#ifndef atomic_andnot
356
-
#define atomic_andnot(i, v) atomic_and(~(int)(i), (v))
357
-
#endif
358
-
359
-
#ifndef atomic_fetch_andnot_relaxed
360
-
361
-
#ifndef atomic_fetch_andnot
362
-
#define atomic_fetch_andnot(i, v) atomic_fetch_and(~(int)(i), (v))
363
-
#define atomic_fetch_andnot_relaxed(i, v) atomic_fetch_and_relaxed(~(int)(i), (v))
364
-
#define atomic_fetch_andnot_acquire(i, v) atomic_fetch_and_acquire(~(int)(i), (v))
365
-
#define atomic_fetch_andnot_release(i, v) atomic_fetch_and_release(~(int)(i), (v))
366
-
#else /* atomic_fetch_andnot */
367
-
#define atomic_fetch_andnot_relaxed atomic_fetch_andnot
368
-
#define atomic_fetch_andnot_acquire atomic_fetch_andnot
369
-
#define atomic_fetch_andnot_release atomic_fetch_andnot
370
-
#endif /* atomic_fetch_andnot */
371
-
372
-
#else /* atomic_fetch_andnot_relaxed */
373
-
374
-
#ifndef atomic_fetch_andnot_acquire
375
-
#define atomic_fetch_andnot_acquire(...) \
376
-
__atomic_op_acquire(atomic_fetch_andnot, __VA_ARGS__)
377
-
#endif
378
-
379
-
#ifndef atomic_fetch_andnot_release
380
-
#define atomic_fetch_andnot_release(...) \
381
-
__atomic_op_release(atomic_fetch_andnot, __VA_ARGS__)
382
-
#endif
383
-
384
-
#ifndef atomic_fetch_andnot
385
-
#define atomic_fetch_andnot(...) \
386
-
__atomic_op_fence(atomic_fetch_andnot, __VA_ARGS__)
387
-
#endif
388
-
#endif /* atomic_fetch_andnot_relaxed */
389
-
390
-
/* atomic_fetch_xor_relaxed */
391
-
#ifndef atomic_fetch_xor_relaxed
392
-
#define atomic_fetch_xor_relaxed atomic_fetch_xor
393
-
#define atomic_fetch_xor_acquire atomic_fetch_xor
394
-
#define atomic_fetch_xor_release atomic_fetch_xor
395
-
396
-
#else /* atomic_fetch_xor_relaxed */
397
-
398
-
#ifndef atomic_fetch_xor_acquire
399
-
#define atomic_fetch_xor_acquire(...) \
400
-
__atomic_op_acquire(atomic_fetch_xor, __VA_ARGS__)
401
-
#endif
402
-
403
-
#ifndef atomic_fetch_xor_release
404
-
#define atomic_fetch_xor_release(...) \
405
-
__atomic_op_release(atomic_fetch_xor, __VA_ARGS__)
406
-
#endif
407
-
408
-
#ifndef atomic_fetch_xor
409
-
#define atomic_fetch_xor(...) \
410
-
__atomic_op_fence(atomic_fetch_xor, __VA_ARGS__)
411
-
#endif
412
-
#endif /* atomic_fetch_xor_relaxed */
413
-
414
-
415
-
/* atomic_xchg_relaxed */
416
-
#ifndef atomic_xchg_relaxed
417
-
#define atomic_xchg_relaxed atomic_xchg
418
-
#define atomic_xchg_acquire atomic_xchg
419
-
#define atomic_xchg_release atomic_xchg
420
-
421
-
#else /* atomic_xchg_relaxed */
422
-
423
-
#ifndef atomic_xchg_acquire
424
-
#define atomic_xchg_acquire(...) \
425
-
__atomic_op_acquire(atomic_xchg, __VA_ARGS__)
426
-
#endif
427
-
428
-
#ifndef atomic_xchg_release
429
-
#define atomic_xchg_release(...) \
430
-
__atomic_op_release(atomic_xchg, __VA_ARGS__)
431
-
#endif
432
-
433
-
#ifndef atomic_xchg
434
-
#define atomic_xchg(...) \
435
-
__atomic_op_fence(atomic_xchg, __VA_ARGS__)
436
-
#endif
437
-
#endif /* atomic_xchg_relaxed */
438
-
439
-
/* atomic_cmpxchg_relaxed */
440
-
#ifndef atomic_cmpxchg_relaxed
441
-
#define atomic_cmpxchg_relaxed atomic_cmpxchg
442
-
#define atomic_cmpxchg_acquire atomic_cmpxchg
443
-
#define atomic_cmpxchg_release atomic_cmpxchg
444
-
445
-
#else /* atomic_cmpxchg_relaxed */
446
-
447
-
#ifndef atomic_cmpxchg_acquire
448
-
#define atomic_cmpxchg_acquire(...) \
449
-
__atomic_op_acquire(atomic_cmpxchg, __VA_ARGS__)
450
-
#endif
451
-
452
-
#ifndef atomic_cmpxchg_release
453
-
#define atomic_cmpxchg_release(...) \
454
-
__atomic_op_release(atomic_cmpxchg, __VA_ARGS__)
455
-
#endif
456
-
457
-
#ifndef atomic_cmpxchg
458
-
#define atomic_cmpxchg(...) \
459
-
__atomic_op_fence(atomic_cmpxchg, __VA_ARGS__)
460
-
#endif
461
-
#endif /* atomic_cmpxchg_relaxed */
462
-
463
-
#ifndef atomic_try_cmpxchg
464
-
465
-
#define __atomic_try_cmpxchg(type, _p, _po, _n) \
466
-
({ \
467
-
typeof(_po) __po = (_po); \
468
-
typeof(*(_po)) __r, __o = *__po; \
469
-
__r = atomic_cmpxchg##type((_p), __o, (_n)); \
470
-
if (unlikely(__r != __o)) \
471
-
*__po = __r; \
472
-
likely(__r == __o); \
473
-
})
474
-
475
-
#define atomic_try_cmpxchg(_p, _po, _n) __atomic_try_cmpxchg(, _p, _po, _n)
476
-
#define atomic_try_cmpxchg_relaxed(_p, _po, _n) __atomic_try_cmpxchg(_relaxed, _p, _po, _n)
477
-
#define atomic_try_cmpxchg_acquire(_p, _po, _n) __atomic_try_cmpxchg(_acquire, _p, _po, _n)
478
-
#define atomic_try_cmpxchg_release(_p, _po, _n) __atomic_try_cmpxchg(_release, _p, _po, _n)
479
-
480
-
#else /* atomic_try_cmpxchg */
481
-
#define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg
482
-
#define atomic_try_cmpxchg_acquire atomic_try_cmpxchg
483
-
#define atomic_try_cmpxchg_release atomic_try_cmpxchg
484
-
#endif /* atomic_try_cmpxchg */
485
-
486
-
/* cmpxchg_relaxed */
487
-
#ifndef cmpxchg_relaxed
488
-
#define cmpxchg_relaxed cmpxchg
489
-
#define cmpxchg_acquire cmpxchg
490
-
#define cmpxchg_release cmpxchg
491
-
492
-
#else /* cmpxchg_relaxed */
493
-
494
-
#ifndef cmpxchg_acquire
495
-
#define cmpxchg_acquire(...) \
496
-
__atomic_op_acquire(cmpxchg, __VA_ARGS__)
497
-
#endif
498
-
499
-
#ifndef cmpxchg_release
500
-
#define cmpxchg_release(...) \
501
-
__atomic_op_release(cmpxchg, __VA_ARGS__)
502
-
#endif
503
-
504
-
#ifndef cmpxchg
505
-
#define cmpxchg(...) \
506
-
__atomic_op_fence(cmpxchg, __VA_ARGS__)
507
-
#endif
508
-
#endif /* cmpxchg_relaxed */
509
-
510
-
/* cmpxchg64_relaxed */
511
-
#ifndef cmpxchg64_relaxed
512
-
#define cmpxchg64_relaxed cmpxchg64
513
-
#define cmpxchg64_acquire cmpxchg64
514
-
#define cmpxchg64_release cmpxchg64
515
-
516
-
#else /* cmpxchg64_relaxed */
517
-
518
-
#ifndef cmpxchg64_acquire
519
-
#define cmpxchg64_acquire(...) \
520
-
__atomic_op_acquire(cmpxchg64, __VA_ARGS__)
521
-
#endif
522
-
523
-
#ifndef cmpxchg64_release
524
-
#define cmpxchg64_release(...) \
525
-
__atomic_op_release(cmpxchg64, __VA_ARGS__)
526
-
#endif
527
-
528
-
#ifndef cmpxchg64
529
-
#define cmpxchg64(...) \
530
-
__atomic_op_fence(cmpxchg64, __VA_ARGS__)
531
-
#endif
532
-
#endif /* cmpxchg64_relaxed */
533
-
534
-
/* xchg_relaxed */
535
-
#ifndef xchg_relaxed
536
-
#define xchg_relaxed xchg
537
-
#define xchg_acquire xchg
538
-
#define xchg_release xchg
539
-
540
-
#else /* xchg_relaxed */
541
-
542
-
#ifndef xchg_acquire
543
-
#define xchg_acquire(...) __atomic_op_acquire(xchg, __VA_ARGS__)
544
-
#endif
545
-
546
-
#ifndef xchg_release
547
-
#define xchg_release(...) __atomic_op_release(xchg, __VA_ARGS__)
548
-
#endif
549
-
550
-
#ifndef xchg
551
-
#define xchg(...) __atomic_op_fence(xchg, __VA_ARGS__)
552
-
#endif
553
-
#endif /* xchg_relaxed */
554
-
555
-
/**
556
-
* atomic_fetch_add_unless - add unless the number is already a given value
557
-
* @v: pointer of type atomic_t
558
-
* @a: the amount to add to v...
559
-
* @u: ...unless v is equal to u.
560
-
*
561
-
* Atomically adds @a to @v, if @v was not already @u.
562
-
* Returns the original value of @v.
563
-
*/
564
-
#ifndef atomic_fetch_add_unless
565
-
static inline int atomic_fetch_add_unless(atomic_t *v, int a, int u)
566
-
{
567
-
int c = atomic_read(v);
568
-
569
-
do {
570
-
if (unlikely(c == u))
571
-
break;
572
-
} while (!atomic_try_cmpxchg(v, &c, c + a));
573
-
574
-
return c;
575
-
}
576
-
#endif
577
-
578
-
/**
579
-
* atomic_add_unless - add unless the number is already a given value
580
-
* @v: pointer of type atomic_t
581
-
* @a: the amount to add to v...
582
-
* @u: ...unless v is equal to u.
583
-
*
584
-
* Atomically adds @a to @v, if @v was not already @u.
585
-
* Returns true if the addition was done.
586
-
*/
587
-
static inline bool atomic_add_unless(atomic_t *v, int a, int u)
588
-
{
589
-
return atomic_fetch_add_unless(v, a, u) != u;
590
-
}
591
-
592
-
/**
593
-
* atomic_inc_not_zero - increment unless the number is zero
594
-
* @v: pointer of type atomic_t
595
-
*
596
-
* Atomically increments @v by 1, if @v is non-zero.
597
-
* Returns true if the increment was done.
598
-
*/
599
-
#ifndef atomic_inc_not_zero
600
-
#define atomic_inc_not_zero(v) atomic_add_unless((v), 1, 0)
601
-
#endif
602
-
603
-
/**
604
-
* atomic_inc_and_test - increment and test
605
-
* @v: pointer of type atomic_t
606
-
*
607
-
* Atomically increments @v by 1
608
-
* and returns true if the result is zero, or false for all
609
-
* other cases.
610
-
*/
611
-
#ifndef atomic_inc_and_test
612
-
static inline bool atomic_inc_and_test(atomic_t *v)
613
-
{
614
-
return atomic_inc_return(v) == 0;
615
-
}
616
-
#endif
617
-
618
-
/**
619
-
* atomic_dec_and_test - decrement and test
620
-
* @v: pointer of type atomic_t
621
-
*
622
-
* Atomically decrements @v by 1 and
623
-
* returns true if the result is 0, or false for all other
624
-
* cases.
625
-
*/
626
-
#ifndef atomic_dec_and_test
627
-
static inline bool atomic_dec_and_test(atomic_t *v)
628
-
{
629
-
return atomic_dec_return(v) == 0;
630
-
}
631
-
#endif
632
-
633
-
/**
634
-
* atomic_sub_and_test - subtract value from variable and test result
635
-
* @i: integer value to subtract
636
-
* @v: pointer of type atomic_t
637
-
*
638
-
* Atomically subtracts @i from @v and returns
639
-
* true if the result is zero, or false for all
640
-
* other cases.
641
-
*/
642
-
#ifndef atomic_sub_and_test
643
-
static inline bool atomic_sub_and_test(int i, atomic_t *v)
644
-
{
645
-
return atomic_sub_return(i, v) == 0;
646
-
}
647
-
#endif
648
-
649
-
/**
650
-
* atomic_add_negative - add and test if negative
651
-
* @i: integer value to add
652
-
* @v: pointer of type atomic_t
653
-
*
654
-
* Atomically adds @i to @v and returns true
655
-
* if the result is negative, or false when
656
-
* result is greater than or equal to zero.
657
-
*/
658
-
#ifndef atomic_add_negative
659
-
static inline bool atomic_add_negative(int i, atomic_t *v)
660
-
{
661
-
return atomic_add_return(i, v) < 0;
662
-
}
663
-
#endif
664
-
665
-
#ifndef atomic_inc_unless_negative
666
-
static inline bool atomic_inc_unless_negative(atomic_t *v)
667
-
{
668
-
int c = atomic_read(v);
669
-
670
-
do {
671
-
if (unlikely(c < 0))
672
-
return false;
673
-
} while (!atomic_try_cmpxchg(v, &c, c + 1));
674
-
675
-
return true;
676
-
}
677
-
#endif
678
-
679
-
#ifndef atomic_dec_unless_positive
680
-
static inline bool atomic_dec_unless_positive(atomic_t *v)
681
-
{
682
-
int c = atomic_read(v);
683
-
684
-
do {
685
-
if (unlikely(c > 0))
686
-
return false;
687
-
} while (!atomic_try_cmpxchg(v, &c, c - 1));
688
-
689
-
return true;
690
-
}
691
-
#endif
692
-
693
-
/*
694
-
* atomic_dec_if_positive - decrement by 1 if old value positive
695
-
* @v: pointer of type atomic_t
696
-
*
697
-
* The function returns the old value of *v minus 1, even if
698
-
* the atomic variable, v, was not decremented.
699
-
*/
700
-
#ifndef atomic_dec_if_positive
701
-
static inline int atomic_dec_if_positive(atomic_t *v)
702
-
{
703
-
int dec, c = atomic_read(v);
704
-
705
-
do {
706
-
dec = c - 1;
707
-
if (unlikely(dec < 0))
708
-
break;
709
-
} while (!atomic_try_cmpxchg(v, &c, dec));
710
-
711
-
return dec;
712
-
}
713
-
#endif
714
-
715
-
#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
716
-
#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
717
-
718
-
#ifdef CONFIG_GENERIC_ATOMIC64
719
-
#include <asm-generic/atomic64.h>
720
-
#endif
721
-
722
-
#ifndef atomic64_read_acquire
723
-
#define atomic64_read_acquire(v) smp_load_acquire(&(v)->counter)
724
-
#endif
725
-
726
-
#ifndef atomic64_set_release
727
-
#define atomic64_set_release(v, i) smp_store_release(&(v)->counter, (i))
728
-
#endif
729
-
730
-
/* atomic64_add_return_relaxed */
731
-
#ifndef atomic64_add_return_relaxed
732
-
#define atomic64_add_return_relaxed atomic64_add_return
733
-
#define atomic64_add_return_acquire atomic64_add_return
734
-
#define atomic64_add_return_release atomic64_add_return
735
-
736
-
#else /* atomic64_add_return_relaxed */
737
-
738
-
#ifndef atomic64_add_return_acquire
739
-
#define atomic64_add_return_acquire(...) \
740
-
__atomic_op_acquire(atomic64_add_return, __VA_ARGS__)
741
-
#endif
742
-
743
-
#ifndef atomic64_add_return_release
744
-
#define atomic64_add_return_release(...) \
745
-
__atomic_op_release(atomic64_add_return, __VA_ARGS__)
746
-
#endif
747
-
748
-
#ifndef atomic64_add_return
749
-
#define atomic64_add_return(...) \
750
-
__atomic_op_fence(atomic64_add_return, __VA_ARGS__)
751
-
#endif
752
-
#endif /* atomic64_add_return_relaxed */
753
-
754
-
#ifndef atomic64_inc
755
-
#define atomic64_inc(v) atomic64_add(1, (v))
756
-
#endif
757
-
758
-
/* atomic64_inc_return_relaxed */
759
-
#ifndef atomic64_inc_return_relaxed
760
-
761
-
#ifndef atomic64_inc_return
762
-
#define atomic64_inc_return(v) atomic64_add_return(1, (v))
763
-
#define atomic64_inc_return_relaxed(v) atomic64_add_return_relaxed(1, (v))
764
-
#define atomic64_inc_return_acquire(v) atomic64_add_return_acquire(1, (v))
765
-
#define atomic64_inc_return_release(v) atomic64_add_return_release(1, (v))
766
-
#else /* atomic64_inc_return */
767
-
#define atomic64_inc_return_relaxed atomic64_inc_return
768
-
#define atomic64_inc_return_acquire atomic64_inc_return
769
-
#define atomic64_inc_return_release atomic64_inc_return
770
-
#endif /* atomic64_inc_return */
771
-
772
-
#else /* atomic64_inc_return_relaxed */
773
-
774
-
#ifndef atomic64_inc_return_acquire
775
-
#define atomic64_inc_return_acquire(...) \
776
-
__atomic_op_acquire(atomic64_inc_return, __VA_ARGS__)
777
-
#endif
778
-
779
-
#ifndef atomic64_inc_return_release
780
-
#define atomic64_inc_return_release(...) \
781
-
__atomic_op_release(atomic64_inc_return, __VA_ARGS__)
782
-
#endif
783
-
784
-
#ifndef atomic64_inc_return
785
-
#define atomic64_inc_return(...) \
786
-
__atomic_op_fence(atomic64_inc_return, __VA_ARGS__)
787
-
#endif
788
-
#endif /* atomic64_inc_return_relaxed */
789
-
790
-
791
-
/* atomic64_sub_return_relaxed */
792
-
#ifndef atomic64_sub_return_relaxed
793
-
#define atomic64_sub_return_relaxed atomic64_sub_return
794
-
#define atomic64_sub_return_acquire atomic64_sub_return
795
-
#define atomic64_sub_return_release atomic64_sub_return
796
-
797
-
#else /* atomic64_sub_return_relaxed */
798
-
799
-
#ifndef atomic64_sub_return_acquire
800
-
#define atomic64_sub_return_acquire(...) \
801
-
__atomic_op_acquire(atomic64_sub_return, __VA_ARGS__)
802
-
#endif
803
-
804
-
#ifndef atomic64_sub_return_release
805
-
#define atomic64_sub_return_release(...) \
806
-
__atomic_op_release(atomic64_sub_return, __VA_ARGS__)
807
-
#endif
808
-
809
-
#ifndef atomic64_sub_return
810
-
#define atomic64_sub_return(...) \
811
-
__atomic_op_fence(atomic64_sub_return, __VA_ARGS__)
812
-
#endif
813
-
#endif /* atomic64_sub_return_relaxed */
814
-
815
-
#ifndef atomic64_dec
816
-
#define atomic64_dec(v) atomic64_sub(1, (v))
817
-
#endif
818
-
819
-
/* atomic64_dec_return_relaxed */
820
-
#ifndef atomic64_dec_return_relaxed
821
-
822
-
#ifndef atomic64_dec_return
823
-
#define atomic64_dec_return(v) atomic64_sub_return(1, (v))
824
-
#define atomic64_dec_return_relaxed(v) atomic64_sub_return_relaxed(1, (v))
825
-
#define atomic64_dec_return_acquire(v) atomic64_sub_return_acquire(1, (v))
826
-
#define atomic64_dec_return_release(v) atomic64_sub_return_release(1, (v))
827
-
#else /* atomic64_dec_return */
828
-
#define atomic64_dec_return_relaxed atomic64_dec_return
829
-
#define atomic64_dec_return_acquire atomic64_dec_return
830
-
#define atomic64_dec_return_release atomic64_dec_return
831
-
#endif /* atomic64_dec_return */
832
-
833
-
#else /* atomic64_dec_return_relaxed */
834
-
835
-
#ifndef atomic64_dec_return_acquire
836
-
#define atomic64_dec_return_acquire(...) \
837
-
__atomic_op_acquire(atomic64_dec_return, __VA_ARGS__)
838
-
#endif
839
-
840
-
#ifndef atomic64_dec_return_release
841
-
#define atomic64_dec_return_release(...) \
842
-
__atomic_op_release(atomic64_dec_return, __VA_ARGS__)
843
-
#endif
844
-
845
-
#ifndef atomic64_dec_return
846
-
#define atomic64_dec_return(...) \
847
-
__atomic_op_fence(atomic64_dec_return, __VA_ARGS__)
848
-
#endif
849
-
#endif /* atomic64_dec_return_relaxed */
850
-
851
-
852
-
/* atomic64_fetch_add_relaxed */
853
-
#ifndef atomic64_fetch_add_relaxed
854
-
#define atomic64_fetch_add_relaxed atomic64_fetch_add
855
-
#define atomic64_fetch_add_acquire atomic64_fetch_add
856
-
#define atomic64_fetch_add_release atomic64_fetch_add
857
-
858
-
#else /* atomic64_fetch_add_relaxed */
859
-
860
-
#ifndef atomic64_fetch_add_acquire
861
-
#define atomic64_fetch_add_acquire(...) \
862
-
__atomic_op_acquire(atomic64_fetch_add, __VA_ARGS__)
863
-
#endif
864
-
865
-
#ifndef atomic64_fetch_add_release
866
-
#define atomic64_fetch_add_release(...) \
867
-
__atomic_op_release(atomic64_fetch_add, __VA_ARGS__)
868
-
#endif
869
-
870
-
#ifndef atomic64_fetch_add
871
-
#define atomic64_fetch_add(...) \
872
-
__atomic_op_fence(atomic64_fetch_add, __VA_ARGS__)
873
-
#endif
874
-
#endif /* atomic64_fetch_add_relaxed */
875
-
876
-
/* atomic64_fetch_inc_relaxed */
877
-
#ifndef atomic64_fetch_inc_relaxed
878
-
879
-
#ifndef atomic64_fetch_inc
880
-
#define atomic64_fetch_inc(v) atomic64_fetch_add(1, (v))
881
-
#define atomic64_fetch_inc_relaxed(v) atomic64_fetch_add_relaxed(1, (v))
882
-
#define atomic64_fetch_inc_acquire(v) atomic64_fetch_add_acquire(1, (v))
883
-
#define atomic64_fetch_inc_release(v) atomic64_fetch_add_release(1, (v))
884
-
#else /* atomic64_fetch_inc */
885
-
#define atomic64_fetch_inc_relaxed atomic64_fetch_inc
886
-
#define atomic64_fetch_inc_acquire atomic64_fetch_inc
887
-
#define atomic64_fetch_inc_release atomic64_fetch_inc
888
-
#endif /* atomic64_fetch_inc */
889
-
890
-
#else /* atomic64_fetch_inc_relaxed */
891
-
892
-
#ifndef atomic64_fetch_inc_acquire
893
-
#define atomic64_fetch_inc_acquire(...) \
894
-
__atomic_op_acquire(atomic64_fetch_inc, __VA_ARGS__)
895
-
#endif
896
-
897
-
#ifndef atomic64_fetch_inc_release
898
-
#define atomic64_fetch_inc_release(...) \
899
-
__atomic_op_release(atomic64_fetch_inc, __VA_ARGS__)
900
-
#endif
901
-
902
-
#ifndef atomic64_fetch_inc
903
-
#define atomic64_fetch_inc(...) \
904
-
__atomic_op_fence(atomic64_fetch_inc, __VA_ARGS__)
905
-
#endif
906
-
#endif /* atomic64_fetch_inc_relaxed */
907
-
908
-
/* atomic64_fetch_sub_relaxed */
909
-
#ifndef atomic64_fetch_sub_relaxed
910
-
#define atomic64_fetch_sub_relaxed atomic64_fetch_sub
911
-
#define atomic64_fetch_sub_acquire atomic64_fetch_sub
912
-
#define atomic64_fetch_sub_release atomic64_fetch_sub
913
-
914
-
#else /* atomic64_fetch_sub_relaxed */
915
-
916
-
#ifndef atomic64_fetch_sub_acquire
917
-
#define atomic64_fetch_sub_acquire(...) \
918
-
__atomic_op_acquire(atomic64_fetch_sub, __VA_ARGS__)
919
-
#endif
920
-
921
-
#ifndef atomic64_fetch_sub_release
922
-
#define atomic64_fetch_sub_release(...) \
923
-
__atomic_op_release(atomic64_fetch_sub, __VA_ARGS__)
924
-
#endif
925
-
926
-
#ifndef atomic64_fetch_sub
927
-
#define atomic64_fetch_sub(...) \
928
-
__atomic_op_fence(atomic64_fetch_sub, __VA_ARGS__)
929
-
#endif
930
-
#endif /* atomic64_fetch_sub_relaxed */
931
-
932
-
/* atomic64_fetch_dec_relaxed */
933
-
#ifndef atomic64_fetch_dec_relaxed
934
-
935
-
#ifndef atomic64_fetch_dec
936
-
#define atomic64_fetch_dec(v) atomic64_fetch_sub(1, (v))
937
-
#define atomic64_fetch_dec_relaxed(v) atomic64_fetch_sub_relaxed(1, (v))
938
-
#define atomic64_fetch_dec_acquire(v) atomic64_fetch_sub_acquire(1, (v))
939
-
#define atomic64_fetch_dec_release(v) atomic64_fetch_sub_release(1, (v))
940
-
#else /* atomic64_fetch_dec */
941
-
#define atomic64_fetch_dec_relaxed atomic64_fetch_dec
942
-
#define atomic64_fetch_dec_acquire atomic64_fetch_dec
943
-
#define atomic64_fetch_dec_release atomic64_fetch_dec
944
-
#endif /* atomic64_fetch_dec */
945
-
946
-
#else /* atomic64_fetch_dec_relaxed */
947
-
948
-
#ifndef atomic64_fetch_dec_acquire
949
-
#define atomic64_fetch_dec_acquire(...) \
950
-
__atomic_op_acquire(atomic64_fetch_dec, __VA_ARGS__)
951
-
#endif
952
-
953
-
#ifndef atomic64_fetch_dec_release
954
-
#define atomic64_fetch_dec_release(...) \
955
-
__atomic_op_release(atomic64_fetch_dec, __VA_ARGS__)
956
-
#endif
957
-
958
-
#ifndef atomic64_fetch_dec
959
-
#define atomic64_fetch_dec(...) \
960
-
__atomic_op_fence(atomic64_fetch_dec, __VA_ARGS__)
961
-
#endif
962
-
#endif /* atomic64_fetch_dec_relaxed */
963
-
964
-
/* atomic64_fetch_or_relaxed */
965
-
#ifndef atomic64_fetch_or_relaxed
966
-
#define atomic64_fetch_or_relaxed atomic64_fetch_or
967
-
#define atomic64_fetch_or_acquire atomic64_fetch_or
968
-
#define atomic64_fetch_or_release atomic64_fetch_or
969
-
970
-
#else /* atomic64_fetch_or_relaxed */
971
-
972
-
#ifndef atomic64_fetch_or_acquire
973
-
#define atomic64_fetch_or_acquire(...) \
974
-
__atomic_op_acquire(atomic64_fetch_or, __VA_ARGS__)
975
-
#endif
976
-
977
-
#ifndef atomic64_fetch_or_release
978
-
#define atomic64_fetch_or_release(...) \
979
-
__atomic_op_release(atomic64_fetch_or, __VA_ARGS__)
980
-
#endif
981
-
982
-
#ifndef atomic64_fetch_or
983
-
#define atomic64_fetch_or(...) \
984
-
__atomic_op_fence(atomic64_fetch_or, __VA_ARGS__)
985
-
#endif
986
-
#endif /* atomic64_fetch_or_relaxed */
987
-
988
-
/* atomic64_fetch_and_relaxed */
989
-
#ifndef atomic64_fetch_and_relaxed
990
-
#define atomic64_fetch_and_relaxed atomic64_fetch_and
991
-
#define atomic64_fetch_and_acquire atomic64_fetch_and
992
-
#define atomic64_fetch_and_release atomic64_fetch_and
993
-
994
-
#else /* atomic64_fetch_and_relaxed */
995
-
996
-
#ifndef atomic64_fetch_and_acquire
997
-
#define atomic64_fetch_and_acquire(...) \
998
-
__atomic_op_acquire(atomic64_fetch_and, __VA_ARGS__)
999
-
#endif
1000
-
1001
-
#ifndef atomic64_fetch_and_release
1002
-
#define atomic64_fetch_and_release(...) \
1003
-
__atomic_op_release(atomic64_fetch_and, __VA_ARGS__)
1004
-
#endif
1005
-
1006
-
#ifndef atomic64_fetch_and
1007
-
#define atomic64_fetch_and(...) \
1008
-
__atomic_op_fence(atomic64_fetch_and, __VA_ARGS__)
1009
-
#endif
1010
-
#endif /* atomic64_fetch_and_relaxed */
1011
-
1012
-
#ifndef atomic64_andnot
1013
-
#define atomic64_andnot(i, v) atomic64_and(~(long long)(i), (v))
1014
-
#endif
1015
-
1016
-
#ifndef atomic64_fetch_andnot_relaxed
1017
-
1018
-
#ifndef atomic64_fetch_andnot
1019
-
#define atomic64_fetch_andnot(i, v) atomic64_fetch_and(~(long long)(i), (v))
1020
-
#define atomic64_fetch_andnot_relaxed(i, v) atomic64_fetch_and_relaxed(~(long long)(i), (v))
1021
-
#define atomic64_fetch_andnot_acquire(i, v) atomic64_fetch_and_acquire(~(long long)(i), (v))
1022
-
#define atomic64_fetch_andnot_release(i, v) atomic64_fetch_and_release(~(long long)(i), (v))
1023
-
#else /* atomic64_fetch_andnot */
1024
-
#define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot
1025
-
#define atomic64_fetch_andnot_acquire atomic64_fetch_andnot
1026
-
#define atomic64_fetch_andnot_release atomic64_fetch_andnot
1027
-
#endif /* atomic64_fetch_andnot */
1028
-
1029
-
#else /* atomic64_fetch_andnot_relaxed */
1030
-
1031
-
#ifndef atomic64_fetch_andnot_acquire
1032
-
#define atomic64_fetch_andnot_acquire(...) \
1033
-
__atomic_op_acquire(atomic64_fetch_andnot, __VA_ARGS__)
1034
-
#endif
1035
-
1036
-
#ifndef atomic64_fetch_andnot_release
1037
-
#define atomic64_fetch_andnot_release(...) \
1038
-
__atomic_op_release(atomic64_fetch_andnot, __VA_ARGS__)
1039
-
#endif
1040
-
1041
-
#ifndef atomic64_fetch_andnot
1042
-
#define atomic64_fetch_andnot(...) \
1043
-
__atomic_op_fence(atomic64_fetch_andnot, __VA_ARGS__)
1044
-
#endif
1045
-
#endif /* atomic64_fetch_andnot_relaxed */
1046
-
1047
-
/* atomic64_fetch_xor_relaxed */
1048
-
#ifndef atomic64_fetch_xor_relaxed
1049
-
#define atomic64_fetch_xor_relaxed atomic64_fetch_xor
1050
-
#define atomic64_fetch_xor_acquire atomic64_fetch_xor
1051
-
#define atomic64_fetch_xor_release atomic64_fetch_xor
1052
-
1053
-
#else /* atomic64_fetch_xor_relaxed */
1054
-
1055
-
#ifndef atomic64_fetch_xor_acquire
1056
-
#define atomic64_fetch_xor_acquire(...) \
1057
-
__atomic_op_acquire(atomic64_fetch_xor, __VA_ARGS__)
1058
-
#endif
1059
-
1060
-
#ifndef atomic64_fetch_xor_release
1061
-
#define atomic64_fetch_xor_release(...) \
1062
-
__atomic_op_release(atomic64_fetch_xor, __VA_ARGS__)
1063
-
#endif
1064
-
1065
-
#ifndef atomic64_fetch_xor
1066
-
#define atomic64_fetch_xor(...) \
1067
-
__atomic_op_fence(atomic64_fetch_xor, __VA_ARGS__)
1068
-
#endif
1069
-
#endif /* atomic64_fetch_xor_relaxed */
1070
-
1071
-
1072
-
/* atomic64_xchg_relaxed */
1073
-
#ifndef atomic64_xchg_relaxed
1074
-
#define atomic64_xchg_relaxed atomic64_xchg
1075
-
#define atomic64_xchg_acquire atomic64_xchg
1076
-
#define atomic64_xchg_release atomic64_xchg
1077
-
1078
-
#else /* atomic64_xchg_relaxed */
1079
-
1080
-
#ifndef atomic64_xchg_acquire
1081
-
#define atomic64_xchg_acquire(...) \
1082
-
__atomic_op_acquire(atomic64_xchg, __VA_ARGS__)
1083
-
#endif
1084
-
1085
-
#ifndef atomic64_xchg_release
1086
-
#define atomic64_xchg_release(...) \
1087
-
__atomic_op_release(atomic64_xchg, __VA_ARGS__)
1088
-
#endif
1089
-
1090
-
#ifndef atomic64_xchg
1091
-
#define atomic64_xchg(...) \
1092
-
__atomic_op_fence(atomic64_xchg, __VA_ARGS__)
1093
-
#endif
1094
-
#endif /* atomic64_xchg_relaxed */
1095
-
1096
-
/* atomic64_cmpxchg_relaxed */
1097
-
#ifndef atomic64_cmpxchg_relaxed
1098
-
#define atomic64_cmpxchg_relaxed atomic64_cmpxchg
1099
-
#define atomic64_cmpxchg_acquire atomic64_cmpxchg
1100
-
#define atomic64_cmpxchg_release atomic64_cmpxchg
1101
-
1102
-
#else /* atomic64_cmpxchg_relaxed */
1103
-
1104
-
#ifndef atomic64_cmpxchg_acquire
1105
-
#define atomic64_cmpxchg_acquire(...) \
1106
-
__atomic_op_acquire(atomic64_cmpxchg, __VA_ARGS__)
1107
-
#endif
1108
-
1109
-
#ifndef atomic64_cmpxchg_release
1110
-
#define atomic64_cmpxchg_release(...) \
1111
-
__atomic_op_release(atomic64_cmpxchg, __VA_ARGS__)
1112
-
#endif
1113
-
1114
-
#ifndef atomic64_cmpxchg
1115
-
#define atomic64_cmpxchg(...) \
1116
-
__atomic_op_fence(atomic64_cmpxchg, __VA_ARGS__)
1117
-
#endif
1118
-
#endif /* atomic64_cmpxchg_relaxed */
1119
-
1120
-
#ifndef atomic64_try_cmpxchg
1121
-
1122
-
#define __atomic64_try_cmpxchg(type, _p, _po, _n) \
1123
-
({ \
1124
-
typeof(_po) __po = (_po); \
1125
-
typeof(*(_po)) __r, __o = *__po; \
1126
-
__r = atomic64_cmpxchg##type((_p), __o, (_n)); \
1127
-
if (unlikely(__r != __o)) \
1128
-
*__po = __r; \
1129
-
likely(__r == __o); \
1130
-
})
1131
-
1132
-
#define atomic64_try_cmpxchg(_p, _po, _n) __atomic64_try_cmpxchg(, _p, _po, _n)
1133
-
#define atomic64_try_cmpxchg_relaxed(_p, _po, _n) __atomic64_try_cmpxchg(_relaxed, _p, _po, _n)
1134
-
#define atomic64_try_cmpxchg_acquire(_p, _po, _n) __atomic64_try_cmpxchg(_acquire, _p, _po, _n)
1135
-
#define atomic64_try_cmpxchg_release(_p, _po, _n) __atomic64_try_cmpxchg(_release, _p, _po, _n)
1136
-
1137
-
#else /* atomic64_try_cmpxchg */
1138
-
#define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg
1139
-
#define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg
1140
-
#define atomic64_try_cmpxchg_release atomic64_try_cmpxchg
1141
-
#endif /* atomic64_try_cmpxchg */
1142
-
1143
-
/**
1144
-
* atomic64_fetch_add_unless - add unless the number is already a given value
1145
-
* @v: pointer of type atomic64_t
1146
-
* @a: the amount to add to v...
1147
-
* @u: ...unless v is equal to u.
1148
-
*
1149
-
* Atomically adds @a to @v, if @v was not already @u.
1150
-
* Returns the original value of @v.
1151
-
*/
1152
-
#ifndef atomic64_fetch_add_unless
1153
-
static inline long long atomic64_fetch_add_unless(atomic64_t *v, long long a,
1154
-
long long u)
1155
-
{
1156
-
long long c = atomic64_read(v);
1157
-
1158
-
do {
1159
-
if (unlikely(c == u))
1160
-
break;
1161
-
} while (!atomic64_try_cmpxchg(v, &c, c + a));
1162
-
1163
-
return c;
1164
-
}
1165
-
#endif
1166
-
1167
-
/**
1168
-
* atomic64_add_unless - add unless the number is already a given value
1169
-
* @v: pointer of type atomic_t
1170
-
* @a: the amount to add to v...
1171
-
* @u: ...unless v is equal to u.
1172
-
*
1173
-
* Atomically adds @a to @v, if @v was not already @u.
1174
-
* Returns true if the addition was done.
1175
-
*/
1176
-
static inline bool atomic64_add_unless(atomic64_t *v, long long a, long long u)
1177
-
{
1178
-
return atomic64_fetch_add_unless(v, a, u) != u;
1179
-
}
1180
-
1181
-
/**
1182
-
* atomic64_inc_not_zero - increment unless the number is zero
1183
-
* @v: pointer of type atomic64_t
1184
-
*
1185
-
* Atomically increments @v by 1, if @v is non-zero.
1186
-
* Returns true if the increment was done.
1187
-
*/
1188
-
#ifndef atomic64_inc_not_zero
1189
-
#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
1190
-
#endif
1191
-
1192
-
/**
1193
-
* atomic64_inc_and_test - increment and test
1194
-
* @v: pointer of type atomic64_t
1195
-
*
1196
-
* Atomically increments @v by 1
1197
-
* and returns true if the result is zero, or false for all
1198
-
* other cases.
1199
-
*/
1200
-
#ifndef atomic64_inc_and_test
1201
-
static inline bool atomic64_inc_and_test(atomic64_t *v)
1202
-
{
1203
-
return atomic64_inc_return(v) == 0;
1204
-
}
1205
-
#endif
1206
-
1207
-
/**
1208
-
* atomic64_dec_and_test - decrement and test
1209
-
* @v: pointer of type atomic64_t
1210
-
*
1211
-
* Atomically decrements @v by 1 and
1212
-
* returns true if the result is 0, or false for all other
1213
-
* cases.
1214
-
*/
1215
-
#ifndef atomic64_dec_and_test
1216
-
static inline bool atomic64_dec_and_test(atomic64_t *v)
1217
-
{
1218
-
return atomic64_dec_return(v) == 0;
1219
-
}
1220
-
#endif
1221
-
1222
-
/**
1223
-
* atomic64_sub_and_test - subtract value from variable and test result
1224
-
* @i: integer value to subtract
1225
-
* @v: pointer of type atomic64_t
1226
-
*
1227
-
* Atomically subtracts @i from @v and returns
1228
-
* true if the result is zero, or false for all
1229
-
* other cases.
1230
-
*/
1231
-
#ifndef atomic64_sub_and_test
1232
-
static inline bool atomic64_sub_and_test(long long i, atomic64_t *v)
1233
-
{
1234
-
return atomic64_sub_return(i, v) == 0;
1235
-
}
1236
-
#endif
1237
-
1238
-
/**
1239
-
* atomic64_add_negative - add and test if negative
1240
-
* @i: integer value to add
1241
-
* @v: pointer of type atomic64_t
1242
-
*
1243
-
* Atomically adds @i to @v and returns true
1244
-
* if the result is negative, or false when
1245
-
* result is greater than or equal to zero.
1246
-
*/
1247
-
#ifndef atomic64_add_negative
1248
-
static inline bool atomic64_add_negative(long long i, atomic64_t *v)
1249
-
{
1250
-
return atomic64_add_return(i, v) < 0;
1251
-
}
1252
-
#endif
1253
-
1254
-
#ifndef atomic64_inc_unless_negative
1255
-
static inline bool atomic64_inc_unless_negative(atomic64_t *v)
1256
-
{
1257
-
long long c = atomic64_read(v);
1258
-
1259
-
do {
1260
-
if (unlikely(c < 0))
1261
-
return false;
1262
-
} while (!atomic64_try_cmpxchg(v, &c, c + 1));
1263
-
1264
-
return true;
1265
-
}
1266
-
#endif
1267
-
1268
-
#ifndef atomic64_dec_unless_positive
1269
-
static inline bool atomic64_dec_unless_positive(atomic64_t *v)
1270
-
{
1271
-
long long c = atomic64_read(v);
1272
-
1273
-
do {
1274
-
if (unlikely(c > 0))
1275
-
return false;
1276
-
} while (!atomic64_try_cmpxchg(v, &c, c - 1));
1277
-
1278
-
return true;
1279
-
}
1280
-
#endif
1281
-
1282
-
/*
1283
-
* atomic64_dec_if_positive - decrement by 1 if old value positive
1284
-
* @v: pointer of type atomic64_t
1285
-
*
1286
-
* The function returns the old value of *v minus 1, even if
1287
-
* the atomic64 variable, v, was not decremented.
1288
-
*/
1289
-
#ifndef atomic64_dec_if_positive
1290
-
static inline long long atomic64_dec_if_positive(atomic64_t *v)
1291
-
{
1292
-
long long dec, c = atomic64_read(v);
1293
-
1294
-
do {
1295
-
dec = c - 1;
1296
-
if (unlikely(dec < 0))
1297
-
break;
1298
-
} while (!atomic64_try_cmpxchg(v, &c, dec));
1299
-
1300
-
return dec;
1301
-
}
1302
-
#endif
1303
-
1304
-
#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
1305
-
#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
82
+
#include <linux/atomic-fallback.h>
1306
83
1307
84
#include <asm-generic/atomic-long.h>
1308
85
+186
scripts/atomic/atomic-tbl.sh
+186
scripts/atomic/atomic-tbl.sh
···
1
+
#!/bin/sh
2
+
# SPDX-License-Identifier: GPL-2.0
3
+
# helpers for dealing with atomics.tbl
4
+
5
+
#meta_in(meta, match)
6
+
meta_in()
7
+
{
8
+
case "$1" in
9
+
[$2]) return 0;;
10
+
esac
11
+
12
+
return 1
13
+
}
14
+
15
+
#meta_has_ret(meta)
16
+
meta_has_ret()
17
+
{
18
+
meta_in "$1" "bBiIfFlR"
19
+
}
20
+
21
+
#meta_has_acquire(meta)
22
+
meta_has_acquire()
23
+
{
24
+
meta_in "$1" "BFIlR"
25
+
}
26
+
27
+
#meta_has_release(meta)
28
+
meta_has_release()
29
+
{
30
+
meta_in "$1" "BFIRs"
31
+
}
32
+
33
+
#meta_has_relaxed(meta)
34
+
meta_has_relaxed()
35
+
{
36
+
meta_in "$1" "BFIR"
37
+
}
38
+
39
+
#find_fallback_template(pfx, name, sfx, order)
40
+
find_fallback_template()
41
+
{
42
+
local pfx="$1"; shift
43
+
local name="$1"; shift
44
+
local sfx="$1"; shift
45
+
local order="$1"; shift
46
+
47
+
local base=""
48
+
local file=""
49
+
50
+
# We may have fallbacks for a specific case (e.g. read_acquire()), or
51
+
# an entire class, e.g. *inc*().
52
+
#
53
+
# Start at the most specific, and fall back to the most general. Once
54
+
# we find a specific fallback, don't bother looking for more.
55
+
for base in "${pfx}${name}${sfx}${order}" "${name}"; do
56
+
file="${ATOMICDIR}/fallbacks/${base}"
57
+
58
+
if [ -f "${file}" ]; then
59
+
printf "${file}"
60
+
break
61
+
fi
62
+
done
63
+
}
64
+
65
+
#gen_ret_type(meta, int)
66
+
gen_ret_type() {
67
+
local meta="$1"; shift
68
+
local int="$1"; shift
69
+
70
+
case "${meta}" in
71
+
[sv]) printf "void";;
72
+
[bB]) printf "bool";;
73
+
[aiIfFlR]) printf "${int}";;
74
+
esac
75
+
}
76
+
77
+
#gen_ret_stmt(meta)
78
+
gen_ret_stmt()
79
+
{
80
+
if meta_has_ret "${meta}"; then
81
+
printf "return ";
82
+
fi
83
+
}
84
+
85
+
# gen_param_name(arg)
86
+
gen_param_name()
87
+
{
88
+
# strip off the leading 'c' for 'cv'
89
+
local name="${1#c}"
90
+
printf "${name#*:}"
91
+
}
92
+
93
+
# gen_param_type(arg, int, atomic)
94
+
gen_param_type()
95
+
{
96
+
local type="${1%%:*}"; shift
97
+
local int="$1"; shift
98
+
local atomic="$1"; shift
99
+
100
+
case "${type}" in
101
+
i) type="${int} ";;
102
+
p) type="${int} *";;
103
+
v) type="${atomic}_t *";;
104
+
cv) type="const ${atomic}_t *";;
105
+
esac
106
+
107
+
printf "${type}"
108
+
}
109
+
110
+
#gen_param(arg, int, atomic)
111
+
gen_param()
112
+
{
113
+
local arg="$1"; shift
114
+
local int="$1"; shift
115
+
local atomic="$1"; shift
116
+
local name="$(gen_param_name "${arg}")"
117
+
local type="$(gen_param_type "${arg}" "${int}" "${atomic}")"
118
+
119
+
printf "${type}${name}"
120
+
}
121
+
122
+
#gen_params(int, atomic, arg...)
123
+
gen_params()
124
+
{
125
+
local int="$1"; shift
126
+
local atomic="$1"; shift
127
+
128
+
while [ "$#" -gt 0 ]; do
129
+
gen_param "$1" "${int}" "${atomic}"
130
+
[ "$#" -gt 1 ] && printf ", "
131
+
shift;
132
+
done
133
+
}
134
+
135
+
#gen_args(arg...)
136
+
gen_args()
137
+
{
138
+
while [ "$#" -gt 0 ]; do
139
+
printf "$(gen_param_name "$1")"
140
+
[ "$#" -gt 1 ] && printf ", "
141
+
shift;
142
+
done
143
+
}
144
+
145
+
#gen_proto_order_variants(meta, pfx, name, sfx, ...)
146
+
gen_proto_order_variants()
147
+
{
148
+
local meta="$1"; shift
149
+
local pfx="$1"; shift
150
+
local name="$1"; shift
151
+
local sfx="$1"; shift
152
+
153
+
gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
154
+
155
+
if meta_has_acquire "${meta}"; then
156
+
gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
157
+
fi
158
+
if meta_has_release "${meta}"; then
159
+
gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
160
+
fi
161
+
if meta_has_relaxed "${meta}"; then
162
+
gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"
163
+
fi
164
+
}
165
+
166
+
#gen_proto_variants(meta, name, ...)
167
+
gen_proto_variants()
168
+
{
169
+
local meta="$1"; shift
170
+
local name="$1"; shift
171
+
local pfx=""
172
+
local sfx=""
173
+
174
+
meta_in "${meta}" "fF" && pfx="fetch_"
175
+
meta_in "${meta}" "R" && sfx="_return"
176
+
177
+
gen_proto_order_variants "${meta}" "${pfx}" "${name}" "${sfx}" "$@"
178
+
}
179
+
180
+
#gen_proto(meta, ...)
181
+
gen_proto() {
182
+
local meta="$1"; shift
183
+
for m in $(echo "${meta}" | fold -w1); do
184
+
gen_proto_variants "${m}" "$@"
185
+
done
186
+
}
+41
scripts/atomic/atomics.tbl
+41
scripts/atomic/atomics.tbl
···
1
+
# name meta args...
2
+
#
3
+
# Where meta contains a string of variants to generate.
4
+
# Upper-case implies _{acquire,release,relaxed} variants.
5
+
# Valid meta values are:
6
+
# * B/b - bool: returns bool
7
+
# * v - void: returns void
8
+
# * I/i - int: returns base type
9
+
# * R - return: returns base type (has _return variants)
10
+
# * F/f - fetch: returns base type (has fetch_ variants)
11
+
# * l - load: returns base type (has _acquire order variant)
12
+
# * s - store: returns void (has _release order variant)
13
+
#
14
+
# Where args contains list of type[:name], where type is:
15
+
# * cv - const pointer to atomic base type (atomic_t/atomic64_t/atomic_long_t)
16
+
# * v - pointer to atomic base type (atomic_t/atomic64_t/atomic_long_t)
17
+
# * i - base type (int/s64/long)
18
+
# * p - pointer to base type (int/s64/long)
19
+
#
20
+
read l cv
21
+
set s v i
22
+
add vRF i v
23
+
sub vRF i v
24
+
inc vRF v
25
+
dec vRF v
26
+
and vF i v
27
+
andnot vF i v
28
+
or vF i v
29
+
xor vF i v
30
+
xchg I v i
31
+
cmpxchg I v i:old i:new
32
+
try_cmpxchg B v p:old i:new
33
+
sub_and_test b i v
34
+
dec_and_test b v
35
+
inc_and_test b v
36
+
add_negative b i v
37
+
add_unless fb v i:a i:u
38
+
inc_not_zero b v
39
+
inc_unless_negative b v
40
+
dec_unless_positive b v
41
+
dec_if_positive i v
+19
scripts/atomic/check-atomics.sh
+19
scripts/atomic/check-atomics.sh
···
1
+
#!/bin/sh
2
+
# SPDX-License-Identifier: GPL-2.0
3
+
#
4
+
# Check if atomic headers are up-to-date
5
+
6
+
ATOMICDIR=$(dirname $0)
7
+
ATOMICTBL=${ATOMICDIR}/atomics.tbl
8
+
LINUXDIR=${ATOMICDIR}/../..
9
+
10
+
cat <<EOF |
11
+
gen-atomic-instrumented.sh asm-generic/atomic-instrumented.h
12
+
gen-atomic-long.sh asm-generic/atomic-long.h
13
+
gen-atomic-fallback.sh linux/atomic-fallback.h
14
+
EOF
15
+
while read script header; do
16
+
if ! (${ATOMICDIR}/${script} ${ATOMICTBL} | diff - ${LINUXDIR}/include/${header} > /dev/null); then
17
+
printf "warning: include/${header} is out-of-date.\n"
18
+
fi
19
+
done
+9
scripts/atomic/fallbacks/acquire
+9
scripts/atomic/fallbacks/acquire
+16
scripts/atomic/fallbacks/add_negative
+16
scripts/atomic/fallbacks/add_negative
···
1
+
cat <<EOF
2
+
/**
3
+
* ${atomic}_add_negative - add and test if negative
4
+
* @i: integer value to add
5
+
* @v: pointer of type ${atomic}_t
6
+
*
7
+
* Atomically adds @i to @v and returns true
8
+
* if the result is negative, or false when
9
+
* result is greater than or equal to zero.
10
+
*/
11
+
static inline bool
12
+
${atomic}_add_negative(${int} i, ${atomic}_t *v)
13
+
{
14
+
return ${atomic}_add_return(i, v) < 0;
15
+
}
16
+
EOF
+16
scripts/atomic/fallbacks/add_unless
+16
scripts/atomic/fallbacks/add_unless
···
1
+
cat << EOF
2
+
/**
3
+
* ${atomic}_add_unless - add unless the number is already a given value
4
+
* @v: pointer of type ${atomic}_t
5
+
* @a: the amount to add to v...
6
+
* @u: ...unless v is equal to u.
7
+
*
8
+
* Atomically adds @a to @v, if @v was not already @u.
9
+
* Returns true if the addition was done.
10
+
*/
11
+
static inline bool
12
+
${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u)
13
+
{
14
+
return ${atomic}_fetch_add_unless(v, a, u) != u;
15
+
}
16
+
EOF
+7
scripts/atomic/fallbacks/andnot
+7
scripts/atomic/fallbacks/andnot
+7
scripts/atomic/fallbacks/dec
+7
scripts/atomic/fallbacks/dec
+15
scripts/atomic/fallbacks/dec_and_test
+15
scripts/atomic/fallbacks/dec_and_test
···
1
+
cat <<EOF
2
+
/**
3
+
* ${atomic}_dec_and_test - decrement and test
4
+
* @v: pointer of type ${atomic}_t
5
+
*
6
+
* Atomically decrements @v by 1 and
7
+
* returns true if the result is 0, or false for all other
8
+
* cases.
9
+
*/
10
+
static inline bool
11
+
${atomic}_dec_and_test(${atomic}_t *v)
12
+
{
13
+
return ${atomic}_dec_return(v) == 0;
14
+
}
15
+
EOF
+15
scripts/atomic/fallbacks/dec_if_positive
+15
scripts/atomic/fallbacks/dec_if_positive
+14
scripts/atomic/fallbacks/dec_unless_positive
+14
scripts/atomic/fallbacks/dec_unless_positive
+11
scripts/atomic/fallbacks/fence
+11
scripts/atomic/fallbacks/fence
+23
scripts/atomic/fallbacks/fetch_add_unless
+23
scripts/atomic/fallbacks/fetch_add_unless
···
1
+
cat << EOF
2
+
/**
3
+
* ${atomic}_fetch_add_unless - add unless the number is already a given value
4
+
* @v: pointer of type ${atomic}_t
5
+
* @a: the amount to add to v...
6
+
* @u: ...unless v is equal to u.
7
+
*
8
+
* Atomically adds @a to @v, so long as @v was not already @u.
9
+
* Returns original value of @v
10
+
*/
11
+
static inline ${int}
12
+
${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u)
13
+
{
14
+
${int} c = ${atomic}_read(v);
15
+
16
+
do {
17
+
if (unlikely(c == u))
18
+
break;
19
+
} while (!${atomic}_try_cmpxchg(v, &c, c + a));
20
+
21
+
return c;
22
+
}
23
+
EOF
+7
scripts/atomic/fallbacks/inc
+7
scripts/atomic/fallbacks/inc
+15
scripts/atomic/fallbacks/inc_and_test
+15
scripts/atomic/fallbacks/inc_and_test
···
1
+
cat <<EOF
2
+
/**
3
+
* ${atomic}_inc_and_test - increment and test
4
+
* @v: pointer of type ${atomic}_t
5
+
*
6
+
* Atomically increments @v by 1
7
+
* and returns true if the result is zero, or false for all
8
+
* other cases.
9
+
*/
10
+
static inline bool
11
+
${atomic}_inc_and_test(${atomic}_t *v)
12
+
{
13
+
return ${atomic}_inc_return(v) == 0;
14
+
}
15
+
EOF
+14
scripts/atomic/fallbacks/inc_not_zero
+14
scripts/atomic/fallbacks/inc_not_zero
···
1
+
cat <<EOF
2
+
/**
3
+
* ${atomic}_inc_not_zero - increment unless the number is zero
4
+
* @v: pointer of type ${atomic}_t
5
+
*
6
+
* Atomically increments @v by 1, if @v is non-zero.
7
+
* Returns true if the increment was done.
8
+
*/
9
+
static inline bool
10
+
${atomic}_inc_not_zero(${atomic}_t *v)
11
+
{
12
+
return ${atomic}_add_unless(v, 1, 0);
13
+
}
14
+
EOF
+14
scripts/atomic/fallbacks/inc_unless_negative
+14
scripts/atomic/fallbacks/inc_unless_negative
+7
scripts/atomic/fallbacks/read_acquire
+7
scripts/atomic/fallbacks/read_acquire
+8
scripts/atomic/fallbacks/release
+8
scripts/atomic/fallbacks/release
+7
scripts/atomic/fallbacks/set_release
+7
scripts/atomic/fallbacks/set_release
+16
scripts/atomic/fallbacks/sub_and_test
+16
scripts/atomic/fallbacks/sub_and_test
···
1
+
cat <<EOF
2
+
/**
3
+
* ${atomic}_sub_and_test - subtract value from variable and test result
4
+
* @i: integer value to subtract
5
+
* @v: pointer of type ${atomic}_t
6
+
*
7
+
* Atomically subtracts @i from @v and returns
8
+
* true if the result is zero, or false for all
9
+
* other cases.
10
+
*/
11
+
static inline bool
12
+
${atomic}_sub_and_test(${int} i, ${atomic}_t *v)
13
+
{
14
+
return ${atomic}_sub_return(i, v) == 0;
15
+
}
16
+
EOF
+11
scripts/atomic/fallbacks/try_cmpxchg
+11
scripts/atomic/fallbacks/try_cmpxchg
+181
scripts/atomic/gen-atomic-fallback.sh
+181
scripts/atomic/gen-atomic-fallback.sh
···
1
+
#!/bin/sh
2
+
# SPDX-License-Identifier: GPL-2.0
3
+
4
+
ATOMICDIR=$(dirname $0)
5
+
6
+
. ${ATOMICDIR}/atomic-tbl.sh
7
+
8
+
#gen_template_fallback(template, meta, pfx, name, sfx, order, atomic, int, args...)
9
+
gen_template_fallback()
10
+
{
11
+
local template="$1"; shift
12
+
local meta="$1"; shift
13
+
local pfx="$1"; shift
14
+
local name="$1"; shift
15
+
local sfx="$1"; shift
16
+
local order="$1"; shift
17
+
local atomic="$1"; shift
18
+
local int="$1"; shift
19
+
20
+
local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
21
+
22
+
local ret="$(gen_ret_type "${meta}" "${int}")"
23
+
local retstmt="$(gen_ret_stmt "${meta}")"
24
+
local params="$(gen_params "${int}" "${atomic}" "$@")"
25
+
local args="$(gen_args "$@")"
26
+
27
+
if [ ! -z "${template}" ]; then
28
+
printf "#ifndef ${atomicname}\n"
29
+
. ${template}
30
+
printf "#define ${atomicname} ${atomicname}\n"
31
+
printf "#endif\n\n"
32
+
fi
33
+
}
34
+
35
+
#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
36
+
gen_proto_fallback()
37
+
{
38
+
local meta="$1"; shift
39
+
local pfx="$1"; shift
40
+
local name="$1"; shift
41
+
local sfx="$1"; shift
42
+
local order="$1"; shift
43
+
44
+
local tmpl="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
45
+
gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
46
+
}
47
+
48
+
#gen_basic_fallbacks(basename)
49
+
gen_basic_fallbacks()
50
+
{
51
+
local basename="$1"; shift
52
+
cat << EOF
53
+
#define ${basename}_acquire ${basename}
54
+
#define ${basename}_release ${basename}
55
+
#define ${basename}_relaxed ${basename}
56
+
EOF
57
+
}
58
+
59
+
#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)
60
+
gen_proto_order_variants()
61
+
{
62
+
local meta="$1"; shift
63
+
local pfx="$1"; shift
64
+
local name="$1"; shift
65
+
local sfx="$1"; shift
66
+
local atomic="$1"
67
+
68
+
local basename="${atomic}_${pfx}${name}${sfx}"
69
+
70
+
local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
71
+
72
+
# If we don't have relaxed atomics, then we don't bother with ordering fallbacks
73
+
# read_acquire and set_release need to be templated, though
74
+
if ! meta_has_relaxed "${meta}"; then
75
+
gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
76
+
77
+
if meta_has_acquire "${meta}"; then
78
+
gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
79
+
fi
80
+
81
+
if meta_has_release "${meta}"; then
82
+
gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
83
+
fi
84
+
85
+
return
86
+
fi
87
+
88
+
printf "#ifndef ${basename}_relaxed\n"
89
+
90
+
if [ ! -z "${template}" ]; then
91
+
printf "#ifdef ${basename}\n"
92
+
fi
93
+
94
+
gen_basic_fallbacks "${basename}"
95
+
96
+
if [ ! -z "${template}" ]; then
97
+
printf "#endif /* ${atomic}_${pfx}${name}${sfx} */\n\n"
98
+
gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
99
+
gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
100
+
gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
101
+
gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"
102
+
fi
103
+
104
+
printf "#else /* ${basename}_relaxed */\n\n"
105
+
106
+
gen_template_fallback "${ATOMICDIR}/fallbacks/acquire" "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
107
+
gen_template_fallback "${ATOMICDIR}/fallbacks/release" "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
108
+
gen_template_fallback "${ATOMICDIR}/fallbacks/fence" "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
109
+
110
+
printf "#endif /* ${basename}_relaxed */\n\n"
111
+
}
112
+
113
+
gen_xchg_fallbacks()
114
+
{
115
+
local xchg="$1"; shift
116
+
cat <<EOF
117
+
#ifndef ${xchg}_relaxed
118
+
#define ${xchg}_relaxed ${xchg}
119
+
#define ${xchg}_acquire ${xchg}
120
+
#define ${xchg}_release ${xchg}
121
+
#else /* ${xchg}_relaxed */
122
+
123
+
#ifndef ${xchg}_acquire
124
+
#define ${xchg}_acquire(...) \\
125
+
__atomic_op_acquire(${xchg}, __VA_ARGS__)
126
+
#endif
127
+
128
+
#ifndef ${xchg}_release
129
+
#define ${xchg}_release(...) \\
130
+
__atomic_op_release(${xchg}, __VA_ARGS__)
131
+
#endif
132
+
133
+
#ifndef ${xchg}
134
+
#define ${xchg}(...) \\
135
+
__atomic_op_fence(${xchg}, __VA_ARGS__)
136
+
#endif
137
+
138
+
#endif /* ${xchg}_relaxed */
139
+
140
+
EOF
141
+
}
142
+
143
+
cat << EOF
144
+
// SPDX-License-Identifier: GPL-2.0
145
+
146
+
// Generated by $0
147
+
// DO NOT MODIFY THIS FILE DIRECTLY
148
+
149
+
#ifndef _LINUX_ATOMIC_FALLBACK_H
150
+
#define _LINUX_ATOMIC_FALLBACK_H
151
+
152
+
EOF
153
+
154
+
for xchg in "xchg" "cmpxchg" "cmpxchg64"; do
155
+
gen_xchg_fallbacks "${xchg}"
156
+
done
157
+
158
+
grep '^[a-z]' "$1" | while read name meta args; do
159
+
gen_proto "${meta}" "${name}" "atomic" "int" ${args}
160
+
done
161
+
162
+
cat <<EOF
163
+
#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
164
+
#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
165
+
166
+
#ifdef CONFIG_GENERIC_ATOMIC64
167
+
#include <asm-generic/atomic64.h>
168
+
#endif
169
+
170
+
EOF
171
+
172
+
grep '^[a-z]' "$1" | while read name meta args; do
173
+
gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}
174
+
done
175
+
176
+
cat <<EOF
177
+
#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c))
178
+
#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c))
179
+
180
+
#endif /* _LINUX_ATOMIC_FALLBACK_H */
181
+
EOF
+182
scripts/atomic/gen-atomic-instrumented.sh
+182
scripts/atomic/gen-atomic-instrumented.sh
···
1
+
#!/bin/sh
2
+
# SPDX-License-Identifier: GPL-2.0
3
+
4
+
ATOMICDIR=$(dirname $0)
5
+
6
+
. ${ATOMICDIR}/atomic-tbl.sh
7
+
8
+
#gen_param_check(arg)
9
+
gen_param_check()
10
+
{
11
+
local arg="$1"; shift
12
+
local type="${arg%%:*}"
13
+
local name="$(gen_param_name "${arg}")"
14
+
local rw="write"
15
+
16
+
case "${type#c}" in
17
+
i) return;;
18
+
esac
19
+
20
+
# We don't write to constant parameters
21
+
[ ${type#c} != ${type} ] && rw="read"
22
+
23
+
printf "\tkasan_check_${rw}(${name}, sizeof(*${name}));\n"
24
+
}
25
+
26
+
#gen_param_check(arg...)
27
+
gen_params_checks()
28
+
{
29
+
while [ "$#" -gt 0 ]; do
30
+
gen_param_check "$1"
31
+
shift;
32
+
done
33
+
}
34
+
35
+
# gen_guard(meta, atomic, pfx, name, sfx, order)
36
+
gen_guard()
37
+
{
38
+
local meta="$1"; shift
39
+
local atomic="$1"; shift
40
+
local pfx="$1"; shift
41
+
local name="$1"; shift
42
+
local sfx="$1"; shift
43
+
local order="$1"; shift
44
+
45
+
local atomicname="arch_${atomic}_${pfx}${name}${sfx}${order}"
46
+
47
+
local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
48
+
49
+
# We definitely need a preprocessor symbol for this atomic if it is an
50
+
# ordering variant, or if there's a generic fallback.
51
+
if [ ! -z "${order}" ] || [ ! -z "${template}" ]; then
52
+
printf "defined(${atomicname})"
53
+
return
54
+
fi
55
+
56
+
# If this is a base variant, but a relaxed variant *may* exist, then we
57
+
# only have a preprocessor symbol if the relaxed variant isn't defined
58
+
if meta_has_relaxed "${meta}"; then
59
+
printf "!defined(${atomicname}_relaxed) || defined(${atomicname})"
60
+
fi
61
+
}
62
+
63
+
#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...)
64
+
gen_proto_order_variant()
65
+
{
66
+
local meta="$1"; shift
67
+
local pfx="$1"; shift
68
+
local name="$1"; shift
69
+
local sfx="$1"; shift
70
+
local order="$1"; shift
71
+
local atomic="$1"; shift
72
+
local int="$1"; shift
73
+
74
+
local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
75
+
76
+
local guard="$(gen_guard "${meta}" "${atomic}" "${pfx}" "${name}" "${sfx}" "${order}")"
77
+
78
+
local ret="$(gen_ret_type "${meta}" "${int}")"
79
+
local params="$(gen_params "${int}" "${atomic}" "$@")"
80
+
local checks="$(gen_params_checks "$@")"
81
+
local args="$(gen_args "$@")"
82
+
local retstmt="$(gen_ret_stmt "${meta}")"
83
+
84
+
[ ! -z "${guard}" ] && printf "#if ${guard}\n"
85
+
86
+
cat <<EOF
87
+
static inline ${ret}
88
+
${atomicname}(${params})
89
+
{
90
+
${checks}
91
+
${retstmt}arch_${atomicname}(${args});
92
+
}
93
+
#define ${atomicname} ${atomicname}
94
+
EOF
95
+
96
+
[ ! -z "${guard}" ] && printf "#endif\n"
97
+
98
+
printf "\n"
99
+
}
100
+
101
+
gen_xchg()
102
+
{
103
+
local xchg="$1"; shift
104
+
local mult="$1"; shift
105
+
106
+
cat <<EOF
107
+
#define ${xchg}(ptr, ...) \\
108
+
({ \\
109
+
typeof(ptr) __ai_ptr = (ptr); \\
110
+
kasan_check_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\
111
+
arch_${xchg}(__ai_ptr, __VA_ARGS__); \\
112
+
})
113
+
EOF
114
+
}
115
+
116
+
gen_optional_xchg()
117
+
{
118
+
local name="$1"; shift
119
+
local sfx="$1"; shift
120
+
local guard="defined(arch_${name}${sfx})"
121
+
122
+
[ -z "${sfx}" ] && guard="!defined(arch_${name}_relaxed) || defined(arch_${name})"
123
+
124
+
printf "#if ${guard}\n"
125
+
gen_xchg "${name}${sfx}" ""
126
+
printf "#endif\n\n"
127
+
}
128
+
129
+
cat << EOF
130
+
// SPDX-License-Identifier: GPL-2.0
131
+
132
+
// Generated by $0
133
+
// DO NOT MODIFY THIS FILE DIRECTLY
134
+
135
+
/*
136
+
* This file provides wrappers with KASAN instrumentation for atomic operations.
137
+
* To use this functionality an arch's atomic.h file needs to define all
138
+
* atomic operations with arch_ prefix (e.g. arch_atomic_read()) and include
139
+
* this file at the end. This file provides atomic_read() that forwards to
140
+
* arch_atomic_read() for actual atomic operation.
141
+
* Note: if an arch atomic operation is implemented by means of other atomic
142
+
* operations (e.g. atomic_read()/atomic_cmpxchg() loop), then it needs to use
143
+
* arch_ variants (i.e. arch_atomic_read()/arch_atomic_cmpxchg()) to avoid
144
+
* double instrumentation.
145
+
*/
146
+
#ifndef _ASM_GENERIC_ATOMIC_INSTRUMENTED_H
147
+
#define _ASM_GENERIC_ATOMIC_INSTRUMENTED_H
148
+
149
+
#include <linux/build_bug.h>
150
+
#include <linux/kasan-checks.h>
151
+
152
+
EOF
153
+
154
+
grep '^[a-z]' "$1" | while read name meta args; do
155
+
gen_proto "${meta}" "${name}" "atomic" "int" ${args}
156
+
done
157
+
158
+
grep '^[a-z]' "$1" | while read name meta args; do
159
+
gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}
160
+
done
161
+
162
+
for xchg in "xchg" "cmpxchg" "cmpxchg64"; do
163
+
for order in "" "_acquire" "_release" "_relaxed"; do
164
+
gen_optional_xchg "${xchg}" "${order}"
165
+
done
166
+
done
167
+
168
+
for xchg in "cmpxchg_local" "cmpxchg64_local" "sync_cmpxchg"; do
169
+
gen_xchg "${xchg}" ""
170
+
printf "\n"
171
+
done
172
+
173
+
gen_xchg "cmpxchg_double" "2 * "
174
+
175
+
printf "\n\n"
176
+
177
+
gen_xchg "cmpxchg_double_local" "2 * "
178
+
179
+
cat <<EOF
180
+
181
+
#endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */
182
+
EOF
+101
scripts/atomic/gen-atomic-long.sh
+101
scripts/atomic/gen-atomic-long.sh
···
1
+
#!/bin/sh
2
+
# SPDX-License-Identifier: GPL-2.0
3
+
4
+
ATOMICDIR=$(dirname $0)
5
+
6
+
. ${ATOMICDIR}/atomic-tbl.sh
7
+
8
+
#gen_cast(arg, int, atomic)
9
+
gen_cast()
10
+
{
11
+
local arg="$1"; shift
12
+
local int="$1"; shift
13
+
local atomic="$1"; shift
14
+
15
+
[ "${arg%%:*}" = "p" ] || return
16
+
17
+
printf "($(gen_param_type "${arg}" "${int}" "${atomic}"))"
18
+
}
19
+
20
+
#gen_args_cast(int, atomic, arg...)
21
+
gen_args_cast()
22
+
{
23
+
local int="$1"; shift
24
+
local atomic="$1"; shift
25
+
26
+
while [ "$#" -gt 0 ]; do
27
+
local cast="$(gen_cast "$1" "${int}" "${atomic}")"
28
+
local arg="$(gen_param_name "$1")"
29
+
printf "${cast}${arg}"
30
+
[ "$#" -gt 1 ] && printf ", "
31
+
shift;
32
+
done
33
+
}
34
+
35
+
#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, arg...)
36
+
gen_proto_order_variant()
37
+
{
38
+
local meta="$1"; shift
39
+
local name="$1$2$3$4"; shift; shift; shift; shift
40
+
local atomic="$1"; shift
41
+
local int="$1"; shift
42
+
43
+
local ret="$(gen_ret_type "${meta}" "long")"
44
+
local params="$(gen_params "long" "atomic_long" "$@")"
45
+
local argscast="$(gen_args_cast "${int}" "${atomic}" "$@")"
46
+
local retstmt="$(gen_ret_stmt "${meta}")"
47
+
48
+
cat <<EOF
49
+
static inline ${ret}
50
+
atomic_long_${name}(${params})
51
+
{
52
+
${retstmt}${atomic}_${name}(${argscast});
53
+
}
54
+
55
+
EOF
56
+
}
57
+
58
+
cat << EOF
59
+
// SPDX-License-Identifier: GPL-2.0
60
+
61
+
// Generated by $0
62
+
// DO NOT MODIFY THIS FILE DIRECTLY
63
+
64
+
#ifndef _ASM_GENERIC_ATOMIC_LONG_H
65
+
#define _ASM_GENERIC_ATOMIC_LONG_H
66
+
67
+
#include <asm/types.h>
68
+
69
+
#ifdef CONFIG_64BIT
70
+
typedef atomic64_t atomic_long_t;
71
+
#define ATOMIC_LONG_INIT(i) ATOMIC64_INIT(i)
72
+
#define atomic_long_cond_read_acquire atomic64_cond_read_acquire
73
+
#define atomic_long_cond_read_relaxed atomic64_cond_read_relaxed
74
+
#else
75
+
typedef atomic_t atomic_long_t;
76
+
#define ATOMIC_LONG_INIT(i) ATOMIC_INIT(i)
77
+
#define atomic_long_cond_read_acquire atomic_cond_read_acquire
78
+
#define atomic_long_cond_read_relaxed atomic_cond_read_relaxed
79
+
#endif
80
+
81
+
#ifdef CONFIG_64BIT
82
+
83
+
EOF
84
+
85
+
grep '^[a-z]' "$1" | while read name meta args; do
86
+
gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}
87
+
done
88
+
89
+
cat <<EOF
90
+
#else /* CONFIG_64BIT */
91
+
92
+
EOF
93
+
94
+
grep '^[a-z]' "$1" | while read name meta args; do
95
+
gen_proto "${meta}" "${name}" "atomic" "int" ${args}
96
+
done
97
+
98
+
cat <<EOF
99
+
#endif /* CONFIG_64BIT */
100
+
#endif /* _ASM_GENERIC_ATOMIC_LONG_H */
101
+
EOF