tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
arm64: atomics: implement native {relaxed, acquire, release} atomics
Commit 654672d4ba1a ("locking/atomics: Add _{acquire|release|relaxed}()
variants of some atomic operation") introduced a relaxed atomic API to
Linux that maps nicely onto the arm64 memory model, including the new
ARMv8.1 atomic instructions.
This patch hooks up the API to our relaxed atomic instructions, rather
than have them all expand to the full-barrier variants as they do
currently.
Signed-off-by: Will Deacon <will.deacon@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
authored by
Will Deacon
and committed by
Catalin Marinas
305d454a
e8f3010f
+372
-263
+59
-4
arch/arm64/include/asm/atomic.h
+59
-4
arch/arm64/include/asm/atomic.h
···
55
55
56
56
#define atomic_read(v) READ_ONCE((v)->counter)
57
57
#define atomic_set(v, i) (((v)->counter) = (i))
58
+
59
+
#define atomic_add_return_relaxed atomic_add_return_relaxed
60
+
#define atomic_add_return_acquire atomic_add_return_acquire
61
+
#define atomic_add_return_release atomic_add_return_release
62
+
#define atomic_add_return atomic_add_return
63
+
64
+
#define atomic_inc_return_relaxed(v) atomic_add_return_relaxed(1, (v))
65
+
#define atomic_inc_return_acquire(v) atomic_add_return_acquire(1, (v))
66
+
#define atomic_inc_return_release(v) atomic_add_return_release(1, (v))
67
+
#define atomic_inc_return(v) atomic_add_return(1, (v))
68
+
69
+
#define atomic_sub_return_relaxed atomic_sub_return_relaxed
70
+
#define atomic_sub_return_acquire atomic_sub_return_acquire
71
+
#define atomic_sub_return_release atomic_sub_return_release
72
+
#define atomic_sub_return atomic_sub_return
73
+
74
+
#define atomic_dec_return_relaxed(v) atomic_sub_return_relaxed(1, (v))
75
+
#define atomic_dec_return_acquire(v) atomic_sub_return_acquire(1, (v))
76
+
#define atomic_dec_return_release(v) atomic_sub_return_release(1, (v))
77
+
#define atomic_dec_return(v) atomic_sub_return(1, (v))
78
+
79
+
#define atomic_xchg_relaxed(v, new) xchg_relaxed(&((v)->counter), (new))
80
+
#define atomic_xchg_acquire(v, new) xchg_acquire(&((v)->counter), (new))
81
+
#define atomic_xchg_release(v, new) xchg_release(&((v)->counter), (new))
58
82
#define atomic_xchg(v, new) xchg(&((v)->counter), (new))
83
+
84
+
#define atomic_cmpxchg_relaxed(v, old, new) \
85
+
cmpxchg_relaxed(&((v)->counter), (old), (new))
86
+
#define atomic_cmpxchg_acquire(v, old, new) \
87
+
cmpxchg_acquire(&((v)->counter), (old), (new))
88
+
#define atomic_cmpxchg_release(v, old, new) \
89
+
cmpxchg_release(&((v)->counter), (old), (new))
59
90
#define atomic_cmpxchg(v, old, new) cmpxchg(&((v)->counter), (old), (new))
60
91
61
92
#define atomic_inc(v) atomic_add(1, (v))
62
93
#define atomic_dec(v) atomic_sub(1, (v))
63
-
#define atomic_inc_return(v) atomic_add_return(1, (v))
64
-
#define atomic_dec_return(v) atomic_sub_return(1, (v))
65
94
#define atomic_inc_and_test(v) (atomic_inc_return(v) == 0)
66
95
#define atomic_dec_and_test(v) (atomic_dec_return(v) == 0)
67
96
#define atomic_sub_and_test(i, v) (atomic_sub_return((i), (v)) == 0)
···
104
75
#define ATOMIC64_INIT ATOMIC_INIT
105
76
#define atomic64_read atomic_read
106
77
#define atomic64_set atomic_set
78
+
79
+
#define atomic64_add_return_relaxed atomic64_add_return_relaxed
80
+
#define atomic64_add_return_acquire atomic64_add_return_acquire
81
+
#define atomic64_add_return_release atomic64_add_return_release
82
+
#define atomic64_add_return atomic64_add_return
83
+
84
+
#define atomic64_inc_return_relaxed(v) atomic64_add_return_relaxed(1, (v))
85
+
#define atomic64_inc_return_acquire(v) atomic64_add_return_acquire(1, (v))
86
+
#define atomic64_inc_return_release(v) atomic64_add_return_release(1, (v))
87
+
#define atomic64_inc_return(v) atomic64_add_return(1, (v))
88
+
89
+
#define atomic64_sub_return_relaxed atomic64_sub_return_relaxed
90
+
#define atomic64_sub_return_acquire atomic64_sub_return_acquire
91
+
#define atomic64_sub_return_release atomic64_sub_return_release
92
+
#define atomic64_sub_return atomic64_sub_return
93
+
94
+
#define atomic64_dec_return_relaxed(v) atomic64_sub_return_relaxed(1, (v))
95
+
#define atomic64_dec_return_acquire(v) atomic64_sub_return_acquire(1, (v))
96
+
#define atomic64_dec_return_release(v) atomic64_sub_return_release(1, (v))
97
+
#define atomic64_dec_return(v) atomic64_sub_return(1, (v))
98
+
99
+
#define atomic64_xchg_relaxed atomic_xchg_relaxed
100
+
#define atomic64_xchg_acquire atomic_xchg_acquire
101
+
#define atomic64_xchg_release atomic_xchg_release
107
102
#define atomic64_xchg atomic_xchg
103
+
104
+
#define atomic64_cmpxchg_relaxed atomic_cmpxchg_relaxed
105
+
#define atomic64_cmpxchg_acquire atomic_cmpxchg_acquire
106
+
#define atomic64_cmpxchg_release atomic_cmpxchg_release
108
107
#define atomic64_cmpxchg atomic_cmpxchg
109
108
110
109
#define atomic64_inc(v) atomic64_add(1, (v))
111
110
#define atomic64_dec(v) atomic64_sub(1, (v))
112
-
#define atomic64_inc_return(v) atomic64_add_return(1, (v))
113
-
#define atomic64_dec_return(v) atomic64_sub_return(1, (v))
114
111
#define atomic64_inc_and_test(v) (atomic64_inc_return(v) == 0)
115
112
#define atomic64_dec_and_test(v) (atomic64_dec_return(v) == 0)
116
113
#define atomic64_sub_and_test(i, v) (atomic64_sub_return((i), (v)) == 0)
+60
-38
arch/arm64/include/asm/atomic_ll_sc.h
+60
-38
arch/arm64/include/asm/atomic_ll_sc.h
···
55
55
} \
56
56
__LL_SC_EXPORT(atomic_##op);
57
57
58
-
#define ATOMIC_OP_RETURN(op, asm_op) \
58
+
#define ATOMIC_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \
59
59
__LL_SC_INLINE int \
60
-
__LL_SC_PREFIX(atomic_##op##_return(int i, atomic_t *v)) \
60
+
__LL_SC_PREFIX(atomic_##op##_return##name(int i, atomic_t *v)) \
61
61
{ \
62
62
unsigned long tmp; \
63
63
int result; \
64
64
\
65
-
asm volatile("// atomic_" #op "_return\n" \
65
+
asm volatile("// atomic_" #op "_return" #name "\n" \
66
66
" prfm pstl1strm, %2\n" \
67
-
"1: ldxr %w0, %2\n" \
67
+
"1: ld" #acq "xr %w0, %2\n" \
68
68
" " #asm_op " %w0, %w0, %w3\n" \
69
-
" stlxr %w1, %w0, %2\n" \
70
-
" cbnz %w1, 1b" \
69
+
" st" #rel "xr %w1, %w0, %2\n" \
70
+
" cbnz %w1, 1b\n" \
71
+
" " #mb \
71
72
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
72
73
: "Ir" (i) \
73
-
: "memory"); \
74
+
: cl); \
74
75
\
75
-
smp_mb(); \
76
76
return result; \
77
77
} \
78
-
__LL_SC_EXPORT(atomic_##op##_return);
78
+
__LL_SC_EXPORT(atomic_##op##_return##name);
79
79
80
-
#define ATOMIC_OPS(op, asm_op) \
81
-
ATOMIC_OP(op, asm_op) \
82
-
ATOMIC_OP_RETURN(op, asm_op)
80
+
#define ATOMIC_OPS(...) \
81
+
ATOMIC_OP(__VA_ARGS__) \
82
+
ATOMIC_OP_RETURN( , dmb ish, , l, "memory", __VA_ARGS__)
83
83
84
-
ATOMIC_OPS(add, add)
85
-
ATOMIC_OPS(sub, sub)
84
+
#define ATOMIC_OPS_RLX(...) \
85
+
ATOMIC_OPS(__VA_ARGS__) \
86
+
ATOMIC_OP_RETURN(_relaxed, , , , , __VA_ARGS__)\
87
+
ATOMIC_OP_RETURN(_acquire, , a, , "memory", __VA_ARGS__)\
88
+
ATOMIC_OP_RETURN(_release, , , l, "memory", __VA_ARGS__)
89
+
90
+
ATOMIC_OPS_RLX(add, add)
91
+
ATOMIC_OPS_RLX(sub, sub)
86
92
87
93
ATOMIC_OP(and, and)
88
94
ATOMIC_OP(andnot, bic)
89
95
ATOMIC_OP(or, orr)
90
96
ATOMIC_OP(xor, eor)
91
97
98
+
#undef ATOMIC_OPS_RLX
92
99
#undef ATOMIC_OPS
93
100
#undef ATOMIC_OP_RETURN
94
101
#undef ATOMIC_OP
···
118
111
} \
119
112
__LL_SC_EXPORT(atomic64_##op);
120
113
121
-
#define ATOMIC64_OP_RETURN(op, asm_op) \
114
+
#define ATOMIC64_OP_RETURN(name, mb, acq, rel, cl, op, asm_op) \
122
115
__LL_SC_INLINE long \
123
-
__LL_SC_PREFIX(atomic64_##op##_return(long i, atomic64_t *v)) \
116
+
__LL_SC_PREFIX(atomic64_##op##_return##name(long i, atomic64_t *v)) \
124
117
{ \
125
118
long result; \
126
119
unsigned long tmp; \
127
120
\
128
-
asm volatile("// atomic64_" #op "_return\n" \
121
+
asm volatile("// atomic64_" #op "_return" #name "\n" \
129
122
" prfm pstl1strm, %2\n" \
130
-
"1: ldxr %0, %2\n" \
123
+
"1: ld" #acq "xr %0, %2\n" \
131
124
" " #asm_op " %0, %0, %3\n" \
132
-
" stlxr %w1, %0, %2\n" \
133
-
" cbnz %w1, 1b" \
125
+
" st" #rel "xr %w1, %0, %2\n" \
126
+
" cbnz %w1, 1b\n" \
127
+
" " #mb \
134
128
: "=&r" (result), "=&r" (tmp), "+Q" (v->counter) \
135
129
: "Ir" (i) \
136
-
: "memory"); \
130
+
: cl); \
137
131
\
138
-
smp_mb(); \
139
132
return result; \
140
133
} \
141
-
__LL_SC_EXPORT(atomic64_##op##_return);
134
+
__LL_SC_EXPORT(atomic64_##op##_return##name);
142
135
143
-
#define ATOMIC64_OPS(op, asm_op) \
144
-
ATOMIC64_OP(op, asm_op) \
145
-
ATOMIC64_OP_RETURN(op, asm_op)
136
+
#define ATOMIC64_OPS(...) \
137
+
ATOMIC64_OP(__VA_ARGS__) \
138
+
ATOMIC64_OP_RETURN(, dmb ish, , l, "memory", __VA_ARGS__)
146
139
147
-
ATOMIC64_OPS(add, add)
148
-
ATOMIC64_OPS(sub, sub)
140
+
#define ATOMIC64_OPS_RLX(...) \
141
+
ATOMIC64_OPS(__VA_ARGS__) \
142
+
ATOMIC64_OP_RETURN(_relaxed,, , , , __VA_ARGS__) \
143
+
ATOMIC64_OP_RETURN(_acquire,, a, , "memory", __VA_ARGS__) \
144
+
ATOMIC64_OP_RETURN(_release,, , l, "memory", __VA_ARGS__)
145
+
146
+
ATOMIC64_OPS_RLX(add, add)
147
+
ATOMIC64_OPS_RLX(sub, sub)
149
148
150
149
ATOMIC64_OP(and, and)
151
150
ATOMIC64_OP(andnot, bic)
152
151
ATOMIC64_OP(or, orr)
153
152
ATOMIC64_OP(xor, eor)
154
153
154
+
#undef ATOMIC64_OPS_RLX
155
155
#undef ATOMIC64_OPS
156
156
#undef ATOMIC64_OP_RETURN
157
157
#undef ATOMIC64_OP
···
186
172
}
187
173
__LL_SC_EXPORT(atomic64_dec_if_positive);
188
174
189
-
#define __CMPXCHG_CASE(w, sz, name, mb, rel, cl) \
175
+
#define __CMPXCHG_CASE(w, sz, name, mb, acq, rel, cl) \
190
176
__LL_SC_INLINE unsigned long \
191
177
__LL_SC_PREFIX(__cmpxchg_case_##name(volatile void *ptr, \
192
178
unsigned long old, \
···
196
182
\
197
183
asm volatile( \
198
184
" prfm pstl1strm, %[v]\n" \
199
-
"1: ldxr" #sz "\t%" #w "[oldval], %[v]\n" \
185
+
"1: ld" #acq "xr" #sz "\t%" #w "[oldval], %[v]\n" \
200
186
" eor %" #w "[tmp], %" #w "[oldval], %" #w "[old]\n" \
201
187
" cbnz %" #w "[tmp], 2f\n" \
202
188
" st" #rel "xr" #sz "\t%w[tmp], %" #w "[new], %[v]\n" \
···
213
199
} \
214
200
__LL_SC_EXPORT(__cmpxchg_case_##name);
215
201
216
-
__CMPXCHG_CASE(w, b, 1, , , )
217
-
__CMPXCHG_CASE(w, h, 2, , , )
218
-
__CMPXCHG_CASE(w, , 4, , , )
219
-
__CMPXCHG_CASE( , , 8, , , )
220
-
__CMPXCHG_CASE(w, b, mb_1, dmb ish, l, "memory")
221
-
__CMPXCHG_CASE(w, h, mb_2, dmb ish, l, "memory")
222
-
__CMPXCHG_CASE(w, , mb_4, dmb ish, l, "memory")
223
-
__CMPXCHG_CASE( , , mb_8, dmb ish, l, "memory")
202
+
__CMPXCHG_CASE(w, b, 1, , , , )
203
+
__CMPXCHG_CASE(w, h, 2, , , , )
204
+
__CMPXCHG_CASE(w, , 4, , , , )
205
+
__CMPXCHG_CASE( , , 8, , , , )
206
+
__CMPXCHG_CASE(w, b, acq_1, , a, , "memory")
207
+
__CMPXCHG_CASE(w, h, acq_2, , a, , "memory")
208
+
__CMPXCHG_CASE(w, , acq_4, , a, , "memory")
209
+
__CMPXCHG_CASE( , , acq_8, , a, , "memory")
210
+
__CMPXCHG_CASE(w, b, rel_1, , , l, "memory")
211
+
__CMPXCHG_CASE(w, h, rel_2, , , l, "memory")
212
+
__CMPXCHG_CASE(w, , rel_4, , , l, "memory")
213
+
__CMPXCHG_CASE( , , rel_8, , , l, "memory")
214
+
__CMPXCHG_CASE(w, b, mb_1, dmb ish, , l, "memory")
215
+
__CMPXCHG_CASE(w, h, mb_2, dmb ish, , l, "memory")
216
+
__CMPXCHG_CASE(w, , mb_4, dmb ish, , l, "memory")
217
+
__CMPXCHG_CASE( , , mb_8, dmb ish, , l, "memory")
224
218
225
219
#undef __CMPXCHG_CASE
226
220
+119
-80
arch/arm64/include/asm/atomic_lse.h
+119
-80
arch/arm64/include/asm/atomic_lse.h
···
75
75
: "x30");
76
76
}
77
77
78
-
static inline int atomic_add_return(int i, atomic_t *v)
79
-
{
80
-
register int w0 asm ("w0") = i;
81
-
register atomic_t *x1 asm ("x1") = v;
82
-
83
-
asm volatile(ARM64_LSE_ATOMIC_INSN(
84
-
/* LL/SC */
85
-
" nop\n"
86
-
__LL_SC_ATOMIC(add_return),
87
-
/* LSE atomics */
88
-
" ldaddal %w[i], w30, %[v]\n"
89
-
" add %w[i], %w[i], w30")
90
-
: [i] "+r" (w0), [v] "+Q" (v->counter)
91
-
: "r" (x1)
92
-
: "x30", "memory");
93
-
94
-
return w0;
78
+
#define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \
79
+
static inline int atomic_add_return##name(int i, atomic_t *v) \
80
+
{ \
81
+
register int w0 asm ("w0") = i; \
82
+
register atomic_t *x1 asm ("x1") = v; \
83
+
\
84
+
asm volatile(ARM64_LSE_ATOMIC_INSN( \
85
+
/* LL/SC */ \
86
+
" nop\n" \
87
+
__LL_SC_ATOMIC(add_return##name), \
88
+
/* LSE atomics */ \
89
+
" ldadd" #mb " %w[i], w30, %[v]\n" \
90
+
" add %w[i], %w[i], w30") \
91
+
: [i] "+r" (w0), [v] "+Q" (v->counter) \
92
+
: "r" (x1) \
93
+
: "x30" , ##cl); \
94
+
\
95
+
return w0; \
95
96
}
97
+
98
+
ATOMIC_OP_ADD_RETURN(_relaxed, )
99
+
ATOMIC_OP_ADD_RETURN(_acquire, a, "memory")
100
+
ATOMIC_OP_ADD_RETURN(_release, l, "memory")
101
+
ATOMIC_OP_ADD_RETURN( , al, "memory")
102
+
103
+
#undef ATOMIC_OP_ADD_RETURN
96
104
97
105
static inline void atomic_and(int i, atomic_t *v)
98
106
{
···
136
128
: "x30");
137
129
}
138
130
139
-
static inline int atomic_sub_return(int i, atomic_t *v)
140
-
{
141
-
register int w0 asm ("w0") = i;
142
-
register atomic_t *x1 asm ("x1") = v;
143
-
144
-
asm volatile(ARM64_LSE_ATOMIC_INSN(
145
-
/* LL/SC */
146
-
" nop\n"
147
-
__LL_SC_ATOMIC(sub_return)
148
-
" nop",
149
-
/* LSE atomics */
150
-
" neg %w[i], %w[i]\n"
151
-
" ldaddal %w[i], w30, %[v]\n"
152
-
" add %w[i], %w[i], w30")
153
-
: [i] "+r" (w0), [v] "+Q" (v->counter)
154
-
: "r" (x1)
155
-
: "x30", "memory");
156
-
157
-
return w0;
131
+
#define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \
132
+
static inline int atomic_sub_return##name(int i, atomic_t *v) \
133
+
{ \
134
+
register int w0 asm ("w0") = i; \
135
+
register atomic_t *x1 asm ("x1") = v; \
136
+
\
137
+
asm volatile(ARM64_LSE_ATOMIC_INSN( \
138
+
/* LL/SC */ \
139
+
" nop\n" \
140
+
__LL_SC_ATOMIC(sub_return##name) \
141
+
" nop", \
142
+
/* LSE atomics */ \
143
+
" neg %w[i], %w[i]\n" \
144
+
" ldadd" #mb " %w[i], w30, %[v]\n" \
145
+
" add %w[i], %w[i], w30") \
146
+
: [i] "+r" (w0), [v] "+Q" (v->counter) \
147
+
: "r" (x1) \
148
+
: "x30" , ##cl); \
149
+
\
150
+
return w0; \
158
151
}
159
152
153
+
ATOMIC_OP_SUB_RETURN(_relaxed, )
154
+
ATOMIC_OP_SUB_RETURN(_acquire, a, "memory")
155
+
ATOMIC_OP_SUB_RETURN(_release, l, "memory")
156
+
ATOMIC_OP_SUB_RETURN( , al, "memory")
157
+
158
+
#undef ATOMIC_OP_SUB_RETURN
160
159
#undef __LL_SC_ATOMIC
161
160
162
161
#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op)
···
216
201
: "x30");
217
202
}
218
203
219
-
static inline long atomic64_add_return(long i, atomic64_t *v)
220
-
{
221
-
register long x0 asm ("x0") = i;
222
-
register atomic64_t *x1 asm ("x1") = v;
223
-
224
-
asm volatile(ARM64_LSE_ATOMIC_INSN(
225
-
/* LL/SC */
226
-
" nop\n"
227
-
__LL_SC_ATOMIC64(add_return),
228
-
/* LSE atomics */
229
-
" ldaddal %[i], x30, %[v]\n"
230
-
" add %[i], %[i], x30")
231
-
: [i] "+r" (x0), [v] "+Q" (v->counter)
232
-
: "r" (x1)
233
-
: "x30", "memory");
234
-
235
-
return x0;
204
+
#define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \
205
+
static inline long atomic64_add_return##name(long i, atomic64_t *v) \
206
+
{ \
207
+
register long x0 asm ("x0") = i; \
208
+
register atomic64_t *x1 asm ("x1") = v; \
209
+
\
210
+
asm volatile(ARM64_LSE_ATOMIC_INSN( \
211
+
/* LL/SC */ \
212
+
" nop\n" \
213
+
__LL_SC_ATOMIC64(add_return##name), \
214
+
/* LSE atomics */ \
215
+
" ldadd" #mb " %[i], x30, %[v]\n" \
216
+
" add %[i], %[i], x30") \
217
+
: [i] "+r" (x0), [v] "+Q" (v->counter) \
218
+
: "r" (x1) \
219
+
: "x30" , ##cl); \
220
+
\
221
+
return x0; \
236
222
}
223
+
224
+
ATOMIC64_OP_ADD_RETURN(_relaxed, )
225
+
ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory")
226
+
ATOMIC64_OP_ADD_RETURN(_release, l, "memory")
227
+
ATOMIC64_OP_ADD_RETURN( , al, "memory")
228
+
229
+
#undef ATOMIC64_OP_ADD_RETURN
237
230
238
231
static inline void atomic64_and(long i, atomic64_t *v)
239
232
{
···
277
254
: "x30");
278
255
}
279
256
280
-
static inline long atomic64_sub_return(long i, atomic64_t *v)
281
-
{
282
-
register long x0 asm ("x0") = i;
283
-
register atomic64_t *x1 asm ("x1") = v;
284
-
285
-
asm volatile(ARM64_LSE_ATOMIC_INSN(
286
-
/* LL/SC */
287
-
" nop\n"
288
-
__LL_SC_ATOMIC64(sub_return)
289
-
" nop",
290
-
/* LSE atomics */
291
-
" neg %[i], %[i]\n"
292
-
" ldaddal %[i], x30, %[v]\n"
293
-
" add %[i], %[i], x30")
294
-
: [i] "+r" (x0), [v] "+Q" (v->counter)
295
-
: "r" (x1)
296
-
: "x30", "memory");
297
-
298
-
return x0;
257
+
#define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \
258
+
static inline long atomic64_sub_return##name(long i, atomic64_t *v) \
259
+
{ \
260
+
register long x0 asm ("x0") = i; \
261
+
register atomic64_t *x1 asm ("x1") = v; \
262
+
\
263
+
asm volatile(ARM64_LSE_ATOMIC_INSN( \
264
+
/* LL/SC */ \
265
+
" nop\n" \
266
+
__LL_SC_ATOMIC64(sub_return##name) \
267
+
" nop", \
268
+
/* LSE atomics */ \
269
+
" neg %[i], %[i]\n" \
270
+
" ldadd" #mb " %[i], x30, %[v]\n" \
271
+
" add %[i], %[i], x30") \
272
+
: [i] "+r" (x0), [v] "+Q" (v->counter) \
273
+
: "r" (x1) \
274
+
: "x30" , ##cl); \
275
+
\
276
+
return x0; \
299
277
}
278
+
279
+
ATOMIC64_OP_SUB_RETURN(_relaxed, )
280
+
ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory")
281
+
ATOMIC64_OP_SUB_RETURN(_release, l, "memory")
282
+
ATOMIC64_OP_SUB_RETURN( , al, "memory")
283
+
284
+
#undef ATOMIC64_OP_SUB_RETURN
300
285
301
286
static inline long atomic64_dec_if_positive(atomic64_t *v)
302
287
{
···
364
333
return x0; \
365
334
}
366
335
367
-
__CMPXCHG_CASE(w, b, 1, )
368
-
__CMPXCHG_CASE(w, h, 2, )
369
-
__CMPXCHG_CASE(w, , 4, )
370
-
__CMPXCHG_CASE(x, , 8, )
371
-
__CMPXCHG_CASE(w, b, mb_1, al, "memory")
372
-
__CMPXCHG_CASE(w, h, mb_2, al, "memory")
373
-
__CMPXCHG_CASE(w, , mb_4, al, "memory")
374
-
__CMPXCHG_CASE(x, , mb_8, al, "memory")
336
+
__CMPXCHG_CASE(w, b, 1, )
337
+
__CMPXCHG_CASE(w, h, 2, )
338
+
__CMPXCHG_CASE(w, , 4, )
339
+
__CMPXCHG_CASE(x, , 8, )
340
+
__CMPXCHG_CASE(w, b, acq_1, a, "memory")
341
+
__CMPXCHG_CASE(w, h, acq_2, a, "memory")
342
+
__CMPXCHG_CASE(w, , acq_4, a, "memory")
343
+
__CMPXCHG_CASE(x, , acq_8, a, "memory")
344
+
__CMPXCHG_CASE(w, b, rel_1, l, "memory")
345
+
__CMPXCHG_CASE(w, h, rel_2, l, "memory")
346
+
__CMPXCHG_CASE(w, , rel_4, l, "memory")
347
+
__CMPXCHG_CASE(x, , rel_8, l, "memory")
348
+
__CMPXCHG_CASE(w, b, mb_1, al, "memory")
349
+
__CMPXCHG_CASE(w, h, mb_2, al, "memory")
350
+
__CMPXCHG_CASE(w, , mb_4, al, "memory")
351
+
__CMPXCHG_CASE(x, , mb_8, al, "memory")
375
352
376
353
#undef __LL_SC_CMPXCHG
377
354
#undef __CMPXCHG_CASE
+134
-141
arch/arm64/include/asm/cmpxchg.h
+134
-141
arch/arm64/include/asm/cmpxchg.h
···
25
25
#include <asm/barrier.h>
26
26
#include <asm/lse.h>
27
27
28
-
static inline unsigned long __xchg(unsigned long x, volatile void *ptr, int size)
29
-
{
30
-
unsigned long ret, tmp;
31
-
32
-
switch (size) {
33
-
case 1:
34
-
asm volatile(ARM64_LSE_ATOMIC_INSN(
35
-
/* LL/SC */
36
-
" prfm pstl1strm, %2\n"
37
-
"1: ldxrb %w0, %2\n"
38
-
" stlxrb %w1, %w3, %2\n"
39
-
" cbnz %w1, 1b\n"
40
-
" dmb ish",
41
-
/* LSE atomics */
42
-
" nop\n"
43
-
" nop\n"
44
-
" swpalb %w3, %w0, %2\n"
45
-
" nop\n"
46
-
" nop")
47
-
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr)
48
-
: "r" (x)
49
-
: "memory");
50
-
break;
51
-
case 2:
52
-
asm volatile(ARM64_LSE_ATOMIC_INSN(
53
-
/* LL/SC */
54
-
" prfm pstl1strm, %2\n"
55
-
"1: ldxrh %w0, %2\n"
56
-
" stlxrh %w1, %w3, %2\n"
57
-
" cbnz %w1, 1b\n"
58
-
" dmb ish",
59
-
/* LSE atomics */
60
-
" nop\n"
61
-
" nop\n"
62
-
" swpalh %w3, %w0, %2\n"
63
-
" nop\n"
64
-
" nop")
65
-
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u16 *)ptr)
66
-
: "r" (x)
67
-
: "memory");
68
-
break;
69
-
case 4:
70
-
asm volatile(ARM64_LSE_ATOMIC_INSN(
71
-
/* LL/SC */
72
-
" prfm pstl1strm, %2\n"
73
-
"1: ldxr %w0, %2\n"
74
-
" stlxr %w1, %w3, %2\n"
75
-
" cbnz %w1, 1b\n"
76
-
" dmb ish",
77
-
/* LSE atomics */
78
-
" nop\n"
79
-
" nop\n"
80
-
" swpal %w3, %w0, %2\n"
81
-
" nop\n"
82
-
" nop")
83
-
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u32 *)ptr)
84
-
: "r" (x)
85
-
: "memory");
86
-
break;
87
-
case 8:
88
-
asm volatile(ARM64_LSE_ATOMIC_INSN(
89
-
/* LL/SC */
90
-
" prfm pstl1strm, %2\n"
91
-
"1: ldxr %0, %2\n"
92
-
" stlxr %w1, %3, %2\n"
93
-
" cbnz %w1, 1b\n"
94
-
" dmb ish",
95
-
/* LSE atomics */
96
-
" nop\n"
97
-
" nop\n"
98
-
" swpal %3, %0, %2\n"
99
-
" nop\n"
100
-
" nop")
101
-
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u64 *)ptr)
102
-
: "r" (x)
103
-
: "memory");
104
-
break;
105
-
default:
106
-
BUILD_BUG();
107
-
}
108
-
109
-
return ret;
28
+
/*
29
+
* We need separate acquire parameters for ll/sc and lse, since the full
30
+
* barrier case is generated as release+dmb for the former and
31
+
* acquire+release for the latter.
32
+
*/
33
+
#define __XCHG_CASE(w, sz, name, mb, nop_lse, acq, acq_lse, rel, cl) \
34
+
static inline unsigned long __xchg_case_##name(unsigned long x, \
35
+
volatile void *ptr) \
36
+
{ \
37
+
unsigned long ret, tmp; \
38
+
\
39
+
asm volatile(ARM64_LSE_ATOMIC_INSN( \
40
+
/* LL/SC */ \
41
+
" prfm pstl1strm, %2\n" \
42
+
"1: ld" #acq "xr" #sz "\t%" #w "0, %2\n" \
43
+
" st" #rel "xr" #sz "\t%w1, %" #w "3, %2\n" \
44
+
" cbnz %w1, 1b\n" \
45
+
" " #mb, \
46
+
/* LSE atomics */ \
47
+
" nop\n" \
48
+
" nop\n" \
49
+
" swp" #acq_lse #rel #sz "\t%" #w "3, %" #w "0, %2\n" \
50
+
" nop\n" \
51
+
" " #nop_lse) \
52
+
: "=&r" (ret), "=&r" (tmp), "+Q" (*(u8 *)ptr) \
53
+
: "r" (x) \
54
+
: cl); \
55
+
\
56
+
return ret; \
110
57
}
111
58
112
-
#define xchg(ptr,x) \
113
-
({ \
114
-
__typeof__(*(ptr)) __ret; \
115
-
__ret = (__typeof__(*(ptr))) \
116
-
__xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \
117
-
__ret; \
118
-
})
59
+
__XCHG_CASE(w, b, 1, , , , , , )
60
+
__XCHG_CASE(w, h, 2, , , , , , )
61
+
__XCHG_CASE(w, , 4, , , , , , )
62
+
__XCHG_CASE( , , 8, , , , , , )
63
+
__XCHG_CASE(w, b, acq_1, , , a, a, , "memory")
64
+
__XCHG_CASE(w, h, acq_2, , , a, a, , "memory")
65
+
__XCHG_CASE(w, , acq_4, , , a, a, , "memory")
66
+
__XCHG_CASE( , , acq_8, , , a, a, , "memory")
67
+
__XCHG_CASE(w, b, rel_1, , , , , l, "memory")
68
+
__XCHG_CASE(w, h, rel_2, , , , , l, "memory")
69
+
__XCHG_CASE(w, , rel_4, , , , , l, "memory")
70
+
__XCHG_CASE( , , rel_8, , , , , l, "memory")
71
+
__XCHG_CASE(w, b, mb_1, dmb ish, nop, , a, l, "memory")
72
+
__XCHG_CASE(w, h, mb_2, dmb ish, nop, , a, l, "memory")
73
+
__XCHG_CASE(w, , mb_4, dmb ish, nop, , a, l, "memory")
74
+
__XCHG_CASE( , , mb_8, dmb ish, nop, , a, l, "memory")
119
75
120
-
static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old,
121
-
unsigned long new, int size)
122
-
{
123
-
switch (size) {
124
-
case 1:
125
-
return __cmpxchg_case_1(ptr, (u8)old, new);
126
-
case 2:
127
-
return __cmpxchg_case_2(ptr, (u16)old, new);
128
-
case 4:
129
-
return __cmpxchg_case_4(ptr, old, new);
130
-
case 8:
131
-
return __cmpxchg_case_8(ptr, old, new);
132
-
default:
133
-
BUILD_BUG();
134
-
}
76
+
#undef __XCHG_CASE
135
77
136
-
unreachable();
78
+
#define __XCHG_GEN(sfx) \
79
+
static inline unsigned long __xchg##sfx(unsigned long x, \
80
+
volatile void *ptr, \
81
+
int size) \
82
+
{ \
83
+
switch (size) { \
84
+
case 1: \
85
+
return __xchg_case##sfx##_1(x, ptr); \
86
+
case 2: \
87
+
return __xchg_case##sfx##_2(x, ptr); \
88
+
case 4: \
89
+
return __xchg_case##sfx##_4(x, ptr); \
90
+
case 8: \
91
+
return __xchg_case##sfx##_8(x, ptr); \
92
+
default: \
93
+
BUILD_BUG(); \
94
+
} \
95
+
\
96
+
unreachable(); \
137
97
}
138
98
139
-
static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old,
140
-
unsigned long new, int size)
141
-
{
142
-
switch (size) {
143
-
case 1:
144
-
return __cmpxchg_case_mb_1(ptr, (u8)old, new);
145
-
case 2:
146
-
return __cmpxchg_case_mb_2(ptr, (u16)old, new);
147
-
case 4:
148
-
return __cmpxchg_case_mb_4(ptr, old, new);
149
-
case 8:
150
-
return __cmpxchg_case_mb_8(ptr, old, new);
151
-
default:
152
-
BUILD_BUG();
153
-
}
99
+
__XCHG_GEN()
100
+
__XCHG_GEN(_acq)
101
+
__XCHG_GEN(_rel)
102
+
__XCHG_GEN(_mb)
154
103
155
-
unreachable();
104
+
#undef __XCHG_GEN
105
+
106
+
#define __xchg_wrapper(sfx, ptr, x) \
107
+
({ \
108
+
__typeof__(*(ptr)) __ret; \
109
+
__ret = (__typeof__(*(ptr))) \
110
+
__xchg##sfx((unsigned long)(x), (ptr), sizeof(*(ptr))); \
111
+
__ret; \
112
+
})
113
+
114
+
/* xchg */
115
+
#define xchg_relaxed(...) __xchg_wrapper( , __VA_ARGS__)
116
+
#define xchg_acquire(...) __xchg_wrapper(_acq, __VA_ARGS__)
117
+
#define xchg_release(...) __xchg_wrapper(_rel, __VA_ARGS__)
118
+
#define xchg(...) __xchg_wrapper( _mb, __VA_ARGS__)
119
+
120
+
#define __CMPXCHG_GEN(sfx) \
121
+
static inline unsigned long __cmpxchg##sfx(volatile void *ptr, \
122
+
unsigned long old, \
123
+
unsigned long new, \
124
+
int size) \
125
+
{ \
126
+
switch (size) { \
127
+
case 1: \
128
+
return __cmpxchg_case##sfx##_1(ptr, (u8)old, new); \
129
+
case 2: \
130
+
return __cmpxchg_case##sfx##_2(ptr, (u16)old, new); \
131
+
case 4: \
132
+
return __cmpxchg_case##sfx##_4(ptr, old, new); \
133
+
case 8: \
134
+
return __cmpxchg_case##sfx##_8(ptr, old, new); \
135
+
default: \
136
+
BUILD_BUG(); \
137
+
} \
138
+
\
139
+
unreachable(); \
156
140
}
157
141
158
-
#define cmpxchg(ptr, o, n) \
159
-
({ \
160
-
__typeof__(*(ptr)) __ret; \
161
-
__ret = (__typeof__(*(ptr))) \
162
-
__cmpxchg_mb((ptr), (unsigned long)(o), (unsigned long)(n), \
163
-
sizeof(*(ptr))); \
164
-
__ret; \
142
+
__CMPXCHG_GEN()
143
+
__CMPXCHG_GEN(_acq)
144
+
__CMPXCHG_GEN(_rel)
145
+
__CMPXCHG_GEN(_mb)
146
+
147
+
#undef __CMPXCHG_GEN
148
+
149
+
#define __cmpxchg_wrapper(sfx, ptr, o, n) \
150
+
({ \
151
+
__typeof__(*(ptr)) __ret; \
152
+
__ret = (__typeof__(*(ptr))) \
153
+
__cmpxchg##sfx((ptr), (unsigned long)(o), \
154
+
(unsigned long)(n), sizeof(*(ptr))); \
155
+
__ret; \
165
156
})
166
157
167
-
#define cmpxchg_local(ptr, o, n) \
168
-
({ \
169
-
__typeof__(*(ptr)) __ret; \
170
-
__ret = (__typeof__(*(ptr))) \
171
-
__cmpxchg((ptr), (unsigned long)(o), \
172
-
(unsigned long)(n), sizeof(*(ptr))); \
173
-
__ret; \
174
-
})
158
+
/* cmpxchg */
159
+
#define cmpxchg_relaxed(...) __cmpxchg_wrapper( , __VA_ARGS__)
160
+
#define cmpxchg_acquire(...) __cmpxchg_wrapper(_acq, __VA_ARGS__)
161
+
#define cmpxchg_release(...) __cmpxchg_wrapper(_rel, __VA_ARGS__)
162
+
#define cmpxchg(...) __cmpxchg_wrapper( _mb, __VA_ARGS__)
163
+
#define cmpxchg_local cmpxchg_relaxed
175
164
165
+
/* cmpxchg64 */
166
+
#define cmpxchg64_relaxed cmpxchg_relaxed
167
+
#define cmpxchg64_acquire cmpxchg_acquire
168
+
#define cmpxchg64_release cmpxchg_release
169
+
#define cmpxchg64 cmpxchg
170
+
#define cmpxchg64_local cmpxchg_local
171
+
172
+
/* cmpxchg_double */
176
173
#define system_has_cmpxchg_double() 1
177
174
178
175
#define __cmpxchg_double_check(ptr1, ptr2) \
···
199
202
__ret; \
200
203
})
201
204
205
+
/* this_cpu_cmpxchg */
202
206
#define _protect_cmpxchg_local(pcp, o, n) \
203
207
({ \
204
208
typeof(*raw_cpu_ptr(&(pcp))) __ret; \
···
224
226
preempt_enable(); \
225
227
__ret; \
226
228
})
227
-
228
-
#define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n))
229
-
#define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n))
230
-
231
-
#define cmpxchg64_relaxed(ptr,o,n) cmpxchg_local((ptr),(o),(n))
232
229
233
230
#endif /* __ASM_CMPXCHG_H */