tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * Based on arch/arm/include/asm/cmpxchg.h
3 *
4 * Copyright (C) 2012 ARM Ltd.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18#ifndef __ASM_CMPXCHG_H
19#define __ASM_CMPXCHG_H
20
21#include <linux/bug.h>
22
23#include <asm/atomic.h>
24#include <asm/barrier.h>
25#include <asm/lse.h>
26
27/*
28 * We need separate acquire parameters for ll/sc and lse, since the full
29 * barrier case is generated as release+dmb for the former and
30 * acquire+release for the latter.
31 */
32#define __XCHG_CASE(w, sz, name, mb, nop_lse, acq, acq_lse, rel, cl) \
33static inline unsigned long __xchg_case_##name(unsigned long x, \
34 volatile void *ptr) \
35{ \
36 unsigned long ret, tmp; \
37 \
38 asm volatile(ARM64_LSE_ATOMIC_INSN( \
39 /* LL/SC */ \
40 " prfm pstl1strm, %2\n" \
41 "1: ld" #acq "xr" #sz "\t%" #w "0, %2\n" \
42 " st" #rel "xr" #sz "\t%w1, %" #w "3, %2\n" \
43 " cbnz %w1, 1b\n" \
44 " " #mb, \
45 /* LSE atomics */ \
46 " swp" #acq_lse #rel #sz "\t%" #w "3, %" #w "0, %2\n" \
47 __nops(3) \
48 " " #nop_lse) \
49 : "=&r" (ret), "=&r" (tmp), "+Q" (*(unsigned long *)ptr) \
50 : "r" (x) \
51 : cl); \
52 \
53 return ret; \
54}
55
56__XCHG_CASE(w, b, 1, , , , , , )
57__XCHG_CASE(w, h, 2, , , , , , )
58__XCHG_CASE(w, , 4, , , , , , )
59__XCHG_CASE( , , 8, , , , , , )
60__XCHG_CASE(w, b, acq_1, , , a, a, , "memory")
61__XCHG_CASE(w, h, acq_2, , , a, a, , "memory")
62__XCHG_CASE(w, , acq_4, , , a, a, , "memory")
63__XCHG_CASE( , , acq_8, , , a, a, , "memory")
64__XCHG_CASE(w, b, rel_1, , , , , l, "memory")
65__XCHG_CASE(w, h, rel_2, , , , , l, "memory")
66__XCHG_CASE(w, , rel_4, , , , , l, "memory")
67__XCHG_CASE( , , rel_8, , , , , l, "memory")
68__XCHG_CASE(w, b, mb_1, dmb ish, nop, , a, l, "memory")
69__XCHG_CASE(w, h, mb_2, dmb ish, nop, , a, l, "memory")
70__XCHG_CASE(w, , mb_4, dmb ish, nop, , a, l, "memory")
71__XCHG_CASE( , , mb_8, dmb ish, nop, , a, l, "memory")
72
73#undef __XCHG_CASE
74
75#define __XCHG_GEN(sfx) \
76static inline unsigned long __xchg##sfx(unsigned long x, \
77 volatile void *ptr, \
78 int size) \
79{ \
80 switch (size) { \
81 case 1: \
82 return __xchg_case##sfx##_1(x, ptr); \
83 case 2: \
84 return __xchg_case##sfx##_2(x, ptr); \
85 case 4: \
86 return __xchg_case##sfx##_4(x, ptr); \
87 case 8: \
88 return __xchg_case##sfx##_8(x, ptr); \
89 default: \
90 BUILD_BUG(); \
91 } \
92 \
93 unreachable(); \
94}
95
96__XCHG_GEN()
97__XCHG_GEN(_acq)
98__XCHG_GEN(_rel)
99__XCHG_GEN(_mb)
100
101#undef __XCHG_GEN
102
103#define __xchg_wrapper(sfx, ptr, x) \
104({ \
105 __typeof__(*(ptr)) __ret; \
106 __ret = (__typeof__(*(ptr))) \
107 __xchg##sfx((unsigned long)(x), (ptr), sizeof(*(ptr))); \
108 __ret; \
109})
110
111/* xchg */
112#define xchg_relaxed(...) __xchg_wrapper( , __VA_ARGS__)
113#define xchg_acquire(...) __xchg_wrapper(_acq, __VA_ARGS__)
114#define xchg_release(...) __xchg_wrapper(_rel, __VA_ARGS__)
115#define xchg(...) __xchg_wrapper( _mb, __VA_ARGS__)
116
117#define __CMPXCHG_GEN(sfx) \
118static inline unsigned long __cmpxchg##sfx(volatile void *ptr, \
119 unsigned long old, \
120 unsigned long new, \
121 int size) \
122{ \
123 switch (size) { \
124 case 1: \
125 return __cmpxchg_case##sfx##_1(ptr, (u8)old, new); \
126 case 2: \
127 return __cmpxchg_case##sfx##_2(ptr, (u16)old, new); \
128 case 4: \
129 return __cmpxchg_case##sfx##_4(ptr, old, new); \
130 case 8: \
131 return __cmpxchg_case##sfx##_8(ptr, old, new); \
132 default: \
133 BUILD_BUG(); \
134 } \
135 \
136 unreachable(); \
137}
138
139__CMPXCHG_GEN()
140__CMPXCHG_GEN(_acq)
141__CMPXCHG_GEN(_rel)
142__CMPXCHG_GEN(_mb)
143
144#undef __CMPXCHG_GEN
145
146#define __cmpxchg_wrapper(sfx, ptr, o, n) \
147({ \
148 __typeof__(*(ptr)) __ret; \
149 __ret = (__typeof__(*(ptr))) \
150 __cmpxchg##sfx((ptr), (unsigned long)(o), \
151 (unsigned long)(n), sizeof(*(ptr))); \
152 __ret; \
153})
154
155/* cmpxchg */
156#define cmpxchg_relaxed(...) __cmpxchg_wrapper( , __VA_ARGS__)
157#define cmpxchg_acquire(...) __cmpxchg_wrapper(_acq, __VA_ARGS__)
158#define cmpxchg_release(...) __cmpxchg_wrapper(_rel, __VA_ARGS__)
159#define cmpxchg(...) __cmpxchg_wrapper( _mb, __VA_ARGS__)
160#define cmpxchg_local cmpxchg_relaxed
161
162/* cmpxchg64 */
163#define cmpxchg64_relaxed cmpxchg_relaxed
164#define cmpxchg64_acquire cmpxchg_acquire
165#define cmpxchg64_release cmpxchg_release
166#define cmpxchg64 cmpxchg
167#define cmpxchg64_local cmpxchg_local
168
169/* cmpxchg_double */
170#define system_has_cmpxchg_double() 1
171
172#define __cmpxchg_double_check(ptr1, ptr2) \
173({ \
174 if (sizeof(*(ptr1)) != 8) \
175 BUILD_BUG(); \
176 VM_BUG_ON((unsigned long *)(ptr2) - (unsigned long *)(ptr1) != 1); \
177})
178
179#define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \
180({\
181 int __ret;\
182 __cmpxchg_double_check(ptr1, ptr2); \
183 __ret = !__cmpxchg_double_mb((unsigned long)(o1), (unsigned long)(o2), \
184 (unsigned long)(n1), (unsigned long)(n2), \
185 ptr1); \
186 __ret; \
187})
188
189#define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \
190({\
191 int __ret;\
192 __cmpxchg_double_check(ptr1, ptr2); \
193 __ret = !__cmpxchg_double((unsigned long)(o1), (unsigned long)(o2), \
194 (unsigned long)(n1), (unsigned long)(n2), \
195 ptr1); \
196 __ret; \
197})
198
199/* this_cpu_cmpxchg */
200#define _protect_cmpxchg_local(pcp, o, n) \
201({ \
202 typeof(*raw_cpu_ptr(&(pcp))) __ret; \
203 preempt_disable(); \
204 __ret = cmpxchg_local(raw_cpu_ptr(&(pcp)), o, n); \
205 preempt_enable(); \
206 __ret; \
207})
208
209#define this_cpu_cmpxchg_1(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
210#define this_cpu_cmpxchg_2(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
211#define this_cpu_cmpxchg_4(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
212#define this_cpu_cmpxchg_8(ptr, o, n) _protect_cmpxchg_local(ptr, o, n)
213
214#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \
215({ \
216 int __ret; \
217 preempt_disable(); \
218 __ret = cmpxchg_double_local( raw_cpu_ptr(&(ptr1)), \
219 raw_cpu_ptr(&(ptr2)), \
220 o1, o2, n1, n2); \
221 preempt_enable(); \
222 __ret; \
223})
224
225#define __CMPWAIT_CASE(w, sz, name) \
226static inline void __cmpwait_case_##name(volatile void *ptr, \
227 unsigned long val) \
228{ \
229 unsigned long tmp; \
230 \
231 asm volatile( \
232 " ldxr" #sz "\t%" #w "[tmp], %[v]\n" \
233 " eor %" #w "[tmp], %" #w "[tmp], %" #w "[val]\n" \
234 " cbnz %" #w "[tmp], 1f\n" \
235 " wfe\n" \
236 "1:" \
237 : [tmp] "=&r" (tmp), [v] "+Q" (*(unsigned long *)ptr) \
238 : [val] "r" (val)); \
239}
240
241__CMPWAIT_CASE(w, b, 1);
242__CMPWAIT_CASE(w, h, 2);
243__CMPWAIT_CASE(w, , 4);
244__CMPWAIT_CASE( , , 8);
245
246#undef __CMPWAIT_CASE
247
248#define __CMPWAIT_GEN(sfx) \
249static inline void __cmpwait##sfx(volatile void *ptr, \
250 unsigned long val, \
251 int size) \
252{ \
253 switch (size) { \
254 case 1: \
255 return __cmpwait_case##sfx##_1(ptr, (u8)val); \
256 case 2: \
257 return __cmpwait_case##sfx##_2(ptr, (u16)val); \
258 case 4: \
259 return __cmpwait_case##sfx##_4(ptr, val); \
260 case 8: \
261 return __cmpwait_case##sfx##_8(ptr, val); \
262 default: \
263 BUILD_BUG(); \
264 } \
265 \
266 unreachable(); \
267}
268
269__CMPWAIT_GEN()
270
271#undef __CMPWAIT_GEN
272
273#define __cmpwait_relaxed(ptr, val) \
274 __cmpwait((ptr), (unsigned long)(val), sizeof(*(ptr)))
275
276#endif /* __ASM_CMPXCHG_H */