tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1#ifndef _ASM_POWERPC_CMPXCHG_H_
2#define _ASM_POWERPC_CMPXCHG_H_
3
4#ifdef __KERNEL__
5#include <linux/compiler.h>
6#include <asm/synch.h>
7#include <asm/asm-compat.h>
8
9/*
10 * Atomic exchange
11 *
12 * Changes the memory location '*ptr' to be val and returns
13 * the previous value stored there.
14 */
15static __always_inline unsigned long
16__xchg_u32(volatile void *p, unsigned long val)
17{
18 unsigned long prev;
19
20 __asm__ __volatile__(
21 PPC_RELEASE_BARRIER
22"1: lwarx %0,0,%2 \n"
23 PPC405_ERR77(0,%2)
24" stwcx. %3,0,%2 \n\
25 bne- 1b"
26 PPC_ACQUIRE_BARRIER
27 : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
28 : "r" (p), "r" (val)
29 : "cc", "memory");
30
31 return prev;
32}
33
34/*
35 * Atomic exchange
36 *
37 * Changes the memory location '*ptr' to be val and returns
38 * the previous value stored there.
39 */
40static __always_inline unsigned long
41__xchg_u32_local(volatile void *p, unsigned long val)
42{
43 unsigned long prev;
44
45 __asm__ __volatile__(
46"1: lwarx %0,0,%2 \n"
47 PPC405_ERR77(0,%2)
48" stwcx. %3,0,%2 \n\
49 bne- 1b"
50 : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
51 : "r" (p), "r" (val)
52 : "cc", "memory");
53
54 return prev;
55}
56
57#ifdef CONFIG_PPC64
58static __always_inline unsigned long
59__xchg_u64(volatile void *p, unsigned long val)
60{
61 unsigned long prev;
62
63 __asm__ __volatile__(
64 PPC_RELEASE_BARRIER
65"1: ldarx %0,0,%2 \n"
66 PPC405_ERR77(0,%2)
67" stdcx. %3,0,%2 \n\
68 bne- 1b"
69 PPC_ACQUIRE_BARRIER
70 : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
71 : "r" (p), "r" (val)
72 : "cc", "memory");
73
74 return prev;
75}
76
77static __always_inline unsigned long
78__xchg_u64_local(volatile void *p, unsigned long val)
79{
80 unsigned long prev;
81
82 __asm__ __volatile__(
83"1: ldarx %0,0,%2 \n"
84 PPC405_ERR77(0,%2)
85" stdcx. %3,0,%2 \n\
86 bne- 1b"
87 : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
88 : "r" (p), "r" (val)
89 : "cc", "memory");
90
91 return prev;
92}
93#endif
94
95/*
96 * This function doesn't exist, so you'll get a linker error
97 * if something tries to do an invalid xchg().
98 */
99extern void __xchg_called_with_bad_pointer(void);
100
101static __always_inline unsigned long
102__xchg(volatile void *ptr, unsigned long x, unsigned int size)
103{
104 switch (size) {
105 case 4:
106 return __xchg_u32(ptr, x);
107#ifdef CONFIG_PPC64
108 case 8:
109 return __xchg_u64(ptr, x);
110#endif
111 }
112 __xchg_called_with_bad_pointer();
113 return x;
114}
115
116static __always_inline unsigned long
117__xchg_local(volatile void *ptr, unsigned long x, unsigned int size)
118{
119 switch (size) {
120 case 4:
121 return __xchg_u32_local(ptr, x);
122#ifdef CONFIG_PPC64
123 case 8:
124 return __xchg_u64_local(ptr, x);
125#endif
126 }
127 __xchg_called_with_bad_pointer();
128 return x;
129}
130#define xchg(ptr,x) \
131 ({ \
132 __typeof__(*(ptr)) _x_ = (x); \
133 (__typeof__(*(ptr))) __xchg((ptr), (unsigned long)_x_, sizeof(*(ptr))); \
134 })
135
136#define xchg_local(ptr,x) \
137 ({ \
138 __typeof__(*(ptr)) _x_ = (x); \
139 (__typeof__(*(ptr))) __xchg_local((ptr), \
140 (unsigned long)_x_, sizeof(*(ptr))); \
141 })
142
143/*
144 * Compare and exchange - if *p == old, set it to new,
145 * and return the old value of *p.
146 */
147
148static __always_inline unsigned long
149__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
150{
151 unsigned int prev;
152
153 __asm__ __volatile__ (
154 PPC_RELEASE_BARRIER
155"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
156 cmpw 0,%0,%3\n\
157 bne- 2f\n"
158 PPC405_ERR77(0,%2)
159" stwcx. %4,0,%2\n\
160 bne- 1b"
161 PPC_ACQUIRE_BARRIER
162 "\n\
1632:"
164 : "=&r" (prev), "+m" (*p)
165 : "r" (p), "r" (old), "r" (new)
166 : "cc", "memory");
167
168 return prev;
169}
170
171static __always_inline unsigned long
172__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
173 unsigned long new)
174{
175 unsigned int prev;
176
177 __asm__ __volatile__ (
178"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
179 cmpw 0,%0,%3\n\
180 bne- 2f\n"
181 PPC405_ERR77(0,%2)
182" stwcx. %4,0,%2\n\
183 bne- 1b"
184 "\n\
1852:"
186 : "=&r" (prev), "+m" (*p)
187 : "r" (p), "r" (old), "r" (new)
188 : "cc", "memory");
189
190 return prev;
191}
192
193#ifdef CONFIG_PPC64
194static __always_inline unsigned long
195__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
196{
197 unsigned long prev;
198
199 __asm__ __volatile__ (
200 PPC_RELEASE_BARRIER
201"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
202 cmpd 0,%0,%3\n\
203 bne- 2f\n\
204 stdcx. %4,0,%2\n\
205 bne- 1b"
206 PPC_ACQUIRE_BARRIER
207 "\n\
2082:"
209 : "=&r" (prev), "+m" (*p)
210 : "r" (p), "r" (old), "r" (new)
211 : "cc", "memory");
212
213 return prev;
214}
215
216static __always_inline unsigned long
217__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
218 unsigned long new)
219{
220 unsigned long prev;
221
222 __asm__ __volatile__ (
223"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
224 cmpd 0,%0,%3\n\
225 bne- 2f\n\
226 stdcx. %4,0,%2\n\
227 bne- 1b"
228 "\n\
2292:"
230 : "=&r" (prev), "+m" (*p)
231 : "r" (p), "r" (old), "r" (new)
232 : "cc", "memory");
233
234 return prev;
235}
236#endif
237
238/* This function doesn't exist, so you'll get a linker error
239 if something tries to do an invalid cmpxchg(). */
240extern void __cmpxchg_called_with_bad_pointer(void);
241
242static __always_inline unsigned long
243__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
244 unsigned int size)
245{
246 switch (size) {
247 case 4:
248 return __cmpxchg_u32(ptr, old, new);
249#ifdef CONFIG_PPC64
250 case 8:
251 return __cmpxchg_u64(ptr, old, new);
252#endif
253 }
254 __cmpxchg_called_with_bad_pointer();
255 return old;
256}
257
258static __always_inline unsigned long
259__cmpxchg_local(volatile void *ptr, unsigned long old, unsigned long new,
260 unsigned int size)
261{
262 switch (size) {
263 case 4:
264 return __cmpxchg_u32_local(ptr, old, new);
265#ifdef CONFIG_PPC64
266 case 8:
267 return __cmpxchg_u64_local(ptr, old, new);
268#endif
269 }
270 __cmpxchg_called_with_bad_pointer();
271 return old;
272}
273
274#define cmpxchg(ptr, o, n) \
275 ({ \
276 __typeof__(*(ptr)) _o_ = (o); \
277 __typeof__(*(ptr)) _n_ = (n); \
278 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
279 (unsigned long)_n_, sizeof(*(ptr))); \
280 })
281
282
283#define cmpxchg_local(ptr, o, n) \
284 ({ \
285 __typeof__(*(ptr)) _o_ = (o); \
286 __typeof__(*(ptr)) _n_ = (n); \
287 (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \
288 (unsigned long)_n_, sizeof(*(ptr))); \
289 })
290
291#ifdef CONFIG_PPC64
292#define cmpxchg64(ptr, o, n) \
293 ({ \
294 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
295 cmpxchg((ptr), (o), (n)); \
296 })
297#define cmpxchg64_local(ptr, o, n) \
298 ({ \
299 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
300 cmpxchg_local((ptr), (o), (n)); \
301 })
302#define cmpxchg64_relaxed cmpxchg64_local
303#else
304#include <asm-generic/cmpxchg-local.h>
305#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
306#endif
307
308#endif /* __KERNEL__ */
309#endif /* _ASM_POWERPC_CMPXCHG_H_ */