tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_POWERPC_CMPXCHG_H_
3#define _ASM_POWERPC_CMPXCHG_H_
4
5#ifdef __KERNEL__
6#include <linux/compiler.h>
7#include <asm/synch.h>
8#include <linux/bug.h>
9#include <asm/asm-405.h>
10
11#ifdef __BIG_ENDIAN
12#define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE)
13#else
14#define BITOFF_CAL(size, off) (off * BITS_PER_BYTE)
15#endif
16
17#define XCHG_GEN(type, sfx, cl) \
18static inline u32 __xchg_##type##sfx(volatile void *p, u32 val) \
19{ \
20 unsigned int prev, prev_mask, tmp, bitoff, off; \
21 \
22 off = (unsigned long)p % sizeof(u32); \
23 bitoff = BITOFF_CAL(sizeof(type), off); \
24 p -= off; \
25 val <<= bitoff; \
26 prev_mask = (u32)(type)-1 << bitoff; \
27 \
28 __asm__ __volatile__( \
29"1: lwarx %0,0,%3\n" \
30" andc %1,%0,%5\n" \
31" or %1,%1,%4\n" \
32 PPC405_ERR77(0,%3) \
33" stwcx. %1,0,%3\n" \
34" bne- 1b\n" \
35 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \
36 : "r" (p), "r" (val), "r" (prev_mask) \
37 : "cc", cl); \
38 \
39 return prev >> bitoff; \
40}
41
42#define CMPXCHG_GEN(type, sfx, br, br2, cl) \
43static inline \
44u32 __cmpxchg_##type##sfx(volatile void *p, u32 old, u32 new) \
45{ \
46 unsigned int prev, prev_mask, tmp, bitoff, off; \
47 \
48 off = (unsigned long)p % sizeof(u32); \
49 bitoff = BITOFF_CAL(sizeof(type), off); \
50 p -= off; \
51 old <<= bitoff; \
52 new <<= bitoff; \
53 prev_mask = (u32)(type)-1 << bitoff; \
54 \
55 __asm__ __volatile__( \
56 br \
57"1: lwarx %0,0,%3\n" \
58" and %1,%0,%6\n" \
59" cmpw 0,%1,%4\n" \
60" bne- 2f\n" \
61" andc %1,%0,%6\n" \
62" or %1,%1,%5\n" \
63 PPC405_ERR77(0,%3) \
64" stwcx. %1,0,%3\n" \
65" bne- 1b\n" \
66 br2 \
67 "\n" \
68"2:" \
69 : "=&r" (prev), "=&r" (tmp), "+m" (*(u32*)p) \
70 : "r" (p), "r" (old), "r" (new), "r" (prev_mask) \
71 : "cc", cl); \
72 \
73 return prev >> bitoff; \
74}
75
76/*
77 * Atomic exchange
78 *
79 * Changes the memory location '*p' to be val and returns
80 * the previous value stored there.
81 */
82
83XCHG_GEN(u8, _local, "memory");
84XCHG_GEN(u8, _relaxed, "cc");
85XCHG_GEN(u16, _local, "memory");
86XCHG_GEN(u16, _relaxed, "cc");
87
88static __always_inline unsigned long
89__xchg_u32_local(volatile void *p, unsigned long val)
90{
91 unsigned long prev;
92
93 __asm__ __volatile__(
94"1: lwarx %0,0,%2 \n"
95 PPC405_ERR77(0,%2)
96" stwcx. %3,0,%2 \n\
97 bne- 1b"
98 : "=&r" (prev), "+m" (*(volatile unsigned int *)p)
99 : "r" (p), "r" (val)
100 : "cc", "memory");
101
102 return prev;
103}
104
105static __always_inline unsigned long
106__xchg_u32_relaxed(u32 *p, unsigned long val)
107{
108 unsigned long prev;
109
110 __asm__ __volatile__(
111"1: lwarx %0,0,%2\n"
112 PPC405_ERR77(0, %2)
113" stwcx. %3,0,%2\n"
114" bne- 1b"
115 : "=&r" (prev), "+m" (*p)
116 : "r" (p), "r" (val)
117 : "cc");
118
119 return prev;
120}
121
122#ifdef CONFIG_PPC64
123static __always_inline unsigned long
124__xchg_u64_local(volatile void *p, unsigned long val)
125{
126 unsigned long prev;
127
128 __asm__ __volatile__(
129"1: ldarx %0,0,%2 \n"
130 PPC405_ERR77(0,%2)
131" stdcx. %3,0,%2 \n\
132 bne- 1b"
133 : "=&r" (prev), "+m" (*(volatile unsigned long *)p)
134 : "r" (p), "r" (val)
135 : "cc", "memory");
136
137 return prev;
138}
139
140static __always_inline unsigned long
141__xchg_u64_relaxed(u64 *p, unsigned long val)
142{
143 unsigned long prev;
144
145 __asm__ __volatile__(
146"1: ldarx %0,0,%2\n"
147 PPC405_ERR77(0, %2)
148" stdcx. %3,0,%2\n"
149" bne- 1b"
150 : "=&r" (prev), "+m" (*p)
151 : "r" (p), "r" (val)
152 : "cc");
153
154 return prev;
155}
156#endif
157
158static __always_inline unsigned long
159__xchg_local(void *ptr, unsigned long x, unsigned int size)
160{
161 switch (size) {
162 case 1:
163 return __xchg_u8_local(ptr, x);
164 case 2:
165 return __xchg_u16_local(ptr, x);
166 case 4:
167 return __xchg_u32_local(ptr, x);
168#ifdef CONFIG_PPC64
169 case 8:
170 return __xchg_u64_local(ptr, x);
171#endif
172 }
173 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg");
174 return x;
175}
176
177static __always_inline unsigned long
178__xchg_relaxed(void *ptr, unsigned long x, unsigned int size)
179{
180 switch (size) {
181 case 1:
182 return __xchg_u8_relaxed(ptr, x);
183 case 2:
184 return __xchg_u16_relaxed(ptr, x);
185 case 4:
186 return __xchg_u32_relaxed(ptr, x);
187#ifdef CONFIG_PPC64
188 case 8:
189 return __xchg_u64_relaxed(ptr, x);
190#endif
191 }
192 BUILD_BUG_ON_MSG(1, "Unsupported size for __xchg_local");
193 return x;
194}
195#define xchg_local(ptr,x) \
196 ({ \
197 __typeof__(*(ptr)) _x_ = (x); \
198 (__typeof__(*(ptr))) __xchg_local((ptr), \
199 (unsigned long)_x_, sizeof(*(ptr))); \
200 })
201
202#define xchg_relaxed(ptr, x) \
203({ \
204 __typeof__(*(ptr)) _x_ = (x); \
205 (__typeof__(*(ptr))) __xchg_relaxed((ptr), \
206 (unsigned long)_x_, sizeof(*(ptr))); \
207})
208/*
209 * Compare and exchange - if *p == old, set it to new,
210 * and return the old value of *p.
211 */
212
213CMPXCHG_GEN(u8, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
214CMPXCHG_GEN(u8, _local, , , "memory");
215CMPXCHG_GEN(u8, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
216CMPXCHG_GEN(u8, _relaxed, , , "cc");
217CMPXCHG_GEN(u16, , PPC_ATOMIC_ENTRY_BARRIER, PPC_ATOMIC_EXIT_BARRIER, "memory");
218CMPXCHG_GEN(u16, _local, , , "memory");
219CMPXCHG_GEN(u16, _acquire, , PPC_ACQUIRE_BARRIER, "memory");
220CMPXCHG_GEN(u16, _relaxed, , , "cc");
221
222static __always_inline unsigned long
223__cmpxchg_u32(volatile unsigned int *p, unsigned long old, unsigned long new)
224{
225 unsigned int prev;
226
227 __asm__ __volatile__ (
228 PPC_ATOMIC_ENTRY_BARRIER
229"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
230 cmpw 0,%0,%3\n\
231 bne- 2f\n"
232 PPC405_ERR77(0,%2)
233" stwcx. %4,0,%2\n\
234 bne- 1b"
235 PPC_ATOMIC_EXIT_BARRIER
236 "\n\
2372:"
238 : "=&r" (prev), "+m" (*p)
239 : "r" (p), "r" (old), "r" (new)
240 : "cc", "memory");
241
242 return prev;
243}
244
245static __always_inline unsigned long
246__cmpxchg_u32_local(volatile unsigned int *p, unsigned long old,
247 unsigned long new)
248{
249 unsigned int prev;
250
251 __asm__ __volatile__ (
252"1: lwarx %0,0,%2 # __cmpxchg_u32\n\
253 cmpw 0,%0,%3\n\
254 bne- 2f\n"
255 PPC405_ERR77(0,%2)
256" stwcx. %4,0,%2\n\
257 bne- 1b"
258 "\n\
2592:"
260 : "=&r" (prev), "+m" (*p)
261 : "r" (p), "r" (old), "r" (new)
262 : "cc", "memory");
263
264 return prev;
265}
266
267static __always_inline unsigned long
268__cmpxchg_u32_relaxed(u32 *p, unsigned long old, unsigned long new)
269{
270 unsigned long prev;
271
272 __asm__ __volatile__ (
273"1: lwarx %0,0,%2 # __cmpxchg_u32_relaxed\n"
274" cmpw 0,%0,%3\n"
275" bne- 2f\n"
276 PPC405_ERR77(0, %2)
277" stwcx. %4,0,%2\n"
278" bne- 1b\n"
279"2:"
280 : "=&r" (prev), "+m" (*p)
281 : "r" (p), "r" (old), "r" (new)
282 : "cc");
283
284 return prev;
285}
286
287/*
288 * cmpxchg family don't have order guarantee if cmp part fails, therefore we
289 * can avoid superfluous barriers if we use assembly code to implement
290 * cmpxchg() and cmpxchg_acquire(), however we don't do the similar for
291 * cmpxchg_release() because that will result in putting a barrier in the
292 * middle of a ll/sc loop, which is probably a bad idea. For example, this
293 * might cause the conditional store more likely to fail.
294 */
295static __always_inline unsigned long
296__cmpxchg_u32_acquire(u32 *p, unsigned long old, unsigned long new)
297{
298 unsigned long prev;
299
300 __asm__ __volatile__ (
301"1: lwarx %0,0,%2 # __cmpxchg_u32_acquire\n"
302" cmpw 0,%0,%3\n"
303" bne- 2f\n"
304 PPC405_ERR77(0, %2)
305" stwcx. %4,0,%2\n"
306" bne- 1b\n"
307 PPC_ACQUIRE_BARRIER
308 "\n"
309"2:"
310 : "=&r" (prev), "+m" (*p)
311 : "r" (p), "r" (old), "r" (new)
312 : "cc", "memory");
313
314 return prev;
315}
316
317#ifdef CONFIG_PPC64
318static __always_inline unsigned long
319__cmpxchg_u64(volatile unsigned long *p, unsigned long old, unsigned long new)
320{
321 unsigned long prev;
322
323 __asm__ __volatile__ (
324 PPC_ATOMIC_ENTRY_BARRIER
325"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
326 cmpd 0,%0,%3\n\
327 bne- 2f\n\
328 stdcx. %4,0,%2\n\
329 bne- 1b"
330 PPC_ATOMIC_EXIT_BARRIER
331 "\n\
3322:"
333 : "=&r" (prev), "+m" (*p)
334 : "r" (p), "r" (old), "r" (new)
335 : "cc", "memory");
336
337 return prev;
338}
339
340static __always_inline unsigned long
341__cmpxchg_u64_local(volatile unsigned long *p, unsigned long old,
342 unsigned long new)
343{
344 unsigned long prev;
345
346 __asm__ __volatile__ (
347"1: ldarx %0,0,%2 # __cmpxchg_u64\n\
348 cmpd 0,%0,%3\n\
349 bne- 2f\n\
350 stdcx. %4,0,%2\n\
351 bne- 1b"
352 "\n\
3532:"
354 : "=&r" (prev), "+m" (*p)
355 : "r" (p), "r" (old), "r" (new)
356 : "cc", "memory");
357
358 return prev;
359}
360
361static __always_inline unsigned long
362__cmpxchg_u64_relaxed(u64 *p, unsigned long old, unsigned long new)
363{
364 unsigned long prev;
365
366 __asm__ __volatile__ (
367"1: ldarx %0,0,%2 # __cmpxchg_u64_relaxed\n"
368" cmpd 0,%0,%3\n"
369" bne- 2f\n"
370" stdcx. %4,0,%2\n"
371" bne- 1b\n"
372"2:"
373 : "=&r" (prev), "+m" (*p)
374 : "r" (p), "r" (old), "r" (new)
375 : "cc");
376
377 return prev;
378}
379
380static __always_inline unsigned long
381__cmpxchg_u64_acquire(u64 *p, unsigned long old, unsigned long new)
382{
383 unsigned long prev;
384
385 __asm__ __volatile__ (
386"1: ldarx %0,0,%2 # __cmpxchg_u64_acquire\n"
387" cmpd 0,%0,%3\n"
388" bne- 2f\n"
389" stdcx. %4,0,%2\n"
390" bne- 1b\n"
391 PPC_ACQUIRE_BARRIER
392 "\n"
393"2:"
394 : "=&r" (prev), "+m" (*p)
395 : "r" (p), "r" (old), "r" (new)
396 : "cc", "memory");
397
398 return prev;
399}
400#endif
401
402static __always_inline unsigned long
403__cmpxchg(volatile void *ptr, unsigned long old, unsigned long new,
404 unsigned int size)
405{
406 switch (size) {
407 case 1:
408 return __cmpxchg_u8(ptr, old, new);
409 case 2:
410 return __cmpxchg_u16(ptr, old, new);
411 case 4:
412 return __cmpxchg_u32(ptr, old, new);
413#ifdef CONFIG_PPC64
414 case 8:
415 return __cmpxchg_u64(ptr, old, new);
416#endif
417 }
418 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg");
419 return old;
420}
421
422static __always_inline unsigned long
423__cmpxchg_local(void *ptr, unsigned long old, unsigned long new,
424 unsigned int size)
425{
426 switch (size) {
427 case 1:
428 return __cmpxchg_u8_local(ptr, old, new);
429 case 2:
430 return __cmpxchg_u16_local(ptr, old, new);
431 case 4:
432 return __cmpxchg_u32_local(ptr, old, new);
433#ifdef CONFIG_PPC64
434 case 8:
435 return __cmpxchg_u64_local(ptr, old, new);
436#endif
437 }
438 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_local");
439 return old;
440}
441
442static __always_inline unsigned long
443__cmpxchg_relaxed(void *ptr, unsigned long old, unsigned long new,
444 unsigned int size)
445{
446 switch (size) {
447 case 1:
448 return __cmpxchg_u8_relaxed(ptr, old, new);
449 case 2:
450 return __cmpxchg_u16_relaxed(ptr, old, new);
451 case 4:
452 return __cmpxchg_u32_relaxed(ptr, old, new);
453#ifdef CONFIG_PPC64
454 case 8:
455 return __cmpxchg_u64_relaxed(ptr, old, new);
456#endif
457 }
458 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_relaxed");
459 return old;
460}
461
462static __always_inline unsigned long
463__cmpxchg_acquire(void *ptr, unsigned long old, unsigned long new,
464 unsigned int size)
465{
466 switch (size) {
467 case 1:
468 return __cmpxchg_u8_acquire(ptr, old, new);
469 case 2:
470 return __cmpxchg_u16_acquire(ptr, old, new);
471 case 4:
472 return __cmpxchg_u32_acquire(ptr, old, new);
473#ifdef CONFIG_PPC64
474 case 8:
475 return __cmpxchg_u64_acquire(ptr, old, new);
476#endif
477 }
478 BUILD_BUG_ON_MSG(1, "Unsupported size for __cmpxchg_acquire");
479 return old;
480}
481#define cmpxchg(ptr, o, n) \
482 ({ \
483 __typeof__(*(ptr)) _o_ = (o); \
484 __typeof__(*(ptr)) _n_ = (n); \
485 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \
486 (unsigned long)_n_, sizeof(*(ptr))); \
487 })
488
489
490#define cmpxchg_local(ptr, o, n) \
491 ({ \
492 __typeof__(*(ptr)) _o_ = (o); \
493 __typeof__(*(ptr)) _n_ = (n); \
494 (__typeof__(*(ptr))) __cmpxchg_local((ptr), (unsigned long)_o_, \
495 (unsigned long)_n_, sizeof(*(ptr))); \
496 })
497
498#define cmpxchg_relaxed(ptr, o, n) \
499({ \
500 __typeof__(*(ptr)) _o_ = (o); \
501 __typeof__(*(ptr)) _n_ = (n); \
502 (__typeof__(*(ptr))) __cmpxchg_relaxed((ptr), \
503 (unsigned long)_o_, (unsigned long)_n_, \
504 sizeof(*(ptr))); \
505})
506
507#define cmpxchg_acquire(ptr, o, n) \
508({ \
509 __typeof__(*(ptr)) _o_ = (o); \
510 __typeof__(*(ptr)) _n_ = (n); \
511 (__typeof__(*(ptr))) __cmpxchg_acquire((ptr), \
512 (unsigned long)_o_, (unsigned long)_n_, \
513 sizeof(*(ptr))); \
514})
515#ifdef CONFIG_PPC64
516#define cmpxchg64(ptr, o, n) \
517 ({ \
518 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
519 cmpxchg((ptr), (o), (n)); \
520 })
521#define cmpxchg64_local(ptr, o, n) \
522 ({ \
523 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
524 cmpxchg_local((ptr), (o), (n)); \
525 })
526#define cmpxchg64_relaxed(ptr, o, n) \
527({ \
528 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
529 cmpxchg_relaxed((ptr), (o), (n)); \
530})
531#define cmpxchg64_acquire(ptr, o, n) \
532({ \
533 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
534 cmpxchg_acquire((ptr), (o), (n)); \
535})
536#else
537#include <asm-generic/cmpxchg-local.h>
538#define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n))
539#endif
540
541#endif /* __KERNEL__ */
542#endif /* _ASM_POWERPC_CMPXCHG_H_ */