Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/*
2 * Copyright IBM Corp. 1999, 2009
3 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>,
4 * Denis Joseph Barrow,
5 * Arnd Bergmann <arndb@de.ibm.com>,
6 *
7 * Atomic operations that C can't guarantee us.
8 * Useful for resource counting etc.
9 * s390 uses 'Compare And Swap' for atomicity in SMP environment.
10 *
11 */
12
13#ifndef __ARCH_S390_ATOMIC__
14#define __ARCH_S390_ATOMIC__
15
16#include <linux/compiler.h>
17#include <linux/types.h>
18#include <asm/barrier.h>
19#include <asm/cmpxchg.h>
20
21#define ATOMIC_INIT(i) { (i) }
22
23#define __ATOMIC_NO_BARRIER "\n"
24
25#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
26
27#define __ATOMIC_OR "lao"
28#define __ATOMIC_AND "lan"
29#define __ATOMIC_ADD "laa"
30#define __ATOMIC_XOR "lax"
31#define __ATOMIC_BARRIER "bcr 14,0\n"
32
33#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier) \
34({ \
35 int old_val; \
36 \
37 typecheck(atomic_t *, ptr); \
38 asm volatile( \
39 op_string " %0,%2,%1\n" \
40 __barrier \
41 : "=d" (old_val), "+Q" ((ptr)->counter) \
42 : "d" (op_val) \
43 : "cc", "memory"); \
44 old_val; \
45})
46
47#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
48
49#define __ATOMIC_OR "or"
50#define __ATOMIC_AND "nr"
51#define __ATOMIC_ADD "ar"
52#define __ATOMIC_XOR "xr"
53#define __ATOMIC_BARRIER "\n"
54
55#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier) \
56({ \
57 int old_val, new_val; \
58 \
59 typecheck(atomic_t *, ptr); \
60 asm volatile( \
61 " l %0,%2\n" \
62 "0: lr %1,%0\n" \
63 op_string " %1,%3\n" \
64 " cs %0,%1,%2\n" \
65 " jl 0b" \
66 : "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\
67 : "d" (op_val) \
68 : "cc", "memory"); \
69 old_val; \
70})
71
72#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
73
74static inline int atomic_read(const atomic_t *v)
75{
76 int c;
77
78 asm volatile(
79 " l %0,%1\n"
80 : "=d" (c) : "Q" (v->counter));
81 return c;
82}
83
84static inline void atomic_set(atomic_t *v, int i)
85{
86 asm volatile(
87 " st %1,%0\n"
88 : "=Q" (v->counter) : "d" (i));
89}
90
91static inline int atomic_add_return(int i, atomic_t *v)
92{
93 return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER) + i;
94}
95
96static inline int atomic_fetch_add(int i, atomic_t *v)
97{
98 return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER);
99}
100
101static inline void atomic_add(int i, atomic_t *v)
102{
103#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
104 if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {
105 asm volatile(
106 "asi %0,%1\n"
107 : "+Q" (v->counter)
108 : "i" (i)
109 : "cc", "memory");
110 return;
111 }
112#endif
113 __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_NO_BARRIER);
114}
115
116#define atomic_add_negative(_i, _v) (atomic_add_return(_i, _v) < 0)
117#define atomic_inc(_v) atomic_add(1, _v)
118#define atomic_inc_return(_v) atomic_add_return(1, _v)
119#define atomic_inc_and_test(_v) (atomic_add_return(1, _v) == 0)
120#define atomic_sub(_i, _v) atomic_add(-(int)(_i), _v)
121#define atomic_sub_return(_i, _v) atomic_add_return(-(int)(_i), _v)
122#define atomic_fetch_sub(_i, _v) atomic_fetch_add(-(int)(_i), _v)
123#define atomic_sub_and_test(_i, _v) (atomic_sub_return(_i, _v) == 0)
124#define atomic_dec(_v) atomic_sub(1, _v)
125#define atomic_dec_return(_v) atomic_sub_return(1, _v)
126#define atomic_dec_and_test(_v) (atomic_sub_return(1, _v) == 0)
127
128#define ATOMIC_OPS(op, OP) \
129static inline void atomic_##op(int i, atomic_t *v) \
130{ \
131 __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_NO_BARRIER); \
132} \
133static inline int atomic_fetch_##op(int i, atomic_t *v) \
134{ \
135 return __ATOMIC_LOOP(v, i, __ATOMIC_##OP, __ATOMIC_BARRIER); \
136}
137
138ATOMIC_OPS(and, AND)
139ATOMIC_OPS(or, OR)
140ATOMIC_OPS(xor, XOR)
141
142#undef ATOMIC_OPS
143
144#define atomic_xchg(v, new) (xchg(&((v)->counter), new))
145
146static inline int atomic_cmpxchg(atomic_t *v, int old, int new)
147{
148 asm volatile(
149 " cs %0,%2,%1"
150 : "+d" (old), "+Q" (v->counter)
151 : "d" (new)
152 : "cc", "memory");
153 return old;
154}
155
156static inline int __atomic_add_unless(atomic_t *v, int a, int u)
157{
158 int c, old;
159 c = atomic_read(v);
160 for (;;) {
161 if (unlikely(c == u))
162 break;
163 old = atomic_cmpxchg(v, c, c + a);
164 if (likely(old == c))
165 break;
166 c = old;
167 }
168 return c;
169}
170
171
172#undef __ATOMIC_LOOP
173
174#define ATOMIC64_INIT(i) { (i) }
175
176#define __ATOMIC64_NO_BARRIER "\n"
177
178#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
179
180#define __ATOMIC64_OR "laog"
181#define __ATOMIC64_AND "lang"
182#define __ATOMIC64_ADD "laag"
183#define __ATOMIC64_XOR "laxg"
184#define __ATOMIC64_BARRIER "bcr 14,0\n"
185
186#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier) \
187({ \
188 long long old_val; \
189 \
190 typecheck(atomic64_t *, ptr); \
191 asm volatile( \
192 op_string " %0,%2,%1\n" \
193 __barrier \
194 : "=d" (old_val), "+Q" ((ptr)->counter) \
195 : "d" (op_val) \
196 : "cc", "memory"); \
197 old_val; \
198})
199
200#else /* CONFIG_HAVE_MARCH_Z196_FEATURES */
201
202#define __ATOMIC64_OR "ogr"
203#define __ATOMIC64_AND "ngr"
204#define __ATOMIC64_ADD "agr"
205#define __ATOMIC64_XOR "xgr"
206#define __ATOMIC64_BARRIER "\n"
207
208#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier) \
209({ \
210 long long old_val, new_val; \
211 \
212 typecheck(atomic64_t *, ptr); \
213 asm volatile( \
214 " lg %0,%2\n" \
215 "0: lgr %1,%0\n" \
216 op_string " %1,%3\n" \
217 " csg %0,%1,%2\n" \
218 " jl 0b" \
219 : "=&d" (old_val), "=&d" (new_val), "+Q" ((ptr)->counter)\
220 : "d" (op_val) \
221 : "cc", "memory"); \
222 old_val; \
223})
224
225#endif /* CONFIG_HAVE_MARCH_Z196_FEATURES */
226
227static inline long long atomic64_read(const atomic64_t *v)
228{
229 long long c;
230
231 asm volatile(
232 " lg %0,%1\n"
233 : "=d" (c) : "Q" (v->counter));
234 return c;
235}
236
237static inline void atomic64_set(atomic64_t *v, long long i)
238{
239 asm volatile(
240 " stg %1,%0\n"
241 : "=Q" (v->counter) : "d" (i));
242}
243
244static inline long long atomic64_add_return(long long i, atomic64_t *v)
245{
246 return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER) + i;
247}
248
249static inline long long atomic64_fetch_add(long long i, atomic64_t *v)
250{
251 return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER);
252}
253
254static inline void atomic64_add(long long i, atomic64_t *v)
255{
256#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
257 if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {
258 asm volatile(
259 "agsi %0,%1\n"
260 : "+Q" (v->counter)
261 : "i" (i)
262 : "cc", "memory");
263 return;
264 }
265#endif
266 __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_NO_BARRIER);
267}
268
269#define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
270
271static inline long long atomic64_cmpxchg(atomic64_t *v,
272 long long old, long long new)
273{
274 asm volatile(
275 " csg %0,%2,%1"
276 : "+d" (old), "+Q" (v->counter)
277 : "d" (new)
278 : "cc", "memory");
279 return old;
280}
281
282#define ATOMIC64_OPS(op, OP) \
283static inline void atomic64_##op(long i, atomic64_t *v) \
284{ \
285 __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_NO_BARRIER); \
286} \
287static inline long atomic64_fetch_##op(long i, atomic64_t *v) \
288{ \
289 return __ATOMIC64_LOOP(v, i, __ATOMIC64_##OP, __ATOMIC64_BARRIER); \
290}
291
292ATOMIC64_OPS(and, AND)
293ATOMIC64_OPS(or, OR)
294ATOMIC64_OPS(xor, XOR)
295
296#undef ATOMIC64_OPS
297#undef __ATOMIC64_LOOP
298
299static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u)
300{
301 long long c, old;
302
303 c = atomic64_read(v);
304 for (;;) {
305 if (unlikely(c == u))
306 break;
307 old = atomic64_cmpxchg(v, c, c + i);
308 if (likely(old == c))
309 break;
310 c = old;
311 }
312 return c != u;
313}
314
315static inline long long atomic64_dec_if_positive(atomic64_t *v)
316{
317 long long c, old, dec;
318
319 c = atomic64_read(v);
320 for (;;) {
321 dec = c - 1;
322 if (unlikely(dec < 0))
323 break;
324 old = atomic64_cmpxchg((v), c, dec);
325 if (likely(old == c))
326 break;
327 c = old;
328 }
329 return dec;
330}
331
332#define atomic64_add_negative(_i, _v) (atomic64_add_return(_i, _v) < 0)
333#define atomic64_inc(_v) atomic64_add(1, _v)
334#define atomic64_inc_return(_v) atomic64_add_return(1, _v)
335#define atomic64_inc_and_test(_v) (atomic64_add_return(1, _v) == 0)
336#define atomic64_sub_return(_i, _v) atomic64_add_return(-(long long)(_i), _v)
337#define atomic64_fetch_sub(_i, _v) atomic64_fetch_add(-(long long)(_i), _v)
338#define atomic64_sub(_i, _v) atomic64_add(-(long long)(_i), _v)
339#define atomic64_sub_and_test(_i, _v) (atomic64_sub_return(_i, _v) == 0)
340#define atomic64_dec(_v) atomic64_sub(1, _v)
341#define atomic64_dec_return(_v) atomic64_sub_return(1, _v)
342#define atomic64_dec_and_test(_v) (atomic64_sub_return(1, _v) == 0)
343#define atomic64_inc_not_zero(v) atomic64_add_unless((v), 1, 0)
344
345#endif /* __ARCH_S390_ATOMIC__ */