Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Low level function for atomic operations
4 *
5 * Copyright IBM Corp. 1999, 2016
6 */
7
8#ifndef __ARCH_S390_ATOMIC_OPS__
9#define __ARCH_S390_ATOMIC_OPS__
10
11#include <linux/limits.h>
12#include <asm/march.h>
13#include <asm/asm.h>
14
15static __always_inline int __atomic_read(const int *ptr)
16{
17 int val;
18
19 asm volatile(
20 " l %[val],%[ptr]"
21 : [val] "=d" (val) : [ptr] "R" (*ptr));
22 return val;
23}
24
25static __always_inline void __atomic_set(int *ptr, int val)
26{
27 if (__builtin_constant_p(val) && val >= S16_MIN && val <= S16_MAX) {
28 asm volatile(
29 " mvhi %[ptr],%[val]"
30 : [ptr] "=Q" (*ptr) : [val] "K" (val));
31 } else {
32 asm volatile(
33 " st %[val],%[ptr]"
34 : [ptr] "=R" (*ptr) : [val] "d" (val));
35 }
36}
37
38static __always_inline long __atomic64_read(const long *ptr)
39{
40 long val;
41
42 asm volatile(
43 " lg %[val],%[ptr]"
44 : [val] "=d" (val) : [ptr] "RT" (*ptr));
45 return val;
46}
47
48static __always_inline void __atomic64_set(long *ptr, long val)
49{
50 if (__builtin_constant_p(val) && val >= S16_MIN && val <= S16_MAX) {
51 asm volatile(
52 " mvghi %[ptr],%[val]"
53 : [ptr] "=Q" (*ptr) : [val] "K" (val));
54 } else {
55 asm volatile(
56 " stg %[val],%[ptr]"
57 : [ptr] "=RT" (*ptr) : [val] "d" (val));
58 }
59}
60
61#ifdef MARCH_HAS_Z196_FEATURES
62
63#define __ATOMIC_OP(op_name, op_type, op_string, op_barrier) \
64static __always_inline op_type op_name(op_type val, op_type *ptr) \
65{ \
66 op_type old; \
67 \
68 asm volatile( \
69 op_string " %[old],%[val],%[ptr]" \
70 op_barrier \
71 : [old] "=d" (old), [ptr] "+QS" (*ptr) \
72 : [val] "d" (val) : "cc", "memory"); \
73 return old; \
74} \
75
76#define __ATOMIC_OPS(op_name, op_type, op_string) \
77 __ATOMIC_OP(op_name, op_type, op_string, "") \
78 __ATOMIC_OP(op_name##_barrier, op_type, op_string, "\nbcr 14,0")
79
80__ATOMIC_OPS(__atomic_add, int, "laa")
81__ATOMIC_OPS(__atomic_and, int, "lan")
82__ATOMIC_OPS(__atomic_or, int, "lao")
83__ATOMIC_OPS(__atomic_xor, int, "lax")
84
85__ATOMIC_OPS(__atomic64_add, long, "laag")
86__ATOMIC_OPS(__atomic64_and, long, "lang")
87__ATOMIC_OPS(__atomic64_or, long, "laog")
88__ATOMIC_OPS(__atomic64_xor, long, "laxg")
89
90#undef __ATOMIC_OPS
91#undef __ATOMIC_OP
92
93#define __ATOMIC_CONST_OP(op_name, op_type, op_string, op_barrier) \
94static __always_inline void op_name(op_type val, op_type *ptr) \
95{ \
96 asm volatile( \
97 op_string " %[ptr],%[val]" \
98 op_barrier \
99 : [ptr] "+QS" (*ptr) : [val] "i" (val) : "cc", "memory");\
100}
101
102#define __ATOMIC_CONST_OPS(op_name, op_type, op_string) \
103 __ATOMIC_CONST_OP(op_name, op_type, op_string, "") \
104 __ATOMIC_CONST_OP(op_name##_barrier, op_type, op_string, "\nbcr 14,0")
105
106__ATOMIC_CONST_OPS(__atomic_add_const, int, "asi")
107__ATOMIC_CONST_OPS(__atomic64_add_const, long, "agsi")
108
109#undef __ATOMIC_CONST_OPS
110#undef __ATOMIC_CONST_OP
111
112#else /* MARCH_HAS_Z196_FEATURES */
113
114#define __ATOMIC_OP(op_name, op_string) \
115static __always_inline int op_name(int val, int *ptr) \
116{ \
117 int old, new; \
118 \
119 asm volatile( \
120 "0: lr %[new],%[old]\n" \
121 op_string " %[new],%[val]\n" \
122 " cs %[old],%[new],%[ptr]\n" \
123 " jl 0b" \
124 : [old] "=d" (old), [new] "=&d" (new), [ptr] "+Q" (*ptr)\
125 : [val] "d" (val), "0" (*ptr) : "cc", "memory"); \
126 return old; \
127}
128
129#define __ATOMIC_OPS(op_name, op_string) \
130 __ATOMIC_OP(op_name, op_string) \
131 __ATOMIC_OP(op_name##_barrier, op_string)
132
133__ATOMIC_OPS(__atomic_add, "ar")
134__ATOMIC_OPS(__atomic_and, "nr")
135__ATOMIC_OPS(__atomic_or, "or")
136__ATOMIC_OPS(__atomic_xor, "xr")
137
138#undef __ATOMIC_OPS
139
140#define __ATOMIC64_OP(op_name, op_string) \
141static __always_inline long op_name(long val, long *ptr) \
142{ \
143 long old, new; \
144 \
145 asm volatile( \
146 "0: lgr %[new],%[old]\n" \
147 op_string " %[new],%[val]\n" \
148 " csg %[old],%[new],%[ptr]\n" \
149 " jl 0b" \
150 : [old] "=d" (old), [new] "=&d" (new), [ptr] "+QS" (*ptr)\
151 : [val] "d" (val), "0" (*ptr) : "cc", "memory"); \
152 return old; \
153}
154
155#define __ATOMIC64_OPS(op_name, op_string) \
156 __ATOMIC64_OP(op_name, op_string) \
157 __ATOMIC64_OP(op_name##_barrier, op_string)
158
159__ATOMIC64_OPS(__atomic64_add, "agr")
160__ATOMIC64_OPS(__atomic64_and, "ngr")
161__ATOMIC64_OPS(__atomic64_or, "ogr")
162__ATOMIC64_OPS(__atomic64_xor, "xgr")
163
164#undef __ATOMIC64_OPS
165
166#define __atomic_add_const(val, ptr) ((void)__atomic_add(val, ptr))
167#define __atomic_add_const_barrier(val, ptr) ((void)__atomic_add(val, ptr))
168#define __atomic64_add_const(val, ptr) ((void)__atomic64_add(val, ptr))
169#define __atomic64_add_const_barrier(val, ptr) ((void)__atomic64_add(val, ptr))
170
171#endif /* MARCH_HAS_Z196_FEATURES */
172
173#if defined(MARCH_HAS_Z196_FEATURES) && defined(__HAVE_ASM_FLAG_OUTPUTS__)
174
175#define __ATOMIC_TEST_OP(op_name, op_type, op_string, op_barrier) \
176static __always_inline bool op_name(op_type val, op_type *ptr) \
177{ \
178 op_type tmp; \
179 int cc; \
180 \
181 asm volatile( \
182 op_string " %[tmp],%[val],%[ptr]" \
183 op_barrier \
184 : "=@cc" (cc), [tmp] "=d" (tmp), [ptr] "+QS" (*ptr) \
185 : [val] "d" (val) \
186 : "memory"); \
187 return (cc == 0) || (cc == 2); \
188} \
189
190#define __ATOMIC_TEST_OPS(op_name, op_type, op_string) \
191 __ATOMIC_TEST_OP(op_name, op_type, op_string, "") \
192 __ATOMIC_TEST_OP(op_name##_barrier, op_type, op_string, "\nbcr 14,0")
193
194__ATOMIC_TEST_OPS(__atomic_add_and_test, int, "laal")
195__ATOMIC_TEST_OPS(__atomic64_add_and_test, long, "laalg")
196
197#undef __ATOMIC_TEST_OPS
198#undef __ATOMIC_TEST_OP
199
200#define __ATOMIC_CONST_TEST_OP(op_name, op_type, op_string, op_barrier) \
201static __always_inline bool op_name(op_type val, op_type *ptr) \
202{ \
203 int cc; \
204 \
205 asm volatile( \
206 op_string " %[ptr],%[val]" \
207 op_barrier \
208 : "=@cc" (cc), [ptr] "+QS" (*ptr) \
209 : [val] "i" (val) \
210 : "memory"); \
211 return (cc == 0) || (cc == 2); \
212}
213
214#define __ATOMIC_CONST_TEST_OPS(op_name, op_type, op_string) \
215 __ATOMIC_CONST_TEST_OP(op_name, op_type, op_string, "") \
216 __ATOMIC_CONST_TEST_OP(op_name##_barrier, op_type, op_string, "\nbcr 14,0")
217
218__ATOMIC_CONST_TEST_OPS(__atomic_add_const_and_test, int, "alsi")
219__ATOMIC_CONST_TEST_OPS(__atomic64_add_const_and_test, long, "algsi")
220
221#undef __ATOMIC_CONST_TEST_OPS
222#undef __ATOMIC_CONST_TEST_OP
223
224#else /* defined(MARCH_HAS_Z196_FEATURES) && defined(__HAVE_ASM_FLAG_OUTPUTS__) */
225
226#define __ATOMIC_TEST_OP(op_name, op_func, op_type) \
227static __always_inline bool op_name(op_type val, op_type *ptr) \
228{ \
229 return op_func(val, ptr) == -val; \
230}
231
232__ATOMIC_TEST_OP(__atomic_add_and_test, __atomic_add, int)
233__ATOMIC_TEST_OP(__atomic_add_and_test_barrier, __atomic_add_barrier, int)
234__ATOMIC_TEST_OP(__atomic_add_const_and_test, __atomic_add, int)
235__ATOMIC_TEST_OP(__atomic_add_const_and_test_barrier, __atomic_add_barrier, int)
236__ATOMIC_TEST_OP(__atomic64_add_and_test, __atomic64_add, long)
237__ATOMIC_TEST_OP(__atomic64_add_and_test_barrier, __atomic64_add_barrier, long)
238__ATOMIC_TEST_OP(__atomic64_add_const_and_test, __atomic64_add, long)
239__ATOMIC_TEST_OP(__atomic64_add_const_and_test_barrier, __atomic64_add_barrier, long)
240
241#undef __ATOMIC_TEST_OP
242
243#endif /* defined(MARCH_HAS_Z196_FEATURES) && defined(__HAVE_ASM_FLAG_OUTPUTS__) */
244
245#endif /* __ARCH_S390_ATOMIC_OPS__ */