tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * Based on arch/arm/include/asm/atomic.h
3 *
4 * Copyright (C) 1996 Russell King.
5 * Copyright (C) 2002 Deep Blue Solutions Ltd.
6 * Copyright (C) 2012 ARM Ltd.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21#ifndef __ASM_ATOMIC_LSE_H
22#define __ASM_ATOMIC_LSE_H
23
24#ifndef __ARM64_IN_ATOMIC_IMPL
25#error "please don't include this file directly"
26#endif
27
28#define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op)
29#define ATOMIC_OP(op, asm_op) \
30static inline void atomic_##op(int i, atomic_t *v) \
31{ \
32 register int w0 asm ("w0") = i; \
33 register atomic_t *x1 asm ("x1") = v; \
34 \
35 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(op), \
36" " #asm_op " %w[i], %[v]\n") \
37 : [i] "+r" (w0), [v] "+Q" (v->counter) \
38 : "r" (x1) \
39 : __LL_SC_CLOBBERS); \
40}
41
42ATOMIC_OP(andnot, stclr)
43ATOMIC_OP(or, stset)
44ATOMIC_OP(xor, steor)
45ATOMIC_OP(add, stadd)
46
47#undef ATOMIC_OP
48
49#define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...) \
50static inline int atomic_fetch_##op##name(int i, atomic_t *v) \
51{ \
52 register int w0 asm ("w0") = i; \
53 register atomic_t *x1 asm ("x1") = v; \
54 \
55 asm volatile(ARM64_LSE_ATOMIC_INSN( \
56 /* LL/SC */ \
57 __LL_SC_ATOMIC(fetch_##op##name), \
58 /* LSE atomics */ \
59" " #asm_op #mb " %w[i], %w[i], %[v]") \
60 : [i] "+r" (w0), [v] "+Q" (v->counter) \
61 : "r" (x1) \
62 : __LL_SC_CLOBBERS, ##cl); \
63 \
64 return w0; \
65}
66
67#define ATOMIC_FETCH_OPS(op, asm_op) \
68 ATOMIC_FETCH_OP(_relaxed, , op, asm_op) \
69 ATOMIC_FETCH_OP(_acquire, a, op, asm_op, "memory") \
70 ATOMIC_FETCH_OP(_release, l, op, asm_op, "memory") \
71 ATOMIC_FETCH_OP( , al, op, asm_op, "memory")
72
73ATOMIC_FETCH_OPS(andnot, ldclr)
74ATOMIC_FETCH_OPS(or, ldset)
75ATOMIC_FETCH_OPS(xor, ldeor)
76ATOMIC_FETCH_OPS(add, ldadd)
77
78#undef ATOMIC_FETCH_OP
79#undef ATOMIC_FETCH_OPS
80
81#define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \
82static inline int atomic_add_return##name(int i, atomic_t *v) \
83{ \
84 register int w0 asm ("w0") = i; \
85 register atomic_t *x1 asm ("x1") = v; \
86 \
87 asm volatile(ARM64_LSE_ATOMIC_INSN( \
88 /* LL/SC */ \
89 __LL_SC_ATOMIC(add_return##name) \
90 __nops(1), \
91 /* LSE atomics */ \
92 " ldadd" #mb " %w[i], w30, %[v]\n" \
93 " add %w[i], %w[i], w30") \
94 : [i] "+r" (w0), [v] "+Q" (v->counter) \
95 : "r" (x1) \
96 : __LL_SC_CLOBBERS, ##cl); \
97 \
98 return w0; \
99}
100
101ATOMIC_OP_ADD_RETURN(_relaxed, )
102ATOMIC_OP_ADD_RETURN(_acquire, a, "memory")
103ATOMIC_OP_ADD_RETURN(_release, l, "memory")
104ATOMIC_OP_ADD_RETURN( , al, "memory")
105
106#undef ATOMIC_OP_ADD_RETURN
107
108static inline void atomic_and(int i, atomic_t *v)
109{
110 register int w0 asm ("w0") = i;
111 register atomic_t *x1 asm ("x1") = v;
112
113 asm volatile(ARM64_LSE_ATOMIC_INSN(
114 /* LL/SC */
115 __LL_SC_ATOMIC(and)
116 __nops(1),
117 /* LSE atomics */
118 " mvn %w[i], %w[i]\n"
119 " stclr %w[i], %[v]")
120 : [i] "+r" (w0), [v] "+Q" (v->counter)
121 : "r" (x1)
122 : __LL_SC_CLOBBERS);
123}
124
125#define ATOMIC_FETCH_OP_AND(name, mb, cl...) \
126static inline int atomic_fetch_and##name(int i, atomic_t *v) \
127{ \
128 register int w0 asm ("w0") = i; \
129 register atomic_t *x1 asm ("x1") = v; \
130 \
131 asm volatile(ARM64_LSE_ATOMIC_INSN( \
132 /* LL/SC */ \
133 __LL_SC_ATOMIC(fetch_and##name) \
134 __nops(1), \
135 /* LSE atomics */ \
136 " mvn %w[i], %w[i]\n" \
137 " ldclr" #mb " %w[i], %w[i], %[v]") \
138 : [i] "+r" (w0), [v] "+Q" (v->counter) \
139 : "r" (x1) \
140 : __LL_SC_CLOBBERS, ##cl); \
141 \
142 return w0; \
143}
144
145ATOMIC_FETCH_OP_AND(_relaxed, )
146ATOMIC_FETCH_OP_AND(_acquire, a, "memory")
147ATOMIC_FETCH_OP_AND(_release, l, "memory")
148ATOMIC_FETCH_OP_AND( , al, "memory")
149
150#undef ATOMIC_FETCH_OP_AND
151
152static inline void atomic_sub(int i, atomic_t *v)
153{
154 register int w0 asm ("w0") = i;
155 register atomic_t *x1 asm ("x1") = v;
156
157 asm volatile(ARM64_LSE_ATOMIC_INSN(
158 /* LL/SC */
159 __LL_SC_ATOMIC(sub)
160 __nops(1),
161 /* LSE atomics */
162 " neg %w[i], %w[i]\n"
163 " stadd %w[i], %[v]")
164 : [i] "+r" (w0), [v] "+Q" (v->counter)
165 : "r" (x1)
166 : __LL_SC_CLOBBERS);
167}
168
169#define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \
170static inline int atomic_sub_return##name(int i, atomic_t *v) \
171{ \
172 register int w0 asm ("w0") = i; \
173 register atomic_t *x1 asm ("x1") = v; \
174 \
175 asm volatile(ARM64_LSE_ATOMIC_INSN( \
176 /* LL/SC */ \
177 __LL_SC_ATOMIC(sub_return##name) \
178 __nops(2), \
179 /* LSE atomics */ \
180 " neg %w[i], %w[i]\n" \
181 " ldadd" #mb " %w[i], w30, %[v]\n" \
182 " add %w[i], %w[i], w30") \
183 : [i] "+r" (w0), [v] "+Q" (v->counter) \
184 : "r" (x1) \
185 : __LL_SC_CLOBBERS , ##cl); \
186 \
187 return w0; \
188}
189
190ATOMIC_OP_SUB_RETURN(_relaxed, )
191ATOMIC_OP_SUB_RETURN(_acquire, a, "memory")
192ATOMIC_OP_SUB_RETURN(_release, l, "memory")
193ATOMIC_OP_SUB_RETURN( , al, "memory")
194
195#undef ATOMIC_OP_SUB_RETURN
196
197#define ATOMIC_FETCH_OP_SUB(name, mb, cl...) \
198static inline int atomic_fetch_sub##name(int i, atomic_t *v) \
199{ \
200 register int w0 asm ("w0") = i; \
201 register atomic_t *x1 asm ("x1") = v; \
202 \
203 asm volatile(ARM64_LSE_ATOMIC_INSN( \
204 /* LL/SC */ \
205 __LL_SC_ATOMIC(fetch_sub##name) \
206 __nops(1), \
207 /* LSE atomics */ \
208 " neg %w[i], %w[i]\n" \
209 " ldadd" #mb " %w[i], %w[i], %[v]") \
210 : [i] "+r" (w0), [v] "+Q" (v->counter) \
211 : "r" (x1) \
212 : __LL_SC_CLOBBERS, ##cl); \
213 \
214 return w0; \
215}
216
217ATOMIC_FETCH_OP_SUB(_relaxed, )
218ATOMIC_FETCH_OP_SUB(_acquire, a, "memory")
219ATOMIC_FETCH_OP_SUB(_release, l, "memory")
220ATOMIC_FETCH_OP_SUB( , al, "memory")
221
222#undef ATOMIC_FETCH_OP_SUB
223#undef __LL_SC_ATOMIC
224
225#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op)
226#define ATOMIC64_OP(op, asm_op) \
227static inline void atomic64_##op(long i, atomic64_t *v) \
228{ \
229 register long x0 asm ("x0") = i; \
230 register atomic64_t *x1 asm ("x1") = v; \
231 \
232 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(op), \
233" " #asm_op " %[i], %[v]\n") \
234 : [i] "+r" (x0), [v] "+Q" (v->counter) \
235 : "r" (x1) \
236 : __LL_SC_CLOBBERS); \
237}
238
239ATOMIC64_OP(andnot, stclr)
240ATOMIC64_OP(or, stset)
241ATOMIC64_OP(xor, steor)
242ATOMIC64_OP(add, stadd)
243
244#undef ATOMIC64_OP
245
246#define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...) \
247static inline long atomic64_fetch_##op##name(long i, atomic64_t *v) \
248{ \
249 register long x0 asm ("x0") = i; \
250 register atomic64_t *x1 asm ("x1") = v; \
251 \
252 asm volatile(ARM64_LSE_ATOMIC_INSN( \
253 /* LL/SC */ \
254 __LL_SC_ATOMIC64(fetch_##op##name), \
255 /* LSE atomics */ \
256" " #asm_op #mb " %[i], %[i], %[v]") \
257 : [i] "+r" (x0), [v] "+Q" (v->counter) \
258 : "r" (x1) \
259 : __LL_SC_CLOBBERS, ##cl); \
260 \
261 return x0; \
262}
263
264#define ATOMIC64_FETCH_OPS(op, asm_op) \
265 ATOMIC64_FETCH_OP(_relaxed, , op, asm_op) \
266 ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \
267 ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \
268 ATOMIC64_FETCH_OP( , al, op, asm_op, "memory")
269
270ATOMIC64_FETCH_OPS(andnot, ldclr)
271ATOMIC64_FETCH_OPS(or, ldset)
272ATOMIC64_FETCH_OPS(xor, ldeor)
273ATOMIC64_FETCH_OPS(add, ldadd)
274
275#undef ATOMIC64_FETCH_OP
276#undef ATOMIC64_FETCH_OPS
277
278#define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \
279static inline long atomic64_add_return##name(long i, atomic64_t *v) \
280{ \
281 register long x0 asm ("x0") = i; \
282 register atomic64_t *x1 asm ("x1") = v; \
283 \
284 asm volatile(ARM64_LSE_ATOMIC_INSN( \
285 /* LL/SC */ \
286 __LL_SC_ATOMIC64(add_return##name) \
287 __nops(1), \
288 /* LSE atomics */ \
289 " ldadd" #mb " %[i], x30, %[v]\n" \
290 " add %[i], %[i], x30") \
291 : [i] "+r" (x0), [v] "+Q" (v->counter) \
292 : "r" (x1) \
293 : __LL_SC_CLOBBERS, ##cl); \
294 \
295 return x0; \
296}
297
298ATOMIC64_OP_ADD_RETURN(_relaxed, )
299ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory")
300ATOMIC64_OP_ADD_RETURN(_release, l, "memory")
301ATOMIC64_OP_ADD_RETURN( , al, "memory")
302
303#undef ATOMIC64_OP_ADD_RETURN
304
305static inline void atomic64_and(long i, atomic64_t *v)
306{
307 register long x0 asm ("x0") = i;
308 register atomic64_t *x1 asm ("x1") = v;
309
310 asm volatile(ARM64_LSE_ATOMIC_INSN(
311 /* LL/SC */
312 __LL_SC_ATOMIC64(and)
313 __nops(1),
314 /* LSE atomics */
315 " mvn %[i], %[i]\n"
316 " stclr %[i], %[v]")
317 : [i] "+r" (x0), [v] "+Q" (v->counter)
318 : "r" (x1)
319 : __LL_SC_CLOBBERS);
320}
321
322#define ATOMIC64_FETCH_OP_AND(name, mb, cl...) \
323static inline long atomic64_fetch_and##name(long i, atomic64_t *v) \
324{ \
325 register long x0 asm ("x0") = i; \
326 register atomic64_t *x1 asm ("x1") = v; \
327 \
328 asm volatile(ARM64_LSE_ATOMIC_INSN( \
329 /* LL/SC */ \
330 __LL_SC_ATOMIC64(fetch_and##name) \
331 __nops(1), \
332 /* LSE atomics */ \
333 " mvn %[i], %[i]\n" \
334 " ldclr" #mb " %[i], %[i], %[v]") \
335 : [i] "+r" (x0), [v] "+Q" (v->counter) \
336 : "r" (x1) \
337 : __LL_SC_CLOBBERS, ##cl); \
338 \
339 return x0; \
340}
341
342ATOMIC64_FETCH_OP_AND(_relaxed, )
343ATOMIC64_FETCH_OP_AND(_acquire, a, "memory")
344ATOMIC64_FETCH_OP_AND(_release, l, "memory")
345ATOMIC64_FETCH_OP_AND( , al, "memory")
346
347#undef ATOMIC64_FETCH_OP_AND
348
349static inline void atomic64_sub(long i, atomic64_t *v)
350{
351 register long x0 asm ("x0") = i;
352 register atomic64_t *x1 asm ("x1") = v;
353
354 asm volatile(ARM64_LSE_ATOMIC_INSN(
355 /* LL/SC */
356 __LL_SC_ATOMIC64(sub)
357 __nops(1),
358 /* LSE atomics */
359 " neg %[i], %[i]\n"
360 " stadd %[i], %[v]")
361 : [i] "+r" (x0), [v] "+Q" (v->counter)
362 : "r" (x1)
363 : __LL_SC_CLOBBERS);
364}
365
366#define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \
367static inline long atomic64_sub_return##name(long i, atomic64_t *v) \
368{ \
369 register long x0 asm ("x0") = i; \
370 register atomic64_t *x1 asm ("x1") = v; \
371 \
372 asm volatile(ARM64_LSE_ATOMIC_INSN( \
373 /* LL/SC */ \
374 __LL_SC_ATOMIC64(sub_return##name) \
375 __nops(2), \
376 /* LSE atomics */ \
377 " neg %[i], %[i]\n" \
378 " ldadd" #mb " %[i], x30, %[v]\n" \
379 " add %[i], %[i], x30") \
380 : [i] "+r" (x0), [v] "+Q" (v->counter) \
381 : "r" (x1) \
382 : __LL_SC_CLOBBERS, ##cl); \
383 \
384 return x0; \
385}
386
387ATOMIC64_OP_SUB_RETURN(_relaxed, )
388ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory")
389ATOMIC64_OP_SUB_RETURN(_release, l, "memory")
390ATOMIC64_OP_SUB_RETURN( , al, "memory")
391
392#undef ATOMIC64_OP_SUB_RETURN
393
394#define ATOMIC64_FETCH_OP_SUB(name, mb, cl...) \
395static inline long atomic64_fetch_sub##name(long i, atomic64_t *v) \
396{ \
397 register long x0 asm ("x0") = i; \
398 register atomic64_t *x1 asm ("x1") = v; \
399 \
400 asm volatile(ARM64_LSE_ATOMIC_INSN( \
401 /* LL/SC */ \
402 __LL_SC_ATOMIC64(fetch_sub##name) \
403 __nops(1), \
404 /* LSE atomics */ \
405 " neg %[i], %[i]\n" \
406 " ldadd" #mb " %[i], %[i], %[v]") \
407 : [i] "+r" (x0), [v] "+Q" (v->counter) \
408 : "r" (x1) \
409 : __LL_SC_CLOBBERS, ##cl); \
410 \
411 return x0; \
412}
413
414ATOMIC64_FETCH_OP_SUB(_relaxed, )
415ATOMIC64_FETCH_OP_SUB(_acquire, a, "memory")
416ATOMIC64_FETCH_OP_SUB(_release, l, "memory")
417ATOMIC64_FETCH_OP_SUB( , al, "memory")
418
419#undef ATOMIC64_FETCH_OP_SUB
420
421static inline long atomic64_dec_if_positive(atomic64_t *v)
422{
423 register long x0 asm ("x0") = (long)v;
424
425 asm volatile(ARM64_LSE_ATOMIC_INSN(
426 /* LL/SC */
427 __LL_SC_ATOMIC64(dec_if_positive)
428 __nops(6),
429 /* LSE atomics */
430 "1: ldr x30, %[v]\n"
431 " subs %[ret], x30, #1\n"
432 " b.lt 2f\n"
433 " casal x30, %[ret], %[v]\n"
434 " sub x30, x30, #1\n"
435 " sub x30, x30, %[ret]\n"
436 " cbnz x30, 1b\n"
437 "2:")
438 : [ret] "+&r" (x0), [v] "+Q" (v->counter)
439 :
440 : __LL_SC_CLOBBERS, "cc", "memory");
441
442 return x0;
443}
444
445#undef __LL_SC_ATOMIC64
446
447#define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op)
448
449#define __CMPXCHG_CASE(w, sz, name, mb, cl...) \
450static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
451 unsigned long old, \
452 unsigned long new) \
453{ \
454 register unsigned long x0 asm ("x0") = (unsigned long)ptr; \
455 register unsigned long x1 asm ("x1") = old; \
456 register unsigned long x2 asm ("x2") = new; \
457 \
458 asm volatile(ARM64_LSE_ATOMIC_INSN( \
459 /* LL/SC */ \
460 __LL_SC_CMPXCHG(name) \
461 __nops(2), \
462 /* LSE atomics */ \
463 " mov " #w "30, %" #w "[old]\n" \
464 " cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \
465 " mov %" #w "[ret], " #w "30") \
466 : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \
467 : [old] "r" (x1), [new] "r" (x2) \
468 : __LL_SC_CLOBBERS, ##cl); \
469 \
470 return x0; \
471}
472
473__CMPXCHG_CASE(w, b, 1, )
474__CMPXCHG_CASE(w, h, 2, )
475__CMPXCHG_CASE(w, , 4, )
476__CMPXCHG_CASE(x, , 8, )
477__CMPXCHG_CASE(w, b, acq_1, a, "memory")
478__CMPXCHG_CASE(w, h, acq_2, a, "memory")
479__CMPXCHG_CASE(w, , acq_4, a, "memory")
480__CMPXCHG_CASE(x, , acq_8, a, "memory")
481__CMPXCHG_CASE(w, b, rel_1, l, "memory")
482__CMPXCHG_CASE(w, h, rel_2, l, "memory")
483__CMPXCHG_CASE(w, , rel_4, l, "memory")
484__CMPXCHG_CASE(x, , rel_8, l, "memory")
485__CMPXCHG_CASE(w, b, mb_1, al, "memory")
486__CMPXCHG_CASE(w, h, mb_2, al, "memory")
487__CMPXCHG_CASE(w, , mb_4, al, "memory")
488__CMPXCHG_CASE(x, , mb_8, al, "memory")
489
490#undef __LL_SC_CMPXCHG
491#undef __CMPXCHG_CASE
492
493#define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op)
494
495#define __CMPXCHG_DBL(name, mb, cl...) \
496static inline long __cmpxchg_double##name(unsigned long old1, \
497 unsigned long old2, \
498 unsigned long new1, \
499 unsigned long new2, \
500 volatile void *ptr) \
501{ \
502 unsigned long oldval1 = old1; \
503 unsigned long oldval2 = old2; \
504 register unsigned long x0 asm ("x0") = old1; \
505 register unsigned long x1 asm ("x1") = old2; \
506 register unsigned long x2 asm ("x2") = new1; \
507 register unsigned long x3 asm ("x3") = new2; \
508 register unsigned long x4 asm ("x4") = (unsigned long)ptr; \
509 \
510 asm volatile(ARM64_LSE_ATOMIC_INSN( \
511 /* LL/SC */ \
512 __LL_SC_CMPXCHG_DBL(name) \
513 __nops(3), \
514 /* LSE atomics */ \
515 " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
516 " eor %[old1], %[old1], %[oldval1]\n" \
517 " eor %[old2], %[old2], %[oldval2]\n" \
518 " orr %[old1], %[old1], %[old2]") \
519 : [old1] "+r" (x0), [old2] "+r" (x1), \
520 [v] "+Q" (*(unsigned long *)ptr) \
521 : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
522 [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \
523 : __LL_SC_CLOBBERS, ##cl); \
524 \
525 return x0; \
526}
527
528__CMPXCHG_DBL( , )
529__CMPXCHG_DBL(_mb, al, "memory")
530
531#undef __LL_SC_CMPXCHG_DBL
532#undef __CMPXCHG_DBL
533
534#endif /* __ASM_ATOMIC_LSE_H */