tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/*
2 * Based on arch/arm/include/asm/atomic.h
3 *
4 * Copyright (C) 1996 Russell King.
5 * Copyright (C) 2002 Deep Blue Solutions Ltd.
6 * Copyright (C) 2012 ARM Ltd.
7 *
8 * This program is free software; you can redistribute it and/or modify
9 * it under the terms of the GNU General Public License version 2 as
10 * published by the Free Software Foundation.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
19 */
20
21#ifndef __ASM_ATOMIC_LSE_H
22#define __ASM_ATOMIC_LSE_H
23
24#ifndef __ARM64_IN_ATOMIC_IMPL
25#error "please don't include this file directly"
26#endif
27
28#define __LL_SC_ATOMIC(op) __LL_SC_CALL(atomic_##op)
29#define ATOMIC_OP(op, asm_op) \
30static inline void atomic_##op(int i, atomic_t *v) \
31{ \
32 register int w0 asm ("w0") = i; \
33 register atomic_t *x1 asm ("x1") = v; \
34 \
35 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC(op), \
36" " #asm_op " %w[i], %[v]\n") \
37 : [i] "+r" (w0), [v] "+Q" (v->counter) \
38 : "r" (x1) \
39 : __LL_SC_CLOBBERS); \
40}
41
42ATOMIC_OP(andnot, stclr)
43ATOMIC_OP(or, stset)
44ATOMIC_OP(xor, steor)
45ATOMIC_OP(add, stadd)
46
47#undef ATOMIC_OP
48
49#define ATOMIC_FETCH_OP(name, mb, op, asm_op, cl...) \
50static inline int atomic_fetch_##op##name(int i, atomic_t *v) \
51{ \
52 register int w0 asm ("w0") = i; \
53 register atomic_t *x1 asm ("x1") = v; \
54 \
55 asm volatile(ARM64_LSE_ATOMIC_INSN( \
56 /* LL/SC */ \
57 __LL_SC_ATOMIC(fetch_##op##name), \
58 /* LSE atomics */ \
59" " #asm_op #mb " %w[i], %w[i], %[v]") \
60 : [i] "+r" (w0), [v] "+Q" (v->counter) \
61 : "r" (x1) \
62 : __LL_SC_CLOBBERS, ##cl); \
63 \
64 return w0; \
65}
66
67#define ATOMIC_FETCH_OPS(op, asm_op) \
68 ATOMIC_FETCH_OP(_relaxed, , op, asm_op) \
69 ATOMIC_FETCH_OP(_acquire, a, op, asm_op, "memory") \
70 ATOMIC_FETCH_OP(_release, l, op, asm_op, "memory") \
71 ATOMIC_FETCH_OP( , al, op, asm_op, "memory")
72
73ATOMIC_FETCH_OPS(andnot, ldclr)
74ATOMIC_FETCH_OPS(or, ldset)
75ATOMIC_FETCH_OPS(xor, ldeor)
76ATOMIC_FETCH_OPS(add, ldadd)
77
78#undef ATOMIC_FETCH_OP
79#undef ATOMIC_FETCH_OPS
80
81#define ATOMIC_OP_ADD_RETURN(name, mb, cl...) \
82static inline int atomic_add_return##name(int i, atomic_t *v) \
83{ \
84 register int w0 asm ("w0") = i; \
85 register atomic_t *x1 asm ("x1") = v; \
86 \
87 asm volatile(ARM64_LSE_ATOMIC_INSN( \
88 /* LL/SC */ \
89 " nop\n" \
90 __LL_SC_ATOMIC(add_return##name), \
91 /* LSE atomics */ \
92 " ldadd" #mb " %w[i], w30, %[v]\n" \
93 " add %w[i], %w[i], w30") \
94 : [i] "+r" (w0), [v] "+Q" (v->counter) \
95 : "r" (x1) \
96 : __LL_SC_CLOBBERS, ##cl); \
97 \
98 return w0; \
99}
100
101ATOMIC_OP_ADD_RETURN(_relaxed, )
102ATOMIC_OP_ADD_RETURN(_acquire, a, "memory")
103ATOMIC_OP_ADD_RETURN(_release, l, "memory")
104ATOMIC_OP_ADD_RETURN( , al, "memory")
105
106#undef ATOMIC_OP_ADD_RETURN
107
108static inline void atomic_and(int i, atomic_t *v)
109{
110 register int w0 asm ("w0") = i;
111 register atomic_t *x1 asm ("x1") = v;
112
113 asm volatile(ARM64_LSE_ATOMIC_INSN(
114 /* LL/SC */
115 " nop\n"
116 __LL_SC_ATOMIC(and),
117 /* LSE atomics */
118 " mvn %w[i], %w[i]\n"
119 " stclr %w[i], %[v]")
120 : [i] "+r" (w0), [v] "+Q" (v->counter)
121 : "r" (x1)
122 : __LL_SC_CLOBBERS);
123}
124
125#define ATOMIC_FETCH_OP_AND(name, mb, cl...) \
126static inline int atomic_fetch_and##name(int i, atomic_t *v) \
127{ \
128 register int w0 asm ("w0") = i; \
129 register atomic_t *x1 asm ("x1") = v; \
130 \
131 asm volatile(ARM64_LSE_ATOMIC_INSN( \
132 /* LL/SC */ \
133 " nop\n" \
134 __LL_SC_ATOMIC(fetch_and##name), \
135 /* LSE atomics */ \
136 " mvn %w[i], %w[i]\n" \
137 " ldclr" #mb " %w[i], %w[i], %[v]") \
138 : [i] "+r" (w0), [v] "+Q" (v->counter) \
139 : "r" (x1) \
140 : __LL_SC_CLOBBERS, ##cl); \
141 \
142 return w0; \
143}
144
145ATOMIC_FETCH_OP_AND(_relaxed, )
146ATOMIC_FETCH_OP_AND(_acquire, a, "memory")
147ATOMIC_FETCH_OP_AND(_release, l, "memory")
148ATOMIC_FETCH_OP_AND( , al, "memory")
149
150#undef ATOMIC_FETCH_OP_AND
151
152static inline void atomic_sub(int i, atomic_t *v)
153{
154 register int w0 asm ("w0") = i;
155 register atomic_t *x1 asm ("x1") = v;
156
157 asm volatile(ARM64_LSE_ATOMIC_INSN(
158 /* LL/SC */
159 " nop\n"
160 __LL_SC_ATOMIC(sub),
161 /* LSE atomics */
162 " neg %w[i], %w[i]\n"
163 " stadd %w[i], %[v]")
164 : [i] "+r" (w0), [v] "+Q" (v->counter)
165 : "r" (x1)
166 : __LL_SC_CLOBBERS);
167}
168
169#define ATOMIC_OP_SUB_RETURN(name, mb, cl...) \
170static inline int atomic_sub_return##name(int i, atomic_t *v) \
171{ \
172 register int w0 asm ("w0") = i; \
173 register atomic_t *x1 asm ("x1") = v; \
174 \
175 asm volatile(ARM64_LSE_ATOMIC_INSN( \
176 /* LL/SC */ \
177 " nop\n" \
178 __LL_SC_ATOMIC(sub_return##name) \
179 " nop", \
180 /* LSE atomics */ \
181 " neg %w[i], %w[i]\n" \
182 " ldadd" #mb " %w[i], w30, %[v]\n" \
183 " add %w[i], %w[i], w30") \
184 : [i] "+r" (w0), [v] "+Q" (v->counter) \
185 : "r" (x1) \
186 : __LL_SC_CLOBBERS , ##cl); \
187 \
188 return w0; \
189}
190
191ATOMIC_OP_SUB_RETURN(_relaxed, )
192ATOMIC_OP_SUB_RETURN(_acquire, a, "memory")
193ATOMIC_OP_SUB_RETURN(_release, l, "memory")
194ATOMIC_OP_SUB_RETURN( , al, "memory")
195
196#undef ATOMIC_OP_SUB_RETURN
197
198#define ATOMIC_FETCH_OP_SUB(name, mb, cl...) \
199static inline int atomic_fetch_sub##name(int i, atomic_t *v) \
200{ \
201 register int w0 asm ("w0") = i; \
202 register atomic_t *x1 asm ("x1") = v; \
203 \
204 asm volatile(ARM64_LSE_ATOMIC_INSN( \
205 /* LL/SC */ \
206 " nop\n" \
207 __LL_SC_ATOMIC(fetch_sub##name), \
208 /* LSE atomics */ \
209 " neg %w[i], %w[i]\n" \
210 " ldadd" #mb " %w[i], %w[i], %[v]") \
211 : [i] "+r" (w0), [v] "+Q" (v->counter) \
212 : "r" (x1) \
213 : __LL_SC_CLOBBERS, ##cl); \
214 \
215 return w0; \
216}
217
218ATOMIC_FETCH_OP_SUB(_relaxed, )
219ATOMIC_FETCH_OP_SUB(_acquire, a, "memory")
220ATOMIC_FETCH_OP_SUB(_release, l, "memory")
221ATOMIC_FETCH_OP_SUB( , al, "memory")
222
223#undef ATOMIC_FETCH_OP_SUB
224#undef __LL_SC_ATOMIC
225
226#define __LL_SC_ATOMIC64(op) __LL_SC_CALL(atomic64_##op)
227#define ATOMIC64_OP(op, asm_op) \
228static inline void atomic64_##op(long i, atomic64_t *v) \
229{ \
230 register long x0 asm ("x0") = i; \
231 register atomic64_t *x1 asm ("x1") = v; \
232 \
233 asm volatile(ARM64_LSE_ATOMIC_INSN(__LL_SC_ATOMIC64(op), \
234" " #asm_op " %[i], %[v]\n") \
235 : [i] "+r" (x0), [v] "+Q" (v->counter) \
236 : "r" (x1) \
237 : __LL_SC_CLOBBERS); \
238}
239
240ATOMIC64_OP(andnot, stclr)
241ATOMIC64_OP(or, stset)
242ATOMIC64_OP(xor, steor)
243ATOMIC64_OP(add, stadd)
244
245#undef ATOMIC64_OP
246
247#define ATOMIC64_FETCH_OP(name, mb, op, asm_op, cl...) \
248static inline long atomic64_fetch_##op##name(long i, atomic64_t *v) \
249{ \
250 register long x0 asm ("x0") = i; \
251 register atomic64_t *x1 asm ("x1") = v; \
252 \
253 asm volatile(ARM64_LSE_ATOMIC_INSN( \
254 /* LL/SC */ \
255 __LL_SC_ATOMIC64(fetch_##op##name), \
256 /* LSE atomics */ \
257" " #asm_op #mb " %[i], %[i], %[v]") \
258 : [i] "+r" (x0), [v] "+Q" (v->counter) \
259 : "r" (x1) \
260 : __LL_SC_CLOBBERS, ##cl); \
261 \
262 return x0; \
263}
264
265#define ATOMIC64_FETCH_OPS(op, asm_op) \
266 ATOMIC64_FETCH_OP(_relaxed, , op, asm_op) \
267 ATOMIC64_FETCH_OP(_acquire, a, op, asm_op, "memory") \
268 ATOMIC64_FETCH_OP(_release, l, op, asm_op, "memory") \
269 ATOMIC64_FETCH_OP( , al, op, asm_op, "memory")
270
271ATOMIC64_FETCH_OPS(andnot, ldclr)
272ATOMIC64_FETCH_OPS(or, ldset)
273ATOMIC64_FETCH_OPS(xor, ldeor)
274ATOMIC64_FETCH_OPS(add, ldadd)
275
276#undef ATOMIC64_FETCH_OP
277#undef ATOMIC64_FETCH_OPS
278
279#define ATOMIC64_OP_ADD_RETURN(name, mb, cl...) \
280static inline long atomic64_add_return##name(long i, atomic64_t *v) \
281{ \
282 register long x0 asm ("x0") = i; \
283 register atomic64_t *x1 asm ("x1") = v; \
284 \
285 asm volatile(ARM64_LSE_ATOMIC_INSN( \
286 /* LL/SC */ \
287 " nop\n" \
288 __LL_SC_ATOMIC64(add_return##name), \
289 /* LSE atomics */ \
290 " ldadd" #mb " %[i], x30, %[v]\n" \
291 " add %[i], %[i], x30") \
292 : [i] "+r" (x0), [v] "+Q" (v->counter) \
293 : "r" (x1) \
294 : __LL_SC_CLOBBERS, ##cl); \
295 \
296 return x0; \
297}
298
299ATOMIC64_OP_ADD_RETURN(_relaxed, )
300ATOMIC64_OP_ADD_RETURN(_acquire, a, "memory")
301ATOMIC64_OP_ADD_RETURN(_release, l, "memory")
302ATOMIC64_OP_ADD_RETURN( , al, "memory")
303
304#undef ATOMIC64_OP_ADD_RETURN
305
306static inline void atomic64_and(long i, atomic64_t *v)
307{
308 register long x0 asm ("x0") = i;
309 register atomic64_t *x1 asm ("x1") = v;
310
311 asm volatile(ARM64_LSE_ATOMIC_INSN(
312 /* LL/SC */
313 " nop\n"
314 __LL_SC_ATOMIC64(and),
315 /* LSE atomics */
316 " mvn %[i], %[i]\n"
317 " stclr %[i], %[v]")
318 : [i] "+r" (x0), [v] "+Q" (v->counter)
319 : "r" (x1)
320 : __LL_SC_CLOBBERS);
321}
322
323#define ATOMIC64_FETCH_OP_AND(name, mb, cl...) \
324static inline long atomic64_fetch_and##name(long i, atomic64_t *v) \
325{ \
326 register long x0 asm ("w0") = i; \
327 register atomic64_t *x1 asm ("x1") = v; \
328 \
329 asm volatile(ARM64_LSE_ATOMIC_INSN( \
330 /* LL/SC */ \
331 " nop\n" \
332 __LL_SC_ATOMIC64(fetch_and##name), \
333 /* LSE atomics */ \
334 " mvn %[i], %[i]\n" \
335 " ldclr" #mb " %[i], %[i], %[v]") \
336 : [i] "+r" (x0), [v] "+Q" (v->counter) \
337 : "r" (x1) \
338 : __LL_SC_CLOBBERS, ##cl); \
339 \
340 return x0; \
341}
342
343ATOMIC64_FETCH_OP_AND(_relaxed, )
344ATOMIC64_FETCH_OP_AND(_acquire, a, "memory")
345ATOMIC64_FETCH_OP_AND(_release, l, "memory")
346ATOMIC64_FETCH_OP_AND( , al, "memory")
347
348#undef ATOMIC64_FETCH_OP_AND
349
350static inline void atomic64_sub(long i, atomic64_t *v)
351{
352 register long x0 asm ("x0") = i;
353 register atomic64_t *x1 asm ("x1") = v;
354
355 asm volatile(ARM64_LSE_ATOMIC_INSN(
356 /* LL/SC */
357 " nop\n"
358 __LL_SC_ATOMIC64(sub),
359 /* LSE atomics */
360 " neg %[i], %[i]\n"
361 " stadd %[i], %[v]")
362 : [i] "+r" (x0), [v] "+Q" (v->counter)
363 : "r" (x1)
364 : __LL_SC_CLOBBERS);
365}
366
367#define ATOMIC64_OP_SUB_RETURN(name, mb, cl...) \
368static inline long atomic64_sub_return##name(long i, atomic64_t *v) \
369{ \
370 register long x0 asm ("x0") = i; \
371 register atomic64_t *x1 asm ("x1") = v; \
372 \
373 asm volatile(ARM64_LSE_ATOMIC_INSN( \
374 /* LL/SC */ \
375 " nop\n" \
376 __LL_SC_ATOMIC64(sub_return##name) \
377 " nop", \
378 /* LSE atomics */ \
379 " neg %[i], %[i]\n" \
380 " ldadd" #mb " %[i], x30, %[v]\n" \
381 " add %[i], %[i], x30") \
382 : [i] "+r" (x0), [v] "+Q" (v->counter) \
383 : "r" (x1) \
384 : __LL_SC_CLOBBERS, ##cl); \
385 \
386 return x0; \
387}
388
389ATOMIC64_OP_SUB_RETURN(_relaxed, )
390ATOMIC64_OP_SUB_RETURN(_acquire, a, "memory")
391ATOMIC64_OP_SUB_RETURN(_release, l, "memory")
392ATOMIC64_OP_SUB_RETURN( , al, "memory")
393
394#undef ATOMIC64_OP_SUB_RETURN
395
396#define ATOMIC64_FETCH_OP_SUB(name, mb, cl...) \
397static inline long atomic64_fetch_sub##name(long i, atomic64_t *v) \
398{ \
399 register long x0 asm ("w0") = i; \
400 register atomic64_t *x1 asm ("x1") = v; \
401 \
402 asm volatile(ARM64_LSE_ATOMIC_INSN( \
403 /* LL/SC */ \
404 " nop\n" \
405 __LL_SC_ATOMIC64(fetch_sub##name), \
406 /* LSE atomics */ \
407 " neg %[i], %[i]\n" \
408 " ldadd" #mb " %[i], %[i], %[v]") \
409 : [i] "+r" (x0), [v] "+Q" (v->counter) \
410 : "r" (x1) \
411 : __LL_SC_CLOBBERS, ##cl); \
412 \
413 return x0; \
414}
415
416ATOMIC64_FETCH_OP_SUB(_relaxed, )
417ATOMIC64_FETCH_OP_SUB(_acquire, a, "memory")
418ATOMIC64_FETCH_OP_SUB(_release, l, "memory")
419ATOMIC64_FETCH_OP_SUB( , al, "memory")
420
421#undef ATOMIC64_FETCH_OP_SUB
422
423static inline long atomic64_dec_if_positive(atomic64_t *v)
424{
425 register long x0 asm ("x0") = (long)v;
426
427 asm volatile(ARM64_LSE_ATOMIC_INSN(
428 /* LL/SC */
429 " nop\n"
430 __LL_SC_ATOMIC64(dec_if_positive)
431 " nop\n"
432 " nop\n"
433 " nop\n"
434 " nop\n"
435 " nop",
436 /* LSE atomics */
437 "1: ldr x30, %[v]\n"
438 " subs %[ret], x30, #1\n"
439 " b.lt 2f\n"
440 " casal x30, %[ret], %[v]\n"
441 " sub x30, x30, #1\n"
442 " sub x30, x30, %[ret]\n"
443 " cbnz x30, 1b\n"
444 "2:")
445 : [ret] "+&r" (x0), [v] "+Q" (v->counter)
446 :
447 : __LL_SC_CLOBBERS, "cc", "memory");
448
449 return x0;
450}
451
452#undef __LL_SC_ATOMIC64
453
454#define __LL_SC_CMPXCHG(op) __LL_SC_CALL(__cmpxchg_case_##op)
455
456#define __CMPXCHG_CASE(w, sz, name, mb, cl...) \
457static inline unsigned long __cmpxchg_case_##name(volatile void *ptr, \
458 unsigned long old, \
459 unsigned long new) \
460{ \
461 register unsigned long x0 asm ("x0") = (unsigned long)ptr; \
462 register unsigned long x1 asm ("x1") = old; \
463 register unsigned long x2 asm ("x2") = new; \
464 \
465 asm volatile(ARM64_LSE_ATOMIC_INSN( \
466 /* LL/SC */ \
467 " nop\n" \
468 __LL_SC_CMPXCHG(name) \
469 " nop", \
470 /* LSE atomics */ \
471 " mov " #w "30, %" #w "[old]\n" \
472 " cas" #mb #sz "\t" #w "30, %" #w "[new], %[v]\n" \
473 " mov %" #w "[ret], " #w "30") \
474 : [ret] "+r" (x0), [v] "+Q" (*(unsigned long *)ptr) \
475 : [old] "r" (x1), [new] "r" (x2) \
476 : __LL_SC_CLOBBERS, ##cl); \
477 \
478 return x0; \
479}
480
481__CMPXCHG_CASE(w, b, 1, )
482__CMPXCHG_CASE(w, h, 2, )
483__CMPXCHG_CASE(w, , 4, )
484__CMPXCHG_CASE(x, , 8, )
485__CMPXCHG_CASE(w, b, acq_1, a, "memory")
486__CMPXCHG_CASE(w, h, acq_2, a, "memory")
487__CMPXCHG_CASE(w, , acq_4, a, "memory")
488__CMPXCHG_CASE(x, , acq_8, a, "memory")
489__CMPXCHG_CASE(w, b, rel_1, l, "memory")
490__CMPXCHG_CASE(w, h, rel_2, l, "memory")
491__CMPXCHG_CASE(w, , rel_4, l, "memory")
492__CMPXCHG_CASE(x, , rel_8, l, "memory")
493__CMPXCHG_CASE(w, b, mb_1, al, "memory")
494__CMPXCHG_CASE(w, h, mb_2, al, "memory")
495__CMPXCHG_CASE(w, , mb_4, al, "memory")
496__CMPXCHG_CASE(x, , mb_8, al, "memory")
497
498#undef __LL_SC_CMPXCHG
499#undef __CMPXCHG_CASE
500
501#define __LL_SC_CMPXCHG_DBL(op) __LL_SC_CALL(__cmpxchg_double##op)
502
503#define __CMPXCHG_DBL(name, mb, cl...) \
504static inline long __cmpxchg_double##name(unsigned long old1, \
505 unsigned long old2, \
506 unsigned long new1, \
507 unsigned long new2, \
508 volatile void *ptr) \
509{ \
510 unsigned long oldval1 = old1; \
511 unsigned long oldval2 = old2; \
512 register unsigned long x0 asm ("x0") = old1; \
513 register unsigned long x1 asm ("x1") = old2; \
514 register unsigned long x2 asm ("x2") = new1; \
515 register unsigned long x3 asm ("x3") = new2; \
516 register unsigned long x4 asm ("x4") = (unsigned long)ptr; \
517 \
518 asm volatile(ARM64_LSE_ATOMIC_INSN( \
519 /* LL/SC */ \
520 " nop\n" \
521 " nop\n" \
522 " nop\n" \
523 __LL_SC_CMPXCHG_DBL(name), \
524 /* LSE atomics */ \
525 " casp" #mb "\t%[old1], %[old2], %[new1], %[new2], %[v]\n"\
526 " eor %[old1], %[old1], %[oldval1]\n" \
527 " eor %[old2], %[old2], %[oldval2]\n" \
528 " orr %[old1], %[old1], %[old2]") \
529 : [old1] "+r" (x0), [old2] "+r" (x1), \
530 [v] "+Q" (*(unsigned long *)ptr) \
531 : [new1] "r" (x2), [new2] "r" (x3), [ptr] "r" (x4), \
532 [oldval1] "r" (oldval1), [oldval2] "r" (oldval2) \
533 : __LL_SC_CLOBBERS, ##cl); \
534 \
535 return x0; \
536}
537
538__CMPXCHG_DBL( , )
539__CMPXCHG_DBL(_mb, al, "memory")
540
541#undef __LL_SC_CMPXCHG_DBL
542#undef __CMPXCHG_DBL
543
544#endif /* __ASM_ATOMIC_LSE_H */