tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* atomic.S: These things are too big to do inline.
2 *
3 * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net)
4 */
5
6#include <linux/linkage.h>
7#include <asm/asi.h>
8#include <asm/backoff.h>
9
10 .text
11
12 /* Two versions of the atomic routines, one that
13 * does not return a value and does not perform
14 * memory barriers, and a second which returns
15 * a value and does the barriers.
16 */
17
18#define ATOMIC_OP(op) \
19ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
20 BACKOFF_SETUP(%o2); \
211: lduw [%o1], %g1; \
22 op %g1, %o0, %g7; \
23 cas [%o1], %g1, %g7; \
24 cmp %g1, %g7; \
25 bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
26 nop; \
27 retl; \
28 nop; \
292: BACKOFF_SPIN(%o2, %o3, 1b); \
30ENDPROC(atomic_##op); \
31
32#define ATOMIC_OP_RETURN(op) \
33ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \
34 BACKOFF_SETUP(%o2); \
351: lduw [%o1], %g1; \
36 op %g1, %o0, %g7; \
37 cas [%o1], %g1, %g7; \
38 cmp %g1, %g7; \
39 bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
40 op %g1, %o0, %g1; \
41 retl; \
42 sra %g1, 0, %o0; \
432: BACKOFF_SPIN(%o2, %o3, 1b); \
44ENDPROC(atomic_##op##_return);
45
46#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op)
47
48ATOMIC_OPS(add)
49ATOMIC_OPS(sub)
50
51#undef ATOMIC_OPS
52#undef ATOMIC_OP_RETURN
53#undef ATOMIC_OP
54
55#define ATOMIC64_OP(op) \
56ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
57 BACKOFF_SETUP(%o2); \
581: ldx [%o1], %g1; \
59 op %g1, %o0, %g7; \
60 casx [%o1], %g1, %g7; \
61 cmp %g1, %g7; \
62 bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
63 nop; \
64 retl; \
65 nop; \
662: BACKOFF_SPIN(%o2, %o3, 1b); \
67ENDPROC(atomic64_##op); \
68
69#define ATOMIC64_OP_RETURN(op) \
70ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \
71 BACKOFF_SETUP(%o2); \
721: ldx [%o1], %g1; \
73 op %g1, %o0, %g7; \
74 casx [%o1], %g1, %g7; \
75 cmp %g1, %g7; \
76 bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
77 nop; \
78 retl; \
79 op %g1, %o0, %o0; \
802: BACKOFF_SPIN(%o2, %o3, 1b); \
81ENDPROC(atomic64_##op##_return);
82
83#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op)
84
85ATOMIC64_OPS(add)
86ATOMIC64_OPS(sub)
87
88#undef ATOMIC64_OPS
89#undef ATOMIC64_OP_RETURN
90#undef ATOMIC64_OP
91
92ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */
93 BACKOFF_SETUP(%o2)
941: ldx [%o0], %g1
95 brlez,pn %g1, 3f
96 sub %g1, 1, %g7
97 casx [%o0], %g1, %g7
98 cmp %g1, %g7
99 bne,pn %xcc, BACKOFF_LABEL(2f, 1b)
100 nop
1013: retl
102 sub %g1, 1, %o0
1032: BACKOFF_SPIN(%o2, %o3, 1b)
104ENDPROC(atomic64_dec_if_positive)