tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* atomic.S: These things are too big to do inline.
2 *
3 * Copyright (C) 1999, 2007 2012 David S. Miller (davem@davemloft.net)
4 */
5
6#include <linux/linkage.h>
7#include <asm/asi.h>
8#include <asm/backoff.h>
9#include <asm/export.h>
10
11 .text
12
13 /* Three versions of the atomic routines, one that
14 * does not return a value and does not perform
15 * memory barriers, and a two which return
16 * a value, the new and old value resp. and does the
17 * barriers.
18 */
19
20#define ATOMIC_OP(op) \
21ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
22 BACKOFF_SETUP(%o2); \
231: lduw [%o1], %g1; \
24 op %g1, %o0, %g7; \
25 cas [%o1], %g1, %g7; \
26 cmp %g1, %g7; \
27 bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
28 nop; \
29 retl; \
30 nop; \
312: BACKOFF_SPIN(%o2, %o3, 1b); \
32ENDPROC(atomic_##op); \
33EXPORT_SYMBOL(atomic_##op);
34
35#define ATOMIC_OP_RETURN(op) \
36ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \
37 BACKOFF_SETUP(%o2); \
381: lduw [%o1], %g1; \
39 op %g1, %o0, %g7; \
40 cas [%o1], %g1, %g7; \
41 cmp %g1, %g7; \
42 bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
43 op %g1, %o0, %g1; \
44 retl; \
45 sra %g1, 0, %o0; \
462: BACKOFF_SPIN(%o2, %o3, 1b); \
47ENDPROC(atomic_##op##_return); \
48EXPORT_SYMBOL(atomic_##op##_return);
49
50#define ATOMIC_FETCH_OP(op) \
51ENTRY(atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
52 BACKOFF_SETUP(%o2); \
531: lduw [%o1], %g1; \
54 op %g1, %o0, %g7; \
55 cas [%o1], %g1, %g7; \
56 cmp %g1, %g7; \
57 bne,pn %icc, BACKOFF_LABEL(2f, 1b); \
58 nop; \
59 retl; \
60 sra %g1, 0, %o0; \
612: BACKOFF_SPIN(%o2, %o3, 1b); \
62ENDPROC(atomic_fetch_##op); \
63EXPORT_SYMBOL(atomic_fetch_##op);
64
65#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op)
66
67ATOMIC_OPS(add)
68ATOMIC_OPS(sub)
69
70#undef ATOMIC_OPS
71#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op)
72
73ATOMIC_OPS(and)
74ATOMIC_OPS(or)
75ATOMIC_OPS(xor)
76
77#undef ATOMIC_OPS
78#undef ATOMIC_FETCH_OP
79#undef ATOMIC_OP_RETURN
80#undef ATOMIC_OP
81
82#define ATOMIC64_OP(op) \
83ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
84 BACKOFF_SETUP(%o2); \
851: ldx [%o1], %g1; \
86 op %g1, %o0, %g7; \
87 casx [%o1], %g1, %g7; \
88 cmp %g1, %g7; \
89 bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
90 nop; \
91 retl; \
92 nop; \
932: BACKOFF_SPIN(%o2, %o3, 1b); \
94ENDPROC(atomic64_##op); \
95EXPORT_SYMBOL(atomic64_##op);
96
97#define ATOMIC64_OP_RETURN(op) \
98ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \
99 BACKOFF_SETUP(%o2); \
1001: ldx [%o1], %g1; \
101 op %g1, %o0, %g7; \
102 casx [%o1], %g1, %g7; \
103 cmp %g1, %g7; \
104 bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
105 nop; \
106 retl; \
107 op %g1, %o0, %o0; \
1082: BACKOFF_SPIN(%o2, %o3, 1b); \
109ENDPROC(atomic64_##op##_return); \
110EXPORT_SYMBOL(atomic64_##op##_return);
111
112#define ATOMIC64_FETCH_OP(op) \
113ENTRY(atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \
114 BACKOFF_SETUP(%o2); \
1151: ldx [%o1], %g1; \
116 op %g1, %o0, %g7; \
117 casx [%o1], %g1, %g7; \
118 cmp %g1, %g7; \
119 bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \
120 nop; \
121 retl; \
122 mov %g1, %o0; \
1232: BACKOFF_SPIN(%o2, %o3, 1b); \
124ENDPROC(atomic64_fetch_##op); \
125EXPORT_SYMBOL(atomic64_fetch_##op);
126
127#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) ATOMIC64_FETCH_OP(op)
128
129ATOMIC64_OPS(add)
130ATOMIC64_OPS(sub)
131
132#undef ATOMIC64_OPS
133#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_FETCH_OP(op)
134
135ATOMIC64_OPS(and)
136ATOMIC64_OPS(or)
137ATOMIC64_OPS(xor)
138
139#undef ATOMIC64_OPS
140#undef ATOMIC64_FETCH_OP
141#undef ATOMIC64_OP_RETURN
142#undef ATOMIC64_OP
143
144ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */
145 BACKOFF_SETUP(%o2)
1461: ldx [%o0], %g1
147 brlez,pn %g1, 3f
148 sub %g1, 1, %g7
149 casx [%o0], %g1, %g7
150 cmp %g1, %g7
151 bne,pn %xcc, BACKOFF_LABEL(2f, 1b)
152 nop
1533: retl
154 sub %g1, 1, %o0
1552: BACKOFF_SPIN(%o2, %o3, 1b)
156ENDPROC(atomic64_dec_if_positive)
157EXPORT_SYMBOL(atomic64_dec_if_positive)