tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
LoongArch: Add atomic operations for 32BIT/64BIT
LoongArch64 has both AMO and LL/SC instructions, while LoongArch32 only
has LL/SC intstructions. So we add a Kconfig option CPU_HAS_AMO here and
implement atomic operations (also including local operations and percpu
operations) for both 32BIT and 64BIT platforms.
Reviewed-by: Arnd Bergmann <arnd@arndb.de>
Signed-off-by: Jiaxun Yang <jiaxun.yang@flygoat.com>
Signed-off-by: Huacai Chen <chenhuacai@loongson.cn>
+413
-219
+4
arch/loongarch/Kconfig
+4
arch/loongarch/Kconfig
+206
arch/loongarch/include/asm/atomic-amo.h
+206
arch/loongarch/include/asm/atomic-amo.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* Atomic operations (AMO).
4
+
*
5
+
* Copyright (C) 2020-2025 Loongson Technology Corporation Limited
6
+
*/
7
+
8
+
#ifndef _ASM_ATOMIC_AMO_H
9
+
#define _ASM_ATOMIC_AMO_H
10
+
11
+
#include <linux/types.h>
12
+
#include <asm/barrier.h>
13
+
#include <asm/cmpxchg.h>
14
+
15
+
#define ATOMIC_OP(op, I, asm_op) \
16
+
static inline void arch_atomic_##op(int i, atomic_t *v) \
17
+
{ \
18
+
__asm__ __volatile__( \
19
+
"am"#asm_op".w" " $zero, %1, %0 \n" \
20
+
: "+ZB" (v->counter) \
21
+
: "r" (I) \
22
+
: "memory"); \
23
+
}
24
+
25
+
#define ATOMIC_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \
26
+
static inline int arch_atomic_##op##_return##suffix(int i, atomic_t *v) \
27
+
{ \
28
+
int result; \
29
+
\
30
+
__asm__ __volatile__( \
31
+
"am"#asm_op#mb".w" " %1, %2, %0 \n" \
32
+
: "+ZB" (v->counter), "=&r" (result) \
33
+
: "r" (I) \
34
+
: "memory"); \
35
+
\
36
+
return result c_op I; \
37
+
}
38
+
39
+
#define ATOMIC_FETCH_OP(op, I, asm_op, mb, suffix) \
40
+
static inline int arch_atomic_fetch_##op##suffix(int i, atomic_t *v) \
41
+
{ \
42
+
int result; \
43
+
\
44
+
__asm__ __volatile__( \
45
+
"am"#asm_op#mb".w" " %1, %2, %0 \n" \
46
+
: "+ZB" (v->counter), "=&r" (result) \
47
+
: "r" (I) \
48
+
: "memory"); \
49
+
\
50
+
return result; \
51
+
}
52
+
53
+
#define ATOMIC_OPS(op, I, asm_op, c_op) \
54
+
ATOMIC_OP(op, I, asm_op) \
55
+
ATOMIC_OP_RETURN(op, I, asm_op, c_op, _db, ) \
56
+
ATOMIC_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \
57
+
ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \
58
+
ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed)
59
+
60
+
ATOMIC_OPS(add, i, add, +)
61
+
ATOMIC_OPS(sub, -i, add, +)
62
+
63
+
#define arch_atomic_add_return arch_atomic_add_return
64
+
#define arch_atomic_add_return_acquire arch_atomic_add_return
65
+
#define arch_atomic_add_return_release arch_atomic_add_return
66
+
#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
67
+
#define arch_atomic_sub_return arch_atomic_sub_return
68
+
#define arch_atomic_sub_return_acquire arch_atomic_sub_return
69
+
#define arch_atomic_sub_return_release arch_atomic_sub_return
70
+
#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
71
+
#define arch_atomic_fetch_add arch_atomic_fetch_add
72
+
#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
73
+
#define arch_atomic_fetch_add_release arch_atomic_fetch_add
74
+
#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
75
+
#define arch_atomic_fetch_sub arch_atomic_fetch_sub
76
+
#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
77
+
#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
78
+
#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
79
+
80
+
#undef ATOMIC_OPS
81
+
82
+
#define ATOMIC_OPS(op, I, asm_op) \
83
+
ATOMIC_OP(op, I, asm_op) \
84
+
ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \
85
+
ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed)
86
+
87
+
ATOMIC_OPS(and, i, and)
88
+
ATOMIC_OPS(or, i, or)
89
+
ATOMIC_OPS(xor, i, xor)
90
+
91
+
#define arch_atomic_fetch_and arch_atomic_fetch_and
92
+
#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
93
+
#define arch_atomic_fetch_and_release arch_atomic_fetch_and
94
+
#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
95
+
#define arch_atomic_fetch_or arch_atomic_fetch_or
96
+
#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
97
+
#define arch_atomic_fetch_or_release arch_atomic_fetch_or
98
+
#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
99
+
#define arch_atomic_fetch_xor arch_atomic_fetch_xor
100
+
#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
101
+
#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
102
+
#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
103
+
104
+
#undef ATOMIC_OPS
105
+
#undef ATOMIC_FETCH_OP
106
+
#undef ATOMIC_OP_RETURN
107
+
#undef ATOMIC_OP
108
+
109
+
#ifdef CONFIG_64BIT
110
+
111
+
#define ATOMIC64_OP(op, I, asm_op) \
112
+
static inline void arch_atomic64_##op(long i, atomic64_t *v) \
113
+
{ \
114
+
__asm__ __volatile__( \
115
+
"am"#asm_op".d " " $zero, %1, %0 \n" \
116
+
: "+ZB" (v->counter) \
117
+
: "r" (I) \
118
+
: "memory"); \
119
+
}
120
+
121
+
#define ATOMIC64_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \
122
+
static inline long arch_atomic64_##op##_return##suffix(long i, atomic64_t *v) \
123
+
{ \
124
+
long result; \
125
+
__asm__ __volatile__( \
126
+
"am"#asm_op#mb".d " " %1, %2, %0 \n" \
127
+
: "+ZB" (v->counter), "=&r" (result) \
128
+
: "r" (I) \
129
+
: "memory"); \
130
+
\
131
+
return result c_op I; \
132
+
}
133
+
134
+
#define ATOMIC64_FETCH_OP(op, I, asm_op, mb, suffix) \
135
+
static inline long arch_atomic64_fetch_##op##suffix(long i, atomic64_t *v) \
136
+
{ \
137
+
long result; \
138
+
\
139
+
__asm__ __volatile__( \
140
+
"am"#asm_op#mb".d " " %1, %2, %0 \n" \
141
+
: "+ZB" (v->counter), "=&r" (result) \
142
+
: "r" (I) \
143
+
: "memory"); \
144
+
\
145
+
return result; \
146
+
}
147
+
148
+
#define ATOMIC64_OPS(op, I, asm_op, c_op) \
149
+
ATOMIC64_OP(op, I, asm_op) \
150
+
ATOMIC64_OP_RETURN(op, I, asm_op, c_op, _db, ) \
151
+
ATOMIC64_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \
152
+
ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \
153
+
ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed)
154
+
155
+
ATOMIC64_OPS(add, i, add, +)
156
+
ATOMIC64_OPS(sub, -i, add, +)
157
+
158
+
#define arch_atomic64_add_return arch_atomic64_add_return
159
+
#define arch_atomic64_add_return_acquire arch_atomic64_add_return
160
+
#define arch_atomic64_add_return_release arch_atomic64_add_return
161
+
#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
162
+
#define arch_atomic64_sub_return arch_atomic64_sub_return
163
+
#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
164
+
#define arch_atomic64_sub_return_release arch_atomic64_sub_return
165
+
#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
166
+
#define arch_atomic64_fetch_add arch_atomic64_fetch_add
167
+
#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
168
+
#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
169
+
#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
170
+
#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
171
+
#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
172
+
#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
173
+
#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
174
+
175
+
#undef ATOMIC64_OPS
176
+
177
+
#define ATOMIC64_OPS(op, I, asm_op) \
178
+
ATOMIC64_OP(op, I, asm_op) \
179
+
ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \
180
+
ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed)
181
+
182
+
ATOMIC64_OPS(and, i, and)
183
+
ATOMIC64_OPS(or, i, or)
184
+
ATOMIC64_OPS(xor, i, xor)
185
+
186
+
#define arch_atomic64_fetch_and arch_atomic64_fetch_and
187
+
#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
188
+
#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
189
+
#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
190
+
#define arch_atomic64_fetch_or arch_atomic64_fetch_or
191
+
#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
192
+
#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
193
+
#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed
194
+
#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
195
+
#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
196
+
#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
197
+
#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
198
+
199
+
#undef ATOMIC64_OPS
200
+
#undef ATOMIC64_FETCH_OP
201
+
#undef ATOMIC64_OP_RETURN
202
+
#undef ATOMIC64_OP
203
+
204
+
#endif
205
+
206
+
#endif /* _ASM_ATOMIC_AMO_H */
+100
arch/loongarch/include/asm/atomic-llsc.h
+100
arch/loongarch/include/asm/atomic-llsc.h
···
1
+
/* SPDX-License-Identifier: GPL-2.0 */
2
+
/*
3
+
* Atomic operations (LLSC).
4
+
*
5
+
* Copyright (C) 2024-2025 Loongson Technology Corporation Limited
6
+
*/
7
+
8
+
#ifndef _ASM_ATOMIC_LLSC_H
9
+
#define _ASM_ATOMIC_LLSC_H
10
+
11
+
#include <linux/types.h>
12
+
#include <asm/barrier.h>
13
+
#include <asm/cmpxchg.h>
14
+
15
+
#define ATOMIC_OP(op, I, asm_op) \
16
+
static inline void arch_atomic_##op(int i, atomic_t *v) \
17
+
{ \
18
+
int temp; \
19
+
\
20
+
__asm__ __volatile__( \
21
+
"1: ll.w %0, %1 #atomic_" #op " \n" \
22
+
" " #asm_op " %0, %0, %2 \n" \
23
+
" sc.w %0, %1 \n" \
24
+
" beq %0, $r0, 1b \n" \
25
+
:"=&r" (temp) , "+ZC"(v->counter) \
26
+
:"r" (I) \
27
+
); \
28
+
}
29
+
30
+
#define ATOMIC_OP_RETURN(op, I, asm_op) \
31
+
static inline int arch_atomic_##op##_return_relaxed(int i, atomic_t *v) \
32
+
{ \
33
+
int result, temp; \
34
+
\
35
+
__asm__ __volatile__( \
36
+
"1: ll.w %1, %2 # atomic_" #op "_return \n" \
37
+
" " #asm_op " %0, %1, %3 \n" \
38
+
" sc.w %0, %2 \n" \
39
+
" beq %0, $r0 ,1b \n" \
40
+
" " #asm_op " %0, %1, %3 \n" \
41
+
: "=&r" (result), "=&r" (temp), "+ZC"(v->counter) \
42
+
: "r" (I)); \
43
+
\
44
+
return result; \
45
+
}
46
+
47
+
#define ATOMIC_FETCH_OP(op, I, asm_op) \
48
+
static inline int arch_atomic_fetch_##op##_relaxed(int i, atomic_t *v) \
49
+
{ \
50
+
int result, temp; \
51
+
\
52
+
__asm__ __volatile__( \
53
+
"1: ll.w %1, %2 # atomic_fetch_" #op " \n" \
54
+
" " #asm_op " %0, %1, %3 \n" \
55
+
" sc.w %0, %2 \n" \
56
+
" beq %0, $r0 ,1b \n" \
57
+
" add.w %0, %1 ,$r0 \n" \
58
+
: "=&r" (result), "=&r" (temp), "+ZC" (v->counter) \
59
+
: "r" (I)); \
60
+
\
61
+
return result; \
62
+
}
63
+
64
+
#define ATOMIC_OPS(op,I ,asm_op, c_op) \
65
+
ATOMIC_OP(op, I, asm_op) \
66
+
ATOMIC_OP_RETURN(op, I , asm_op) \
67
+
ATOMIC_FETCH_OP(op, I, asm_op)
68
+
69
+
ATOMIC_OPS(add, i , add.w ,+=)
70
+
ATOMIC_OPS(sub, -i , add.w ,+=)
71
+
72
+
#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
73
+
#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
74
+
#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
75
+
#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
76
+
77
+
#undef ATOMIC_OPS
78
+
79
+
#define ATOMIC_OPS(op, I, asm_op) \
80
+
ATOMIC_OP(op, I, asm_op) \
81
+
ATOMIC_FETCH_OP(op, I, asm_op)
82
+
83
+
ATOMIC_OPS(and, i, and)
84
+
ATOMIC_OPS(or, i, or)
85
+
ATOMIC_OPS(xor, i, xor)
86
+
87
+
#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
88
+
#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
89
+
#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
90
+
91
+
#undef ATOMIC_OPS
92
+
#undef ATOMIC_FETCH_OP
93
+
#undef ATOMIC_OP_RETURN
94
+
#undef ATOMIC_OP
95
+
96
+
#ifdef CONFIG_64BIT
97
+
#error "64-bit LLSC atomic operations are not supported"
98
+
#endif
99
+
100
+
#endif /* _ASM_ATOMIC_LLSC_H */
+10
-187
arch/loongarch/include/asm/atomic.h
+10
-187
arch/loongarch/include/asm/atomic.h
···
11
11
#include <asm/barrier.h>
12
12
#include <asm/cmpxchg.h>
13
13
14
+
#ifdef CONFIG_CPU_HAS_AMO
15
+
#include <asm/atomic-amo.h>
16
+
#else
17
+
#include <asm/atomic-llsc.h>
18
+
#endif
19
+
20
+
#ifdef CONFIG_GENERIC_ATOMIC64
21
+
#include <asm-generic/atomic64.h>
22
+
#endif
23
+
14
24
#if __SIZEOF_LONG__ == 4
15
25
#define __LL "ll.w "
16
26
#define __SC "sc.w "
···
43
33
44
34
#define arch_atomic_read(v) READ_ONCE((v)->counter)
45
35
#define arch_atomic_set(v, i) WRITE_ONCE((v)->counter, (i))
46
-
47
-
#define ATOMIC_OP(op, I, asm_op) \
48
-
static inline void arch_atomic_##op(int i, atomic_t *v) \
49
-
{ \
50
-
__asm__ __volatile__( \
51
-
"am"#asm_op".w" " $zero, %1, %0 \n" \
52
-
: "+ZB" (v->counter) \
53
-
: "r" (I) \
54
-
: "memory"); \
55
-
}
56
-
57
-
#define ATOMIC_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \
58
-
static inline int arch_atomic_##op##_return##suffix(int i, atomic_t *v) \
59
-
{ \
60
-
int result; \
61
-
\
62
-
__asm__ __volatile__( \
63
-
"am"#asm_op#mb".w" " %1, %2, %0 \n" \
64
-
: "+ZB" (v->counter), "=&r" (result) \
65
-
: "r" (I) \
66
-
: "memory"); \
67
-
\
68
-
return result c_op I; \
69
-
}
70
-
71
-
#define ATOMIC_FETCH_OP(op, I, asm_op, mb, suffix) \
72
-
static inline int arch_atomic_fetch_##op##suffix(int i, atomic_t *v) \
73
-
{ \
74
-
int result; \
75
-
\
76
-
__asm__ __volatile__( \
77
-
"am"#asm_op#mb".w" " %1, %2, %0 \n" \
78
-
: "+ZB" (v->counter), "=&r" (result) \
79
-
: "r" (I) \
80
-
: "memory"); \
81
-
\
82
-
return result; \
83
-
}
84
-
85
-
#define ATOMIC_OPS(op, I, asm_op, c_op) \
86
-
ATOMIC_OP(op, I, asm_op) \
87
-
ATOMIC_OP_RETURN(op, I, asm_op, c_op, _db, ) \
88
-
ATOMIC_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \
89
-
ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \
90
-
ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed)
91
-
92
-
ATOMIC_OPS(add, i, add, +)
93
-
ATOMIC_OPS(sub, -i, add, +)
94
-
95
-
#define arch_atomic_add_return arch_atomic_add_return
96
-
#define arch_atomic_add_return_acquire arch_atomic_add_return
97
-
#define arch_atomic_add_return_release arch_atomic_add_return
98
-
#define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed
99
-
#define arch_atomic_sub_return arch_atomic_sub_return
100
-
#define arch_atomic_sub_return_acquire arch_atomic_sub_return
101
-
#define arch_atomic_sub_return_release arch_atomic_sub_return
102
-
#define arch_atomic_sub_return_relaxed arch_atomic_sub_return_relaxed
103
-
#define arch_atomic_fetch_add arch_atomic_fetch_add
104
-
#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add
105
-
#define arch_atomic_fetch_add_release arch_atomic_fetch_add
106
-
#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add_relaxed
107
-
#define arch_atomic_fetch_sub arch_atomic_fetch_sub
108
-
#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub
109
-
#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub
110
-
#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub_relaxed
111
-
112
-
#undef ATOMIC_OPS
113
-
114
-
#define ATOMIC_OPS(op, I, asm_op) \
115
-
ATOMIC_OP(op, I, asm_op) \
116
-
ATOMIC_FETCH_OP(op, I, asm_op, _db, ) \
117
-
ATOMIC_FETCH_OP(op, I, asm_op, , _relaxed)
118
-
119
-
ATOMIC_OPS(and, i, and)
120
-
ATOMIC_OPS(or, i, or)
121
-
ATOMIC_OPS(xor, i, xor)
122
-
123
-
#define arch_atomic_fetch_and arch_atomic_fetch_and
124
-
#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and
125
-
#define arch_atomic_fetch_and_release arch_atomic_fetch_and
126
-
#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and_relaxed
127
-
#define arch_atomic_fetch_or arch_atomic_fetch_or
128
-
#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or
129
-
#define arch_atomic_fetch_or_release arch_atomic_fetch_or
130
-
#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or_relaxed
131
-
#define arch_atomic_fetch_xor arch_atomic_fetch_xor
132
-
#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor
133
-
#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor
134
-
#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor_relaxed
135
-
136
-
#undef ATOMIC_OPS
137
-
#undef ATOMIC_FETCH_OP
138
-
#undef ATOMIC_OP_RETURN
139
-
#undef ATOMIC_OP
140
36
141
37
static inline int arch_atomic_fetch_add_unless(atomic_t *v, int a, int u)
142
38
{
···
109
193
110
194
#define arch_atomic64_read(v) READ_ONCE((v)->counter)
111
195
#define arch_atomic64_set(v, i) WRITE_ONCE((v)->counter, (i))
112
-
113
-
#define ATOMIC64_OP(op, I, asm_op) \
114
-
static inline void arch_atomic64_##op(long i, atomic64_t *v) \
115
-
{ \
116
-
__asm__ __volatile__( \
117
-
"am"#asm_op".d " " $zero, %1, %0 \n" \
118
-
: "+ZB" (v->counter) \
119
-
: "r" (I) \
120
-
: "memory"); \
121
-
}
122
-
123
-
#define ATOMIC64_OP_RETURN(op, I, asm_op, c_op, mb, suffix) \
124
-
static inline long arch_atomic64_##op##_return##suffix(long i, atomic64_t *v) \
125
-
{ \
126
-
long result; \
127
-
__asm__ __volatile__( \
128
-
"am"#asm_op#mb".d " " %1, %2, %0 \n" \
129
-
: "+ZB" (v->counter), "=&r" (result) \
130
-
: "r" (I) \
131
-
: "memory"); \
132
-
\
133
-
return result c_op I; \
134
-
}
135
-
136
-
#define ATOMIC64_FETCH_OP(op, I, asm_op, mb, suffix) \
137
-
static inline long arch_atomic64_fetch_##op##suffix(long i, atomic64_t *v) \
138
-
{ \
139
-
long result; \
140
-
\
141
-
__asm__ __volatile__( \
142
-
"am"#asm_op#mb".d " " %1, %2, %0 \n" \
143
-
: "+ZB" (v->counter), "=&r" (result) \
144
-
: "r" (I) \
145
-
: "memory"); \
146
-
\
147
-
return result; \
148
-
}
149
-
150
-
#define ATOMIC64_OPS(op, I, asm_op, c_op) \
151
-
ATOMIC64_OP(op, I, asm_op) \
152
-
ATOMIC64_OP_RETURN(op, I, asm_op, c_op, _db, ) \
153
-
ATOMIC64_OP_RETURN(op, I, asm_op, c_op, , _relaxed) \
154
-
ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \
155
-
ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed)
156
-
157
-
ATOMIC64_OPS(add, i, add, +)
158
-
ATOMIC64_OPS(sub, -i, add, +)
159
-
160
-
#define arch_atomic64_add_return arch_atomic64_add_return
161
-
#define arch_atomic64_add_return_acquire arch_atomic64_add_return
162
-
#define arch_atomic64_add_return_release arch_atomic64_add_return
163
-
#define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed
164
-
#define arch_atomic64_sub_return arch_atomic64_sub_return
165
-
#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return
166
-
#define arch_atomic64_sub_return_release arch_atomic64_sub_return
167
-
#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed
168
-
#define arch_atomic64_fetch_add arch_atomic64_fetch_add
169
-
#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add
170
-
#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add
171
-
#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed
172
-
#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub
173
-
#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub
174
-
#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub
175
-
#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed
176
-
177
-
#undef ATOMIC64_OPS
178
-
179
-
#define ATOMIC64_OPS(op, I, asm_op) \
180
-
ATOMIC64_OP(op, I, asm_op) \
181
-
ATOMIC64_FETCH_OP(op, I, asm_op, _db, ) \
182
-
ATOMIC64_FETCH_OP(op, I, asm_op, , _relaxed)
183
-
184
-
ATOMIC64_OPS(and, i, and)
185
-
ATOMIC64_OPS(or, i, or)
186
-
ATOMIC64_OPS(xor, i, xor)
187
-
188
-
#define arch_atomic64_fetch_and arch_atomic64_fetch_and
189
-
#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and
190
-
#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and
191
-
#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed
192
-
#define arch_atomic64_fetch_or arch_atomic64_fetch_or
193
-
#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or
194
-
#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or
195
-
#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed
196
-
#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor
197
-
#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor
198
-
#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor
199
-
#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed
200
-
201
-
#undef ATOMIC64_OPS
202
-
#undef ATOMIC64_FETCH_OP
203
-
#undef ATOMIC64_OP_RETURN
204
-
#undef ATOMIC64_OP
205
196
206
197
static inline long arch_atomic64_fetch_add_unless(atomic64_t *v, long a, long u)
207
198
{
+37
-11
arch/loongarch/include/asm/cmpxchg.h
+37
-11
arch/loongarch/include/asm/cmpxchg.h
···
9
9
#include <linux/build_bug.h>
10
10
#include <asm/barrier.h>
11
11
12
-
#define __xchg_asm(amswap_db, m, val) \
12
+
#define __xchg_amo_asm(amswap_db, m, val) \
13
13
({ \
14
-
__typeof(val) __ret; \
14
+
__typeof(val) __ret; \
15
15
\
16
-
__asm__ __volatile__ ( \
17
-
" "amswap_db" %1, %z2, %0 \n" \
18
-
: "+ZB" (*m), "=&r" (__ret) \
19
-
: "Jr" (val) \
20
-
: "memory"); \
16
+
__asm__ __volatile__ ( \
17
+
" "amswap_db" %1, %z2, %0 \n" \
18
+
: "+ZB" (*m), "=&r" (__ret) \
19
+
: "Jr" (val) \
20
+
: "memory"); \
21
21
\
22
-
__ret; \
22
+
__ret; \
23
+
})
24
+
25
+
#define __xchg_llsc_asm(ld, st, m, val) \
26
+
({ \
27
+
__typeof(val) __ret, __tmp; \
28
+
\
29
+
asm volatile ( \
30
+
"1: ll.w %0, %3 \n" \
31
+
" move %1, %z4 \n" \
32
+
" sc.w %1, %2 \n" \
33
+
" beqz %1, 1b \n" \
34
+
: "=&r" (__ret), "=&r" (__tmp), "=ZC" (*m) \
35
+
: "ZC" (*m), "Jr" (val) \
36
+
: "memory"); \
37
+
\
38
+
__ret; \
23
39
})
24
40
25
41
static inline unsigned int __xchg_small(volatile void *ptr, unsigned int val,
···
83
67
switch (size) {
84
68
case 1:
85
69
case 2:
86
-
return __xchg_small(ptr, x, size);
70
+
return __xchg_small((volatile void *)ptr, x, size);
87
71
88
72
case 4:
89
-
return __xchg_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x);
73
+
#ifdef CONFIG_CPU_HAS_AMO
74
+
return __xchg_amo_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x);
75
+
#else
76
+
return __xchg_llsc_asm("ll.w", "sc.w", (volatile u32 *)ptr, (u32)x);
77
+
#endif /* CONFIG_CPU_HAS_AMO */
90
78
79
+
#ifdef CONFIG_64BIT
91
80
case 8:
92
-
return __xchg_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x);
81
+
#ifdef CONFIG_CPU_HAS_AMO
82
+
return __xchg_amo_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x);
83
+
#else
84
+
return __xchg_llsc_asm("ll.d", "sc.d", (volatile u64 *)ptr, (u64)x);
85
+
#endif /* CONFIG_CPU_HAS_AMO */
86
+
#endif /* CONFIG_64BIT */
93
87
94
88
default:
95
89
BUILD_BUG();
+37
arch/loongarch/include/asm/local.h
+37
arch/loongarch/include/asm/local.h
···
8
8
#include <linux/percpu.h>
9
9
#include <linux/bitops.h>
10
10
#include <linux/atomic.h>
11
+
#include <asm/asm.h>
11
12
#include <asm/cmpxchg.h>
12
13
13
14
typedef struct {
···
28
27
/*
29
28
* Same as above, but return the result value
30
29
*/
30
+
#ifdef CONFIG_CPU_HAS_AMO
31
31
static inline long local_add_return(long i, local_t *l)
32
32
{
33
33
unsigned long result;
···
57
55
58
56
return result;
59
57
}
58
+
#else
59
+
static inline long local_add_return(long i, local_t *l)
60
+
{
61
+
unsigned long result, temp;
62
+
63
+
__asm__ __volatile__(
64
+
"1:" __LL "%1, %2 # local_add_return \n"
65
+
__stringify(LONG_ADD) " %0, %1, %3 \n"
66
+
__SC "%0, %2 \n"
67
+
" beq %0, $r0, 1b \n"
68
+
__stringify(LONG_ADD) " %0, %1, %3 \n"
69
+
: "=&r" (result), "=&r" (temp), "=ZC" (l->a.counter)
70
+
: "r" (i), "ZC" (l->a.counter)
71
+
: "memory");
72
+
73
+
return result;
74
+
}
75
+
76
+
static inline long local_sub_return(long i, local_t *l)
77
+
{
78
+
unsigned long result, temp;
79
+
80
+
__asm__ __volatile__(
81
+
"1:" __LL "%1, %2 # local_sub_return \n"
82
+
__stringify(LONG_SUB) " %0, %1, %3 \n"
83
+
__SC "%0, %2 \n"
84
+
" beq %0, $r0, 1b \n"
85
+
__stringify(LONG_SUB) " %0, %1, %3 \n"
86
+
: "=&r" (result), "=&r" (temp), "=ZC" (l->a.counter)
87
+
: "r" (i), "ZC" (l->a.counter)
88
+
: "memory");
89
+
90
+
return result;
91
+
}
92
+
#endif
60
93
61
94
static inline long local_cmpxchg(local_t *l, long old, long new)
62
95
{
+19
-21
arch/loongarch/include/asm/percpu.h
+19
-21
arch/loongarch/include/asm/percpu.h
···
36
36
__my_cpu_offset; \
37
37
})
38
38
39
+
#ifdef CONFIG_CPU_HAS_AMO
40
+
39
41
#define PERCPU_OP(op, asm_op, c_op) \
40
42
static __always_inline unsigned long __percpu_##op(void *ptr, \
41
43
unsigned long val, int size) \
···
70
68
PERCPU_OP(or, or, |)
71
69
#undef PERCPU_OP
72
70
73
-
static __always_inline unsigned long __percpu_xchg(void *ptr, unsigned long val, int size)
74
-
{
75
-
switch (size) {
76
-
case 1:
77
-
case 2:
78
-
return __xchg_small((volatile void *)ptr, val, size);
71
+
#endif
79
72
80
-
case 4:
81
-
return __xchg_asm("amswap.w", (volatile u32 *)ptr, (u32)val);
82
-
83
-
case 8:
84
-
return __xchg_asm("amswap.d", (volatile u64 *)ptr, (u64)val);
85
-
86
-
default:
87
-
BUILD_BUG();
88
-
}
89
-
90
-
return 0;
91
-
}
73
+
#ifdef CONFIG_64BIT
92
74
93
75
#define __pcpu_op_1(op) op ".b "
94
76
#define __pcpu_op_2(op) op ".h "
···
101
115
: "memory"); \
102
116
} while (0)
103
117
118
+
#endif
119
+
120
+
#define __percpu_xchg __arch_xchg
121
+
104
122
/* this_cpu_cmpxchg */
105
123
#define _protect_cmpxchg_local(pcp, o, n) \
106
124
({ \
···
125
135
__retval; \
126
136
})
127
137
138
+
#ifdef CONFIG_CPU_HAS_AMO
139
+
128
140
#define _percpu_add(pcp, val) \
129
141
_pcp_protect(__percpu_add, pcp, val)
130
142
···
137
145
138
146
#define _percpu_or(pcp, val) \
139
147
_pcp_protect(__percpu_or, pcp, val)
140
-
141
-
#define _percpu_xchg(pcp, val) ((typeof(pcp)) \
142
-
_pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
143
148
144
149
#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val)
145
150
#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val)
···
150
161
#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val)
151
162
#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val)
152
163
164
+
#endif
165
+
166
+
#ifdef CONFIG_64BIT
167
+
153
168
#define this_cpu_read_1(pcp) _percpu_read(1, pcp)
154
169
#define this_cpu_read_2(pcp) _percpu_read(2, pcp)
155
170
#define this_cpu_read_4(pcp) _percpu_read(4, pcp)
···
163
170
#define this_cpu_write_2(pcp, val) _percpu_write(2, pcp, val)
164
171
#define this_cpu_write_4(pcp, val) _percpu_write(4, pcp, val)
165
172
#define this_cpu_write_8(pcp, val) _percpu_write(8, pcp, val)
173
+
174
+
#endif
175
+
176
+
#define _percpu_xchg(pcp, val) ((typeof(pcp)) \
177
+
_pcp_protect(__percpu_xchg, pcp, (unsigned long)(val)))
166
178
167
179
#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val)
168
180
#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val)