tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright (C) 2014 Regents of the University of California
4 */
5
6#ifndef _ASM_RISCV_CMPXCHG_H
7#define _ASM_RISCV_CMPXCHG_H
8
9#include <linux/bug.h>
10
11#include <asm/fence.h>
12
13#define __arch_xchg_masked(sc_sfx, prepend, append, r, p, n) \
14({ \
15 u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \
16 ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \
17 ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \
18 << __s; \
19 ulong __newx = (ulong)(n) << __s; \
20 ulong __retx; \
21 ulong __rc; \
22 \
23 __asm__ __volatile__ ( \
24 prepend \
25 "0: lr.w %0, %2\n" \
26 " and %1, %0, %z4\n" \
27 " or %1, %1, %z3\n" \
28 " sc.w" sc_sfx " %1, %1, %2\n" \
29 " bnez %1, 0b\n" \
30 append \
31 : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
32 : "rJ" (__newx), "rJ" (~__mask) \
33 : "memory"); \
34 \
35 r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
36})
37
38#define __arch_xchg(sfx, prepend, append, r, p, n) \
39({ \
40 __asm__ __volatile__ ( \
41 prepend \
42 " amoswap" sfx " %0, %2, %1\n" \
43 append \
44 : "=r" (r), "+A" (*(p)) \
45 : "r" (n) \
46 : "memory"); \
47})
48
49#define _arch_xchg(ptr, new, sc_sfx, swap_sfx, prepend, \
50 sc_append, swap_append) \
51({ \
52 __typeof__(ptr) __ptr = (ptr); \
53 __typeof__(*(__ptr)) __new = (new); \
54 __typeof__(*(__ptr)) __ret; \
55 \
56 switch (sizeof(*__ptr)) { \
57 case 1: \
58 case 2: \
59 __arch_xchg_masked(sc_sfx, prepend, sc_append, \
60 __ret, __ptr, __new); \
61 break; \
62 case 4: \
63 __arch_xchg(".w" swap_sfx, prepend, swap_append, \
64 __ret, __ptr, __new); \
65 break; \
66 case 8: \
67 __arch_xchg(".d" swap_sfx, prepend, swap_append, \
68 __ret, __ptr, __new); \
69 break; \
70 default: \
71 BUILD_BUG(); \
72 } \
73 (__typeof__(*(__ptr)))__ret; \
74})
75
76#define arch_xchg_relaxed(ptr, x) \
77 _arch_xchg(ptr, x, "", "", "", "", "")
78
79#define arch_xchg_acquire(ptr, x) \
80 _arch_xchg(ptr, x, "", "", "", \
81 RISCV_ACQUIRE_BARRIER, RISCV_ACQUIRE_BARRIER)
82
83#define arch_xchg_release(ptr, x) \
84 _arch_xchg(ptr, x, "", "", RISCV_RELEASE_BARRIER, "", "")
85
86#define arch_xchg(ptr, x) \
87 _arch_xchg(ptr, x, ".rl", ".aqrl", "", RISCV_FULL_BARRIER, "")
88
89#define xchg32(ptr, x) \
90({ \
91 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \
92 arch_xchg((ptr), (x)); \
93})
94
95#define xchg64(ptr, x) \
96({ \
97 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
98 arch_xchg((ptr), (x)); \
99})
100
101/*
102 * Atomic compare and exchange. Compare OLD with MEM, if identical,
103 * store NEW in MEM. Return the initial value in MEM. Success is
104 * indicated by comparing RETURN with OLD.
105 */
106
107#define __arch_cmpxchg_masked(sc_sfx, prepend, append, r, p, o, n) \
108({ \
109 u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3); \
110 ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \
111 ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0) \
112 << __s; \
113 ulong __newx = (ulong)(n) << __s; \
114 ulong __oldx = (ulong)(o) << __s; \
115 ulong __retx; \
116 ulong __rc; \
117 \
118 __asm__ __volatile__ ( \
119 prepend \
120 "0: lr.w %0, %2\n" \
121 " and %1, %0, %z5\n" \
122 " bne %1, %z3, 1f\n" \
123 " and %1, %0, %z6\n" \
124 " or %1, %1, %z4\n" \
125 " sc.w" sc_sfx " %1, %1, %2\n" \
126 " bnez %1, 0b\n" \
127 append \
128 "1:\n" \
129 : "=&r" (__retx), "=&r" (__rc), "+A" (*(__ptr32b)) \
130 : "rJ" ((long)__oldx), "rJ" (__newx), \
131 "rJ" (__mask), "rJ" (~__mask) \
132 : "memory"); \
133 \
134 r = (__typeof__(*(p)))((__retx & __mask) >> __s); \
135})
136
137#define __arch_cmpxchg(lr_sfx, sc_sfx, prepend, append, r, p, co, o, n) \
138({ \
139 register unsigned int __rc; \
140 \
141 __asm__ __volatile__ ( \
142 prepend \
143 "0: lr" lr_sfx " %0, %2\n" \
144 " bne %0, %z3, 1f\n" \
145 " sc" sc_sfx " %1, %z4, %2\n" \
146 " bnez %1, 0b\n" \
147 append \
148 "1:\n" \
149 : "=&r" (r), "=&r" (__rc), "+A" (*(p)) \
150 : "rJ" (co o), "rJ" (n) \
151 : "memory"); \
152})
153
154#define _arch_cmpxchg(ptr, old, new, sc_sfx, prepend, append) \
155({ \
156 __typeof__(ptr) __ptr = (ptr); \
157 __typeof__(*(__ptr)) __old = (old); \
158 __typeof__(*(__ptr)) __new = (new); \
159 __typeof__(*(__ptr)) __ret; \
160 \
161 switch (sizeof(*__ptr)) { \
162 case 1: \
163 case 2: \
164 __arch_cmpxchg_masked(sc_sfx, prepend, append, \
165 __ret, __ptr, __old, __new); \
166 break; \
167 case 4: \
168 __arch_cmpxchg(".w", ".w" sc_sfx, prepend, append, \
169 __ret, __ptr, (long), __old, __new); \
170 break; \
171 case 8: \
172 __arch_cmpxchg(".d", ".d" sc_sfx, prepend, append, \
173 __ret, __ptr, /**/, __old, __new); \
174 break; \
175 default: \
176 BUILD_BUG(); \
177 } \
178 (__typeof__(*(__ptr)))__ret; \
179})
180
181#define arch_cmpxchg_relaxed(ptr, o, n) \
182 _arch_cmpxchg((ptr), (o), (n), "", "", "")
183
184#define arch_cmpxchg_acquire(ptr, o, n) \
185 _arch_cmpxchg((ptr), (o), (n), "", "", RISCV_ACQUIRE_BARRIER)
186
187#define arch_cmpxchg_release(ptr, o, n) \
188 _arch_cmpxchg((ptr), (o), (n), "", RISCV_RELEASE_BARRIER, "")
189
190#define arch_cmpxchg(ptr, o, n) \
191 _arch_cmpxchg((ptr), (o), (n), ".rl", "", " fence rw, rw\n")
192
193#define arch_cmpxchg_local(ptr, o, n) \
194 arch_cmpxchg_relaxed((ptr), (o), (n))
195
196#define arch_cmpxchg64(ptr, o, n) \
197({ \
198 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
199 arch_cmpxchg((ptr), (o), (n)); \
200})
201
202#define arch_cmpxchg64_local(ptr, o, n) \
203({ \
204 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
205 arch_cmpxchg_relaxed((ptr), (o), (n)); \
206})
207
208#define arch_cmpxchg64_relaxed(ptr, o, n) \
209({ \
210 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
211 arch_cmpxchg_relaxed((ptr), (o), (n)); \
212})
213
214#define arch_cmpxchg64_acquire(ptr, o, n) \
215({ \
216 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
217 arch_cmpxchg_acquire((ptr), (o), (n)); \
218})
219
220#define arch_cmpxchg64_release(ptr, o, n) \
221({ \
222 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
223 arch_cmpxchg_release((ptr), (o), (n)); \
224})
225
226#endif /* _ASM_RISCV_CMPXCHG_H */