Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_X86_PERCPU_H
3#define _ASM_X86_PERCPU_H
4
5#ifdef CONFIG_X86_64
6# define __percpu_seg gs
7# define __percpu_rel (%rip)
8#else
9# define __percpu_seg fs
10# define __percpu_rel
11#endif
12
13#ifdef __ASSEMBLER__
14
15#ifdef CONFIG_SMP
16# define __percpu %__percpu_seg:
17#else
18# define __percpu
19#endif
20
21#define PER_CPU_VAR(var) __percpu(var)__percpu_rel
22
23#else /* !__ASSEMBLY__: */
24
25#include <linux/args.h>
26#include <linux/bits.h>
27#include <linux/build_bug.h>
28#include <linux/stringify.h>
29#include <asm/asm.h>
30
31#ifdef CONFIG_SMP
32
33#define __force_percpu_prefix "%%"__stringify(__percpu_seg)":"
34
35#ifdef CONFIG_CC_HAS_NAMED_AS
36
37#ifdef __CHECKER__
38# define __seg_gs __attribute__((address_space(__seg_gs)))
39# define __seg_fs __attribute__((address_space(__seg_fs)))
40#endif
41
42#define __percpu_prefix
43#define __percpu_seg_override CONCATENATE(__seg_, __percpu_seg)
44
45#else /* !CONFIG_CC_HAS_NAMED_AS: */
46
47#define __percpu_prefix __force_percpu_prefix
48#define __percpu_seg_override
49
50#endif /* CONFIG_CC_HAS_NAMED_AS */
51
52/*
53 * Compared to the generic __my_cpu_offset version, the following
54 * saves one instruction and avoids clobbering a temp register.
55 */
56#define __my_cpu_offset this_cpu_read(this_cpu_off)
57
58/*
59 * arch_raw_cpu_ptr should not be used in 32-bit VDSO for a 64-bit
60 * kernel, because games are played with CONFIG_X86_64 there and
61 * sizeof(this_cpu_off) becames 4.
62 */
63#ifndef BUILD_VDSO32_64
64#define arch_raw_cpu_ptr(_ptr) \
65({ \
66 unsigned long tcp_ptr__ = raw_cpu_read_long(this_cpu_off); \
67 \
68 tcp_ptr__ += (__force unsigned long)(_ptr); \
69 (TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)tcp_ptr__; \
70})
71#else
72#define arch_raw_cpu_ptr(_ptr) \
73({ \
74 BUILD_BUG(); \
75 (TYPEOF_UNQUAL(*(_ptr)) __force __kernel *)0; \
76})
77#endif
78
79#define PER_CPU_VAR(var) %__percpu_seg:(var)__percpu_rel
80
81#else /* !CONFIG_SMP: */
82
83#define __force_percpu_prefix
84#define __percpu_prefix
85#define __percpu_seg_override
86
87#define PER_CPU_VAR(var) (var)__percpu_rel
88
89#endif /* CONFIG_SMP */
90
91#if defined(CONFIG_USE_X86_SEG_SUPPORT) && defined(USE_TYPEOF_UNQUAL)
92# define __my_cpu_type(var) typeof(var)
93# define __my_cpu_ptr(ptr) (ptr)
94# define __my_cpu_var(var) (var)
95
96# define __percpu_qual __percpu_seg_override
97#else
98# define __my_cpu_type(var) typeof(var) __percpu_seg_override
99# define __my_cpu_ptr(ptr) (__my_cpu_type(*(ptr))*)(__force uintptr_t)(ptr)
100# define __my_cpu_var(var) (*__my_cpu_ptr(&(var)))
101#endif
102
103#define __force_percpu_arg(x) __force_percpu_prefix "%" #x
104#define __percpu_arg(x) __percpu_prefix "%" #x
105
106/*
107 * For arch-specific code, we can use direct single-insn ops (they
108 * don't give an lvalue though).
109 */
110
111#define __pcpu_type_1 u8
112#define __pcpu_type_2 u16
113#define __pcpu_type_4 u32
114#define __pcpu_type_8 u64
115
116#define __pcpu_cast_1(val) ((u8)(((unsigned long) val) & 0xff))
117#define __pcpu_cast_2(val) ((u16)(((unsigned long) val) & 0xffff))
118#define __pcpu_cast_4(val) ((u32)(((unsigned long) val) & 0xffffffff))
119#define __pcpu_cast_8(val) ((u64)(val))
120
121#define __pcpu_op_1(op) op "b "
122#define __pcpu_op_2(op) op "w "
123#define __pcpu_op_4(op) op "l "
124#define __pcpu_op_8(op) op "q "
125
126#define __pcpu_reg_1(mod, x) mod "q" (x)
127#define __pcpu_reg_2(mod, x) mod "r" (x)
128#define __pcpu_reg_4(mod, x) mod "r" (x)
129#define __pcpu_reg_8(mod, x) mod "r" (x)
130
131#define __pcpu_reg_imm_1(x) "qi" (x)
132#define __pcpu_reg_imm_2(x) "ri" (x)
133#define __pcpu_reg_imm_4(x) "ri" (x)
134#define __pcpu_reg_imm_8(x) "re" (x)
135
136#ifdef CONFIG_USE_X86_SEG_SUPPORT
137
138#define __raw_cpu_read(size, qual, pcp) \
139({ \
140 *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)); \
141})
142
143#define __raw_cpu_write(size, qual, pcp, val) \
144do { \
145 *(qual __my_cpu_type(pcp) *)__my_cpu_ptr(&(pcp)) = (val); \
146} while (0)
147
148#define __raw_cpu_read_const(pcp) __raw_cpu_read(, , pcp)
149
150#else /* !CONFIG_USE_X86_SEG_SUPPORT: */
151
152#define __raw_cpu_read(size, qual, _var) \
153({ \
154 __pcpu_type_##size pfo_val__; \
155 \
156 asm qual (__pcpu_op_##size("mov") \
157 __percpu_arg([var]) ", %[val]" \
158 : [val] __pcpu_reg_##size("=", pfo_val__) \
159 : [var] "m" (__my_cpu_var(_var))); \
160 \
161 (typeof(_var))(unsigned long) pfo_val__; \
162})
163
164#define __raw_cpu_write(size, qual, _var, _val) \
165do { \
166 __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \
167 \
168 if (0) { \
169 TYPEOF_UNQUAL(_var) pto_tmp__; \
170 pto_tmp__ = (_val); \
171 (void)pto_tmp__; \
172 } \
173 asm qual (__pcpu_op_##size("mov") "%[val], " \
174 __percpu_arg([var]) \
175 : [var] "=m" (__my_cpu_var(_var)) \
176 : [val] __pcpu_reg_imm_##size(pto_val__)); \
177} while (0)
178
179/*
180 * The generic per-CPU infrastrucutre is not suitable for
181 * reading const-qualified variables.
182 */
183#define __raw_cpu_read_const(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
184
185#endif /* CONFIG_USE_X86_SEG_SUPPORT */
186
187#define __raw_cpu_read_stable(size, _var) \
188({ \
189 __pcpu_type_##size pfo_val__; \
190 \
191 asm(__pcpu_op_##size("mov") \
192 __force_percpu_arg(a[var]) ", %[val]" \
193 : [val] __pcpu_reg_##size("=", pfo_val__) \
194 : [var] "i" (&(_var))); \
195 \
196 (typeof(_var))(unsigned long) pfo_val__; \
197})
198
199#define percpu_unary_op(size, qual, op, _var) \
200({ \
201 asm qual (__pcpu_op_##size(op) __percpu_arg([var]) \
202 : [var] "+m" (__my_cpu_var(_var))); \
203})
204
205#define percpu_binary_op(size, qual, op, _var, _val) \
206do { \
207 __pcpu_type_##size pto_val__ = __pcpu_cast_##size(_val); \
208 \
209 if (0) { \
210 TYPEOF_UNQUAL(_var) pto_tmp__; \
211 pto_tmp__ = (_val); \
212 (void)pto_tmp__; \
213 } \
214 asm qual (__pcpu_op_##size(op) "%[val], " __percpu_arg([var]) \
215 : [var] "+m" (__my_cpu_var(_var)) \
216 : [val] __pcpu_reg_imm_##size(pto_val__)); \
217} while (0)
218
219/*
220 * Generate a per-CPU add to memory instruction and optimize code
221 * if one is added or subtracted.
222 */
223#define percpu_add_op(size, qual, var, val) \
224do { \
225 const int pao_ID__ = \
226 (__builtin_constant_p(val) && \
227 ((val) == 1 || \
228 (val) == (typeof(val))-1)) ? (int)(val) : 0; \
229 \
230 if (0) { \
231 TYPEOF_UNQUAL(var) pao_tmp__; \
232 pao_tmp__ = (val); \
233 (void)pao_tmp__; \
234 } \
235 if (pao_ID__ == 1) \
236 percpu_unary_op(size, qual, "inc", var); \
237 else if (pao_ID__ == -1) \
238 percpu_unary_op(size, qual, "dec", var); \
239 else \
240 percpu_binary_op(size, qual, "add", var, val); \
241} while (0)
242
243/*
244 * Add return operation
245 */
246#define percpu_add_return_op(size, qual, _var, _val) \
247({ \
248 __pcpu_type_##size paro_tmp__ = __pcpu_cast_##size(_val); \
249 \
250 asm qual (__pcpu_op_##size("xadd") "%[tmp], " \
251 __percpu_arg([var]) \
252 : [tmp] __pcpu_reg_##size("+", paro_tmp__), \
253 [var] "+m" (__my_cpu_var(_var)) \
254 : : "memory"); \
255 (typeof(_var))(unsigned long) (paro_tmp__ + _val); \
256})
257
258/*
259 * raw_cpu_xchg() can use a load-store since
260 * it is not required to be IRQ-safe.
261 */
262#define raw_percpu_xchg_op(_var, _nval) \
263({ \
264 TYPEOF_UNQUAL(_var) pxo_old__ = raw_cpu_read(_var); \
265 \
266 raw_cpu_write(_var, _nval); \
267 \
268 pxo_old__; \
269})
270
271/*
272 * this_cpu_xchg() is implemented using CMPXCHG without a LOCK prefix.
273 * XCHG is expensive due to the implied LOCK prefix. The processor
274 * cannot prefetch cachelines if XCHG is used.
275 */
276#define this_percpu_xchg_op(_var, _nval) \
277({ \
278 TYPEOF_UNQUAL(_var) pxo_old__ = this_cpu_read(_var); \
279 \
280 do { } while (!this_cpu_try_cmpxchg(_var, &pxo_old__, _nval)); \
281 \
282 pxo_old__; \
283})
284
285/*
286 * CMPXCHG has no such implied lock semantics as a result it is much
287 * more efficient for CPU-local operations.
288 */
289#define percpu_cmpxchg_op(size, qual, _var, _oval, _nval) \
290({ \
291 __pcpu_type_##size pco_old__ = __pcpu_cast_##size(_oval); \
292 __pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval); \
293 \
294 asm qual (__pcpu_op_##size("cmpxchg") "%[nval], " \
295 __percpu_arg([var]) \
296 : [oval] "+a" (pco_old__), \
297 [var] "+m" (__my_cpu_var(_var)) \
298 : [nval] __pcpu_reg_##size(, pco_new__) \
299 : "memory"); \
300 \
301 (typeof(_var))(unsigned long) pco_old__; \
302})
303
304#define percpu_try_cmpxchg_op(size, qual, _var, _ovalp, _nval) \
305({ \
306 bool success; \
307 __pcpu_type_##size *pco_oval__ = (__pcpu_type_##size *)(_ovalp); \
308 __pcpu_type_##size pco_old__ = *pco_oval__; \
309 __pcpu_type_##size pco_new__ = __pcpu_cast_##size(_nval); \
310 \
311 asm qual (__pcpu_op_##size("cmpxchg") "%[nval], " \
312 __percpu_arg([var]) \
313 : "=@ccz" (success), \
314 [oval] "+a" (pco_old__), \
315 [var] "+m" (__my_cpu_var(_var)) \
316 : [nval] __pcpu_reg_##size(, pco_new__) \
317 : "memory"); \
318 if (unlikely(!success)) \
319 *pco_oval__ = pco_old__; \
320 \
321 likely(success); \
322})
323
324#if defined(CONFIG_X86_32) && !defined(CONFIG_UML)
325
326#define percpu_cmpxchg64_op(size, qual, _var, _oval, _nval) \
327({ \
328 union { \
329 u64 var; \
330 struct { \
331 u32 low, high; \
332 }; \
333 } old__, new__; \
334 \
335 old__.var = _oval; \
336 new__.var = _nval; \
337 \
338 asm_inline qual ( \
339 ALTERNATIVE("call this_cpu_cmpxchg8b_emu", \
340 "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
341 : ALT_OUTPUT_SP([var] "+m" (__my_cpu_var(_var)), \
342 "+a" (old__.low), "+d" (old__.high)) \
343 : "b" (new__.low), "c" (new__.high), \
344 "S" (&(_var)) \
345 : "memory"); \
346 \
347 old__.var; \
348})
349
350#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, , pcp, oval, nval)
351#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg64_op(8, volatile, pcp, oval, nval)
352
353#define percpu_try_cmpxchg64_op(size, qual, _var, _ovalp, _nval) \
354({ \
355 bool success; \
356 u64 *_oval = (u64 *)(_ovalp); \
357 union { \
358 u64 var; \
359 struct { \
360 u32 low, high; \
361 }; \
362 } old__, new__; \
363 \
364 old__.var = *_oval; \
365 new__.var = _nval; \
366 \
367 asm_inline qual ( \
368 ALTERNATIVE("call this_cpu_cmpxchg8b_emu", \
369 "cmpxchg8b " __percpu_arg([var]), X86_FEATURE_CX8) \
370 : ALT_OUTPUT_SP("=@ccz" (success), \
371 [var] "+m" (__my_cpu_var(_var)), \
372 "+a" (old__.low), "+d" (old__.high)) \
373 : "b" (new__.low), "c" (new__.high), \
374 "S" (&(_var)) \
375 : "memory"); \
376 if (unlikely(!success)) \
377 *_oval = old__.var; \
378 \
379 likely(success); \
380})
381
382#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg64_op(8, , pcp, ovalp, nval)
383#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg64_op(8, volatile, pcp, ovalp, nval)
384
385#endif /* defined(CONFIG_X86_32) && !defined(CONFIG_UML) */
386
387#ifdef CONFIG_X86_64
388#define raw_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval);
389#define this_cpu_cmpxchg64(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval);
390
391#define raw_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval);
392#define this_cpu_try_cmpxchg64(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval);
393
394#define percpu_cmpxchg128_op(size, qual, _var, _oval, _nval) \
395({ \
396 union { \
397 u128 var; \
398 struct { \
399 u64 low, high; \
400 }; \
401 } old__, new__; \
402 \
403 old__.var = _oval; \
404 new__.var = _nval; \
405 \
406 asm_inline qual ( \
407 ALTERNATIVE("call this_cpu_cmpxchg16b_emu", \
408 "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
409 : ALT_OUTPUT_SP([var] "+m" (__my_cpu_var(_var)), \
410 "+a" (old__.low), "+d" (old__.high)) \
411 : "b" (new__.low), "c" (new__.high), \
412 "S" (&(_var)) \
413 : "memory"); \
414 \
415 old__.var; \
416})
417
418#define raw_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, , pcp, oval, nval)
419#define this_cpu_cmpxchg128(pcp, oval, nval) percpu_cmpxchg128_op(16, volatile, pcp, oval, nval)
420
421#define percpu_try_cmpxchg128_op(size, qual, _var, _ovalp, _nval) \
422({ \
423 bool success; \
424 u128 *_oval = (u128 *)(_ovalp); \
425 union { \
426 u128 var; \
427 struct { \
428 u64 low, high; \
429 }; \
430 } old__, new__; \
431 \
432 old__.var = *_oval; \
433 new__.var = _nval; \
434 \
435 asm_inline qual ( \
436 ALTERNATIVE("call this_cpu_cmpxchg16b_emu", \
437 "cmpxchg16b " __percpu_arg([var]), X86_FEATURE_CX16) \
438 : ALT_OUTPUT_SP("=@ccz" (success), \
439 [var] "+m" (__my_cpu_var(_var)), \
440 "+a" (old__.low), "+d" (old__.high)) \
441 : "b" (new__.low), "c" (new__.high), \
442 "S" (&(_var)) \
443 : "memory"); \
444 if (unlikely(!success)) \
445 *_oval = old__.var; \
446 \
447 likely(success); \
448})
449
450#define raw_cpu_try_cmpxchg128(pcp, ovalp, nval) percpu_try_cmpxchg128_op(16, , pcp, ovalp, nval)
451#define this_cpu_try_cmpxchg128(pcp, ovalp, nval) percpu_try_cmpxchg128_op(16, volatile, pcp, ovalp, nval)
452
453#endif /* CONFIG_X86_64 */
454
455#define raw_cpu_read_1(pcp) __raw_cpu_read(1, , pcp)
456#define raw_cpu_read_2(pcp) __raw_cpu_read(2, , pcp)
457#define raw_cpu_read_4(pcp) __raw_cpu_read(4, , pcp)
458#define raw_cpu_write_1(pcp, val) __raw_cpu_write(1, , pcp, val)
459#define raw_cpu_write_2(pcp, val) __raw_cpu_write(2, , pcp, val)
460#define raw_cpu_write_4(pcp, val) __raw_cpu_write(4, , pcp, val)
461
462#define this_cpu_read_1(pcp) __raw_cpu_read(1, volatile, pcp)
463#define this_cpu_read_2(pcp) __raw_cpu_read(2, volatile, pcp)
464#define this_cpu_read_4(pcp) __raw_cpu_read(4, volatile, pcp)
465#define this_cpu_write_1(pcp, val) __raw_cpu_write(1, volatile, pcp, val)
466#define this_cpu_write_2(pcp, val) __raw_cpu_write(2, volatile, pcp, val)
467#define this_cpu_write_4(pcp, val) __raw_cpu_write(4, volatile, pcp, val)
468
469#define this_cpu_read_stable_1(pcp) __raw_cpu_read_stable(1, pcp)
470#define this_cpu_read_stable_2(pcp) __raw_cpu_read_stable(2, pcp)
471#define this_cpu_read_stable_4(pcp) __raw_cpu_read_stable(4, pcp)
472
473#define raw_cpu_add_1(pcp, val) percpu_add_op(1, , (pcp), val)
474#define raw_cpu_add_2(pcp, val) percpu_add_op(2, , (pcp), val)
475#define raw_cpu_add_4(pcp, val) percpu_add_op(4, , (pcp), val)
476#define raw_cpu_and_1(pcp, val) percpu_binary_op(1, , "and", (pcp), val)
477#define raw_cpu_and_2(pcp, val) percpu_binary_op(2, , "and", (pcp), val)
478#define raw_cpu_and_4(pcp, val) percpu_binary_op(4, , "and", (pcp), val)
479#define raw_cpu_or_1(pcp, val) percpu_binary_op(1, , "or", (pcp), val)
480#define raw_cpu_or_2(pcp, val) percpu_binary_op(2, , "or", (pcp), val)
481#define raw_cpu_or_4(pcp, val) percpu_binary_op(4, , "or", (pcp), val)
482#define raw_cpu_xchg_1(pcp, val) raw_percpu_xchg_op(pcp, val)
483#define raw_cpu_xchg_2(pcp, val) raw_percpu_xchg_op(pcp, val)
484#define raw_cpu_xchg_4(pcp, val) raw_percpu_xchg_op(pcp, val)
485
486#define this_cpu_add_1(pcp, val) percpu_add_op(1, volatile, (pcp), val)
487#define this_cpu_add_2(pcp, val) percpu_add_op(2, volatile, (pcp), val)
488#define this_cpu_add_4(pcp, val) percpu_add_op(4, volatile, (pcp), val)
489#define this_cpu_and_1(pcp, val) percpu_binary_op(1, volatile, "and", (pcp), val)
490#define this_cpu_and_2(pcp, val) percpu_binary_op(2, volatile, "and", (pcp), val)
491#define this_cpu_and_4(pcp, val) percpu_binary_op(4, volatile, "and", (pcp), val)
492#define this_cpu_or_1(pcp, val) percpu_binary_op(1, volatile, "or", (pcp), val)
493#define this_cpu_or_2(pcp, val) percpu_binary_op(2, volatile, "or", (pcp), val)
494#define this_cpu_or_4(pcp, val) percpu_binary_op(4, volatile, "or", (pcp), val)
495#define this_cpu_xchg_1(pcp, nval) this_percpu_xchg_op(pcp, nval)
496#define this_cpu_xchg_2(pcp, nval) this_percpu_xchg_op(pcp, nval)
497#define this_cpu_xchg_4(pcp, nval) this_percpu_xchg_op(pcp, nval)
498
499#define raw_cpu_add_return_1(pcp, val) percpu_add_return_op(1, , pcp, val)
500#define raw_cpu_add_return_2(pcp, val) percpu_add_return_op(2, , pcp, val)
501#define raw_cpu_add_return_4(pcp, val) percpu_add_return_op(4, , pcp, val)
502#define raw_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, , pcp, oval, nval)
503#define raw_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, , pcp, oval, nval)
504#define raw_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, , pcp, oval, nval)
505#define raw_cpu_try_cmpxchg_1(pcp, ovalp, nval) percpu_try_cmpxchg_op(1, , pcp, ovalp, nval)
506#define raw_cpu_try_cmpxchg_2(pcp, ovalp, nval) percpu_try_cmpxchg_op(2, , pcp, ovalp, nval)
507#define raw_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, , pcp, ovalp, nval)
508
509#define this_cpu_add_return_1(pcp, val) percpu_add_return_op(1, volatile, pcp, val)
510#define this_cpu_add_return_2(pcp, val) percpu_add_return_op(2, volatile, pcp, val)
511#define this_cpu_add_return_4(pcp, val) percpu_add_return_op(4, volatile, pcp, val)
512#define this_cpu_cmpxchg_1(pcp, oval, nval) percpu_cmpxchg_op(1, volatile, pcp, oval, nval)
513#define this_cpu_cmpxchg_2(pcp, oval, nval) percpu_cmpxchg_op(2, volatile, pcp, oval, nval)
514#define this_cpu_cmpxchg_4(pcp, oval, nval) percpu_cmpxchg_op(4, volatile, pcp, oval, nval)
515#define this_cpu_try_cmpxchg_1(pcp, ovalp, nval) percpu_try_cmpxchg_op(1, volatile, pcp, ovalp, nval)
516#define this_cpu_try_cmpxchg_2(pcp, ovalp, nval) percpu_try_cmpxchg_op(2, volatile, pcp, ovalp, nval)
517#define this_cpu_try_cmpxchg_4(pcp, ovalp, nval) percpu_try_cmpxchg_op(4, volatile, pcp, ovalp, nval)
518
519/*
520 * Per-CPU atomic 64-bit operations are only available under 64-bit kernels.
521 * 32-bit kernels must fall back to generic operations.
522 */
523#ifdef CONFIG_X86_64
524
525#define raw_cpu_read_8(pcp) __raw_cpu_read(8, , pcp)
526#define raw_cpu_write_8(pcp, val) __raw_cpu_write(8, , pcp, val)
527
528#define this_cpu_read_8(pcp) __raw_cpu_read(8, volatile, pcp)
529#define this_cpu_write_8(pcp, val) __raw_cpu_write(8, volatile, pcp, val)
530
531#define this_cpu_read_stable_8(pcp) __raw_cpu_read_stable(8, pcp)
532
533#define raw_cpu_add_8(pcp, val) percpu_add_op(8, , (pcp), val)
534#define raw_cpu_and_8(pcp, val) percpu_binary_op(8, , "and", (pcp), val)
535#define raw_cpu_or_8(pcp, val) percpu_binary_op(8, , "or", (pcp), val)
536#define raw_cpu_add_return_8(pcp, val) percpu_add_return_op(8, , pcp, val)
537#define raw_cpu_xchg_8(pcp, nval) raw_percpu_xchg_op(pcp, nval)
538#define raw_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, , pcp, oval, nval)
539#define raw_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, , pcp, ovalp, nval)
540
541#define this_cpu_add_8(pcp, val) percpu_add_op(8, volatile, (pcp), val)
542#define this_cpu_and_8(pcp, val) percpu_binary_op(8, volatile, "and", (pcp), val)
543#define this_cpu_or_8(pcp, val) percpu_binary_op(8, volatile, "or", (pcp), val)
544#define this_cpu_add_return_8(pcp, val) percpu_add_return_op(8, volatile, pcp, val)
545#define this_cpu_xchg_8(pcp, nval) this_percpu_xchg_op(pcp, nval)
546#define this_cpu_cmpxchg_8(pcp, oval, nval) percpu_cmpxchg_op(8, volatile, pcp, oval, nval)
547#define this_cpu_try_cmpxchg_8(pcp, ovalp, nval) percpu_try_cmpxchg_op(8, volatile, pcp, ovalp, nval)
548
549#define raw_cpu_read_long(pcp) raw_cpu_read_8(pcp)
550
551#else /* !CONFIG_X86_64: */
552
553/* There is no generic 64-bit read stable operation for 32-bit targets. */
554#define this_cpu_read_stable_8(pcp) ({ BUILD_BUG(); (typeof(pcp))0; })
555
556#define raw_cpu_read_long(pcp) raw_cpu_read_4(pcp)
557
558#endif /* CONFIG_X86_64 */
559
560#define this_cpu_read_const(pcp) __raw_cpu_read_const(pcp)
561
562/*
563 * this_cpu_read() makes the compiler load the per-CPU variable every time
564 * it is accessed while this_cpu_read_stable() allows the value to be cached.
565 * this_cpu_read_stable() is more efficient and can be used if its value
566 * is guaranteed to be valid across CPUs. The current users include
567 * current_task and cpu_current_top_of_stack, both of which are
568 * actually per-thread variables implemented as per-CPU variables and
569 * thus stable for the duration of the respective task.
570 */
571#define this_cpu_read_stable(pcp) __pcpu_size_call_return(this_cpu_read_stable_, pcp)
572
573#define x86_this_cpu_constant_test_bit(_nr, _var) \
574({ \
575 unsigned long __percpu *addr__ = \
576 (unsigned long __percpu *)&(_var) + BIT_WORD(_nr); \
577 \
578 !!(BIT_MASK(_nr) & raw_cpu_read(*addr__)); \
579})
580
581#define x86_this_cpu_variable_test_bit(_nr, _var) \
582({ \
583 bool oldbit; \
584 \
585 asm volatile("btl %[nr], " __percpu_arg([var]) \
586 : "=@ccc" (oldbit) \
587 : [var] "m" (__my_cpu_var(_var)), \
588 [nr] "rI" (_nr)); \
589 oldbit; \
590})
591
592#define x86_this_cpu_test_bit(_nr, _var) \
593 (__builtin_constant_p(_nr) \
594 ? x86_this_cpu_constant_test_bit(_nr, _var) \
595 : x86_this_cpu_variable_test_bit(_nr, _var))
596
597
598#include <asm-generic/percpu.h>
599
600/* We can use this directly for local CPU (faster). */
601DECLARE_PER_CPU_CACHE_HOT(unsigned long, this_cpu_off);
602
603#endif /* !__ASSEMBLER__ */
604
605#ifdef CONFIG_SMP
606
607/*
608 * Define the "EARLY_PER_CPU" macros. These are used for some per_cpu
609 * variables that are initialized and accessed before there are per_cpu
610 * areas allocated.
611 */
612
613#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
614 DEFINE_PER_CPU(_type, _name) = _initvalue; \
615 __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \
616 { [0 ... NR_CPUS-1] = _initvalue }; \
617 __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
618
619#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \
620 DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue; \
621 __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \
622 { [0 ... NR_CPUS-1] = _initvalue }; \
623 __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
624
625#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
626 EXPORT_PER_CPU_SYMBOL(_name)
627
628#define DECLARE_EARLY_PER_CPU(_type, _name) \
629 DECLARE_PER_CPU(_type, _name); \
630 extern __typeof__(_type) *_name##_early_ptr; \
631 extern __typeof__(_type) _name##_early_map[]
632
633#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \
634 DECLARE_PER_CPU_READ_MOSTLY(_type, _name); \
635 extern __typeof__(_type) *_name##_early_ptr; \
636 extern __typeof__(_type) _name##_early_map[]
637
638#define early_per_cpu_ptr(_name) (_name##_early_ptr)
639#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
640
641#define early_per_cpu(_name, _cpu) \
642 *(early_per_cpu_ptr(_name) ? \
643 &early_per_cpu_ptr(_name)[_cpu] : \
644 &per_cpu(_name, _cpu))
645
646#else /* !CONFIG_SMP: */
647#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
648 DEFINE_PER_CPU(_type, _name) = _initvalue
649
650#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \
651 DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue
652
653#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
654 EXPORT_PER_CPU_SYMBOL(_name)
655
656#define DECLARE_EARLY_PER_CPU(_type, _name) \
657 DECLARE_PER_CPU(_type, _name)
658
659#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \
660 DECLARE_PER_CPU_READ_MOSTLY(_type, _name)
661
662#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu)
663#define early_per_cpu_ptr(_name) NULL
664/* no early_per_cpu_map() */
665
666#endif /* !CONFIG_SMP */
667
668#endif /* _ASM_X86_PERCPU_H */