Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1#ifndef _ASM_X86_PERCPU_H_
2#define _ASM_X86_PERCPU_H_
3
4#ifdef CONFIG_X86_64
5#include <linux/compiler.h>
6
7/* Same as asm-generic/percpu.h, except that we store the per cpu offset
8 in the PDA. Longer term the PDA and every per cpu variable
9 should be just put into a single section and referenced directly
10 from %gs */
11
12#ifdef CONFIG_SMP
13#include <asm/pda.h>
14
15#define __per_cpu_offset(cpu) (cpu_pda(cpu)->data_offset)
16#define __my_cpu_offset read_pda(data_offset)
17
18#define per_cpu_offset(x) (__per_cpu_offset(x))
19
20#endif
21#include <asm-generic/percpu.h>
22
23DECLARE_PER_CPU(struct x8664_pda, pda);
24
25#else /* CONFIG_X86_64 */
26
27#ifdef __ASSEMBLY__
28
29/*
30 * PER_CPU finds an address of a per-cpu variable.
31 *
32 * Args:
33 * var - variable name
34 * reg - 32bit register
35 *
36 * The resulting address is stored in the "reg" argument.
37 *
38 * Example:
39 * PER_CPU(cpu_gdt_descr, %ebx)
40 */
41#ifdef CONFIG_SMP
42#define PER_CPU(var, reg) \
43 movl %fs:per_cpu__##this_cpu_off, reg; \
44 lea per_cpu__##var(reg), reg
45#define PER_CPU_VAR(var) %fs:per_cpu__##var
46#else /* ! SMP */
47#define PER_CPU(var, reg) \
48 movl $per_cpu__##var, reg
49#define PER_CPU_VAR(var) per_cpu__##var
50#endif /* SMP */
51
52#else /* ...!ASSEMBLY */
53
54/*
55 * PER_CPU finds an address of a per-cpu variable.
56 *
57 * Args:
58 * var - variable name
59 * cpu - 32bit register containing the current CPU number
60 *
61 * The resulting address is stored in the "cpu" argument.
62 *
63 * Example:
64 * PER_CPU(cpu_gdt_descr, %ebx)
65 */
66#ifdef CONFIG_SMP
67
68#define __my_cpu_offset x86_read_percpu(this_cpu_off)
69
70/* fs segment starts at (positive) offset == __per_cpu_offset[cpu] */
71#define __percpu_seg "%%fs:"
72
73#else /* !SMP */
74
75#define __percpu_seg ""
76
77#endif /* SMP */
78
79#include <asm-generic/percpu.h>
80
81/* We can use this directly for local CPU (faster). */
82DECLARE_PER_CPU(unsigned long, this_cpu_off);
83
84/* For arch-specific code, we can use direct single-insn ops (they
85 * don't give an lvalue though). */
86extern void __bad_percpu_size(void);
87
88#define percpu_to_op(op,var,val) \
89 do { \
90 typedef typeof(var) T__; \
91 if (0) { T__ tmp__; tmp__ = (val); } \
92 switch (sizeof(var)) { \
93 case 1: \
94 asm(op "b %1,"__percpu_seg"%0" \
95 : "+m" (var) \
96 :"ri" ((T__)val)); \
97 break; \
98 case 2: \
99 asm(op "w %1,"__percpu_seg"%0" \
100 : "+m" (var) \
101 :"ri" ((T__)val)); \
102 break; \
103 case 4: \
104 asm(op "l %1,"__percpu_seg"%0" \
105 : "+m" (var) \
106 :"ri" ((T__)val)); \
107 break; \
108 default: __bad_percpu_size(); \
109 } \
110 } while (0)
111
112#define percpu_from_op(op,var) \
113 ({ \
114 typeof(var) ret__; \
115 switch (sizeof(var)) { \
116 case 1: \
117 asm(op "b "__percpu_seg"%1,%0" \
118 : "=r" (ret__) \
119 : "m" (var)); \
120 break; \
121 case 2: \
122 asm(op "w "__percpu_seg"%1,%0" \
123 : "=r" (ret__) \
124 : "m" (var)); \
125 break; \
126 case 4: \
127 asm(op "l "__percpu_seg"%1,%0" \
128 : "=r" (ret__) \
129 : "m" (var)); \
130 break; \
131 default: __bad_percpu_size(); \
132 } \
133 ret__; })
134
135#define x86_read_percpu(var) percpu_from_op("mov", per_cpu__##var)
136#define x86_write_percpu(var,val) percpu_to_op("mov", per_cpu__##var, val)
137#define x86_add_percpu(var,val) percpu_to_op("add", per_cpu__##var, val)
138#define x86_sub_percpu(var,val) percpu_to_op("sub", per_cpu__##var, val)
139#define x86_or_percpu(var,val) percpu_to_op("or", per_cpu__##var, val)
140#endif /* !__ASSEMBLY__ */
141#endif /* !CONFIG_X86_64 */
142#endif /* _ASM_X86_PERCPU_H_ */