Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _ASM_GENERIC_PERCPU_H_
3#define _ASM_GENERIC_PERCPU_H_
4
5#include <linux/compiler.h>
6#include <linux/threads.h>
7#include <linux/percpu-defs.h>
8
9#ifdef CONFIG_SMP
10
11/*
12 * per_cpu_offset() is the offset that has to be added to a
13 * percpu variable to get to the instance for a certain processor.
14 *
15 * Most arches use the __per_cpu_offset array for those offsets but
16 * some arches have their own ways of determining the offset (x86_64, s390).
17 */
18#ifndef __per_cpu_offset
19extern unsigned long __per_cpu_offset[NR_CPUS];
20
21#define per_cpu_offset(x) (__per_cpu_offset[x])
22#endif
23
24/*
25 * Determine the offset for the currently active processor.
26 * An arch may define __my_cpu_offset to provide a more effective
27 * means of obtaining the offset to the per cpu variables of the
28 * current processor.
29 */
30#ifndef __my_cpu_offset
31#define __my_cpu_offset per_cpu_offset(raw_smp_processor_id())
32#endif
33#ifdef CONFIG_DEBUG_PREEMPT
34#define my_cpu_offset per_cpu_offset(smp_processor_id())
35#else
36#define my_cpu_offset __my_cpu_offset
37#endif
38
39/*
40 * Arch may define arch_raw_cpu_ptr() to provide more efficient address
41 * translations for raw_cpu_ptr().
42 */
43#ifndef arch_raw_cpu_ptr
44#define arch_raw_cpu_ptr(ptr) SHIFT_PERCPU_PTR(ptr, __my_cpu_offset)
45#endif
46
47#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
48extern void setup_per_cpu_areas(void);
49#endif
50
51#endif /* SMP */
52
53#ifndef PER_CPU_BASE_SECTION
54#ifdef CONFIG_SMP
55#define PER_CPU_BASE_SECTION ".data..percpu"
56#else
57#define PER_CPU_BASE_SECTION ".data"
58#endif
59#endif
60
61#ifndef PER_CPU_ATTRIBUTES
62#define PER_CPU_ATTRIBUTES
63#endif
64
65#ifndef PER_CPU_DEF_ATTRIBUTES
66#define PER_CPU_DEF_ATTRIBUTES
67#endif
68
69#define raw_cpu_generic_read(pcp) \
70({ \
71 *raw_cpu_ptr(&(pcp)); \
72})
73
74#define raw_cpu_generic_to_op(pcp, val, op) \
75do { \
76 *raw_cpu_ptr(&(pcp)) op val; \
77} while (0)
78
79#define raw_cpu_generic_add_return(pcp, val) \
80({ \
81 typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \
82 \
83 *__p += val; \
84 *__p; \
85})
86
87#define raw_cpu_generic_xchg(pcp, nval) \
88({ \
89 typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \
90 typeof(pcp) __ret; \
91 __ret = *__p; \
92 *__p = nval; \
93 __ret; \
94})
95
96#define raw_cpu_generic_cmpxchg(pcp, oval, nval) \
97({ \
98 typeof(&(pcp)) __p = raw_cpu_ptr(&(pcp)); \
99 typeof(pcp) __ret; \
100 __ret = *__p; \
101 if (__ret == (oval)) \
102 *__p = nval; \
103 __ret; \
104})
105
106#define raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
107({ \
108 typeof(&(pcp1)) __p1 = raw_cpu_ptr(&(pcp1)); \
109 typeof(&(pcp2)) __p2 = raw_cpu_ptr(&(pcp2)); \
110 int __ret = 0; \
111 if (*__p1 == (oval1) && *__p2 == (oval2)) { \
112 *__p1 = nval1; \
113 *__p2 = nval2; \
114 __ret = 1; \
115 } \
116 (__ret); \
117})
118
119#define __this_cpu_generic_read_nopreempt(pcp) \
120({ \
121 typeof(pcp) __ret; \
122 preempt_disable_notrace(); \
123 __ret = READ_ONCE(*raw_cpu_ptr(&(pcp))); \
124 preempt_enable_notrace(); \
125 __ret; \
126})
127
128#define __this_cpu_generic_read_noirq(pcp) \
129({ \
130 typeof(pcp) __ret; \
131 unsigned long __flags; \
132 raw_local_irq_save(__flags); \
133 __ret = raw_cpu_generic_read(pcp); \
134 raw_local_irq_restore(__flags); \
135 __ret; \
136})
137
138#define this_cpu_generic_read(pcp) \
139({ \
140 typeof(pcp) __ret; \
141 if (__native_word(pcp)) \
142 __ret = __this_cpu_generic_read_nopreempt(pcp); \
143 else \
144 __ret = __this_cpu_generic_read_noirq(pcp); \
145 __ret; \
146})
147
148#define this_cpu_generic_to_op(pcp, val, op) \
149do { \
150 unsigned long __flags; \
151 raw_local_irq_save(__flags); \
152 raw_cpu_generic_to_op(pcp, val, op); \
153 raw_local_irq_restore(__flags); \
154} while (0)
155
156
157#define this_cpu_generic_add_return(pcp, val) \
158({ \
159 typeof(pcp) __ret; \
160 unsigned long __flags; \
161 raw_local_irq_save(__flags); \
162 __ret = raw_cpu_generic_add_return(pcp, val); \
163 raw_local_irq_restore(__flags); \
164 __ret; \
165})
166
167#define this_cpu_generic_xchg(pcp, nval) \
168({ \
169 typeof(pcp) __ret; \
170 unsigned long __flags; \
171 raw_local_irq_save(__flags); \
172 __ret = raw_cpu_generic_xchg(pcp, nval); \
173 raw_local_irq_restore(__flags); \
174 __ret; \
175})
176
177#define this_cpu_generic_cmpxchg(pcp, oval, nval) \
178({ \
179 typeof(pcp) __ret; \
180 unsigned long __flags; \
181 raw_local_irq_save(__flags); \
182 __ret = raw_cpu_generic_cmpxchg(pcp, oval, nval); \
183 raw_local_irq_restore(__flags); \
184 __ret; \
185})
186
187#define this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \
188({ \
189 int __ret; \
190 unsigned long __flags; \
191 raw_local_irq_save(__flags); \
192 __ret = raw_cpu_generic_cmpxchg_double(pcp1, pcp2, \
193 oval1, oval2, nval1, nval2); \
194 raw_local_irq_restore(__flags); \
195 __ret; \
196})
197
198#ifndef raw_cpu_read_1
199#define raw_cpu_read_1(pcp) raw_cpu_generic_read(pcp)
200#endif
201#ifndef raw_cpu_read_2
202#define raw_cpu_read_2(pcp) raw_cpu_generic_read(pcp)
203#endif
204#ifndef raw_cpu_read_4
205#define raw_cpu_read_4(pcp) raw_cpu_generic_read(pcp)
206#endif
207#ifndef raw_cpu_read_8
208#define raw_cpu_read_8(pcp) raw_cpu_generic_read(pcp)
209#endif
210
211#ifndef raw_cpu_write_1
212#define raw_cpu_write_1(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
213#endif
214#ifndef raw_cpu_write_2
215#define raw_cpu_write_2(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
216#endif
217#ifndef raw_cpu_write_4
218#define raw_cpu_write_4(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
219#endif
220#ifndef raw_cpu_write_8
221#define raw_cpu_write_8(pcp, val) raw_cpu_generic_to_op(pcp, val, =)
222#endif
223
224#ifndef raw_cpu_add_1
225#define raw_cpu_add_1(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
226#endif
227#ifndef raw_cpu_add_2
228#define raw_cpu_add_2(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
229#endif
230#ifndef raw_cpu_add_4
231#define raw_cpu_add_4(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
232#endif
233#ifndef raw_cpu_add_8
234#define raw_cpu_add_8(pcp, val) raw_cpu_generic_to_op(pcp, val, +=)
235#endif
236
237#ifndef raw_cpu_and_1
238#define raw_cpu_and_1(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
239#endif
240#ifndef raw_cpu_and_2
241#define raw_cpu_and_2(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
242#endif
243#ifndef raw_cpu_and_4
244#define raw_cpu_and_4(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
245#endif
246#ifndef raw_cpu_and_8
247#define raw_cpu_and_8(pcp, val) raw_cpu_generic_to_op(pcp, val, &=)
248#endif
249
250#ifndef raw_cpu_or_1
251#define raw_cpu_or_1(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
252#endif
253#ifndef raw_cpu_or_2
254#define raw_cpu_or_2(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
255#endif
256#ifndef raw_cpu_or_4
257#define raw_cpu_or_4(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
258#endif
259#ifndef raw_cpu_or_8
260#define raw_cpu_or_8(pcp, val) raw_cpu_generic_to_op(pcp, val, |=)
261#endif
262
263#ifndef raw_cpu_add_return_1
264#define raw_cpu_add_return_1(pcp, val) raw_cpu_generic_add_return(pcp, val)
265#endif
266#ifndef raw_cpu_add_return_2
267#define raw_cpu_add_return_2(pcp, val) raw_cpu_generic_add_return(pcp, val)
268#endif
269#ifndef raw_cpu_add_return_4
270#define raw_cpu_add_return_4(pcp, val) raw_cpu_generic_add_return(pcp, val)
271#endif
272#ifndef raw_cpu_add_return_8
273#define raw_cpu_add_return_8(pcp, val) raw_cpu_generic_add_return(pcp, val)
274#endif
275
276#ifndef raw_cpu_xchg_1
277#define raw_cpu_xchg_1(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
278#endif
279#ifndef raw_cpu_xchg_2
280#define raw_cpu_xchg_2(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
281#endif
282#ifndef raw_cpu_xchg_4
283#define raw_cpu_xchg_4(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
284#endif
285#ifndef raw_cpu_xchg_8
286#define raw_cpu_xchg_8(pcp, nval) raw_cpu_generic_xchg(pcp, nval)
287#endif
288
289#ifndef raw_cpu_cmpxchg_1
290#define raw_cpu_cmpxchg_1(pcp, oval, nval) \
291 raw_cpu_generic_cmpxchg(pcp, oval, nval)
292#endif
293#ifndef raw_cpu_cmpxchg_2
294#define raw_cpu_cmpxchg_2(pcp, oval, nval) \
295 raw_cpu_generic_cmpxchg(pcp, oval, nval)
296#endif
297#ifndef raw_cpu_cmpxchg_4
298#define raw_cpu_cmpxchg_4(pcp, oval, nval) \
299 raw_cpu_generic_cmpxchg(pcp, oval, nval)
300#endif
301#ifndef raw_cpu_cmpxchg_8
302#define raw_cpu_cmpxchg_8(pcp, oval, nval) \
303 raw_cpu_generic_cmpxchg(pcp, oval, nval)
304#endif
305
306#ifndef raw_cpu_cmpxchg_double_1
307#define raw_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
308 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
309#endif
310#ifndef raw_cpu_cmpxchg_double_2
311#define raw_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
312 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
313#endif
314#ifndef raw_cpu_cmpxchg_double_4
315#define raw_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
316 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
317#endif
318#ifndef raw_cpu_cmpxchg_double_8
319#define raw_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
320 raw_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
321#endif
322
323#ifndef this_cpu_read_1
324#define this_cpu_read_1(pcp) this_cpu_generic_read(pcp)
325#endif
326#ifndef this_cpu_read_2
327#define this_cpu_read_2(pcp) this_cpu_generic_read(pcp)
328#endif
329#ifndef this_cpu_read_4
330#define this_cpu_read_4(pcp) this_cpu_generic_read(pcp)
331#endif
332#ifndef this_cpu_read_8
333#define this_cpu_read_8(pcp) this_cpu_generic_read(pcp)
334#endif
335
336#ifndef this_cpu_write_1
337#define this_cpu_write_1(pcp, val) this_cpu_generic_to_op(pcp, val, =)
338#endif
339#ifndef this_cpu_write_2
340#define this_cpu_write_2(pcp, val) this_cpu_generic_to_op(pcp, val, =)
341#endif
342#ifndef this_cpu_write_4
343#define this_cpu_write_4(pcp, val) this_cpu_generic_to_op(pcp, val, =)
344#endif
345#ifndef this_cpu_write_8
346#define this_cpu_write_8(pcp, val) this_cpu_generic_to_op(pcp, val, =)
347#endif
348
349#ifndef this_cpu_add_1
350#define this_cpu_add_1(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
351#endif
352#ifndef this_cpu_add_2
353#define this_cpu_add_2(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
354#endif
355#ifndef this_cpu_add_4
356#define this_cpu_add_4(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
357#endif
358#ifndef this_cpu_add_8
359#define this_cpu_add_8(pcp, val) this_cpu_generic_to_op(pcp, val, +=)
360#endif
361
362#ifndef this_cpu_and_1
363#define this_cpu_and_1(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
364#endif
365#ifndef this_cpu_and_2
366#define this_cpu_and_2(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
367#endif
368#ifndef this_cpu_and_4
369#define this_cpu_and_4(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
370#endif
371#ifndef this_cpu_and_8
372#define this_cpu_and_8(pcp, val) this_cpu_generic_to_op(pcp, val, &=)
373#endif
374
375#ifndef this_cpu_or_1
376#define this_cpu_or_1(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
377#endif
378#ifndef this_cpu_or_2
379#define this_cpu_or_2(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
380#endif
381#ifndef this_cpu_or_4
382#define this_cpu_or_4(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
383#endif
384#ifndef this_cpu_or_8
385#define this_cpu_or_8(pcp, val) this_cpu_generic_to_op(pcp, val, |=)
386#endif
387
388#ifndef this_cpu_add_return_1
389#define this_cpu_add_return_1(pcp, val) this_cpu_generic_add_return(pcp, val)
390#endif
391#ifndef this_cpu_add_return_2
392#define this_cpu_add_return_2(pcp, val) this_cpu_generic_add_return(pcp, val)
393#endif
394#ifndef this_cpu_add_return_4
395#define this_cpu_add_return_4(pcp, val) this_cpu_generic_add_return(pcp, val)
396#endif
397#ifndef this_cpu_add_return_8
398#define this_cpu_add_return_8(pcp, val) this_cpu_generic_add_return(pcp, val)
399#endif
400
401#ifndef this_cpu_xchg_1
402#define this_cpu_xchg_1(pcp, nval) this_cpu_generic_xchg(pcp, nval)
403#endif
404#ifndef this_cpu_xchg_2
405#define this_cpu_xchg_2(pcp, nval) this_cpu_generic_xchg(pcp, nval)
406#endif
407#ifndef this_cpu_xchg_4
408#define this_cpu_xchg_4(pcp, nval) this_cpu_generic_xchg(pcp, nval)
409#endif
410#ifndef this_cpu_xchg_8
411#define this_cpu_xchg_8(pcp, nval) this_cpu_generic_xchg(pcp, nval)
412#endif
413
414#ifndef this_cpu_cmpxchg_1
415#define this_cpu_cmpxchg_1(pcp, oval, nval) \
416 this_cpu_generic_cmpxchg(pcp, oval, nval)
417#endif
418#ifndef this_cpu_cmpxchg_2
419#define this_cpu_cmpxchg_2(pcp, oval, nval) \
420 this_cpu_generic_cmpxchg(pcp, oval, nval)
421#endif
422#ifndef this_cpu_cmpxchg_4
423#define this_cpu_cmpxchg_4(pcp, oval, nval) \
424 this_cpu_generic_cmpxchg(pcp, oval, nval)
425#endif
426#ifndef this_cpu_cmpxchg_8
427#define this_cpu_cmpxchg_8(pcp, oval, nval) \
428 this_cpu_generic_cmpxchg(pcp, oval, nval)
429#endif
430
431#ifndef this_cpu_cmpxchg_double_1
432#define this_cpu_cmpxchg_double_1(pcp1, pcp2, oval1, oval2, nval1, nval2) \
433 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
434#endif
435#ifndef this_cpu_cmpxchg_double_2
436#define this_cpu_cmpxchg_double_2(pcp1, pcp2, oval1, oval2, nval1, nval2) \
437 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
438#endif
439#ifndef this_cpu_cmpxchg_double_4
440#define this_cpu_cmpxchg_double_4(pcp1, pcp2, oval1, oval2, nval1, nval2) \
441 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
442#endif
443#ifndef this_cpu_cmpxchg_double_8
444#define this_cpu_cmpxchg_double_8(pcp1, pcp2, oval1, oval2, nval1, nval2) \
445 this_cpu_generic_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2)
446#endif
447
448#endif /* _ASM_GENERIC_PERCPU_H_ */