Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1#ifndef __LINUX_PERCPU_H
2#define __LINUX_PERCPU_H
3
4#include <linux/preempt.h>
5#include <linux/slab.h> /* For kmalloc() */
6#include <linux/smp.h>
7#include <linux/cpumask.h>
8
9#include <asm/percpu.h>
10
11#ifdef CONFIG_SMP
12#define DEFINE_PER_CPU(type, name) \
13 __attribute__((__section__(".data.percpu"))) \
14 PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
15
16#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \
17 __attribute__((__section__(".data.percpu.shared_aligned"))) \
18 PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name \
19 ____cacheline_aligned_in_smp
20#else
21#define DEFINE_PER_CPU(type, name) \
22 PER_CPU_ATTRIBUTES __typeof__(type) per_cpu__##name
23
24#define DEFINE_PER_CPU_SHARED_ALIGNED(type, name) \
25 DEFINE_PER_CPU(type, name)
26#endif
27
28#define EXPORT_PER_CPU_SYMBOL(var) EXPORT_SYMBOL(per_cpu__##var)
29#define EXPORT_PER_CPU_SYMBOL_GPL(var) EXPORT_SYMBOL_GPL(per_cpu__##var)
30
31/* Enough to cover all DEFINE_PER_CPUs in kernel, including modules. */
32#ifndef PERCPU_ENOUGH_ROOM
33#ifdef CONFIG_MODULES
34#define PERCPU_MODULE_RESERVE 8192
35#else
36#define PERCPU_MODULE_RESERVE 0
37#endif
38
39#define PERCPU_ENOUGH_ROOM \
40 (__per_cpu_end - __per_cpu_start + PERCPU_MODULE_RESERVE)
41#endif /* PERCPU_ENOUGH_ROOM */
42
43/*
44 * Must be an lvalue. Since @var must be a simple identifier,
45 * we force a syntax error here if it isn't.
46 */
47#define get_cpu_var(var) (*({ \
48 extern int simple_identifier_##var(void); \
49 preempt_disable(); \
50 &__get_cpu_var(var); }))
51#define put_cpu_var(var) preempt_enable()
52
53#ifdef CONFIG_SMP
54
55struct percpu_data {
56 void *ptrs[1];
57};
58
59#define __percpu_disguise(pdata) (struct percpu_data *)~(unsigned long)(pdata)
60/*
61 * Use this to get to a cpu's version of the per-cpu object dynamically
62 * allocated. Non-atomic access to the current CPU's version should
63 * probably be combined with get_cpu()/put_cpu().
64 */
65#define percpu_ptr(ptr, cpu) \
66({ \
67 struct percpu_data *__p = __percpu_disguise(ptr); \
68 (__typeof__(ptr))__p->ptrs[(cpu)]; \
69})
70
71extern void *percpu_populate(void *__pdata, size_t size, gfp_t gfp, int cpu);
72extern void percpu_depopulate(void *__pdata, int cpu);
73extern int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
74 cpumask_t *mask);
75extern void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask);
76extern void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask);
77extern void percpu_free(void *__pdata);
78
79#else /* CONFIG_SMP */
80
81#define percpu_ptr(ptr, cpu) ({ (void)(cpu); (ptr); })
82
83static inline void percpu_depopulate(void *__pdata, int cpu)
84{
85}
86
87static inline void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask)
88{
89}
90
91static inline void *percpu_populate(void *__pdata, size_t size, gfp_t gfp,
92 int cpu)
93{
94 return percpu_ptr(__pdata, cpu);
95}
96
97static inline int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp,
98 cpumask_t *mask)
99{
100 return 0;
101}
102
103static __always_inline void *__percpu_alloc_mask(size_t size, gfp_t gfp, cpumask_t *mask)
104{
105 return kzalloc(size, gfp);
106}
107
108static inline void percpu_free(void *__pdata)
109{
110 kfree(__pdata);
111}
112
113#endif /* CONFIG_SMP */
114
115#define percpu_populate_mask(__pdata, size, gfp, mask) \
116 __percpu_populate_mask((__pdata), (size), (gfp), &(mask))
117#define percpu_depopulate_mask(__pdata, mask) \
118 __percpu_depopulate_mask((__pdata), &(mask))
119#define percpu_alloc_mask(size, gfp, mask) \
120 __percpu_alloc_mask((size), (gfp), &(mask))
121
122#define percpu_alloc(size, gfp) percpu_alloc_mask((size), (gfp), cpu_online_map)
123
124/* (legacy) interface for use without CPU hotplug handling */
125
126#define __alloc_percpu(size) percpu_alloc_mask((size), GFP_KERNEL, \
127 cpu_possible_map)
128#define alloc_percpu(type) (type *)__alloc_percpu(sizeof(type))
129#define free_percpu(ptr) percpu_free((ptr))
130#define per_cpu_ptr(ptr, cpu) percpu_ptr((ptr), (cpu))
131
132#endif /* __LINUX_PERCPU_H */