Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1#ifndef _LINUX_SLAB_DEF_H
2#define _LINUX_SLAB_DEF_H
3
4/*
5 * Definitions unique to the original Linux SLAB allocator.
6 *
7 * What we provide here is a way to optimize the frequent kmalloc
8 * calls in the kernel by selecting the appropriate general cache
9 * if kmalloc was called with a size that can be established at
10 * compile time.
11 */
12
13#include <linux/init.h>
14#include <linux/compiler.h>
15
16/*
17 * struct kmem_cache
18 *
19 * manages a cache.
20 */
21
22struct kmem_cache {
23/* 1) Cache tunables. Protected by cache_chain_mutex */
24 unsigned int batchcount;
25 unsigned int limit;
26 unsigned int shared;
27
28 unsigned int size;
29 u32 reciprocal_buffer_size;
30/* 2) touched by every alloc & free from the backend */
31
32 unsigned int flags; /* constant flags */
33 unsigned int num; /* # of objs per slab */
34
35/* 3) cache_grow/shrink */
36 /* order of pgs per slab (2^n) */
37 unsigned int gfporder;
38
39 /* force GFP flags, e.g. GFP_DMA */
40 gfp_t allocflags;
41
42 size_t colour; /* cache colouring range */
43 unsigned int colour_off; /* colour offset */
44 struct kmem_cache *slabp_cache;
45 unsigned int slab_size;
46
47 /* constructor func */
48 void (*ctor)(void *obj);
49
50/* 4) cache creation/removal */
51 const char *name;
52 struct list_head list;
53 int refcount;
54 int object_size;
55 int align;
56
57/* 5) statistics */
58#ifdef CONFIG_DEBUG_SLAB
59 unsigned long num_active;
60 unsigned long num_allocations;
61 unsigned long high_mark;
62 unsigned long grown;
63 unsigned long reaped;
64 unsigned long errors;
65 unsigned long max_freeable;
66 unsigned long node_allocs;
67 unsigned long node_frees;
68 unsigned long node_overflow;
69 atomic_t allochit;
70 atomic_t allocmiss;
71 atomic_t freehit;
72 atomic_t freemiss;
73
74 /*
75 * If debugging is enabled, then the allocator can add additional
76 * fields and/or padding to every object. size contains the total
77 * object size including these internal fields, the following two
78 * variables contain the offset to the user object and its size.
79 */
80 int obj_offset;
81#endif /* CONFIG_DEBUG_SLAB */
82#ifdef CONFIG_MEMCG_KMEM
83 struct memcg_cache_params *memcg_params;
84#endif
85
86/* 6) per-cpu/per-node data, touched during every alloc/free */
87 /*
88 * We put array[] at the end of kmem_cache, because we want to size
89 * this array to nr_cpu_ids slots instead of NR_CPUS
90 * (see kmem_cache_init())
91 * We still use [NR_CPUS] and not [1] or [0] because cache_cache
92 * is statically defined, so we reserve the max number of cpus.
93 *
94 * We also need to guarantee that the list is able to accomodate a
95 * pointer for each node since "nodelists" uses the remainder of
96 * available pointers.
97 */
98 struct kmem_cache_node **node;
99 struct array_cache *array[NR_CPUS + MAX_NUMNODES];
100 /*
101 * Do not add fields after array[]
102 */
103};
104
105void *kmem_cache_alloc(struct kmem_cache *, gfp_t);
106void *__kmalloc(size_t size, gfp_t flags);
107
108#ifdef CONFIG_TRACING
109extern void *kmem_cache_alloc_trace(struct kmem_cache *, gfp_t, size_t);
110#else
111static __always_inline void *
112kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
113{
114 return kmem_cache_alloc(cachep, flags);
115}
116#endif
117
118static __always_inline void *kmalloc(size_t size, gfp_t flags)
119{
120 struct kmem_cache *cachep;
121 void *ret;
122
123 if (__builtin_constant_p(size)) {
124 int i;
125
126 if (!size)
127 return ZERO_SIZE_PTR;
128
129 if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE))
130 return NULL;
131
132 i = kmalloc_index(size);
133
134#ifdef CONFIG_ZONE_DMA
135 if (flags & GFP_DMA)
136 cachep = kmalloc_dma_caches[i];
137 else
138#endif
139 cachep = kmalloc_caches[i];
140
141 ret = kmem_cache_alloc_trace(cachep, flags, size);
142
143 return ret;
144 }
145 return __kmalloc(size, flags);
146}
147
148#ifdef CONFIG_NUMA
149extern void *__kmalloc_node(size_t size, gfp_t flags, int node);
150extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node);
151
152#ifdef CONFIG_TRACING
153extern void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
154 gfp_t flags,
155 int nodeid,
156 size_t size);
157#else
158static __always_inline void *
159kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
160 gfp_t flags,
161 int nodeid,
162 size_t size)
163{
164 return kmem_cache_alloc_node(cachep, flags, nodeid);
165}
166#endif
167
168static __always_inline void *kmalloc_node(size_t size, gfp_t flags, int node)
169{
170 struct kmem_cache *cachep;
171
172 if (__builtin_constant_p(size)) {
173 int i;
174
175 if (!size)
176 return ZERO_SIZE_PTR;
177
178 if (WARN_ON_ONCE(size > KMALLOC_MAX_SIZE))
179 return NULL;
180
181 i = kmalloc_index(size);
182
183#ifdef CONFIG_ZONE_DMA
184 if (flags & GFP_DMA)
185 cachep = kmalloc_dma_caches[i];
186 else
187#endif
188 cachep = kmalloc_caches[i];
189
190 return kmem_cache_alloc_node_trace(cachep, flags, node, size);
191 }
192 return __kmalloc_node(size, flags, node);
193}
194
195#endif /* CONFIG_NUMA */
196
197#endif /* _LINUX_SLAB_DEF_H */