Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_VMSTAT_H
3#define _LINUX_VMSTAT_H
4
5#include <linux/types.h>
6#include <linux/percpu.h>
7#include <linux/mmzone.h>
8#include <linux/vm_event_item.h>
9#include <linux/atomic.h>
10#include <linux/static_key.h>
11
12extern int sysctl_stat_interval;
13
14#ifdef CONFIG_NUMA
15#define ENABLE_NUMA_STAT 1
16#define DISABLE_NUMA_STAT 0
17extern int sysctl_vm_numa_stat;
18DECLARE_STATIC_KEY_TRUE(vm_numa_stat_key);
19extern int sysctl_vm_numa_stat_handler(struct ctl_table *table,
20 int write, void __user *buffer, size_t *length, loff_t *ppos);
21#endif
22
23#ifdef CONFIG_VM_EVENT_COUNTERS
24/*
25 * Light weight per cpu counter implementation.
26 *
27 * Counters should only be incremented and no critical kernel component
28 * should rely on the counter values.
29 *
30 * Counters are handled completely inline. On many platforms the code
31 * generated will simply be the increment of a global address.
32 */
33
34struct vm_event_state {
35 unsigned long event[NR_VM_EVENT_ITEMS];
36};
37
38DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
39
40/*
41 * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
42 * local_irq_disable overhead.
43 */
44static inline void __count_vm_event(enum vm_event_item item)
45{
46 raw_cpu_inc(vm_event_states.event[item]);
47}
48
49static inline void count_vm_event(enum vm_event_item item)
50{
51 this_cpu_inc(vm_event_states.event[item]);
52}
53
54static inline void __count_vm_events(enum vm_event_item item, long delta)
55{
56 raw_cpu_add(vm_event_states.event[item], delta);
57}
58
59static inline void count_vm_events(enum vm_event_item item, long delta)
60{
61 this_cpu_add(vm_event_states.event[item], delta);
62}
63
64extern void all_vm_events(unsigned long *);
65
66extern void vm_events_fold_cpu(int cpu);
67
68#else
69
70/* Disable counters */
71static inline void count_vm_event(enum vm_event_item item)
72{
73}
74static inline void count_vm_events(enum vm_event_item item, long delta)
75{
76}
77static inline void __count_vm_event(enum vm_event_item item)
78{
79}
80static inline void __count_vm_events(enum vm_event_item item, long delta)
81{
82}
83static inline void all_vm_events(unsigned long *ret)
84{
85}
86static inline void vm_events_fold_cpu(int cpu)
87{
88}
89
90#endif /* CONFIG_VM_EVENT_COUNTERS */
91
92#ifdef CONFIG_NUMA_BALANCING
93#define count_vm_numa_event(x) count_vm_event(x)
94#define count_vm_numa_events(x, y) count_vm_events(x, y)
95#else
96#define count_vm_numa_event(x) do {} while (0)
97#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
98#endif /* CONFIG_NUMA_BALANCING */
99
100#ifdef CONFIG_DEBUG_TLBFLUSH
101#define count_vm_tlb_event(x) count_vm_event(x)
102#define count_vm_tlb_events(x, y) count_vm_events(x, y)
103#else
104#define count_vm_tlb_event(x) do {} while (0)
105#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
106#endif
107
108#ifdef CONFIG_DEBUG_VM_VMACACHE
109#define count_vm_vmacache_event(x) count_vm_event(x)
110#else
111#define count_vm_vmacache_event(x) do {} while (0)
112#endif
113
114#define __count_zid_vm_events(item, zid, delta) \
115 __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
116
117/*
118 * Zone and node-based page accounting with per cpu differentials.
119 */
120extern atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS];
121extern atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS];
122extern atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS];
123
124#ifdef CONFIG_NUMA
125static inline void zone_numa_state_add(long x, struct zone *zone,
126 enum numa_stat_item item)
127{
128 atomic_long_add(x, &zone->vm_numa_stat[item]);
129 atomic_long_add(x, &vm_numa_stat[item]);
130}
131
132static inline unsigned long global_numa_state(enum numa_stat_item item)
133{
134 long x = atomic_long_read(&vm_numa_stat[item]);
135
136 return x;
137}
138
139static inline unsigned long zone_numa_state_snapshot(struct zone *zone,
140 enum numa_stat_item item)
141{
142 long x = atomic_long_read(&zone->vm_numa_stat[item]);
143 int cpu;
144
145 for_each_online_cpu(cpu)
146 x += per_cpu_ptr(zone->pageset, cpu)->vm_numa_stat_diff[item];
147
148 return x;
149}
150#endif /* CONFIG_NUMA */
151
152static inline void zone_page_state_add(long x, struct zone *zone,
153 enum zone_stat_item item)
154{
155 atomic_long_add(x, &zone->vm_stat[item]);
156 atomic_long_add(x, &vm_zone_stat[item]);
157}
158
159static inline void node_page_state_add(long x, struct pglist_data *pgdat,
160 enum node_stat_item item)
161{
162 atomic_long_add(x, &pgdat->vm_stat[item]);
163 atomic_long_add(x, &vm_node_stat[item]);
164}
165
166static inline unsigned long global_zone_page_state(enum zone_stat_item item)
167{
168 long x = atomic_long_read(&vm_zone_stat[item]);
169#ifdef CONFIG_SMP
170 if (x < 0)
171 x = 0;
172#endif
173 return x;
174}
175
176static inline unsigned long global_node_page_state(enum node_stat_item item)
177{
178 long x = atomic_long_read(&vm_node_stat[item]);
179#ifdef CONFIG_SMP
180 if (x < 0)
181 x = 0;
182#endif
183 return x;
184}
185
186static inline unsigned long zone_page_state(struct zone *zone,
187 enum zone_stat_item item)
188{
189 long x = atomic_long_read(&zone->vm_stat[item]);
190#ifdef CONFIG_SMP
191 if (x < 0)
192 x = 0;
193#endif
194 return x;
195}
196
197/*
198 * More accurate version that also considers the currently pending
199 * deltas. For that we need to loop over all cpus to find the current
200 * deltas. There is no synchronization so the result cannot be
201 * exactly accurate either.
202 */
203static inline unsigned long zone_page_state_snapshot(struct zone *zone,
204 enum zone_stat_item item)
205{
206 long x = atomic_long_read(&zone->vm_stat[item]);
207
208#ifdef CONFIG_SMP
209 int cpu;
210 for_each_online_cpu(cpu)
211 x += per_cpu_ptr(zone->pageset, cpu)->vm_stat_diff[item];
212
213 if (x < 0)
214 x = 0;
215#endif
216 return x;
217}
218
219#ifdef CONFIG_NUMA
220extern void __inc_numa_state(struct zone *zone, enum numa_stat_item item);
221extern unsigned long sum_zone_node_page_state(int node,
222 enum zone_stat_item item);
223extern unsigned long sum_zone_numa_state(int node, enum numa_stat_item item);
224extern unsigned long node_page_state(struct pglist_data *pgdat,
225 enum node_stat_item item);
226#else
227#define sum_zone_node_page_state(node, item) global_zone_page_state(item)
228#define node_page_state(node, item) global_node_page_state(item)
229#endif /* CONFIG_NUMA */
230
231#define add_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, __d)
232#define sub_zone_page_state(__z, __i, __d) mod_zone_page_state(__z, __i, -(__d))
233#define add_node_page_state(__p, __i, __d) mod_node_page_state(__p, __i, __d)
234#define sub_node_page_state(__p, __i, __d) mod_node_page_state(__p, __i, -(__d))
235
236#ifdef CONFIG_SMP
237void __mod_zone_page_state(struct zone *, enum zone_stat_item item, long);
238void __inc_zone_page_state(struct page *, enum zone_stat_item);
239void __dec_zone_page_state(struct page *, enum zone_stat_item);
240
241void __mod_node_page_state(struct pglist_data *, enum node_stat_item item, long);
242void __inc_node_page_state(struct page *, enum node_stat_item);
243void __dec_node_page_state(struct page *, enum node_stat_item);
244
245void mod_zone_page_state(struct zone *, enum zone_stat_item, long);
246void inc_zone_page_state(struct page *, enum zone_stat_item);
247void dec_zone_page_state(struct page *, enum zone_stat_item);
248
249void mod_node_page_state(struct pglist_data *, enum node_stat_item, long);
250void inc_node_page_state(struct page *, enum node_stat_item);
251void dec_node_page_state(struct page *, enum node_stat_item);
252
253extern void inc_node_state(struct pglist_data *, enum node_stat_item);
254extern void __inc_zone_state(struct zone *, enum zone_stat_item);
255extern void __inc_node_state(struct pglist_data *, enum node_stat_item);
256extern void dec_zone_state(struct zone *, enum zone_stat_item);
257extern void __dec_zone_state(struct zone *, enum zone_stat_item);
258extern void __dec_node_state(struct pglist_data *, enum node_stat_item);
259
260void quiet_vmstat(void);
261void cpu_vm_stats_fold(int cpu);
262void refresh_zone_stat_thresholds(void);
263
264struct ctl_table;
265int vmstat_refresh(struct ctl_table *, int write,
266 void __user *buffer, size_t *lenp, loff_t *ppos);
267
268void drain_zonestat(struct zone *zone, struct per_cpu_pageset *);
269
270int calculate_pressure_threshold(struct zone *zone);
271int calculate_normal_threshold(struct zone *zone);
272void set_pgdat_percpu_threshold(pg_data_t *pgdat,
273 int (*calculate_pressure)(struct zone *));
274#else /* CONFIG_SMP */
275
276/*
277 * We do not maintain differentials in a single processor configuration.
278 * The functions directly modify the zone and global counters.
279 */
280static inline void __mod_zone_page_state(struct zone *zone,
281 enum zone_stat_item item, long delta)
282{
283 zone_page_state_add(delta, zone, item);
284}
285
286static inline void __mod_node_page_state(struct pglist_data *pgdat,
287 enum node_stat_item item, int delta)
288{
289 node_page_state_add(delta, pgdat, item);
290}
291
292static inline void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
293{
294 atomic_long_inc(&zone->vm_stat[item]);
295 atomic_long_inc(&vm_zone_stat[item]);
296}
297
298static inline void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
299{
300 atomic_long_inc(&pgdat->vm_stat[item]);
301 atomic_long_inc(&vm_node_stat[item]);
302}
303
304static inline void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
305{
306 atomic_long_dec(&zone->vm_stat[item]);
307 atomic_long_dec(&vm_zone_stat[item]);
308}
309
310static inline void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
311{
312 atomic_long_dec(&pgdat->vm_stat[item]);
313 atomic_long_dec(&vm_node_stat[item]);
314}
315
316static inline void __inc_zone_page_state(struct page *page,
317 enum zone_stat_item item)
318{
319 __inc_zone_state(page_zone(page), item);
320}
321
322static inline void __inc_node_page_state(struct page *page,
323 enum node_stat_item item)
324{
325 __inc_node_state(page_pgdat(page), item);
326}
327
328
329static inline void __dec_zone_page_state(struct page *page,
330 enum zone_stat_item item)
331{
332 __dec_zone_state(page_zone(page), item);
333}
334
335static inline void __dec_node_page_state(struct page *page,
336 enum node_stat_item item)
337{
338 __dec_node_state(page_pgdat(page), item);
339}
340
341
342/*
343 * We only use atomic operations to update counters. So there is no need to
344 * disable interrupts.
345 */
346#define inc_zone_page_state __inc_zone_page_state
347#define dec_zone_page_state __dec_zone_page_state
348#define mod_zone_page_state __mod_zone_page_state
349
350#define inc_node_page_state __inc_node_page_state
351#define dec_node_page_state __dec_node_page_state
352#define mod_node_page_state __mod_node_page_state
353
354#define inc_zone_state __inc_zone_state
355#define inc_node_state __inc_node_state
356#define dec_zone_state __dec_zone_state
357
358#define set_pgdat_percpu_threshold(pgdat, callback) { }
359
360static inline void refresh_zone_stat_thresholds(void) { }
361static inline void cpu_vm_stats_fold(int cpu) { }
362static inline void quiet_vmstat(void) { }
363
364static inline void drain_zonestat(struct zone *zone,
365 struct per_cpu_pageset *pset) { }
366#endif /* CONFIG_SMP */
367
368static inline void __mod_zone_freepage_state(struct zone *zone, int nr_pages,
369 int migratetype)
370{
371 __mod_zone_page_state(zone, NR_FREE_PAGES, nr_pages);
372 if (is_migrate_cma(migratetype))
373 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES, nr_pages);
374}
375
376extern const char * const vmstat_text[];
377
378#endif /* _LINUX_VMSTAT_H */