Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef _LINUX_PAGE_REF_H
3#define _LINUX_PAGE_REF_H
4
5#include <linux/atomic.h>
6#include <linux/mm_types.h>
7#include <linux/page-flags.h>
8#include <linux/tracepoint-defs.h>
9
10DECLARE_TRACEPOINT(page_ref_set);
11DECLARE_TRACEPOINT(page_ref_mod);
12DECLARE_TRACEPOINT(page_ref_mod_and_test);
13DECLARE_TRACEPOINT(page_ref_mod_and_return);
14DECLARE_TRACEPOINT(page_ref_mod_unless);
15DECLARE_TRACEPOINT(page_ref_freeze);
16DECLARE_TRACEPOINT(page_ref_unfreeze);
17
18#ifdef CONFIG_DEBUG_PAGE_REF
19
20/*
21 * Ideally we would want to use the trace_<tracepoint>_enabled() helper
22 * functions. But due to include header file issues, that is not
23 * feasible. Instead we have to open code the static key functions.
24 *
25 * See trace_##name##_enabled(void) in include/linux/tracepoint.h
26 */
27#define page_ref_tracepoint_active(t) tracepoint_enabled(t)
28
29extern void __page_ref_set(struct page *page, int v);
30extern void __page_ref_mod(struct page *page, int v);
31extern void __page_ref_mod_and_test(struct page *page, int v, int ret);
32extern void __page_ref_mod_and_return(struct page *page, int v, int ret);
33extern void __page_ref_mod_unless(struct page *page, int v, int u);
34extern void __page_ref_freeze(struct page *page, int v, int ret);
35extern void __page_ref_unfreeze(struct page *page, int v);
36
37#else
38
39#define page_ref_tracepoint_active(t) false
40
41static inline void __page_ref_set(struct page *page, int v)
42{
43}
44static inline void __page_ref_mod(struct page *page, int v)
45{
46}
47static inline void __page_ref_mod_and_test(struct page *page, int v, int ret)
48{
49}
50static inline void __page_ref_mod_and_return(struct page *page, int v, int ret)
51{
52}
53static inline void __page_ref_mod_unless(struct page *page, int v, int u)
54{
55}
56static inline void __page_ref_freeze(struct page *page, int v, int ret)
57{
58}
59static inline void __page_ref_unfreeze(struct page *page, int v)
60{
61}
62
63#endif
64
65static inline int page_ref_count(struct page *page)
66{
67 return atomic_read(&page->_refcount);
68}
69
70static inline int page_count(struct page *page)
71{
72 return atomic_read(&compound_head(page)->_refcount);
73}
74
75static inline void set_page_count(struct page *page, int v)
76{
77 atomic_set(&page->_refcount, v);
78 if (page_ref_tracepoint_active(page_ref_set))
79 __page_ref_set(page, v);
80}
81
82/*
83 * Setup the page count before being freed into the page allocator for
84 * the first time (boot or memory hotplug)
85 */
86static inline void init_page_count(struct page *page)
87{
88 set_page_count(page, 1);
89}
90
91static inline void page_ref_add(struct page *page, int nr)
92{
93 atomic_add(nr, &page->_refcount);
94 if (page_ref_tracepoint_active(page_ref_mod))
95 __page_ref_mod(page, nr);
96}
97
98static inline void page_ref_sub(struct page *page, int nr)
99{
100 atomic_sub(nr, &page->_refcount);
101 if (page_ref_tracepoint_active(page_ref_mod))
102 __page_ref_mod(page, -nr);
103}
104
105static inline int page_ref_sub_return(struct page *page, int nr)
106{
107 int ret = atomic_sub_return(nr, &page->_refcount);
108
109 if (page_ref_tracepoint_active(page_ref_mod_and_return))
110 __page_ref_mod_and_return(page, -nr, ret);
111 return ret;
112}
113
114static inline void page_ref_inc(struct page *page)
115{
116 atomic_inc(&page->_refcount);
117 if (page_ref_tracepoint_active(page_ref_mod))
118 __page_ref_mod(page, 1);
119}
120
121static inline void page_ref_dec(struct page *page)
122{
123 atomic_dec(&page->_refcount);
124 if (page_ref_tracepoint_active(page_ref_mod))
125 __page_ref_mod(page, -1);
126}
127
128static inline int page_ref_sub_and_test(struct page *page, int nr)
129{
130 int ret = atomic_sub_and_test(nr, &page->_refcount);
131
132 if (page_ref_tracepoint_active(page_ref_mod_and_test))
133 __page_ref_mod_and_test(page, -nr, ret);
134 return ret;
135}
136
137static inline int page_ref_inc_return(struct page *page)
138{
139 int ret = atomic_inc_return(&page->_refcount);
140
141 if (page_ref_tracepoint_active(page_ref_mod_and_return))
142 __page_ref_mod_and_return(page, 1, ret);
143 return ret;
144}
145
146static inline int page_ref_dec_and_test(struct page *page)
147{
148 int ret = atomic_dec_and_test(&page->_refcount);
149
150 if (page_ref_tracepoint_active(page_ref_mod_and_test))
151 __page_ref_mod_and_test(page, -1, ret);
152 return ret;
153}
154
155static inline int page_ref_dec_return(struct page *page)
156{
157 int ret = atomic_dec_return(&page->_refcount);
158
159 if (page_ref_tracepoint_active(page_ref_mod_and_return))
160 __page_ref_mod_and_return(page, -1, ret);
161 return ret;
162}
163
164static inline int page_ref_add_unless(struct page *page, int nr, int u)
165{
166 int ret = atomic_add_unless(&page->_refcount, nr, u);
167
168 if (page_ref_tracepoint_active(page_ref_mod_unless))
169 __page_ref_mod_unless(page, nr, ret);
170 return ret;
171}
172
173static inline int page_ref_freeze(struct page *page, int count)
174{
175 int ret = likely(atomic_cmpxchg(&page->_refcount, count, 0) == count);
176
177 if (page_ref_tracepoint_active(page_ref_freeze))
178 __page_ref_freeze(page, count, ret);
179 return ret;
180}
181
182static inline void page_ref_unfreeze(struct page *page, int count)
183{
184 VM_BUG_ON_PAGE(page_count(page) != 0, page);
185 VM_BUG_ON(count == 0);
186
187 atomic_set_release(&page->_refcount, count);
188 if (page_ref_tracepoint_active(page_ref_unfreeze))
189 __page_ref_unfreeze(page, count);
190}
191
192#endif