tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Memory Controller-related BPF kfuncs and auxiliary code
4 *
5 * Author: Roman Gushchin <roman.gushchin@linux.dev>
6 */
7
8#include <linux/memcontrol.h>
9#include <linux/bpf.h>
10
11__bpf_kfunc_start_defs();
12
13/**
14 * bpf_get_root_mem_cgroup - Returns a pointer to the root memory cgroup
15 *
16 * The function has KF_ACQUIRE semantics, even though the root memory
17 * cgroup is never destroyed after being created and doesn't require
18 * reference counting. And it's perfectly safe to pass it to
19 * bpf_put_mem_cgroup()
20 *
21 * Return: A pointer to the root memory cgroup.
22 */
23__bpf_kfunc struct mem_cgroup *bpf_get_root_mem_cgroup(void)
24{
25 if (mem_cgroup_disabled())
26 return NULL;
27
28 /* css_get() is not needed */
29 return root_mem_cgroup;
30}
31
32/**
33 * bpf_get_mem_cgroup - Get a reference to a memory cgroup
34 * @css: pointer to the css structure
35 *
36 * It's fine to pass a css which belongs to any cgroup controller,
37 * e.g. unified hierarchy's main css.
38 *
39 * Implements KF_ACQUIRE semantics.
40 *
41 * Return: A pointer to a mem_cgroup structure after bumping
42 * the corresponding css's reference counter.
43 */
44__bpf_kfunc struct mem_cgroup *
45bpf_get_mem_cgroup(struct cgroup_subsys_state *css)
46{
47 struct mem_cgroup *memcg = NULL;
48 bool rcu_unlock = false;
49
50 if (mem_cgroup_disabled() || !root_mem_cgroup)
51 return NULL;
52
53 if (root_mem_cgroup->css.ss != css->ss) {
54 struct cgroup *cgroup = css->cgroup;
55 int ssid = root_mem_cgroup->css.ss->id;
56
57 rcu_read_lock();
58 rcu_unlock = true;
59 css = rcu_dereference_raw(cgroup->subsys[ssid]);
60 }
61
62 if (css && css_tryget(css))
63 memcg = container_of(css, struct mem_cgroup, css);
64
65 if (rcu_unlock)
66 rcu_read_unlock();
67
68 return memcg;
69}
70
71/**
72 * bpf_put_mem_cgroup - Put a reference to a memory cgroup
73 * @memcg: memory cgroup to release
74 *
75 * Releases a previously acquired memcg reference.
76 * Implements KF_RELEASE semantics.
77 */
78__bpf_kfunc void bpf_put_mem_cgroup(struct mem_cgroup *memcg)
79{
80 css_put(&memcg->css);
81}
82
83/**
84 * bpf_mem_cgroup_vm_events - Read memory cgroup's vm event counter
85 * @memcg: memory cgroup
86 * @event: event id
87 *
88 * Allows to read memory cgroup event counters.
89 *
90 * Return: The current value of the corresponding events counter.
91 */
92__bpf_kfunc unsigned long bpf_mem_cgroup_vm_events(struct mem_cgroup *memcg,
93 enum vm_event_item event)
94{
95 if (unlikely(!memcg_vm_event_item_valid(event)))
96 return (unsigned long)-1;
97
98 return memcg_events(memcg, event);
99}
100
101/**
102 * bpf_mem_cgroup_usage - Read memory cgroup's usage
103 * @memcg: memory cgroup
104 *
105 * Please, note that the root memory cgroup it special and is exempt
106 * from the memory accounting. The returned value is a sum of sub-cgroup's
107 * usages and it not reflecting the size of the root memory cgroup itself.
108 * If you need to get an approximation, you can use root level statistics:
109 * e.g. NR_FILE_PAGES + NR_ANON_MAPPED.
110 *
111 * Return: The current memory cgroup size in bytes.
112 */
113__bpf_kfunc unsigned long bpf_mem_cgroup_usage(struct mem_cgroup *memcg)
114{
115 return page_counter_read(&memcg->memory) * PAGE_SIZE;
116}
117
118/**
119 * bpf_mem_cgroup_memory_events - Read memory cgroup's memory event value
120 * @memcg: memory cgroup
121 * @event: memory event id
122 *
123 * Return: The current value of the memory event counter.
124 */
125__bpf_kfunc unsigned long bpf_mem_cgroup_memory_events(struct mem_cgroup *memcg,
126 enum memcg_memory_event event)
127{
128 if (unlikely(event >= MEMCG_NR_MEMORY_EVENTS))
129 return (unsigned long)-1;
130
131 return atomic_long_read(&memcg->memory_events[event]);
132}
133
134/**
135 * bpf_mem_cgroup_page_state - Read memory cgroup's page state counter
136 * @memcg: memory cgroup
137 * @idx: counter idx
138 *
139 * Allows to read memory cgroup statistics. The output is in bytes.
140 *
141 * Return: The value of the page state counter in bytes.
142 */
143__bpf_kfunc unsigned long bpf_mem_cgroup_page_state(struct mem_cgroup *memcg, int idx)
144{
145 if (unlikely(!memcg_stat_item_valid(idx)))
146 return (unsigned long)-1;
147
148 return memcg_page_state_output(memcg, idx);
149}
150
151/**
152 * bpf_mem_cgroup_flush_stats - Flush memory cgroup's statistics
153 * @memcg: memory cgroup
154 *
155 * Propagate memory cgroup's statistics up the cgroup tree.
156 */
157__bpf_kfunc void bpf_mem_cgroup_flush_stats(struct mem_cgroup *memcg)
158{
159 mem_cgroup_flush_stats(memcg);
160}
161
162__bpf_kfunc_end_defs();
163
164BTF_KFUNCS_START(bpf_memcontrol_kfuncs)
165BTF_ID_FLAGS(func, bpf_get_root_mem_cgroup, KF_ACQUIRE | KF_RET_NULL)
166BTF_ID_FLAGS(func, bpf_get_mem_cgroup, KF_ACQUIRE | KF_RET_NULL | KF_RCU)
167BTF_ID_FLAGS(func, bpf_put_mem_cgroup, KF_RELEASE)
168
169BTF_ID_FLAGS(func, bpf_mem_cgroup_vm_events)
170BTF_ID_FLAGS(func, bpf_mem_cgroup_memory_events)
171BTF_ID_FLAGS(func, bpf_mem_cgroup_usage)
172BTF_ID_FLAGS(func, bpf_mem_cgroup_page_state)
173BTF_ID_FLAGS(func, bpf_mem_cgroup_flush_stats, KF_SLEEPABLE)
174
175BTF_KFUNCS_END(bpf_memcontrol_kfuncs)
176
177static const struct btf_kfunc_id_set bpf_memcontrol_kfunc_set = {
178 .owner = THIS_MODULE,
179 .set = &bpf_memcontrol_kfuncs,
180};
181
182static int __init bpf_memcontrol_init(void)
183{
184 int err;
185
186 err = register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC,
187 &bpf_memcontrol_kfunc_set);
188 if (err)
189 pr_warn("error while registering bpf memcontrol kfuncs: %d", err);
190
191 return err;
192}
193late_initcall(bpf_memcontrol_init);