Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2024 Google */#include <linux/bpf.h>#include <linux/btf_ids.h>#include <linux/slab.h>#include <linux/kernel.h>#include <linux/seq_file.h>
#include "../../mm/slab.h" /* kmem_cache, slab_caches and slab_mutex */
/* open-coded version */struct bpf_iter_kmem_cache { __u64 __opaque[1];} __attribute__((aligned(8)));
struct bpf_iter_kmem_cache_kern { struct kmem_cache *pos;} __attribute__((aligned(8)));
#define KMEM_CACHE_POS_START ((void *)1L)
__bpf_kfunc_start_defs();
__bpf_kfunc int bpf_iter_kmem_cache_new(struct bpf_iter_kmem_cache *it){ struct bpf_iter_kmem_cache_kern *kit = (void *)it;
BUILD_BUG_ON(sizeof(*kit) > sizeof(*it)); BUILD_BUG_ON(__alignof__(*kit) != __alignof__(*it));
kit->pos = KMEM_CACHE_POS_START; return 0;}
__bpf_kfunc struct kmem_cache *bpf_iter_kmem_cache_next(struct bpf_iter_kmem_cache *it){ struct bpf_iter_kmem_cache_kern *kit = (void *)it; struct kmem_cache *prev = kit->pos; struct kmem_cache *next; bool destroy = false;
if (!prev) return NULL;
mutex_lock(&slab_mutex);
if (list_empty(&slab_caches)) { mutex_unlock(&slab_mutex); return NULL; }
if (prev == KMEM_CACHE_POS_START) next = list_first_entry(&slab_caches, struct kmem_cache, list); else if (list_last_entry(&slab_caches, struct kmem_cache, list) == prev) next = NULL; else next = list_next_entry(prev, list);
/* boot_caches have negative refcount, don't touch them */ if (next && next->refcount > 0) next->refcount++;
/* Skip kmem_cache_destroy() for active entries */ if (prev && prev != KMEM_CACHE_POS_START) { if (prev->refcount > 1) prev->refcount--; else if (prev->refcount == 1) destroy = true; }
mutex_unlock(&slab_mutex);
if (destroy) kmem_cache_destroy(prev);
kit->pos = next; return next;}
__bpf_kfunc void bpf_iter_kmem_cache_destroy(struct bpf_iter_kmem_cache *it){ struct bpf_iter_kmem_cache_kern *kit = (void *)it; struct kmem_cache *s = kit->pos; bool destroy = false;
if (s == NULL || s == KMEM_CACHE_POS_START) return;
mutex_lock(&slab_mutex);
/* Skip kmem_cache_destroy() for active entries */ if (s->refcount > 1) s->refcount--; else if (s->refcount == 1) destroy = true;
mutex_unlock(&slab_mutex);
if (destroy) kmem_cache_destroy(s);}
__bpf_kfunc_end_defs();
struct bpf_iter__kmem_cache { __bpf_md_ptr(struct bpf_iter_meta *, meta); __bpf_md_ptr(struct kmem_cache *, s);};
union kmem_cache_iter_priv { struct bpf_iter_kmem_cache it; struct bpf_iter_kmem_cache_kern kit;};
static void *kmem_cache_iter_seq_start(struct seq_file *seq, loff_t *pos){ loff_t cnt = 0; bool found = false; struct kmem_cache *s; union kmem_cache_iter_priv *p = seq->private;
mutex_lock(&slab_mutex);
/* Find an entry at the given position in the slab_caches list instead * of keeping a reference (of the last visited entry, if any) out of * slab_mutex. It might miss something if one is deleted in the middle * while it releases the lock. But it should be rare and there's not * much we can do about it. */ list_for_each_entry(s, &slab_caches, list) { if (cnt == *pos) { /* Make sure this entry remains in the list by getting * a new reference count. Note that boot_cache entries * have a negative refcount, so don't touch them. */ if (s->refcount > 0) s->refcount++; found = true; break; } cnt++; } mutex_unlock(&slab_mutex);
if (!found) s = NULL;
p->kit.pos = s; return s;}
static void kmem_cache_iter_seq_stop(struct seq_file *seq, void *v){ struct bpf_iter_meta meta; struct bpf_iter__kmem_cache ctx = { .meta = &meta, .s = v, }; union kmem_cache_iter_priv *p = seq->private; struct bpf_prog *prog;
meta.seq = seq; prog = bpf_iter_get_info(&meta, true); if (prog && !ctx.s) bpf_iter_run_prog(prog, &ctx);
bpf_iter_kmem_cache_destroy(&p->it);}
static void *kmem_cache_iter_seq_next(struct seq_file *seq, void *v, loff_t *pos){ union kmem_cache_iter_priv *p = seq->private;
++*pos;
return bpf_iter_kmem_cache_next(&p->it);}
static int kmem_cache_iter_seq_show(struct seq_file *seq, void *v){ struct bpf_iter_meta meta; struct bpf_iter__kmem_cache ctx = { .meta = &meta, .s = v, }; struct bpf_prog *prog; int ret = 0;
meta.seq = seq; prog = bpf_iter_get_info(&meta, false); if (prog) ret = bpf_iter_run_prog(prog, &ctx);
return ret;}
static const struct seq_operations kmem_cache_iter_seq_ops = { .start = kmem_cache_iter_seq_start, .next = kmem_cache_iter_seq_next, .stop = kmem_cache_iter_seq_stop, .show = kmem_cache_iter_seq_show,};
BTF_ID_LIST_GLOBAL_SINGLE(bpf_kmem_cache_btf_id, struct, kmem_cache)
static const struct bpf_iter_seq_info kmem_cache_iter_seq_info = { .seq_ops = &kmem_cache_iter_seq_ops, .seq_priv_size = sizeof(union kmem_cache_iter_priv),};
static void bpf_iter_kmem_cache_show_fdinfo(const struct bpf_iter_aux_info *aux, struct seq_file *seq){ seq_puts(seq, "kmem_cache iter\n");}
DEFINE_BPF_ITER_FUNC(kmem_cache, struct bpf_iter_meta *meta, struct kmem_cache *s)
static struct bpf_iter_reg bpf_kmem_cache_reg_info = { .target = "kmem_cache", .feature = BPF_ITER_RESCHED, .show_fdinfo = bpf_iter_kmem_cache_show_fdinfo, .ctx_arg_info_size = 1, .ctx_arg_info = { { offsetof(struct bpf_iter__kmem_cache, s), PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED }, }, .seq_info = &kmem_cache_iter_seq_info,};
static int __init bpf_kmem_cache_iter_init(void){ bpf_kmem_cache_reg_info.ctx_arg_info[0].btf_id = bpf_kmem_cache_btf_id[0]; return bpf_iter_reg_target(&bpf_kmem_cache_reg_info);}
late_initcall(bpf_kmem_cache_iter_init);