Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (c) 2019 Facebook
4 * Copyright 2020 Google LLC.
5 */
6
7#ifndef _BPF_LOCAL_STORAGE_H
8#define _BPF_LOCAL_STORAGE_H
9
10#include <linux/bpf.h>
11#include <linux/filter.h>
12#include <linux/rculist.h>
13#include <linux/list.h>
14#include <linux/hash.h>
15#include <linux/types.h>
16#include <uapi/linux/btf.h>
17
18#define BPF_LOCAL_STORAGE_CACHE_SIZE 16
19
20#define bpf_rcu_lock_held() \
21 (rcu_read_lock_held() || rcu_read_lock_trace_held() || \
22 rcu_read_lock_bh_held())
23struct bpf_local_storage_map_bucket {
24 struct hlist_head list;
25 raw_spinlock_t lock;
26};
27
28/* Thp map is not the primary owner of a bpf_local_storage_elem.
29 * Instead, the container object (eg. sk->sk_bpf_storage) is.
30 *
31 * The map (bpf_local_storage_map) is for two purposes
32 * 1. Define the size of the "local storage". It is
33 * the map's value_size.
34 *
35 * 2. Maintain a list to keep track of all elems such
36 * that they can be cleaned up during the map destruction.
37 *
38 * When a bpf local storage is being looked up for a
39 * particular object, the "bpf_map" pointer is actually used
40 * as the "key" to search in the list of elem in
41 * the respective bpf_local_storage owned by the object.
42 *
43 * e.g. sk->sk_bpf_storage is the mini-map with the "bpf_map" pointer
44 * as the searching key.
45 */
46struct bpf_local_storage_map {
47 struct bpf_map map;
48 /* Lookup elem does not require accessing the map.
49 *
50 * Updating/Deleting requires a bucket lock to
51 * link/unlink the elem from the map. Having
52 * multiple buckets to improve contention.
53 */
54 struct bpf_local_storage_map_bucket *buckets;
55 u32 bucket_log;
56 u16 elem_size;
57 u16 cache_idx;
58};
59
60struct bpf_local_storage_data {
61 /* smap is used as the searching key when looking up
62 * from the object's bpf_local_storage.
63 *
64 * Put it in the same cacheline as the data to minimize
65 * the number of cachelines accessed during the cache hit case.
66 */
67 struct bpf_local_storage_map __rcu *smap;
68 u8 data[] __aligned(8);
69};
70
71/* Linked to bpf_local_storage and bpf_local_storage_map */
72struct bpf_local_storage_elem {
73 struct hlist_node map_node; /* Linked to bpf_local_storage_map */
74 struct hlist_node snode; /* Linked to bpf_local_storage */
75 struct bpf_local_storage __rcu *local_storage;
76 struct rcu_head rcu;
77 /* 8 bytes hole */
78 /* The data is stored in another cacheline to minimize
79 * the number of cachelines access during a cache hit.
80 */
81 struct bpf_local_storage_data sdata ____cacheline_aligned;
82};
83
84struct bpf_local_storage {
85 struct bpf_local_storage_data __rcu *cache[BPF_LOCAL_STORAGE_CACHE_SIZE];
86 struct hlist_head list; /* List of bpf_local_storage_elem */
87 void *owner; /* The object that owns the above "list" of
88 * bpf_local_storage_elem.
89 */
90 struct rcu_head rcu;
91 raw_spinlock_t lock; /* Protect adding/removing from the "list" */
92};
93
94/* U16_MAX is much more than enough for sk local storage
95 * considering a tcp_sock is ~2k.
96 */
97#define BPF_LOCAL_STORAGE_MAX_VALUE_SIZE \
98 min_t(u32, \
99 (KMALLOC_MAX_SIZE - MAX_BPF_STACK - \
100 sizeof(struct bpf_local_storage_elem)), \
101 (U16_MAX - sizeof(struct bpf_local_storage_elem)))
102
103#define SELEM(_SDATA) \
104 container_of((_SDATA), struct bpf_local_storage_elem, sdata)
105#define SDATA(_SELEM) (&(_SELEM)->sdata)
106
107#define BPF_LOCAL_STORAGE_CACHE_SIZE 16
108
109struct bpf_local_storage_cache {
110 spinlock_t idx_lock;
111 u64 idx_usage_counts[BPF_LOCAL_STORAGE_CACHE_SIZE];
112};
113
114#define DEFINE_BPF_STORAGE_CACHE(name) \
115static struct bpf_local_storage_cache name = { \
116 .idx_lock = __SPIN_LOCK_UNLOCKED(name.idx_lock), \
117}
118
119/* Helper functions for bpf_local_storage */
120int bpf_local_storage_map_alloc_check(union bpf_attr *attr);
121
122struct bpf_map *
123bpf_local_storage_map_alloc(union bpf_attr *attr,
124 struct bpf_local_storage_cache *cache);
125
126struct bpf_local_storage_data *
127bpf_local_storage_lookup(struct bpf_local_storage *local_storage,
128 struct bpf_local_storage_map *smap,
129 bool cacheit_lockit);
130
131bool bpf_local_storage_unlink_nolock(struct bpf_local_storage *local_storage);
132
133void bpf_local_storage_map_free(struct bpf_map *map,
134 struct bpf_local_storage_cache *cache,
135 int __percpu *busy_counter);
136
137int bpf_local_storage_map_check_btf(const struct bpf_map *map,
138 const struct btf *btf,
139 const struct btf_type *key_type,
140 const struct btf_type *value_type);
141
142void bpf_selem_link_storage_nolock(struct bpf_local_storage *local_storage,
143 struct bpf_local_storage_elem *selem);
144
145void bpf_selem_unlink(struct bpf_local_storage_elem *selem, bool use_trace_rcu);
146
147void bpf_selem_link_map(struct bpf_local_storage_map *smap,
148 struct bpf_local_storage_elem *selem);
149
150void bpf_selem_unlink_map(struct bpf_local_storage_elem *selem);
151
152struct bpf_local_storage_elem *
153bpf_selem_alloc(struct bpf_local_storage_map *smap, void *owner, void *value,
154 bool charge_mem, gfp_t gfp_flags);
155
156int
157bpf_local_storage_alloc(void *owner,
158 struct bpf_local_storage_map *smap,
159 struct bpf_local_storage_elem *first_selem,
160 gfp_t gfp_flags);
161
162struct bpf_local_storage_data *
163bpf_local_storage_update(void *owner, struct bpf_local_storage_map *smap,
164 void *value, u64 map_flags, gfp_t gfp_flags);
165
166void bpf_local_storage_free_rcu(struct rcu_head *rcu);
167
168#endif /* _BPF_LOCAL_STORAGE_H */