tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */
2
3/*
4 * AF_XDP user-space access library.
5 *
6 * Copyright(c) 2018 - 2019 Intel Corporation.
7 *
8 * Author(s): Magnus Karlsson <magnus.karlsson@intel.com>
9 */
10
11#ifndef __LIBBPF_XSK_H
12#define __LIBBPF_XSK_H
13
14#include <stdio.h>
15#include <stdint.h>
16#include <linux/if_xdp.h>
17
18#include "libbpf.h"
19#include "libbpf_util.h"
20
21#ifdef __cplusplus
22extern "C" {
23#endif
24
25/* Do not access these members directly. Use the functions below. */
26#define DEFINE_XSK_RING(name) \
27struct name { \
28 __u32 cached_prod; \
29 __u32 cached_cons; \
30 __u32 mask; \
31 __u32 size; \
32 __u32 *producer; \
33 __u32 *consumer; \
34 void *ring; \
35}
36
37DEFINE_XSK_RING(xsk_ring_prod);
38DEFINE_XSK_RING(xsk_ring_cons);
39
40/* For a detailed explanation on the memory barriers associated with the
41 * ring, please take a look at net/xdp/xsk_queue.h.
42 */
43
44struct xsk_umem;
45struct xsk_socket;
46
47static inline __u64 *xsk_ring_prod__fill_addr(struct xsk_ring_prod *fill,
48 __u32 idx)
49{
50 __u64 *addrs = (__u64 *)fill->ring;
51
52 return &addrs[idx & fill->mask];
53}
54
55static inline const __u64 *
56xsk_ring_cons__comp_addr(const struct xsk_ring_cons *comp, __u32 idx)
57{
58 const __u64 *addrs = (const __u64 *)comp->ring;
59
60 return &addrs[idx & comp->mask];
61}
62
63static inline struct xdp_desc *xsk_ring_prod__tx_desc(struct xsk_ring_prod *tx,
64 __u32 idx)
65{
66 struct xdp_desc *descs = (struct xdp_desc *)tx->ring;
67
68 return &descs[idx & tx->mask];
69}
70
71static inline const struct xdp_desc *
72xsk_ring_cons__rx_desc(const struct xsk_ring_cons *rx, __u32 idx)
73{
74 const struct xdp_desc *descs = (const struct xdp_desc *)rx->ring;
75
76 return &descs[idx & rx->mask];
77}
78
79static inline __u32 xsk_prod_nb_free(struct xsk_ring_prod *r, __u32 nb)
80{
81 __u32 free_entries = r->cached_cons - r->cached_prod;
82
83 if (free_entries >= nb)
84 return free_entries;
85
86 /* Refresh the local tail pointer.
87 * cached_cons is r->size bigger than the real consumer pointer so
88 * that this addition can be avoided in the more frequently
89 * executed code that computs free_entries in the beginning of
90 * this function. Without this optimization it whould have been
91 * free_entries = r->cached_prod - r->cached_cons + r->size.
92 */
93 r->cached_cons = *r->consumer + r->size;
94
95 return r->cached_cons - r->cached_prod;
96}
97
98static inline __u32 xsk_cons_nb_avail(struct xsk_ring_cons *r, __u32 nb)
99{
100 __u32 entries = r->cached_prod - r->cached_cons;
101
102 if (entries == 0) {
103 r->cached_prod = *r->producer;
104 entries = r->cached_prod - r->cached_cons;
105 }
106
107 return (entries > nb) ? nb : entries;
108}
109
110static inline size_t xsk_ring_prod__reserve(struct xsk_ring_prod *prod,
111 size_t nb, __u32 *idx)
112{
113 if (xsk_prod_nb_free(prod, nb) < nb)
114 return 0;
115
116 *idx = prod->cached_prod;
117 prod->cached_prod += nb;
118
119 return nb;
120}
121
122static inline void xsk_ring_prod__submit(struct xsk_ring_prod *prod, size_t nb)
123{
124 /* Make sure everything has been written to the ring before indicating
125 * this to the kernel by writing the producer pointer.
126 */
127 libbpf_smp_wmb();
128
129 *prod->producer += nb;
130}
131
132static inline size_t xsk_ring_cons__peek(struct xsk_ring_cons *cons,
133 size_t nb, __u32 *idx)
134{
135 size_t entries = xsk_cons_nb_avail(cons, nb);
136
137 if (entries > 0) {
138 /* Make sure we do not speculatively read the data before
139 * we have received the packet buffers from the ring.
140 */
141 libbpf_smp_rmb();
142
143 *idx = cons->cached_cons;
144 cons->cached_cons += entries;
145 }
146
147 return entries;
148}
149
150static inline void xsk_ring_cons__release(struct xsk_ring_cons *cons, size_t nb)
151{
152 /* Make sure data has been read before indicating we are done
153 * with the entries by updating the consumer pointer.
154 */
155 libbpf_smp_rwmb();
156
157 *cons->consumer += nb;
158}
159
160static inline void *xsk_umem__get_data(void *umem_area, __u64 addr)
161{
162 return &((char *)umem_area)[addr];
163}
164
165LIBBPF_API int xsk_umem__fd(const struct xsk_umem *umem);
166LIBBPF_API int xsk_socket__fd(const struct xsk_socket *xsk);
167
168#define XSK_RING_CONS__DEFAULT_NUM_DESCS 2048
169#define XSK_RING_PROD__DEFAULT_NUM_DESCS 2048
170#define XSK_UMEM__DEFAULT_FRAME_SHIFT 12 /* 4096 bytes */
171#define XSK_UMEM__DEFAULT_FRAME_SIZE (1 << XSK_UMEM__DEFAULT_FRAME_SHIFT)
172#define XSK_UMEM__DEFAULT_FRAME_HEADROOM 0
173
174struct xsk_umem_config {
175 __u32 fill_size;
176 __u32 comp_size;
177 __u32 frame_size;
178 __u32 frame_headroom;
179};
180
181/* Flags for the libbpf_flags field. */
182#define XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD (1 << 0)
183
184struct xsk_socket_config {
185 __u32 rx_size;
186 __u32 tx_size;
187 __u32 libbpf_flags;
188 __u32 xdp_flags;
189 __u16 bind_flags;
190};
191
192/* Set config to NULL to get the default configuration. */
193LIBBPF_API int xsk_umem__create(struct xsk_umem **umem,
194 void *umem_area, __u64 size,
195 struct xsk_ring_prod *fill,
196 struct xsk_ring_cons *comp,
197 const struct xsk_umem_config *config);
198LIBBPF_API int xsk_socket__create(struct xsk_socket **xsk,
199 const char *ifname, __u32 queue_id,
200 struct xsk_umem *umem,
201 struct xsk_ring_cons *rx,
202 struct xsk_ring_prod *tx,
203 const struct xsk_socket_config *config);
204
205/* Returns 0 for success and -EBUSY if the umem is still in use. */
206LIBBPF_API int xsk_umem__delete(struct xsk_umem *umem);
207LIBBPF_API void xsk_socket__delete(struct xsk_socket *xsk);
208
209#ifdef __cplusplus
210} /* extern "C" */
211#endif
212
213#endif /* __LIBBPF_XSK_H */