tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
fork
Configure Feed
Select the types of activity you want to include in your feed.
1// SPDX-License-Identifier: GPL-2.0-only
2
3#include <linux/netdevice.h>
4#include <linux/notifier.h>
5#include <linux/rtnetlink.h>
6#include <net/busy_poll.h>
7#include <net/net_namespace.h>
8#include <net/netdev_queues.h>
9#include <net/netdev_rx_queue.h>
10#include <net/sock.h>
11#include <net/xdp.h>
12#include <net/xdp_sock.h>
13#include <net/page_pool/memory_provider.h>
14
15#include "dev.h"
16#include "devmem.h"
17#include "netdev-genl-gen.h"
18
19struct netdev_nl_dump_ctx {
20 unsigned long ifindex;
21 unsigned int rxq_idx;
22 unsigned int txq_idx;
23 unsigned int napi_id;
24};
25
26static struct netdev_nl_dump_ctx *netdev_dump_ctx(struct netlink_callback *cb)
27{
28 NL_ASSERT_CTX_FITS(struct netdev_nl_dump_ctx);
29
30 return (struct netdev_nl_dump_ctx *)cb->ctx;
31}
32
33static int
34netdev_nl_dev_fill(struct net_device *netdev, struct sk_buff *rsp,
35 const struct genl_info *info)
36{
37 u64 xsk_features = 0;
38 u64 xdp_rx_meta = 0;
39 void *hdr;
40
41 hdr = genlmsg_iput(rsp, info);
42 if (!hdr)
43 return -EMSGSIZE;
44
45#define XDP_METADATA_KFUNC(_, flag, __, xmo) \
46 if (netdev->xdp_metadata_ops && netdev->xdp_metadata_ops->xmo) \
47 xdp_rx_meta |= flag;
48XDP_METADATA_KFUNC_xxx
49#undef XDP_METADATA_KFUNC
50
51 if (netdev->xsk_tx_metadata_ops) {
52 if (netdev->xsk_tx_metadata_ops->tmo_fill_timestamp)
53 xsk_features |= NETDEV_XSK_FLAGS_TX_TIMESTAMP;
54 if (netdev->xsk_tx_metadata_ops->tmo_request_checksum)
55 xsk_features |= NETDEV_XSK_FLAGS_TX_CHECKSUM;
56 if (netdev->xsk_tx_metadata_ops->tmo_request_launch_time)
57 xsk_features |= NETDEV_XSK_FLAGS_TX_LAUNCH_TIME_FIFO;
58 }
59
60 if (nla_put_u32(rsp, NETDEV_A_DEV_IFINDEX, netdev->ifindex) ||
61 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_FEATURES,
62 netdev->xdp_features, NETDEV_A_DEV_PAD) ||
63 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XDP_RX_METADATA_FEATURES,
64 xdp_rx_meta, NETDEV_A_DEV_PAD) ||
65 nla_put_u64_64bit(rsp, NETDEV_A_DEV_XSK_FEATURES,
66 xsk_features, NETDEV_A_DEV_PAD))
67 goto err_cancel_msg;
68
69 if (netdev->xdp_features & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
70 if (nla_put_u32(rsp, NETDEV_A_DEV_XDP_ZC_MAX_SEGS,
71 netdev->xdp_zc_max_segs))
72 goto err_cancel_msg;
73 }
74
75 genlmsg_end(rsp, hdr);
76
77 return 0;
78
79err_cancel_msg:
80 genlmsg_cancel(rsp, hdr);
81 return -EMSGSIZE;
82}
83
84static void
85netdev_genl_dev_notify(struct net_device *netdev, int cmd)
86{
87 struct genl_info info;
88 struct sk_buff *ntf;
89
90 if (!genl_has_listeners(&netdev_nl_family, dev_net(netdev),
91 NETDEV_NLGRP_MGMT))
92 return;
93
94 genl_info_init_ntf(&info, &netdev_nl_family, cmd);
95
96 ntf = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
97 if (!ntf)
98 return;
99
100 if (netdev_nl_dev_fill(netdev, ntf, &info)) {
101 nlmsg_free(ntf);
102 return;
103 }
104
105 genlmsg_multicast_netns(&netdev_nl_family, dev_net(netdev), ntf,
106 0, NETDEV_NLGRP_MGMT, GFP_KERNEL);
107}
108
109int netdev_nl_dev_get_doit(struct sk_buff *skb, struct genl_info *info)
110{
111 struct net_device *netdev;
112 struct sk_buff *rsp;
113 u32 ifindex;
114 int err;
115
116 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX))
117 return -EINVAL;
118
119 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
120
121 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
122 if (!rsp)
123 return -ENOMEM;
124
125 rtnl_lock();
126
127 netdev = __dev_get_by_index(genl_info_net(info), ifindex);
128 if (netdev)
129 err = netdev_nl_dev_fill(netdev, rsp, info);
130 else
131 err = -ENODEV;
132
133 rtnl_unlock();
134
135 if (err)
136 goto err_free_msg;
137
138 return genlmsg_reply(rsp, info);
139
140err_free_msg:
141 nlmsg_free(rsp);
142 return err;
143}
144
145int netdev_nl_dev_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
146{
147 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
148 struct net *net = sock_net(skb->sk);
149 struct net_device *netdev;
150 int err = 0;
151
152 rtnl_lock();
153 for_each_netdev_dump(net, netdev, ctx->ifindex) {
154 err = netdev_nl_dev_fill(netdev, skb, genl_info_dump(cb));
155 if (err < 0)
156 break;
157 }
158 rtnl_unlock();
159
160 return err;
161}
162
163static int
164netdev_nl_napi_fill_one(struct sk_buff *rsp, struct napi_struct *napi,
165 const struct genl_info *info)
166{
167 unsigned long irq_suspend_timeout;
168 unsigned long gro_flush_timeout;
169 u32 napi_defer_hard_irqs;
170 void *hdr;
171 pid_t pid;
172
173 if (!napi->dev->up)
174 return 0;
175
176 hdr = genlmsg_iput(rsp, info);
177 if (!hdr)
178 return -EMSGSIZE;
179
180 if (nla_put_u32(rsp, NETDEV_A_NAPI_ID, napi->napi_id))
181 goto nla_put_failure;
182
183 if (nla_put_u32(rsp, NETDEV_A_NAPI_IFINDEX, napi->dev->ifindex))
184 goto nla_put_failure;
185
186 if (napi->irq >= 0 && nla_put_u32(rsp, NETDEV_A_NAPI_IRQ, napi->irq))
187 goto nla_put_failure;
188
189 if (napi->thread) {
190 pid = task_pid_nr(napi->thread);
191 if (nla_put_u32(rsp, NETDEV_A_NAPI_PID, pid))
192 goto nla_put_failure;
193 }
194
195 napi_defer_hard_irqs = napi_get_defer_hard_irqs(napi);
196 if (nla_put_s32(rsp, NETDEV_A_NAPI_DEFER_HARD_IRQS,
197 napi_defer_hard_irqs))
198 goto nla_put_failure;
199
200 irq_suspend_timeout = napi_get_irq_suspend_timeout(napi);
201 if (nla_put_uint(rsp, NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT,
202 irq_suspend_timeout))
203 goto nla_put_failure;
204
205 gro_flush_timeout = napi_get_gro_flush_timeout(napi);
206 if (nla_put_uint(rsp, NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT,
207 gro_flush_timeout))
208 goto nla_put_failure;
209
210 genlmsg_end(rsp, hdr);
211
212 return 0;
213
214nla_put_failure:
215 genlmsg_cancel(rsp, hdr);
216 return -EMSGSIZE;
217}
218
219int netdev_nl_napi_get_doit(struct sk_buff *skb, struct genl_info *info)
220{
221 struct napi_struct *napi;
222 struct sk_buff *rsp;
223 u32 napi_id;
224 int err;
225
226 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
227 return -EINVAL;
228
229 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
230
231 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
232 if (!rsp)
233 return -ENOMEM;
234
235 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
236 if (napi) {
237 err = netdev_nl_napi_fill_one(rsp, napi, info);
238 netdev_unlock(napi->dev);
239 } else {
240 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
241 err = -ENOENT;
242 }
243
244 if (err) {
245 goto err_free_msg;
246 } else if (!rsp->len) {
247 err = -ENOENT;
248 goto err_free_msg;
249 }
250
251 return genlmsg_reply(rsp, info);
252
253err_free_msg:
254 nlmsg_free(rsp);
255 return err;
256}
257
258static int
259netdev_nl_napi_dump_one(struct net_device *netdev, struct sk_buff *rsp,
260 const struct genl_info *info,
261 struct netdev_nl_dump_ctx *ctx)
262{
263 struct napi_struct *napi;
264 unsigned int prev_id;
265 int err = 0;
266
267 if (!netdev->up)
268 return err;
269
270 prev_id = UINT_MAX;
271 list_for_each_entry(napi, &netdev->napi_list, dev_list) {
272 if (!napi_id_valid(napi->napi_id))
273 continue;
274
275 /* Dump continuation below depends on the list being sorted */
276 WARN_ON_ONCE(napi->napi_id >= prev_id);
277 prev_id = napi->napi_id;
278
279 if (ctx->napi_id && napi->napi_id >= ctx->napi_id)
280 continue;
281
282 err = netdev_nl_napi_fill_one(rsp, napi, info);
283 if (err)
284 return err;
285 ctx->napi_id = napi->napi_id;
286 }
287 return err;
288}
289
290int netdev_nl_napi_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
291{
292 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
293 const struct genl_info *info = genl_info_dump(cb);
294 struct net *net = sock_net(skb->sk);
295 struct net_device *netdev;
296 u32 ifindex = 0;
297 int err = 0;
298
299 if (info->attrs[NETDEV_A_NAPI_IFINDEX])
300 ifindex = nla_get_u32(info->attrs[NETDEV_A_NAPI_IFINDEX]);
301
302 if (ifindex) {
303 netdev = netdev_get_by_index_lock(net, ifindex);
304 if (netdev) {
305 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
306 netdev_unlock(netdev);
307 } else {
308 err = -ENODEV;
309 }
310 } else {
311 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
312 err = netdev_nl_napi_dump_one(netdev, skb, info, ctx);
313 if (err < 0)
314 break;
315 ctx->napi_id = 0;
316 }
317 }
318
319 return err;
320}
321
322static int
323netdev_nl_napi_set_config(struct napi_struct *napi, struct genl_info *info)
324{
325 u64 irq_suspend_timeout = 0;
326 u64 gro_flush_timeout = 0;
327 u32 defer = 0;
328
329 if (info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]) {
330 defer = nla_get_u32(info->attrs[NETDEV_A_NAPI_DEFER_HARD_IRQS]);
331 napi_set_defer_hard_irqs(napi, defer);
332 }
333
334 if (info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]) {
335 irq_suspend_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_IRQ_SUSPEND_TIMEOUT]);
336 napi_set_irq_suspend_timeout(napi, irq_suspend_timeout);
337 }
338
339 if (info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]) {
340 gro_flush_timeout = nla_get_uint(info->attrs[NETDEV_A_NAPI_GRO_FLUSH_TIMEOUT]);
341 napi_set_gro_flush_timeout(napi, gro_flush_timeout);
342 }
343
344 return 0;
345}
346
347int netdev_nl_napi_set_doit(struct sk_buff *skb, struct genl_info *info)
348{
349 struct napi_struct *napi;
350 unsigned int napi_id;
351 int err;
352
353 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_NAPI_ID))
354 return -EINVAL;
355
356 napi_id = nla_get_u32(info->attrs[NETDEV_A_NAPI_ID]);
357
358 napi = netdev_napi_by_id_lock(genl_info_net(info), napi_id);
359 if (napi) {
360 err = netdev_nl_napi_set_config(napi, info);
361 netdev_unlock(napi->dev);
362 } else {
363 NL_SET_BAD_ATTR(info->extack, info->attrs[NETDEV_A_NAPI_ID]);
364 err = -ENOENT;
365 }
366
367 return err;
368}
369
370static int nla_put_napi_id(struct sk_buff *skb, const struct napi_struct *napi)
371{
372 if (napi && napi_id_valid(napi->napi_id))
373 return nla_put_u32(skb, NETDEV_A_QUEUE_NAPI_ID, napi->napi_id);
374 return 0;
375}
376
377static int
378netdev_nl_queue_fill_one(struct sk_buff *rsp, struct net_device *netdev,
379 u32 q_idx, u32 q_type, const struct genl_info *info)
380{
381 struct pp_memory_provider_params *params;
382 struct netdev_rx_queue *rxq;
383 struct netdev_queue *txq;
384 void *hdr;
385
386 hdr = genlmsg_iput(rsp, info);
387 if (!hdr)
388 return -EMSGSIZE;
389
390 if (nla_put_u32(rsp, NETDEV_A_QUEUE_ID, q_idx) ||
391 nla_put_u32(rsp, NETDEV_A_QUEUE_TYPE, q_type) ||
392 nla_put_u32(rsp, NETDEV_A_QUEUE_IFINDEX, netdev->ifindex))
393 goto nla_put_failure;
394
395 switch (q_type) {
396 case NETDEV_QUEUE_TYPE_RX:
397 rxq = __netif_get_rx_queue(netdev, q_idx);
398 if (nla_put_napi_id(rsp, rxq->napi))
399 goto nla_put_failure;
400
401 params = &rxq->mp_params;
402 if (params->mp_ops &&
403 params->mp_ops->nl_fill(params->mp_priv, rsp, rxq))
404 goto nla_put_failure;
405#ifdef CONFIG_XDP_SOCKETS
406 if (rxq->pool)
407 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK))
408 goto nla_put_failure;
409#endif
410
411 break;
412 case NETDEV_QUEUE_TYPE_TX:
413 txq = netdev_get_tx_queue(netdev, q_idx);
414 if (nla_put_napi_id(rsp, txq->napi))
415 goto nla_put_failure;
416#ifdef CONFIG_XDP_SOCKETS
417 if (txq->pool)
418 if (nla_put_empty_nest(rsp, NETDEV_A_QUEUE_XSK))
419 goto nla_put_failure;
420#endif
421 break;
422 }
423
424 genlmsg_end(rsp, hdr);
425
426 return 0;
427
428nla_put_failure:
429 genlmsg_cancel(rsp, hdr);
430 return -EMSGSIZE;
431}
432
433static int netdev_nl_queue_validate(struct net_device *netdev, u32 q_id,
434 u32 q_type)
435{
436 switch (q_type) {
437 case NETDEV_QUEUE_TYPE_RX:
438 if (q_id >= netdev->real_num_rx_queues)
439 return -EINVAL;
440 return 0;
441 case NETDEV_QUEUE_TYPE_TX:
442 if (q_id >= netdev->real_num_tx_queues)
443 return -EINVAL;
444 }
445 return 0;
446}
447
448static int
449netdev_nl_queue_fill(struct sk_buff *rsp, struct net_device *netdev, u32 q_idx,
450 u32 q_type, const struct genl_info *info)
451{
452 int err;
453
454 if (!netdev->up)
455 return -ENOENT;
456
457 err = netdev_nl_queue_validate(netdev, q_idx, q_type);
458 if (err)
459 return err;
460
461 return netdev_nl_queue_fill_one(rsp, netdev, q_idx, q_type, info);
462}
463
464int netdev_nl_queue_get_doit(struct sk_buff *skb, struct genl_info *info)
465{
466 u32 q_id, q_type, ifindex;
467 struct net_device *netdev;
468 struct sk_buff *rsp;
469 int err;
470
471 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_ID) ||
472 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_TYPE) ||
473 GENL_REQ_ATTR_CHECK(info, NETDEV_A_QUEUE_IFINDEX))
474 return -EINVAL;
475
476 q_id = nla_get_u32(info->attrs[NETDEV_A_QUEUE_ID]);
477 q_type = nla_get_u32(info->attrs[NETDEV_A_QUEUE_TYPE]);
478 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
479
480 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
481 if (!rsp)
482 return -ENOMEM;
483
484 rtnl_lock();
485
486 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
487 if (netdev) {
488 err = netdev_nl_queue_fill(rsp, netdev, q_id, q_type, info);
489 netdev_unlock(netdev);
490 } else {
491 err = -ENODEV;
492 }
493
494 rtnl_unlock();
495
496 if (err)
497 goto err_free_msg;
498
499 return genlmsg_reply(rsp, info);
500
501err_free_msg:
502 nlmsg_free(rsp);
503 return err;
504}
505
506static int
507netdev_nl_queue_dump_one(struct net_device *netdev, struct sk_buff *rsp,
508 const struct genl_info *info,
509 struct netdev_nl_dump_ctx *ctx)
510{
511 int err = 0;
512
513 if (!netdev->up)
514 return err;
515
516 for (; ctx->rxq_idx < netdev->real_num_rx_queues; ctx->rxq_idx++) {
517 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->rxq_idx,
518 NETDEV_QUEUE_TYPE_RX, info);
519 if (err)
520 return err;
521 }
522 for (; ctx->txq_idx < netdev->real_num_tx_queues; ctx->txq_idx++) {
523 err = netdev_nl_queue_fill_one(rsp, netdev, ctx->txq_idx,
524 NETDEV_QUEUE_TYPE_TX, info);
525 if (err)
526 return err;
527 }
528
529 return err;
530}
531
532int netdev_nl_queue_get_dumpit(struct sk_buff *skb, struct netlink_callback *cb)
533{
534 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
535 const struct genl_info *info = genl_info_dump(cb);
536 struct net *net = sock_net(skb->sk);
537 struct net_device *netdev;
538 u32 ifindex = 0;
539 int err = 0;
540
541 if (info->attrs[NETDEV_A_QUEUE_IFINDEX])
542 ifindex = nla_get_u32(info->attrs[NETDEV_A_QUEUE_IFINDEX]);
543
544 rtnl_lock();
545 if (ifindex) {
546 netdev = netdev_get_by_index_lock(net, ifindex);
547 if (netdev) {
548 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
549 netdev_unlock(netdev);
550 } else {
551 err = -ENODEV;
552 }
553 } else {
554 for_each_netdev_lock_scoped(net, netdev, ctx->ifindex) {
555 err = netdev_nl_queue_dump_one(netdev, skb, info, ctx);
556 if (err < 0)
557 break;
558 ctx->rxq_idx = 0;
559 ctx->txq_idx = 0;
560 }
561 }
562 rtnl_unlock();
563
564 return err;
565}
566
567#define NETDEV_STAT_NOT_SET (~0ULL)
568
569static void netdev_nl_stats_add(void *_sum, const void *_add, size_t size)
570{
571 const u64 *add = _add;
572 u64 *sum = _sum;
573
574 while (size) {
575 if (*add != NETDEV_STAT_NOT_SET && *sum != NETDEV_STAT_NOT_SET)
576 *sum += *add;
577 sum++;
578 add++;
579 size -= 8;
580 }
581}
582
583static int netdev_stat_put(struct sk_buff *rsp, unsigned int attr_id, u64 value)
584{
585 if (value == NETDEV_STAT_NOT_SET)
586 return 0;
587 return nla_put_uint(rsp, attr_id, value);
588}
589
590static int
591netdev_nl_stats_write_rx(struct sk_buff *rsp, struct netdev_queue_stats_rx *rx)
592{
593 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_PACKETS, rx->packets) ||
594 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_BYTES, rx->bytes) ||
595 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_ALLOC_FAIL, rx->alloc_fail) ||
596 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROPS, rx->hw_drops) ||
597 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_OVERRUNS, rx->hw_drop_overruns) ||
598 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_COMPLETE, rx->csum_complete) ||
599 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_UNNECESSARY, rx->csum_unnecessary) ||
600 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_NONE, rx->csum_none) ||
601 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_CSUM_BAD, rx->csum_bad) ||
602 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_PACKETS, rx->hw_gro_packets) ||
603 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_BYTES, rx->hw_gro_bytes) ||
604 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_PACKETS, rx->hw_gro_wire_packets) ||
605 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_GRO_WIRE_BYTES, rx->hw_gro_wire_bytes) ||
606 netdev_stat_put(rsp, NETDEV_A_QSTATS_RX_HW_DROP_RATELIMITS, rx->hw_drop_ratelimits))
607 return -EMSGSIZE;
608 return 0;
609}
610
611static int
612netdev_nl_stats_write_tx(struct sk_buff *rsp, struct netdev_queue_stats_tx *tx)
613{
614 if (netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_PACKETS, tx->packets) ||
615 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_BYTES, tx->bytes) ||
616 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROPS, tx->hw_drops) ||
617 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_ERRORS, tx->hw_drop_errors) ||
618 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_CSUM_NONE, tx->csum_none) ||
619 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_NEEDS_CSUM, tx->needs_csum) ||
620 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_PACKETS, tx->hw_gso_packets) ||
621 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_BYTES, tx->hw_gso_bytes) ||
622 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_PACKETS, tx->hw_gso_wire_packets) ||
623 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_GSO_WIRE_BYTES, tx->hw_gso_wire_bytes) ||
624 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_HW_DROP_RATELIMITS, tx->hw_drop_ratelimits) ||
625 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_STOP, tx->stop) ||
626 netdev_stat_put(rsp, NETDEV_A_QSTATS_TX_WAKE, tx->wake))
627 return -EMSGSIZE;
628 return 0;
629}
630
631static int
632netdev_nl_stats_queue(struct net_device *netdev, struct sk_buff *rsp,
633 u32 q_type, int i, const struct genl_info *info)
634{
635 const struct netdev_stat_ops *ops = netdev->stat_ops;
636 struct netdev_queue_stats_rx rx;
637 struct netdev_queue_stats_tx tx;
638 void *hdr;
639
640 hdr = genlmsg_iput(rsp, info);
641 if (!hdr)
642 return -EMSGSIZE;
643 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex) ||
644 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_TYPE, q_type) ||
645 nla_put_u32(rsp, NETDEV_A_QSTATS_QUEUE_ID, i))
646 goto nla_put_failure;
647
648 switch (q_type) {
649 case NETDEV_QUEUE_TYPE_RX:
650 memset(&rx, 0xff, sizeof(rx));
651 ops->get_queue_stats_rx(netdev, i, &rx);
652 if (!memchr_inv(&rx, 0xff, sizeof(rx)))
653 goto nla_cancel;
654 if (netdev_nl_stats_write_rx(rsp, &rx))
655 goto nla_put_failure;
656 break;
657 case NETDEV_QUEUE_TYPE_TX:
658 memset(&tx, 0xff, sizeof(tx));
659 ops->get_queue_stats_tx(netdev, i, &tx);
660 if (!memchr_inv(&tx, 0xff, sizeof(tx)))
661 goto nla_cancel;
662 if (netdev_nl_stats_write_tx(rsp, &tx))
663 goto nla_put_failure;
664 break;
665 }
666
667 genlmsg_end(rsp, hdr);
668 return 0;
669
670nla_cancel:
671 genlmsg_cancel(rsp, hdr);
672 return 0;
673nla_put_failure:
674 genlmsg_cancel(rsp, hdr);
675 return -EMSGSIZE;
676}
677
678static int
679netdev_nl_stats_by_queue(struct net_device *netdev, struct sk_buff *rsp,
680 const struct genl_info *info,
681 struct netdev_nl_dump_ctx *ctx)
682{
683 const struct netdev_stat_ops *ops = netdev->stat_ops;
684 int i, err;
685
686 if (!(netdev->flags & IFF_UP))
687 return 0;
688
689 i = ctx->rxq_idx;
690 while (ops->get_queue_stats_rx && i < netdev->real_num_rx_queues) {
691 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_RX,
692 i, info);
693 if (err)
694 return err;
695 ctx->rxq_idx = ++i;
696 }
697 i = ctx->txq_idx;
698 while (ops->get_queue_stats_tx && i < netdev->real_num_tx_queues) {
699 err = netdev_nl_stats_queue(netdev, rsp, NETDEV_QUEUE_TYPE_TX,
700 i, info);
701 if (err)
702 return err;
703 ctx->txq_idx = ++i;
704 }
705
706 ctx->rxq_idx = 0;
707 ctx->txq_idx = 0;
708 return 0;
709}
710
711static int
712netdev_nl_stats_by_netdev(struct net_device *netdev, struct sk_buff *rsp,
713 const struct genl_info *info)
714{
715 struct netdev_queue_stats_rx rx_sum, rx;
716 struct netdev_queue_stats_tx tx_sum, tx;
717 const struct netdev_stat_ops *ops;
718 void *hdr;
719 int i;
720
721 ops = netdev->stat_ops;
722 /* Netdev can't guarantee any complete counters */
723 if (!ops->get_base_stats)
724 return 0;
725
726 memset(&rx_sum, 0xff, sizeof(rx_sum));
727 memset(&tx_sum, 0xff, sizeof(tx_sum));
728
729 ops->get_base_stats(netdev, &rx_sum, &tx_sum);
730
731 /* The op was there, but nothing reported, don't bother */
732 if (!memchr_inv(&rx_sum, 0xff, sizeof(rx_sum)) &&
733 !memchr_inv(&tx_sum, 0xff, sizeof(tx_sum)))
734 return 0;
735
736 hdr = genlmsg_iput(rsp, info);
737 if (!hdr)
738 return -EMSGSIZE;
739 if (nla_put_u32(rsp, NETDEV_A_QSTATS_IFINDEX, netdev->ifindex))
740 goto nla_put_failure;
741
742 for (i = 0; i < netdev->real_num_rx_queues; i++) {
743 memset(&rx, 0xff, sizeof(rx));
744 if (ops->get_queue_stats_rx)
745 ops->get_queue_stats_rx(netdev, i, &rx);
746 netdev_nl_stats_add(&rx_sum, &rx, sizeof(rx));
747 }
748 for (i = 0; i < netdev->real_num_tx_queues; i++) {
749 memset(&tx, 0xff, sizeof(tx));
750 if (ops->get_queue_stats_tx)
751 ops->get_queue_stats_tx(netdev, i, &tx);
752 netdev_nl_stats_add(&tx_sum, &tx, sizeof(tx));
753 }
754
755 if (netdev_nl_stats_write_rx(rsp, &rx_sum) ||
756 netdev_nl_stats_write_tx(rsp, &tx_sum))
757 goto nla_put_failure;
758
759 genlmsg_end(rsp, hdr);
760 return 0;
761
762nla_put_failure:
763 genlmsg_cancel(rsp, hdr);
764 return -EMSGSIZE;
765}
766
767static int
768netdev_nl_qstats_get_dump_one(struct net_device *netdev, unsigned int scope,
769 struct sk_buff *skb, const struct genl_info *info,
770 struct netdev_nl_dump_ctx *ctx)
771{
772 if (!netdev->stat_ops)
773 return 0;
774
775 switch (scope) {
776 case 0:
777 return netdev_nl_stats_by_netdev(netdev, skb, info);
778 case NETDEV_QSTATS_SCOPE_QUEUE:
779 return netdev_nl_stats_by_queue(netdev, skb, info, ctx);
780 }
781
782 return -EINVAL; /* Should not happen, per netlink policy */
783}
784
785int netdev_nl_qstats_get_dumpit(struct sk_buff *skb,
786 struct netlink_callback *cb)
787{
788 struct netdev_nl_dump_ctx *ctx = netdev_dump_ctx(cb);
789 const struct genl_info *info = genl_info_dump(cb);
790 struct net *net = sock_net(skb->sk);
791 struct net_device *netdev;
792 unsigned int ifindex;
793 unsigned int scope;
794 int err = 0;
795
796 scope = 0;
797 if (info->attrs[NETDEV_A_QSTATS_SCOPE])
798 scope = nla_get_uint(info->attrs[NETDEV_A_QSTATS_SCOPE]);
799
800 ifindex = 0;
801 if (info->attrs[NETDEV_A_QSTATS_IFINDEX])
802 ifindex = nla_get_u32(info->attrs[NETDEV_A_QSTATS_IFINDEX]);
803
804 rtnl_lock();
805 if (ifindex) {
806 netdev = __dev_get_by_index(net, ifindex);
807 if (netdev && netdev->stat_ops) {
808 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
809 info, ctx);
810 } else {
811 NL_SET_BAD_ATTR(info->extack,
812 info->attrs[NETDEV_A_QSTATS_IFINDEX]);
813 err = netdev ? -EOPNOTSUPP : -ENODEV;
814 }
815 } else {
816 for_each_netdev_dump(net, netdev, ctx->ifindex) {
817 err = netdev_nl_qstats_get_dump_one(netdev, scope, skb,
818 info, ctx);
819 if (err < 0)
820 break;
821 }
822 }
823 rtnl_unlock();
824
825 return err;
826}
827
828int netdev_nl_bind_rx_doit(struct sk_buff *skb, struct genl_info *info)
829{
830 struct nlattr *tb[ARRAY_SIZE(netdev_queue_id_nl_policy)];
831 struct net_devmem_dmabuf_binding *binding;
832 u32 ifindex, dmabuf_fd, rxq_idx;
833 struct netdev_nl_sock *priv;
834 struct net_device *netdev;
835 struct sk_buff *rsp;
836 struct nlattr *attr;
837 int rem, err = 0;
838 void *hdr;
839
840 if (GENL_REQ_ATTR_CHECK(info, NETDEV_A_DEV_IFINDEX) ||
841 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_FD) ||
842 GENL_REQ_ATTR_CHECK(info, NETDEV_A_DMABUF_QUEUES))
843 return -EINVAL;
844
845 ifindex = nla_get_u32(info->attrs[NETDEV_A_DEV_IFINDEX]);
846 dmabuf_fd = nla_get_u32(info->attrs[NETDEV_A_DMABUF_FD]);
847
848 priv = genl_sk_priv_get(&netdev_nl_family, NETLINK_CB(skb).sk);
849 if (IS_ERR(priv))
850 return PTR_ERR(priv);
851
852 rsp = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
853 if (!rsp)
854 return -ENOMEM;
855
856 hdr = genlmsg_iput(rsp, info);
857 if (!hdr) {
858 err = -EMSGSIZE;
859 goto err_genlmsg_free;
860 }
861
862 mutex_lock(&priv->lock);
863
864 err = 0;
865 netdev = netdev_get_by_index_lock(genl_info_net(info), ifindex);
866 if (!netdev) {
867 err = -ENODEV;
868 goto err_unlock_sock;
869 }
870 if (!netif_device_present(netdev))
871 err = -ENODEV;
872 else if (!netdev_need_ops_lock(netdev))
873 err = -EOPNOTSUPP;
874 if (err) {
875 NL_SET_BAD_ATTR(info->extack,
876 info->attrs[NETDEV_A_DEV_IFINDEX]);
877 goto err_unlock;
878 }
879
880 binding = net_devmem_bind_dmabuf(netdev, dmabuf_fd, info->extack);
881 if (IS_ERR(binding)) {
882 err = PTR_ERR(binding);
883 goto err_unlock;
884 }
885
886 nla_for_each_attr_type(attr, NETDEV_A_DMABUF_QUEUES,
887 genlmsg_data(info->genlhdr),
888 genlmsg_len(info->genlhdr), rem) {
889 err = nla_parse_nested(
890 tb, ARRAY_SIZE(netdev_queue_id_nl_policy) - 1, attr,
891 netdev_queue_id_nl_policy, info->extack);
892 if (err < 0)
893 goto err_unbind;
894
895 if (NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_ID) ||
896 NL_REQ_ATTR_CHECK(info->extack, attr, tb, NETDEV_A_QUEUE_TYPE)) {
897 err = -EINVAL;
898 goto err_unbind;
899 }
900
901 if (nla_get_u32(tb[NETDEV_A_QUEUE_TYPE]) != NETDEV_QUEUE_TYPE_RX) {
902 NL_SET_BAD_ATTR(info->extack, tb[NETDEV_A_QUEUE_TYPE]);
903 err = -EINVAL;
904 goto err_unbind;
905 }
906
907 rxq_idx = nla_get_u32(tb[NETDEV_A_QUEUE_ID]);
908
909 err = net_devmem_bind_dmabuf_to_queue(netdev, rxq_idx, binding,
910 info->extack);
911 if (err)
912 goto err_unbind;
913 }
914
915 list_add(&binding->list, &priv->bindings);
916
917 nla_put_u32(rsp, NETDEV_A_DMABUF_ID, binding->id);
918 genlmsg_end(rsp, hdr);
919
920 err = genlmsg_reply(rsp, info);
921 if (err)
922 goto err_unbind;
923
924 netdev_unlock(netdev);
925
926 mutex_unlock(&priv->lock);
927
928 return 0;
929
930err_unbind:
931 net_devmem_unbind_dmabuf(binding);
932err_unlock:
933 netdev_unlock(netdev);
934err_unlock_sock:
935 mutex_unlock(&priv->lock);
936err_genlmsg_free:
937 nlmsg_free(rsp);
938 return err;
939}
940
941void netdev_nl_sock_priv_init(struct netdev_nl_sock *priv)
942{
943 INIT_LIST_HEAD(&priv->bindings);
944 mutex_init(&priv->lock);
945}
946
947void netdev_nl_sock_priv_destroy(struct netdev_nl_sock *priv)
948{
949 struct net_devmem_dmabuf_binding *binding;
950 struct net_devmem_dmabuf_binding *temp;
951 struct net_device *dev;
952
953 mutex_lock(&priv->lock);
954 list_for_each_entry_safe(binding, temp, &priv->bindings, list) {
955 dev = binding->dev;
956 netdev_lock(dev);
957 net_devmem_unbind_dmabuf(binding);
958 netdev_unlock(dev);
959 }
960 mutex_unlock(&priv->lock);
961}
962
963static int netdev_genl_netdevice_event(struct notifier_block *nb,
964 unsigned long event, void *ptr)
965{
966 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
967
968 switch (event) {
969 case NETDEV_REGISTER:
970 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_ADD_NTF);
971 break;
972 case NETDEV_UNREGISTER:
973 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_DEL_NTF);
974 break;
975 case NETDEV_XDP_FEAT_CHANGE:
976 netdev_genl_dev_notify(netdev, NETDEV_CMD_DEV_CHANGE_NTF);
977 break;
978 }
979
980 return NOTIFY_OK;
981}
982
983static struct notifier_block netdev_genl_nb = {
984 .notifier_call = netdev_genl_netdevice_event,
985};
986
987static int __init netdev_genl_init(void)
988{
989 int err;
990
991 err = register_netdevice_notifier(&netdev_genl_nb);
992 if (err)
993 return err;
994
995 err = genl_register_family(&netdev_nl_family);
996 if (err)
997 goto err_unreg_ntf;
998
999 return 0;
1000
1001err_unreg_ntf:
1002 unregister_netdevice_notifier(&netdev_genl_nb);
1003 return err;
1004}
1005
1006subsys_initcall(netdev_genl_init);