Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309// SPDX-License-Identifier: GPL-2.0-only/* Copyright (c) 2023 Isovalent */
#include <linux/netdevice.h>#include <linux/ethtool.h>#include <linux/etherdevice.h>#include <linux/filter.h>#include <linux/netfilter_netdev.h>#include <linux/bpf_mprog.h>#include <linux/indirect_call_wrapper.h>
#include <net/netdev_lock.h>#include <net/netdev_queues.h>#include <net/netdev_rx_queue.h>#include <net/xdp_sock_drv.h>#include <net/netkit.h>#include <net/dst.h>#include <net/tcx.h>
#define NETKIT_DRV_NAME "netkit"
#define NETKIT_NUM_RX_QUEUES_MAX 1024#define NETKIT_NUM_TX_QUEUES_MAX 1
#define NETKIT_NUM_RX_QUEUES_REAL 1#define NETKIT_NUM_TX_QUEUES_REAL 1
struct netkit { __cacheline_group_begin(netkit_fastpath); struct net_device __rcu *peer; struct bpf_mprog_entry __rcu *active; enum netkit_action policy; enum netkit_scrub scrub; struct bpf_mprog_bundle bundle; __cacheline_group_end(netkit_fastpath);
__cacheline_group_begin(netkit_slowpath); enum netkit_mode mode; enum netkit_pairing pair; bool primary; u32 headroom; __cacheline_group_end(netkit_slowpath);};
struct netkit_link { struct bpf_link link; struct net_device *dev;};
static struct rtnl_link_ops netkit_link_ops;
static __always_inline intnetkit_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb, enum netkit_action ret){ const struct bpf_mprog_fp *fp; const struct bpf_prog *prog;
bpf_mprog_foreach_prog(entry, fp, prog) { bpf_compute_data_pointers(skb); ret = bpf_prog_run(prog, skb); if (ret != NETKIT_NEXT) break; } return ret;}
static void netkit_xnet(struct sk_buff *skb){ skb->priority = 0; skb->mark = 0;}
static void netkit_prep_forward(struct sk_buff *skb, bool xnet, bool xnet_scrub){ skb_scrub_packet(skb, false); nf_skip_egress(skb, true); skb_reset_mac_header(skb); if (!xnet) return; skb_clear_tstamp(skb); if (xnet_scrub) netkit_xnet(skb);}
static struct netkit *netkit_priv(const struct net_device *dev){ return netdev_priv(dev);}
static netdev_tx_t netkit_xmit(struct sk_buff *skb, struct net_device *dev){ struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; struct netkit *nk = netkit_priv(dev); enum netkit_action ret = READ_ONCE(nk->policy); netdev_tx_t ret_dev = NET_XMIT_SUCCESS; const struct bpf_mprog_entry *entry; struct net_device *peer; int len = skb->len;
bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); rcu_read_lock(); peer = rcu_dereference(nk->peer); if (unlikely(!peer || !(peer->flags & IFF_UP) || !pskb_may_pull(skb, ETH_HLEN) || skb_orphan_frags(skb, GFP_ATOMIC))) goto drop; netkit_prep_forward(skb, !net_eq(dev_net(dev), dev_net(peer)), nk->scrub); eth_skb_pkt_type(skb, peer); skb->dev = peer; entry = rcu_dereference(nk->active); if (entry) ret = netkit_run(entry, skb, ret); switch (ret) { case NETKIT_NEXT: case NETKIT_PASS: eth_skb_pull_mac(skb); skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); if (likely(__netif_rx(skb) == NET_RX_SUCCESS)) { dev_sw_netstats_tx_add(dev, 1, len); dev_sw_netstats_rx_add(peer, len); } else { goto drop_stats; } break; case NETKIT_REDIRECT: dev_sw_netstats_tx_add(dev, 1, len); skb_do_redirect(skb); break; case NETKIT_DROP: default:drop: kfree_skb(skb);drop_stats: dev_core_stats_tx_dropped_inc(dev); ret_dev = NET_XMIT_DROP; break; } rcu_read_unlock(); bpf_net_ctx_clear(bpf_net_ctx); return ret_dev;}
static int netkit_open(struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer);
if (nk->pair == NETKIT_DEVICE_SINGLE) { netif_carrier_on(dev); return 0; } if (!peer) return -ENOTCONN; if (peer->flags & IFF_UP) { netif_carrier_on(dev); netif_carrier_on(peer); } return 0;}
static int netkit_close(struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer);
netif_carrier_off(dev); if (peer) netif_carrier_off(peer); return 0;}
static int netkit_get_iflink(const struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer; int iflink = 0;
rcu_read_lock(); peer = rcu_dereference(nk->peer); if (peer) iflink = READ_ONCE(peer->ifindex); rcu_read_unlock(); return iflink;}
static int netkit_set_multicast(struct net_device *dev, struct netdev_hw_addr_list *uc, struct netdev_hw_addr_list *mc){ /* Nothing to do, we receive whatever gets pushed to us! */ return 0;}
static int netkit_set_macaddr(struct net_device *dev, void *sa){ struct netkit *nk = netkit_priv(dev);
if (nk->mode != NETKIT_L2) return -EOPNOTSUPP;
return eth_mac_addr(dev, sa);}
static void netkit_set_headroom(struct net_device *dev, int headroom){ struct netkit *nk = netkit_priv(dev), *nk2; struct net_device *peer;
if (headroom < 0) headroom = NET_SKB_PAD;
rcu_read_lock(); peer = rcu_dereference(nk->peer); if (!peer) { nk->headroom = headroom; dev->needed_headroom = headroom; } else { nk2 = netkit_priv(peer); nk->headroom = headroom; headroom = max(nk->headroom, nk2->headroom);
peer->needed_headroom = headroom; dev->needed_headroom = headroom; } rcu_read_unlock();}
INDIRECT_CALLABLE_SCOPE struct net_device *netkit_peer_dev(struct net_device *dev){ return rcu_dereference(netkit_priv(dev)->peer);}
static void netkit_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats){ dev_fetch_sw_netstats(stats, dev->tstats); stats->tx_dropped = DEV_STATS_READ(dev, tx_dropped);}
static bool netkit_xsk_supported_at_phys(const struct net_device *dev){ if (!dev->netdev_ops->ndo_bpf || !dev->netdev_ops->ndo_xdp_xmit || !dev->netdev_ops->ndo_xsk_wakeup) return false; return true;}
static int netkit_xsk(struct net_device *dev, struct netdev_bpf *xdp){ struct netkit *nk = netkit_priv(dev); struct netdev_bpf xdp_lower; struct netdev_rx_queue *rxq; struct net_device *phys; bool create = false; int ret = -EBUSY;
switch (xdp->command) { case XDP_SETUP_XSK_POOL: if (nk->pair == NETKIT_DEVICE_PAIR) return -EOPNOTSUPP; if (xdp->xsk.queue_id >= dev->real_num_rx_queues) return -EINVAL;
rxq = __netif_get_rx_queue(dev, xdp->xsk.queue_id); if (!rxq->lease) return -EOPNOTSUPP;
phys = rxq->lease->dev; if (!netkit_xsk_supported_at_phys(phys)) return -EOPNOTSUPP;
create = xdp->xsk.pool; memcpy(&xdp_lower, xdp, sizeof(xdp_lower)); xdp_lower.xsk.queue_id = get_netdev_rx_queue_index(rxq->lease); break; case XDP_SETUP_PROG: return -EOPNOTSUPP; default: return -EINVAL; }
netdev_lock(phys); if (create && (phys->xdp_features & NETDEV_XDP_ACT_XSK) != NETDEV_XDP_ACT_XSK) { ret = -EOPNOTSUPP; goto out; } if (!create || !dev_get_min_mp_channel_count(phys)) ret = phys->netdev_ops->ndo_bpf(phys, &xdp_lower);out: netdev_unlock(phys); return ret;}
static int netkit_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags){ struct netdev_rx_queue *rxq, *rxq_lease; struct net_device *phys;
if (queue_id >= dev->real_num_rx_queues) return -EINVAL;
rxq = __netif_get_rx_queue(dev, queue_id); rxq_lease = READ_ONCE(rxq->lease); if (unlikely(!rxq_lease)) return -EOPNOTSUPP;
/* netkit_xsk already validated full xsk support, hence it's * fine to call into ndo_xsk_wakeup right away given this * was a prerequisite to get here in the first place. The * phys xsk support cannot change without tearing down the * device (which clears the lease first). */ phys = rxq_lease->dev; return phys->netdev_ops->ndo_xsk_wakeup(phys, get_netdev_rx_queue_index(rxq_lease), flags);}
static int netkit_init(struct net_device *dev){ netdev_lockdep_set_classes(dev); return 0;}
static void netkit_uninit(struct net_device *dev);
static const struct net_device_ops netkit_netdev_ops = { .ndo_init = netkit_init, .ndo_open = netkit_open, .ndo_stop = netkit_close, .ndo_start_xmit = netkit_xmit, .ndo_set_rx_mode_async = netkit_set_multicast, .ndo_set_rx_headroom = netkit_set_headroom, .ndo_set_mac_address = netkit_set_macaddr, .ndo_get_iflink = netkit_get_iflink, .ndo_get_peer_dev = netkit_peer_dev, .ndo_get_stats64 = netkit_get_stats, .ndo_uninit = netkit_uninit, .ndo_bpf = netkit_xsk, .ndo_xsk_wakeup = netkit_xsk_wakeup, .ndo_features_check = passthru_features_check,};
static void netkit_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info){ strscpy(info->driver, NETKIT_DRV_NAME, sizeof(info->driver));}
static const struct ethtool_ops netkit_ethtool_ops = { .get_drvinfo = netkit_get_drvinfo,};
static int netkit_queue_create(struct net_device *dev, struct netlink_ext_ack *extack){ struct netkit *nk = netkit_priv(dev); u32 rxq_count_old, rxq_count_new; int err;
rxq_count_old = dev->real_num_rx_queues; rxq_count_new = rxq_count_old + 1;
/* In paired mode, only the non-primary (peer) device can * create leased queues since the primary is the management * side. In single device mode, leasing is always allowed. */ if (nk->pair == NETKIT_DEVICE_PAIR && nk->primary) { NL_SET_ERR_MSG(extack, "netkit can only lease against the peer device"); return -EOPNOTSUPP; }
err = netif_set_real_num_rx_queues(dev, rxq_count_new); if (err) { if (rxq_count_new > dev->num_rx_queues) NL_SET_ERR_MSG(extack, "netkit maximum queue limit reached"); else NL_SET_ERR_MSG_FMT(extack, "netkit cannot create more queues err=%d", err); return err; }
return rxq_count_old;}
static const struct netdev_queue_mgmt_ops netkit_queue_mgmt_ops = { .ndo_queue_create = netkit_queue_create,};
static struct net_device *netkit_alloc(struct nlattr *tb[], const char *ifname, unsigned char name_assign_type, unsigned int num_tx_queues, unsigned int num_rx_queues){ const struct rtnl_link_ops *ops = &netkit_link_ops; struct net_device *dev;
if (num_tx_queues > NETKIT_NUM_TX_QUEUES_MAX || num_rx_queues > NETKIT_NUM_RX_QUEUES_MAX) return ERR_PTR(-EOPNOTSUPP);
dev = alloc_netdev_mqs(ops->priv_size, ifname, name_assign_type, ops->setup, num_tx_queues, num_rx_queues); if (dev) { dev->real_num_tx_queues = NETKIT_NUM_TX_QUEUES_REAL; dev->real_num_rx_queues = NETKIT_NUM_RX_QUEUES_REAL; } return dev;}
static void netkit_queue_unlease(struct net_device *dev){ struct netdev_rx_queue *rxq, *rxq_lease; struct net_device *dev_lease; int i;
if (dev->real_num_rx_queues == 1) return;
netdev_lock(dev); for (i = 1; i < dev->real_num_rx_queues; i++) { rxq = __netif_get_rx_queue(dev, i); rxq_lease = rxq->lease; dev_lease = rxq_lease->dev;
netdev_lock(dev_lease); netdev_rx_queue_unlease(rxq, rxq_lease); netdev_unlock(dev_lease); } netdev_unlock(dev);}
static void netkit_setup(struct net_device *dev){ static const netdev_features_t netkit_features_hw_vlan = NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_STAG_RX; static const netdev_features_t netkit_features = netkit_features_hw_vlan | NETIF_F_SG | NETIF_F_FRAGLIST | NETIF_F_HW_CSUM | NETIF_F_RXCSUM | NETIF_F_SCTP_CRC | NETIF_F_HIGHDMA | NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL;
ether_setup(dev); dev->max_mtu = ETH_MAX_MTU; dev->pcpu_stat_type = NETDEV_PCPU_STAT_TSTATS;
dev->flags |= IFF_NOARP; dev->priv_flags &= ~IFF_TX_SKB_SHARING; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; dev->priv_flags |= IFF_PHONY_HEADROOM; dev->priv_flags |= IFF_NO_QUEUE; dev->priv_flags |= IFF_DISABLE_NETPOLL; dev->lltx = true; dev->netmem_tx = NETMEM_TX_NO_DMA;
dev->netdev_ops = &netkit_netdev_ops; dev->ethtool_ops = &netkit_ethtool_ops; dev->queue_mgmt_ops = &netkit_queue_mgmt_ops;
dev->features |= netkit_features; dev->hw_features = netkit_features; dev->hw_enc_features = netkit_features; dev->mpls_features = NETIF_F_HW_CSUM | NETIF_F_GSO_SOFTWARE; dev->vlan_features = dev->features & ~netkit_features_hw_vlan;
dev->needs_free_netdev = true;
netif_set_tso_max_size(dev, GSO_MAX_SIZE);}
static struct net *netkit_get_link_net(const struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer);
return peer ? dev_net(peer) : dev_net(dev);}
static int netkit_check_policy(int policy, struct nlattr *tb, struct netlink_ext_ack *extack){ switch (policy) { case NETKIT_PASS: case NETKIT_DROP: return 0; default: NL_SET_ERR_MSG_ATTR(extack, tb, "Provided default xmit policy not supported"); return -EINVAL; }}
static int netkit_validate(struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack){ struct nlattr *attr = tb[IFLA_ADDRESS];
if (!attr) return 0; if (nla_len(attr) != ETH_ALEN) return -EINVAL; if (!is_valid_ether_addr(nla_data(attr))) return -EADDRNOTAVAIL; return 0;}
static int netkit_new_link(struct net_device *dev, struct rtnl_newlink_params *params, struct netlink_ext_ack *extack){ struct net *peer_net = rtnl_newlink_peer_net(params); enum netkit_scrub scrub_prim = NETKIT_SCRUB_DEFAULT; enum netkit_scrub scrub_peer = NETKIT_SCRUB_DEFAULT; struct nlattr *peer_tb[IFLA_MAX + 1], **tbp, *attr; enum netkit_pairing pair = NETKIT_DEVICE_PAIR; enum netkit_action policy_prim = NETKIT_PASS; enum netkit_action policy_peer = NETKIT_PASS; bool seen_peer = false, seen_scrub = false; struct nlattr **data = params->data; enum netkit_mode mode = NETKIT_L3; unsigned char ifname_assign_type; struct nlattr **tb = params->tb; u16 headroom = 0, tailroom = 0; struct ifinfomsg *ifmp = NULL; struct net_device *peer = NULL; char ifname[IFNAMSIZ]; struct netkit *nk; int err;
tbp = tb; if (data) { if (data[IFLA_NETKIT_MODE]) mode = nla_get_u32(data[IFLA_NETKIT_MODE]); if (data[IFLA_NETKIT_PEER_INFO]) { attr = data[IFLA_NETKIT_PEER_INFO]; ifmp = nla_data(attr); rtnl_nla_parse_ifinfomsg(peer_tb, attr, extack); tbp = peer_tb; } if (data[IFLA_NETKIT_SCRUB]) scrub_prim = nla_get_u32(data[IFLA_NETKIT_SCRUB]); if (data[IFLA_NETKIT_PEER_SCRUB]) scrub_peer = nla_get_u32(data[IFLA_NETKIT_PEER_SCRUB]); if (data[IFLA_NETKIT_POLICY]) { attr = data[IFLA_NETKIT_POLICY]; policy_prim = nla_get_u32(attr); err = netkit_check_policy(policy_prim, attr, extack); if (err < 0) return err; } if (data[IFLA_NETKIT_PEER_POLICY]) { attr = data[IFLA_NETKIT_PEER_POLICY]; policy_peer = nla_get_u32(attr); err = netkit_check_policy(policy_peer, attr, extack); if (err < 0) return err; } if (data[IFLA_NETKIT_HEADROOM]) headroom = nla_get_u16(data[IFLA_NETKIT_HEADROOM]); if (data[IFLA_NETKIT_TAILROOM]) tailroom = nla_get_u16(data[IFLA_NETKIT_TAILROOM]); if (data[IFLA_NETKIT_PAIRING]) pair = nla_get_u32(data[IFLA_NETKIT_PAIRING]);
seen_scrub = data[IFLA_NETKIT_SCRUB]; seen_peer = data[IFLA_NETKIT_PEER_INFO] || data[IFLA_NETKIT_PEER_SCRUB] || data[IFLA_NETKIT_PEER_POLICY]; }
if (ifmp && tbp[IFLA_IFNAME]) { nla_strscpy(ifname, tbp[IFLA_IFNAME], IFNAMSIZ); ifname_assign_type = NET_NAME_USER; } else { strscpy(ifname, "nk%d", IFNAMSIZ); ifname_assign_type = NET_NAME_ENUM; } if (mode != NETKIT_L2 && (tb[IFLA_ADDRESS] || tbp[IFLA_ADDRESS])) return -EOPNOTSUPP; if (pair == NETKIT_DEVICE_SINGLE && (tb != tbp || seen_peer || seen_scrub || policy_prim != NETKIT_PASS)) return -EOPNOTSUPP;
if (pair == NETKIT_DEVICE_PAIR) { peer = rtnl_create_link(peer_net, ifname, ifname_assign_type, &netkit_link_ops, tbp, extack); if (IS_ERR(peer)) return PTR_ERR(peer);
netif_inherit_tso_max(peer, dev); if (headroom) peer->needed_headroom = headroom; if (tailroom) peer->needed_tailroom = tailroom; if (mode == NETKIT_L2 && !(ifmp && tbp[IFLA_ADDRESS])) eth_hw_addr_random(peer); if (ifmp && dev->ifindex) peer->ifindex = ifmp->ifi_index;
nk = netkit_priv(peer); nk->primary = false; nk->policy = policy_peer; nk->scrub = scrub_peer; nk->mode = mode; nk->pair = pair; nk->headroom = headroom; bpf_mprog_bundle_init(&nk->bundle);
err = register_netdevice(peer); if (err < 0) goto err_register_peer; netif_carrier_off(peer); if (mode == NETKIT_L2) dev_change_flags(peer, peer->flags & ~IFF_NOARP, NULL);
err = rtnl_configure_link(peer, NULL, 0, NULL); if (err < 0) goto err_configure_peer; }
if (mode == NETKIT_L2 && !tb[IFLA_ADDRESS]) eth_hw_addr_random(dev); if (tb[IFLA_IFNAME]) nla_strscpy(dev->name, tb[IFLA_IFNAME], IFNAMSIZ); else strscpy(dev->name, "nk%d", IFNAMSIZ); if (headroom) dev->needed_headroom = headroom; if (tailroom) dev->needed_tailroom = tailroom;
nk = netkit_priv(dev); nk->primary = true; nk->policy = policy_prim; nk->scrub = scrub_prim; nk->mode = mode; nk->pair = pair; nk->headroom = headroom; bpf_mprog_bundle_init(&nk->bundle);
if (pair == NETKIT_DEVICE_SINGLE) xdp_set_features_flag(dev, NETDEV_XDP_ACT_XSK);
err = register_netdevice(dev); if (err < 0) goto err_configure_peer; netif_carrier_off(dev); if (mode == NETKIT_L2) dev_change_flags(dev, dev->flags & ~IFF_NOARP, NULL);
rcu_assign_pointer(netkit_priv(dev)->peer, peer); if (peer) rcu_assign_pointer(netkit_priv(peer)->peer, dev); return 0;err_configure_peer: if (peer) unregister_netdevice(peer); return err;err_register_peer: free_netdev(peer); return err;}
static struct bpf_mprog_entry *netkit_entry_fetch(struct net_device *dev, bool bundle_fallback){ struct netkit *nk = netkit_priv(dev); struct bpf_mprog_entry *entry;
ASSERT_RTNL(); entry = rcu_dereference_rtnl(nk->active); if (entry) return entry; if (bundle_fallback) return &nk->bundle.a; return NULL;}
static void netkit_entry_update(struct net_device *dev, struct bpf_mprog_entry *entry){ struct netkit *nk = netkit_priv(dev);
ASSERT_RTNL(); rcu_assign_pointer(nk->active, entry);}
static void netkit_entry_sync(void){ synchronize_rcu();}
static struct net_device *netkit_dev_fetch(struct net *net, u32 ifindex, u32 which){ struct net_device *dev; struct netkit *nk;
ASSERT_RTNL();
switch (which) { case BPF_NETKIT_PRIMARY: case BPF_NETKIT_PEER: break; default: return ERR_PTR(-EINVAL); }
dev = __dev_get_by_index(net, ifindex); if (!dev) return ERR_PTR(-ENODEV); if (dev->netdev_ops != &netkit_netdev_ops) return ERR_PTR(-ENXIO);
nk = netkit_priv(dev); if (!nk->primary) return ERR_PTR(-EACCES); if (nk->pair == NETKIT_DEVICE_SINGLE) return ERR_PTR(-EOPNOTSUPP); if (which == BPF_NETKIT_PEER) { dev = rcu_dereference_rtnl(nk->peer); if (!dev) return ERR_PTR(-ENODEV); } return dev;}
int netkit_prog_attach(const union bpf_attr *attr, struct bpf_prog *prog){ struct bpf_mprog_entry *entry, *entry_new; struct bpf_prog *replace_prog = NULL; struct net_device *dev; int ret;
rtnl_lock(); dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex, attr->attach_type); if (IS_ERR(dev)) { ret = PTR_ERR(dev); goto out; } entry = netkit_entry_fetch(dev, true); if (attr->attach_flags & BPF_F_REPLACE) { replace_prog = bpf_prog_get_type(attr->replace_bpf_fd, prog->type); if (IS_ERR(replace_prog)) { ret = PTR_ERR(replace_prog); replace_prog = NULL; goto out; } } ret = bpf_mprog_attach(entry, &entry_new, prog, NULL, replace_prog, attr->attach_flags, attr->relative_fd, attr->expected_revision); if (!ret) { if (entry != entry_new) { netkit_entry_update(dev, entry_new); netkit_entry_sync(); } bpf_mprog_commit(entry); }out: if (replace_prog) bpf_prog_put(replace_prog); rtnl_unlock(); return ret;}
int netkit_prog_detach(const union bpf_attr *attr, struct bpf_prog *prog){ struct bpf_mprog_entry *entry, *entry_new; struct net_device *dev; int ret;
rtnl_lock(); dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->target_ifindex, attr->attach_type); if (IS_ERR(dev)) { ret = PTR_ERR(dev); goto out; } entry = netkit_entry_fetch(dev, false); if (!entry) { ret = -ENOENT; goto out; } ret = bpf_mprog_detach(entry, &entry_new, prog, NULL, attr->attach_flags, attr->relative_fd, attr->expected_revision); if (!ret) { if (!bpf_mprog_total(entry_new)) entry_new = NULL; netkit_entry_update(dev, entry_new); netkit_entry_sync(); bpf_mprog_commit(entry); }out: rtnl_unlock(); return ret;}
int netkit_prog_query(const union bpf_attr *attr, union bpf_attr __user *uattr){ struct net_device *dev; int ret;
rtnl_lock(); dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->query.target_ifindex, attr->query.attach_type); if (IS_ERR(dev)) { ret = PTR_ERR(dev); goto out; } ret = bpf_mprog_query(attr, uattr, netkit_entry_fetch(dev, false));out: rtnl_unlock(); return ret;}
static struct netkit_link *netkit_link(const struct bpf_link *link){ return container_of(link, struct netkit_link, link);}
static int netkit_link_prog_attach(struct bpf_link *link, u32 flags, u32 id_or_fd, u64 revision){ struct netkit_link *nkl = netkit_link(link); struct bpf_mprog_entry *entry, *entry_new; struct net_device *dev = nkl->dev; int ret;
ASSERT_RTNL(); entry = netkit_entry_fetch(dev, true); ret = bpf_mprog_attach(entry, &entry_new, link->prog, link, NULL, flags, id_or_fd, revision); if (!ret) { if (entry != entry_new) { netkit_entry_update(dev, entry_new); netkit_entry_sync(); } bpf_mprog_commit(entry); } return ret;}
static void netkit_link_release(struct bpf_link *link){ struct netkit_link *nkl = netkit_link(link); struct bpf_mprog_entry *entry, *entry_new; struct net_device *dev; int ret = 0;
rtnl_lock(); dev = nkl->dev; if (!dev) goto out; entry = netkit_entry_fetch(dev, false); if (!entry) { ret = -ENOENT; goto out; } ret = bpf_mprog_detach(entry, &entry_new, link->prog, link, 0, 0, 0); if (!ret) { if (!bpf_mprog_total(entry_new)) entry_new = NULL; netkit_entry_update(dev, entry_new); netkit_entry_sync(); bpf_mprog_commit(entry); nkl->dev = NULL; }out: WARN_ON_ONCE(ret); rtnl_unlock();}
static int netkit_link_update(struct bpf_link *link, struct bpf_prog *nprog, struct bpf_prog *oprog){ struct netkit_link *nkl = netkit_link(link); struct bpf_mprog_entry *entry, *entry_new; struct net_device *dev; int ret = 0;
rtnl_lock(); dev = nkl->dev; if (!dev) { ret = -ENOLINK; goto out; } if (oprog && link->prog != oprog) { ret = -EPERM; goto out; } oprog = link->prog; if (oprog == nprog) { bpf_prog_put(nprog); goto out; } entry = netkit_entry_fetch(dev, false); if (!entry) { ret = -ENOENT; goto out; } ret = bpf_mprog_attach(entry, &entry_new, nprog, link, oprog, BPF_F_REPLACE | BPF_F_ID, link->prog->aux->id, 0); if (!ret) { WARN_ON_ONCE(entry != entry_new); oprog = xchg(&link->prog, nprog); bpf_prog_put(oprog); bpf_mprog_commit(entry); }out: rtnl_unlock(); return ret;}
static void netkit_link_dealloc(struct bpf_link *link){ kfree(netkit_link(link));}
static void netkit_link_fdinfo(const struct bpf_link *link, struct seq_file *seq){ const struct netkit_link *nkl = netkit_link(link); u32 ifindex = 0;
rtnl_lock(); if (nkl->dev) ifindex = nkl->dev->ifindex; rtnl_unlock();
seq_printf(seq, "ifindex:\t%u\n", ifindex); seq_printf(seq, "attach_type:\t%u (%s)\n", link->attach_type, link->attach_type == BPF_NETKIT_PRIMARY ? "primary" : "peer");}
static int netkit_link_fill_info(const struct bpf_link *link, struct bpf_link_info *info){ const struct netkit_link *nkl = netkit_link(link); u32 ifindex = 0;
rtnl_lock(); if (nkl->dev) ifindex = nkl->dev->ifindex; rtnl_unlock();
info->netkit.ifindex = ifindex; info->netkit.attach_type = link->attach_type; return 0;}
static int netkit_link_detach(struct bpf_link *link){ netkit_link_release(link); return 0;}
static const struct bpf_link_ops netkit_link_lops = { .release = netkit_link_release, .detach = netkit_link_detach, .dealloc = netkit_link_dealloc, .update_prog = netkit_link_update, .show_fdinfo = netkit_link_fdinfo, .fill_link_info = netkit_link_fill_info,};
static int netkit_link_init(struct netkit_link *nkl, struct bpf_link_primer *link_primer, const union bpf_attr *attr, struct net_device *dev, struct bpf_prog *prog){ bpf_link_init(&nkl->link, BPF_LINK_TYPE_NETKIT, &netkit_link_lops, prog, attr->link_create.attach_type); nkl->dev = dev; return bpf_link_prime(&nkl->link, link_primer);}
int netkit_link_attach(const union bpf_attr *attr, struct bpf_prog *prog){ struct bpf_link_primer link_primer; struct netkit_link *nkl; struct net_device *dev; int ret;
rtnl_lock(); dev = netkit_dev_fetch(current->nsproxy->net_ns, attr->link_create.target_ifindex, attr->link_create.attach_type); if (IS_ERR(dev)) { ret = PTR_ERR(dev); goto out; } nkl = kzalloc_obj(*nkl, GFP_KERNEL_ACCOUNT); if (!nkl) { ret = -ENOMEM; goto out; } ret = netkit_link_init(nkl, &link_primer, attr, dev, prog); if (ret) { kfree(nkl); goto out; } ret = netkit_link_prog_attach(&nkl->link, attr->link_create.flags, attr->link_create.netkit.relative_fd, attr->link_create.netkit.expected_revision); if (ret) { nkl->dev = NULL; bpf_link_cleanup(&link_primer); goto out; } ret = bpf_link_settle(&link_primer);out: rtnl_unlock(); return ret;}
static void netkit_release_all(struct net_device *dev){ struct bpf_mprog_entry *entry; struct bpf_tuple tuple = {}; struct bpf_mprog_fp *fp; struct bpf_mprog_cp *cp;
entry = netkit_entry_fetch(dev, false); if (!entry) return; netkit_entry_update(dev, NULL); netkit_entry_sync(); bpf_mprog_foreach_tuple(entry, fp, cp, tuple) { if (tuple.link) netkit_link(tuple.link)->dev = NULL; else bpf_prog_put(tuple.prog); }}
static void netkit_uninit(struct net_device *dev){ netkit_release_all(dev); netkit_queue_unlease(dev);}
static void netkit_del_link(struct net_device *dev, struct list_head *head){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer);
RCU_INIT_POINTER(nk->peer, NULL); unregister_netdevice_queue(dev, head); if (peer) { nk = netkit_priv(peer); RCU_INIT_POINTER(nk->peer, NULL); /* Guard against the peer already being in an unregister * list (e.g. same-namespace teardown where the peer is * in the caller's dev_kill_list). list_move_tail() on an * already-queued device would otherwise corrupt that * list's iteration. This situation can occur via netkit * notifier, hence guard against this scenario. */ if (!unregister_netdevice_queued(peer)) unregister_netdevice_queue(peer, head); }}
static int netkit_change_link(struct net_device *dev, struct nlattr *tb[], struct nlattr *data[], struct netlink_ext_ack *extack){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer); enum netkit_action policy; struct nlattr *attr; int err, i; static const struct { u32 attr; char *name; } fixed_params[] = { { IFLA_NETKIT_MODE, "operating mode" }, { IFLA_NETKIT_SCRUB, "scrubbing" }, { IFLA_NETKIT_PEER_SCRUB, "peer scrubbing" }, { IFLA_NETKIT_PEER_INFO, "peer info" }, { IFLA_NETKIT_HEADROOM, "headroom" }, { IFLA_NETKIT_TAILROOM, "tailroom" }, { IFLA_NETKIT_PAIRING, "pairing" }, };
if (!nk->primary) { NL_SET_ERR_MSG(extack, "netkit link settings can be changed only through the primary device"); return -EACCES; }
for (i = 0; i < ARRAY_SIZE(fixed_params); i++) { attr = data[fixed_params[i].attr]; if (attr) { NL_SET_ERR_MSG_ATTR_FMT(extack, attr, "netkit link %s cannot be changed after device creation", fixed_params[i].name); return -EACCES; } }
if (data[IFLA_NETKIT_POLICY]) { err = -EOPNOTSUPP; attr = data[IFLA_NETKIT_POLICY]; policy = nla_get_u32(attr); if (nk->pair == NETKIT_DEVICE_PAIR) err = netkit_check_policy(policy, attr, extack); if (err) return err; WRITE_ONCE(nk->policy, policy); }
if (data[IFLA_NETKIT_PEER_POLICY]) { err = -EOPNOTSUPP; attr = data[IFLA_NETKIT_PEER_POLICY]; policy = nla_get_u32(attr); if (peer) err = netkit_check_policy(policy, attr, extack); if (err) return err; nk = netkit_priv(peer); WRITE_ONCE(nk->policy, policy); }
return 0;}
static void netkit_check_lease_unregister(struct net_device *dev){ LIST_HEAD(list_kill); u32 q_idx;
if (READ_ONCE(dev->reg_state) != NETREG_UNREGISTERING || !dev->dev.parent) return;
netdev_lock_ops(dev); for (q_idx = 0; q_idx < dev->real_num_rx_queues; q_idx++) { struct net_device *tmp = dev; struct netdev_rx_queue *rxq; u32 tmp_q_idx = q_idx;
rxq = __netif_get_rx_queue_lease(&tmp, &tmp_q_idx, NETIF_PHYS_TO_VIRT); if (rxq && tmp != dev && tmp->netdev_ops == &netkit_netdev_ops) { /* A single phys device can have multiple queues leased * to one netkit device. We can only queue that netkit * device once to the list_kill. Queues of that phys * device can be leased with different individual netkit * devices, hence we batch via list_kill. */ if (unregister_netdevice_queued(tmp)) continue; netkit_del_link(tmp, &list_kill); } } netdev_unlock_ops(dev); unregister_netdevice_many(&list_kill);}
static int netkit_notifier(struct notifier_block *this, unsigned long event, void *ptr){ struct net_device *dev = netdev_notifier_info_to_dev(ptr);
if (event == NETDEV_UNREGISTER) netkit_check_lease_unregister(dev); return NOTIFY_DONE;}
static size_t netkit_get_size(const struct net_device *dev){ return nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_POLICY */ nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_POLICY */ nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_SCRUB */ nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PEER_SCRUB */ nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_MODE */ nla_total_size(sizeof(u8)) + /* IFLA_NETKIT_PRIMARY */ nla_total_size(sizeof(u16)) + /* IFLA_NETKIT_HEADROOM */ nla_total_size(sizeof(u16)) + /* IFLA_NETKIT_TAILROOM */ nla_total_size(sizeof(u32)) + /* IFLA_NETKIT_PAIRING */ 0;}
static int netkit_fill_info(struct sk_buff *skb, const struct net_device *dev){ struct netkit *nk = netkit_priv(dev); struct net_device *peer = rtnl_dereference(nk->peer);
if (nla_put_u8(skb, IFLA_NETKIT_PRIMARY, nk->primary)) return -EMSGSIZE; if (nla_put_u32(skb, IFLA_NETKIT_POLICY, nk->policy)) return -EMSGSIZE; if (nla_put_u32(skb, IFLA_NETKIT_MODE, nk->mode)) return -EMSGSIZE; if (nk->pair == NETKIT_DEVICE_PAIR && nla_put_u32(skb, IFLA_NETKIT_SCRUB, nk->scrub)) return -EMSGSIZE; if (nla_put_u16(skb, IFLA_NETKIT_HEADROOM, dev->needed_headroom)) return -EMSGSIZE; if (nla_put_u16(skb, IFLA_NETKIT_TAILROOM, dev->needed_tailroom)) return -EMSGSIZE; if (nla_put_u32(skb, IFLA_NETKIT_PAIRING, nk->pair)) return -EMSGSIZE;
if (peer) { nk = netkit_priv(peer); if (nla_put_u32(skb, IFLA_NETKIT_PEER_POLICY, nk->policy)) return -EMSGSIZE; if (nla_put_u32(skb, IFLA_NETKIT_PEER_SCRUB, nk->scrub)) return -EMSGSIZE; }
return 0;}
static const struct nla_policy netkit_policy[IFLA_NETKIT_MAX + 1] = { [IFLA_NETKIT_PEER_INFO] = { .len = sizeof(struct ifinfomsg) }, [IFLA_NETKIT_MODE] = NLA_POLICY_MAX(NLA_U32, NETKIT_L3), [IFLA_NETKIT_POLICY] = { .type = NLA_U32 }, [IFLA_NETKIT_PEER_POLICY] = { .type = NLA_U32 }, [IFLA_NETKIT_HEADROOM] = { .type = NLA_U16 }, [IFLA_NETKIT_TAILROOM] = { .type = NLA_U16 }, [IFLA_NETKIT_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT), [IFLA_NETKIT_PEER_SCRUB] = NLA_POLICY_MAX(NLA_U32, NETKIT_SCRUB_DEFAULT), [IFLA_NETKIT_PAIRING] = NLA_POLICY_MAX(NLA_U32, NETKIT_DEVICE_SINGLE), [IFLA_NETKIT_PRIMARY] = { .type = NLA_REJECT, .reject_message = "Primary attribute is read-only" },};
static struct rtnl_link_ops netkit_link_ops = { .kind = NETKIT_DRV_NAME, .priv_size = sizeof(struct netkit), .alloc = netkit_alloc, .setup = netkit_setup, .newlink = netkit_new_link, .dellink = netkit_del_link, .changelink = netkit_change_link, .get_link_net = netkit_get_link_net, .get_size = netkit_get_size, .fill_info = netkit_fill_info, .policy = netkit_policy, .validate = netkit_validate, .peer_type = IFLA_NETKIT_PEER_INFO, .maxtype = IFLA_NETKIT_MAX,};
static struct notifier_block netkit_netdev_notifier = { .notifier_call = netkit_notifier,};
static __init int netkit_mod_init(void){ int ret;
BUILD_BUG_ON((int)NETKIT_NEXT != (int)TCX_NEXT || (int)NETKIT_PASS != (int)TCX_PASS || (int)NETKIT_DROP != (int)TCX_DROP || (int)NETKIT_REDIRECT != (int)TCX_REDIRECT);
ret = rtnl_link_register(&netkit_link_ops); if (ret) return ret; ret = register_netdevice_notifier(&netkit_netdev_notifier); if (ret) rtnl_link_unregister(&netkit_link_ops); return ret;}
static __exit void netkit_mod_exit(void){ unregister_netdevice_notifier(&netkit_netdev_notifier); rtnl_link_unregister(&netkit_link_ops);}
module_init(netkit_mod_init);module_exit(netkit_mod_exit);
MODULE_DESCRIPTION("BPF-programmable network device");MODULE_AUTHOR("Daniel Borkmann <daniel@iogearbox.net>");MODULE_AUTHOR("Nikolay Aleksandrov <razor@blackwall.org>");MODULE_LICENSE("GPL");MODULE_ALIAS_RTNL_LINK(NETKIT_DRV_NAME);