Something went wrong. Try again.
Linux kernel mirror (for testing) git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270/* SPDX-License-Identifier: GPL-2.0 */#ifndef __NET_TC_WRAPPER_H#define __NET_TC_WRAPPER_H
#include <net/pkt_cls.h>
#if IS_ENABLED(CONFIG_MITIGATION_RETPOLINE)
#include <linux/cpufeature.h>#include <linux/static_key.h>#include <linux/indirect_call_wrapper.h>
#define TC_INDIRECT_SCOPE
extern struct static_key_false tc_skip_wrapper_act;extern struct static_key_false tc_skip_wrapper_cls;
/* TC Actions */#ifdef CONFIG_NET_CLS_ACT
#define TC_INDIRECT_ACTION_DECLARE(fname) \ INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \ const struct tc_action *a, \ struct tcf_result *res))
TC_INDIRECT_ACTION_DECLARE(tcf_bpf_act);TC_INDIRECT_ACTION_DECLARE(tcf_connmark_act);TC_INDIRECT_ACTION_DECLARE(tcf_csum_act);TC_INDIRECT_ACTION_DECLARE(tcf_ct_act);TC_INDIRECT_ACTION_DECLARE(tcf_ctinfo_act);TC_INDIRECT_ACTION_DECLARE(tcf_gact_act);TC_INDIRECT_ACTION_DECLARE(tcf_gate_act);TC_INDIRECT_ACTION_DECLARE(tcf_ife_act);TC_INDIRECT_ACTION_DECLARE(tcf_ipt_act);TC_INDIRECT_ACTION_DECLARE(tcf_mirred_act);TC_INDIRECT_ACTION_DECLARE(tcf_mpls_act);TC_INDIRECT_ACTION_DECLARE(tcf_nat_act);TC_INDIRECT_ACTION_DECLARE(tcf_pedit_act);TC_INDIRECT_ACTION_DECLARE(tcf_police_act);TC_INDIRECT_ACTION_DECLARE(tcf_sample_act);TC_INDIRECT_ACTION_DECLARE(tcf_simp_act);TC_INDIRECT_ACTION_DECLARE(tcf_skbedit_act);TC_INDIRECT_ACTION_DECLARE(tcf_skbmod_act);TC_INDIRECT_ACTION_DECLARE(tcf_vlan_act);TC_INDIRECT_ACTION_DECLARE(tunnel_key_act);
static inline int tc_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res){ if (static_branch_likely(&tc_skip_wrapper_act)) goto skip;
#if IS_BUILTIN(CONFIG_NET_ACT_GACT) if (a->ops->act == tcf_gact_act) return tcf_gact_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_MIRRED) if (a->ops->act == tcf_mirred_act) return tcf_mirred_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_PEDIT) if (a->ops->act == tcf_pedit_act) return tcf_pedit_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT) if (a->ops->act == tcf_skbedit_act) return tcf_skbedit_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_SKBMOD) if (a->ops->act == tcf_skbmod_act) return tcf_skbmod_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_POLICE) if (a->ops->act == tcf_police_act) return tcf_police_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_BPF) if (a->ops->act == tcf_bpf_act) return tcf_bpf_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_CONNMARK) if (a->ops->act == tcf_connmark_act) return tcf_connmark_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_CSUM) if (a->ops->act == tcf_csum_act) return tcf_csum_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_CT) if (a->ops->act == tcf_ct_act) return tcf_ct_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_CTINFO) if (a->ops->act == tcf_ctinfo_act) return tcf_ctinfo_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_GATE) if (a->ops->act == tcf_gate_act) return tcf_gate_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_MPLS) if (a->ops->act == tcf_mpls_act) return tcf_mpls_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_NAT) if (a->ops->act == tcf_nat_act) return tcf_nat_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY) if (a->ops->act == tunnel_key_act) return tunnel_key_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_VLAN) if (a->ops->act == tcf_vlan_act) return tcf_vlan_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_IFE) if (a->ops->act == tcf_ife_act) return tcf_ife_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_SIMP) if (a->ops->act == tcf_simp_act) return tcf_simp_act(skb, a, res);#endif#if IS_BUILTIN(CONFIG_NET_ACT_SAMPLE) if (a->ops->act == tcf_sample_act) return tcf_sample_act(skb, a, res);#endif
skip: return a->ops->act(skb, a, res);}
#endif /* CONFIG_NET_CLS_ACT */
/* TC Filters */#ifdef CONFIG_NET_CLS
#define TC_INDIRECT_FILTER_DECLARE(fname) \ INDIRECT_CALLABLE_DECLARE(int fname(struct sk_buff *skb, \ const struct tcf_proto *tp, \ struct tcf_result *res))
TC_INDIRECT_FILTER_DECLARE(basic_classify);TC_INDIRECT_FILTER_DECLARE(cls_bpf_classify);TC_INDIRECT_FILTER_DECLARE(cls_cgroup_classify);TC_INDIRECT_FILTER_DECLARE(fl_classify);TC_INDIRECT_FILTER_DECLARE(flow_classify);TC_INDIRECT_FILTER_DECLARE(fw_classify);TC_INDIRECT_FILTER_DECLARE(mall_classify);TC_INDIRECT_FILTER_DECLARE(route4_classify);TC_INDIRECT_FILTER_DECLARE(u32_classify);
static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp, struct tcf_result *res){ if (static_branch_likely(&tc_skip_wrapper_cls)) goto skip;
#if IS_BUILTIN(CONFIG_NET_CLS_BPF) if (tp->classify == cls_bpf_classify) return cls_bpf_classify(skb, tp, res);#endif#if IS_BUILTIN(CONFIG_NET_CLS_U32) if (tp->classify == u32_classify) return u32_classify(skb, tp, res);#endif#if IS_BUILTIN(CONFIG_NET_CLS_FLOWER) if (tp->classify == fl_classify) return fl_classify(skb, tp, res);#endif#if IS_BUILTIN(CONFIG_NET_CLS_FW) if (tp->classify == fw_classify) return fw_classify(skb, tp, res);#endif#if IS_BUILTIN(CONFIG_NET_CLS_MATCHALL) if (tp->classify == mall_classify) return mall_classify(skb, tp, res);#endif#if IS_BUILTIN(CONFIG_NET_CLS_BASIC) if (tp->classify == basic_classify) return basic_classify(skb, tp, res);#endif#if IS_BUILTIN(CONFIG_NET_CLS_CGROUP) if (tp->classify == cls_cgroup_classify) return cls_cgroup_classify(skb, tp, res);#endif#if IS_BUILTIN(CONFIG_NET_CLS_FLOW) if (tp->classify == flow_classify) return flow_classify(skb, tp, res);#endif#if IS_BUILTIN(CONFIG_NET_CLS_ROUTE4) if (tp->classify == route4_classify) return route4_classify(skb, tp, res);#endif
skip: return tp->classify(skb, tp, res);}
#endif /* CONFIG_NET_CLS */
static inline void tc_wrapper_init(void){#ifdef CONFIG_X86 int cnt_cls = IS_BUILTIN(CONFIG_NET_CLS_BPF) + IS_BUILTIN(CONFIG_NET_CLS_U32) + IS_BUILTIN(CONFIG_NET_CLS_FLOWER) + IS_BUILTIN(CONFIG_NET_CLS_FW) + IS_BUILTIN(CONFIG_NET_CLS_MATCHALL) + IS_BUILTIN(CONFIG_NET_CLS_BASIC) + IS_BUILTIN(CONFIG_NET_CLS_CGROUP) + IS_BUILTIN(CONFIG_NET_CLS_FLOW) + IS_BUILTIN(CONFIG_NET_CLS_ROUTE4);
int cnt_act = IS_BUILTIN(CONFIG_NET_ACT_GACT) + IS_BUILTIN(CONFIG_NET_ACT_MIRRED) + IS_BUILTIN(CONFIG_NET_ACT_PEDIT) + IS_BUILTIN(CONFIG_NET_ACT_SKBEDIT) + IS_BUILTIN(CONFIG_NET_ACT_SKBMOD) + IS_BUILTIN(CONFIG_NET_ACT_POLICE) + IS_BUILTIN(CONFIG_NET_ACT_BPF) + IS_BUILTIN(CONFIG_NET_ACT_CONNMARK) + IS_BUILTIN(CONFIG_NET_ACT_CSUM) + IS_BUILTIN(CONFIG_NET_ACT_CT) + IS_BUILTIN(CONFIG_NET_ACT_CTINFO) + IS_BUILTIN(CONFIG_NET_ACT_GATE) + IS_BUILTIN(CONFIG_NET_ACT_MPLS) + IS_BUILTIN(CONFIG_NET_ACT_NAT) + IS_BUILTIN(CONFIG_NET_ACT_TUNNEL_KEY) + IS_BUILTIN(CONFIG_NET_ACT_VLAN) + IS_BUILTIN(CONFIG_NET_ACT_IFE) + IS_BUILTIN(CONFIG_NET_ACT_SIMP) + IS_BUILTIN(CONFIG_NET_ACT_SAMPLE);
if (cpu_feature_enabled(X86_FEATURE_RETPOLINE)) return;
if (cnt_cls > 1) static_branch_enable(&tc_skip_wrapper_cls);
if (cnt_act > 1) static_branch_enable(&tc_skip_wrapper_act);#endif}
#else
#define TC_INDIRECT_SCOPE static
static inline int tc_act(struct sk_buff *skb, const struct tc_action *a, struct tcf_result *res){ return a->ops->act(skb, a, res);}
static inline int tc_classify(struct sk_buff *skb, const struct tcf_proto *tp, struct tcf_result *res){ return tp->classify(skb, tp, res);}
static inline void tc_wrapper_init(void){}
#endif
#endif /* __NET_TC_WRAPPER_H */