tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net>
4 * Copyright (c) 2016 Pablo Neira Ayuso <pablo@netfilter.org>
5 *
6 * Development of this code funded by Astaro AG (http://www.astaro.com/)
7 */
8
9#include <linux/kernel.h>
10#include <linux/if_vlan.h>
11#include <linux/init.h>
12#include <linux/module.h>
13#include <linux/netlink.h>
14#include <linux/netfilter.h>
15#include <linux/netfilter/nf_tables.h>
16#include <net/netfilter/nf_tables_core.h>
17#include <net/netfilter/nf_tables.h>
18#include <net/netfilter/nf_tables_offload.h>
19/* For layer 4 checksum field offset. */
20#include <linux/tcp.h>
21#include <linux/udp.h>
22#include <net/gre.h>
23#include <linux/icmpv6.h>
24#include <linux/ip.h>
25#include <linux/ipv6.h>
26#include <net/sctp/checksum.h>
27
28static bool nft_payload_rebuild_vlan_hdr(const struct sk_buff *skb, int mac_off,
29 struct vlan_ethhdr *veth)
30{
31 if (skb_copy_bits(skb, mac_off, veth, ETH_HLEN))
32 return false;
33
34 veth->h_vlan_proto = skb->vlan_proto;
35 veth->h_vlan_TCI = htons(skb_vlan_tag_get(skb));
36 veth->h_vlan_encapsulated_proto = skb->protocol;
37
38 return true;
39}
40
41/* add vlan header into the user buffer for if tag was removed by offloads */
42static bool
43nft_payload_copy_vlan(u32 *d, const struct sk_buff *skb, u8 offset, u8 len)
44{
45 int mac_off = skb_mac_header(skb) - skb->data;
46 u8 *vlanh, *dst_u8 = (u8 *) d;
47 struct vlan_ethhdr veth;
48 u8 vlan_hlen = 0;
49
50 if ((skb->protocol == htons(ETH_P_8021AD) ||
51 skb->protocol == htons(ETH_P_8021Q)) &&
52 offset >= VLAN_ETH_HLEN && offset < VLAN_ETH_HLEN + VLAN_HLEN)
53 vlan_hlen += VLAN_HLEN;
54
55 vlanh = (u8 *) &veth;
56 if (offset < VLAN_ETH_HLEN + vlan_hlen) {
57 u8 ethlen = len;
58
59 if (vlan_hlen &&
60 skb_copy_bits(skb, mac_off, &veth, VLAN_ETH_HLEN) < 0)
61 return false;
62 else if (!nft_payload_rebuild_vlan_hdr(skb, mac_off, &veth))
63 return false;
64
65 if (offset + len > VLAN_ETH_HLEN + vlan_hlen)
66 ethlen -= offset + len - VLAN_ETH_HLEN - vlan_hlen;
67
68 memcpy(dst_u8, vlanh + offset - vlan_hlen, ethlen);
69
70 len -= ethlen;
71 if (len == 0)
72 return true;
73
74 dst_u8 += ethlen;
75 offset = ETH_HLEN + vlan_hlen;
76 } else {
77 offset -= VLAN_HLEN + vlan_hlen;
78 }
79
80 return skb_copy_bits(skb, offset + mac_off, dst_u8, len) == 0;
81}
82
83static int __nft_payload_inner_offset(struct nft_pktinfo *pkt)
84{
85 unsigned int thoff = nft_thoff(pkt);
86
87 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
88 return -1;
89
90 switch (pkt->tprot) {
91 case IPPROTO_UDP:
92 pkt->inneroff = thoff + sizeof(struct udphdr);
93 break;
94 case IPPROTO_TCP: {
95 struct tcphdr *th, _tcph;
96
97 th = skb_header_pointer(pkt->skb, thoff, sizeof(_tcph), &_tcph);
98 if (!th)
99 return -1;
100
101 pkt->inneroff = thoff + __tcp_hdrlen(th);
102 }
103 break;
104 case IPPROTO_GRE: {
105 u32 offset = sizeof(struct gre_base_hdr);
106 struct gre_base_hdr *gre, _gre;
107 __be16 version;
108
109 gre = skb_header_pointer(pkt->skb, thoff, sizeof(_gre), &_gre);
110 if (!gre)
111 return -1;
112
113 version = gre->flags & GRE_VERSION;
114 switch (version) {
115 case GRE_VERSION_0:
116 if (gre->flags & GRE_ROUTING)
117 return -1;
118
119 if (gre->flags & GRE_CSUM) {
120 offset += sizeof_field(struct gre_full_hdr, csum) +
121 sizeof_field(struct gre_full_hdr, reserved1);
122 }
123 if (gre->flags & GRE_KEY)
124 offset += sizeof_field(struct gre_full_hdr, key);
125
126 if (gre->flags & GRE_SEQ)
127 offset += sizeof_field(struct gre_full_hdr, seq);
128 break;
129 default:
130 return -1;
131 }
132
133 pkt->inneroff = thoff + offset;
134 }
135 break;
136 case IPPROTO_IPIP:
137 pkt->inneroff = thoff;
138 break;
139 default:
140 return -1;
141 }
142
143 pkt->flags |= NFT_PKTINFO_INNER;
144
145 return 0;
146}
147
148int nft_payload_inner_offset(const struct nft_pktinfo *pkt)
149{
150 if (!(pkt->flags & NFT_PKTINFO_INNER) &&
151 __nft_payload_inner_offset((struct nft_pktinfo *)pkt) < 0)
152 return -1;
153
154 return pkt->inneroff;
155}
156
157void nft_payload_eval(const struct nft_expr *expr,
158 struct nft_regs *regs,
159 const struct nft_pktinfo *pkt)
160{
161 const struct nft_payload *priv = nft_expr_priv(expr);
162 const struct sk_buff *skb = pkt->skb;
163 u32 *dest = ®s->data[priv->dreg];
164 int offset;
165
166 if (priv->len % NFT_REG32_SIZE)
167 dest[priv->len / NFT_REG32_SIZE] = 0;
168
169 switch (priv->base) {
170 case NFT_PAYLOAD_LL_HEADER:
171 if (!skb_mac_header_was_set(skb))
172 goto err;
173
174 if (skb_vlan_tag_present(skb) &&
175 priv->offset >= offsetof(struct ethhdr, h_proto)) {
176 if (!nft_payload_copy_vlan(dest, skb,
177 priv->offset, priv->len))
178 goto err;
179 return;
180 }
181 offset = skb_mac_header(skb) - skb->data;
182 break;
183 case NFT_PAYLOAD_NETWORK_HEADER:
184 offset = skb_network_offset(skb);
185 break;
186 case NFT_PAYLOAD_TRANSPORT_HEADER:
187 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
188 goto err;
189 offset = nft_thoff(pkt);
190 break;
191 case NFT_PAYLOAD_INNER_HEADER:
192 offset = nft_payload_inner_offset(pkt);
193 if (offset < 0)
194 goto err;
195 break;
196 default:
197 WARN_ON_ONCE(1);
198 goto err;
199 }
200 offset += priv->offset;
201
202 if (skb_copy_bits(skb, offset, dest, priv->len) < 0)
203 goto err;
204 return;
205err:
206 regs->verdict.code = NFT_BREAK;
207}
208
209static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = {
210 [NFTA_PAYLOAD_SREG] = { .type = NLA_U32 },
211 [NFTA_PAYLOAD_DREG] = { .type = NLA_U32 },
212 [NFTA_PAYLOAD_BASE] = { .type = NLA_U32 },
213 [NFTA_PAYLOAD_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255),
214 [NFTA_PAYLOAD_LEN] = NLA_POLICY_MAX(NLA_BE32, 255),
215 [NFTA_PAYLOAD_CSUM_TYPE] = { .type = NLA_U32 },
216 [NFTA_PAYLOAD_CSUM_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255),
217 [NFTA_PAYLOAD_CSUM_FLAGS] = { .type = NLA_U32 },
218};
219
220static int nft_payload_init(const struct nft_ctx *ctx,
221 const struct nft_expr *expr,
222 const struct nlattr * const tb[])
223{
224 struct nft_payload *priv = nft_expr_priv(expr);
225
226 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
227 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
228 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
229
230 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG],
231 &priv->dreg, NULL, NFT_DATA_VALUE,
232 priv->len);
233}
234
235static int nft_payload_dump(struct sk_buff *skb,
236 const struct nft_expr *expr, bool reset)
237{
238 const struct nft_payload *priv = nft_expr_priv(expr);
239
240 if (nft_dump_register(skb, NFTA_PAYLOAD_DREG, priv->dreg) ||
241 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
242 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
243 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)))
244 goto nla_put_failure;
245 return 0;
246
247nla_put_failure:
248 return -1;
249}
250
251static bool nft_payload_reduce(struct nft_regs_track *track,
252 const struct nft_expr *expr)
253{
254 const struct nft_payload *priv = nft_expr_priv(expr);
255 const struct nft_payload *payload;
256
257 if (!nft_reg_track_cmp(track, expr, priv->dreg)) {
258 nft_reg_track_update(track, expr, priv->dreg, priv->len);
259 return false;
260 }
261
262 payload = nft_expr_priv(track->regs[priv->dreg].selector);
263 if (priv->base != payload->base ||
264 priv->offset != payload->offset ||
265 priv->len != payload->len) {
266 nft_reg_track_update(track, expr, priv->dreg, priv->len);
267 return false;
268 }
269
270 if (!track->regs[priv->dreg].bitwise)
271 return true;
272
273 return nft_expr_reduce_bitwise(track, expr);
274}
275
276static bool nft_payload_offload_mask(struct nft_offload_reg *reg,
277 u32 priv_len, u32 field_len)
278{
279 unsigned int remainder, delta, k;
280 struct nft_data mask = {};
281 __be32 remainder_mask;
282
283 if (priv_len == field_len) {
284 memset(®->mask, 0xff, priv_len);
285 return true;
286 } else if (priv_len > field_len) {
287 return false;
288 }
289
290 memset(&mask, 0xff, field_len);
291 remainder = priv_len % sizeof(u32);
292 if (remainder) {
293 k = priv_len / sizeof(u32);
294 delta = field_len - priv_len;
295 remainder_mask = htonl(~((1 << (delta * BITS_PER_BYTE)) - 1));
296 mask.data[k] = (__force u32)remainder_mask;
297 }
298
299 memcpy(®->mask, &mask, field_len);
300
301 return true;
302}
303
304static int nft_payload_offload_ll(struct nft_offload_ctx *ctx,
305 struct nft_flow_rule *flow,
306 const struct nft_payload *priv)
307{
308 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
309
310 switch (priv->offset) {
311 case offsetof(struct ethhdr, h_source):
312 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN))
313 return -EOPNOTSUPP;
314
315 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
316 src, ETH_ALEN, reg);
317 break;
318 case offsetof(struct ethhdr, h_dest):
319 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN))
320 return -EOPNOTSUPP;
321
322 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs,
323 dst, ETH_ALEN, reg);
324 break;
325 case offsetof(struct ethhdr, h_proto):
326 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
327 return -EOPNOTSUPP;
328
329 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic,
330 n_proto, sizeof(__be16), reg);
331 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
332 break;
333 case offsetof(struct vlan_ethhdr, h_vlan_TCI):
334 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
335 return -EOPNOTSUPP;
336
337 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_VLAN, vlan,
338 vlan_tci, sizeof(__be16), reg,
339 NFT_OFFLOAD_F_NETWORK2HOST);
340 break;
341 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto):
342 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
343 return -EOPNOTSUPP;
344
345 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_VLAN, vlan,
346 vlan_tpid, sizeof(__be16), reg);
347 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
348 break;
349 case offsetof(struct vlan_ethhdr, h_vlan_TCI) + sizeof(struct vlan_hdr):
350 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
351 return -EOPNOTSUPP;
352
353 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_CVLAN, cvlan,
354 vlan_tci, sizeof(__be16), reg,
355 NFT_OFFLOAD_F_NETWORK2HOST);
356 break;
357 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto) +
358 sizeof(struct vlan_hdr):
359 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
360 return -EOPNOTSUPP;
361
362 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_CVLAN, cvlan,
363 vlan_tpid, sizeof(__be16), reg);
364 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK);
365 break;
366 default:
367 return -EOPNOTSUPP;
368 }
369
370 return 0;
371}
372
373static int nft_payload_offload_ip(struct nft_offload_ctx *ctx,
374 struct nft_flow_rule *flow,
375 const struct nft_payload *priv)
376{
377 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
378
379 switch (priv->offset) {
380 case offsetof(struct iphdr, saddr):
381 if (!nft_payload_offload_mask(reg, priv->len,
382 sizeof(struct in_addr)))
383 return -EOPNOTSUPP;
384
385 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, src,
386 sizeof(struct in_addr), reg);
387 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
388 break;
389 case offsetof(struct iphdr, daddr):
390 if (!nft_payload_offload_mask(reg, priv->len,
391 sizeof(struct in_addr)))
392 return -EOPNOTSUPP;
393
394 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, dst,
395 sizeof(struct in_addr), reg);
396 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
397 break;
398 case offsetof(struct iphdr, protocol):
399 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8)))
400 return -EOPNOTSUPP;
401
402 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
403 sizeof(__u8), reg);
404 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
405 break;
406 default:
407 return -EOPNOTSUPP;
408 }
409
410 return 0;
411}
412
413static int nft_payload_offload_ip6(struct nft_offload_ctx *ctx,
414 struct nft_flow_rule *flow,
415 const struct nft_payload *priv)
416{
417 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
418
419 switch (priv->offset) {
420 case offsetof(struct ipv6hdr, saddr):
421 if (!nft_payload_offload_mask(reg, priv->len,
422 sizeof(struct in6_addr)))
423 return -EOPNOTSUPP;
424
425 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, src,
426 sizeof(struct in6_addr), reg);
427 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
428 break;
429 case offsetof(struct ipv6hdr, daddr):
430 if (!nft_payload_offload_mask(reg, priv->len,
431 sizeof(struct in6_addr)))
432 return -EOPNOTSUPP;
433
434 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, dst,
435 sizeof(struct in6_addr), reg);
436 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
437 break;
438 case offsetof(struct ipv6hdr, nexthdr):
439 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8)))
440 return -EOPNOTSUPP;
441
442 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto,
443 sizeof(__u8), reg);
444 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT);
445 break;
446 default:
447 return -EOPNOTSUPP;
448 }
449
450 return 0;
451}
452
453static int nft_payload_offload_nh(struct nft_offload_ctx *ctx,
454 struct nft_flow_rule *flow,
455 const struct nft_payload *priv)
456{
457 int err;
458
459 switch (ctx->dep.l3num) {
460 case htons(ETH_P_IP):
461 err = nft_payload_offload_ip(ctx, flow, priv);
462 break;
463 case htons(ETH_P_IPV6):
464 err = nft_payload_offload_ip6(ctx, flow, priv);
465 break;
466 default:
467 return -EOPNOTSUPP;
468 }
469
470 return err;
471}
472
473static int nft_payload_offload_tcp(struct nft_offload_ctx *ctx,
474 struct nft_flow_rule *flow,
475 const struct nft_payload *priv)
476{
477 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
478
479 switch (priv->offset) {
480 case offsetof(struct tcphdr, source):
481 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
482 return -EOPNOTSUPP;
483
484 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
485 sizeof(__be16), reg);
486 break;
487 case offsetof(struct tcphdr, dest):
488 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
489 return -EOPNOTSUPP;
490
491 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
492 sizeof(__be16), reg);
493 break;
494 default:
495 return -EOPNOTSUPP;
496 }
497
498 return 0;
499}
500
501static int nft_payload_offload_udp(struct nft_offload_ctx *ctx,
502 struct nft_flow_rule *flow,
503 const struct nft_payload *priv)
504{
505 struct nft_offload_reg *reg = &ctx->regs[priv->dreg];
506
507 switch (priv->offset) {
508 case offsetof(struct udphdr, source):
509 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
510 return -EOPNOTSUPP;
511
512 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src,
513 sizeof(__be16), reg);
514 break;
515 case offsetof(struct udphdr, dest):
516 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16)))
517 return -EOPNOTSUPP;
518
519 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst,
520 sizeof(__be16), reg);
521 break;
522 default:
523 return -EOPNOTSUPP;
524 }
525
526 return 0;
527}
528
529static int nft_payload_offload_th(struct nft_offload_ctx *ctx,
530 struct nft_flow_rule *flow,
531 const struct nft_payload *priv)
532{
533 int err;
534
535 switch (ctx->dep.protonum) {
536 case IPPROTO_TCP:
537 err = nft_payload_offload_tcp(ctx, flow, priv);
538 break;
539 case IPPROTO_UDP:
540 err = nft_payload_offload_udp(ctx, flow, priv);
541 break;
542 default:
543 return -EOPNOTSUPP;
544 }
545
546 return err;
547}
548
549static int nft_payload_offload(struct nft_offload_ctx *ctx,
550 struct nft_flow_rule *flow,
551 const struct nft_expr *expr)
552{
553 const struct nft_payload *priv = nft_expr_priv(expr);
554 int err;
555
556 switch (priv->base) {
557 case NFT_PAYLOAD_LL_HEADER:
558 err = nft_payload_offload_ll(ctx, flow, priv);
559 break;
560 case NFT_PAYLOAD_NETWORK_HEADER:
561 err = nft_payload_offload_nh(ctx, flow, priv);
562 break;
563 case NFT_PAYLOAD_TRANSPORT_HEADER:
564 err = nft_payload_offload_th(ctx, flow, priv);
565 break;
566 default:
567 err = -EOPNOTSUPP;
568 break;
569 }
570 return err;
571}
572
573static const struct nft_expr_ops nft_payload_ops = {
574 .type = &nft_payload_type,
575 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
576 .eval = nft_payload_eval,
577 .init = nft_payload_init,
578 .dump = nft_payload_dump,
579 .reduce = nft_payload_reduce,
580 .offload = nft_payload_offload,
581};
582
583const struct nft_expr_ops nft_payload_fast_ops = {
584 .type = &nft_payload_type,
585 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
586 .eval = nft_payload_eval,
587 .init = nft_payload_init,
588 .dump = nft_payload_dump,
589 .reduce = nft_payload_reduce,
590 .offload = nft_payload_offload,
591};
592
593void nft_payload_inner_eval(const struct nft_expr *expr, struct nft_regs *regs,
594 const struct nft_pktinfo *pkt,
595 struct nft_inner_tun_ctx *tun_ctx)
596{
597 const struct nft_payload *priv = nft_expr_priv(expr);
598 const struct sk_buff *skb = pkt->skb;
599 u32 *dest = ®s->data[priv->dreg];
600 int offset;
601
602 if (priv->len % NFT_REG32_SIZE)
603 dest[priv->len / NFT_REG32_SIZE] = 0;
604
605 switch (priv->base) {
606 case NFT_PAYLOAD_TUN_HEADER:
607 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TUN))
608 goto err;
609
610 offset = tun_ctx->inner_tunoff;
611 break;
612 case NFT_PAYLOAD_LL_HEADER:
613 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_LL))
614 goto err;
615
616 offset = tun_ctx->inner_lloff;
617 break;
618 case NFT_PAYLOAD_NETWORK_HEADER:
619 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_NH))
620 goto err;
621
622 offset = tun_ctx->inner_nhoff;
623 break;
624 case NFT_PAYLOAD_TRANSPORT_HEADER:
625 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TH))
626 goto err;
627
628 offset = tun_ctx->inner_thoff;
629 break;
630 default:
631 WARN_ON_ONCE(1);
632 goto err;
633 }
634 offset += priv->offset;
635
636 if (skb_copy_bits(skb, offset, dest, priv->len) < 0)
637 goto err;
638
639 return;
640err:
641 regs->verdict.code = NFT_BREAK;
642}
643
644static int nft_payload_inner_init(const struct nft_ctx *ctx,
645 const struct nft_expr *expr,
646 const struct nlattr * const tb[])
647{
648 struct nft_payload *priv = nft_expr_priv(expr);
649 u32 base;
650
651 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
652 switch (base) {
653 case NFT_PAYLOAD_TUN_HEADER:
654 case NFT_PAYLOAD_LL_HEADER:
655 case NFT_PAYLOAD_NETWORK_HEADER:
656 case NFT_PAYLOAD_TRANSPORT_HEADER:
657 break;
658 default:
659 return -EOPNOTSUPP;
660 }
661
662 priv->base = base;
663 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
664 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
665
666 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG],
667 &priv->dreg, NULL, NFT_DATA_VALUE,
668 priv->len);
669}
670
671static const struct nft_expr_ops nft_payload_inner_ops = {
672 .type = &nft_payload_type,
673 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)),
674 .init = nft_payload_inner_init,
675 .dump = nft_payload_dump,
676 /* direct call to nft_payload_inner_eval(). */
677};
678
679static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum)
680{
681 *sum = csum_fold(csum_add(csum_sub(~csum_unfold(*sum), fsum), tsum));
682 if (*sum == 0)
683 *sum = CSUM_MANGLED_0;
684}
685
686static bool nft_payload_udp_checksum(struct sk_buff *skb, unsigned int thoff)
687{
688 struct udphdr *uh, _uh;
689
690 uh = skb_header_pointer(skb, thoff, sizeof(_uh), &_uh);
691 if (!uh)
692 return false;
693
694 return (__force bool)uh->check;
695}
696
697static int nft_payload_l4csum_offset(const struct nft_pktinfo *pkt,
698 struct sk_buff *skb,
699 unsigned int *l4csum_offset)
700{
701 if (pkt->fragoff)
702 return -1;
703
704 switch (pkt->tprot) {
705 case IPPROTO_TCP:
706 *l4csum_offset = offsetof(struct tcphdr, check);
707 break;
708 case IPPROTO_UDP:
709 if (!nft_payload_udp_checksum(skb, nft_thoff(pkt)))
710 return -1;
711 fallthrough;
712 case IPPROTO_UDPLITE:
713 *l4csum_offset = offsetof(struct udphdr, check);
714 break;
715 case IPPROTO_ICMPV6:
716 *l4csum_offset = offsetof(struct icmp6hdr, icmp6_cksum);
717 break;
718 default:
719 return -1;
720 }
721
722 *l4csum_offset += nft_thoff(pkt);
723 return 0;
724}
725
726static int nft_payload_csum_sctp(struct sk_buff *skb, int offset)
727{
728 struct sctphdr *sh;
729
730 if (skb_ensure_writable(skb, offset + sizeof(*sh)))
731 return -1;
732
733 sh = (struct sctphdr *)(skb->data + offset);
734 sh->checksum = sctp_compute_cksum(skb, offset);
735 skb->ip_summed = CHECKSUM_UNNECESSARY;
736 return 0;
737}
738
739static int nft_payload_l4csum_update(const struct nft_pktinfo *pkt,
740 struct sk_buff *skb,
741 __wsum fsum, __wsum tsum)
742{
743 int l4csum_offset;
744 __sum16 sum;
745
746 /* If we cannot determine layer 4 checksum offset or this packet doesn't
747 * require layer 4 checksum recalculation, skip this packet.
748 */
749 if (nft_payload_l4csum_offset(pkt, skb, &l4csum_offset) < 0)
750 return 0;
751
752 if (skb_copy_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
753 return -1;
754
755 /* Checksum mangling for an arbitrary amount of bytes, based on
756 * inet_proto_csum_replace*() functions.
757 */
758 if (skb->ip_summed != CHECKSUM_PARTIAL) {
759 nft_csum_replace(&sum, fsum, tsum);
760 if (skb->ip_summed == CHECKSUM_COMPLETE) {
761 skb->csum = ~csum_add(csum_sub(~(skb->csum), fsum),
762 tsum);
763 }
764 } else {
765 sum = ~csum_fold(csum_add(csum_sub(csum_unfold(sum), fsum),
766 tsum));
767 }
768
769 if (skb_ensure_writable(skb, l4csum_offset + sizeof(sum)) ||
770 skb_store_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0)
771 return -1;
772
773 return 0;
774}
775
776static int nft_payload_csum_inet(struct sk_buff *skb, const u32 *src,
777 __wsum fsum, __wsum tsum, int csum_offset)
778{
779 __sum16 sum;
780
781 if (skb_copy_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
782 return -1;
783
784 nft_csum_replace(&sum, fsum, tsum);
785 if (skb_ensure_writable(skb, csum_offset + sizeof(sum)) ||
786 skb_store_bits(skb, csum_offset, &sum, sizeof(sum)) < 0)
787 return -1;
788
789 return 0;
790}
791
792struct nft_payload_set {
793 enum nft_payload_bases base:8;
794 u8 offset;
795 u8 len;
796 u8 sreg;
797 u8 csum_type;
798 u8 csum_offset;
799 u8 csum_flags;
800};
801
802static void nft_payload_set_eval(const struct nft_expr *expr,
803 struct nft_regs *regs,
804 const struct nft_pktinfo *pkt)
805{
806 const struct nft_payload_set *priv = nft_expr_priv(expr);
807 struct sk_buff *skb = pkt->skb;
808 const u32 *src = ®s->data[priv->sreg];
809 int offset, csum_offset;
810 __wsum fsum, tsum;
811
812 switch (priv->base) {
813 case NFT_PAYLOAD_LL_HEADER:
814 if (!skb_mac_header_was_set(skb))
815 goto err;
816 offset = skb_mac_header(skb) - skb->data;
817 break;
818 case NFT_PAYLOAD_NETWORK_HEADER:
819 offset = skb_network_offset(skb);
820 break;
821 case NFT_PAYLOAD_TRANSPORT_HEADER:
822 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff)
823 goto err;
824 offset = nft_thoff(pkt);
825 break;
826 case NFT_PAYLOAD_INNER_HEADER:
827 offset = nft_payload_inner_offset(pkt);
828 if (offset < 0)
829 goto err;
830 break;
831 default:
832 WARN_ON_ONCE(1);
833 goto err;
834 }
835
836 csum_offset = offset + priv->csum_offset;
837 offset += priv->offset;
838
839 if ((priv->csum_type == NFT_PAYLOAD_CSUM_INET || priv->csum_flags) &&
840 ((priv->base != NFT_PAYLOAD_TRANSPORT_HEADER &&
841 priv->base != NFT_PAYLOAD_INNER_HEADER) ||
842 skb->ip_summed != CHECKSUM_PARTIAL)) {
843 fsum = skb_checksum(skb, offset, priv->len, 0);
844 tsum = csum_partial(src, priv->len, 0);
845
846 if (priv->csum_type == NFT_PAYLOAD_CSUM_INET &&
847 nft_payload_csum_inet(skb, src, fsum, tsum, csum_offset))
848 goto err;
849
850 if (priv->csum_flags &&
851 nft_payload_l4csum_update(pkt, skb, fsum, tsum) < 0)
852 goto err;
853 }
854
855 if (skb_ensure_writable(skb, max(offset + priv->len, 0)) ||
856 skb_store_bits(skb, offset, src, priv->len) < 0)
857 goto err;
858
859 if (priv->csum_type == NFT_PAYLOAD_CSUM_SCTP &&
860 pkt->tprot == IPPROTO_SCTP &&
861 skb->ip_summed != CHECKSUM_PARTIAL) {
862 if (pkt->fragoff == 0 &&
863 nft_payload_csum_sctp(skb, nft_thoff(pkt)))
864 goto err;
865 }
866
867 return;
868err:
869 regs->verdict.code = NFT_BREAK;
870}
871
872static int nft_payload_set_init(const struct nft_ctx *ctx,
873 const struct nft_expr *expr,
874 const struct nlattr * const tb[])
875{
876 struct nft_payload_set *priv = nft_expr_priv(expr);
877 u32 csum_offset, csum_type = NFT_PAYLOAD_CSUM_NONE;
878 int err;
879
880 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
881 priv->offset = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_OFFSET]));
882 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN]));
883
884 if (tb[NFTA_PAYLOAD_CSUM_TYPE])
885 csum_type = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE]));
886 if (tb[NFTA_PAYLOAD_CSUM_OFFSET]) {
887 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_CSUM_OFFSET], U8_MAX,
888 &csum_offset);
889 if (err < 0)
890 return err;
891
892 priv->csum_offset = csum_offset;
893 }
894 if (tb[NFTA_PAYLOAD_CSUM_FLAGS]) {
895 u32 flags;
896
897 flags = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_FLAGS]));
898 if (flags & ~NFT_PAYLOAD_L4CSUM_PSEUDOHDR)
899 return -EINVAL;
900
901 priv->csum_flags = flags;
902 }
903
904 switch (csum_type) {
905 case NFT_PAYLOAD_CSUM_NONE:
906 case NFT_PAYLOAD_CSUM_INET:
907 break;
908 case NFT_PAYLOAD_CSUM_SCTP:
909 if (priv->base != NFT_PAYLOAD_TRANSPORT_HEADER)
910 return -EINVAL;
911
912 if (priv->csum_offset != offsetof(struct sctphdr, checksum))
913 return -EINVAL;
914 break;
915 default:
916 return -EOPNOTSUPP;
917 }
918 priv->csum_type = csum_type;
919
920 return nft_parse_register_load(tb[NFTA_PAYLOAD_SREG], &priv->sreg,
921 priv->len);
922}
923
924static int nft_payload_set_dump(struct sk_buff *skb,
925 const struct nft_expr *expr, bool reset)
926{
927 const struct nft_payload_set *priv = nft_expr_priv(expr);
928
929 if (nft_dump_register(skb, NFTA_PAYLOAD_SREG, priv->sreg) ||
930 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) ||
931 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) ||
932 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)) ||
933 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_TYPE, htonl(priv->csum_type)) ||
934 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_OFFSET,
935 htonl(priv->csum_offset)) ||
936 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_FLAGS, htonl(priv->csum_flags)))
937 goto nla_put_failure;
938 return 0;
939
940nla_put_failure:
941 return -1;
942}
943
944static bool nft_payload_set_reduce(struct nft_regs_track *track,
945 const struct nft_expr *expr)
946{
947 int i;
948
949 for (i = 0; i < NFT_REG32_NUM; i++) {
950 if (!track->regs[i].selector)
951 continue;
952
953 if (track->regs[i].selector->ops != &nft_payload_ops &&
954 track->regs[i].selector->ops != &nft_payload_fast_ops)
955 continue;
956
957 __nft_reg_track_cancel(track, i);
958 }
959
960 return false;
961}
962
963static const struct nft_expr_ops nft_payload_set_ops = {
964 .type = &nft_payload_type,
965 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload_set)),
966 .eval = nft_payload_set_eval,
967 .init = nft_payload_set_init,
968 .dump = nft_payload_set_dump,
969 .reduce = nft_payload_set_reduce,
970};
971
972static const struct nft_expr_ops *
973nft_payload_select_ops(const struct nft_ctx *ctx,
974 const struct nlattr * const tb[])
975{
976 enum nft_payload_bases base;
977 unsigned int offset, len;
978 int err;
979
980 if (tb[NFTA_PAYLOAD_BASE] == NULL ||
981 tb[NFTA_PAYLOAD_OFFSET] == NULL ||
982 tb[NFTA_PAYLOAD_LEN] == NULL)
983 return ERR_PTR(-EINVAL);
984
985 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE]));
986 switch (base) {
987 case NFT_PAYLOAD_LL_HEADER:
988 case NFT_PAYLOAD_NETWORK_HEADER:
989 case NFT_PAYLOAD_TRANSPORT_HEADER:
990 case NFT_PAYLOAD_INNER_HEADER:
991 break;
992 default:
993 return ERR_PTR(-EOPNOTSUPP);
994 }
995
996 if (tb[NFTA_PAYLOAD_SREG] != NULL) {
997 if (tb[NFTA_PAYLOAD_DREG] != NULL)
998 return ERR_PTR(-EINVAL);
999 return &nft_payload_set_ops;
1000 }
1001
1002 if (tb[NFTA_PAYLOAD_DREG] == NULL)
1003 return ERR_PTR(-EINVAL);
1004
1005 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U8_MAX, &offset);
1006 if (err < 0)
1007 return ERR_PTR(err);
1008
1009 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_LEN], U8_MAX, &len);
1010 if (err < 0)
1011 return ERR_PTR(err);
1012
1013 if (len <= 4 && is_power_of_2(len) && IS_ALIGNED(offset, len) &&
1014 base != NFT_PAYLOAD_LL_HEADER && base != NFT_PAYLOAD_INNER_HEADER)
1015 return &nft_payload_fast_ops;
1016 else
1017 return &nft_payload_ops;
1018}
1019
1020struct nft_expr_type nft_payload_type __read_mostly = {
1021 .name = "payload",
1022 .select_ops = nft_payload_select_ops,
1023 .inner_ops = &nft_payload_inner_ops,
1024 .policy = nft_payload_policy,
1025 .maxattr = NFTA_PAYLOAD_MAX,
1026 .owner = THIS_MODULE,
1027};