Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
tjh.dev
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * IPV6 GSO/GRO offload support
4 * Linux INET6 implementation
5 *
6 * TCPv6 GSO/GRO support
7 */
8#include <linux/indirect_call_wrapper.h>
9#include <linux/skbuff.h>
10#include <net/inet6_hashtables.h>
11#include <net/gro.h>
12#include <net/protocol.h>
13#include <net/tcp.h>
14#include <net/ip6_checksum.h>
15#include "ip6_offload.h"
16
17static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb,
18 struct tcphdr *th)
19{
20#if IS_ENABLED(CONFIG_IPV6)
21 const struct ipv6hdr *hdr;
22 struct sk_buff *p;
23 struct sock *sk;
24 struct net *net;
25 int iif, sdif;
26
27 if (likely(!(skb->dev->features & NETIF_F_GRO_FRAGLIST)))
28 return;
29
30 p = tcp_gro_lookup(head, th);
31 if (p) {
32 NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist;
33 return;
34 }
35
36 inet6_get_iif_sdif(skb, &iif, &sdif);
37 hdr = skb_gro_network_header(skb);
38 net = dev_net_rcu(skb->dev);
39 sk = __inet6_lookup_established(net, &hdr->saddr, th->source,
40 &hdr->daddr, ntohs(th->dest),
41 iif, sdif);
42 NAPI_GRO_CB(skb)->is_flist = !sk;
43 if (sk)
44 sock_gen_put(sk);
45#endif /* IS_ENABLED(CONFIG_IPV6) */
46}
47
48INDIRECT_CALLABLE_SCOPE
49struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb)
50{
51 struct tcphdr *th;
52
53 /* Don't bother verifying checksum if we're going to flush anyway. */
54 if (!NAPI_GRO_CB(skb)->flush &&
55 skb_gro_checksum_validate(skb, IPPROTO_TCP,
56 ip6_gro_compute_pseudo))
57 goto flush;
58
59 th = tcp_gro_pull_header(skb);
60 if (!th)
61 goto flush;
62
63 tcp6_check_fraglist_gro(head, skb, th);
64
65 return tcp_gro_receive(head, skb, th);
66
67flush:
68 NAPI_GRO_CB(skb)->flush = 1;
69 return NULL;
70}
71
72INDIRECT_CALLABLE_SCOPE int tcp6_gro_complete(struct sk_buff *skb, int thoff)
73{
74 const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
75 const struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + offset);
76 struct tcphdr *th = tcp_hdr(skb);
77
78 if (unlikely(NAPI_GRO_CB(skb)->is_flist)) {
79 skb_shinfo(skb)->gso_type |= SKB_GSO_FRAGLIST | SKB_GSO_TCPV6;
80 skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
81
82 __skb_incr_checksum_unnecessary(skb);
83
84 return 0;
85 }
86
87 th->check = ~tcp_v6_check(skb->len - thoff, &iph->saddr,
88 &iph->daddr, 0);
89 skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
90
91 tcp_gro_complete(skb);
92 return 0;
93}
94
95static void __tcpv6_gso_segment_csum(struct sk_buff *seg,
96 struct in6_addr *oldip,
97 const struct in6_addr *newip,
98 __be16 *oldport, __be16 newport)
99{
100 struct tcphdr *th = tcp_hdr(seg);
101
102 if (!ipv6_addr_equal(oldip, newip)) {
103 inet_proto_csum_replace16(&th->check, seg,
104 oldip->s6_addr32,
105 newip->s6_addr32,
106 true);
107 *oldip = *newip;
108 }
109
110 if (*oldport == newport)
111 return;
112
113 inet_proto_csum_replace2(&th->check, seg, *oldport, newport, false);
114 *oldport = newport;
115}
116
117static struct sk_buff *__tcpv6_gso_segment_list_csum(struct sk_buff *segs)
118{
119 const struct tcphdr *th;
120 const struct ipv6hdr *iph;
121 struct sk_buff *seg;
122 struct tcphdr *th2;
123 struct ipv6hdr *iph2;
124
125 seg = segs;
126 th = tcp_hdr(seg);
127 iph = ipv6_hdr(seg);
128 th2 = tcp_hdr(seg->next);
129 iph2 = ipv6_hdr(seg->next);
130
131 if (!(*(const u32 *)&th->source ^ *(const u32 *)&th2->source) &&
132 ipv6_addr_equal(&iph->saddr, &iph2->saddr) &&
133 ipv6_addr_equal(&iph->daddr, &iph2->daddr))
134 return segs;
135
136 while ((seg = seg->next)) {
137 th2 = tcp_hdr(seg);
138 iph2 = ipv6_hdr(seg);
139
140 __tcpv6_gso_segment_csum(seg, &iph2->saddr, &iph->saddr,
141 &th2->source, th->source);
142 __tcpv6_gso_segment_csum(seg, &iph2->daddr, &iph->daddr,
143 &th2->dest, th->dest);
144 }
145
146 return segs;
147}
148
149static struct sk_buff *__tcp6_gso_segment_list(struct sk_buff *skb,
150 netdev_features_t features)
151{
152 skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
153 if (IS_ERR(skb))
154 return skb;
155
156 return __tcpv6_gso_segment_list_csum(skb);
157}
158
159static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb,
160 netdev_features_t features)
161{
162 struct tcphdr *th;
163
164 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6))
165 return ERR_PTR(-EINVAL);
166
167 if (!pskb_may_pull(skb, sizeof(*th)))
168 return ERR_PTR(-EINVAL);
169
170 if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST) {
171 struct tcphdr *th = tcp_hdr(skb);
172
173 if (skb_pagelen(skb) - th->doff * 4 == skb_shinfo(skb)->gso_size)
174 return __tcp6_gso_segment_list(skb, features);
175
176 skb->ip_summed = CHECKSUM_NONE;
177 }
178
179 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
180 const struct ipv6hdr *ipv6h = ipv6_hdr(skb);
181 struct tcphdr *th = tcp_hdr(skb);
182
183 /* Set up pseudo header, usually expect stack to have done
184 * this.
185 */
186
187 th->check = 0;
188 skb->ip_summed = CHECKSUM_PARTIAL;
189 __tcp_v6_send_check(skb, &ipv6h->saddr, &ipv6h->daddr);
190 }
191
192 return tcp_gso_segment(skb, features);
193}
194
195int __init tcpv6_offload_init(void)
196{
197 net_hotdata.tcpv6_offload = (struct net_offload) {
198 .callbacks = {
199 .gso_segment = tcp6_gso_segment,
200 .gro_receive = tcp6_gro_receive,
201 .gro_complete = tcp6_gro_complete,
202 },
203 };
204 return inet6_add_offload(&net_hotdata.tcpv6_offload, IPPROTO_TCP);
205}