tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
1// SPDX-License-Identifier: GPL-2.0
2
3#include "vmlinux.h"
4
5#include <bpf/bpf_helpers.h>
6#include <bpf/bpf_tracing.h>
7
8char _license[] SEC("license") = "GPL";
9
10#define USEC_PER_SEC 1000000UL
11
12#define min(a, b) ((a) < (b) ? (a) : (b))
13
14static inline struct tcp_sock *tcp_sk(const struct sock *sk)
15{
16 return (struct tcp_sock *)sk;
17}
18
19SEC("struct_ops/write_sk_pacing_init")
20void BPF_PROG(write_sk_pacing_init, struct sock *sk)
21{
22#ifdef ENABLE_ATOMICS_TESTS
23 __sync_bool_compare_and_swap(&sk->sk_pacing_status, SK_PACING_NONE,
24 SK_PACING_NEEDED);
25#else
26 sk->sk_pacing_status = SK_PACING_NEEDED;
27#endif
28}
29
30SEC("struct_ops/write_sk_pacing_cong_control")
31void BPF_PROG(write_sk_pacing_cong_control, struct sock *sk,
32 const struct rate_sample *rs)
33{
34 const struct tcp_sock *tp = tcp_sk(sk);
35 unsigned long rate =
36 ((tp->snd_cwnd * tp->mss_cache * USEC_PER_SEC) << 3) /
37 (tp->srtt_us ?: 1U << 3);
38 sk->sk_pacing_rate = min(rate, sk->sk_max_pacing_rate);
39}
40
41SEC("struct_ops/write_sk_pacing_ssthresh")
42__u32 BPF_PROG(write_sk_pacing_ssthresh, struct sock *sk)
43{
44 return tcp_sk(sk)->snd_ssthresh;
45}
46
47SEC("struct_ops/write_sk_pacing_undo_cwnd")
48__u32 BPF_PROG(write_sk_pacing_undo_cwnd, struct sock *sk)
49{
50 return tcp_sk(sk)->snd_cwnd;
51}
52
53SEC(".struct_ops")
54struct tcp_congestion_ops write_sk_pacing = {
55 .init = (void *)write_sk_pacing_init,
56 .cong_control = (void *)write_sk_pacing_cong_control,
57 .ssthresh = (void *)write_sk_pacing_ssthresh,
58 .undo_cwnd = (void *)write_sk_pacing_undo_cwnd,
59 .name = "bpf_w_sk_pacing",
60};