1 /* SPDX-License-Identifier: GPL-2.0
3 * Copyright (c) 2019 Facebook
5 * This program is free software; you can redistribute it and/or
6 * modify it under the terms of version 2 of the GNU General Public
7 * License as published by the Free Software Foundation.
9 * Include file for sample Host Bandwidth Manager (HBM) BPF programs
11 #define KBUILD_MODNAME "foo"
14 #include <uapi/linux/bpf.h>
15 #include <uapi/linux/if_ether.h>
16 #include <uapi/linux/if_packet.h>
17 #include <uapi/linux/ip.h>
18 #include <uapi/linux/ipv6.h>
19 #include <uapi/linux/in.h>
20 #include <uapi/linux/tcp.h>
21 #include <uapi/linux/filter.h>
22 #include <uapi/linux/pkt_cls.h>
24 #include <net/inet_ecn.h>
25 #include <bpf/bpf_endian.h>
26 #include <bpf/bpf_helpers.h>
34 #ifndef HBM_DEBUG // Define HBM_DEBUG to enable debugging
36 #define bpf_printk(fmt, ...)
39 #define INITIAL_CREDIT_PACKETS 100
40 #define MAX_BYTES_PER_PACKET 1500
41 #define MARK_THRESH (40 * MAX_BYTES_PER_PACKET)
42 #define DROP_THRESH (80 * 5 * MAX_BYTES_PER_PACKET)
43 #define LARGE_PKT_DROP_THRESH (DROP_THRESH - (15 * MAX_BYTES_PER_PACKET))
44 #define MARK_REGION_SIZE (LARGE_PKT_DROP_THRESH - MARK_THRESH)
45 #define LARGE_PKT_THRESH 120
46 #define MAX_CREDIT (100 * MAX_BYTES_PER_PACKET)
47 #define INIT_CREDIT (INITIAL_CREDIT_PACKETS * MAX_BYTES_PER_PACKET)
49 // Time base accounting for fq's EDT
50 #define BURST_SIZE_NS 100000 // 100us
51 #define MARK_THRESH_NS 50000 // 50us
52 #define DROP_THRESH_NS 500000 // 500us
53 // Reserve 20us of queuing for small packets (less than 120 bytes)
54 #define LARGE_PKT_DROP_THRESH_NS (DROP_THRESH_NS - 20000)
55 #define MARK_REGION_SIZE_NS (LARGE_PKT_DROP_THRESH_NS - MARK_THRESH_NS)
57 // rate in bytes per ns << 20
58 #define CREDIT_PER_NS(delta, rate) ((((u64)(delta)) * (rate)) >> 20)
59 #define BYTES_PER_NS(delta, rate) ((((u64)(delta)) * (rate)) >> 20)
60 #define BYTES_TO_NS(bytes, rate) div64_u64(((u64)(bytes)) << 20, (u64)(rate))
63 __uint(type, BPF_MAP_TYPE_CGROUP_STORAGE);
64 __type(key, struct bpf_cgroup_storage_key);
65 __type(value, struct hbm_vqueue);
66 } queue_state SEC(".maps");
69 __uint(type, BPF_MAP_TYPE_ARRAY);
70 __uint(max_entries, 1);
72 __type(value, struct hvm_queue_stats);
73 } queue_stats SEC(".maps");
84 static int get_tcp_info(struct __sk_buff *skb, struct hbm_pkt_info *pkti)
87 struct bpf_tcp_sock *tp;
91 sk = bpf_sk_fullsock(sk);
93 if (sk->protocol == IPPROTO_TCP) {
94 tp = bpf_tcp_sock(sk);
96 pkti->cwnd = tp->snd_cwnd;
97 pkti->rtt = tp->srtt_us >> 3;
98 pkti->packets_out = tp->packets_out;
106 pkti->packets_out = 0;
110 static void hbm_get_pkt_info(struct __sk_buff *skb,
111 struct hbm_pkt_info *pkti)
114 struct ipv6hdr *ip6h;
118 bpf_skb_load_bytes(skb, 0, &iph, 12);
119 if (iph.version == 6) {
120 ip6h = (struct ipv6hdr *)&iph;
122 pkti->is_tcp = (ip6h->nexthdr == 6);
123 pkti->ecn = (ip6h->flow_lbl[0] >> 4) & INET_ECN_MASK;
124 } else if (iph.version == 4) {
126 pkti->is_tcp = (iph.protocol == 6);
127 pkti->ecn = iph.tos & INET_ECN_MASK;
130 pkti->is_tcp = false;
134 get_tcp_info(skb, pkti);
137 static __always_inline void hbm_init_vqueue(struct hbm_vqueue *qdp, int rate)
139 bpf_printk("Initializing queue_state, rate:%d\n", rate * 128);
140 qdp->lasttime = bpf_ktime_get_ns();
141 qdp->credit = INIT_CREDIT;
142 qdp->rate = rate * 128;
145 static __always_inline void hbm_init_edt_vqueue(struct hbm_vqueue *qdp,
148 unsigned long long curtime;
150 curtime = bpf_ktime_get_ns();
151 bpf_printk("Initializing queue_state, rate:%d\n", rate * 128);
152 qdp->lasttime = curtime - BURST_SIZE_NS; // support initial burst
153 qdp->credit = 0; // not used
154 qdp->rate = rate * 128;
157 static __always_inline void hbm_update_stats(struct hbm_queue_stats *qsp,
159 unsigned long long curtime,
160 bool congestion_flag,
164 struct hbm_pkt_info *pkti,
170 // Following is needed for work conserving
171 __sync_add_and_fetch(&(qsp->bytes_total), len);
173 // Optionally update statistics
174 if (qsp->firstPacketTime == 0)
175 qsp->firstPacketTime = curtime;
176 qsp->lastPacketTime = curtime;
177 __sync_add_and_fetch(&(qsp->pkts_total), 1);
178 if (congestion_flag) {
179 __sync_add_and_fetch(&(qsp->pkts_marked), 1);
180 __sync_add_and_fetch(&(qsp->bytes_marked), len);
183 __sync_add_and_fetch(&(qsp->pkts_dropped), 1);
184 __sync_add_and_fetch(&(qsp->bytes_dropped),
188 __sync_add_and_fetch(&(qsp->pkts_ecn_ce), 1);
190 __sync_add_and_fetch(&(qsp->sum_cwnd),
192 __sync_add_and_fetch(&(qsp->sum_cwnd_cnt), 1);
195 __sync_add_and_fetch(&(qsp->sum_rtt),
197 __sync_add_and_fetch(&(qsp->sum_credit), credit);
204 __sync_add_and_fetch(&(qsp->returnValCount[0]),
206 else if (rv == ALLOW_PKT)
207 __sync_add_and_fetch(&(qsp->returnValCount[1]),
210 __sync_add_and_fetch(&(qsp->returnValCount[2]),
213 __sync_add_and_fetch(&(qsp->returnValCount[3]),