2 # SPDX-License-Identifier: GPL-2.0
4 # Copyright (c) 2019 Facebook
6 # This program is free software; you can redistribute it and/or
7 # modify it under the terms of version 2 of the GNU General Public
8 # License as published by the Free Software Foundation.
11 echo "Script for testing HBM (Host Bandwidth Manager) framework."
12 echo "It creates a cgroup to use for testing and load a BPF program to limit"
13 echo "egress or ingress bandwidht. It then uses iperf3 or netperf to create"
14 echo "loads. The output is the goodput in Mbps (unless -D was used)."
16 echo "USAGE: $name [out] [-b=<prog>|--bpf=<prog>] [-c=<cc>|--cc=<cc>]"
17 echo " [-D] [-d=<delay>|--delay=<delay>] [--debug] [-E] [--edt]"
18 echo " [-f=<#flows>|--flows=<#flows>] [-h] [-i=<id>|--id=<id >]"
19 echo " [-l] [-N] [--no_cn] [-p=<port>|--port=<port>] [-P]"
20 echo " [-q=<qdisc>] [-R] [-s=<server>|--server=<server]"
21 echo " [-S|--stats] -t=<time>|--time=<time>] [-w] [cubic|dctcp]"
23 echo " out egress (default)"
24 echo " -b or --bpf BPF program filename to load and attach."
25 echo " Default is hbm_out_kern.o for egress,"
26 echo " -c or -cc TCP congestion control (cubic or dctcp)"
27 echo " --debug print BPF trace buffer"
28 echo " -d or --delay add a delay in ms using netem"
29 echo " -D In addition to the goodput in Mbps, it also outputs"
30 echo " other detailed information. This information is"
31 echo " test dependent (i.e. iperf3 or netperf)."
32 echo " -E enable ECN (not required for dctcp)"
33 echo " --edt use fq's Earliest Departure Time (requires fq)"
34 echo " -f or --flows number of concurrent flows (default=1)"
35 echo " -i or --id cgroup id (an integer, default is 1)"
36 echo " -N use netperf instead of iperf3"
37 echo " --no_cn Do not return CN notifications"
38 echo " -l do not limit flows using loopback"
40 echo " -p or --port iperf3 port (default is 5201)"
41 echo " -P use an iperf3 instance for each flow"
42 echo " -q use the specified qdisc"
43 echo " -r or --rate rate in Mbps (default 1s 1Gbps)"
44 echo " -R Use TCP_RR for netperf. 1st flow has req"
45 echo " size of 10KB, rest of 1MB. Reply in all"
46 echo " cases is 1 byte."
47 echo " More detailed output for each flow can be found"
48 echo " in the files netperf.<cg>.<flow>, where <cg> is the"
49 echo " cgroup id as specified with the -i flag, and <flow>"
50 echo " is the flow id starting at 1 and increasing by 1 for"
51 echo " flow (as specified by -f)."
52 echo " -s or --server hostname of netperf server. Used to create netperf"
53 echo " test traffic between to hosts (default is within host)"
54 echo " netserver must be running on the host."
55 echo " -S or --stats whether to update hbm stats (default is yes)."
56 echo " -t or --time duration of iperf3 in seconds (default=5)"
57 echo " -w Work conserving flag. cgroup can increase its"
58 echo " bandwidth beyond the rate limit specified"
59 echo " while there is available bandwidth. Current"
60 echo " implementation assumes there is only one NIC"
61 echo " (eth0), but can be extended to support multiple"
63 echo " cubic or dctcp specify which TCP CC to use"
94 function start_hbm () {
96 echo "./hbm $dir -n $id -r $rate -t $dur $flags $dbg $prog" > hbm.out
98 ./hbm $dir -n $id -r $rate -t $dur $flags $dbg $prog >> hbm.out 2>&1 &
105 # Support for upcomming ingress rate limiting
106 #in) # support for upcoming ingress rate limiting
121 flags="$flags --no_cn"
187 echo "Unknown arg:$i"
196 if [ $debug_flag -eq 1 ] ; then
204 cg_base_dir=/sys/fs/cgroup
205 cg_dir="$cg_base_dir/cgroup-test-work-dir/hbm$id"
207 echo $$ >> $cg_dir/cgroup.procs
212 rm -f hbm.[0-9]*.$dir_name
213 if [ $ecn -ne 0 ] ; then
214 sysctl -w -q -n net.ipv4.tcp_ecn=1
217 if [ $use_netperf -eq 0 ] ; then
218 cur_cc=`sysctl -n net.ipv4.tcp_congestion_control`
219 if [ "$cc" != "x" ] ; then
220 sysctl -w -q -n net.ipv4.tcp_congestion_control=$cc
224 if [ "$netem" -ne "0" ] ; then
225 if [ "$qdisc" != "" ] ; then
226 echo "WARNING: Ignoring -q options because -d option used"
228 tc qdisc del dev lo root > /dev/null 2>&1
229 tc qdisc add dev lo root netem delay $netem\ms > /dev/null 2>&1
230 elif [ "$qdisc" != "" ] ; then
231 tc qdisc del dev eth0 root > /dev/null 2>&1
232 tc qdisc add dev eth0 root $qdisc > /dev/null 2>&1
238 if [ $use_netperf -ne 0 ] ; then
239 if [ "$server" != "" ] ; then
244 ( ping6 -i 0.2 -c $m $hn > ping.out 2>&1 ) &
246 if [ $use_netperf -ne 0 ] ; then
247 begNetserverPid=`ps ax | grep netserver | grep --invert-match "grep" | \
249 if [ "$begNetserverPid" == "" ] ; then
250 if [ "$server" == "" ] ; then
251 ( ./netserver > /dev/null 2>&1) &
256 if [ "$server" == "" ] ; then
261 if [ "$cc" == "x" ] ; then
267 while [ $flow_cnt -le $flows ] ; do
268 if [ $rr -ne 0 ] ; then
270 if [ $flow_cnt -eq 1 ] ; then
273 if [ "$dir" == "-i" ] ; then
277 ( ./netperf -H $np_server -l $dur -f m -j -t TCP_RR -- -r $reqSize,$replySize $np_cc -k P50_lATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,REMOTE_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,LOCAL_RECV_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &
279 if [ "$dir" == "-i" ] ; then
280 ( ./netperf -H $np_server -l $dur -f m -j -t TCP_RR -- -r 1,10M $np_cc -k P50_LATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,REMOTE_TRANSPORT_RETRANS,REMOTE_SEND_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &
282 ( ./netperf -H $np_server -l $dur -f m -j -t TCP_STREAM -- $np_cc -k P50_lATENCY,P90_LATENCY,LOCAL_TRANSPORT_RETRANS,LOCAL_SEND_THROUGHPUT,REQUEST_SIZE,RESPONSE_SIZE > netperf.$id.$flow_cnt ) &
285 flow_cnt=$[flow_cnt+1]
288 # sleep for duration of test (plus some buffer)
292 # force graceful termination of netperf
300 if [ $details -ne 0 ] ; then
302 echo "Details for HBM in cgroup $id"
303 if [ $do_stats -eq 1 ] ; then
304 if [ -e hbm.$id.$dir_name ] ; then
305 cat hbm.$id.$dir_name
309 while [ $flow_cnt -le $flows ] ; do
310 if [ "$dir" == "-i" ] ; then
311 r=`cat netperf.$id.$flow_cnt | grep -o "REMOTE_SEND_THROUGHPUT=[0-9]*" | grep -o "[0-9]*"`
313 r=`cat netperf.$id.$flow_cnt | grep -o "LOCAL_SEND_THROUGHPUT=[0-9]*" | grep -o "[0-9]*"`
315 echo "rate for flow $flow_cnt: $r"
317 if [ $details -ne 0 ] ; then
319 echo "Details for cgroup $id, flow $flow_cnt"
320 cat netperf.$id.$flow_cnt
322 flow_cnt=$[flow_cnt+1]
324 if [ $details -ne 0 ] ; then
326 delay=`grep "avg" ping.out | grep -o "= [0-9.]*/[0-9.]*" | grep -o "[0-9.]*$"`
327 echo "PING AVG DELAY:$delay"
328 echo "AGGREGATE_GOODPUT:$rate"
332 elif [ $multi_iperf -eq 0 ] ; then
333 (iperf3 -s -p $port -1 > /dev/null 2>&1) &
335 iperf3 -c $host -p $port -i 0 -P $flows -f m -t $dur > iperf.$id
336 rates=`grep receiver iperf.$id | grep -o "[0-9.]* Mbits" | grep -o "^[0-9]*"`
337 rate=`echo $rates | grep -o "[0-9]*$"`
339 if [ $details -ne 0 ] ; then
341 echo "Details for HBM in cgroup $id"
342 if [ $do_stats -eq 1 ] ; then
343 if [ -e hbm.$id.$dir_name ] ; then
344 cat hbm.$id.$dir_name
347 delay=`grep "avg" ping.out | grep -o "= [0-9.]*/[0-9.]*" | grep -o "[0-9.]*$"`
348 echo "PING AVG DELAY:$delay"
349 echo "AGGREGATE_GOODPUT:$rate"
355 while [ $flow_cnt -le $flows ] ; do
356 (iperf3 -s -p $port -1 > /dev/null 2>&1) &
357 ( iperf3 -c $host -p $port -i 0 -P 1 -f m -t $dur | grep receiver | grep -o "[0-9.]* Mbits" | grep -o "^[0-9]*" | grep -o "[0-9]*$" > iperf3.$id.$flow_cnt ) &
359 flow_cnt=$[flow_cnt+1]
365 if [ $details -ne 0 ] ; then
367 echo "Details for HBM in cgroup $id"
368 if [ $do_stats -eq 1 ] ; then
369 if [ -e hbm.$id.$dir_name ] ; then
370 cat hbm.$id.$dir_name
375 while [ $flow_cnt -le $flows ] ; do
376 r=`cat iperf3.$id.$flow_cnt`
377 # echo "rate for flow $flow_cnt: $r"
378 if [ $details -ne 0 ] ; then
379 echo "Rate for cgroup $id, flow $flow_cnt LOCAL_SEND_THROUGHPUT=$r"
382 flow_cnt=$[flow_cnt+1]
384 if [ $details -ne 0 ] ; then
385 delay=`grep "avg" ping.out | grep -o "= [0-9.]*/[0-9.]*" | grep -o "[0-9.]*$"`
386 echo "PING AVG DELAY:$delay"
387 echo "AGGREGATE_GOODPUT:$rate"
393 if [ $use_netperf -eq 0 ] ; then
394 sysctl -w -q -n net.ipv4.tcp_congestion_control=$cur_cc
396 if [ $ecn -ne 0 ] ; then
397 sysctl -w -q -n net.ipv4.tcp_ecn=0
399 if [ "$netem" -ne "0" ] ; then
400 tc qdisc del dev lo root > /dev/null 2>&1
402 if [ "$qdisc" != "" ] ; then
403 tc qdisc del dev eth0 root > /dev/null 2>&1
407 hbmPid=`ps ax | grep "hbm " | grep --invert-match "grep" | awk '{ print $1 }'`
408 if [ "$hbmPid" == "$hbm_pid" ] ; then
414 # Detach any BPF programs that may have lingered
415 ttx=`bpftool cgroup tree | grep hbm`
418 if [ "${x:0:36}" == "/sys/fs/cgroup/cgroup-test-work-dir/" ] ; then
421 if [ $v -eq 0 ] ; then
424 if [ $v -eq 1 ] ; then
425 type=$x ; bpftool cgroup detach $cg $type id $id
432 if [ $use_netperf -ne 0 ] ; then
433 if [ "$server" == "" ] ; then
434 if [ "$begNetserverPid" == "" ] ; then
435 netserverPid=`ps ax | grep netserver | grep --invert-match "grep" | awk '{ print $1 }'`
436 if [ "$netserverPid" != "" ] ; then