2 * Copyright (C) 2016 Netronome Systems, Inc.
4 * This software is dual licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this
6 * source tree or the BSD 2-Clause License provided below. You have the
7 * option to license this software under the complete terms of either license.
9 * The BSD 2-Clause License:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * 1. Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * 2. Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 * Netronome network device driver: TC offload functions for PF and VF
39 #include <linux/kernel.h>
40 #include <linux/netdevice.h>
41 #include <linux/pci.h>
42 #include <linux/jiffies.h>
43 #include <linux/timer.h>
44 #include <linux/list.h>
46 #include <net/pkt_cls.h>
47 #include <net/tc_act/tc_gact.h>
48 #include <net/tc_act/tc_mirred.h>
51 #include "../nfp_net_ctrl.h"
52 #include "../nfp_net.h"
54 void nfp_net_filter_stats_timer(unsigned long data)
56 struct nfp_net *nn = (void *)data;
57 struct nfp_net_bpf_priv *priv;
58 struct nfp_stat_pair latest;
62 spin_lock_bh(&priv->rx_filter_lock);
64 if (nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
65 mod_timer(&priv->rx_filter_stats_timer,
66 jiffies + NFP_NET_STAT_POLL_IVL);
68 spin_unlock_bh(&priv->rx_filter_lock);
70 latest.pkts = nn_readq(nn, NFP_NET_CFG_STATS_APP1_FRAMES);
71 latest.bytes = nn_readq(nn, NFP_NET_CFG_STATS_APP1_BYTES);
73 if (latest.pkts != priv->rx_filter.pkts)
74 priv->rx_filter_change = jiffies;
76 priv->rx_filter = latest;
79 static void nfp_net_bpf_stats_reset(struct nfp_net *nn)
81 struct nfp_net_bpf_priv *priv = nn->app_priv;
83 priv->rx_filter.pkts = nn_readq(nn, NFP_NET_CFG_STATS_APP1_FRAMES);
84 priv->rx_filter.bytes = nn_readq(nn, NFP_NET_CFG_STATS_APP1_BYTES);
85 priv->rx_filter_prev = priv->rx_filter;
86 priv->rx_filter_change = jiffies;
90 nfp_net_bpf_stats_update(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
92 struct nfp_net_bpf_priv *priv = nn->app_priv;
95 pkts = priv->rx_filter.pkts - priv->rx_filter_prev.pkts;
96 bytes = priv->rx_filter.bytes - priv->rx_filter_prev.bytes;
97 bytes -= pkts * ETH_HLEN;
99 priv->rx_filter_prev = priv->rx_filter;
101 tcf_exts_stats_update(cls_bpf->exts,
102 bytes, pkts, priv->rx_filter_change);
108 nfp_net_bpf_get_act(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
110 const struct tc_action *a;
116 /* TC direct action */
117 if (cls_bpf->exts_integrated) {
118 if (!tcf_exts_has_actions(cls_bpf->exts))
119 return NN_ACT_DIRECT;
125 if (!tcf_exts_has_one_action(cls_bpf->exts))
128 tcf_exts_to_list(cls_bpf->exts, &actions);
129 list_for_each_entry(a, &actions, list) {
130 if (is_tcf_gact_shot(a))
131 return NN_ACT_TC_DROP;
133 if (is_tcf_mirred_egress_redirect(a) &&
134 tcf_mirred_ifindex(a) == nn->dp.netdev->ifindex)
135 return NN_ACT_TC_REDIR;
142 nfp_net_bpf_offload_prepare(struct nfp_net *nn,
143 struct tc_cls_bpf_offload *cls_bpf,
144 struct nfp_bpf_result *res,
145 void **code, dma_addr_t *dma_addr, u16 max_instr)
147 unsigned int code_sz = max_instr * sizeof(u64);
148 enum nfp_bpf_action_type act;
149 u16 start_off, done_off;
150 unsigned int max_mtu;
153 if (!IS_ENABLED(CONFIG_BPF_SYSCALL))
156 ret = nfp_net_bpf_get_act(nn, cls_bpf);
161 max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
162 if (max_mtu < nn->dp.netdev->mtu) {
163 nn_info(nn, "BPF offload not supported with MTU larger than HW packet split boundary\n");
167 start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
168 done_off = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
170 *code = dma_zalloc_coherent(nn->dp.dev, code_sz, dma_addr, GFP_KERNEL);
174 ret = nfp_bpf_jit(cls_bpf->prog, *code, act, start_off, done_off,
182 dma_free_coherent(nn->dp.dev, code_sz, *code, *dma_addr);
187 nfp_net_bpf_load_and_start(struct nfp_net *nn, u32 tc_flags,
188 void *code, dma_addr_t dma_addr,
189 unsigned int code_sz, unsigned int n_instr,
192 struct nfp_net_bpf_priv *priv = nn->app_priv;
193 u64 bpf_addr = dma_addr;
196 nn->dp.bpf_offload_skip_sw = !!(tc_flags & TCA_CLS_FLAGS_SKIP_SW);
199 bpf_addr |= NFP_NET_CFG_BPF_CFG_8CTX;
201 nn_writew(nn, NFP_NET_CFG_BPF_SIZE, n_instr);
202 nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, bpf_addr);
204 /* Load up the JITed code */
205 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
207 nn_err(nn, "FW command error while loading BPF: %d\n", err);
209 /* Enable passing packets through BPF function */
210 nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
211 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
212 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
214 nn_err(nn, "FW command error while enabling BPF: %d\n", err);
216 dma_free_coherent(nn->dp.dev, code_sz, code, dma_addr);
218 nfp_net_bpf_stats_reset(nn);
219 mod_timer(&priv->rx_filter_stats_timer,
220 jiffies + NFP_NET_STAT_POLL_IVL);
223 static int nfp_net_bpf_stop(struct nfp_net *nn)
225 struct nfp_net_bpf_priv *priv = nn->app_priv;
227 if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
230 spin_lock_bh(&priv->rx_filter_lock);
231 nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
232 spin_unlock_bh(&priv->rx_filter_lock);
233 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
235 del_timer_sync(&priv->rx_filter_stats_timer);
236 nn->dp.bpf_offload_skip_sw = 0;
238 return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
241 int nfp_net_bpf_offload(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
243 struct nfp_bpf_result res;
249 max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
251 switch (cls_bpf->command) {
252 case TC_CLSBPF_REPLACE:
253 /* There is nothing stopping us from implementing seamless
254 * replace but the simple method of loading I adopted in
255 * the firmware does not handle atomic replace (i.e. we have to
256 * stop the BPF offload and re-enable it). Leaking-in a few
257 * frames which didn't have BPF applied in the hardware should
258 * be fine if software fallback is available, though.
260 if (nn->dp.bpf_offload_skip_sw)
263 err = nfp_net_bpf_offload_prepare(nn, cls_bpf, &res, &code,
264 &dma_addr, max_instr);
268 nfp_net_bpf_stop(nn);
269 nfp_net_bpf_load_and_start(nn, cls_bpf->gen_flags, code,
270 dma_addr, max_instr * sizeof(u64),
271 res.n_instr, res.dense_mode);
275 if (nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
278 err = nfp_net_bpf_offload_prepare(nn, cls_bpf, &res, &code,
279 &dma_addr, max_instr);
283 nfp_net_bpf_load_and_start(nn, cls_bpf->gen_flags, code,
284 dma_addr, max_instr * sizeof(u64),
285 res.n_instr, res.dense_mode);
288 case TC_CLSBPF_DESTROY:
289 return nfp_net_bpf_stop(nn);
291 case TC_CLSBPF_STATS:
292 return nfp_net_bpf_stats_update(nn, cls_bpf);