drivers/net/ethernet/netronome/nfp/bpf/offload.c

   1 /*
   2  * Copyright (C) 2016 Netronome Systems, Inc.
   3  *
   4  * This software is dual licensed under the GNU General License Version 2,
   5  * June 1991 as shown in the file COPYING in the top-level directory of this
   6  * source tree or the BSD 2-Clause License provided below.  You have the
   7  * option to license this software under the complete terms of either license.
   8  *
   9  * The BSD 2-Clause License:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      1. Redistributions of source code must retain the above
  16  *         copyright notice, this list of conditions and the following
  17  *         disclaimer.
  18  *
  19  *      2. Redistributions in binary form must reproduce the above
  20  *         copyright notice, this list of conditions and the following
  21  *         disclaimer in the documentation and/or other materials
  22  *         provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 /*
  35  * nfp_net_offload.c
  36  * Netronome network device driver: TC offload functions for PF and VF
  37  */
  38
  39 #include <linux/kernel.h>
  40 #include <linux/netdevice.h>
  41 #include <linux/pci.h>
  42 #include <linux/jiffies.h>
  43 #include <linux/timer.h>
  44 #include <linux/list.h>
  45
  46 #include <net/pkt_cls.h>
  47 #include <net/tc_act/tc_gact.h>
  48 #include <net/tc_act/tc_mirred.h>
  49
  50 #include "main.h"
  51 #include "../nfp_net_ctrl.h"
  52 #include "../nfp_net.h"
  53
  54 void nfp_net_filter_stats_timer(unsigned long data)
  55 {
  56         struct nfp_net *nn = (void *)data;
  57         struct nfp_net_bpf_priv *priv;
  58         struct nfp_stat_pair latest;
  59
  60         priv = nn->app_priv;
  61
  62         spin_lock_bh(&priv->rx_filter_lock);
  63
  64         if (nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
  65                 mod_timer(&priv->rx_filter_stats_timer,
  66                           jiffies + NFP_NET_STAT_POLL_IVL);
  67
  68         spin_unlock_bh(&priv->rx_filter_lock);
  69
  70         latest.pkts = nn_readq(nn, NFP_NET_CFG_STATS_APP1_FRAMES);
  71         latest.bytes = nn_readq(nn, NFP_NET_CFG_STATS_APP1_BYTES);
  72
  73         if (latest.pkts != priv->rx_filter.pkts)
  74                 priv->rx_filter_change = jiffies;
  75
  76         priv->rx_filter = latest;
  77 }
  78
  79 static void nfp_net_bpf_stats_reset(struct nfp_net *nn)
  80 {
  81         struct nfp_net_bpf_priv *priv = nn->app_priv;
  82
  83         priv->rx_filter.pkts = nn_readq(nn, NFP_NET_CFG_STATS_APP1_FRAMES);
  84         priv->rx_filter.bytes = nn_readq(nn, NFP_NET_CFG_STATS_APP1_BYTES);
  85         priv->rx_filter_prev = priv->rx_filter;
  86         priv->rx_filter_change = jiffies;
  87 }
  88
  89 static int
  90 nfp_net_bpf_stats_update(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
  91 {
  92         struct nfp_net_bpf_priv *priv = nn->app_priv;
  93         u64 bytes, pkts;
  94
  95         pkts = priv->rx_filter.pkts - priv->rx_filter_prev.pkts;
  96         bytes = priv->rx_filter.bytes - priv->rx_filter_prev.bytes;
  97         bytes -= pkts * ETH_HLEN;
  98
  99         priv->rx_filter_prev = priv->rx_filter;
 100
 101         tcf_exts_stats_update(cls_bpf->exts,
 102                               bytes, pkts, priv->rx_filter_change);
 103
 104         return 0;
 105 }
 106
 107 static int
 108 nfp_net_bpf_get_act(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
 109 {
 110         const struct tc_action *a;
 111         LIST_HEAD(actions);
 112
 113         if (!cls_bpf->exts)
 114                 return NN_ACT_XDP;
 115
 116         /* TC direct action */
 117         if (cls_bpf->exts_integrated) {
 118                 if (!tcf_exts_has_actions(cls_bpf->exts))
 119                         return NN_ACT_DIRECT;
 120
 121                 return -EOPNOTSUPP;
 122         }
 123
 124         /* TC legacy mode */
 125         if (!tcf_exts_has_one_action(cls_bpf->exts))
 126                 return -EOPNOTSUPP;
 127
 128         tcf_exts_to_list(cls_bpf->exts, &actions);
 129         list_for_each_entry(a, &actions, list) {
 130                 if (is_tcf_gact_shot(a))
 131                         return NN_ACT_TC_DROP;
 132
 133                 if (is_tcf_mirred_egress_redirect(a) &&
 134                     tcf_mirred_ifindex(a) == nn->dp.netdev->ifindex)
 135                         return NN_ACT_TC_REDIR;
 136         }
 137
 138         return -EOPNOTSUPP;
 139 }
 140
 141 static int
 142 nfp_net_bpf_offload_prepare(struct nfp_net *nn,
 143                             struct tc_cls_bpf_offload *cls_bpf,
 144                             struct nfp_bpf_result *res,
 145                             void **code, dma_addr_t *dma_addr, u16 max_instr)
 146 {
 147         unsigned int code_sz = max_instr * sizeof(u64);
 148         enum nfp_bpf_action_type act;
 149         u16 start_off, done_off;
 150         unsigned int max_mtu;
 151         int ret;
 152
 153         if (!IS_ENABLED(CONFIG_BPF_SYSCALL))
 154                 return -EOPNOTSUPP;
 155
 156         ret = nfp_net_bpf_get_act(nn, cls_bpf);
 157         if (ret < 0)
 158                 return ret;
 159         act = ret;
 160
 161         max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32;
 162         if (max_mtu < nn->dp.netdev->mtu) {
 163                 nn_info(nn, "BPF offload not supported with MTU larger than HW packet split boundary\n");
 164                 return -EOPNOTSUPP;
 165         }
 166
 167         start_off = nn_readw(nn, NFP_NET_CFG_BPF_START);
 168         done_off = nn_readw(nn, NFP_NET_CFG_BPF_DONE);
 169
 170         *code = dma_zalloc_coherent(nn->dp.dev, code_sz, dma_addr, GFP_KERNEL);
 171         if (!*code)
 172                 return -ENOMEM;
 173
 174         ret = nfp_bpf_jit(cls_bpf->prog, *code, act, start_off, done_off,
 175                           max_instr, res);
 176         if (ret)
 177                 goto out;
 178
 179         return 0;
 180
 181 out:
 182         dma_free_coherent(nn->dp.dev, code_sz, *code, *dma_addr);
 183         return ret;
 184 }
 185
 186 static void
 187 nfp_net_bpf_load_and_start(struct nfp_net *nn, u32 tc_flags,
 188                            void *code, dma_addr_t dma_addr,
 189                            unsigned int code_sz, unsigned int n_instr,
 190                            bool dense_mode)
 191 {
 192         struct nfp_net_bpf_priv *priv = nn->app_priv;
 193         u64 bpf_addr = dma_addr;
 194         int err;
 195
 196         nn->dp.bpf_offload_skip_sw = !!(tc_flags & TCA_CLS_FLAGS_SKIP_SW);
 197
 198         if (dense_mode)
 199                 bpf_addr |= NFP_NET_CFG_BPF_CFG_8CTX;
 200
 201         nn_writew(nn, NFP_NET_CFG_BPF_SIZE, n_instr);
 202         nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, bpf_addr);
 203
 204         /* Load up the JITed code */
 205         err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF);
 206         if (err)
 207                 nn_err(nn, "FW command error while loading BPF: %d\n", err);
 208
 209         /* Enable passing packets through BPF function */
 210         nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF;
 211         nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
 212         err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
 213         if (err)
 214                 nn_err(nn, "FW command error while enabling BPF: %d\n", err);
 215
 216         dma_free_coherent(nn->dp.dev, code_sz, code, dma_addr);
 217
 218         nfp_net_bpf_stats_reset(nn);
 219         mod_timer(&priv->rx_filter_stats_timer,
 220                   jiffies + NFP_NET_STAT_POLL_IVL);
 221 }
 222
 223 static int nfp_net_bpf_stop(struct nfp_net *nn)
 224 {
 225         struct nfp_net_bpf_priv *priv = nn->app_priv;
 226
 227         if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF))
 228                 return 0;
 229
 230         spin_lock_bh(&priv->rx_filter_lock);
 231         nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF;
 232         spin_unlock_bh(&priv->rx_filter_lock);
 233         nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl);
 234
 235         del_timer_sync(&priv->rx_filter_stats_timer);
 236         nn->dp.bpf_offload_skip_sw = 0;
 237
 238         return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN);
 239 }
 240
 241 int nfp_net_bpf_offload(struct nfp_net *nn, struct tc_cls_bpf_offload *cls_bpf)
 242 {
 243         struct nfp_bpf_result res;
 244         dma_addr_t dma_addr;
 245         u16 max_instr;
 246         void *code;
 247         int err;
 248
 249         max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN);
 250
 251         switch (cls_bpf->command) {
 252         case TC_CLSBPF_REPLACE:
 253                 /* There is nothing stopping us from implementing seamless
 254                  * replace but the simple method of loading I adopted in
 255                  * the firmware does not handle atomic replace (i.e. we have to
 256                  * stop the BPF offload and re-enable it).  Leaking-in a few
 257                  * frames which didn't have BPF applied in the hardware should
 258                  * be fine if software fallback is available, though.
 259                  */
 260                 if (nn->dp.bpf_offload_skip_sw)
 261                         return -EBUSY;
 262
 263                 err = nfp_net_bpf_offload_prepare(nn, cls_bpf, &res, &code,
 264                                                   &dma_addr, max_instr);
 265                 if (err)
 266                         return err;
 267
 268                 nfp_net_bpf_stop(nn);
 269                 nfp_net_bpf_load_and_start(nn, cls_bpf->gen_flags, code,
 270                                            dma_addr, max_instr * sizeof(u64),
 271                                            res.n_instr, res.dense_mode);
 272                 return 0;
 273
 274         case TC_CLSBPF_ADD:
 275                 if (nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)
 276                         return -EBUSY;
 277
 278                 err = nfp_net_bpf_offload_prepare(nn, cls_bpf, &res, &code,
 279                                                   &dma_addr, max_instr);
 280                 if (err)
 281                         return err;
 282
 283                 nfp_net_bpf_load_and_start(nn, cls_bpf->gen_flags, code,
 284                                            dma_addr, max_instr * sizeof(u64),
 285                                            res.n_instr, res.dense_mode);
 286                 return 0;
 287
 288         case TC_CLSBPF_DESTROY:
 289                 return nfp_net_bpf_stop(nn);
 290
 291         case TC_CLSBPF_STATS:
 292                 return nfp_net_bpf_stats_update(nn, cls_bpf);
 293
 294         default:
 295                 return -EOPNOTSUPP;
 296         }
 297 }