drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c

   1 /* Broadcom NetXtreme-C/E network driver.
   2  *
   3  * Copyright (c) 2017 Broadcom Limited
   4  *
   5  * This program is free software; you can redistribute it and/or modify
   6  * it under the terms of the GNU General Public License as published by
   7  * the Free Software Foundation.
   8  */
   9
  10 #include <linux/netdevice.h>
  11 #include <linux/inetdevice.h>
  12 #include <linux/if_vlan.h>
  13 #include <net/flow_dissector.h>
  14 #include <net/pkt_cls.h>
  15 #include <net/tc_act/tc_gact.h>
  16 #include <net/tc_act/tc_skbedit.h>
  17 #include <net/tc_act/tc_mirred.h>
  18 #include <net/tc_act/tc_vlan.h>
  19 #include <net/tc_act/tc_tunnel_key.h>
  20
  21 #include "bnxt_hsi.h"
  22 #include "bnxt.h"
  23 #include "bnxt_sriov.h"
  24 #include "bnxt_tc.h"
  25 #include "bnxt_vfr.h"
  26
  27 #define BNXT_FID_INVALID                        0xffff
  28 #define VLAN_TCI(vid, prio)     ((vid) | ((prio) << VLAN_PRIO_SHIFT))
  29
  30 #define is_vlan_pcp_wildcarded(vlan_tci_mask)   \
  31         ((ntohs(vlan_tci_mask) & VLAN_PRIO_MASK) == 0x0000)
  32 #define is_vlan_pcp_exactmatch(vlan_tci_mask)   \
  33         ((ntohs(vlan_tci_mask) & VLAN_PRIO_MASK) == VLAN_PRIO_MASK)
  34 #define is_vlan_pcp_zero(vlan_tci)      \
  35         ((ntohs(vlan_tci) & VLAN_PRIO_MASK) == 0x0000)
  36 #define is_vid_exactmatch(vlan_tci_mask)        \
  37         ((ntohs(vlan_tci_mask) & VLAN_VID_MASK) == VLAN_VID_MASK)
  38
  39 /* Return the dst fid of the func for flow forwarding
  40  * For PFs: src_fid is the fid of the PF
  41  * For VF-reps: src_fid the fid of the VF
  42  */
  43 static u16 bnxt_flow_get_dst_fid(struct bnxt *pf_bp, struct net_device *dev)
  44 {
  45         struct bnxt *bp;
  46
  47         /* check if dev belongs to the same switch */
  48         if (!switchdev_port_same_parent_id(pf_bp->dev, dev)) {
  49                 netdev_info(pf_bp->dev, "dev(ifindex=%d) not on same switch",
  50                             dev->ifindex);
  51                 return BNXT_FID_INVALID;
  52         }
  53
  54         /* Is dev a VF-rep? */
  55         if (bnxt_dev_is_vf_rep(dev))
  56                 return bnxt_vf_rep_get_fid(dev);
  57
  58         bp = netdev_priv(dev);
  59         return bp->pf.fw_fid;
  60 }
  61
  62 static int bnxt_tc_parse_redir(struct bnxt *bp,
  63                                struct bnxt_tc_actions *actions,
  64                                const struct tc_action *tc_act)
  65 {
  66         struct net_device *dev = tcf_mirred_dev(tc_act);
  67
  68         if (!dev) {
  69                 netdev_info(bp->dev, "no dev in mirred action");
  70                 return -EINVAL;
  71         }
  72
  73         actions->flags |= BNXT_TC_ACTION_FLAG_FWD;
  74         actions->dst_dev = dev;
  75         return 0;
  76 }
  77
  78 static int bnxt_tc_parse_vlan(struct bnxt *bp,
  79                               struct bnxt_tc_actions *actions,
  80                               const struct tc_action *tc_act)
  81 {
  82         switch (tcf_vlan_action(tc_act)) {
  83         case TCA_VLAN_ACT_POP:
  84                 actions->flags |= BNXT_TC_ACTION_FLAG_POP_VLAN;
  85                 break;
  86         case TCA_VLAN_ACT_PUSH:
  87                 actions->flags |= BNXT_TC_ACTION_FLAG_PUSH_VLAN;
  88                 actions->push_vlan_tci = htons(tcf_vlan_push_vid(tc_act));
  89                 actions->push_vlan_tpid = tcf_vlan_push_proto(tc_act);
  90                 break;
  91         default:
  92                 return -EOPNOTSUPP;
  93         }
  94         return 0;
  95 }
  96
  97 static int bnxt_tc_parse_tunnel_set(struct bnxt *bp,
  98                                     struct bnxt_tc_actions *actions,
  99                                     const struct tc_action *tc_act)
 100 {
 101         struct ip_tunnel_info *tun_info = tcf_tunnel_info(tc_act);
 102         struct ip_tunnel_key *tun_key = &tun_info->key;
 103
 104         if (ip_tunnel_info_af(tun_info) != AF_INET) {
 105                 netdev_info(bp->dev, "only IPv4 tunnel-encap is supported");
 106                 return -EOPNOTSUPP;
 107         }
 108
 109         actions->tun_encap_key = *tun_key;
 110         actions->flags |= BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP;
 111         return 0;
 112 }
 113
 114 static int bnxt_tc_parse_actions(struct bnxt *bp,
 115                                  struct bnxt_tc_actions *actions,
 116                                  struct tcf_exts *tc_exts)
 117 {
 118         const struct tc_action *tc_act;
 119         int i, rc;
 120
 121         if (!tcf_exts_has_actions(tc_exts)) {
 122                 netdev_info(bp->dev, "no actions");
 123                 return -EINVAL;
 124         }
 125
 126         tcf_exts_for_each_action(i, tc_act, tc_exts) {
 127                 /* Drop action */
 128                 if (is_tcf_gact_shot(tc_act)) {
 129                         actions->flags |= BNXT_TC_ACTION_FLAG_DROP;
 130                         return 0; /* don't bother with other actions */
 131                 }
 132
 133                 /* Redirect action */
 134                 if (is_tcf_mirred_egress_redirect(tc_act)) {
 135                         rc = bnxt_tc_parse_redir(bp, actions, tc_act);
 136                         if (rc)
 137                                 return rc;
 138                         continue;
 139                 }
 140
 141                 /* Push/pop VLAN */
 142                 if (is_tcf_vlan(tc_act)) {
 143                         rc = bnxt_tc_parse_vlan(bp, actions, tc_act);
 144                         if (rc)
 145                                 return rc;
 146                         continue;
 147                 }
 148
 149                 /* Tunnel encap */
 150                 if (is_tcf_tunnel_set(tc_act)) {
 151                         rc = bnxt_tc_parse_tunnel_set(bp, actions, tc_act);
 152                         if (rc)
 153                                 return rc;
 154                         continue;
 155                 }
 156
 157                 /* Tunnel decap */
 158                 if (is_tcf_tunnel_release(tc_act)) {
 159                         actions->flags |= BNXT_TC_ACTION_FLAG_TUNNEL_DECAP;
 160                         continue;
 161                 }
 162         }
 163
 164         if (actions->flags & BNXT_TC_ACTION_FLAG_FWD) {
 165                 if (actions->flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP) {
 166                         /* dst_fid is PF's fid */
 167                         actions->dst_fid = bp->pf.fw_fid;
 168                 } else {
 169                         /* find the FID from dst_dev */
 170                         actions->dst_fid =
 171                                 bnxt_flow_get_dst_fid(bp, actions->dst_dev);
 172                         if (actions->dst_fid == BNXT_FID_INVALID)
 173                                 return -EINVAL;
 174                 }
 175         }
 176
 177         return 0;
 178 }
 179
 180 #define GET_KEY(flow_cmd, key_type)                                     \
 181                 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\
 182                                           (flow_cmd)->key)
 183 #define GET_MASK(flow_cmd, key_type)                                    \
 184                 skb_flow_dissector_target((flow_cmd)->dissector, key_type,\
 185                                           (flow_cmd)->mask)
 186
 187 static int bnxt_tc_parse_flow(struct bnxt *bp,
 188                               struct tc_cls_flower_offload *tc_flow_cmd,
 189                               struct bnxt_tc_flow *flow)
 190 {
 191         struct flow_dissector *dissector = tc_flow_cmd->dissector;
 192         u16 addr_type = 0;
 193
 194         /* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */
 195         if ((dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_CONTROL)) == 0 ||
 196             (dissector->used_keys & BIT(FLOW_DISSECTOR_KEY_BASIC)) == 0) {
 197                 netdev_info(bp->dev, "cannot form TC key: used_keys = 0x%x",
 198                             dissector->used_keys);
 199                 return -EOPNOTSUPP;
 200         }
 201
 202         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_CONTROL)) {
 203                 struct flow_dissector_key_control *key =
 204                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_CONTROL);
 205
 206                 addr_type = key->addr_type;
 207         }
 208
 209         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_BASIC)) {
 210                 struct flow_dissector_key_basic *key =
 211                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC);
 212                 struct flow_dissector_key_basic *mask =
 213                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_BASIC);
 214
 215                 flow->l2_key.ether_type = key->n_proto;
 216                 flow->l2_mask.ether_type = mask->n_proto;
 217
 218                 if (key->n_proto == htons(ETH_P_IP) ||
 219                     key->n_proto == htons(ETH_P_IPV6)) {
 220                         flow->l4_key.ip_proto = key->ip_proto;
 221                         flow->l4_mask.ip_proto = mask->ip_proto;
 222                 }
 223         }
 224
 225         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
 226                 struct flow_dissector_key_eth_addrs *key =
 227                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS);
 228                 struct flow_dissector_key_eth_addrs *mask =
 229                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ETH_ADDRS);
 230
 231                 flow->flags |= BNXT_TC_FLOW_FLAGS_ETH_ADDRS;
 232                 ether_addr_copy(flow->l2_key.dmac, key->dst);
 233                 ether_addr_copy(flow->l2_mask.dmac, mask->dst);
 234                 ether_addr_copy(flow->l2_key.smac, key->src);
 235                 ether_addr_copy(flow->l2_mask.smac, mask->src);
 236         }
 237
 238         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_VLAN)) {
 239                 struct flow_dissector_key_vlan *key =
 240                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN);
 241                 struct flow_dissector_key_vlan *mask =
 242                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_VLAN);
 243
 244                 flow->l2_key.inner_vlan_tci =
 245                    cpu_to_be16(VLAN_TCI(key->vlan_id, key->vlan_priority));
 246                 flow->l2_mask.inner_vlan_tci =
 247                    cpu_to_be16((VLAN_TCI(mask->vlan_id, mask->vlan_priority)));
 248                 flow->l2_key.inner_vlan_tpid = htons(ETH_P_8021Q);
 249                 flow->l2_mask.inner_vlan_tpid = htons(0xffff);
 250                 flow->l2_key.num_vlans = 1;
 251         }
 252
 253         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_IPV4_ADDRS)) {
 254                 struct flow_dissector_key_ipv4_addrs *key =
 255                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
 256                 struct flow_dissector_key_ipv4_addrs *mask =
 257                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV4_ADDRS);
 258
 259                 flow->flags |= BNXT_TC_FLOW_FLAGS_IPV4_ADDRS;
 260                 flow->l3_key.ipv4.daddr.s_addr = key->dst;
 261                 flow->l3_mask.ipv4.daddr.s_addr = mask->dst;
 262                 flow->l3_key.ipv4.saddr.s_addr = key->src;
 263                 flow->l3_mask.ipv4.saddr.s_addr = mask->src;
 264         } else if (dissector_uses_key(dissector,
 265                                       FLOW_DISSECTOR_KEY_IPV6_ADDRS)) {
 266                 struct flow_dissector_key_ipv6_addrs *key =
 267                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
 268                 struct flow_dissector_key_ipv6_addrs *mask =
 269                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_IPV6_ADDRS);
 270
 271                 flow->flags |= BNXT_TC_FLOW_FLAGS_IPV6_ADDRS;
 272                 flow->l3_key.ipv6.daddr = key->dst;
 273                 flow->l3_mask.ipv6.daddr = mask->dst;
 274                 flow->l3_key.ipv6.saddr = key->src;
 275                 flow->l3_mask.ipv6.saddr = mask->src;
 276         }
 277
 278         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_PORTS)) {
 279                 struct flow_dissector_key_ports *key =
 280                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS);
 281                 struct flow_dissector_key_ports *mask =
 282                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_PORTS);
 283
 284                 flow->flags |= BNXT_TC_FLOW_FLAGS_PORTS;
 285                 flow->l4_key.ports.dport = key->dst;
 286                 flow->l4_mask.ports.dport = mask->dst;
 287                 flow->l4_key.ports.sport = key->src;
 288                 flow->l4_mask.ports.sport = mask->src;
 289         }
 290
 291         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ICMP)) {
 292                 struct flow_dissector_key_icmp *key =
 293                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP);
 294                 struct flow_dissector_key_icmp *mask =
 295                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ICMP);
 296
 297                 flow->flags |= BNXT_TC_FLOW_FLAGS_ICMP;
 298                 flow->l4_key.icmp.type = key->type;
 299                 flow->l4_key.icmp.code = key->code;
 300                 flow->l4_mask.icmp.type = mask->type;
 301                 flow->l4_mask.icmp.code = mask->code;
 302         }
 303
 304         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
 305                 struct flow_dissector_key_control *key =
 306                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_CONTROL);
 307
 308                 addr_type = key->addr_type;
 309         }
 310
 311         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS)) {
 312                 struct flow_dissector_key_ipv4_addrs *key =
 313                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS);
 314                 struct flow_dissector_key_ipv4_addrs *mask =
 315                                 GET_MASK(tc_flow_cmd,
 316                                          FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS);
 317
 318                 flow->flags |= BNXT_TC_FLOW_FLAGS_TUNL_IPV4_ADDRS;
 319                 flow->tun_key.u.ipv4.dst = key->dst;
 320                 flow->tun_mask.u.ipv4.dst = mask->dst;
 321                 flow->tun_key.u.ipv4.src = key->src;
 322                 flow->tun_mask.u.ipv4.src = mask->src;
 323         } else if (dissector_uses_key(dissector,
 324                                       FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS)) {
 325                 return -EOPNOTSUPP;
 326         }
 327
 328         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
 329                 struct flow_dissector_key_keyid *key =
 330                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_KEYID);
 331                 struct flow_dissector_key_keyid *mask =
 332                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_KEYID);
 333
 334                 flow->flags |= BNXT_TC_FLOW_FLAGS_TUNL_ID;
 335                 flow->tun_key.tun_id = key32_to_tunnel_id(key->keyid);
 336                 flow->tun_mask.tun_id = key32_to_tunnel_id(mask->keyid);
 337         }
 338
 339         if (dissector_uses_key(dissector, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
 340                 struct flow_dissector_key_ports *key =
 341                         GET_KEY(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_PORTS);
 342                 struct flow_dissector_key_ports *mask =
 343                         GET_MASK(tc_flow_cmd, FLOW_DISSECTOR_KEY_ENC_PORTS);
 344
 345                 flow->flags |= BNXT_TC_FLOW_FLAGS_TUNL_PORTS;
 346                 flow->tun_key.tp_dst = key->dst;
 347                 flow->tun_mask.tp_dst = mask->dst;
 348                 flow->tun_key.tp_src = key->src;
 349                 flow->tun_mask.tp_src = mask->src;
 350         }
 351
 352         return bnxt_tc_parse_actions(bp, &flow->actions, tc_flow_cmd->exts);
 353 }
 354
 355 static int bnxt_hwrm_cfa_flow_free(struct bnxt *bp, __le16 flow_handle)
 356 {
 357         struct hwrm_cfa_flow_free_input req = { 0 };
 358         int rc;
 359
 360         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_FREE, -1, -1);
 361         req.flow_handle = flow_handle;
 362
 363         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 364         if (rc)
 365                 netdev_info(bp->dev, "Error: %s: flow_handle=0x%x rc=%d",
 366                             __func__, flow_handle, rc);
 367
 368         if (rc)
 369                 rc = -EIO;
 370         return rc;
 371 }
 372
 373 static int ipv6_mask_len(struct in6_addr *mask)
 374 {
 375         int mask_len = 0, i;
 376
 377         for (i = 0; i < 4; i++)
 378                 mask_len += inet_mask_len(mask->s6_addr32[i]);
 379
 380         return mask_len;
 381 }
 382
 383 static bool is_wildcard(void *mask, int len)
 384 {
 385         const u8 *p = mask;
 386         int i;
 387
 388         for (i = 0; i < len; i++) {
 389                 if (p[i] != 0)
 390                         return false;
 391         }
 392         return true;
 393 }
 394
 395 static bool is_exactmatch(void *mask, int len)
 396 {
 397         const u8 *p = mask;
 398         int i;
 399
 400         for (i = 0; i < len; i++)
 401                 if (p[i] != 0xff)
 402                         return false;
 403
 404         return true;
 405 }
 406
 407 static bool is_vlan_tci_allowed(__be16  vlan_tci_mask,
 408                                 __be16  vlan_tci)
 409 {
 410         /* VLAN priority must be either exactly zero or fully wildcarded and
 411          * VLAN id must be exact match.
 412          */
 413         if (is_vid_exactmatch(vlan_tci_mask) &&
 414             ((is_vlan_pcp_exactmatch(vlan_tci_mask) &&
 415               is_vlan_pcp_zero(vlan_tci)) ||
 416              is_vlan_pcp_wildcarded(vlan_tci_mask)))
 417                 return true;
 418
 419         return false;
 420 }
 421
 422 static bool bits_set(void *key, int len)
 423 {
 424         const u8 *p = key;
 425         int i;
 426
 427         for (i = 0; i < len; i++)
 428                 if (p[i] != 0)
 429                         return true;
 430
 431         return false;
 432 }
 433
 434 static int bnxt_hwrm_cfa_flow_alloc(struct bnxt *bp, struct bnxt_tc_flow *flow,
 435                                     __le16 ref_flow_handle,
 436                                     __le32 tunnel_handle, __le16 *flow_handle)
 437 {
 438         struct hwrm_cfa_flow_alloc_output *resp = bp->hwrm_cmd_resp_addr;
 439         struct bnxt_tc_actions *actions = &flow->actions;
 440         struct bnxt_tc_l3_key *l3_mask = &flow->l3_mask;
 441         struct bnxt_tc_l3_key *l3_key = &flow->l3_key;
 442         struct hwrm_cfa_flow_alloc_input req = { 0 };
 443         u16 flow_flags = 0, action_flags = 0;
 444         int rc;
 445
 446         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_ALLOC, -1, -1);
 447
 448         req.src_fid = cpu_to_le16(flow->src_fid);
 449         req.ref_flow_handle = ref_flow_handle;
 450
 451         if (actions->flags & BNXT_TC_ACTION_FLAG_TUNNEL_DECAP ||
 452             actions->flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP) {
 453                 req.tunnel_handle = tunnel_handle;
 454                 flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_TUNNEL;
 455                 action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_TUNNEL;
 456         }
 457
 458         req.ethertype = flow->l2_key.ether_type;
 459         req.ip_proto = flow->l4_key.ip_proto;
 460
 461         if (flow->flags & BNXT_TC_FLOW_FLAGS_ETH_ADDRS) {
 462                 memcpy(req.dmac, flow->l2_key.dmac, ETH_ALEN);
 463                 memcpy(req.smac, flow->l2_key.smac, ETH_ALEN);
 464         }
 465
 466         if (flow->l2_key.num_vlans > 0) {
 467                 flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_NUM_VLAN_ONE;
 468                 /* FW expects the inner_vlan_tci value to be set
 469                  * in outer_vlan_tci when num_vlans is 1 (which is
 470                  * always the case in TC.)
 471                  */
 472                 req.outer_vlan_tci = flow->l2_key.inner_vlan_tci;
 473         }
 474
 475         /* If all IP and L4 fields are wildcarded then this is an L2 flow */
 476         if (is_wildcard(l3_mask, sizeof(*l3_mask)) &&
 477             is_wildcard(&flow->l4_mask, sizeof(flow->l4_mask))) {
 478                 flow_flags |= CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_L2;
 479         } else {
 480                 flow_flags |= flow->l2_key.ether_type == htons(ETH_P_IP) ?
 481                                 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV4 :
 482                                 CFA_FLOW_ALLOC_REQ_FLAGS_FLOWTYPE_IPV6;
 483
 484                 if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV4_ADDRS) {
 485                         req.ip_dst[0] = l3_key->ipv4.daddr.s_addr;
 486                         req.ip_dst_mask_len =
 487                                 inet_mask_len(l3_mask->ipv4.daddr.s_addr);
 488                         req.ip_src[0] = l3_key->ipv4.saddr.s_addr;
 489                         req.ip_src_mask_len =
 490                                 inet_mask_len(l3_mask->ipv4.saddr.s_addr);
 491                 } else if (flow->flags & BNXT_TC_FLOW_FLAGS_IPV6_ADDRS) {
 492                         memcpy(req.ip_dst, l3_key->ipv6.daddr.s6_addr32,
 493                                sizeof(req.ip_dst));
 494                         req.ip_dst_mask_len =
 495                                         ipv6_mask_len(&l3_mask->ipv6.daddr);
 496                         memcpy(req.ip_src, l3_key->ipv6.saddr.s6_addr32,
 497                                sizeof(req.ip_src));
 498                         req.ip_src_mask_len =
 499                                         ipv6_mask_len(&l3_mask->ipv6.saddr);
 500                 }
 501         }
 502
 503         if (flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) {
 504                 req.l4_src_port = flow->l4_key.ports.sport;
 505                 req.l4_src_port_mask = flow->l4_mask.ports.sport;
 506                 req.l4_dst_port = flow->l4_key.ports.dport;
 507                 req.l4_dst_port_mask = flow->l4_mask.ports.dport;
 508         } else if (flow->flags & BNXT_TC_FLOW_FLAGS_ICMP) {
 509                 /* l4 ports serve as type/code when ip_proto is ICMP */
 510                 req.l4_src_port = htons(flow->l4_key.icmp.type);
 511                 req.l4_src_port_mask = htons(flow->l4_mask.icmp.type);
 512                 req.l4_dst_port = htons(flow->l4_key.icmp.code);
 513                 req.l4_dst_port_mask = htons(flow->l4_mask.icmp.code);
 514         }
 515         req.flags = cpu_to_le16(flow_flags);
 516
 517         if (actions->flags & BNXT_TC_ACTION_FLAG_DROP) {
 518                 action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_DROP;
 519         } else {
 520                 if (actions->flags & BNXT_TC_ACTION_FLAG_FWD) {
 521                         action_flags |= CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_FWD;
 522                         req.dst_fid = cpu_to_le16(actions->dst_fid);
 523                 }
 524                 if (actions->flags & BNXT_TC_ACTION_FLAG_PUSH_VLAN) {
 525                         action_flags |=
 526                             CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE;
 527                         req.l2_rewrite_vlan_tpid = actions->push_vlan_tpid;
 528                         req.l2_rewrite_vlan_tci = actions->push_vlan_tci;
 529                         memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN);
 530                         memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN);
 531                 }
 532                 if (actions->flags & BNXT_TC_ACTION_FLAG_POP_VLAN) {
 533                         action_flags |=
 534                             CFA_FLOW_ALLOC_REQ_ACTION_FLAGS_L2_HEADER_REWRITE;
 535                         /* Rewrite config with tpid = 0 implies vlan pop */
 536                         req.l2_rewrite_vlan_tpid = 0;
 537                         memcpy(&req.l2_rewrite_dmac, &req.dmac, ETH_ALEN);
 538                         memcpy(&req.l2_rewrite_smac, &req.smac, ETH_ALEN);
 539                 }
 540         }
 541         req.action_flags = cpu_to_le16(action_flags);
 542
 543         mutex_lock(&bp->hwrm_cmd_lock);
 544         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 545         if (!rc)
 546                 *flow_handle = resp->flow_handle;
 547         mutex_unlock(&bp->hwrm_cmd_lock);
 548
 549         if (rc == HWRM_ERR_CODE_RESOURCE_ALLOC_ERROR)
 550                 rc = -ENOSPC;
 551         else if (rc)
 552                 rc = -EIO;
 553         return rc;
 554 }
 555
 556 static int hwrm_cfa_decap_filter_alloc(struct bnxt *bp,
 557                                        struct bnxt_tc_flow *flow,
 558                                        struct bnxt_tc_l2_key *l2_info,
 559                                        __le32 ref_decap_handle,
 560                                        __le32 *decap_filter_handle)
 561 {
 562         struct hwrm_cfa_decap_filter_alloc_output *resp =
 563                                                 bp->hwrm_cmd_resp_addr;
 564         struct hwrm_cfa_decap_filter_alloc_input req = { 0 };
 565         struct ip_tunnel_key *tun_key = &flow->tun_key;
 566         u32 enables = 0;
 567         int rc;
 568
 569         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_DECAP_FILTER_ALLOC, -1, -1);
 570
 571         req.flags = cpu_to_le32(CFA_DECAP_FILTER_ALLOC_REQ_FLAGS_OVS_TUNNEL);
 572         enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_TYPE |
 573                    CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IP_PROTOCOL;
 574         req.tunnel_type = CFA_DECAP_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN;
 575         req.ip_protocol = CFA_DECAP_FILTER_ALLOC_REQ_IP_PROTOCOL_UDP;
 576
 577         if (flow->flags & BNXT_TC_FLOW_FLAGS_TUNL_ID) {
 578                 enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_TUNNEL_ID;
 579                 /* tunnel_id is wrongly defined in hsi defn. as __le32 */
 580                 req.tunnel_id = tunnel_id_to_key32(tun_key->tun_id);
 581         }
 582
 583         if (flow->flags & BNXT_TC_FLOW_FLAGS_TUNL_ETH_ADDRS) {
 584                 enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_MACADDR;
 585                 ether_addr_copy(req.dst_macaddr, l2_info->dmac);
 586         }
 587         if (l2_info->num_vlans) {
 588                 enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_T_IVLAN_VID;
 589                 req.t_ivlan_vid = l2_info->inner_vlan_tci;
 590         }
 591
 592         enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_ETHERTYPE;
 593         req.ethertype = htons(ETH_P_IP);
 594
 595         if (flow->flags & BNXT_TC_FLOW_FLAGS_TUNL_IPV4_ADDRS) {
 596                 enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_SRC_IPADDR |
 597                            CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_IPADDR |
 598                            CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_IPADDR_TYPE;
 599                 req.ip_addr_type = CFA_DECAP_FILTER_ALLOC_REQ_IP_ADDR_TYPE_IPV4;
 600                 req.dst_ipaddr[0] = tun_key->u.ipv4.dst;
 601                 req.src_ipaddr[0] = tun_key->u.ipv4.src;
 602         }
 603
 604         if (flow->flags & BNXT_TC_FLOW_FLAGS_TUNL_PORTS) {
 605                 enables |= CFA_DECAP_FILTER_ALLOC_REQ_ENABLES_DST_PORT;
 606                 req.dst_port = tun_key->tp_dst;
 607         }
 608
 609         /* Eventhough the decap_handle returned by hwrm_cfa_decap_filter_alloc
 610          * is defined as __le32, l2_ctxt_ref_id is defined in HSI as __le16.
 611          */
 612         req.l2_ctxt_ref_id = (__force __le16)ref_decap_handle;
 613         req.enables = cpu_to_le32(enables);
 614
 615         mutex_lock(&bp->hwrm_cmd_lock);
 616         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 617         if (!rc)
 618                 *decap_filter_handle = resp->decap_filter_id;
 619         else
 620                 netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc);
 621         mutex_unlock(&bp->hwrm_cmd_lock);
 622
 623         if (rc)
 624                 rc = -EIO;
 625         return rc;
 626 }
 627
 628 static int hwrm_cfa_decap_filter_free(struct bnxt *bp,
 629                                       __le32 decap_filter_handle)
 630 {
 631         struct hwrm_cfa_decap_filter_free_input req = { 0 };
 632         int rc;
 633
 634         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_DECAP_FILTER_FREE, -1, -1);
 635         req.decap_filter_id = decap_filter_handle;
 636
 637         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 638         if (rc)
 639                 netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc);
 640
 641         if (rc)
 642                 rc = -EIO;
 643         return rc;
 644 }
 645
 646 static int hwrm_cfa_encap_record_alloc(struct bnxt *bp,
 647                                        struct ip_tunnel_key *encap_key,
 648                                        struct bnxt_tc_l2_key *l2_info,
 649                                        __le32 *encap_record_handle)
 650 {
 651         struct hwrm_cfa_encap_record_alloc_output *resp =
 652                                                 bp->hwrm_cmd_resp_addr;
 653         struct hwrm_cfa_encap_record_alloc_input req = { 0 };
 654         struct hwrm_cfa_encap_data_vxlan *encap =
 655                         (struct hwrm_cfa_encap_data_vxlan *)&req.encap_data;
 656         struct hwrm_vxlan_ipv4_hdr *encap_ipv4 =
 657                                 (struct hwrm_vxlan_ipv4_hdr *)encap->l3;
 658         int rc;
 659
 660         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ENCAP_RECORD_ALLOC, -1, -1);
 661
 662         req.encap_type = CFA_ENCAP_RECORD_ALLOC_REQ_ENCAP_TYPE_VXLAN;
 663
 664         ether_addr_copy(encap->dst_mac_addr, l2_info->dmac);
 665         ether_addr_copy(encap->src_mac_addr, l2_info->smac);
 666         if (l2_info->num_vlans) {
 667                 encap->num_vlan_tags = l2_info->num_vlans;
 668                 encap->ovlan_tci = l2_info->inner_vlan_tci;
 669                 encap->ovlan_tpid = l2_info->inner_vlan_tpid;
 670         }
 671
 672         encap_ipv4->ver_hlen = 4 << VXLAN_IPV4_HDR_VER_HLEN_VERSION_SFT;
 673         encap_ipv4->ver_hlen |= 5 << VXLAN_IPV4_HDR_VER_HLEN_HEADER_LENGTH_SFT;
 674         encap_ipv4->ttl = encap_key->ttl;
 675
 676         encap_ipv4->dest_ip_addr = encap_key->u.ipv4.dst;
 677         encap_ipv4->src_ip_addr = encap_key->u.ipv4.src;
 678         encap_ipv4->protocol = IPPROTO_UDP;
 679
 680         encap->dst_port = encap_key->tp_dst;
 681         encap->vni = tunnel_id_to_key32(encap_key->tun_id);
 682
 683         mutex_lock(&bp->hwrm_cmd_lock);
 684         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 685         if (!rc)
 686                 *encap_record_handle = resp->encap_record_id;
 687         else
 688                 netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc);
 689         mutex_unlock(&bp->hwrm_cmd_lock);
 690
 691         if (rc)
 692                 rc = -EIO;
 693         return rc;
 694 }
 695
 696 static int hwrm_cfa_encap_record_free(struct bnxt *bp,
 697                                       __le32 encap_record_handle)
 698 {
 699         struct hwrm_cfa_encap_record_free_input req = { 0 };
 700         int rc;
 701
 702         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_ENCAP_RECORD_FREE, -1, -1);
 703         req.encap_record_id = encap_record_handle;
 704
 705         rc = hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
 706         if (rc)
 707                 netdev_info(bp->dev, "%s: Error rc=%d", __func__, rc);
 708
 709         if (rc)
 710                 rc = -EIO;
 711         return rc;
 712 }
 713
 714 static int bnxt_tc_put_l2_node(struct bnxt *bp,
 715                                struct bnxt_tc_flow_node *flow_node)
 716 {
 717         struct bnxt_tc_l2_node *l2_node = flow_node->l2_node;
 718         struct bnxt_tc_info *tc_info = bp->tc_info;
 719         int rc;
 720
 721         /* remove flow_node from the L2 shared flow list */
 722         list_del(&flow_node->l2_list_node);
 723         if (--l2_node->refcount == 0) {
 724                 rc =  rhashtable_remove_fast(&tc_info->l2_table, &l2_node->node,
 725                                              tc_info->l2_ht_params);
 726                 if (rc)
 727                         netdev_err(bp->dev,
 728                                    "Error: %s: rhashtable_remove_fast: %d",
 729                                    __func__, rc);
 730                 kfree_rcu(l2_node, rcu);
 731         }
 732         return 0;
 733 }
 734
 735 static struct bnxt_tc_l2_node *
 736 bnxt_tc_get_l2_node(struct bnxt *bp, struct rhashtable *l2_table,
 737                     struct rhashtable_params ht_params,
 738                     struct bnxt_tc_l2_key *l2_key)
 739 {
 740         struct bnxt_tc_l2_node *l2_node;
 741         int rc;
 742
 743         l2_node = rhashtable_lookup_fast(l2_table, l2_key, ht_params);
 744         if (!l2_node) {
 745                 l2_node = kzalloc(sizeof(*l2_node), GFP_KERNEL);
 746                 if (!l2_node) {
 747                         rc = -ENOMEM;
 748                         return NULL;
 749                 }
 750
 751                 l2_node->key = *l2_key;
 752                 rc = rhashtable_insert_fast(l2_table, &l2_node->node,
 753                                             ht_params);
 754                 if (rc) {
 755                         kfree_rcu(l2_node, rcu);
 756                         netdev_err(bp->dev,
 757                                    "Error: %s: rhashtable_insert_fast: %d",
 758                                    __func__, rc);
 759                         return NULL;
 760                 }
 761                 INIT_LIST_HEAD(&l2_node->common_l2_flows);
 762         }
 763         return l2_node;
 764 }
 765
 766 /* Get the ref_flow_handle for a flow by checking if there are any other
 767  * flows that share the same L2 key as this flow.
 768  */
 769 static int
 770 bnxt_tc_get_ref_flow_handle(struct bnxt *bp, struct bnxt_tc_flow *flow,
 771                             struct bnxt_tc_flow_node *flow_node,
 772                             __le16 *ref_flow_handle)
 773 {
 774         struct bnxt_tc_info *tc_info = bp->tc_info;
 775         struct bnxt_tc_flow_node *ref_flow_node;
 776         struct bnxt_tc_l2_node *l2_node;
 777
 778         l2_node = bnxt_tc_get_l2_node(bp, &tc_info->l2_table,
 779                                       tc_info->l2_ht_params,
 780                                       &flow->l2_key);
 781         if (!l2_node)
 782                 return -1;
 783
 784         /* If any other flow is using this l2_node, use it's flow_handle
 785          * as the ref_flow_handle
 786          */
 787         if (l2_node->refcount > 0) {
 788                 ref_flow_node = list_first_entry(&l2_node->common_l2_flows,
 789                                                  struct bnxt_tc_flow_node,
 790                                                  l2_list_node);
 791                 *ref_flow_handle = ref_flow_node->flow_handle;
 792         } else {
 793                 *ref_flow_handle = cpu_to_le16(0xffff);
 794         }
 795
 796         /* Insert the l2_node into the flow_node so that subsequent flows
 797          * with a matching l2 key can use the flow_handle of this flow
 798          * as their ref_flow_handle
 799          */
 800         flow_node->l2_node = l2_node;
 801         list_add(&flow_node->l2_list_node, &l2_node->common_l2_flows);
 802         l2_node->refcount++;
 803         return 0;
 804 }
 805
 806 /* After the flow parsing is done, this routine is used for checking
 807  * if there are any aspects of the flow that prevent it from being
 808  * offloaded.
 809  */
 810 static bool bnxt_tc_can_offload(struct bnxt *bp, struct bnxt_tc_flow *flow)
 811 {
 812         /* If L4 ports are specified then ip_proto must be TCP or UDP */
 813         if ((flow->flags & BNXT_TC_FLOW_FLAGS_PORTS) &&
 814             (flow->l4_key.ip_proto != IPPROTO_TCP &&
 815              flow->l4_key.ip_proto != IPPROTO_UDP)) {
 816                 netdev_info(bp->dev, "Cannot offload non-TCP/UDP (%d) ports",
 817                             flow->l4_key.ip_proto);
 818                 return false;
 819         }
 820
 821         /* Currently source/dest MAC cannot be partial wildcard  */
 822         if (bits_set(&flow->l2_key.smac, sizeof(flow->l2_key.smac)) &&
 823             !is_exactmatch(flow->l2_mask.smac, sizeof(flow->l2_mask.smac))) {
 824                 netdev_info(bp->dev, "Wildcard match unsupported for Source MAC\n");
 825                 return false;
 826         }
 827         if (bits_set(&flow->l2_key.dmac, sizeof(flow->l2_key.dmac)) &&
 828             !is_exactmatch(&flow->l2_mask.dmac, sizeof(flow->l2_mask.dmac))) {
 829                 netdev_info(bp->dev, "Wildcard match unsupported for Dest MAC\n");
 830                 return false;
 831         }
 832
 833         /* Currently VLAN fields cannot be partial wildcard */
 834         if (bits_set(&flow->l2_key.inner_vlan_tci,
 835                      sizeof(flow->l2_key.inner_vlan_tci)) &&
 836             !is_vlan_tci_allowed(flow->l2_mask.inner_vlan_tci,
 837                                  flow->l2_key.inner_vlan_tci)) {
 838                 netdev_info(bp->dev, "Unsupported VLAN TCI\n");
 839                 return false;
 840         }
 841         if (bits_set(&flow->l2_key.inner_vlan_tpid,
 842                      sizeof(flow->l2_key.inner_vlan_tpid)) &&
 843             !is_exactmatch(&flow->l2_mask.inner_vlan_tpid,
 844                            sizeof(flow->l2_mask.inner_vlan_tpid))) {
 845                 netdev_info(bp->dev, "Wildcard match unsupported for VLAN TPID\n");
 846                 return false;
 847         }
 848
 849         /* Currently Ethertype must be set */
 850         if (!is_exactmatch(&flow->l2_mask.ether_type,
 851                            sizeof(flow->l2_mask.ether_type))) {
 852                 netdev_info(bp->dev, "Wildcard match unsupported for Ethertype\n");
 853                 return false;
 854         }
 855
 856         return true;
 857 }
 858
 859 /* Returns the final refcount of the node on success
 860  * or a -ve error code on failure
 861  */
 862 static int bnxt_tc_put_tunnel_node(struct bnxt *bp,
 863                                    struct rhashtable *tunnel_table,
 864                                    struct rhashtable_params *ht_params,
 865                                    struct bnxt_tc_tunnel_node *tunnel_node)
 866 {
 867         int rc;
 868
 869         if (--tunnel_node->refcount == 0) {
 870                 rc =  rhashtable_remove_fast(tunnel_table, &tunnel_node->node,
 871                                              *ht_params);
 872                 if (rc) {
 873                         netdev_err(bp->dev, "rhashtable_remove_fast rc=%d", rc);
 874                         rc = -1;
 875                 }
 876                 kfree_rcu(tunnel_node, rcu);
 877                 return rc;
 878         } else {
 879                 return tunnel_node->refcount;
 880         }
 881 }
 882
 883 /* Get (or add) either encap or decap tunnel node from/to the supplied
 884  * hash table.
 885  */
 886 static struct bnxt_tc_tunnel_node *
 887 bnxt_tc_get_tunnel_node(struct bnxt *bp, struct rhashtable *tunnel_table,
 888                         struct rhashtable_params *ht_params,
 889                         struct ip_tunnel_key *tun_key)
 890 {
 891         struct bnxt_tc_tunnel_node *tunnel_node;
 892         int rc;
 893
 894         tunnel_node = rhashtable_lookup_fast(tunnel_table, tun_key, *ht_params);
 895         if (!tunnel_node) {
 896                 tunnel_node = kzalloc(sizeof(*tunnel_node), GFP_KERNEL);
 897                 if (!tunnel_node) {
 898                         rc = -ENOMEM;
 899                         goto err;
 900                 }
 901
 902                 tunnel_node->key = *tun_key;
 903                 tunnel_node->tunnel_handle = INVALID_TUNNEL_HANDLE;
 904                 rc = rhashtable_insert_fast(tunnel_table, &tunnel_node->node,
 905                                             *ht_params);
 906                 if (rc) {
 907                         kfree_rcu(tunnel_node, rcu);
 908                         goto err;
 909                 }
 910         }
 911         tunnel_node->refcount++;
 912         return tunnel_node;
 913 err:
 914         netdev_info(bp->dev, "error rc=%d", rc);
 915         return NULL;
 916 }
 917
 918 static int bnxt_tc_get_ref_decap_handle(struct bnxt *bp,
 919                                         struct bnxt_tc_flow *flow,
 920                                         struct bnxt_tc_l2_key *l2_key,
 921                                         struct bnxt_tc_flow_node *flow_node,
 922                                         __le32 *ref_decap_handle)
 923 {
 924         struct bnxt_tc_info *tc_info = bp->tc_info;
 925         struct bnxt_tc_flow_node *ref_flow_node;
 926         struct bnxt_tc_l2_node *decap_l2_node;
 927
 928         decap_l2_node = bnxt_tc_get_l2_node(bp, &tc_info->decap_l2_table,
 929                                             tc_info->decap_l2_ht_params,
 930                                             l2_key);
 931         if (!decap_l2_node)
 932                 return -1;
 933
 934         /* If any other flow is using this decap_l2_node, use it's decap_handle
 935          * as the ref_decap_handle
 936          */
 937         if (decap_l2_node->refcount > 0) {
 938                 ref_flow_node =
 939                         list_first_entry(&decap_l2_node->common_l2_flows,
 940                                          struct bnxt_tc_flow_node,
 941                                          decap_l2_list_node);
 942                 *ref_decap_handle = ref_flow_node->decap_node->tunnel_handle;
 943         } else {
 944                 *ref_decap_handle = INVALID_TUNNEL_HANDLE;
 945         }
 946
 947         /* Insert the l2_node into the flow_node so that subsequent flows
 948          * with a matching decap l2 key can use the decap_filter_handle of
 949          * this flow as their ref_decap_handle
 950          */
 951         flow_node->decap_l2_node = decap_l2_node;
 952         list_add(&flow_node->decap_l2_list_node,
 953                  &decap_l2_node->common_l2_flows);
 954         decap_l2_node->refcount++;
 955         return 0;
 956 }
 957
 958 static void bnxt_tc_put_decap_l2_node(struct bnxt *bp,
 959                                       struct bnxt_tc_flow_node *flow_node)
 960 {
 961         struct bnxt_tc_l2_node *decap_l2_node = flow_node->decap_l2_node;
 962         struct bnxt_tc_info *tc_info = bp->tc_info;
 963         int rc;
 964
 965         /* remove flow_node from the decap L2 sharing flow list */
 966         list_del(&flow_node->decap_l2_list_node);
 967         if (--decap_l2_node->refcount == 0) {
 968                 rc =  rhashtable_remove_fast(&tc_info->decap_l2_table,
 969                                              &decap_l2_node->node,
 970                                              tc_info->decap_l2_ht_params);
 971                 if (rc)
 972                         netdev_err(bp->dev, "rhashtable_remove_fast rc=%d", rc);
 973                 kfree_rcu(decap_l2_node, rcu);
 974         }
 975 }
 976
 977 static void bnxt_tc_put_decap_handle(struct bnxt *bp,
 978                                      struct bnxt_tc_flow_node *flow_node)
 979 {
 980         __le32 decap_handle = flow_node->decap_node->tunnel_handle;
 981         struct bnxt_tc_info *tc_info = bp->tc_info;
 982         int rc;
 983
 984         if (flow_node->decap_l2_node)
 985                 bnxt_tc_put_decap_l2_node(bp, flow_node);
 986
 987         rc = bnxt_tc_put_tunnel_node(bp, &tc_info->decap_table,
 988                                      &tc_info->decap_ht_params,
 989                                      flow_node->decap_node);
 990         if (!rc && decap_handle != INVALID_TUNNEL_HANDLE)
 991                 hwrm_cfa_decap_filter_free(bp, decap_handle);
 992 }
 993
 994 static int bnxt_tc_resolve_tunnel_hdrs(struct bnxt *bp,
 995                                        struct ip_tunnel_key *tun_key,
 996                                        struct bnxt_tc_l2_key *l2_info)
 997 {
 998 #ifdef CONFIG_INET
 999         struct net_device *real_dst_dev = bp->dev;
1000         struct flowi4 flow = { {0} };
1001         struct net_device *dst_dev;
1002         struct neighbour *nbr;
1003         struct rtable *rt;
1004         int rc;
1005
1006         flow.flowi4_proto = IPPROTO_UDP;
1007         flow.fl4_dport = tun_key->tp_dst;
1008         flow.daddr = tun_key->u.ipv4.dst;
1009
1010         rt = ip_route_output_key(dev_net(real_dst_dev), &flow);
1011         if (IS_ERR(rt)) {
1012                 netdev_info(bp->dev, "no route to %pI4b", &flow.daddr);
1013                 return -EOPNOTSUPP;
1014         }
1015
1016         /* The route must either point to the real_dst_dev or a dst_dev that
1017          * uses the real_dst_dev.
1018          */
1019         dst_dev = rt->dst.dev;
1020         if (is_vlan_dev(dst_dev)) {
1021 #if IS_ENABLED(CONFIG_VLAN_8021Q)
1022                 struct vlan_dev_priv *vlan = vlan_dev_priv(dst_dev);
1023
1024                 if (vlan->real_dev != real_dst_dev) {
1025                         netdev_info(bp->dev,
1026                                     "dst_dev(%s) doesn't use PF-if(%s)",
1027                                     netdev_name(dst_dev),
1028                                     netdev_name(real_dst_dev));
1029                         rc = -EOPNOTSUPP;
1030                         goto put_rt;
1031                 }
1032                 l2_info->inner_vlan_tci = htons(vlan->vlan_id);
1033                 l2_info->inner_vlan_tpid = vlan->vlan_proto;
1034                 l2_info->num_vlans = 1;
1035 #endif
1036         } else if (dst_dev != real_dst_dev) {
1037                 netdev_info(bp->dev,
1038                             "dst_dev(%s) for %pI4b is not PF-if(%s)",
1039                             netdev_name(dst_dev), &flow.daddr,
1040                             netdev_name(real_dst_dev));
1041                 rc = -EOPNOTSUPP;
1042                 goto put_rt;
1043         }
1044
1045         nbr = dst_neigh_lookup(&rt->dst, &flow.daddr);
1046         if (!nbr) {
1047                 netdev_info(bp->dev, "can't lookup neighbor for %pI4b",
1048                             &flow.daddr);
1049                 rc = -EOPNOTSUPP;
1050                 goto put_rt;
1051         }
1052
1053         tun_key->u.ipv4.src = flow.saddr;
1054         tun_key->ttl = ip4_dst_hoplimit(&rt->dst);
1055         neigh_ha_snapshot(l2_info->dmac, nbr, dst_dev);
1056         ether_addr_copy(l2_info->smac, dst_dev->dev_addr);
1057         neigh_release(nbr);
1058         ip_rt_put(rt);
1059
1060         return 0;
1061 put_rt:
1062         ip_rt_put(rt);
1063         return rc;
1064 #else
1065         return -EOPNOTSUPP;
1066 #endif
1067 }
1068
1069 static int bnxt_tc_get_decap_handle(struct bnxt *bp, struct bnxt_tc_flow *flow,
1070                                     struct bnxt_tc_flow_node *flow_node,
1071                                     __le32 *decap_filter_handle)
1072 {
1073         struct ip_tunnel_key *decap_key = &flow->tun_key;
1074         struct bnxt_tc_info *tc_info = bp->tc_info;
1075         struct bnxt_tc_l2_key l2_info = { {0} };
1076         struct bnxt_tc_tunnel_node *decap_node;
1077         struct ip_tunnel_key tun_key = { 0 };
1078         struct bnxt_tc_l2_key *decap_l2_info;
1079         __le32 ref_decap_handle;
1080         int rc;
1081
1082         /* Check if there's another flow using the same tunnel decap.
1083          * If not, add this tunnel to the table and resolve the other
1084          * tunnel header fileds. Ignore src_port in the tunnel_key,
1085          * since it is not required for decap filters.
1086          */
1087         decap_key->tp_src = 0;
1088         decap_node = bnxt_tc_get_tunnel_node(bp, &tc_info->decap_table,
1089                                              &tc_info->decap_ht_params,
1090                                              decap_key);
1091         if (!decap_node)
1092                 return -ENOMEM;
1093
1094         flow_node->decap_node = decap_node;
1095
1096         if (decap_node->tunnel_handle != INVALID_TUNNEL_HANDLE)
1097                 goto done;
1098
1099         /* Resolve the L2 fields for tunnel decap
1100          * Resolve the route for remote vtep (saddr) of the decap key
1101          * Find it's next-hop mac addrs
1102          */
1103         tun_key.u.ipv4.dst = flow->tun_key.u.ipv4.src;
1104         tun_key.tp_dst = flow->tun_key.tp_dst;
1105         rc = bnxt_tc_resolve_tunnel_hdrs(bp, &tun_key, &l2_info);
1106         if (rc)
1107                 goto put_decap;
1108
1109         decap_l2_info = &decap_node->l2_info;
1110         /* decap smac is wildcarded */
1111         ether_addr_copy(decap_l2_info->dmac, l2_info.smac);
1112         if (l2_info.num_vlans) {
1113                 decap_l2_info->num_vlans = l2_info.num_vlans;
1114                 decap_l2_info->inner_vlan_tpid = l2_info.inner_vlan_tpid;
1115                 decap_l2_info->inner_vlan_tci = l2_info.inner_vlan_tci;
1116         }
1117         flow->flags |= BNXT_TC_FLOW_FLAGS_TUNL_ETH_ADDRS;
1118
1119         /* For getting a decap_filter_handle we first need to check if
1120          * there are any other decap flows that share the same tunnel L2
1121          * key and if so, pass that flow's decap_filter_handle as the
1122          * ref_decap_handle for this flow.
1123          */
1124         rc = bnxt_tc_get_ref_decap_handle(bp, flow, decap_l2_info, flow_node,
1125                                           &ref_decap_handle);
1126         if (rc)
1127                 goto put_decap;
1128
1129         /* Issue the hwrm cmd to allocate a decap filter handle */
1130         rc = hwrm_cfa_decap_filter_alloc(bp, flow, decap_l2_info,
1131                                          ref_decap_handle,
1132                                          &decap_node->tunnel_handle);
1133         if (rc)
1134                 goto put_decap_l2;
1135
1136 done:
1137         *decap_filter_handle = decap_node->tunnel_handle;
1138         return 0;
1139
1140 put_decap_l2:
1141         bnxt_tc_put_decap_l2_node(bp, flow_node);
1142 put_decap:
1143         bnxt_tc_put_tunnel_node(bp, &tc_info->decap_table,
1144                                 &tc_info->decap_ht_params,
1145                                 flow_node->decap_node);
1146         return rc;
1147 }
1148
1149 static void bnxt_tc_put_encap_handle(struct bnxt *bp,
1150                                      struct bnxt_tc_tunnel_node *encap_node)
1151 {
1152         __le32 encap_handle = encap_node->tunnel_handle;
1153         struct bnxt_tc_info *tc_info = bp->tc_info;
1154         int rc;
1155
1156         rc = bnxt_tc_put_tunnel_node(bp, &tc_info->encap_table,
1157                                      &tc_info->encap_ht_params, encap_node);
1158         if (!rc && encap_handle != INVALID_TUNNEL_HANDLE)
1159                 hwrm_cfa_encap_record_free(bp, encap_handle);
1160 }
1161
1162 /* Lookup the tunnel encap table and check if there's an encap_handle
1163  * alloc'd already.
1164  * If not, query L2 info via a route lookup and issue an encap_record_alloc
1165  * cmd to FW.
1166  */
1167 static int bnxt_tc_get_encap_handle(struct bnxt *bp, struct bnxt_tc_flow *flow,
1168                                     struct bnxt_tc_flow_node *flow_node,
1169                                     __le32 *encap_handle)
1170 {
1171         struct ip_tunnel_key *encap_key = &flow->actions.tun_encap_key;
1172         struct bnxt_tc_info *tc_info = bp->tc_info;
1173         struct bnxt_tc_tunnel_node *encap_node;
1174         int rc;
1175
1176         /* Check if there's another flow using the same tunnel encap.
1177          * If not, add this tunnel to the table and resolve the other
1178          * tunnel header fileds
1179          */
1180         encap_node = bnxt_tc_get_tunnel_node(bp, &tc_info->encap_table,
1181                                              &tc_info->encap_ht_params,
1182                                              encap_key);
1183         if (!encap_node)
1184                 return -ENOMEM;
1185
1186         flow_node->encap_node = encap_node;
1187
1188         if (encap_node->tunnel_handle != INVALID_TUNNEL_HANDLE)
1189                 goto done;
1190
1191         rc = bnxt_tc_resolve_tunnel_hdrs(bp, encap_key, &encap_node->l2_info);
1192         if (rc)
1193                 goto put_encap;
1194
1195         /* Allocate a new tunnel encap record */
1196         rc = hwrm_cfa_encap_record_alloc(bp, encap_key, &encap_node->l2_info,
1197                                          &encap_node->tunnel_handle);
1198         if (rc)
1199                 goto put_encap;
1200
1201 done:
1202         *encap_handle = encap_node->tunnel_handle;
1203         return 0;
1204
1205 put_encap:
1206         bnxt_tc_put_tunnel_node(bp, &tc_info->encap_table,
1207                                 &tc_info->encap_ht_params, encap_node);
1208         return rc;
1209 }
1210
1211 static void bnxt_tc_put_tunnel_handle(struct bnxt *bp,
1212                                       struct bnxt_tc_flow *flow,
1213                                       struct bnxt_tc_flow_node *flow_node)
1214 {
1215         if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_DECAP)
1216                 bnxt_tc_put_decap_handle(bp, flow_node);
1217         else if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP)
1218                 bnxt_tc_put_encap_handle(bp, flow_node->encap_node);
1219 }
1220
1221 static int bnxt_tc_get_tunnel_handle(struct bnxt *bp,
1222                                      struct bnxt_tc_flow *flow,
1223                                      struct bnxt_tc_flow_node *flow_node,
1224                                      __le32 *tunnel_handle)
1225 {
1226         if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_DECAP)
1227                 return bnxt_tc_get_decap_handle(bp, flow, flow_node,
1228                                                 tunnel_handle);
1229         else if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_ENCAP)
1230                 return bnxt_tc_get_encap_handle(bp, flow, flow_node,
1231                                                 tunnel_handle);
1232         else
1233                 return 0;
1234 }
1235 static int __bnxt_tc_del_flow(struct bnxt *bp,
1236                               struct bnxt_tc_flow_node *flow_node)
1237 {
1238         struct bnxt_tc_info *tc_info = bp->tc_info;
1239         int rc;
1240
1241         /* send HWRM cmd to free the flow-id */
1242         bnxt_hwrm_cfa_flow_free(bp, flow_node->flow_handle);
1243
1244         mutex_lock(&tc_info->lock);
1245
1246         /* release references to any tunnel encap/decap nodes */
1247         bnxt_tc_put_tunnel_handle(bp, &flow_node->flow, flow_node);
1248
1249         /* release reference to l2 node */
1250         bnxt_tc_put_l2_node(bp, flow_node);
1251
1252         mutex_unlock(&tc_info->lock);
1253
1254         rc = rhashtable_remove_fast(&tc_info->flow_table, &flow_node->node,
1255                                     tc_info->flow_ht_params);
1256         if (rc)
1257                 netdev_err(bp->dev, "Error: %s: rhashtable_remove_fast rc=%d",
1258                            __func__, rc);
1259
1260         kfree_rcu(flow_node, rcu);
1261         return 0;
1262 }
1263
1264 static void bnxt_tc_set_src_fid(struct bnxt *bp, struct bnxt_tc_flow *flow,
1265                                 u16 src_fid)
1266 {
1267         if (flow->actions.flags & BNXT_TC_ACTION_FLAG_TUNNEL_DECAP)
1268                 flow->src_fid = bp->pf.fw_fid;
1269         else
1270                 flow->src_fid = src_fid;
1271 }
1272
1273 /* Add a new flow or replace an existing flow.
1274  * Notes on locking:
1275  * There are essentially two critical sections here.
1276  * 1. while adding a new flow
1277  *    a) lookup l2-key
1278  *    b) issue HWRM cmd and get flow_handle
1279  *    c) link l2-key with flow
1280  * 2. while deleting a flow
1281  *    a) unlinking l2-key from flow
1282  * A lock is needed to protect these two critical sections.
1283  *
1284  * The hash-tables are already protected by the rhashtable API.
1285  */
1286 static int bnxt_tc_add_flow(struct bnxt *bp, u16 src_fid,
1287                             struct tc_cls_flower_offload *tc_flow_cmd)
1288 {
1289         struct bnxt_tc_flow_node *new_node, *old_node;
1290         struct bnxt_tc_info *tc_info = bp->tc_info;
1291         struct bnxt_tc_flow *flow;
1292         __le32 tunnel_handle = 0;
1293         __le16 ref_flow_handle;
1294         int rc;
1295
1296         /* allocate memory for the new flow and it's node */
1297         new_node = kzalloc(sizeof(*new_node), GFP_KERNEL);
1298         if (!new_node) {
1299                 rc = -ENOMEM;
1300                 goto done;
1301         }
1302         new_node->cookie = tc_flow_cmd->cookie;
1303         flow = &new_node->flow;
1304
1305         rc = bnxt_tc_parse_flow(bp, tc_flow_cmd, flow);
1306         if (rc)
1307                 goto free_node;
1308
1309         bnxt_tc_set_src_fid(bp, flow, src_fid);
1310
1311         if (!bnxt_tc_can_offload(bp, flow)) {
1312                 rc = -ENOSPC;
1313                 goto free_node;
1314         }
1315
1316         /* If a flow exists with the same cookie, delete it */
1317         old_node = rhashtable_lookup_fast(&tc_info->flow_table,
1318                                           &tc_flow_cmd->cookie,
1319                                           tc_info->flow_ht_params);
1320         if (old_node)
1321                 __bnxt_tc_del_flow(bp, old_node);
1322
1323         /* Check if the L2 part of the flow has been offloaded already.
1324          * If so, bump up it's refcnt and get it's reference handle.
1325          */
1326         mutex_lock(&tc_info->lock);
1327         rc = bnxt_tc_get_ref_flow_handle(bp, flow, new_node, &ref_flow_handle);
1328         if (rc)
1329                 goto unlock;
1330
1331         /* If the flow involves tunnel encap/decap, get tunnel_handle */
1332         rc = bnxt_tc_get_tunnel_handle(bp, flow, new_node, &tunnel_handle);
1333         if (rc)
1334                 goto put_l2;
1335
1336         /* send HWRM cmd to alloc the flow */
1337         rc = bnxt_hwrm_cfa_flow_alloc(bp, flow, ref_flow_handle,
1338                                       tunnel_handle, &new_node->flow_handle);
1339         if (rc)
1340                 goto put_tunnel;
1341
1342         flow->lastused = jiffies;
1343         spin_lock_init(&flow->stats_lock);
1344         /* add new flow to flow-table */
1345         rc = rhashtable_insert_fast(&tc_info->flow_table, &new_node->node,
1346                                     tc_info->flow_ht_params);
1347         if (rc)
1348                 goto hwrm_flow_free;
1349
1350         mutex_unlock(&tc_info->lock);
1351         return 0;
1352
1353 hwrm_flow_free:
1354         bnxt_hwrm_cfa_flow_free(bp, new_node->flow_handle);
1355 put_tunnel:
1356         bnxt_tc_put_tunnel_handle(bp, flow, new_node);
1357 put_l2:
1358         bnxt_tc_put_l2_node(bp, new_node);
1359 unlock:
1360         mutex_unlock(&tc_info->lock);
1361 free_node:
1362         kfree_rcu(new_node, rcu);
1363 done:
1364         netdev_err(bp->dev, "Error: %s: cookie=0x%lx error=%d",
1365                    __func__, tc_flow_cmd->cookie, rc);
1366         return rc;
1367 }
1368
1369 static int bnxt_tc_del_flow(struct bnxt *bp,
1370                             struct tc_cls_flower_offload *tc_flow_cmd)
1371 {
1372         struct bnxt_tc_info *tc_info = bp->tc_info;
1373         struct bnxt_tc_flow_node *flow_node;
1374
1375         flow_node = rhashtable_lookup_fast(&tc_info->flow_table,
1376                                            &tc_flow_cmd->cookie,
1377                                            tc_info->flow_ht_params);
1378         if (!flow_node)
1379                 return -EINVAL;
1380
1381         return __bnxt_tc_del_flow(bp, flow_node);
1382 }
1383
1384 static int bnxt_tc_get_flow_stats(struct bnxt *bp,
1385                                   struct tc_cls_flower_offload *tc_flow_cmd)
1386 {
1387         struct bnxt_tc_flow_stats stats, *curr_stats, *prev_stats;
1388         struct bnxt_tc_info *tc_info = bp->tc_info;
1389         struct bnxt_tc_flow_node *flow_node;
1390         struct bnxt_tc_flow *flow;
1391         unsigned long lastused;
1392
1393         flow_node = rhashtable_lookup_fast(&tc_info->flow_table,
1394                                            &tc_flow_cmd->cookie,
1395                                            tc_info->flow_ht_params);
1396         if (!flow_node)
1397                 return -1;
1398
1399         flow = &flow_node->flow;
1400         curr_stats = &flow->stats;
1401         prev_stats = &flow->prev_stats;
1402
1403         spin_lock(&flow->stats_lock);
1404         stats.packets = curr_stats->packets - prev_stats->packets;
1405         stats.bytes = curr_stats->bytes - prev_stats->bytes;
1406         *prev_stats = *curr_stats;
1407         lastused = flow->lastused;
1408         spin_unlock(&flow->stats_lock);
1409
1410         tcf_exts_stats_update(tc_flow_cmd->exts, stats.bytes, stats.packets,
1411                               lastused);
1412         return 0;
1413 }
1414
1415 static int
1416 bnxt_hwrm_cfa_flow_stats_get(struct bnxt *bp, int num_flows,
1417                              struct bnxt_tc_stats_batch stats_batch[])
1418 {
1419         struct hwrm_cfa_flow_stats_output *resp = bp->hwrm_cmd_resp_addr;
1420         struct hwrm_cfa_flow_stats_input req = { 0 };
1421         __le16 *req_flow_handles = &req.flow_handle_0;
1422         int rc, i;
1423
1424         bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_CFA_FLOW_STATS, -1, -1);
1425         req.num_flows = cpu_to_le16(num_flows);
1426         for (i = 0; i < num_flows; i++) {
1427                 struct bnxt_tc_flow_node *flow_node = stats_batch[i].flow_node;
1428
1429                 req_flow_handles[i] = flow_node->flow_handle;
1430         }
1431
1432         mutex_lock(&bp->hwrm_cmd_lock);
1433         rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT);
1434         if (!rc) {
1435                 __le64 *resp_packets = &resp->packet_0;
1436                 __le64 *resp_bytes = &resp->byte_0;
1437
1438                 for (i = 0; i < num_flows; i++) {
1439                         stats_batch[i].hw_stats.packets =
1440                                                 le64_to_cpu(resp_packets[i]);
1441                         stats_batch[i].hw_stats.bytes =
1442                                                 le64_to_cpu(resp_bytes[i]);
1443                 }
1444         } else {
1445                 netdev_info(bp->dev, "error rc=%d", rc);
1446         }
1447         mutex_unlock(&bp->hwrm_cmd_lock);
1448
1449         if (rc)
1450                 rc = -EIO;
1451         return rc;
1452 }
1453
1454 /* Add val to accum while handling a possible wraparound
1455  * of val. Eventhough val is of type u64, its actual width
1456  * is denoted by mask and will wrap-around beyond that width.
1457  */
1458 static void accumulate_val(u64 *accum, u64 val, u64 mask)
1459 {
1460 #define low_bits(x, mask)               ((x) & (mask))
1461 #define high_bits(x, mask)              ((x) & ~(mask))
1462         bool wrapped = val < low_bits(*accum, mask);
1463
1464         *accum = high_bits(*accum, mask) + val;
1465         if (wrapped)
1466                 *accum += (mask + 1);
1467 }
1468
1469 /* The HW counters' width is much less than 64bits.
1470  * Handle possible wrap-around while updating the stat counters
1471  */
1472 static void bnxt_flow_stats_accum(struct bnxt_tc_info *tc_info,
1473                                   struct bnxt_tc_flow_stats *acc_stats,
1474                                   struct bnxt_tc_flow_stats *hw_stats)
1475 {
1476         accumulate_val(&acc_stats->bytes, hw_stats->bytes, tc_info->bytes_mask);
1477         accumulate_val(&acc_stats->packets, hw_stats->packets,
1478                        tc_info->packets_mask);
1479 }
1480
1481 static int
1482 bnxt_tc_flow_stats_batch_update(struct bnxt *bp, int num_flows,
1483                                 struct bnxt_tc_stats_batch stats_batch[])
1484 {
1485         struct bnxt_tc_info *tc_info = bp->tc_info;
1486         int rc, i;
1487
1488         rc = bnxt_hwrm_cfa_flow_stats_get(bp, num_flows, stats_batch);
1489         if (rc)
1490                 return rc;
1491
1492         for (i = 0; i < num_flows; i++) {
1493                 struct bnxt_tc_flow_node *flow_node = stats_batch[i].flow_node;
1494                 struct bnxt_tc_flow *flow = &flow_node->flow;
1495
1496                 spin_lock(&flow->stats_lock);
1497                 bnxt_flow_stats_accum(tc_info, &flow->stats,
1498                                       &stats_batch[i].hw_stats);
1499                 if (flow->stats.packets != flow->prev_stats.packets)
1500                         flow->lastused = jiffies;
1501                 spin_unlock(&flow->stats_lock);
1502         }
1503
1504         return 0;
1505 }
1506
1507 static int
1508 bnxt_tc_flow_stats_batch_prep(struct bnxt *bp,
1509                               struct bnxt_tc_stats_batch stats_batch[],
1510                               int *num_flows)
1511 {
1512         struct bnxt_tc_info *tc_info = bp->tc_info;
1513         struct rhashtable_iter *iter = &tc_info->iter;
1514         void *flow_node;
1515         int rc, i;
1516
1517         rhashtable_walk_start(iter);
1518
1519         rc = 0;
1520         for (i = 0; i < BNXT_FLOW_STATS_BATCH_MAX; i++) {
1521                 flow_node = rhashtable_walk_next(iter);
1522                 if (IS_ERR(flow_node)) {
1523                         i = 0;
1524                         if (PTR_ERR(flow_node) == -EAGAIN) {
1525                                 continue;
1526                         } else {
1527                                 rc = PTR_ERR(flow_node);
1528                                 goto done;
1529                         }
1530                 }
1531
1532                 /* No more flows */
1533                 if (!flow_node)
1534                         goto done;
1535
1536                 stats_batch[i].flow_node = flow_node;
1537         }
1538 done:
1539         rhashtable_walk_stop(iter);
1540         *num_flows = i;
1541         return rc;
1542 }
1543
1544 void bnxt_tc_flow_stats_work(struct bnxt *bp)
1545 {
1546         struct bnxt_tc_info *tc_info = bp->tc_info;
1547         int num_flows, rc;
1548
1549         num_flows = atomic_read(&tc_info->flow_table.nelems);
1550         if (!num_flows)
1551                 return;
1552
1553         rhashtable_walk_enter(&tc_info->flow_table, &tc_info->iter);
1554
1555         for (;;) {
1556                 rc = bnxt_tc_flow_stats_batch_prep(bp, tc_info->stats_batch,
1557                                                    &num_flows);
1558                 if (rc) {
1559                         if (rc == -EAGAIN)
1560                                 continue;
1561                         break;
1562                 }
1563
1564                 if (!num_flows)
1565                         break;
1566
1567                 bnxt_tc_flow_stats_batch_update(bp, num_flows,
1568                                                 tc_info->stats_batch);
1569         }
1570
1571         rhashtable_walk_exit(&tc_info->iter);
1572 }
1573
1574 int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid,
1575                          struct tc_cls_flower_offload *cls_flower)
1576 {
1577         switch (cls_flower->command) {
1578         case TC_CLSFLOWER_REPLACE:
1579                 return bnxt_tc_add_flow(bp, src_fid, cls_flower);
1580         case TC_CLSFLOWER_DESTROY:
1581                 return bnxt_tc_del_flow(bp, cls_flower);
1582         case TC_CLSFLOWER_STATS:
1583                 return bnxt_tc_get_flow_stats(bp, cls_flower);
1584         default:
1585                 return -EOPNOTSUPP;
1586         }
1587 }
1588
1589 static const struct rhashtable_params bnxt_tc_flow_ht_params = {
1590         .head_offset = offsetof(struct bnxt_tc_flow_node, node),
1591         .key_offset = offsetof(struct bnxt_tc_flow_node, cookie),
1592         .key_len = sizeof(((struct bnxt_tc_flow_node *)0)->cookie),
1593         .automatic_shrinking = true
1594 };
1595
1596 static const struct rhashtable_params bnxt_tc_l2_ht_params = {
1597         .head_offset = offsetof(struct bnxt_tc_l2_node, node),
1598         .key_offset = offsetof(struct bnxt_tc_l2_node, key),
1599         .key_len = BNXT_TC_L2_KEY_LEN,
1600         .automatic_shrinking = true
1601 };
1602
1603 static const struct rhashtable_params bnxt_tc_decap_l2_ht_params = {
1604         .head_offset = offsetof(struct bnxt_tc_l2_node, node),
1605         .key_offset = offsetof(struct bnxt_tc_l2_node, key),
1606         .key_len = BNXT_TC_L2_KEY_LEN,
1607         .automatic_shrinking = true
1608 };
1609
1610 static const struct rhashtable_params bnxt_tc_tunnel_ht_params = {
1611         .head_offset = offsetof(struct bnxt_tc_tunnel_node, node),
1612         .key_offset = offsetof(struct bnxt_tc_tunnel_node, key),
1613         .key_len = sizeof(struct ip_tunnel_key),
1614         .automatic_shrinking = true
1615 };
1616
1617 /* convert counter width in bits to a mask */
1618 #define mask(width)             ((u64)~0 >> (64 - (width)))
1619
1620 int bnxt_init_tc(struct bnxt *bp)
1621 {
1622         struct bnxt_tc_info *tc_info;
1623         int rc;
1624
1625         if (bp->hwrm_spec_code < 0x10803) {
1626                 netdev_warn(bp->dev,
1627                             "Firmware does not support TC flower offload.\n");
1628                 return -ENOTSUPP;
1629         }
1630
1631         tc_info = kzalloc(sizeof(*tc_info), GFP_KERNEL);
1632         if (!tc_info)
1633                 return -ENOMEM;
1634         mutex_init(&tc_info->lock);
1635
1636         /* Counter widths are programmed by FW */
1637         tc_info->bytes_mask = mask(36);
1638         tc_info->packets_mask = mask(28);
1639
1640         tc_info->flow_ht_params = bnxt_tc_flow_ht_params;
1641         rc = rhashtable_init(&tc_info->flow_table, &tc_info->flow_ht_params);
1642         if (rc)
1643                 goto free_tc_info;
1644
1645         tc_info->l2_ht_params = bnxt_tc_l2_ht_params;
1646         rc = rhashtable_init(&tc_info->l2_table, &tc_info->l2_ht_params);
1647         if (rc)
1648                 goto destroy_flow_table;
1649
1650         tc_info->decap_l2_ht_params = bnxt_tc_decap_l2_ht_params;
1651         rc = rhashtable_init(&tc_info->decap_l2_table,
1652                              &tc_info->decap_l2_ht_params);
1653         if (rc)
1654                 goto destroy_l2_table;
1655
1656         tc_info->decap_ht_params = bnxt_tc_tunnel_ht_params;
1657         rc = rhashtable_init(&tc_info->decap_table,
1658                              &tc_info->decap_ht_params);
1659         if (rc)
1660                 goto destroy_decap_l2_table;
1661
1662         tc_info->encap_ht_params = bnxt_tc_tunnel_ht_params;
1663         rc = rhashtable_init(&tc_info->encap_table,
1664                              &tc_info->encap_ht_params);
1665         if (rc)
1666                 goto destroy_decap_table;
1667
1668         tc_info->enabled = true;
1669         bp->dev->hw_features |= NETIF_F_HW_TC;
1670         bp->dev->features |= NETIF_F_HW_TC;
1671         bp->tc_info = tc_info;
1672         return 0;
1673
1674 destroy_decap_table:
1675         rhashtable_destroy(&tc_info->decap_table);
1676 destroy_decap_l2_table:
1677         rhashtable_destroy(&tc_info->decap_l2_table);
1678 destroy_l2_table:
1679         rhashtable_destroy(&tc_info->l2_table);
1680 destroy_flow_table:
1681         rhashtable_destroy(&tc_info->flow_table);
1682 free_tc_info:
1683         kfree(tc_info);
1684         return rc;
1685 }
1686
1687 void bnxt_shutdown_tc(struct bnxt *bp)
1688 {
1689         struct bnxt_tc_info *tc_info = bp->tc_info;
1690
1691         if (!bnxt_tc_flower_enabled(bp))
1692                 return;
1693
1694         rhashtable_destroy(&tc_info->flow_table);
1695         rhashtable_destroy(&tc_info->l2_table);
1696         rhashtable_destroy(&tc_info->decap_l2_table);
1697         rhashtable_destroy(&tc_info->decap_table);
1698         rhashtable_destroy(&tc_info->encap_table);
1699         kfree(tc_info);
1700         bp->tc_info = NULL;
1701 }