drivers/net/ethernet/sfc/tc.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /****************************************************************************
   3  * Driver for Solarflare network controllers and boards
   4  * Copyright 2019 Solarflare Communications Inc.
   5  * Copyright 2020-2022 Xilinx Inc.
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 as published
   9  * by the Free Software Foundation, incorporated herein by reference.
  10  */
  11
  12 #include <net/pkt_cls.h>
  13 #include <net/vxlan.h>
  14 #include <net/geneve.h>
  15 #include <net/tc_act/tc_ct.h>
  16 #include "tc.h"
  17 #include "tc_bindings.h"
  18 #include "tc_encap_actions.h"
  19 #include "tc_conntrack.h"
  20 #include "mae.h"
  21 #include "ef100_rep.h"
  22 #include "efx.h"
  23
  24 enum efx_encap_type efx_tc_indr_netdev_type(struct net_device *net_dev)
  25 {
  26         if (netif_is_vxlan(net_dev))
  27                 return EFX_ENCAP_TYPE_VXLAN;
  28         if (netif_is_geneve(net_dev))
  29                 return EFX_ENCAP_TYPE_GENEVE;
  30
  31         return EFX_ENCAP_TYPE_NONE;
  32 }
  33
  34 #define EFX_TC_HDR_TYPE_TTL_MASK ((u32)0xff)
  35 /* Hoplimit is stored in the most significant byte in the pedit ipv6 header action */
  36 #define EFX_TC_HDR_TYPE_HLIMIT_MASK ~((u32)0xff000000)
  37 #define EFX_EFV_PF      NULL
  38 /* Look up the representor information (efv) for a device.
  39  * May return NULL for the PF (us), or an error pointer for a device that
  40  * isn't supported as a TC offload endpoint
  41  */
  42 struct efx_rep *efx_tc_flower_lookup_efv(struct efx_nic *efx,
  43                                          struct net_device *dev)
  44 {
  45         struct efx_rep *efv;
  46
  47         if (!dev)
  48                 return ERR_PTR(-EOPNOTSUPP);
  49         /* Is it us (the PF)? */
  50         if (dev == efx->net_dev)
  51                 return EFX_EFV_PF;
  52         /* Is it an efx vfrep at all? */
  53         if (dev->netdev_ops != &efx_ef100_rep_netdev_ops)
  54                 return ERR_PTR(-EOPNOTSUPP);
  55         /* Is it ours?  We don't support TC rules that include another
  56          * EF100's netdevices (not even on another port of the same NIC).
  57          */
  58         efv = netdev_priv(dev);
  59         if (efv->parent != efx)
  60                 return ERR_PTR(-EOPNOTSUPP);
  61         return efv;
  62 }
  63
  64 /* Convert a driver-internal vport ID into an internal device (PF or VF) */
  65 static s64 efx_tc_flower_internal_mport(struct efx_nic *efx, struct efx_rep *efv)
  66 {
  67         u32 mport;
  68
  69         if (IS_ERR(efv))
  70                 return PTR_ERR(efv);
  71         if (!efv) /* device is PF (us) */
  72                 efx_mae_mport_uplink(efx, &mport);
  73         else /* device is repr */
  74                 efx_mae_mport_mport(efx, efv->mport, &mport);
  75         return mport;
  76 }
  77
  78 /* Convert a driver-internal vport ID into an external device (wire or VF) */
  79 s64 efx_tc_flower_external_mport(struct efx_nic *efx, struct efx_rep *efv)
  80 {
  81         u32 mport;
  82
  83         if (IS_ERR(efv))
  84                 return PTR_ERR(efv);
  85         if (!efv) /* device is PF (us) */
  86                 efx_mae_mport_wire(efx, &mport);
  87         else /* device is repr */
  88                 efx_mae_mport_mport(efx, efv->mport, &mport);
  89         return mport;
  90 }
  91
  92 static const struct rhashtable_params efx_tc_mac_ht_params = {
  93         .key_len        = offsetofend(struct efx_tc_mac_pedit_action, h_addr),
  94         .key_offset     = 0,
  95         .head_offset    = offsetof(struct efx_tc_mac_pedit_action, linkage),
  96 };
  97
  98 static const struct rhashtable_params efx_tc_encap_match_ht_params = {
  99         .key_len        = offsetof(struct efx_tc_encap_match, linkage),
 100         .key_offset     = 0,
 101         .head_offset    = offsetof(struct efx_tc_encap_match, linkage),
 102 };
 103
 104 static const struct rhashtable_params efx_tc_match_action_ht_params = {
 105         .key_len        = sizeof(unsigned long),
 106         .key_offset     = offsetof(struct efx_tc_flow_rule, cookie),
 107         .head_offset    = offsetof(struct efx_tc_flow_rule, linkage),
 108 };
 109
 110 static const struct rhashtable_params efx_tc_lhs_rule_ht_params = {
 111         .key_len        = sizeof(unsigned long),
 112         .key_offset     = offsetof(struct efx_tc_lhs_rule, cookie),
 113         .head_offset    = offsetof(struct efx_tc_lhs_rule, linkage),
 114 };
 115
 116 static const struct rhashtable_params efx_tc_recirc_ht_params = {
 117         .key_len        = offsetof(struct efx_tc_recirc_id, linkage),
 118         .key_offset     = 0,
 119         .head_offset    = offsetof(struct efx_tc_recirc_id, linkage),
 120 };
 121
 122 static struct efx_tc_mac_pedit_action *efx_tc_flower_get_mac(struct efx_nic *efx,
 123                                                              unsigned char h_addr[ETH_ALEN],
 124                                                              struct netlink_ext_ack *extack)
 125 {
 126         struct efx_tc_mac_pedit_action *ped, *old;
 127         int rc;
 128
 129         ped = kzalloc(sizeof(*ped), GFP_USER);
 130         if (!ped)
 131                 return ERR_PTR(-ENOMEM);
 132         memcpy(ped->h_addr, h_addr, ETH_ALEN);
 133         old = rhashtable_lookup_get_insert_fast(&efx->tc->mac_ht,
 134                                                 &ped->linkage,
 135                                                 efx_tc_mac_ht_params);
 136         if (old) {
 137                 /* don't need our new entry */
 138                 kfree(ped);
 139                 if (IS_ERR(old)) /* oh dear, it's actually an error */
 140                         return ERR_CAST(old);
 141                 if (!refcount_inc_not_zero(&old->ref))
 142                         return ERR_PTR(-EAGAIN);
 143                 /* existing entry found, ref taken */
 144                 return old;
 145         }
 146
 147         rc = efx_mae_allocate_pedit_mac(efx, ped);
 148         if (rc < 0) {
 149                 NL_SET_ERR_MSG_MOD(extack, "Failed to store pedit MAC address in hw");
 150                 goto out_remove;
 151         }
 152
 153         /* ref and return */
 154         refcount_set(&ped->ref, 1);
 155         return ped;
 156 out_remove:
 157         rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage,
 158                                efx_tc_mac_ht_params);
 159         kfree(ped);
 160         return ERR_PTR(rc);
 161 }
 162
 163 static void efx_tc_flower_put_mac(struct efx_nic *efx,
 164                                   struct efx_tc_mac_pedit_action *ped)
 165 {
 166         if (!refcount_dec_and_test(&ped->ref))
 167                 return; /* still in use */
 168         rhashtable_remove_fast(&efx->tc->mac_ht, &ped->linkage,
 169                                efx_tc_mac_ht_params);
 170         efx_mae_free_pedit_mac(efx, ped);
 171         kfree(ped);
 172 }
 173
 174 static void efx_tc_free_action_set(struct efx_nic *efx,
 175                                    struct efx_tc_action_set *act, bool in_hw)
 176 {
 177         /* Failure paths calling this on the 'cursor' action set in_hw=false,
 178          * because if the alloc had succeeded we'd've put it in acts.list and
 179          * not still have it in act.
 180          */
 181         if (in_hw) {
 182                 efx_mae_free_action_set(efx, act->fw_id);
 183                 /* in_hw is true iff we are on an acts.list; make sure to
 184                  * remove ourselves from that list before we are freed.
 185                  */
 186                 list_del(&act->list);
 187         }
 188         if (act->count) {
 189                 spin_lock_bh(&act->count->cnt->lock);
 190                 if (!list_empty(&act->count_user))
 191                         list_del(&act->count_user);
 192                 spin_unlock_bh(&act->count->cnt->lock);
 193                 efx_tc_flower_put_counter_index(efx, act->count);
 194         }
 195         if (act->encap_md) {
 196                 list_del(&act->encap_user);
 197                 efx_tc_flower_release_encap_md(efx, act->encap_md);
 198         }
 199         if (act->src_mac)
 200                 efx_tc_flower_put_mac(efx, act->src_mac);
 201         if (act->dst_mac)
 202                 efx_tc_flower_put_mac(efx, act->dst_mac);
 203         kfree(act);
 204 }
 205
 206 static void efx_tc_free_action_set_list(struct efx_nic *efx,
 207                                         struct efx_tc_action_set_list *acts,
 208                                         bool in_hw)
 209 {
 210         struct efx_tc_action_set *act, *next;
 211
 212         /* Failure paths set in_hw=false, because usually the acts didn't get
 213          * to efx_mae_alloc_action_set_list(); if they did, the failure tree
 214          * has a separate efx_mae_free_action_set_list() before calling us.
 215          */
 216         if (in_hw)
 217                 efx_mae_free_action_set_list(efx, acts);
 218         /* Any act that's on the list will be in_hw even if the list isn't */
 219         list_for_each_entry_safe(act, next, &acts->list, list)
 220                 efx_tc_free_action_set(efx, act, true);
 221         /* Don't kfree, as acts is embedded inside a struct efx_tc_flow_rule */
 222 }
 223
 224 /* Boilerplate for the simple 'copy a field' cases */
 225 #define _MAP_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field)       \
 226 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_##_name)) {            \
 227         struct flow_match_##_type fm;                                   \
 228                                                                         \
 229         flow_rule_match_##_tcget(rule, &fm);                            \
 230         match->value._field = fm.key->_tcfield;                         \
 231         match->mask._field = fm.mask->_tcfield;                         \
 232 }
 233 #define MAP_KEY_AND_MASK(_name, _type, _tcfield, _field)        \
 234         _MAP_KEY_AND_MASK(_name, _type, _type, _tcfield, _field)
 235 #define MAP_ENC_KEY_AND_MASK(_name, _type, _tcget, _tcfield, _field)    \
 236         _MAP_KEY_AND_MASK(ENC_##_name, _type, _tcget, _tcfield, _field)
 237
 238 static int efx_tc_flower_parse_match(struct efx_nic *efx,
 239                                      struct flow_rule *rule,
 240                                      struct efx_tc_match *match,
 241                                      struct netlink_ext_ack *extack)
 242 {
 243         struct flow_dissector *dissector = rule->match.dissector;
 244         unsigned char ipv = 0;
 245
 246         /* Owing to internal TC infelicities, the IPV6_ADDRS key might be set
 247          * even on IPv4 filters; so rather than relying on dissector->used_keys
 248          * we check the addr_type in the CONTROL key.  If we don't find it (or
 249          * it's masked, which should never happen), we treat both IPV4_ADDRS
 250          * and IPV6_ADDRS as absent.
 251          */
 252         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
 253                 struct flow_match_control fm;
 254
 255                 flow_rule_match_control(rule, &fm);
 256                 if (IS_ALL_ONES(fm.mask->addr_type))
 257                         switch (fm.key->addr_type) {
 258                         case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
 259                                 ipv = 4;
 260                                 break;
 261                         case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
 262                                 ipv = 6;
 263                                 break;
 264                         default:
 265                                 break;
 266                         }
 267
 268                 if (fm.mask->flags & FLOW_DIS_IS_FRAGMENT) {
 269                         match->value.ip_frag = fm.key->flags & FLOW_DIS_IS_FRAGMENT;
 270                         match->mask.ip_frag = true;
 271                 }
 272                 if (fm.mask->flags & FLOW_DIS_FIRST_FRAG) {
 273                         match->value.ip_firstfrag = fm.key->flags & FLOW_DIS_FIRST_FRAG;
 274                         match->mask.ip_firstfrag = true;
 275                 }
 276                 if (fm.mask->flags & ~(FLOW_DIS_IS_FRAGMENT | FLOW_DIS_FIRST_FRAG)) {
 277                         NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on control.flags %#x",
 278                                                fm.mask->flags);
 279                         return -EOPNOTSUPP;
 280                 }
 281         }
 282         if (dissector->used_keys &
 283             ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
 284               BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
 285               BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
 286               BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
 287               BIT_ULL(FLOW_DISSECTOR_KEY_CVLAN) |
 288               BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
 289               BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
 290               BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
 291               BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |
 292               BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
 293               BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
 294               BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) |
 295               BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS) |
 296               BIT_ULL(FLOW_DISSECTOR_KEY_ENC_CONTROL) |
 297               BIT_ULL(FLOW_DISSECTOR_KEY_CT) |
 298               BIT_ULL(FLOW_DISSECTOR_KEY_TCP) |
 299               BIT_ULL(FLOW_DISSECTOR_KEY_IP))) {
 300                 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported flower keys %#llx",
 301                                        dissector->used_keys);
 302                 return -EOPNOTSUPP;
 303         }
 304
 305         MAP_KEY_AND_MASK(BASIC, basic, n_proto, eth_proto);
 306         /* Make sure we're IP if any L3/L4 keys used. */
 307         if (!IS_ALL_ONES(match->mask.eth_proto) ||
 308             !(match->value.eth_proto == htons(ETH_P_IP) ||
 309               match->value.eth_proto == htons(ETH_P_IPV6)))
 310                 if (dissector->used_keys &
 311                     (BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
 312                      BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
 313                      BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
 314                      BIT_ULL(FLOW_DISSECTOR_KEY_IP) |
 315                      BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) {
 316                         NL_SET_ERR_MSG_FMT_MOD(extack,
 317                                                "L3/L4 flower keys %#llx require protocol ipv[46]",
 318                                                dissector->used_keys);
 319                         return -EINVAL;
 320                 }
 321
 322         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
 323                 struct flow_match_vlan fm;
 324
 325                 flow_rule_match_vlan(rule, &fm);
 326                 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {
 327                         match->value.vlan_proto[0] = fm.key->vlan_tpid;
 328                         match->mask.vlan_proto[0] = fm.mask->vlan_tpid;
 329                         match->value.vlan_tci[0] = cpu_to_be16(fm.key->vlan_priority << 13 |
 330                                                                fm.key->vlan_id);
 331                         match->mask.vlan_tci[0] = cpu_to_be16(fm.mask->vlan_priority << 13 |
 332                                                               fm.mask->vlan_id);
 333                 }
 334         }
 335
 336         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CVLAN)) {
 337                 struct flow_match_vlan fm;
 338
 339                 flow_rule_match_cvlan(rule, &fm);
 340                 if (fm.mask->vlan_id || fm.mask->vlan_priority || fm.mask->vlan_tpid) {
 341                         match->value.vlan_proto[1] = fm.key->vlan_tpid;
 342                         match->mask.vlan_proto[1] = fm.mask->vlan_tpid;
 343                         match->value.vlan_tci[1] = cpu_to_be16(fm.key->vlan_priority << 13 |
 344                                                                fm.key->vlan_id);
 345                         match->mask.vlan_tci[1] = cpu_to_be16(fm.mask->vlan_priority << 13 |
 346                                                               fm.mask->vlan_id);
 347                 }
 348         }
 349
 350         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
 351                 struct flow_match_eth_addrs fm;
 352
 353                 flow_rule_match_eth_addrs(rule, &fm);
 354                 ether_addr_copy(match->value.eth_saddr, fm.key->src);
 355                 ether_addr_copy(match->value.eth_daddr, fm.key->dst);
 356                 ether_addr_copy(match->mask.eth_saddr, fm.mask->src);
 357                 ether_addr_copy(match->mask.eth_daddr, fm.mask->dst);
 358         }
 359
 360         MAP_KEY_AND_MASK(BASIC, basic, ip_proto, ip_proto);
 361         /* Make sure we're TCP/UDP if any L4 keys used. */
 362         if ((match->value.ip_proto != IPPROTO_UDP &&
 363              match->value.ip_proto != IPPROTO_TCP) || !IS_ALL_ONES(match->mask.ip_proto))
 364                 if (dissector->used_keys &
 365                     (BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
 366                      BIT_ULL(FLOW_DISSECTOR_KEY_TCP))) {
 367                         NL_SET_ERR_MSG_FMT_MOD(extack,
 368                                                "L4 flower keys %#llx require ipproto udp or tcp",
 369                                                dissector->used_keys);
 370                         return -EINVAL;
 371                 }
 372         MAP_KEY_AND_MASK(IP, ip, tos, ip_tos);
 373         MAP_KEY_AND_MASK(IP, ip, ttl, ip_ttl);
 374         if (ipv == 4) {
 375                 MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, src, src_ip);
 376                 MAP_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, dst, dst_ip);
 377         }
 378 #ifdef CONFIG_IPV6
 379         else if (ipv == 6) {
 380                 MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, src, src_ip6);
 381                 MAP_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, dst, dst_ip6);
 382         }
 383 #endif
 384         MAP_KEY_AND_MASK(PORTS, ports, src, l4_sport);
 385         MAP_KEY_AND_MASK(PORTS, ports, dst, l4_dport);
 386         MAP_KEY_AND_MASK(TCP, tcp, flags, tcp_flags);
 387         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
 388                 struct flow_match_control fm;
 389
 390                 flow_rule_match_enc_control(rule, &fm);
 391                 if (fm.mask->flags) {
 392                         NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported match on enc_control.flags %#x",
 393                                                fm.mask->flags);
 394                         return -EOPNOTSUPP;
 395                 }
 396                 if (!IS_ALL_ONES(fm.mask->addr_type)) {
 397                         NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported enc addr_type mask %u (key %u)",
 398                                                fm.mask->addr_type,
 399                                                fm.key->addr_type);
 400                         return -EOPNOTSUPP;
 401                 }
 402                 switch (fm.key->addr_type) {
 403                 case FLOW_DISSECTOR_KEY_IPV4_ADDRS:
 404                         MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,
 405                                              src, enc_src_ip);
 406                         MAP_ENC_KEY_AND_MASK(IPV4_ADDRS, ipv4_addrs, enc_ipv4_addrs,
 407                                              dst, enc_dst_ip);
 408                         break;
 409 #ifdef CONFIG_IPV6
 410                 case FLOW_DISSECTOR_KEY_IPV6_ADDRS:
 411                         MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,
 412                                              src, enc_src_ip6);
 413                         MAP_ENC_KEY_AND_MASK(IPV6_ADDRS, ipv6_addrs, enc_ipv6_addrs,
 414                                              dst, enc_dst_ip6);
 415                         break;
 416 #endif
 417                 default:
 418                         NL_SET_ERR_MSG_FMT_MOD(extack,
 419                                                "Unsupported enc addr_type %u (supported are IPv4, IPv6)",
 420                                                fm.key->addr_type);
 421                         return -EOPNOTSUPP;
 422                 }
 423                 MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, tos, enc_ip_tos);
 424                 MAP_ENC_KEY_AND_MASK(IP, ip, enc_ip, ttl, enc_ip_ttl);
 425                 MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, src, enc_sport);
 426                 MAP_ENC_KEY_AND_MASK(PORTS, ports, enc_ports, dst, enc_dport);
 427                 MAP_ENC_KEY_AND_MASK(KEYID, enc_keyid, enc_keyid, keyid, enc_keyid);
 428         } else if (dissector->used_keys &
 429                    (BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID) |
 430                     BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS) |
 431                     BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS) |
 432                     BIT_ULL(FLOW_DISSECTOR_KEY_ENC_IP) |
 433                     BIT_ULL(FLOW_DISSECTOR_KEY_ENC_PORTS))) {
 434                 NL_SET_ERR_MSG_FMT_MOD(extack,
 435                                        "Flower enc keys require enc_control (keys: %#llx)",
 436                                        dissector->used_keys);
 437                 return -EOPNOTSUPP;
 438         }
 439         if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT)) {
 440                 struct flow_match_ct fm;
 441
 442                 flow_rule_match_ct(rule, &fm);
 443                 match->value.ct_state_trk = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);
 444                 match->mask.ct_state_trk = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED);
 445                 match->value.ct_state_est = !!(fm.key->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);
 446                 match->mask.ct_state_est = !!(fm.mask->ct_state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED);
 447                 if (fm.mask->ct_state & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
 448                                           TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED)) {
 449                         NL_SET_ERR_MSG_FMT_MOD(extack,
 450                                                "Unsupported ct_state match %#x",
 451                                                fm.mask->ct_state);
 452                         return -EOPNOTSUPP;
 453                 }
 454                 match->value.ct_mark = fm.key->ct_mark;
 455                 match->mask.ct_mark = fm.mask->ct_mark;
 456                 match->value.ct_zone = fm.key->ct_zone;
 457                 match->mask.ct_zone = fm.mask->ct_zone;
 458
 459                 if (memchr_inv(fm.mask->ct_labels, 0, sizeof(fm.mask->ct_labels))) {
 460                         NL_SET_ERR_MSG_MOD(extack, "Matching on ct_label not supported");
 461                         return -EOPNOTSUPP;
 462                 }
 463         }
 464
 465         return 0;
 466 }
 467
 468 static void efx_tc_flower_release_encap_match(struct efx_nic *efx,
 469                                               struct efx_tc_encap_match *encap)
 470 {
 471         int rc;
 472
 473         if (!refcount_dec_and_test(&encap->ref))
 474                 return; /* still in use */
 475
 476         if (encap->type == EFX_TC_EM_DIRECT) {
 477                 rc = efx_mae_unregister_encap_match(efx, encap);
 478                 if (rc)
 479                         /* Display message but carry on and remove entry from our
 480                          * SW tables, because there's not much we can do about it.
 481                          */
 482                         netif_err(efx, drv, efx->net_dev,
 483                                   "Failed to release encap match %#x, rc %d\n",
 484                                   encap->fw_id, rc);
 485         }
 486         rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,
 487                                efx_tc_encap_match_ht_params);
 488         if (encap->pseudo)
 489                 efx_tc_flower_release_encap_match(efx, encap->pseudo);
 490         kfree(encap);
 491 }
 492
 493 static int efx_tc_flower_record_encap_match(struct efx_nic *efx,
 494                                             struct efx_tc_match *match,
 495                                             enum efx_encap_type type,
 496                                             enum efx_tc_em_pseudo_type em_type,
 497                                             u8 child_ip_tos_mask,
 498                                             __be16 child_udp_sport_mask,
 499                                             struct netlink_ext_ack *extack)
 500 {
 501         struct efx_tc_encap_match *encap, *old, *pseudo = NULL;
 502         bool ipv6 = false;
 503         int rc;
 504
 505         /* We require that the socket-defining fields (IP addrs and UDP dest
 506          * port) are present and exact-match.  Other fields may only be used
 507          * if the field-set (and any masks) are the same for all encap
 508          * matches on the same <sip,dip,dport> tuple; this is enforced by
 509          * pseudo encap matches.
 510          */
 511         if (match->mask.enc_dst_ip | match->mask.enc_src_ip) {
 512                 if (!IS_ALL_ONES(match->mask.enc_dst_ip)) {
 513                         NL_SET_ERR_MSG_MOD(extack,
 514                                            "Egress encap match is not exact on dst IP address");
 515                         return -EOPNOTSUPP;
 516                 }
 517                 if (!IS_ALL_ONES(match->mask.enc_src_ip)) {
 518                         NL_SET_ERR_MSG_MOD(extack,
 519                                            "Egress encap match is not exact on src IP address");
 520                         return -EOPNOTSUPP;
 521                 }
 522 #ifdef CONFIG_IPV6
 523                 if (!ipv6_addr_any(&match->mask.enc_dst_ip6) ||
 524                     !ipv6_addr_any(&match->mask.enc_src_ip6)) {
 525                         NL_SET_ERR_MSG_MOD(extack,
 526                                            "Egress encap match on both IPv4 and IPv6, don't understand");
 527                         return -EOPNOTSUPP;
 528                 }
 529         } else {
 530                 ipv6 = true;
 531                 if (!efx_ipv6_addr_all_ones(&match->mask.enc_dst_ip6)) {
 532                         NL_SET_ERR_MSG_MOD(extack,
 533                                            "Egress encap match is not exact on dst IP address");
 534                         return -EOPNOTSUPP;
 535                 }
 536                 if (!efx_ipv6_addr_all_ones(&match->mask.enc_src_ip6)) {
 537                         NL_SET_ERR_MSG_MOD(extack,
 538                                            "Egress encap match is not exact on src IP address");
 539                         return -EOPNOTSUPP;
 540                 }
 541 #endif
 542         }
 543         if (!IS_ALL_ONES(match->mask.enc_dport)) {
 544                 NL_SET_ERR_MSG_MOD(extack, "Egress encap match is not exact on dst UDP port");
 545                 return -EOPNOTSUPP;
 546         }
 547         if (match->mask.enc_sport || match->mask.enc_ip_tos) {
 548                 struct efx_tc_match pmatch = *match;
 549
 550                 if (em_type == EFX_TC_EM_PSEUDO_MASK) { /* can't happen */
 551                         NL_SET_ERR_MSG_MOD(extack, "Bad recursion in egress encap match handler");
 552                         return -EOPNOTSUPP;
 553                 }
 554                 pmatch.value.enc_ip_tos = 0;
 555                 pmatch.mask.enc_ip_tos = 0;
 556                 pmatch.value.enc_sport = 0;
 557                 pmatch.mask.enc_sport = 0;
 558                 rc = efx_tc_flower_record_encap_match(efx, &pmatch, type,
 559                                                       EFX_TC_EM_PSEUDO_MASK,
 560                                                       match->mask.enc_ip_tos,
 561                                                       match->mask.enc_sport,
 562                                                       extack);
 563                 if (rc)
 564                         return rc;
 565                 pseudo = pmatch.encap;
 566         }
 567         if (match->mask.enc_ip_ttl) {
 568                 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on IP TTL not supported");
 569                 rc = -EOPNOTSUPP;
 570                 goto fail_pseudo;
 571         }
 572
 573         rc = efx_mae_check_encap_match_caps(efx, ipv6, match->mask.enc_ip_tos,
 574                                             match->mask.enc_sport, extack);
 575         if (rc)
 576                 goto fail_pseudo;
 577
 578         encap = kzalloc(sizeof(*encap), GFP_USER);
 579         if (!encap) {
 580                 rc = -ENOMEM;
 581                 goto fail_pseudo;
 582         }
 583         encap->src_ip = match->value.enc_src_ip;
 584         encap->dst_ip = match->value.enc_dst_ip;
 585 #ifdef CONFIG_IPV6
 586         encap->src_ip6 = match->value.enc_src_ip6;
 587         encap->dst_ip6 = match->value.enc_dst_ip6;
 588 #endif
 589         encap->udp_dport = match->value.enc_dport;
 590         encap->tun_type = type;
 591         encap->ip_tos = match->value.enc_ip_tos;
 592         encap->ip_tos_mask = match->mask.enc_ip_tos;
 593         encap->child_ip_tos_mask = child_ip_tos_mask;
 594         encap->udp_sport = match->value.enc_sport;
 595         encap->udp_sport_mask = match->mask.enc_sport;
 596         encap->child_udp_sport_mask = child_udp_sport_mask;
 597         encap->type = em_type;
 598         encap->pseudo = pseudo;
 599         old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_match_ht,
 600                                                 &encap->linkage,
 601                                                 efx_tc_encap_match_ht_params);
 602         if (old) {
 603                 /* don't need our new entry */
 604                 kfree(encap);
 605                 if (pseudo) /* don't need our new pseudo either */
 606                         efx_tc_flower_release_encap_match(efx, pseudo);
 607                 if (IS_ERR(old)) /* oh dear, it's actually an error */
 608                         return PTR_ERR(old);
 609                 /* check old and new em_types are compatible */
 610                 switch (old->type) {
 611                 case EFX_TC_EM_DIRECT:
 612                         /* old EM is in hardware, so mustn't overlap with a
 613                          * pseudo, but may be shared with another direct EM
 614                          */
 615                         if (em_type == EFX_TC_EM_DIRECT)
 616                                 break;
 617                         NL_SET_ERR_MSG_MOD(extack, "Pseudo encap match conflicts with existing direct entry");
 618                         return -EEXIST;
 619                 case EFX_TC_EM_PSEUDO_MASK:
 620                         /* old EM is protecting a ToS- or src port-qualified
 621                          * filter, so may only be shared with another pseudo
 622                          * for the same ToS and src port masks.
 623                          */
 624                         if (em_type != EFX_TC_EM_PSEUDO_MASK) {
 625                                 NL_SET_ERR_MSG_FMT_MOD(extack,
 626                                                        "%s encap match conflicts with existing pseudo(MASK) entry",
 627                                                        em_type ? "Pseudo" : "Direct");
 628                                 return -EEXIST;
 629                         }
 630                         if (child_ip_tos_mask != old->child_ip_tos_mask) {
 631                                 NL_SET_ERR_MSG_FMT_MOD(extack,
 632                                                        "Pseudo encap match for TOS mask %#04x conflicts with existing mask %#04x",
 633                                                        child_ip_tos_mask,
 634                                                        old->child_ip_tos_mask);
 635                                 return -EEXIST;
 636                         }
 637                         if (child_udp_sport_mask != old->child_udp_sport_mask) {
 638                                 NL_SET_ERR_MSG_FMT_MOD(extack,
 639                                                        "Pseudo encap match for UDP src port mask %#x conflicts with existing mask %#x",
 640                                                        child_udp_sport_mask,
 641                                                        old->child_udp_sport_mask);
 642                                 return -EEXIST;
 643                         }
 644                         break;
 645                 case EFX_TC_EM_PSEUDO_OR:
 646                         /* old EM corresponds to an OR that has to be unique
 647                          * (it must not overlap with any other OR, whether
 648                          * direct-EM or pseudo).
 649                          */
 650                         NL_SET_ERR_MSG_FMT_MOD(extack,
 651                                                "%s encap match conflicts with existing pseudo(OR) entry",
 652                                                em_type ? "Pseudo" : "Direct");
 653                         return -EEXIST;
 654                 default: /* Unrecognised pseudo-type.  Just say no */
 655                         NL_SET_ERR_MSG_FMT_MOD(extack,
 656                                                "%s encap match conflicts with existing pseudo(%d) entry",
 657                                                em_type ? "Pseudo" : "Direct",
 658                                                old->type);
 659                         return -EEXIST;
 660                 }
 661                 /* check old and new tun_types are compatible */
 662                 if (old->tun_type != type) {
 663                         NL_SET_ERR_MSG_FMT_MOD(extack,
 664                                                "Egress encap match with conflicting tun_type %u != %u",
 665                                                old->tun_type, type);
 666                         return -EEXIST;
 667                 }
 668                 if (!refcount_inc_not_zero(&old->ref))
 669                         return -EAGAIN;
 670                 /* existing entry found */
 671                 encap = old;
 672         } else {
 673                 if (em_type == EFX_TC_EM_DIRECT) {
 674                         rc = efx_mae_register_encap_match(efx, encap);
 675                         if (rc) {
 676                                 NL_SET_ERR_MSG_MOD(extack, "Failed to record egress encap match in HW");
 677                                 goto fail;
 678                         }
 679                 }
 680                 refcount_set(&encap->ref, 1);
 681         }
 682         match->encap = encap;
 683         return 0;
 684 fail:
 685         rhashtable_remove_fast(&efx->tc->encap_match_ht, &encap->linkage,
 686                                efx_tc_encap_match_ht_params);
 687         kfree(encap);
 688 fail_pseudo:
 689         if (pseudo)
 690                 efx_tc_flower_release_encap_match(efx, pseudo);
 691         return rc;
 692 }
 693
 694 static struct efx_tc_recirc_id *efx_tc_get_recirc_id(struct efx_nic *efx,
 695                                                      u32 chain_index,
 696                                                      struct net_device *net_dev)
 697 {
 698         struct efx_tc_recirc_id *rid, *old;
 699         int rc;
 700
 701         rid = kzalloc(sizeof(*rid), GFP_USER);
 702         if (!rid)
 703                 return ERR_PTR(-ENOMEM);
 704         rid->chain_index = chain_index;
 705         /* We don't take a reference here, because it's implied - if there's
 706          * a rule on the net_dev that's been offloaded to us, then the net_dev
 707          * can't go away until the rule has been deoffloaded.
 708          */
 709         rid->net_dev = net_dev;
 710         old = rhashtable_lookup_get_insert_fast(&efx->tc->recirc_ht,
 711                                                 &rid->linkage,
 712                                                 efx_tc_recirc_ht_params);
 713         if (old) {
 714                 /* don't need our new entry */
 715                 kfree(rid);
 716                 if (IS_ERR(old)) /* oh dear, it's actually an error */
 717                         return ERR_CAST(old);
 718                 if (!refcount_inc_not_zero(&old->ref))
 719                         return ERR_PTR(-EAGAIN);
 720                 /* existing entry found */
 721                 rid = old;
 722         } else {
 723                 rc = ida_alloc_range(&efx->tc->recirc_ida, 1, U8_MAX, GFP_USER);
 724                 if (rc < 0) {
 725                         rhashtable_remove_fast(&efx->tc->recirc_ht,
 726                                                &rid->linkage,
 727                                                efx_tc_recirc_ht_params);
 728                         kfree(rid);
 729                         return ERR_PTR(rc);
 730                 }
 731                 rid->fw_id = rc;
 732                 refcount_set(&rid->ref, 1);
 733         }
 734         return rid;
 735 }
 736
 737 static void efx_tc_put_recirc_id(struct efx_nic *efx, struct efx_tc_recirc_id *rid)
 738 {
 739         if (!refcount_dec_and_test(&rid->ref))
 740                 return; /* still in use */
 741         rhashtable_remove_fast(&efx->tc->recirc_ht, &rid->linkage,
 742                                efx_tc_recirc_ht_params);
 743         ida_free(&efx->tc->recirc_ida, rid->fw_id);
 744         kfree(rid);
 745 }
 746
 747 static void efx_tc_delete_rule(struct efx_nic *efx, struct efx_tc_flow_rule *rule)
 748 {
 749         efx_mae_delete_rule(efx, rule->fw_id);
 750
 751         /* Release entries in subsidiary tables */
 752         efx_tc_free_action_set_list(efx, &rule->acts, true);
 753         if (rule->match.rid)
 754                 efx_tc_put_recirc_id(efx, rule->match.rid);
 755         if (rule->match.encap)
 756                 efx_tc_flower_release_encap_match(efx, rule->match.encap);
 757         rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
 758 }
 759
 760 static const char *efx_tc_encap_type_name(enum efx_encap_type typ)
 761 {
 762         switch (typ) {
 763         case EFX_ENCAP_TYPE_NONE:
 764                 return "none";
 765         case EFX_ENCAP_TYPE_VXLAN:
 766                 return "vxlan";
 767         case EFX_ENCAP_TYPE_GENEVE:
 768                 return "geneve";
 769         default:
 770                 pr_warn_once("Unknown efx_encap_type %d encountered\n", typ);
 771                 return "unknown";
 772         }
 773 }
 774
 775 /* For details of action order constraints refer to SF-123102-TC-1§12.6.1 */
 776 enum efx_tc_action_order {
 777         EFX_TC_AO_DECAP,
 778         EFX_TC_AO_DEC_TTL,
 779         EFX_TC_AO_PEDIT_MAC_ADDRS,
 780         EFX_TC_AO_VLAN_POP,
 781         EFX_TC_AO_VLAN_PUSH,
 782         EFX_TC_AO_COUNT,
 783         EFX_TC_AO_ENCAP,
 784         EFX_TC_AO_DELIVER
 785 };
 786 /* Determine whether we can add @new action without violating order */
 787 static bool efx_tc_flower_action_order_ok(const struct efx_tc_action_set *act,
 788                                           enum efx_tc_action_order new)
 789 {
 790         switch (new) {
 791         case EFX_TC_AO_DECAP:
 792                 if (act->decap)
 793                         return false;
 794                 /* PEDIT_MAC_ADDRS must not happen before DECAP, though it
 795                  * can wait until much later
 796                  */
 797                 if (act->dst_mac || act->src_mac)
 798                         return false;
 799
 800                 /* Decrementing ttl must not happen before DECAP */
 801                 if (act->do_ttl_dec)
 802                         return false;
 803                 fallthrough;
 804         case EFX_TC_AO_VLAN_POP:
 805                 if (act->vlan_pop >= 2)
 806                         return false;
 807                 /* If we've already pushed a VLAN, we can't then pop it;
 808                  * the hardware would instead try to pop an existing VLAN
 809                  * before pushing the new one.
 810                  */
 811                 if (act->vlan_push)
 812                         return false;
 813                 fallthrough;
 814         case EFX_TC_AO_VLAN_PUSH:
 815                 if (act->vlan_push >= 2)
 816                         return false;
 817                 fallthrough;
 818         case EFX_TC_AO_COUNT:
 819                 if (act->count)
 820                         return false;
 821                 fallthrough;
 822         case EFX_TC_AO_PEDIT_MAC_ADDRS:
 823         case EFX_TC_AO_ENCAP:
 824                 if (act->encap_md)
 825                         return false;
 826                 fallthrough;
 827         case EFX_TC_AO_DELIVER:
 828                 return !act->deliver;
 829         case EFX_TC_AO_DEC_TTL:
 830                 if (act->encap_md)
 831                         return false;
 832                 return !act->do_ttl_dec;
 833         default:
 834                 /* Bad caller.  Whatever they wanted to do, say they can't. */
 835                 WARN_ON_ONCE(1);
 836                 return false;
 837         }
 838 }
 839
 840 /**
 841  * DOC: TC conntrack sequences
 842  *
 843  * The MAE hardware can handle at most two rounds of action rule matching,
 844  * consequently we support conntrack through the notion of a "left-hand side
 845  * rule".  This is a rule which typically contains only the actions "ct" and
 846  * "goto chain N", and corresponds to one or more "right-hand side rules" in
 847  * chain N, which typically match on +trk+est, and may perform ct(nat) actions.
 848  * RHS rules go in the Action Rule table as normal but with a nonzero recirc_id
 849  * (the hardware equivalent of chain_index), while LHS rules may go in either
 850  * the Action Rule or the Outer Rule table, the latter being preferred for
 851  * performance reasons, and set both DO_CT and a recirc_id in their response.
 852  *
 853  * Besides the RHS rules, there are often also similar rules matching on
 854  * +trk+new which perform the ct(commit) action.  These are not offloaded.
 855  */
 856
 857 static bool efx_tc_rule_is_lhs_rule(struct flow_rule *fr,
 858                                     struct efx_tc_match *match)
 859 {
 860         const struct flow_action_entry *fa;
 861         int i;
 862
 863         flow_action_for_each(i, fa, &fr->action) {
 864                 switch (fa->id) {
 865                 case FLOW_ACTION_GOTO:
 866                         return true;
 867                 case FLOW_ACTION_CT:
 868                         /* If rule is -trk, or doesn't mention trk at all, then
 869                          * a CT action implies a conntrack lookup (hence it's an
 870                          * LHS rule).  If rule is +trk, then a CT action could
 871                          * just be ct(nat) or even ct(commit) (though the latter
 872                          * can't be offloaded).
 873                          */
 874                         if (!match->mask.ct_state_trk || !match->value.ct_state_trk)
 875                                 return true;
 876                         break;
 877                 default:
 878                         break;
 879                 }
 880         }
 881         return false;
 882 }
 883
 884 /* A foreign LHS rule has matches on enc_ keys at the TC layer (including an
 885  * implied match on enc_ip_proto UDP).  Translate these into non-enc_ keys,
 886  * so that we can use the same MAE machinery as local LHS rules (and so that
 887  * the lhs_rules entries have uniform semantics).  It may seem odd to do it
 888  * this way round, given that the corresponding fields in the MAE MCDIs are
 889  * all ENC_, but (a) we don't have enc_L2 or enc_ip_proto in struct
 890  * efx_tc_match_fields and (b) semantically an LHS rule doesn't have inner
 891  * fields so it's just matching on *the* header rather than the outer header.
 892  * Make sure that the non-enc_ keys were not already being matched on, as that
 893  * would imply a rule that needed a triple lookup.  (Hardware can do that,
 894  * with OR-AR-CT-AR, but it halves packet rate so we avoid it where possible;
 895  * see efx_tc_flower_flhs_needs_ar().)
 896  */
 897 static int efx_tc_flower_translate_flhs_match(struct efx_tc_match *match)
 898 {
 899         int rc = 0;
 900
 901 #define COPY_MASK_AND_VALUE(_key, _ekey)        ({      \
 902         if (match->mask._key) {                         \
 903                 rc = -EOPNOTSUPP;                       \
 904         } else {                                        \
 905                 match->mask._key = match->mask._ekey;   \
 906                 match->mask._ekey = 0;                  \
 907                 match->value._key = match->value._ekey; \
 908                 match->value._ekey = 0;                 \
 909         }                                               \
 910         rc;                                             \
 911 })
 912 #define COPY_FROM_ENC(_key)     COPY_MASK_AND_VALUE(_key, enc_##_key)
 913         if (match->mask.ip_proto)
 914                 return -EOPNOTSUPP;
 915         match->mask.ip_proto = ~0;
 916         match->value.ip_proto = IPPROTO_UDP;
 917         if (COPY_FROM_ENC(src_ip) || COPY_FROM_ENC(dst_ip))
 918                 return rc;
 919 #ifdef CONFIG_IPV6
 920         if (!ipv6_addr_any(&match->mask.src_ip6))
 921                 return -EOPNOTSUPP;
 922         match->mask.src_ip6 = match->mask.enc_src_ip6;
 923         memset(&match->mask.enc_src_ip6, 0, sizeof(struct in6_addr));
 924         if (!ipv6_addr_any(&match->mask.dst_ip6))
 925                 return -EOPNOTSUPP;
 926         match->mask.dst_ip6 = match->mask.enc_dst_ip6;
 927         memset(&match->mask.enc_dst_ip6, 0, sizeof(struct in6_addr));
 928 #endif
 929         if (COPY_FROM_ENC(ip_tos) || COPY_FROM_ENC(ip_ttl))
 930                 return rc;
 931         /* should really copy enc_ip_frag but we don't have that in
 932          * parse_match yet
 933          */
 934         if (COPY_MASK_AND_VALUE(l4_sport, enc_sport) ||
 935             COPY_MASK_AND_VALUE(l4_dport, enc_dport))
 936                 return rc;
 937         return 0;
 938 #undef COPY_FROM_ENC
 939 #undef COPY_MASK_AND_VALUE
 940 }
 941
 942 /* If a foreign LHS rule wants to match on keys that are only available after
 943  * encap header identification and parsing, then it can't be done in the Outer
 944  * Rule lookup, because that lookup determines the encap type used to parse
 945  * beyond the outer headers.  Thus, such rules must use the OR-AR-CT-AR lookup
 946  * sequence, with an EM (struct efx_tc_encap_match) in the OR step.
 947  * Return true iff the passed match requires this.
 948  */
 949 static bool efx_tc_flower_flhs_needs_ar(struct efx_tc_match *match)
 950 {
 951         /* matches on inner-header keys can't be done in OR */
 952         return match->mask.eth_proto ||
 953                match->mask.vlan_tci[0] || match->mask.vlan_tci[1] ||
 954                match->mask.vlan_proto[0] || match->mask.vlan_proto[1] ||
 955                memchr_inv(match->mask.eth_saddr, 0, ETH_ALEN) ||
 956                memchr_inv(match->mask.eth_daddr, 0, ETH_ALEN) ||
 957                match->mask.ip_proto ||
 958                match->mask.ip_tos || match->mask.ip_ttl ||
 959                match->mask.src_ip || match->mask.dst_ip ||
 960 #ifdef CONFIG_IPV6
 961                !ipv6_addr_any(&match->mask.src_ip6) ||
 962                !ipv6_addr_any(&match->mask.dst_ip6) ||
 963 #endif
 964                match->mask.ip_frag || match->mask.ip_firstfrag ||
 965                match->mask.l4_sport || match->mask.l4_dport ||
 966                match->mask.tcp_flags ||
 967         /* nor can VNI */
 968                match->mask.enc_keyid;
 969 }
 970
 971 static int efx_tc_flower_handle_lhs_actions(struct efx_nic *efx,
 972                                             struct flow_cls_offload *tc,
 973                                             struct flow_rule *fr,
 974                                             struct net_device *net_dev,
 975                                             struct efx_tc_lhs_rule *rule)
 976
 977 {
 978         struct netlink_ext_ack *extack = tc->common.extack;
 979         struct efx_tc_lhs_action *act = &rule->lhs_act;
 980         const struct flow_action_entry *fa;
 981         enum efx_tc_counter_type ctype;
 982         bool pipe = true;
 983         int i;
 984
 985         ctype = rule->is_ar ? EFX_TC_COUNTER_TYPE_AR : EFX_TC_COUNTER_TYPE_OR;
 986
 987         flow_action_for_each(i, fa, &fr->action) {
 988                 struct efx_tc_ct_zone *ct_zone;
 989                 struct efx_tc_recirc_id *rid;
 990
 991                 if (!pipe) {
 992                         /* more actions after a non-pipe action */
 993                         NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action");
 994                         return -EINVAL;
 995                 }
 996                 switch (fa->id) {
 997                 case FLOW_ACTION_GOTO:
 998                         if (!fa->chain_index) {
 999                                 NL_SET_ERR_MSG_MOD(extack, "Can't goto chain 0, no looping in hw");
1000                                 return -EOPNOTSUPP;
1001                         }
1002                         rid = efx_tc_get_recirc_id(efx, fa->chain_index,
1003                                                    net_dev);
1004                         if (IS_ERR(rid)) {
1005                                 NL_SET_ERR_MSG_MOD(extack, "Failed to allocate a hardware recirculation ID for this chain_index");
1006                                 return PTR_ERR(rid);
1007                         }
1008                         act->rid = rid;
1009                         if (fa->hw_stats) {
1010                                 struct efx_tc_counter_index *cnt;
1011
1012                                 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
1013                                         NL_SET_ERR_MSG_FMT_MOD(extack,
1014                                                                "hw_stats_type %u not supported (only 'delayed')",
1015                                                                fa->hw_stats);
1016                                         return -EOPNOTSUPP;
1017                                 }
1018                                 cnt = efx_tc_flower_get_counter_index(efx, tc->cookie,
1019                                                                       ctype);
1020                                 if (IS_ERR(cnt)) {
1021                                         NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
1022                                         return PTR_ERR(cnt);
1023                                 }
1024                                 WARN_ON(act->count); /* can't happen */
1025                                 act->count = cnt;
1026                         }
1027                         pipe = false;
1028                         break;
1029                 case FLOW_ACTION_CT:
1030                         if (act->zone) {
1031                                 NL_SET_ERR_MSG_MOD(extack, "Can't offload multiple ct actions");
1032                                 return -EOPNOTSUPP;
1033                         }
1034                         if (fa->ct.action & (TCA_CT_ACT_COMMIT |
1035                                              TCA_CT_ACT_FORCE)) {
1036                                 NL_SET_ERR_MSG_MOD(extack, "Can't offload ct commit/force");
1037                                 return -EOPNOTSUPP;
1038                         }
1039                         if (fa->ct.action & TCA_CT_ACT_CLEAR) {
1040                                 NL_SET_ERR_MSG_MOD(extack, "Can't clear ct in LHS rule");
1041                                 return -EOPNOTSUPP;
1042                         }
1043                         if (fa->ct.action & (TCA_CT_ACT_NAT |
1044                                              TCA_CT_ACT_NAT_SRC |
1045                                              TCA_CT_ACT_NAT_DST)) {
1046                                 NL_SET_ERR_MSG_MOD(extack, "Can't perform NAT in LHS rule - packet isn't conntracked yet");
1047                                 return -EOPNOTSUPP;
1048                         }
1049                         if (fa->ct.action) {
1050                                 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled ct.action %u for LHS rule\n",
1051                                                        fa->ct.action);
1052                                 return -EOPNOTSUPP;
1053                         }
1054                         ct_zone = efx_tc_ct_register_zone(efx, fa->ct.zone,
1055                                                           fa->ct.flow_table);
1056                         if (IS_ERR(ct_zone)) {
1057                                 NL_SET_ERR_MSG_MOD(extack, "Failed to register for CT updates");
1058                                 return PTR_ERR(ct_zone);
1059                         }
1060                         act->zone = ct_zone;
1061                         break;
1062                 default:
1063                         NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u for LHS rule\n",
1064                                                fa->id);
1065                         return -EOPNOTSUPP;
1066                 }
1067         }
1068
1069         if (pipe) {
1070                 NL_SET_ERR_MSG_MOD(extack, "Missing goto chain in LHS rule");
1071                 return -EOPNOTSUPP;
1072         }
1073         return 0;
1074 }
1075
1076 static void efx_tc_flower_release_lhs_actions(struct efx_nic *efx,
1077                                               struct efx_tc_lhs_action *act)
1078 {
1079         if (act->rid)
1080                 efx_tc_put_recirc_id(efx, act->rid);
1081         if (act->zone)
1082                 efx_tc_ct_unregister_zone(efx, act->zone);
1083         if (act->count)
1084                 efx_tc_flower_put_counter_index(efx, act->count);
1085 }
1086
1087 /**
1088  * struct efx_tc_mangler_state - accumulates 32-bit pedits into fields
1089  *
1090  * @dst_mac_32: dst_mac[0:3] has been populated
1091  * @dst_mac_16: dst_mac[4:5] has been populated
1092  * @src_mac_16: src_mac[0:1] has been populated
1093  * @src_mac_32: src_mac[2:5] has been populated
1094  * @dst_mac:    h_dest field of ethhdr
1095  * @src_mac:    h_source field of ethhdr
1096  *
1097  * Since FLOW_ACTION_MANGLE comes in 32-bit chunks that do not
1098  * necessarily equate to whole fields of the packet header, this
1099  * structure is used to hold the cumulative effect of the partial
1100  * field pedits that have been processed so far.
1101  */
1102 struct efx_tc_mangler_state {
1103         u8 dst_mac_32:1; /* eth->h_dest[0:3] */
1104         u8 dst_mac_16:1; /* eth->h_dest[4:5] */
1105         u8 src_mac_16:1; /* eth->h_source[0:1] */
1106         u8 src_mac_32:1; /* eth->h_source[2:5] */
1107         unsigned char dst_mac[ETH_ALEN];
1108         unsigned char src_mac[ETH_ALEN];
1109 };
1110
1111 /** efx_tc_complete_mac_mangle() - pull complete field pedits out of @mung
1112  * @efx:        NIC we're installing a flow rule on
1113  * @act:        action set (cursor) to update
1114  * @mung:       accumulated partial mangles
1115  * @extack:     netlink extended ack for reporting errors
1116  *
1117  * Check @mung to find any combinations of partial mangles that can be
1118  * combined into a complete packet field edit, add that edit to @act,
1119  * and consume the partial mangles from @mung.
1120  */
1121
1122 static int efx_tc_complete_mac_mangle(struct efx_nic *efx,
1123                                       struct efx_tc_action_set *act,
1124                                       struct efx_tc_mangler_state *mung,
1125                                       struct netlink_ext_ack *extack)
1126 {
1127         struct efx_tc_mac_pedit_action *ped;
1128
1129         if (mung->dst_mac_32 && mung->dst_mac_16) {
1130                 ped = efx_tc_flower_get_mac(efx, mung->dst_mac, extack);
1131                 if (IS_ERR(ped))
1132                         return PTR_ERR(ped);
1133
1134                 /* Check that we have not already populated dst_mac */
1135                 if (act->dst_mac)
1136                         efx_tc_flower_put_mac(efx, act->dst_mac);
1137
1138                 act->dst_mac = ped;
1139
1140                 /* consume the incomplete state */
1141                 mung->dst_mac_32 = 0;
1142                 mung->dst_mac_16 = 0;
1143         }
1144         if (mung->src_mac_16 && mung->src_mac_32) {
1145                 ped = efx_tc_flower_get_mac(efx, mung->src_mac, extack);
1146                 if (IS_ERR(ped))
1147                         return PTR_ERR(ped);
1148
1149                 /* Check that we have not already populated src_mac */
1150                 if (act->src_mac)
1151                         efx_tc_flower_put_mac(efx, act->src_mac);
1152
1153                 act->src_mac = ped;
1154
1155                 /* consume the incomplete state */
1156                 mung->src_mac_32 = 0;
1157                 mung->src_mac_16 = 0;
1158         }
1159         return 0;
1160 }
1161
1162 static int efx_tc_pedit_add(struct efx_nic *efx, struct efx_tc_action_set *act,
1163                             const struct flow_action_entry *fa,
1164                             struct netlink_ext_ack *extack)
1165 {
1166         switch (fa->mangle.htype) {
1167         case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
1168                 switch (fa->mangle.offset) {
1169                 case offsetof(struct iphdr, ttl):
1170                         /* check that pedit applies to ttl only */
1171                         if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK)
1172                                 break;
1173
1174                         /* Adding 0xff is equivalent to decrementing the ttl.
1175                          * Other added values are not supported.
1176                          */
1177                         if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) != U8_MAX)
1178                                 break;
1179
1180                         /* check that we do not decrement ttl twice */
1181                         if (!efx_tc_flower_action_order_ok(act,
1182                                                            EFX_TC_AO_DEC_TTL)) {
1183                                 NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported");
1184                                 return -EOPNOTSUPP;
1185                         }
1186                         act->do_ttl_dec = 1;
1187                         return 0;
1188                 default:
1189                         break;
1190                 }
1191                 break;
1192         case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
1193                 switch (fa->mangle.offset) {
1194                 case round_down(offsetof(struct ipv6hdr, hop_limit), 4):
1195                         /* check that pedit applies to hoplimit only */
1196                         if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK)
1197                                 break;
1198
1199                         /* Adding 0xff is equivalent to decrementing the hoplimit.
1200                          * Other added values are not supported.
1201                          */
1202                         if ((fa->mangle.val >> 24) != U8_MAX)
1203                                 break;
1204
1205                         /* check that we do not decrement hoplimit twice */
1206                         if (!efx_tc_flower_action_order_ok(act,
1207                                                            EFX_TC_AO_DEC_TTL)) {
1208                                 NL_SET_ERR_MSG_MOD(extack, "multiple dec ttl are not supported");
1209                                 return -EOPNOTSUPP;
1210                         }
1211                         act->do_ttl_dec = 1;
1212                         return 0;
1213                 default:
1214                         break;
1215                 }
1216                 break;
1217         default:
1218                 break;
1219         }
1220
1221         NL_SET_ERR_MSG_FMT_MOD(extack,
1222                                "ttl add action type %x %x %x/%x is not supported",
1223                                fa->mangle.htype, fa->mangle.offset,
1224                                fa->mangle.val, fa->mangle.mask);
1225         return -EOPNOTSUPP;
1226 }
1227
1228 /**
1229  * efx_tc_mangle() - handle a single 32-bit (or less) pedit
1230  * @efx:        NIC we're installing a flow rule on
1231  * @act:        action set (cursor) to update
1232  * @fa:         FLOW_ACTION_MANGLE action metadata
1233  * @mung:       accumulator for partial mangles
1234  * @extack:     netlink extended ack for reporting errors
1235  * @match:      original match used along with the mangle action
1236  *
1237  * Identify the fields written by a FLOW_ACTION_MANGLE, and record
1238  * the partial mangle state in @mung.  If this mangle completes an
1239  * earlier partial mangle, consume and apply to @act by calling
1240  * efx_tc_complete_mac_mangle().
1241  */
1242
1243 static int efx_tc_mangle(struct efx_nic *efx, struct efx_tc_action_set *act,
1244                          const struct flow_action_entry *fa,
1245                          struct efx_tc_mangler_state *mung,
1246                          struct netlink_ext_ack *extack,
1247                          struct efx_tc_match *match)
1248 {
1249         __le32 mac32;
1250         __le16 mac16;
1251         u8 tr_ttl;
1252
1253         switch (fa->mangle.htype) {
1254         case FLOW_ACT_MANGLE_HDR_TYPE_ETH:
1255                 BUILD_BUG_ON(offsetof(struct ethhdr, h_dest) != 0);
1256                 BUILD_BUG_ON(offsetof(struct ethhdr, h_source) != 6);
1257                 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_PEDIT_MAC_ADDRS)) {
1258                         NL_SET_ERR_MSG_MOD(extack,
1259                                            "Pedit mangle mac action violates action order");
1260                         return -EOPNOTSUPP;
1261                 }
1262                 switch (fa->mangle.offset) {
1263                 case 0:
1264                         if (fa->mangle.mask) {
1265                                 NL_SET_ERR_MSG_FMT_MOD(extack,
1266                                                        "mask (%#x) of eth.dst32 mangle is not supported",
1267                                                        fa->mangle.mask);
1268                                 return -EOPNOTSUPP;
1269                         }
1270                         /* Ethernet address is little-endian */
1271                         mac32 = cpu_to_le32(fa->mangle.val);
1272                         memcpy(mung->dst_mac, &mac32, sizeof(mac32));
1273                         mung->dst_mac_32 = 1;
1274                         return efx_tc_complete_mac_mangle(efx, act, mung, extack);
1275                 case 4:
1276                         if (fa->mangle.mask == 0xffff) {
1277                                 mac16 = cpu_to_le16(fa->mangle.val >> 16);
1278                                 memcpy(mung->src_mac, &mac16, sizeof(mac16));
1279                                 mung->src_mac_16 = 1;
1280                         } else if (fa->mangle.mask == 0xffff0000) {
1281                                 mac16 = cpu_to_le16((u16)fa->mangle.val);
1282                                 memcpy(mung->dst_mac + 4, &mac16, sizeof(mac16));
1283                                 mung->dst_mac_16 = 1;
1284                         } else {
1285                                 NL_SET_ERR_MSG_FMT_MOD(extack,
1286                                                        "mask (%#x) of eth+4 mangle is not high or low 16b",
1287                                                        fa->mangle.mask);
1288                                 return -EOPNOTSUPP;
1289                         }
1290                         return efx_tc_complete_mac_mangle(efx, act, mung, extack);
1291                 case 8:
1292                         if (fa->mangle.mask) {
1293                                 NL_SET_ERR_MSG_FMT_MOD(extack,
1294                                                        "mask (%#x) of eth.src32 mangle is not supported",
1295                                                        fa->mangle.mask);
1296                                 return -EOPNOTSUPP;
1297                         }
1298                         mac32 = cpu_to_le32(fa->mangle.val);
1299                         memcpy(mung->src_mac + 2, &mac32, sizeof(mac32));
1300                         mung->src_mac_32 = 1;
1301                         return efx_tc_complete_mac_mangle(efx, act, mung, extack);
1302                 default:
1303                         NL_SET_ERR_MSG_FMT_MOD(extack, "mangle eth+%u %x/%x is not supported",
1304                                                fa->mangle.offset, fa->mangle.val, fa->mangle.mask);
1305                         return -EOPNOTSUPP;
1306                 }
1307                 break;
1308         case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
1309                 switch (fa->mangle.offset) {
1310                 case offsetof(struct iphdr, ttl):
1311                         /* we currently only support pedit IP4 when it applies
1312                          * to TTL and then only when it can be achieved with a
1313                          * decrement ttl action
1314                          */
1315
1316                         /* check that pedit applies to ttl only */
1317                         if (fa->mangle.mask != ~EFX_TC_HDR_TYPE_TTL_MASK) {
1318                                 NL_SET_ERR_MSG_FMT_MOD(extack,
1319                                                        "mask (%#x) out of range, only support mangle action on ipv4.ttl",
1320                                                        fa->mangle.mask);
1321                                 return -EOPNOTSUPP;
1322                         }
1323
1324                         /* we can only convert to a dec ttl when we have an
1325                          * exact match on the ttl field
1326                          */
1327                         if (match->mask.ip_ttl != U8_MAX) {
1328                                 NL_SET_ERR_MSG_FMT_MOD(extack,
1329                                                        "only support mangle ttl when we have an exact match, current mask (%#x)",
1330                                                        match->mask.ip_ttl);
1331                                 return -EOPNOTSUPP;
1332                         }
1333
1334                         /* check that we don't try to decrement 0, which equates
1335                          * to setting the ttl to 0xff
1336                          */
1337                         if (match->value.ip_ttl == 0) {
1338                                 NL_SET_ERR_MSG_MOD(extack,
1339                                                    "decrement ttl past 0 is not supported");
1340                                 return -EOPNOTSUPP;
1341                         }
1342
1343                         /* check that we do not decrement ttl twice */
1344                         if (!efx_tc_flower_action_order_ok(act,
1345                                                            EFX_TC_AO_DEC_TTL)) {
1346                                 NL_SET_ERR_MSG_MOD(extack,
1347                                                    "multiple dec ttl is not supported");
1348                                 return -EOPNOTSUPP;
1349                         }
1350
1351                         /* check pedit can be achieved with decrement action */
1352                         tr_ttl = match->value.ip_ttl - 1;
1353                         if ((fa->mangle.val & EFX_TC_HDR_TYPE_TTL_MASK) == tr_ttl) {
1354                                 act->do_ttl_dec = 1;
1355                                 return 0;
1356                         }
1357
1358                         fallthrough;
1359                 default:
1360                         NL_SET_ERR_MSG_FMT_MOD(extack,
1361                                                "only support mangle on the ttl field (offset is %u)",
1362                                                fa->mangle.offset);
1363                         return -EOPNOTSUPP;
1364                 }
1365                 break;
1366         case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
1367                 switch (fa->mangle.offset) {
1368                 case round_down(offsetof(struct ipv6hdr, hop_limit), 4):
1369                         /* we currently only support pedit IP6 when it applies
1370                          * to the hoplimit and then only when it can be achieved
1371                          * with a decrement hoplimit action
1372                          */
1373
1374                         /* check that pedit applies to ttl only */
1375                         if (fa->mangle.mask != EFX_TC_HDR_TYPE_HLIMIT_MASK) {
1376                                 NL_SET_ERR_MSG_FMT_MOD(extack,
1377                                                        "mask (%#x) out of range, only support mangle action on ipv6.hop_limit",
1378                                                        fa->mangle.mask);
1379
1380                                 return -EOPNOTSUPP;
1381                         }
1382
1383                         /* we can only convert to a dec ttl when we have an
1384                          * exact match on the ttl field
1385                          */
1386                         if (match->mask.ip_ttl != U8_MAX) {
1387                                 NL_SET_ERR_MSG_FMT_MOD(extack,
1388                                                        "only support hop_limit when we have an exact match, current mask (%#x)",
1389                                                        match->mask.ip_ttl);
1390                                 return -EOPNOTSUPP;
1391                         }
1392
1393                         /* check that we don't try to decrement 0, which equates
1394                          * to setting the ttl to 0xff
1395                          */
1396                         if (match->value.ip_ttl == 0) {
1397                                 NL_SET_ERR_MSG_MOD(extack,
1398                                                    "decrementing hop_limit past 0 is not supported");
1399                                 return -EOPNOTSUPP;
1400                         }
1401
1402                         /* check that we do not decrement hoplimit twice */
1403                         if (!efx_tc_flower_action_order_ok(act,
1404                                                            EFX_TC_AO_DEC_TTL)) {
1405                                 NL_SET_ERR_MSG_MOD(extack,
1406                                                    "multiple dec ttl is not supported");
1407                                 return -EOPNOTSUPP;
1408                         }
1409
1410                         /* check pedit can be achieved with decrement action */
1411                         tr_ttl = match->value.ip_ttl - 1;
1412                         if ((fa->mangle.val >> 24) == tr_ttl) {
1413                                 act->do_ttl_dec = 1;
1414                                 return 0;
1415                         }
1416
1417                         fallthrough;
1418                 default:
1419                         NL_SET_ERR_MSG_FMT_MOD(extack,
1420                                                "only support mangle on the hop_limit field");
1421                         return -EOPNOTSUPP;
1422                 }
1423         default:
1424                 NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled mangle htype %u for action rule",
1425                                        fa->mangle.htype);
1426                 return -EOPNOTSUPP;
1427         }
1428         return 0;
1429 }
1430
1431 /**
1432  * efx_tc_incomplete_mangle() - check for leftover partial pedits
1433  * @mung:       accumulator for partial mangles
1434  * @extack:     netlink extended ack for reporting errors
1435  *
1436  * Since the MAE can only overwrite whole fields, any partial
1437  * field mangle left over on reaching packet delivery (mirred or
1438  * end of TC actions) cannot be offloaded.  Check for any such
1439  * and reject them with -%EOPNOTSUPP.
1440  */
1441
1442 static int efx_tc_incomplete_mangle(struct efx_tc_mangler_state *mung,
1443                                     struct netlink_ext_ack *extack)
1444 {
1445         if (mung->dst_mac_32 || mung->dst_mac_16) {
1446                 NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of destination MAC address");
1447                 return -EOPNOTSUPP;
1448         }
1449         if (mung->src_mac_16 || mung->src_mac_32) {
1450                 NL_SET_ERR_MSG_MOD(extack, "Incomplete pedit of source MAC address");
1451                 return -EOPNOTSUPP;
1452         }
1453         return 0;
1454 }
1455
1456 static int efx_tc_flower_replace_foreign_lhs_ar(struct efx_nic *efx,
1457                                                 struct flow_cls_offload *tc,
1458                                                 struct flow_rule *fr,
1459                                                 struct efx_tc_match *match,
1460                                                 struct net_device *net_dev)
1461 {
1462         struct netlink_ext_ack *extack = tc->common.extack;
1463         struct efx_tc_lhs_rule *rule, *old;
1464         enum efx_encap_type type;
1465         int rc;
1466
1467         type = efx_tc_indr_netdev_type(net_dev);
1468         if (type == EFX_ENCAP_TYPE_NONE) {
1469                 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device");
1470                 return -EOPNOTSUPP;
1471         }
1472
1473         rc = efx_mae_check_encap_type_supported(efx, type);
1474         if (rc) {
1475                 NL_SET_ERR_MSG_FMT_MOD(extack,
1476                                        "Firmware reports no support for %s encap match",
1477                                        efx_tc_encap_type_name(type));
1478                 return rc;
1479         }
1480         /* This is an Action Rule, so it needs a separate Encap Match in the
1481          * Outer Rule table.  Insert that now.
1482          */
1483         rc = efx_tc_flower_record_encap_match(efx, match, type,
1484                                               EFX_TC_EM_DIRECT, 0, 0, extack);
1485         if (rc)
1486                 return rc;
1487
1488         match->mask.recirc_id = 0xff;
1489         if (match->mask.ct_state_trk && match->value.ct_state_trk) {
1490                 NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");
1491                 rc = -EOPNOTSUPP;
1492                 goto release_encap_match;
1493         }
1494         /* LHS rules are always -trk, so we don't need to match on that */
1495         match->mask.ct_state_trk = 0;
1496         match->value.ct_state_trk = 0;
1497         /* We must inhibit match on TCP SYN/FIN/RST, so that SW can see
1498          * the packet and update the conntrack table.
1499          * Outer Rules will do that with CT_TCP_FLAGS_INHIBIT, but Action
1500          * Rules don't have that; instead they support matching on
1501          * TCP_SYN_FIN_RST (aka TCP_INTERESTING_FLAGS), so use that.
1502          * This is only strictly needed if there will be a DO_CT action,
1503          * which we don't know yet, but typically there will be and it's
1504          * simpler not to bother checking here.
1505          */
1506         match->mask.tcp_syn_fin_rst = true;
1507
1508         rc = efx_mae_match_check_caps(efx, &match->mask, extack);
1509         if (rc)
1510                 goto release_encap_match;
1511
1512         rule = kzalloc(sizeof(*rule), GFP_USER);
1513         if (!rule) {
1514                 rc = -ENOMEM;
1515                 goto release_encap_match;
1516         }
1517         rule->cookie = tc->cookie;
1518         rule->is_ar = true;
1519         old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,
1520                                                 &rule->linkage,
1521                                                 efx_tc_lhs_rule_ht_params);
1522         if (old) {
1523                 netif_dbg(efx, drv, efx->net_dev,
1524                           "Already offloaded rule (cookie %lx)\n", tc->cookie);
1525                 rc = -EEXIST;
1526                 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
1527                 goto release;
1528         }
1529
1530         /* Parse actions */
1531         rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);
1532         if (rc)
1533                 goto release;
1534
1535         rule->match = *match;
1536         rule->lhs_act.tun_type = type;
1537
1538         rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);
1539         if (rc) {
1540                 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1541                 goto release;
1542         }
1543         netif_dbg(efx, drv, efx->net_dev,
1544                   "Successfully parsed lhs rule (cookie %lx)\n",
1545                   tc->cookie);
1546         return 0;
1547
1548 release:
1549         efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);
1550         if (!old)
1551                 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,
1552                                        efx_tc_lhs_rule_ht_params);
1553         kfree(rule);
1554 release_encap_match:
1555         if (match->encap)
1556                 efx_tc_flower_release_encap_match(efx, match->encap);
1557         return rc;
1558 }
1559
1560 static int efx_tc_flower_replace_foreign_lhs(struct efx_nic *efx,
1561                                              struct flow_cls_offload *tc,
1562                                              struct flow_rule *fr,
1563                                              struct efx_tc_match *match,
1564                                              struct net_device *net_dev)
1565 {
1566         struct netlink_ext_ack *extack = tc->common.extack;
1567         struct efx_tc_lhs_rule *rule, *old;
1568         enum efx_encap_type type;
1569         int rc;
1570
1571         if (tc->common.chain_index) {
1572                 NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");
1573                 return -EOPNOTSUPP;
1574         }
1575
1576         if (!efx_tc_match_is_encap(&match->mask)) {
1577                 /* This is not a tunnel decap rule, ignore it */
1578                 netif_dbg(efx, drv, efx->net_dev, "Ignoring foreign LHS filter without encap match\n");
1579                 return -EOPNOTSUPP;
1580         }
1581
1582         if (efx_tc_flower_flhs_needs_ar(match))
1583                 return efx_tc_flower_replace_foreign_lhs_ar(efx, tc, fr, match,
1584                                                             net_dev);
1585
1586         type = efx_tc_indr_netdev_type(net_dev);
1587         if (type == EFX_ENCAP_TYPE_NONE) {
1588                 NL_SET_ERR_MSG_MOD(extack, "Egress encap match on unsupported tunnel device\n");
1589                 return -EOPNOTSUPP;
1590         }
1591
1592         rc = efx_mae_check_encap_type_supported(efx, type);
1593         if (rc) {
1594                 NL_SET_ERR_MSG_FMT_MOD(extack,
1595                                        "Firmware reports no support for %s encap match",
1596                                        efx_tc_encap_type_name(type));
1597                 return rc;
1598         }
1599         /* Reserve the outer tuple with a pseudo Encap Match */
1600         rc = efx_tc_flower_record_encap_match(efx, match, type,
1601                                               EFX_TC_EM_PSEUDO_OR, 0, 0,
1602                                               extack);
1603         if (rc)
1604                 return rc;
1605
1606         if (match->mask.ct_state_trk && match->value.ct_state_trk) {
1607                 NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");
1608                 rc = -EOPNOTSUPP;
1609                 goto release_encap_match;
1610         }
1611         /* LHS rules are always -trk, so we don't need to match on that */
1612         match->mask.ct_state_trk = 0;
1613         match->value.ct_state_trk = 0;
1614
1615         rc = efx_tc_flower_translate_flhs_match(match);
1616         if (rc) {
1617                 NL_SET_ERR_MSG_MOD(extack, "LHS rule cannot match on inner fields");
1618                 goto release_encap_match;
1619         }
1620
1621         rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack);
1622         if (rc)
1623                 goto release_encap_match;
1624
1625         rule = kzalloc(sizeof(*rule), GFP_USER);
1626         if (!rule) {
1627                 rc = -ENOMEM;
1628                 goto release_encap_match;
1629         }
1630         rule->cookie = tc->cookie;
1631         old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,
1632                                                 &rule->linkage,
1633                                                 efx_tc_lhs_rule_ht_params);
1634         if (old) {
1635                 netif_dbg(efx, drv, efx->net_dev,
1636                           "Already offloaded rule (cookie %lx)\n", tc->cookie);
1637                 rc = -EEXIST;
1638                 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
1639                 goto release;
1640         }
1641
1642         /* Parse actions */
1643         rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, net_dev, rule);
1644         if (rc)
1645                 goto release;
1646
1647         rule->match = *match;
1648         rule->lhs_act.tun_type = type;
1649
1650         rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);
1651         if (rc) {
1652                 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1653                 goto release;
1654         }
1655         netif_dbg(efx, drv, efx->net_dev,
1656                   "Successfully parsed lhs rule (cookie %lx)\n",
1657                   tc->cookie);
1658         return 0;
1659
1660 release:
1661         efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);
1662         if (!old)
1663                 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,
1664                                        efx_tc_lhs_rule_ht_params);
1665         kfree(rule);
1666 release_encap_match:
1667         if (match->encap)
1668                 efx_tc_flower_release_encap_match(efx, match->encap);
1669         return rc;
1670 }
1671
1672 static int efx_tc_flower_replace_foreign(struct efx_nic *efx,
1673                                          struct net_device *net_dev,
1674                                          struct flow_cls_offload *tc)
1675 {
1676         struct flow_rule *fr = flow_cls_offload_flow_rule(tc);
1677         struct netlink_ext_ack *extack = tc->common.extack;
1678         struct efx_tc_flow_rule *rule = NULL, *old = NULL;
1679         struct efx_tc_action_set *act = NULL;
1680         bool found = false, uplinked = false;
1681         const struct flow_action_entry *fa;
1682         struct efx_tc_match match;
1683         struct efx_rep *to_efv;
1684         s64 rc;
1685         int i;
1686
1687         /* Parse match */
1688         memset(&match, 0, sizeof(match));
1689         rc = efx_tc_flower_parse_match(efx, fr, &match, extack);
1690         if (rc)
1691                 return rc;
1692         /* The rule as given to us doesn't specify a source netdevice.
1693          * But, determining whether packets from a VF should match it is
1694          * complicated, so leave those to the software slowpath: qualify
1695          * the filter with source m-port == wire.
1696          */
1697         rc = efx_tc_flower_external_mport(efx, EFX_EFV_PF);
1698         if (rc < 0) {
1699                 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port for foreign filter");
1700                 return rc;
1701         }
1702         match.value.ingress_port = rc;
1703         match.mask.ingress_port = ~0;
1704
1705         if (efx_tc_rule_is_lhs_rule(fr, &match))
1706                 return efx_tc_flower_replace_foreign_lhs(efx, tc, fr, &match,
1707                                                          net_dev);
1708
1709         if (tc->common.chain_index) {
1710                 struct efx_tc_recirc_id *rid;
1711
1712                 rid = efx_tc_get_recirc_id(efx, tc->common.chain_index, net_dev);
1713                 if (IS_ERR(rid)) {
1714                         NL_SET_ERR_MSG_FMT_MOD(extack,
1715                                                "Failed to allocate a hardware recirculation ID for chain_index %u",
1716                                                tc->common.chain_index);
1717                         return PTR_ERR(rid);
1718                 }
1719                 match.rid = rid;
1720                 match.value.recirc_id = rid->fw_id;
1721         }
1722         match.mask.recirc_id = 0xff;
1723
1724         /* AR table can't match on DO_CT (+trk).  But a commonly used pattern is
1725          * +trk+est, which is strictly implied by +est, so rewrite it to that.
1726          */
1727         if (match.mask.ct_state_trk && match.value.ct_state_trk &&
1728             match.mask.ct_state_est && match.value.ct_state_est)
1729                 match.mask.ct_state_trk = 0;
1730         /* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could
1731          * match +trk-est (CT_HIT=0) despite being on an established connection.
1732          * So make -est imply -tcp_syn_fin_rst match to ensure these packets
1733          * still hit the software path.
1734          */
1735         if (match.mask.ct_state_est && !match.value.ct_state_est) {
1736                 if (match.value.tcp_syn_fin_rst) {
1737                         /* Can't offload this combination */
1738                         NL_SET_ERR_MSG_MOD(extack, "TCP flags and -est conflict for offload");
1739                         rc = -EOPNOTSUPP;
1740                         goto release;
1741                 }
1742                 match.mask.tcp_syn_fin_rst = true;
1743         }
1744
1745         flow_action_for_each(i, fa, &fr->action) {
1746                 switch (fa->id) {
1747                 case FLOW_ACTION_REDIRECT:
1748                 case FLOW_ACTION_MIRRED: /* mirred means mirror here */
1749                         to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
1750                         if (IS_ERR(to_efv))
1751                                 continue;
1752                         found = true;
1753                         break;
1754                 default:
1755                         break;
1756                 }
1757         }
1758         if (!found) { /* We don't care. */
1759                 netif_dbg(efx, drv, efx->net_dev,
1760                           "Ignoring foreign filter that doesn't egdev us\n");
1761                 rc = -EOPNOTSUPP;
1762                 goto release;
1763         }
1764
1765         rc = efx_mae_match_check_caps(efx, &match.mask, extack);
1766         if (rc)
1767                 goto release;
1768
1769         if (efx_tc_match_is_encap(&match.mask)) {
1770                 enum efx_encap_type type;
1771
1772                 type = efx_tc_indr_netdev_type(net_dev);
1773                 if (type == EFX_ENCAP_TYPE_NONE) {
1774                         NL_SET_ERR_MSG_MOD(extack,
1775                                            "Egress encap match on unsupported tunnel device");
1776                         rc = -EOPNOTSUPP;
1777                         goto release;
1778                 }
1779
1780                 rc = efx_mae_check_encap_type_supported(efx, type);
1781                 if (rc) {
1782                         NL_SET_ERR_MSG_FMT_MOD(extack,
1783                                                "Firmware reports no support for %s encap match",
1784                                                efx_tc_encap_type_name(type));
1785                         goto release;
1786                 }
1787
1788                 rc = efx_tc_flower_record_encap_match(efx, &match, type,
1789                                                       EFX_TC_EM_DIRECT, 0, 0,
1790                                                       extack);
1791                 if (rc)
1792                         goto release;
1793         } else if (!tc->common.chain_index) {
1794                 /* This is not a tunnel decap rule, ignore it */
1795                 netif_dbg(efx, drv, efx->net_dev,
1796                           "Ignoring foreign filter without encap match\n");
1797                 rc = -EOPNOTSUPP;
1798                 goto release;
1799         }
1800
1801         rule = kzalloc(sizeof(*rule), GFP_USER);
1802         if (!rule) {
1803                 rc = -ENOMEM;
1804                 goto release;
1805         }
1806         INIT_LIST_HEAD(&rule->acts.list);
1807         rule->cookie = tc->cookie;
1808         old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,
1809                                                 &rule->linkage,
1810                                                 efx_tc_match_action_ht_params);
1811         if (IS_ERR(old)) {
1812                 rc = PTR_ERR(old);
1813                 goto release;
1814         } else if (old) {
1815                 netif_dbg(efx, drv, efx->net_dev,
1816                           "Ignoring already-offloaded rule (cookie %lx)\n",
1817                           tc->cookie);
1818                 rc = -EEXIST;
1819                 goto release;
1820         }
1821
1822         act = kzalloc(sizeof(*act), GFP_USER);
1823         if (!act) {
1824                 rc = -ENOMEM;
1825                 goto release;
1826         }
1827
1828         /* Parse actions.  For foreign rules we only support decap & redirect.
1829          * See corresponding code in efx_tc_flower_replace() for theory of
1830          * operation & how 'act' cursor is used.
1831          */
1832         flow_action_for_each(i, fa, &fr->action) {
1833                 struct efx_tc_action_set save;
1834
1835                 switch (fa->id) {
1836                 case FLOW_ACTION_REDIRECT:
1837                 case FLOW_ACTION_MIRRED:
1838                         /* See corresponding code in efx_tc_flower_replace() for
1839                          * long explanations of what's going on here.
1840                          */
1841                         save = *act;
1842                         if (fa->hw_stats) {
1843                                 struct efx_tc_counter_index *ctr;
1844
1845                                 if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
1846                                         NL_SET_ERR_MSG_FMT_MOD(extack,
1847                                                                "hw_stats_type %u not supported (only 'delayed')",
1848                                                                fa->hw_stats);
1849                                         rc = -EOPNOTSUPP;
1850                                         goto release;
1851                                 }
1852                                 if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {
1853                                         NL_SET_ERR_MSG_MOD(extack, "Count action violates action order (can't happen)");
1854                                         rc = -EOPNOTSUPP;
1855                                         goto release;
1856                                 }
1857
1858                                 ctr = efx_tc_flower_get_counter_index(efx,
1859                                                                       tc->cookie,
1860                                                                       EFX_TC_COUNTER_TYPE_AR);
1861                                 if (IS_ERR(ctr)) {
1862                                         rc = PTR_ERR(ctr);
1863                                         NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
1864                                         goto release;
1865                                 }
1866                                 act->count = ctr;
1867                                 INIT_LIST_HEAD(&act->count_user);
1868                         }
1869
1870                         if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {
1871                                 /* can't happen */
1872                                 rc = -EOPNOTSUPP;
1873                                 NL_SET_ERR_MSG_MOD(extack,
1874                                                    "Deliver action violates action order (can't happen)");
1875                                 goto release;
1876                         }
1877                         to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
1878                         /* PF implies egdev is us, in which case we really
1879                          * want to deliver to the uplink (because this is an
1880                          * ingress filter).  If we don't recognise the egdev
1881                          * at all, then we'd better trap so SW can handle it.
1882                          */
1883                         if (IS_ERR(to_efv))
1884                                 to_efv = EFX_EFV_PF;
1885                         if (to_efv == EFX_EFV_PF) {
1886                                 if (uplinked)
1887                                         break;
1888                                 uplinked = true;
1889                         }
1890                         rc = efx_tc_flower_internal_mport(efx, to_efv);
1891                         if (rc < 0) {
1892                                 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");
1893                                 goto release;
1894                         }
1895                         act->dest_mport = rc;
1896                         act->deliver = 1;
1897                         rc = efx_mae_alloc_action_set(efx, act);
1898                         if (rc) {
1899                                 NL_SET_ERR_MSG_MOD(extack,
1900                                                    "Failed to write action set to hw (mirred)");
1901                                 goto release;
1902                         }
1903                         list_add_tail(&act->list, &rule->acts.list);
1904                         act = NULL;
1905                         if (fa->id == FLOW_ACTION_REDIRECT)
1906                                 break; /* end of the line */
1907                         /* Mirror, so continue on with saved act */
1908                         act = kzalloc(sizeof(*act), GFP_USER);
1909                         if (!act) {
1910                                 rc = -ENOMEM;
1911                                 goto release;
1912                         }
1913                         *act = save;
1914                         break;
1915                 case FLOW_ACTION_TUNNEL_DECAP:
1916                         if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DECAP)) {
1917                                 rc = -EINVAL;
1918                                 NL_SET_ERR_MSG_MOD(extack, "Decap action violates action order");
1919                                 goto release;
1920                         }
1921                         act->decap = 1;
1922                         /* If we previously delivered/trapped to uplink, now
1923                          * that we've decapped we'll want another copy if we
1924                          * try to deliver/trap to uplink again.
1925                          */
1926                         uplinked = false;
1927                         break;
1928                 default:
1929                         NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
1930                                                fa->id);
1931                         rc = -EOPNOTSUPP;
1932                         goto release;
1933                 }
1934         }
1935
1936         if (act) {
1937                 if (!uplinked) {
1938                         /* Not shot/redirected, so deliver to default dest (which is
1939                          * the uplink, as this is an ingress filter)
1940                          */
1941                         efx_mae_mport_uplink(efx, &act->dest_mport);
1942                         act->deliver = 1;
1943                 }
1944                 rc = efx_mae_alloc_action_set(efx, act);
1945                 if (rc) {
1946                         NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");
1947                         goto release;
1948                 }
1949                 list_add_tail(&act->list, &rule->acts.list);
1950                 act = NULL; /* Prevent double-free in error path */
1951         }
1952
1953         rule->match = match;
1954
1955         netif_dbg(efx, drv, efx->net_dev,
1956                   "Successfully parsed foreign filter (cookie %lx)\n",
1957                   tc->cookie);
1958
1959         rc = efx_mae_alloc_action_set_list(efx, &rule->acts);
1960         if (rc) {
1961                 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");
1962                 goto release;
1963         }
1964         rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,
1965                                  rule->acts.fw_id, &rule->fw_id);
1966         if (rc) {
1967                 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
1968                 goto release_acts;
1969         }
1970         return 0;
1971
1972 release_acts:
1973         efx_mae_free_action_set_list(efx, &rule->acts);
1974 release:
1975         /* We failed to insert the rule, so free up any entries we created in
1976          * subsidiary tables.
1977          */
1978         if (match.rid)
1979                 efx_tc_put_recirc_id(efx, match.rid);
1980         if (act)
1981                 efx_tc_free_action_set(efx, act, false);
1982         if (rule) {
1983                 if (!old)
1984                         rhashtable_remove_fast(&efx->tc->match_action_ht,
1985                                                &rule->linkage,
1986                                                efx_tc_match_action_ht_params);
1987                 efx_tc_free_action_set_list(efx, &rule->acts, false);
1988         }
1989         kfree(rule);
1990         if (match.encap)
1991                 efx_tc_flower_release_encap_match(efx, match.encap);
1992         return rc;
1993 }
1994
1995 static int efx_tc_flower_replace_lhs(struct efx_nic *efx,
1996                                      struct flow_cls_offload *tc,
1997                                      struct flow_rule *fr,
1998                                      struct efx_tc_match *match,
1999                                      struct efx_rep *efv,
2000                                      struct net_device *net_dev)
2001 {
2002         struct netlink_ext_ack *extack = tc->common.extack;
2003         struct efx_tc_lhs_rule *rule, *old;
2004         int rc;
2005
2006         if (tc->common.chain_index) {
2007                 NL_SET_ERR_MSG_MOD(extack, "LHS rule only allowed in chain 0");
2008                 return -EOPNOTSUPP;
2009         }
2010
2011         if (match->mask.ct_state_trk && match->value.ct_state_trk) {
2012                 NL_SET_ERR_MSG_MOD(extack, "LHS rule can never match +trk");
2013                 return -EOPNOTSUPP;
2014         }
2015         /* LHS rules are always -trk, so we don't need to match on that */
2016         match->mask.ct_state_trk = 0;
2017         match->value.ct_state_trk = 0;
2018
2019         rc = efx_mae_match_check_caps_lhs(efx, &match->mask, extack);
2020         if (rc)
2021                 return rc;
2022
2023         rule = kzalloc(sizeof(*rule), GFP_USER);
2024         if (!rule)
2025                 return -ENOMEM;
2026         rule->cookie = tc->cookie;
2027         old = rhashtable_lookup_get_insert_fast(&efx->tc->lhs_rule_ht,
2028                                                 &rule->linkage,
2029                                                 efx_tc_lhs_rule_ht_params);
2030         if (IS_ERR(old)) {
2031                 rc = PTR_ERR(old);
2032                 goto release;
2033         } else if (old) {
2034                 netif_dbg(efx, drv, efx->net_dev,
2035                           "Already offloaded rule (cookie %lx)\n", tc->cookie);
2036                 rc = -EEXIST;
2037                 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
2038                 goto release;
2039         }
2040
2041         /* Parse actions */
2042         /* See note in efx_tc_flower_replace() regarding passed net_dev
2043          * (used for efx_tc_get_recirc_id()).
2044          */
2045         rc = efx_tc_flower_handle_lhs_actions(efx, tc, fr, efx->net_dev, rule);
2046         if (rc)
2047                 goto release;
2048
2049         rule->match = *match;
2050
2051         rc = efx_mae_insert_lhs_rule(efx, rule, EFX_TC_PRIO_TC);
2052         if (rc) {
2053                 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
2054                 goto release;
2055         }
2056         netif_dbg(efx, drv, efx->net_dev,
2057                   "Successfully parsed lhs rule (cookie %lx)\n",
2058                   tc->cookie);
2059         return 0;
2060
2061 release:
2062         efx_tc_flower_release_lhs_actions(efx, &rule->lhs_act);
2063         if (!old)
2064                 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &rule->linkage,
2065                                        efx_tc_lhs_rule_ht_params);
2066         kfree(rule);
2067         return rc;
2068 }
2069
2070 static int efx_tc_flower_replace(struct efx_nic *efx,
2071                                  struct net_device *net_dev,
2072                                  struct flow_cls_offload *tc,
2073                                  struct efx_rep *efv)
2074 {
2075         struct flow_rule *fr = flow_cls_offload_flow_rule(tc);
2076         struct netlink_ext_ack *extack = tc->common.extack;
2077         const struct ip_tunnel_info *encap_info = NULL;
2078         struct efx_tc_flow_rule *rule = NULL, *old;
2079         struct efx_tc_mangler_state mung = {};
2080         struct efx_tc_action_set *act = NULL;
2081         const struct flow_action_entry *fa;
2082         struct efx_rep *from_efv, *to_efv;
2083         struct efx_tc_match match;
2084         u32 acts_id;
2085         s64 rc;
2086         int i;
2087
2088         if (!tc_can_offload_extack(efx->net_dev, extack))
2089                 return -EOPNOTSUPP;
2090         if (WARN_ON(!efx->tc))
2091                 return -ENETDOWN;
2092         if (WARN_ON(!efx->tc->up))
2093                 return -ENETDOWN;
2094
2095         from_efv = efx_tc_flower_lookup_efv(efx, net_dev);
2096         if (IS_ERR(from_efv)) {
2097                 /* Not from our PF or representors, so probably a tunnel dev */
2098                 return efx_tc_flower_replace_foreign(efx, net_dev, tc);
2099         }
2100
2101         if (efv != from_efv) {
2102                 /* can't happen */
2103                 NL_SET_ERR_MSG_FMT_MOD(extack, "for %s efv is %snull but from_efv is %snull (can't happen)",
2104                                        netdev_name(net_dev), efv ? "non-" : "",
2105                                        from_efv ? "non-" : "");
2106                 return -EINVAL;
2107         }
2108
2109         /* Parse match */
2110         memset(&match, 0, sizeof(match));
2111         rc = efx_tc_flower_external_mport(efx, from_efv);
2112         if (rc < 0) {
2113                 NL_SET_ERR_MSG_MOD(extack, "Failed to identify ingress m-port");
2114                 return rc;
2115         }
2116         match.value.ingress_port = rc;
2117         match.mask.ingress_port = ~0;
2118         rc = efx_tc_flower_parse_match(efx, fr, &match, extack);
2119         if (rc)
2120                 return rc;
2121         if (efx_tc_match_is_encap(&match.mask)) {
2122                 NL_SET_ERR_MSG_MOD(extack, "Ingress enc_key matches not supported");
2123                 return -EOPNOTSUPP;
2124         }
2125
2126         if (efx_tc_rule_is_lhs_rule(fr, &match))
2127                 return efx_tc_flower_replace_lhs(efx, tc, fr, &match, efv,
2128                                                  net_dev);
2129
2130         /* chain_index 0 is always recirc_id 0 (and does not appear in recirc_ht).
2131          * Conveniently, match.rid == NULL and match.value.recirc_id == 0 owing
2132          * to the initial memset(), so we don't need to do anything in that case.
2133          */
2134         if (tc->common.chain_index) {
2135                 struct efx_tc_recirc_id *rid;
2136
2137                 /* Note regarding passed net_dev:
2138                  * VFreps and PF can share chain namespace, as they have
2139                  * distinct ingress_mports.  So we don't need to burn an
2140                  * extra recirc_id if both use the same chain_index.
2141                  * (Strictly speaking, we could give each VFrep its own
2142                  * recirc_id namespace that doesn't take IDs away from the
2143                  * PF, but that would require a bunch of additional IDAs -
2144                  * one for each representor - and that's not likely to be
2145                  * the main cause of recirc_id exhaustion anyway.)
2146                  */
2147                 rid = efx_tc_get_recirc_id(efx, tc->common.chain_index,
2148                                            efx->net_dev);
2149                 if (IS_ERR(rid)) {
2150                         NL_SET_ERR_MSG_FMT_MOD(extack,
2151                                                "Failed to allocate a hardware recirculation ID for chain_index %u",
2152                                                tc->common.chain_index);
2153                         return PTR_ERR(rid);
2154                 }
2155                 match.rid = rid;
2156                 match.value.recirc_id = rid->fw_id;
2157         }
2158         match.mask.recirc_id = 0xff;
2159
2160         /* AR table can't match on DO_CT (+trk).  But a commonly used pattern is
2161          * +trk+est, which is strictly implied by +est, so rewrite it to that.
2162          */
2163         if (match.mask.ct_state_trk && match.value.ct_state_trk &&
2164             match.mask.ct_state_est && match.value.ct_state_est)
2165                 match.mask.ct_state_trk = 0;
2166         /* Thanks to CT_TCP_FLAGS_INHIBIT, packets with interesting flags could
2167          * match +trk-est (CT_HIT=0) despite being on an established connection.
2168          * So make -est imply -tcp_syn_fin_rst match to ensure these packets
2169          * still hit the software path.
2170          */
2171         if (match.mask.ct_state_est && !match.value.ct_state_est) {
2172                 if (match.value.tcp_syn_fin_rst) {
2173                         /* Can't offload this combination */
2174                         rc = -EOPNOTSUPP;
2175                         goto release;
2176                 }
2177                 match.mask.tcp_syn_fin_rst = true;
2178         }
2179
2180         rc = efx_mae_match_check_caps(efx, &match.mask, extack);
2181         if (rc)
2182                 goto release;
2183
2184         rule = kzalloc(sizeof(*rule), GFP_USER);
2185         if (!rule) {
2186                 rc = -ENOMEM;
2187                 goto release;
2188         }
2189         INIT_LIST_HEAD(&rule->acts.list);
2190         rule->cookie = tc->cookie;
2191         old = rhashtable_lookup_get_insert_fast(&efx->tc->match_action_ht,
2192                                                 &rule->linkage,
2193                                                 efx_tc_match_action_ht_params);
2194         if (IS_ERR(old)) {
2195                 rc = PTR_ERR(old);
2196                 goto release;
2197         } else if (old) {
2198                 netif_dbg(efx, drv, efx->net_dev,
2199                           "Already offloaded rule (cookie %lx)\n", tc->cookie);
2200                 NL_SET_ERR_MSG_MOD(extack, "Rule already offloaded");
2201                 rc = -EEXIST;
2202                 goto release;
2203         }
2204
2205         /* Parse actions */
2206         act = kzalloc(sizeof(*act), GFP_USER);
2207         if (!act) {
2208                 rc = -ENOMEM;
2209                 goto release;
2210         }
2211
2212         /**
2213          * DOC: TC action translation
2214          *
2215          * Actions in TC are sequential and cumulative, with delivery actions
2216          * potentially anywhere in the order.  The EF100 MAE, however, takes
2217          * an 'action set list' consisting of 'action sets', each of which is
2218          * applied to the _original_ packet, and consists of a set of optional
2219          * actions in a fixed order with delivery at the end.
2220          * To translate between these two models, we maintain a 'cursor', @act,
2221          * which describes the cumulative effect of all the packet-mutating
2222          * actions encountered so far; on handling a delivery (mirred or drop)
2223          * action, once the action-set has been inserted into hardware, we
2224          * append @act to the action-set list (@rule->acts); if this is a pipe
2225          * action (mirred mirror) we then allocate a new @act with a copy of
2226          * the cursor state _before_ the delivery action, otherwise we set @act
2227          * to %NULL.
2228          * This ensures that every allocated action-set is either attached to
2229          * @rule->acts or pointed to by @act (and never both), and that only
2230          * those action-sets in @rule->acts exist in hardware.  Consequently,
2231          * in the failure path, @act only needs to be freed in memory, whereas
2232          * for @rule->acts we remove each action-set from hardware before
2233          * freeing it (efx_tc_free_action_set_list()), even if the action-set
2234          * list itself is not in hardware.
2235          */
2236         flow_action_for_each(i, fa, &fr->action) {
2237                 struct efx_tc_action_set save;
2238                 u16 tci;
2239
2240                 if (!act) {
2241                         /* more actions after a non-pipe action */
2242                         NL_SET_ERR_MSG_MOD(extack, "Action follows non-pipe action");
2243                         rc = -EINVAL;
2244                         goto release;
2245                 }
2246
2247                 if ((fa->id == FLOW_ACTION_REDIRECT ||
2248                      fa->id == FLOW_ACTION_MIRRED ||
2249                      fa->id == FLOW_ACTION_DROP) && fa->hw_stats) {
2250                         struct efx_tc_counter_index *ctr;
2251
2252                         /* Currently the only actions that want stats are
2253                          * mirred and gact (ok, shot, trap, goto-chain), which
2254                          * means we want stats just before delivery.  Also,
2255                          * note that tunnel_key set shouldn't change the length
2256                          * — it's only the subsequent mirred that does that,
2257                          * and the stats are taken _before_ the mirred action
2258                          * happens.
2259                          */
2260                         if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_COUNT)) {
2261                                 /* All supported actions that count either steal
2262                                  * (gact shot, mirred redirect) or clone act
2263                                  * (mirred mirror), so we should never get two
2264                                  * count actions on one action_set.
2265                                  */
2266                                 NL_SET_ERR_MSG_MOD(extack, "Count-action conflict (can't happen)");
2267                                 rc = -EOPNOTSUPP;
2268                                 goto release;
2269                         }
2270
2271                         if (!(fa->hw_stats & FLOW_ACTION_HW_STATS_DELAYED)) {
2272                                 NL_SET_ERR_MSG_FMT_MOD(extack, "hw_stats_type %u not supported (only 'delayed')",
2273                                                        fa->hw_stats);
2274                                 rc = -EOPNOTSUPP;
2275                                 goto release;
2276                         }
2277
2278                         ctr = efx_tc_flower_get_counter_index(efx, tc->cookie,
2279                                                               EFX_TC_COUNTER_TYPE_AR);
2280                         if (IS_ERR(ctr)) {
2281                                 rc = PTR_ERR(ctr);
2282                                 NL_SET_ERR_MSG_MOD(extack, "Failed to obtain a counter");
2283                                 goto release;
2284                         }
2285                         act->count = ctr;
2286                         INIT_LIST_HEAD(&act->count_user);
2287                 }
2288
2289                 switch (fa->id) {
2290                 case FLOW_ACTION_DROP:
2291                         rc = efx_mae_alloc_action_set(efx, act);
2292                         if (rc) {
2293                                 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (drop)");
2294                                 goto release;
2295                         }
2296                         list_add_tail(&act->list, &rule->acts.list);
2297                         act = NULL; /* end of the line */
2298                         break;
2299                 case FLOW_ACTION_REDIRECT:
2300                 case FLOW_ACTION_MIRRED:
2301                         save = *act;
2302
2303                         if (encap_info) {
2304                                 struct efx_tc_encap_action *encap;
2305
2306                                 if (!efx_tc_flower_action_order_ok(act,
2307                                                                    EFX_TC_AO_ENCAP)) {
2308                                         rc = -EOPNOTSUPP;
2309                                         NL_SET_ERR_MSG_MOD(extack, "Encap action violates action order");
2310                                         goto release;
2311                                 }
2312                                 encap = efx_tc_flower_create_encap_md(
2313                                                 efx, encap_info, fa->dev, extack);
2314                                 if (IS_ERR_OR_NULL(encap)) {
2315                                         rc = PTR_ERR(encap);
2316                                         if (!rc)
2317                                                 rc = -EIO; /* arbitrary */
2318                                         goto release;
2319                                 }
2320                                 act->encap_md = encap;
2321                                 list_add_tail(&act->encap_user, &encap->users);
2322                                 act->dest_mport = encap->dest_mport;
2323                                 act->deliver = 1;
2324                                 if (act->count && !WARN_ON(!act->count->cnt)) {
2325                                         /* This counter is used by an encap
2326                                          * action, which needs a reference back
2327                                          * so it can prod neighbouring whenever
2328                                          * traffic is seen.
2329                                          */
2330                                         spin_lock_bh(&act->count->cnt->lock);
2331                                         list_add_tail(&act->count_user,
2332                                                       &act->count->cnt->users);
2333                                         spin_unlock_bh(&act->count->cnt->lock);
2334                                 }
2335                                 rc = efx_mae_alloc_action_set(efx, act);
2336                                 if (rc) {
2337                                         NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (encap)");
2338                                         goto release;
2339                                 }
2340                                 list_add_tail(&act->list, &rule->acts.list);
2341                                 act->user = &rule->acts;
2342                                 act = NULL;
2343                                 if (fa->id == FLOW_ACTION_REDIRECT)
2344                                         break; /* end of the line */
2345                                 /* Mirror, so continue on with saved act */
2346                                 save.count = NULL;
2347                                 act = kzalloc(sizeof(*act), GFP_USER);
2348                                 if (!act) {
2349                                         rc = -ENOMEM;
2350                                         goto release;
2351                                 }
2352                                 *act = save;
2353                                 break;
2354                         }
2355
2356                         if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_DELIVER)) {
2357                                 /* can't happen */
2358                                 rc = -EOPNOTSUPP;
2359                                 NL_SET_ERR_MSG_MOD(extack, "Deliver action violates action order (can't happen)");
2360                                 goto release;
2361                         }
2362
2363                         to_efv = efx_tc_flower_lookup_efv(efx, fa->dev);
2364                         if (IS_ERR(to_efv)) {
2365                                 NL_SET_ERR_MSG_MOD(extack, "Mirred egress device not on switch");
2366                                 rc = PTR_ERR(to_efv);
2367                                 goto release;
2368                         }
2369                         rc = efx_tc_flower_external_mport(efx, to_efv);
2370                         if (rc < 0) {
2371                                 NL_SET_ERR_MSG_MOD(extack, "Failed to identify egress m-port");
2372                                 goto release;
2373                         }
2374                         act->dest_mport = rc;
2375                         act->deliver = 1;
2376                         rc = efx_mae_alloc_action_set(efx, act);
2377                         if (rc) {
2378                                 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (mirred)");
2379                                 goto release;
2380                         }
2381                         list_add_tail(&act->list, &rule->acts.list);
2382                         act = NULL;
2383                         if (fa->id == FLOW_ACTION_REDIRECT)
2384                                 break; /* end of the line */
2385                         /* Mirror, so continue on with saved act */
2386                         save.count = NULL;
2387                         act = kzalloc(sizeof(*act), GFP_USER);
2388                         if (!act) {
2389                                 rc = -ENOMEM;
2390                                 goto release;
2391                         }
2392                         *act = save;
2393                         break;
2394                 case FLOW_ACTION_VLAN_POP:
2395                         if (act->vlan_push) {
2396                                 act->vlan_push--;
2397                         } else if (efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_POP)) {
2398                                 act->vlan_pop++;
2399                         } else {
2400                                 NL_SET_ERR_MSG_MOD(extack,
2401                                                    "More than two VLAN pops, or action order violated");
2402                                 rc = -EINVAL;
2403                                 goto release;
2404                         }
2405                         break;
2406                 case FLOW_ACTION_VLAN_PUSH:
2407                         if (!efx_tc_flower_action_order_ok(act, EFX_TC_AO_VLAN_PUSH)) {
2408                                 rc = -EINVAL;
2409                                 NL_SET_ERR_MSG_MOD(extack,
2410                                                    "More than two VLAN pushes, or action order violated");
2411                                 goto release;
2412                         }
2413                         tci = fa->vlan.vid & VLAN_VID_MASK;
2414                         tci |= fa->vlan.prio << VLAN_PRIO_SHIFT;
2415                         act->vlan_tci[act->vlan_push] = cpu_to_be16(tci);
2416                         act->vlan_proto[act->vlan_push] = fa->vlan.proto;
2417                         act->vlan_push++;
2418                         break;
2419                 case FLOW_ACTION_ADD:
2420                         rc = efx_tc_pedit_add(efx, act, fa, extack);
2421                         if (rc < 0)
2422                                 goto release;
2423                         break;
2424                 case FLOW_ACTION_MANGLE:
2425                         rc = efx_tc_mangle(efx, act, fa, &mung, extack, &match);
2426                         if (rc < 0)
2427                                 goto release;
2428                         break;
2429                 case FLOW_ACTION_TUNNEL_ENCAP:
2430                         if (encap_info) {
2431                                 /* Can't specify encap multiple times.
2432                                  * If you want to overwrite an existing
2433                                  * encap_info, use an intervening
2434                                  * FLOW_ACTION_TUNNEL_DECAP to clear it.
2435                                  */
2436                                 NL_SET_ERR_MSG_MOD(extack, "Tunnel key set when already set");
2437                                 rc = -EINVAL;
2438                                 goto release;
2439                         }
2440                         if (!fa->tunnel) {
2441                                 NL_SET_ERR_MSG_MOD(extack, "Tunnel key set is missing key");
2442                                 rc = -EOPNOTSUPP;
2443                                 goto release;
2444                         }
2445                         encap_info = fa->tunnel;
2446                         break;
2447                 case FLOW_ACTION_TUNNEL_DECAP:
2448                         if (encap_info) {
2449                                 encap_info = NULL;
2450                                 break;
2451                         }
2452                         /* Since we don't support enc_key matches on ingress
2453                          * (and if we did there'd be no tunnel-device to give
2454                          * us a type), we can't offload a decap that's not
2455                          * just undoing a previous encap action.
2456                          */
2457                         NL_SET_ERR_MSG_MOD(extack, "Cannot offload tunnel decap action without tunnel device");
2458                         rc = -EOPNOTSUPP;
2459                         goto release;
2460                 case FLOW_ACTION_CT:
2461                         if (fa->ct.action != TCA_CT_ACT_NAT) {
2462                                 rc = -EOPNOTSUPP;
2463                                 NL_SET_ERR_MSG_FMT_MOD(extack, "Can only offload CT 'nat' action in RHS rules, not %d", fa->ct.action);
2464                                 goto release;
2465                         }
2466                         act->do_nat = 1;
2467                         break;
2468                 default:
2469                         NL_SET_ERR_MSG_FMT_MOD(extack, "Unhandled action %u",
2470                                                fa->id);
2471                         rc = -EOPNOTSUPP;
2472                         goto release;
2473                 }
2474         }
2475
2476         rc = efx_tc_incomplete_mangle(&mung, extack);
2477         if (rc < 0)
2478                 goto release;
2479         if (act) {
2480                 /* Not shot/redirected, so deliver to default dest */
2481                 if (from_efv == EFX_EFV_PF)
2482                         /* Rule applies to traffic from the wire,
2483                          * and default dest is thus the PF
2484                          */
2485                         efx_mae_mport_uplink(efx, &act->dest_mport);
2486                 else
2487                         /* Representor, so rule applies to traffic from
2488                          * representee, and default dest is thus the rep.
2489                          * All reps use the same mport for delivery
2490                          */
2491                         efx_mae_mport_mport(efx, efx->tc->reps_mport_id,
2492                                             &act->dest_mport);
2493                 act->deliver = 1;
2494                 rc = efx_mae_alloc_action_set(efx, act);
2495                 if (rc) {
2496                         NL_SET_ERR_MSG_MOD(extack, "Failed to write action set to hw (deliver)");
2497                         goto release;
2498                 }
2499                 list_add_tail(&act->list, &rule->acts.list);
2500                 act = NULL; /* Prevent double-free in error path */
2501         }
2502
2503         netif_dbg(efx, drv, efx->net_dev,
2504                   "Successfully parsed filter (cookie %lx)\n",
2505                   tc->cookie);
2506
2507         rule->match = match;
2508
2509         rc = efx_mae_alloc_action_set_list(efx, &rule->acts);
2510         if (rc) {
2511                 NL_SET_ERR_MSG_MOD(extack, "Failed to write action set list to hw");
2512                 goto release;
2513         }
2514         if (from_efv == EFX_EFV_PF)
2515                 /* PF netdev, so rule applies to traffic from wire */
2516                 rule->fallback = &efx->tc->facts.pf;
2517         else
2518                 /* repdev, so rule applies to traffic from representee */
2519                 rule->fallback = &efx->tc->facts.reps;
2520         if (!efx_tc_check_ready(efx, rule)) {
2521                 netif_dbg(efx, drv, efx->net_dev, "action not ready for hw\n");
2522                 acts_id = rule->fallback->fw_id;
2523         } else {
2524                 netif_dbg(efx, drv, efx->net_dev, "ready for hw\n");
2525                 acts_id = rule->acts.fw_id;
2526         }
2527         rc = efx_mae_insert_rule(efx, &rule->match, EFX_TC_PRIO_TC,
2528                                  acts_id, &rule->fw_id);
2529         if (rc) {
2530                 NL_SET_ERR_MSG_MOD(extack, "Failed to insert rule in hw");
2531                 goto release_acts;
2532         }
2533         return 0;
2534
2535 release_acts:
2536         efx_mae_free_action_set_list(efx, &rule->acts);
2537 release:
2538         /* We failed to insert the rule, so free up any entries we created in
2539          * subsidiary tables.
2540          */
2541         if (match.rid)
2542                 efx_tc_put_recirc_id(efx, match.rid);
2543         if (act)
2544                 efx_tc_free_action_set(efx, act, false);
2545         if (rule) {
2546                 if (!old)
2547                         rhashtable_remove_fast(&efx->tc->match_action_ht,
2548                                                &rule->linkage,
2549                                                efx_tc_match_action_ht_params);
2550                 efx_tc_free_action_set_list(efx, &rule->acts, false);
2551         }
2552         kfree(rule);
2553         return rc;
2554 }
2555
2556 static int efx_tc_flower_destroy(struct efx_nic *efx,
2557                                  struct net_device *net_dev,
2558                                  struct flow_cls_offload *tc)
2559 {
2560         struct netlink_ext_ack *extack = tc->common.extack;
2561         struct efx_tc_lhs_rule *lhs_rule;
2562         struct efx_tc_flow_rule *rule;
2563
2564         lhs_rule = rhashtable_lookup_fast(&efx->tc->lhs_rule_ht, &tc->cookie,
2565                                           efx_tc_lhs_rule_ht_params);
2566         if (lhs_rule) {
2567                 /* Remove it from HW */
2568                 efx_mae_remove_lhs_rule(efx, lhs_rule);
2569                 /* Delete it from SW */
2570                 efx_tc_flower_release_lhs_actions(efx, &lhs_rule->lhs_act);
2571                 rhashtable_remove_fast(&efx->tc->lhs_rule_ht, &lhs_rule->linkage,
2572                                        efx_tc_lhs_rule_ht_params);
2573                 if (lhs_rule->match.encap)
2574                         efx_tc_flower_release_encap_match(efx, lhs_rule->match.encap);
2575                 netif_dbg(efx, drv, efx->net_dev, "Removed (lhs) filter %lx\n",
2576                           lhs_rule->cookie);
2577                 kfree(lhs_rule);
2578                 return 0;
2579         }
2580
2581         rule = rhashtable_lookup_fast(&efx->tc->match_action_ht, &tc->cookie,
2582                                       efx_tc_match_action_ht_params);
2583         if (!rule) {
2584                 /* Only log a message if we're the ingress device.  Otherwise
2585                  * it's a foreign filter and we might just not have been
2586                  * interested (e.g. we might not have been the egress device
2587                  * either).
2588                  */
2589                 if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))
2590                         netif_warn(efx, drv, efx->net_dev,
2591                                    "Filter %lx not found to remove\n", tc->cookie);
2592                 NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
2593                 return -ENOENT;
2594         }
2595
2596         /* Remove it from HW */
2597         efx_tc_delete_rule(efx, rule);
2598         /* Delete it from SW */
2599         rhashtable_remove_fast(&efx->tc->match_action_ht, &rule->linkage,
2600                                efx_tc_match_action_ht_params);
2601         netif_dbg(efx, drv, efx->net_dev, "Removed filter %lx\n", rule->cookie);
2602         kfree(rule);
2603         return 0;
2604 }
2605
2606 static int efx_tc_flower_stats(struct efx_nic *efx, struct net_device *net_dev,
2607                                struct flow_cls_offload *tc)
2608 {
2609         struct netlink_ext_ack *extack = tc->common.extack;
2610         struct efx_tc_counter_index *ctr;
2611         struct efx_tc_counter *cnt;
2612         u64 packets, bytes;
2613
2614         ctr = efx_tc_flower_find_counter_index(efx, tc->cookie);
2615         if (!ctr) {
2616                 /* See comment in efx_tc_flower_destroy() */
2617                 if (!IS_ERR(efx_tc_flower_lookup_efv(efx, net_dev)))
2618                         if (net_ratelimit())
2619                                 netif_warn(efx, drv, efx->net_dev,
2620                                            "Filter %lx not found for stats\n",
2621                                            tc->cookie);
2622                 NL_SET_ERR_MSG_MOD(extack, "Flow cookie not found in offloaded rules");
2623                 return -ENOENT;
2624         }
2625         if (WARN_ON(!ctr->cnt)) /* can't happen */
2626                 return -EIO;
2627         cnt = ctr->cnt;
2628
2629         spin_lock_bh(&cnt->lock);
2630         /* Report only new pkts/bytes since last time TC asked */
2631         packets = cnt->packets;
2632         bytes = cnt->bytes;
2633         flow_stats_update(&tc->stats, bytes - cnt->old_bytes,
2634                           packets - cnt->old_packets, 0, cnt->touched,
2635                           FLOW_ACTION_HW_STATS_DELAYED);
2636         cnt->old_packets = packets;
2637         cnt->old_bytes = bytes;
2638         spin_unlock_bh(&cnt->lock);
2639         return 0;
2640 }
2641
2642 int efx_tc_flower(struct efx_nic *efx, struct net_device *net_dev,
2643                   struct flow_cls_offload *tc, struct efx_rep *efv)
2644 {
2645         int rc;
2646
2647         if (!efx->tc)
2648                 return -EOPNOTSUPP;
2649
2650         mutex_lock(&efx->tc->mutex);
2651         switch (tc->command) {
2652         case FLOW_CLS_REPLACE:
2653                 rc = efx_tc_flower_replace(efx, net_dev, tc, efv);
2654                 break;
2655         case FLOW_CLS_DESTROY:
2656                 rc = efx_tc_flower_destroy(efx, net_dev, tc);
2657                 break;
2658         case FLOW_CLS_STATS:
2659                 rc = efx_tc_flower_stats(efx, net_dev, tc);
2660                 break;
2661         default:
2662                 rc = -EOPNOTSUPP;
2663                 break;
2664         }
2665         mutex_unlock(&efx->tc->mutex);
2666         return rc;
2667 }
2668
2669 static int efx_tc_configure_default_rule(struct efx_nic *efx, u32 ing_port,
2670                                          u32 eg_port, struct efx_tc_flow_rule *rule)
2671 {
2672         struct efx_tc_action_set_list *acts = &rule->acts;
2673         struct efx_tc_match *match = &rule->match;
2674         struct efx_tc_action_set *act;
2675         int rc;
2676
2677         match->value.ingress_port = ing_port;
2678         match->mask.ingress_port = ~0;
2679         act = kzalloc(sizeof(*act), GFP_KERNEL);
2680         if (!act)
2681                 return -ENOMEM;
2682         act->deliver = 1;
2683         act->dest_mport = eg_port;
2684         rc = efx_mae_alloc_action_set(efx, act);
2685         if (rc)
2686                 goto fail1;
2687         EFX_WARN_ON_PARANOID(!list_empty(&acts->list));
2688         list_add_tail(&act->list, &acts->list);
2689         rc = efx_mae_alloc_action_set_list(efx, acts);
2690         if (rc)
2691                 goto fail2;
2692         rc = efx_mae_insert_rule(efx, match, EFX_TC_PRIO_DFLT,
2693                                  acts->fw_id, &rule->fw_id);
2694         if (rc)
2695                 goto fail3;
2696         return 0;
2697 fail3:
2698         efx_mae_free_action_set_list(efx, acts);
2699 fail2:
2700         list_del(&act->list);
2701         efx_mae_free_action_set(efx, act->fw_id);
2702 fail1:
2703         kfree(act);
2704         return rc;
2705 }
2706
2707 static int efx_tc_configure_default_rule_pf(struct efx_nic *efx)
2708 {
2709         struct efx_tc_flow_rule *rule = &efx->tc->dflt.pf;
2710         u32 ing_port, eg_port;
2711
2712         efx_mae_mport_uplink(efx, &ing_port);
2713         efx_mae_mport_wire(efx, &eg_port);
2714         return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
2715 }
2716
2717 static int efx_tc_configure_default_rule_wire(struct efx_nic *efx)
2718 {
2719         struct efx_tc_flow_rule *rule = &efx->tc->dflt.wire;
2720         u32 ing_port, eg_port;
2721
2722         efx_mae_mport_wire(efx, &ing_port);
2723         efx_mae_mport_uplink(efx, &eg_port);
2724         return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
2725 }
2726
2727 int efx_tc_configure_default_rule_rep(struct efx_rep *efv)
2728 {
2729         struct efx_tc_flow_rule *rule = &efv->dflt;
2730         struct efx_nic *efx = efv->parent;
2731         u32 ing_port, eg_port;
2732
2733         efx_mae_mport_mport(efx, efv->mport, &ing_port);
2734         efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port);
2735         return efx_tc_configure_default_rule(efx, ing_port, eg_port, rule);
2736 }
2737
2738 void efx_tc_deconfigure_default_rule(struct efx_nic *efx,
2739                                      struct efx_tc_flow_rule *rule)
2740 {
2741         if (rule->fw_id != MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL)
2742                 efx_tc_delete_rule(efx, rule);
2743         rule->fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
2744 }
2745
2746 static int efx_tc_configure_fallback_acts(struct efx_nic *efx, u32 eg_port,
2747                                           struct efx_tc_action_set_list *acts)
2748 {
2749         struct efx_tc_action_set *act;
2750         int rc;
2751
2752         act = kzalloc(sizeof(*act), GFP_KERNEL);
2753         if (!act)
2754                 return -ENOMEM;
2755         act->deliver = 1;
2756         act->dest_mport = eg_port;
2757         rc = efx_mae_alloc_action_set(efx, act);
2758         if (rc)
2759                 goto fail1;
2760         EFX_WARN_ON_PARANOID(!list_empty(&acts->list));
2761         list_add_tail(&act->list, &acts->list);
2762         rc = efx_mae_alloc_action_set_list(efx, acts);
2763         if (rc)
2764                 goto fail2;
2765         return 0;
2766 fail2:
2767         list_del(&act->list);
2768         efx_mae_free_action_set(efx, act->fw_id);
2769 fail1:
2770         kfree(act);
2771         return rc;
2772 }
2773
2774 static int efx_tc_configure_fallback_acts_pf(struct efx_nic *efx)
2775 {
2776         struct efx_tc_action_set_list *acts = &efx->tc->facts.pf;
2777         u32 eg_port;
2778
2779         efx_mae_mport_uplink(efx, &eg_port);
2780         return efx_tc_configure_fallback_acts(efx, eg_port, acts);
2781 }
2782
2783 static int efx_tc_configure_fallback_acts_reps(struct efx_nic *efx)
2784 {
2785         struct efx_tc_action_set_list *acts = &efx->tc->facts.reps;
2786         u32 eg_port;
2787
2788         efx_mae_mport_mport(efx, efx->tc->reps_mport_id, &eg_port);
2789         return efx_tc_configure_fallback_acts(efx, eg_port, acts);
2790 }
2791
2792 static void efx_tc_deconfigure_fallback_acts(struct efx_nic *efx,
2793                                              struct efx_tc_action_set_list *acts)
2794 {
2795         efx_tc_free_action_set_list(efx, acts, true);
2796 }
2797
2798 static int efx_tc_configure_rep_mport(struct efx_nic *efx)
2799 {
2800         u32 rep_mport_label;
2801         int rc;
2802
2803         rc = efx_mae_allocate_mport(efx, &efx->tc->reps_mport_id, &rep_mport_label);
2804         if (rc)
2805                 return rc;
2806         pci_dbg(efx->pci_dev, "created rep mport 0x%08x (0x%04x)\n",
2807                 efx->tc->reps_mport_id, rep_mport_label);
2808         /* Use mport *selector* as vport ID */
2809         efx_mae_mport_mport(efx, efx->tc->reps_mport_id,
2810                             &efx->tc->reps_mport_vport_id);
2811         return 0;
2812 }
2813
2814 static void efx_tc_deconfigure_rep_mport(struct efx_nic *efx)
2815 {
2816         efx_mae_free_mport(efx, efx->tc->reps_mport_id);
2817         efx->tc->reps_mport_id = MAE_MPORT_SELECTOR_NULL;
2818 }
2819
2820 int efx_tc_insert_rep_filters(struct efx_nic *efx)
2821 {
2822         struct efx_filter_spec promisc, allmulti;
2823         int rc;
2824
2825         if (efx->type->is_vf)
2826                 return 0;
2827         if (!efx->tc)
2828                 return 0;
2829         efx_filter_init_rx(&promisc, EFX_FILTER_PRI_REQUIRED, 0, 0);
2830         efx_filter_set_uc_def(&promisc);
2831         efx_filter_set_vport_id(&promisc, efx->tc->reps_mport_vport_id);
2832         rc = efx_filter_insert_filter(efx, &promisc, false);
2833         if (rc < 0)
2834                 return rc;
2835         efx->tc->reps_filter_uc = rc;
2836         efx_filter_init_rx(&allmulti, EFX_FILTER_PRI_REQUIRED, 0, 0);
2837         efx_filter_set_mc_def(&allmulti);
2838         efx_filter_set_vport_id(&allmulti, efx->tc->reps_mport_vport_id);
2839         rc = efx_filter_insert_filter(efx, &allmulti, false);
2840         if (rc < 0)
2841                 return rc;
2842         efx->tc->reps_filter_mc = rc;
2843         return 0;
2844 }
2845
2846 void efx_tc_remove_rep_filters(struct efx_nic *efx)
2847 {
2848         if (efx->type->is_vf)
2849                 return;
2850         if (!efx->tc)
2851                 return;
2852         if (efx->tc->reps_filter_mc >= 0)
2853                 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_mc);
2854         efx->tc->reps_filter_mc = -1;
2855         if (efx->tc->reps_filter_uc >= 0)
2856                 efx_filter_remove_id_safe(efx, EFX_FILTER_PRI_REQUIRED, efx->tc->reps_filter_uc);
2857         efx->tc->reps_filter_uc = -1;
2858 }
2859
2860 int efx_init_tc(struct efx_nic *efx)
2861 {
2862         int rc;
2863
2864         rc = efx_mae_get_caps(efx, efx->tc->caps);
2865         if (rc)
2866                 return rc;
2867         if (efx->tc->caps->match_field_count > MAE_NUM_FIELDS)
2868                 /* Firmware supports some match fields the driver doesn't know
2869                  * about.  Not fatal, unless any of those fields are required
2870                  * (MAE_FIELD_SUPPORTED_MATCH_ALWAYS) but if so we don't know.
2871                  */
2872                 netif_warn(efx, probe, efx->net_dev,
2873                            "FW reports additional match fields %u\n",
2874                            efx->tc->caps->match_field_count);
2875         if (efx->tc->caps->action_prios < EFX_TC_PRIO__NUM) {
2876                 netif_err(efx, probe, efx->net_dev,
2877                           "Too few action prios supported (have %u, need %u)\n",
2878                           efx->tc->caps->action_prios, EFX_TC_PRIO__NUM);
2879                 return -EIO;
2880         }
2881         rc = efx_tc_configure_default_rule_pf(efx);
2882         if (rc)
2883                 return rc;
2884         rc = efx_tc_configure_default_rule_wire(efx);
2885         if (rc)
2886                 return rc;
2887         rc = efx_tc_configure_rep_mport(efx);
2888         if (rc)
2889                 return rc;
2890         rc = efx_tc_configure_fallback_acts_pf(efx);
2891         if (rc)
2892                 return rc;
2893         rc = efx_tc_configure_fallback_acts_reps(efx);
2894         if (rc)
2895                 return rc;
2896         rc = efx_mae_get_tables(efx);
2897         if (rc)
2898                 return rc;
2899         rc = flow_indr_dev_register(efx_tc_indr_setup_cb, efx);
2900         if (rc)
2901                 goto out_free;
2902         efx->tc->up = true;
2903         return 0;
2904 out_free:
2905         efx_mae_free_tables(efx);
2906         return rc;
2907 }
2908
2909 void efx_fini_tc(struct efx_nic *efx)
2910 {
2911         /* We can get called even if efx_init_struct_tc() failed */
2912         if (!efx->tc)
2913                 return;
2914         if (efx->tc->up)
2915                 flow_indr_dev_unregister(efx_tc_indr_setup_cb, efx, efx_tc_block_unbind);
2916         efx_tc_deconfigure_rep_mport(efx);
2917         efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.pf);
2918         efx_tc_deconfigure_default_rule(efx, &efx->tc->dflt.wire);
2919         efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.pf);
2920         efx_tc_deconfigure_fallback_acts(efx, &efx->tc->facts.reps);
2921         efx->tc->up = false;
2922         efx_mae_free_tables(efx);
2923 }
2924
2925 /* At teardown time, all TC filter rules (and thus all resources they created)
2926  * should already have been removed.  If we find any in our hashtables, make a
2927  * cursory attempt to clean up the software side.
2928  */
2929 static void efx_tc_encap_match_free(void *ptr, void *__unused)
2930 {
2931         struct efx_tc_encap_match *encap = ptr;
2932
2933         WARN_ON(refcount_read(&encap->ref));
2934         kfree(encap);
2935 }
2936
2937 static void efx_tc_recirc_free(void *ptr, void *arg)
2938 {
2939         struct efx_tc_recirc_id *rid = ptr;
2940         struct efx_nic *efx = arg;
2941
2942         WARN_ON(refcount_read(&rid->ref));
2943         ida_free(&efx->tc->recirc_ida, rid->fw_id);
2944         kfree(rid);
2945 }
2946
2947 static void efx_tc_lhs_free(void *ptr, void *arg)
2948 {
2949         struct efx_tc_lhs_rule *rule = ptr;
2950         struct efx_nic *efx = arg;
2951
2952         netif_err(efx, drv, efx->net_dev,
2953                   "tc lhs_rule %lx still present at teardown, removing\n",
2954                   rule->cookie);
2955
2956         if (rule->lhs_act.zone)
2957                 efx_tc_ct_unregister_zone(efx, rule->lhs_act.zone);
2958         if (rule->lhs_act.count)
2959                 efx_tc_flower_put_counter_index(efx, rule->lhs_act.count);
2960         efx_mae_remove_lhs_rule(efx, rule);
2961
2962         kfree(rule);
2963 }
2964
2965 static void efx_tc_mac_free(void *ptr, void *__unused)
2966 {
2967         struct efx_tc_mac_pedit_action *ped = ptr;
2968
2969         WARN_ON(refcount_read(&ped->ref));
2970         kfree(ped);
2971 }
2972
2973 static void efx_tc_flow_free(void *ptr, void *arg)
2974 {
2975         struct efx_tc_flow_rule *rule = ptr;
2976         struct efx_nic *efx = arg;
2977
2978         netif_err(efx, drv, efx->net_dev,
2979                   "tc rule %lx still present at teardown, removing\n",
2980                   rule->cookie);
2981
2982         /* Also releases entries in subsidiary tables */
2983         efx_tc_delete_rule(efx, rule);
2984
2985         kfree(rule);
2986 }
2987
2988 int efx_init_struct_tc(struct efx_nic *efx)
2989 {
2990         int rc;
2991
2992         if (efx->type->is_vf)
2993                 return 0;
2994
2995         efx->tc = kzalloc(sizeof(*efx->tc), GFP_KERNEL);
2996         if (!efx->tc)
2997                 return -ENOMEM;
2998         efx->tc->caps = kzalloc(sizeof(struct mae_caps), GFP_KERNEL);
2999         if (!efx->tc->caps) {
3000                 rc = -ENOMEM;
3001                 goto fail_alloc_caps;
3002         }
3003         INIT_LIST_HEAD(&efx->tc->block_list);
3004
3005         mutex_init(&efx->tc->mutex);
3006         init_waitqueue_head(&efx->tc->flush_wq);
3007         rc = efx_tc_init_encap_actions(efx);
3008         if (rc < 0)
3009                 goto fail_encap_actions;
3010         rc = efx_tc_init_counters(efx);
3011         if (rc < 0)
3012                 goto fail_counters;
3013         rc = rhashtable_init(&efx->tc->mac_ht, &efx_tc_mac_ht_params);
3014         if (rc < 0)
3015                 goto fail_mac_ht;
3016         rc = rhashtable_init(&efx->tc->encap_match_ht, &efx_tc_encap_match_ht_params);
3017         if (rc < 0)
3018                 goto fail_encap_match_ht;
3019         rc = rhashtable_init(&efx->tc->match_action_ht, &efx_tc_match_action_ht_params);
3020         if (rc < 0)
3021                 goto fail_match_action_ht;
3022         rc = rhashtable_init(&efx->tc->lhs_rule_ht, &efx_tc_lhs_rule_ht_params);
3023         if (rc < 0)
3024                 goto fail_lhs_rule_ht;
3025         rc = efx_tc_init_conntrack(efx);
3026         if (rc < 0)
3027                 goto fail_conntrack;
3028         rc = rhashtable_init(&efx->tc->recirc_ht, &efx_tc_recirc_ht_params);
3029         if (rc < 0)
3030                 goto fail_recirc_ht;
3031         ida_init(&efx->tc->recirc_ida);
3032         efx->tc->reps_filter_uc = -1;
3033         efx->tc->reps_filter_mc = -1;
3034         INIT_LIST_HEAD(&efx->tc->dflt.pf.acts.list);
3035         efx->tc->dflt.pf.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
3036         INIT_LIST_HEAD(&efx->tc->dflt.wire.acts.list);
3037         efx->tc->dflt.wire.fw_id = MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL;
3038         INIT_LIST_HEAD(&efx->tc->facts.pf.list);
3039         efx->tc->facts.pf.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL;
3040         INIT_LIST_HEAD(&efx->tc->facts.reps.list);
3041         efx->tc->facts.reps.fw_id = MC_CMD_MAE_ACTION_SET_ALLOC_OUT_ACTION_SET_ID_NULL;
3042         efx->extra_channel_type[EFX_EXTRA_CHANNEL_TC] = &efx_tc_channel_type;
3043         return 0;
3044 fail_recirc_ht:
3045         efx_tc_destroy_conntrack(efx);
3046 fail_conntrack:
3047         rhashtable_destroy(&efx->tc->lhs_rule_ht);
3048 fail_lhs_rule_ht:
3049         rhashtable_destroy(&efx->tc->match_action_ht);
3050 fail_match_action_ht:
3051         rhashtable_destroy(&efx->tc->encap_match_ht);
3052 fail_encap_match_ht:
3053         rhashtable_destroy(&efx->tc->mac_ht);
3054 fail_mac_ht:
3055         efx_tc_destroy_counters(efx);
3056 fail_counters:
3057         efx_tc_destroy_encap_actions(efx);
3058 fail_encap_actions:
3059         mutex_destroy(&efx->tc->mutex);
3060         kfree(efx->tc->caps);
3061 fail_alloc_caps:
3062         kfree(efx->tc);
3063         efx->tc = NULL;
3064         return rc;
3065 }
3066
3067 void efx_fini_struct_tc(struct efx_nic *efx)
3068 {
3069         if (!efx->tc)
3070                 return;
3071
3072         mutex_lock(&efx->tc->mutex);
3073         EFX_WARN_ON_PARANOID(efx->tc->dflt.pf.fw_id !=
3074                              MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
3075         EFX_WARN_ON_PARANOID(efx->tc->dflt.wire.fw_id !=
3076                              MC_CMD_MAE_ACTION_RULE_INSERT_OUT_ACTION_RULE_ID_NULL);
3077         EFX_WARN_ON_PARANOID(efx->tc->facts.pf.fw_id !=
3078                              MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL);
3079         EFX_WARN_ON_PARANOID(efx->tc->facts.reps.fw_id !=
3080                              MC_CMD_MAE_ACTION_SET_LIST_ALLOC_OUT_ACTION_SET_LIST_ID_NULL);
3081         rhashtable_free_and_destroy(&efx->tc->lhs_rule_ht, efx_tc_lhs_free, efx);
3082         rhashtable_free_and_destroy(&efx->tc->match_action_ht, efx_tc_flow_free,
3083                                     efx);
3084         rhashtable_free_and_destroy(&efx->tc->encap_match_ht,
3085                                     efx_tc_encap_match_free, NULL);
3086         efx_tc_fini_conntrack(efx);
3087         rhashtable_free_and_destroy(&efx->tc->recirc_ht, efx_tc_recirc_free, efx);
3088         WARN_ON(!ida_is_empty(&efx->tc->recirc_ida));
3089         ida_destroy(&efx->tc->recirc_ida);
3090         rhashtable_free_and_destroy(&efx->tc->mac_ht, efx_tc_mac_free, NULL);
3091         efx_tc_fini_counters(efx);
3092         efx_tc_fini_encap_actions(efx);
3093         mutex_unlock(&efx->tc->mutex);
3094         mutex_destroy(&efx->tc->mutex);
3095         kfree(efx->tc->caps);
3096         kfree(efx->tc);
3097         efx->tc = NULL;
3098 }