drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_atcam.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/err.h>
   6 #include <linux/errno.h>
   7 #include <linux/gfp.h>
   8 #include <linux/refcount.h>
   9 #include <linux/rhashtable.h>
  10 #define CREATE_TRACE_POINTS
  11 #include <trace/events/mlxsw.h>
  12
  13 #include "reg.h"
  14 #include "core.h"
  15 #include "spectrum.h"
  16 #include "spectrum_acl_tcam.h"
  17 #include "core_acl_flex_keys.h"
  18
  19 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START    0
  20 #define MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END      5
  21
  22 struct mlxsw_sp_acl_atcam_lkey_id_ht_key {
  23         char enc_key[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; /* MSB blocks */
  24         u8 erp_id;
  25 };
  26
  27 struct mlxsw_sp_acl_atcam_lkey_id {
  28         struct rhash_head ht_node;
  29         struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key;
  30         refcount_t refcnt;
  31         u32 id;
  32 };
  33
  34 struct mlxsw_sp_acl_atcam_region_ops {
  35         int (*init)(struct mlxsw_sp_acl_atcam_region *aregion);
  36         void (*fini)(struct mlxsw_sp_acl_atcam_region *aregion);
  37         struct mlxsw_sp_acl_atcam_lkey_id *
  38                 (*lkey_id_get)(struct mlxsw_sp_acl_atcam_region *aregion,
  39                                char *enc_key, u8 erp_id);
  40         void (*lkey_id_put)(struct mlxsw_sp_acl_atcam_region *aregion,
  41                             struct mlxsw_sp_acl_atcam_lkey_id *lkey_id);
  42 };
  43
  44 struct mlxsw_sp_acl_atcam_region_generic {
  45         struct mlxsw_sp_acl_atcam_lkey_id dummy_lkey_id;
  46 };
  47
  48 struct mlxsw_sp_acl_atcam_region_12kb {
  49         struct rhashtable lkey_ht;
  50         unsigned int max_lkey_id;
  51         unsigned long *used_lkey_id;
  52 };
  53
  54 static const struct rhashtable_params mlxsw_sp_acl_atcam_lkey_id_ht_params = {
  55         .key_len = sizeof(struct mlxsw_sp_acl_atcam_lkey_id_ht_key),
  56         .key_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_key),
  57         .head_offset = offsetof(struct mlxsw_sp_acl_atcam_lkey_id, ht_node),
  58 };
  59
  60 static const struct rhashtable_params mlxsw_sp_acl_atcam_entries_ht_params = {
  61         .key_len = sizeof(struct mlxsw_sp_acl_atcam_entry_ht_key),
  62         .key_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_key),
  63         .head_offset = offsetof(struct mlxsw_sp_acl_atcam_entry, ht_node),
  64 };
  65
  66 static bool
  67 mlxsw_sp_acl_atcam_is_centry(const struct mlxsw_sp_acl_atcam_entry *aentry)
  68 {
  69         return mlxsw_sp_acl_erp_mask_is_ctcam(aentry->erp_mask);
  70 }
  71
  72 static int
  73 mlxsw_sp_acl_atcam_region_generic_init(struct mlxsw_sp_acl_atcam_region *aregion)
  74 {
  75         struct mlxsw_sp_acl_atcam_region_generic *region_generic;
  76
  77         region_generic = kzalloc(sizeof(*region_generic), GFP_KERNEL);
  78         if (!region_generic)
  79                 return -ENOMEM;
  80
  81         refcount_set(&region_generic->dummy_lkey_id.refcnt, 1);
  82         aregion->priv = region_generic;
  83
  84         return 0;
  85 }
  86
  87 static void
  88 mlxsw_sp_acl_atcam_region_generic_fini(struct mlxsw_sp_acl_atcam_region *aregion)
  89 {
  90         kfree(aregion->priv);
  91 }
  92
  93 static struct mlxsw_sp_acl_atcam_lkey_id *
  94 mlxsw_sp_acl_atcam_generic_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
  95                                        char *enc_key, u8 erp_id)
  96 {
  97         struct mlxsw_sp_acl_atcam_region_generic *region_generic;
  98
  99         region_generic = aregion->priv;
 100         return &region_generic->dummy_lkey_id;
 101 }
 102
 103 static void
 104 mlxsw_sp_acl_atcam_generic_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
 105                                        struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
 106 {
 107 }
 108
 109 static const struct mlxsw_sp_acl_atcam_region_ops
 110 mlxsw_sp_acl_atcam_region_generic_ops = {
 111         .init           = mlxsw_sp_acl_atcam_region_generic_init,
 112         .fini           = mlxsw_sp_acl_atcam_region_generic_fini,
 113         .lkey_id_get    = mlxsw_sp_acl_atcam_generic_lkey_id_get,
 114         .lkey_id_put    = mlxsw_sp_acl_atcam_generic_lkey_id_put,
 115 };
 116
 117 static int
 118 mlxsw_sp_acl_atcam_region_12kb_init(struct mlxsw_sp_acl_atcam_region *aregion)
 119 {
 120         struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp;
 121         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb;
 122         u64 max_lkey_id;
 123         int err;
 124
 125         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID))
 126                 return -EIO;
 127
 128         max_lkey_id = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_LARGE_KEY_ID);
 129         region_12kb = kzalloc(sizeof(*region_12kb), GFP_KERNEL);
 130         if (!region_12kb)
 131                 return -ENOMEM;
 132
 133         region_12kb->used_lkey_id = bitmap_zalloc(max_lkey_id, GFP_KERNEL);
 134         if (!region_12kb->used_lkey_id) {
 135                 err = -ENOMEM;
 136                 goto err_used_lkey_id_alloc;
 137         }
 138
 139         err = rhashtable_init(&region_12kb->lkey_ht,
 140                               &mlxsw_sp_acl_atcam_lkey_id_ht_params);
 141         if (err)
 142                 goto err_rhashtable_init;
 143
 144         region_12kb->max_lkey_id = max_lkey_id;
 145         aregion->priv = region_12kb;
 146
 147         return 0;
 148
 149 err_rhashtable_init:
 150         bitmap_free(region_12kb->used_lkey_id);
 151 err_used_lkey_id_alloc:
 152         kfree(region_12kb);
 153         return err;
 154 }
 155
 156 static void
 157 mlxsw_sp_acl_atcam_region_12kb_fini(struct mlxsw_sp_acl_atcam_region *aregion)
 158 {
 159         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
 160
 161         rhashtable_destroy(&region_12kb->lkey_ht);
 162         bitmap_free(region_12kb->used_lkey_id);
 163         kfree(region_12kb);
 164 }
 165
 166 static struct mlxsw_sp_acl_atcam_lkey_id *
 167 mlxsw_sp_acl_atcam_lkey_id_create(struct mlxsw_sp_acl_atcam_region *aregion,
 168                                   struct mlxsw_sp_acl_atcam_lkey_id_ht_key *ht_key)
 169 {
 170         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
 171         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
 172         u32 id;
 173         int err;
 174
 175         id = find_first_zero_bit(region_12kb->used_lkey_id,
 176                                  region_12kb->max_lkey_id);
 177         if (id < region_12kb->max_lkey_id)
 178                 __set_bit(id, region_12kb->used_lkey_id);
 179         else
 180                 return ERR_PTR(-ENOBUFS);
 181
 182         lkey_id = kzalloc(sizeof(*lkey_id), GFP_KERNEL);
 183         if (!lkey_id) {
 184                 err = -ENOMEM;
 185                 goto err_lkey_id_alloc;
 186         }
 187
 188         lkey_id->id = id;
 189         memcpy(&lkey_id->ht_key, ht_key, sizeof(*ht_key));
 190         refcount_set(&lkey_id->refcnt, 1);
 191
 192         err = rhashtable_insert_fast(&region_12kb->lkey_ht,
 193                                      &lkey_id->ht_node,
 194                                      mlxsw_sp_acl_atcam_lkey_id_ht_params);
 195         if (err)
 196                 goto err_rhashtable_insert;
 197
 198         return lkey_id;
 199
 200 err_rhashtable_insert:
 201         kfree(lkey_id);
 202 err_lkey_id_alloc:
 203         __clear_bit(id, region_12kb->used_lkey_id);
 204         return ERR_PTR(err);
 205 }
 206
 207 static void
 208 mlxsw_sp_acl_atcam_lkey_id_destroy(struct mlxsw_sp_acl_atcam_region *aregion,
 209                                    struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
 210 {
 211         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
 212         u32 id = lkey_id->id;
 213
 214         rhashtable_remove_fast(&region_12kb->lkey_ht, &lkey_id->ht_node,
 215                                mlxsw_sp_acl_atcam_lkey_id_ht_params);
 216         kfree(lkey_id);
 217         __clear_bit(id, region_12kb->used_lkey_id);
 218 }
 219
 220 static struct mlxsw_sp_acl_atcam_lkey_id *
 221 mlxsw_sp_acl_atcam_12kb_lkey_id_get(struct mlxsw_sp_acl_atcam_region *aregion,
 222                                     char *enc_key, u8 erp_id)
 223 {
 224         struct mlxsw_sp_acl_atcam_region_12kb *region_12kb = aregion->priv;
 225         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 226         struct mlxsw_sp_acl_atcam_lkey_id_ht_key ht_key = {{ 0 } };
 227         struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp;
 228         struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
 229         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
 230
 231         memcpy(ht_key.enc_key, enc_key, sizeof(ht_key.enc_key));
 232         mlxsw_afk_clear(afk, ht_key.enc_key,
 233                         MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_START,
 234                         MLXSW_SP_ACL_ATCAM_LKEY_ID_BLOCK_CLEAR_END);
 235         ht_key.erp_id = erp_id;
 236         lkey_id = rhashtable_lookup_fast(&region_12kb->lkey_ht, &ht_key,
 237                                          mlxsw_sp_acl_atcam_lkey_id_ht_params);
 238         if (lkey_id) {
 239                 refcount_inc(&lkey_id->refcnt);
 240                 return lkey_id;
 241         }
 242
 243         return mlxsw_sp_acl_atcam_lkey_id_create(aregion, &ht_key);
 244 }
 245
 246 static void
 247 mlxsw_sp_acl_atcam_12kb_lkey_id_put(struct mlxsw_sp_acl_atcam_region *aregion,
 248                                     struct mlxsw_sp_acl_atcam_lkey_id *lkey_id)
 249 {
 250         if (refcount_dec_and_test(&lkey_id->refcnt))
 251                 mlxsw_sp_acl_atcam_lkey_id_destroy(aregion, lkey_id);
 252 }
 253
 254 static const struct mlxsw_sp_acl_atcam_region_ops
 255 mlxsw_sp_acl_atcam_region_12kb_ops = {
 256         .init           = mlxsw_sp_acl_atcam_region_12kb_init,
 257         .fini           = mlxsw_sp_acl_atcam_region_12kb_fini,
 258         .lkey_id_get    = mlxsw_sp_acl_atcam_12kb_lkey_id_get,
 259         .lkey_id_put    = mlxsw_sp_acl_atcam_12kb_lkey_id_put,
 260 };
 261
 262 static const struct mlxsw_sp_acl_atcam_region_ops *
 263 mlxsw_sp_acl_atcam_region_ops_arr[] = {
 264         [MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB]    =
 265                 &mlxsw_sp_acl_atcam_region_generic_ops,
 266         [MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB]    =
 267                 &mlxsw_sp_acl_atcam_region_generic_ops,
 268         [MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB]    =
 269                 &mlxsw_sp_acl_atcam_region_generic_ops,
 270         [MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB]   =
 271                 &mlxsw_sp_acl_atcam_region_12kb_ops,
 272 };
 273
 274 int mlxsw_sp_acl_atcam_region_associate(struct mlxsw_sp *mlxsw_sp,
 275                                         u16 region_id)
 276 {
 277         char perar_pl[MLXSW_REG_PERAR_LEN];
 278         /* For now, just assume that every region has 12 key blocks */
 279         u16 hw_region = region_id * 3;
 280         u64 max_regions;
 281
 282         max_regions = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_MAX_REGIONS);
 283         if (hw_region >= max_regions)
 284                 return -ENOBUFS;
 285
 286         mlxsw_reg_perar_pack(perar_pl, region_id, hw_region);
 287         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perar), perar_pl);
 288 }
 289
 290 static void
 291 mlxsw_sp_acl_atcam_region_type_init(struct mlxsw_sp_acl_atcam_region *aregion)
 292 {
 293         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 294         enum mlxsw_sp_acl_atcam_region_type region_type;
 295         unsigned int blocks_count;
 296
 297         /* We already know the blocks count can not exceed the maximum
 298          * blocks count.
 299          */
 300         blocks_count = mlxsw_afk_key_info_blocks_count_get(region->key_info);
 301         if (blocks_count <= 2)
 302                 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB;
 303         else if (blocks_count <= 4)
 304                 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB;
 305         else if (blocks_count <= 8)
 306                 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB;
 307         else
 308                 region_type = MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB;
 309
 310         aregion->type = region_type;
 311         aregion->ops = mlxsw_sp_acl_atcam_region_ops_arr[region_type];
 312 }
 313
 314 int
 315 mlxsw_sp_acl_atcam_region_init(struct mlxsw_sp *mlxsw_sp,
 316                                struct mlxsw_sp_acl_atcam *atcam,
 317                                struct mlxsw_sp_acl_atcam_region *aregion,
 318                                struct mlxsw_sp_acl_tcam_region *region,
 319                                void *hints_priv,
 320                                const struct mlxsw_sp_acl_ctcam_region_ops *ops)
 321 {
 322         int err;
 323
 324         aregion->region = region;
 325         aregion->atcam = atcam;
 326         mlxsw_sp_acl_atcam_region_type_init(aregion);
 327         INIT_LIST_HEAD(&aregion->entries_list);
 328
 329         err = rhashtable_init(&aregion->entries_ht,
 330                               &mlxsw_sp_acl_atcam_entries_ht_params);
 331         if (err)
 332                 return err;
 333         err = aregion->ops->init(aregion);
 334         if (err)
 335                 goto err_ops_init;
 336         err = mlxsw_sp_acl_erp_region_init(aregion, hints_priv);
 337         if (err)
 338                 goto err_erp_region_init;
 339         err = mlxsw_sp_acl_ctcam_region_init(mlxsw_sp, &aregion->cregion,
 340                                              region, ops);
 341         if (err)
 342                 goto err_ctcam_region_init;
 343
 344         return 0;
 345
 346 err_ctcam_region_init:
 347         mlxsw_sp_acl_erp_region_fini(aregion);
 348 err_erp_region_init:
 349         aregion->ops->fini(aregion);
 350 err_ops_init:
 351         rhashtable_destroy(&aregion->entries_ht);
 352         return err;
 353 }
 354
 355 void mlxsw_sp_acl_atcam_region_fini(struct mlxsw_sp_acl_atcam_region *aregion)
 356 {
 357         mlxsw_sp_acl_ctcam_region_fini(&aregion->cregion);
 358         mlxsw_sp_acl_erp_region_fini(aregion);
 359         aregion->ops->fini(aregion);
 360         rhashtable_destroy(&aregion->entries_ht);
 361         WARN_ON(!list_empty(&aregion->entries_list));
 362 }
 363
 364 void mlxsw_sp_acl_atcam_chunk_init(struct mlxsw_sp_acl_atcam_region *aregion,
 365                                    struct mlxsw_sp_acl_atcam_chunk *achunk,
 366                                    unsigned int priority)
 367 {
 368         mlxsw_sp_acl_ctcam_chunk_init(&aregion->cregion, &achunk->cchunk,
 369                                       priority);
 370 }
 371
 372 void mlxsw_sp_acl_atcam_chunk_fini(struct mlxsw_sp_acl_atcam_chunk *achunk)
 373 {
 374         mlxsw_sp_acl_ctcam_chunk_fini(&achunk->cchunk);
 375 }
 376
 377 static int
 378 mlxsw_sp_acl_atcam_region_entry_insert(struct mlxsw_sp *mlxsw_sp,
 379                                        struct mlxsw_sp_acl_atcam_region *aregion,
 380                                        struct mlxsw_sp_acl_atcam_entry *aentry,
 381                                        struct mlxsw_sp_acl_rule_info *rulei)
 382 {
 383         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 384         u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
 385         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id;
 386         char ptce3_pl[MLXSW_REG_PTCE3_LEN];
 387         u32 kvdl_index, priority;
 388         int err;
 389
 390         err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
 391         if (err)
 392                 return err;
 393
 394         lkey_id = aregion->ops->lkey_id_get(aregion, aentry->enc_key, erp_id);
 395         if (IS_ERR(lkey_id))
 396                 return PTR_ERR(lkey_id);
 397         aentry->lkey_id = lkey_id;
 398
 399         kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
 400         mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_WRITE,
 401                              priority, region->tcam_region_info,
 402                              aentry->enc_key, erp_id,
 403                              aentry->delta_info.start,
 404                              aentry->delta_info.mask,
 405                              aentry->delta_info.value,
 406                              refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
 407                              kvdl_index);
 408         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
 409         if (err)
 410                 goto err_ptce3_write;
 411
 412         return 0;
 413
 414 err_ptce3_write:
 415         aregion->ops->lkey_id_put(aregion, lkey_id);
 416         return err;
 417 }
 418
 419 static void
 420 mlxsw_sp_acl_atcam_region_entry_remove(struct mlxsw_sp *mlxsw_sp,
 421                                        struct mlxsw_sp_acl_atcam_region *aregion,
 422                                        struct mlxsw_sp_acl_atcam_entry *aentry)
 423 {
 424         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
 425         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 426         u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
 427         char ptce3_pl[MLXSW_REG_PTCE3_LEN];
 428
 429         mlxsw_reg_ptce3_pack(ptce3_pl, false, MLXSW_REG_PTCE3_OP_WRITE_WRITE, 0,
 430                              region->tcam_region_info,
 431                              aentry->enc_key, erp_id,
 432                              aentry->delta_info.start,
 433                              aentry->delta_info.mask,
 434                              aentry->delta_info.value,
 435                              refcount_read(&lkey_id->refcnt) != 1,
 436                              lkey_id->id, 0);
 437         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
 438         aregion->ops->lkey_id_put(aregion, lkey_id);
 439 }
 440
 441 static int
 442 mlxsw_sp_acl_atcam_region_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
 443                                                struct mlxsw_sp_acl_atcam_region *aregion,
 444                                                struct mlxsw_sp_acl_atcam_entry *aentry,
 445                                                struct mlxsw_sp_acl_rule_info *rulei)
 446 {
 447         struct mlxsw_sp_acl_atcam_lkey_id *lkey_id = aentry->lkey_id;
 448         u8 erp_id = mlxsw_sp_acl_erp_mask_erp_id(aentry->erp_mask);
 449         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 450         char ptce3_pl[MLXSW_REG_PTCE3_LEN];
 451         u32 kvdl_index, priority;
 452         int err;
 453
 454         err = mlxsw_sp_acl_tcam_priority_get(mlxsw_sp, rulei, &priority, true);
 455         if (err)
 456                 return err;
 457         kvdl_index = mlxsw_afa_block_first_kvdl_index(rulei->act_block);
 458         mlxsw_reg_ptce3_pack(ptce3_pl, true, MLXSW_REG_PTCE3_OP_WRITE_UPDATE,
 459                              priority, region->tcam_region_info,
 460                              aentry->enc_key, erp_id,
 461                              aentry->delta_info.start,
 462                              aentry->delta_info.mask,
 463                              aentry->delta_info.value,
 464                              refcount_read(&lkey_id->refcnt) != 1, lkey_id->id,
 465                              kvdl_index);
 466         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ptce3), ptce3_pl);
 467 }
 468
 469 static int
 470 __mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
 471                                struct mlxsw_sp_acl_atcam_region *aregion,
 472                                struct mlxsw_sp_acl_atcam_entry *aentry,
 473                                struct mlxsw_sp_acl_rule_info *rulei)
 474 {
 475         struct mlxsw_sp_acl_tcam_region *region = aregion->region;
 476         char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 };
 477         struct mlxsw_afk *afk = mlxsw_sp_acl_afk(mlxsw_sp->acl);
 478         const struct mlxsw_sp_acl_erp_delta *delta;
 479         struct mlxsw_sp_acl_erp_mask *erp_mask;
 480         int err;
 481
 482         mlxsw_afk_encode(afk, region->key_info, &rulei->values,
 483                          aentry->ht_key.full_enc_key, mask);
 484
 485         erp_mask = mlxsw_sp_acl_erp_mask_get(aregion, mask, false);
 486         if (IS_ERR(erp_mask))
 487                 return PTR_ERR(erp_mask);
 488         aentry->erp_mask = erp_mask;
 489         aentry->ht_key.erp_id = mlxsw_sp_acl_erp_mask_erp_id(erp_mask);
 490         memcpy(aentry->enc_key, aentry->ht_key.full_enc_key,
 491                sizeof(aentry->enc_key));
 492
 493         /* Compute all needed delta information and clear the delta bits
 494          * from the encrypted key.
 495          */
 496         delta = mlxsw_sp_acl_erp_delta(aentry->erp_mask);
 497         aentry->delta_info.start = mlxsw_sp_acl_erp_delta_start(delta);
 498         aentry->delta_info.mask = mlxsw_sp_acl_erp_delta_mask(delta);
 499         aentry->delta_info.value =
 500                 mlxsw_sp_acl_erp_delta_value(delta,
 501                                              aentry->ht_key.full_enc_key);
 502         mlxsw_sp_acl_erp_delta_clear(delta, aentry->enc_key);
 503
 504         /* Add rule to the list of A-TCAM rules, assuming this
 505          * rule is intended to A-TCAM. In case this rule does
 506          * not fit into A-TCAM it will be removed from the list.
 507          */
 508         list_add(&aentry->list, &aregion->entries_list);
 509
 510         /* We can't insert identical rules into the A-TCAM, so fail and
 511          * let the rule spill into C-TCAM
 512          */
 513         err = rhashtable_lookup_insert_fast(&aregion->entries_ht,
 514                                             &aentry->ht_node,
 515                                             mlxsw_sp_acl_atcam_entries_ht_params);
 516         if (err)
 517                 goto err_rhashtable_insert;
 518
 519         /* Bloom filter must be updated here, before inserting the rule into
 520          * the A-TCAM.
 521          */
 522         err = mlxsw_sp_acl_erp_bf_insert(mlxsw_sp, aregion, erp_mask, aentry);
 523         if (err)
 524                 goto err_bf_insert;
 525
 526         err = mlxsw_sp_acl_atcam_region_entry_insert(mlxsw_sp, aregion, aentry,
 527                                                      rulei);
 528         if (err)
 529                 goto err_rule_insert;
 530
 531         return 0;
 532
 533 err_rule_insert:
 534         mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, erp_mask, aentry);
 535 err_bf_insert:
 536         rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
 537                                mlxsw_sp_acl_atcam_entries_ht_params);
 538 err_rhashtable_insert:
 539         list_del(&aentry->list);
 540         mlxsw_sp_acl_erp_mask_put(aregion, erp_mask);
 541         return err;
 542 }
 543
 544 static void
 545 __mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
 546                                struct mlxsw_sp_acl_atcam_region *aregion,
 547                                struct mlxsw_sp_acl_atcam_entry *aentry)
 548 {
 549         mlxsw_sp_acl_atcam_region_entry_remove(mlxsw_sp, aregion, aentry);
 550         mlxsw_sp_acl_erp_bf_remove(mlxsw_sp, aregion, aentry->erp_mask, aentry);
 551         rhashtable_remove_fast(&aregion->entries_ht, &aentry->ht_node,
 552                                mlxsw_sp_acl_atcam_entries_ht_params);
 553         list_del(&aentry->list);
 554         mlxsw_sp_acl_erp_mask_put(aregion, aentry->erp_mask);
 555 }
 556
 557 static int
 558 __mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
 559                                           struct mlxsw_sp_acl_atcam_region *aregion,
 560                                           struct mlxsw_sp_acl_atcam_entry *aentry,
 561                                           struct mlxsw_sp_acl_rule_info *rulei)
 562 {
 563         return mlxsw_sp_acl_atcam_region_entry_action_replace(mlxsw_sp, aregion,
 564                                                               aentry, rulei);
 565 }
 566
 567 int mlxsw_sp_acl_atcam_entry_add(struct mlxsw_sp *mlxsw_sp,
 568                                  struct mlxsw_sp_acl_atcam_region *aregion,
 569                                  struct mlxsw_sp_acl_atcam_chunk *achunk,
 570                                  struct mlxsw_sp_acl_atcam_entry *aentry,
 571                                  struct mlxsw_sp_acl_rule_info *rulei)
 572 {
 573         int err;
 574
 575         err = __mlxsw_sp_acl_atcam_entry_add(mlxsw_sp, aregion, aentry, rulei);
 576         if (!err)
 577                 return 0;
 578
 579         /* It is possible we failed to add the rule to the A-TCAM due to
 580          * exceeded number of masks. Try to spill into C-TCAM.
 581          */
 582         trace_mlxsw_sp_acl_atcam_entry_add_ctcam_spill(mlxsw_sp, aregion);
 583         err = mlxsw_sp_acl_ctcam_entry_add(mlxsw_sp, &aregion->cregion,
 584                                            &achunk->cchunk, &aentry->centry,
 585                                            rulei, true);
 586         if (!err)
 587                 return 0;
 588
 589         return err;
 590 }
 591
 592 void mlxsw_sp_acl_atcam_entry_del(struct mlxsw_sp *mlxsw_sp,
 593                                   struct mlxsw_sp_acl_atcam_region *aregion,
 594                                   struct mlxsw_sp_acl_atcam_chunk *achunk,
 595                                   struct mlxsw_sp_acl_atcam_entry *aentry)
 596 {
 597         if (mlxsw_sp_acl_atcam_is_centry(aentry))
 598                 mlxsw_sp_acl_ctcam_entry_del(mlxsw_sp, &aregion->cregion,
 599                                              &achunk->cchunk, &aentry->centry);
 600         else
 601                 __mlxsw_sp_acl_atcam_entry_del(mlxsw_sp, aregion, aentry);
 602 }
 603
 604 int
 605 mlxsw_sp_acl_atcam_entry_action_replace(struct mlxsw_sp *mlxsw_sp,
 606                                         struct mlxsw_sp_acl_atcam_region *aregion,
 607                                         struct mlxsw_sp_acl_atcam_entry *aentry,
 608                                         struct mlxsw_sp_acl_rule_info *rulei)
 609 {
 610         int err;
 611
 612         if (mlxsw_sp_acl_atcam_is_centry(aentry))
 613                 err = mlxsw_sp_acl_ctcam_entry_action_replace(mlxsw_sp,
 614                                                               &aregion->cregion,
 615                                                               &aentry->centry,
 616                                                               rulei);
 617         else
 618                 err = __mlxsw_sp_acl_atcam_entry_action_replace(mlxsw_sp,
 619                                                                 aregion, aentry,
 620                                                                 rulei);
 621
 622         return err;
 623 }
 624
 625 int mlxsw_sp_acl_atcam_init(struct mlxsw_sp *mlxsw_sp,
 626                             struct mlxsw_sp_acl_atcam *atcam)
 627 {
 628         return mlxsw_sp_acl_erps_init(mlxsw_sp, atcam);
 629 }
 630
 631 void mlxsw_sp_acl_atcam_fini(struct mlxsw_sp *mlxsw_sp,
 632                              struct mlxsw_sp_acl_atcam *atcam)
 633 {
 634         mlxsw_sp_acl_erps_fini(mlxsw_sp, atcam);
 635 }
 636
 637 void *
 638 mlxsw_sp_acl_atcam_rehash_hints_get(struct mlxsw_sp_acl_atcam_region *aregion)
 639 {
 640         return mlxsw_sp_acl_erp_rehash_hints_get(aregion);
 641 }
 642
 643 void mlxsw_sp_acl_atcam_rehash_hints_put(void *hints_priv)
 644 {
 645         mlxsw_sp_acl_erp_rehash_hints_put(hints_priv);
 646 }