drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c

   1 /*
   2  * drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c
   3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
   4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
   5  * Copyright (c) 2016 Ido Schimmel <idosch@mellanox.com>
   6  * Copyright (c) 2016 Yotam Gigi <yotamg@mellanox.com>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  * 3. Neither the names of the copyright holders nor the names of its
  17  *    contributors may be used to endorse or promote products derived from
  18  *    this software without specific prior written permission.
  19  *
  20  * Alternatively, this software may be distributed under the terms of the
  21  * GNU General Public License ("GPL") version 2 as published by the Free
  22  * Software Foundation.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  34  * POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <linux/kernel.h>
  38 #include <linux/types.h>
  39 #include <linux/rhashtable.h>
  40 #include <linux/bitops.h>
  41 #include <linux/in6.h>
  42 #include <linux/notifier.h>
  43 #include <net/netevent.h>
  44 #include <net/neighbour.h>
  45 #include <net/arp.h>
  46 #include <net/ip_fib.h>
  47
  48 #include "spectrum.h"
  49 #include "core.h"
  50 #include "reg.h"
  51
  52 #define mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) \
  53         for_each_set_bit(prefix, (prefix_usage)->b, MLXSW_SP_PREFIX_COUNT)
  54
  55 static bool
  56 mlxsw_sp_prefix_usage_subset(struct mlxsw_sp_prefix_usage *prefix_usage1,
  57                              struct mlxsw_sp_prefix_usage *prefix_usage2)
  58 {
  59         unsigned char prefix;
  60
  61         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage1) {
  62                 if (!test_bit(prefix, prefix_usage2->b))
  63                         return false;
  64         }
  65         return true;
  66 }
  67
  68 static bool
  69 mlxsw_sp_prefix_usage_eq(struct mlxsw_sp_prefix_usage *prefix_usage1,
  70                          struct mlxsw_sp_prefix_usage *prefix_usage2)
  71 {
  72         return !memcmp(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
  73 }
  74
  75 static bool
  76 mlxsw_sp_prefix_usage_none(struct mlxsw_sp_prefix_usage *prefix_usage)
  77 {
  78         struct mlxsw_sp_prefix_usage prefix_usage_none = {{ 0 } };
  79
  80         return mlxsw_sp_prefix_usage_eq(prefix_usage, &prefix_usage_none);
  81 }
  82
  83 static void
  84 mlxsw_sp_prefix_usage_cpy(struct mlxsw_sp_prefix_usage *prefix_usage1,
  85                           struct mlxsw_sp_prefix_usage *prefix_usage2)
  86 {
  87         memcpy(prefix_usage1, prefix_usage2, sizeof(*prefix_usage1));
  88 }
  89
  90 static void
  91 mlxsw_sp_prefix_usage_zero(struct mlxsw_sp_prefix_usage *prefix_usage)
  92 {
  93         memset(prefix_usage, 0, sizeof(*prefix_usage));
  94 }
  95
  96 static void
  97 mlxsw_sp_prefix_usage_set(struct mlxsw_sp_prefix_usage *prefix_usage,
  98                           unsigned char prefix_len)
  99 {
 100         set_bit(prefix_len, prefix_usage->b);
 101 }
 102
 103 static void
 104 mlxsw_sp_prefix_usage_clear(struct mlxsw_sp_prefix_usage *prefix_usage,
 105                             unsigned char prefix_len)
 106 {
 107         clear_bit(prefix_len, prefix_usage->b);
 108 }
 109
 110 struct mlxsw_sp_fib_key {
 111         struct net_device *dev;
 112         unsigned char addr[sizeof(struct in6_addr)];
 113         unsigned char prefix_len;
 114 };
 115
 116 enum mlxsw_sp_fib_entry_type {
 117         MLXSW_SP_FIB_ENTRY_TYPE_REMOTE,
 118         MLXSW_SP_FIB_ENTRY_TYPE_LOCAL,
 119         MLXSW_SP_FIB_ENTRY_TYPE_TRAP,
 120 };
 121
 122 struct mlxsw_sp_nexthop_group;
 123
 124 struct mlxsw_sp_fib_entry {
 125         struct rhash_head ht_node;
 126         struct list_head list;
 127         struct mlxsw_sp_fib_key key;
 128         enum mlxsw_sp_fib_entry_type type;
 129         unsigned int ref_count;
 130         u16 rif; /* used for action local */
 131         struct mlxsw_sp_vr *vr;
 132         struct fib_info *fi;
 133         struct list_head nexthop_group_node;
 134         struct mlxsw_sp_nexthop_group *nh_group;
 135 };
 136
 137 struct mlxsw_sp_fib {
 138         struct rhashtable ht;
 139         struct list_head entry_list;
 140         unsigned long prefix_ref_count[MLXSW_SP_PREFIX_COUNT];
 141         struct mlxsw_sp_prefix_usage prefix_usage;
 142 };
 143
 144 static const struct rhashtable_params mlxsw_sp_fib_ht_params = {
 145         .key_offset = offsetof(struct mlxsw_sp_fib_entry, key),
 146         .head_offset = offsetof(struct mlxsw_sp_fib_entry, ht_node),
 147         .key_len = sizeof(struct mlxsw_sp_fib_key),
 148         .automatic_shrinking = true,
 149 };
 150
 151 static int mlxsw_sp_fib_entry_insert(struct mlxsw_sp_fib *fib,
 152                                      struct mlxsw_sp_fib_entry *fib_entry)
 153 {
 154         unsigned char prefix_len = fib_entry->key.prefix_len;
 155         int err;
 156
 157         err = rhashtable_insert_fast(&fib->ht, &fib_entry->ht_node,
 158                                      mlxsw_sp_fib_ht_params);
 159         if (err)
 160                 return err;
 161         list_add_tail(&fib_entry->list, &fib->entry_list);
 162         if (fib->prefix_ref_count[prefix_len]++ == 0)
 163                 mlxsw_sp_prefix_usage_set(&fib->prefix_usage, prefix_len);
 164         return 0;
 165 }
 166
 167 static void mlxsw_sp_fib_entry_remove(struct mlxsw_sp_fib *fib,
 168                                       struct mlxsw_sp_fib_entry *fib_entry)
 169 {
 170         unsigned char prefix_len = fib_entry->key.prefix_len;
 171
 172         if (--fib->prefix_ref_count[prefix_len] == 0)
 173                 mlxsw_sp_prefix_usage_clear(&fib->prefix_usage, prefix_len);
 174         list_del(&fib_entry->list);
 175         rhashtable_remove_fast(&fib->ht, &fib_entry->ht_node,
 176                                mlxsw_sp_fib_ht_params);
 177 }
 178
 179 static struct mlxsw_sp_fib_entry *
 180 mlxsw_sp_fib_entry_create(struct mlxsw_sp_fib *fib, const void *addr,
 181                           size_t addr_len, unsigned char prefix_len,
 182                           struct net_device *dev)
 183 {
 184         struct mlxsw_sp_fib_entry *fib_entry;
 185
 186         fib_entry = kzalloc(sizeof(*fib_entry), GFP_KERNEL);
 187         if (!fib_entry)
 188                 return NULL;
 189         fib_entry->key.dev = dev;
 190         memcpy(fib_entry->key.addr, addr, addr_len);
 191         fib_entry->key.prefix_len = prefix_len;
 192         return fib_entry;
 193 }
 194
 195 static void mlxsw_sp_fib_entry_destroy(struct mlxsw_sp_fib_entry *fib_entry)
 196 {
 197         kfree(fib_entry);
 198 }
 199
 200 static struct mlxsw_sp_fib_entry *
 201 mlxsw_sp_fib_entry_lookup(struct mlxsw_sp_fib *fib, const void *addr,
 202                           size_t addr_len, unsigned char prefix_len,
 203                           struct net_device *dev)
 204 {
 205         struct mlxsw_sp_fib_key key;
 206
 207         memset(&key, 0, sizeof(key));
 208         key.dev = dev;
 209         memcpy(key.addr, addr, addr_len);
 210         key.prefix_len = prefix_len;
 211         return rhashtable_lookup_fast(&fib->ht, &key, mlxsw_sp_fib_ht_params);
 212 }
 213
 214 static struct mlxsw_sp_fib *mlxsw_sp_fib_create(void)
 215 {
 216         struct mlxsw_sp_fib *fib;
 217         int err;
 218
 219         fib = kzalloc(sizeof(*fib), GFP_KERNEL);
 220         if (!fib)
 221                 return ERR_PTR(-ENOMEM);
 222         err = rhashtable_init(&fib->ht, &mlxsw_sp_fib_ht_params);
 223         if (err)
 224                 goto err_rhashtable_init;
 225         INIT_LIST_HEAD(&fib->entry_list);
 226         return fib;
 227
 228 err_rhashtable_init:
 229         kfree(fib);
 230         return ERR_PTR(err);
 231 }
 232
 233 static void mlxsw_sp_fib_destroy(struct mlxsw_sp_fib *fib)
 234 {
 235         rhashtable_destroy(&fib->ht);
 236         kfree(fib);
 237 }
 238
 239 static struct mlxsw_sp_lpm_tree *
 240 mlxsw_sp_lpm_tree_find_unused(struct mlxsw_sp *mlxsw_sp, bool one_reserved)
 241 {
 242         static struct mlxsw_sp_lpm_tree *lpm_tree;
 243         int i;
 244
 245         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 246                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 247                 if (lpm_tree->ref_count == 0) {
 248                         if (one_reserved)
 249                                 one_reserved = false;
 250                         else
 251                                 return lpm_tree;
 252                 }
 253         }
 254         return NULL;
 255 }
 256
 257 static int mlxsw_sp_lpm_tree_alloc(struct mlxsw_sp *mlxsw_sp,
 258                                    struct mlxsw_sp_lpm_tree *lpm_tree)
 259 {
 260         char ralta_pl[MLXSW_REG_RALTA_LEN];
 261
 262         mlxsw_reg_ralta_pack(ralta_pl, true,
 263                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 264                              lpm_tree->id);
 265         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 266 }
 267
 268 static int mlxsw_sp_lpm_tree_free(struct mlxsw_sp *mlxsw_sp,
 269                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 270 {
 271         char ralta_pl[MLXSW_REG_RALTA_LEN];
 272
 273         mlxsw_reg_ralta_pack(ralta_pl, false,
 274                              (enum mlxsw_reg_ralxx_protocol) lpm_tree->proto,
 275                              lpm_tree->id);
 276         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
 277 }
 278
 279 static int
 280 mlxsw_sp_lpm_tree_left_struct_set(struct mlxsw_sp *mlxsw_sp,
 281                                   struct mlxsw_sp_prefix_usage *prefix_usage,
 282                                   struct mlxsw_sp_lpm_tree *lpm_tree)
 283 {
 284         char ralst_pl[MLXSW_REG_RALST_LEN];
 285         u8 root_bin = 0;
 286         u8 prefix;
 287         u8 last_prefix = MLXSW_REG_RALST_BIN_NO_CHILD;
 288
 289         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage)
 290                 root_bin = prefix;
 291
 292         mlxsw_reg_ralst_pack(ralst_pl, root_bin, lpm_tree->id);
 293         mlxsw_sp_prefix_usage_for_each(prefix, prefix_usage) {
 294                 if (prefix == 0)
 295                         continue;
 296                 mlxsw_reg_ralst_bin_pack(ralst_pl, prefix, last_prefix,
 297                                          MLXSW_REG_RALST_BIN_NO_CHILD);
 298                 last_prefix = prefix;
 299         }
 300         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
 301 }
 302
 303 static struct mlxsw_sp_lpm_tree *
 304 mlxsw_sp_lpm_tree_create(struct mlxsw_sp *mlxsw_sp,
 305                          struct mlxsw_sp_prefix_usage *prefix_usage,
 306                          enum mlxsw_sp_l3proto proto, bool one_reserved)
 307 {
 308         struct mlxsw_sp_lpm_tree *lpm_tree;
 309         int err;
 310
 311         lpm_tree = mlxsw_sp_lpm_tree_find_unused(mlxsw_sp, one_reserved);
 312         if (!lpm_tree)
 313                 return ERR_PTR(-EBUSY);
 314         lpm_tree->proto = proto;
 315         err = mlxsw_sp_lpm_tree_alloc(mlxsw_sp, lpm_tree);
 316         if (err)
 317                 return ERR_PTR(err);
 318
 319         err = mlxsw_sp_lpm_tree_left_struct_set(mlxsw_sp, prefix_usage,
 320                                                 lpm_tree);
 321         if (err)
 322                 goto err_left_struct_set;
 323         memcpy(&lpm_tree->prefix_usage, prefix_usage,
 324                sizeof(lpm_tree->prefix_usage));
 325         return lpm_tree;
 326
 327 err_left_struct_set:
 328         mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 329         return ERR_PTR(err);
 330 }
 331
 332 static int mlxsw_sp_lpm_tree_destroy(struct mlxsw_sp *mlxsw_sp,
 333                                      struct mlxsw_sp_lpm_tree *lpm_tree)
 334 {
 335         return mlxsw_sp_lpm_tree_free(mlxsw_sp, lpm_tree);
 336 }
 337
 338 static struct mlxsw_sp_lpm_tree *
 339 mlxsw_sp_lpm_tree_get(struct mlxsw_sp *mlxsw_sp,
 340                       struct mlxsw_sp_prefix_usage *prefix_usage,
 341                       enum mlxsw_sp_l3proto proto, bool one_reserved)
 342 {
 343         struct mlxsw_sp_lpm_tree *lpm_tree;
 344         int i;
 345
 346         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 347                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 348                 if (lpm_tree->ref_count != 0 &&
 349                     lpm_tree->proto == proto &&
 350                     mlxsw_sp_prefix_usage_eq(&lpm_tree->prefix_usage,
 351                                              prefix_usage))
 352                         goto inc_ref_count;
 353         }
 354         lpm_tree = mlxsw_sp_lpm_tree_create(mlxsw_sp, prefix_usage,
 355                                             proto, one_reserved);
 356         if (IS_ERR(lpm_tree))
 357                 return lpm_tree;
 358
 359 inc_ref_count:
 360         lpm_tree->ref_count++;
 361         return lpm_tree;
 362 }
 363
 364 static int mlxsw_sp_lpm_tree_put(struct mlxsw_sp *mlxsw_sp,
 365                                  struct mlxsw_sp_lpm_tree *lpm_tree)
 366 {
 367         if (--lpm_tree->ref_count == 0)
 368                 return mlxsw_sp_lpm_tree_destroy(mlxsw_sp, lpm_tree);
 369         return 0;
 370 }
 371
 372 static void mlxsw_sp_lpm_init(struct mlxsw_sp *mlxsw_sp)
 373 {
 374         struct mlxsw_sp_lpm_tree *lpm_tree;
 375         int i;
 376
 377         for (i = 0; i < MLXSW_SP_LPM_TREE_COUNT; i++) {
 378                 lpm_tree = &mlxsw_sp->router.lpm_trees[i];
 379                 lpm_tree->id = i + MLXSW_SP_LPM_TREE_MIN;
 380         }
 381 }
 382
 383 static struct mlxsw_sp_vr *mlxsw_sp_vr_find_unused(struct mlxsw_sp *mlxsw_sp)
 384 {
 385         struct mlxsw_resources *resources;
 386         struct mlxsw_sp_vr *vr;
 387         int i;
 388
 389         resources = mlxsw_core_resources_get(mlxsw_sp->core);
 390         for (i = 0; i < resources->max_virtual_routers; i++) {
 391                 vr = &mlxsw_sp->router.vrs[i];
 392                 if (!vr->used)
 393                         return vr;
 394         }
 395         return NULL;
 396 }
 397
 398 static int mlxsw_sp_vr_lpm_tree_bind(struct mlxsw_sp *mlxsw_sp,
 399                                      struct mlxsw_sp_vr *vr)
 400 {
 401         char raltb_pl[MLXSW_REG_RALTB_LEN];
 402
 403         mlxsw_reg_raltb_pack(raltb_pl, vr->id,
 404                              (enum mlxsw_reg_ralxx_protocol) vr->proto,
 405                              vr->lpm_tree->id);
 406         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 407 }
 408
 409 static int mlxsw_sp_vr_lpm_tree_unbind(struct mlxsw_sp *mlxsw_sp,
 410                                        struct mlxsw_sp_vr *vr)
 411 {
 412         char raltb_pl[MLXSW_REG_RALTB_LEN];
 413
 414         /* Bind to tree 0 which is default */
 415         mlxsw_reg_raltb_pack(raltb_pl, vr->id,
 416                              (enum mlxsw_reg_ralxx_protocol) vr->proto, 0);
 417         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
 418 }
 419
 420 static u32 mlxsw_sp_fix_tb_id(u32 tb_id)
 421 {
 422         /* For our purpose, squash main and local table into one */
 423         if (tb_id == RT_TABLE_LOCAL)
 424                 tb_id = RT_TABLE_MAIN;
 425         return tb_id;
 426 }
 427
 428 static struct mlxsw_sp_vr *mlxsw_sp_vr_find(struct mlxsw_sp *mlxsw_sp,
 429                                             u32 tb_id,
 430                                             enum mlxsw_sp_l3proto proto)
 431 {
 432         struct mlxsw_resources *resources;
 433         struct mlxsw_sp_vr *vr;
 434         int i;
 435
 436         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 437
 438         resources = mlxsw_core_resources_get(mlxsw_sp->core);
 439         for (i = 0; i < resources->max_virtual_routers; i++) {
 440                 vr = &mlxsw_sp->router.vrs[i];
 441                 if (vr->used && vr->proto == proto && vr->tb_id == tb_id)
 442                         return vr;
 443         }
 444         return NULL;
 445 }
 446
 447 static struct mlxsw_sp_vr *mlxsw_sp_vr_create(struct mlxsw_sp *mlxsw_sp,
 448                                               unsigned char prefix_len,
 449                                               u32 tb_id,
 450                                               enum mlxsw_sp_l3proto proto)
 451 {
 452         struct mlxsw_sp_prefix_usage req_prefix_usage;
 453         struct mlxsw_sp_lpm_tree *lpm_tree;
 454         struct mlxsw_sp_vr *vr;
 455         int err;
 456
 457         vr = mlxsw_sp_vr_find_unused(mlxsw_sp);
 458         if (!vr)
 459                 return ERR_PTR(-EBUSY);
 460         vr->fib = mlxsw_sp_fib_create();
 461         if (IS_ERR(vr->fib))
 462                 return ERR_CAST(vr->fib);
 463
 464         vr->proto = proto;
 465         vr->tb_id = tb_id;
 466         mlxsw_sp_prefix_usage_zero(&req_prefix_usage);
 467         mlxsw_sp_prefix_usage_set(&req_prefix_usage, prefix_len);
 468         lpm_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, &req_prefix_usage,
 469                                          proto, true);
 470         if (IS_ERR(lpm_tree)) {
 471                 err = PTR_ERR(lpm_tree);
 472                 goto err_tree_get;
 473         }
 474         vr->lpm_tree = lpm_tree;
 475         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
 476         if (err)
 477                 goto err_tree_bind;
 478
 479         vr->used = true;
 480         return vr;
 481
 482 err_tree_bind:
 483         mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
 484 err_tree_get:
 485         mlxsw_sp_fib_destroy(vr->fib);
 486
 487         return ERR_PTR(err);
 488 }
 489
 490 static void mlxsw_sp_vr_destroy(struct mlxsw_sp *mlxsw_sp,
 491                                 struct mlxsw_sp_vr *vr)
 492 {
 493         mlxsw_sp_vr_lpm_tree_unbind(mlxsw_sp, vr);
 494         mlxsw_sp_lpm_tree_put(mlxsw_sp, vr->lpm_tree);
 495         mlxsw_sp_fib_destroy(vr->fib);
 496         vr->used = false;
 497 }
 498
 499 static int
 500 mlxsw_sp_vr_lpm_tree_check(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr,
 501                            struct mlxsw_sp_prefix_usage *req_prefix_usage)
 502 {
 503         struct mlxsw_sp_lpm_tree *lpm_tree = vr->lpm_tree;
 504         struct mlxsw_sp_lpm_tree *new_tree;
 505         int err;
 506
 507         if (mlxsw_sp_prefix_usage_eq(req_prefix_usage, &lpm_tree->prefix_usage))
 508                 return 0;
 509
 510         new_tree = mlxsw_sp_lpm_tree_get(mlxsw_sp, req_prefix_usage,
 511                                          vr->proto, false);
 512         if (IS_ERR(new_tree)) {
 513                 /* We failed to get a tree according to the required
 514                  * prefix usage. However, the current tree might be still good
 515                  * for us if our requirement is subset of the prefixes used
 516                  * in the tree.
 517                  */
 518                 if (mlxsw_sp_prefix_usage_subset(req_prefix_usage,
 519                                                  &lpm_tree->prefix_usage))
 520                         return 0;
 521                 return PTR_ERR(new_tree);
 522         }
 523
 524         /* Prevent packet loss by overwriting existing binding */
 525         vr->lpm_tree = new_tree;
 526         err = mlxsw_sp_vr_lpm_tree_bind(mlxsw_sp, vr);
 527         if (err)
 528                 goto err_tree_bind;
 529         mlxsw_sp_lpm_tree_put(mlxsw_sp, lpm_tree);
 530
 531         return 0;
 532
 533 err_tree_bind:
 534         vr->lpm_tree = lpm_tree;
 535         mlxsw_sp_lpm_tree_put(mlxsw_sp, new_tree);
 536         return err;
 537 }
 538
 539 static struct mlxsw_sp_vr *mlxsw_sp_vr_get(struct mlxsw_sp *mlxsw_sp,
 540                                            unsigned char prefix_len,
 541                                            u32 tb_id,
 542                                            enum mlxsw_sp_l3proto proto)
 543 {
 544         struct mlxsw_sp_vr *vr;
 545         int err;
 546
 547         tb_id = mlxsw_sp_fix_tb_id(tb_id);
 548         vr = mlxsw_sp_vr_find(mlxsw_sp, tb_id, proto);
 549         if (!vr) {
 550                 vr = mlxsw_sp_vr_create(mlxsw_sp, prefix_len, tb_id, proto);
 551                 if (IS_ERR(vr))
 552                         return vr;
 553         } else {
 554                 struct mlxsw_sp_prefix_usage req_prefix_usage;
 555
 556                 mlxsw_sp_prefix_usage_cpy(&req_prefix_usage,
 557                                           &vr->fib->prefix_usage);
 558                 mlxsw_sp_prefix_usage_set(&req_prefix_usage, prefix_len);
 559                 /* Need to replace LPM tree in case new prefix is required. */
 560                 err = mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, vr,
 561                                                  &req_prefix_usage);
 562                 if (err)
 563                         return ERR_PTR(err);
 564         }
 565         return vr;
 566 }
 567
 568 static void mlxsw_sp_vr_put(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_vr *vr)
 569 {
 570         /* Destroy virtual router entity in case the associated FIB is empty
 571          * and allow it to be used for other tables in future. Otherwise,
 572          * check if some prefix usage did not disappear and change tree if
 573          * that is the case. Note that in case new, smaller tree cannot be
 574          * allocated, the original one will be kept being used.
 575          */
 576         if (mlxsw_sp_prefix_usage_none(&vr->fib->prefix_usage))
 577                 mlxsw_sp_vr_destroy(mlxsw_sp, vr);
 578         else
 579                 mlxsw_sp_vr_lpm_tree_check(mlxsw_sp, vr,
 580                                            &vr->fib->prefix_usage);
 581 }
 582
 583 static int mlxsw_sp_vrs_init(struct mlxsw_sp *mlxsw_sp)
 584 {
 585         struct mlxsw_resources *resources;
 586         struct mlxsw_sp_vr *vr;
 587         int i;
 588
 589         resources = mlxsw_core_resources_get(mlxsw_sp->core);
 590         if (!resources->max_virtual_routers_valid)
 591                 return -EIO;
 592
 593         mlxsw_sp->router.vrs = kcalloc(resources->max_virtual_routers,
 594                                        sizeof(struct mlxsw_sp_vr),
 595                                        GFP_KERNEL);
 596         if (!mlxsw_sp->router.vrs)
 597                 return -ENOMEM;
 598
 599         for (i = 0; i < resources->max_virtual_routers; i++) {
 600                 vr = &mlxsw_sp->router.vrs[i];
 601                 vr->id = i;
 602         }
 603
 604         return 0;
 605 }
 606
 607 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp);
 608
 609 static void mlxsw_sp_vrs_fini(struct mlxsw_sp *mlxsw_sp)
 610 {
 611         mlxsw_sp_router_fib_flush(mlxsw_sp);
 612         kfree(mlxsw_sp->router.vrs);
 613 }
 614
 615 struct mlxsw_sp_neigh_key {
 616         struct neighbour *n;
 617 };
 618
 619 struct mlxsw_sp_neigh_entry {
 620         struct rhash_head ht_node;
 621         struct mlxsw_sp_neigh_key key;
 622         u16 rif;
 623         bool offloaded;
 624         struct delayed_work dw;
 625         struct mlxsw_sp_port *mlxsw_sp_port;
 626         unsigned char ha[ETH_ALEN];
 627         struct list_head nexthop_list; /* list of nexthops using
 628                                         * this neigh entry
 629                                         */
 630         struct list_head nexthop_neighs_list_node;
 631 };
 632
 633 static const struct rhashtable_params mlxsw_sp_neigh_ht_params = {
 634         .key_offset = offsetof(struct mlxsw_sp_neigh_entry, key),
 635         .head_offset = offsetof(struct mlxsw_sp_neigh_entry, ht_node),
 636         .key_len = sizeof(struct mlxsw_sp_neigh_key),
 637 };
 638
 639 static int
 640 mlxsw_sp_neigh_entry_insert(struct mlxsw_sp *mlxsw_sp,
 641                             struct mlxsw_sp_neigh_entry *neigh_entry)
 642 {
 643         return rhashtable_insert_fast(&mlxsw_sp->router.neigh_ht,
 644                                       &neigh_entry->ht_node,
 645                                       mlxsw_sp_neigh_ht_params);
 646 }
 647
 648 static void
 649 mlxsw_sp_neigh_entry_remove(struct mlxsw_sp *mlxsw_sp,
 650                             struct mlxsw_sp_neigh_entry *neigh_entry)
 651 {
 652         rhashtable_remove_fast(&mlxsw_sp->router.neigh_ht,
 653                                &neigh_entry->ht_node,
 654                                mlxsw_sp_neigh_ht_params);
 655 }
 656
 657 static void mlxsw_sp_router_neigh_update_hw(struct work_struct *work);
 658
 659 static struct mlxsw_sp_neigh_entry *
 660 mlxsw_sp_neigh_entry_create(struct neighbour *n, u16 rif)
 661 {
 662         struct mlxsw_sp_neigh_entry *neigh_entry;
 663
 664         neigh_entry = kzalloc(sizeof(*neigh_entry), GFP_ATOMIC);
 665         if (!neigh_entry)
 666                 return NULL;
 667         neigh_entry->key.n = n;
 668         neigh_entry->rif = rif;
 669         INIT_DELAYED_WORK(&neigh_entry->dw, mlxsw_sp_router_neigh_update_hw);
 670         INIT_LIST_HEAD(&neigh_entry->nexthop_list);
 671         return neigh_entry;
 672 }
 673
 674 static void
 675 mlxsw_sp_neigh_entry_destroy(struct mlxsw_sp_neigh_entry *neigh_entry)
 676 {
 677         kfree(neigh_entry);
 678 }
 679
 680 static struct mlxsw_sp_neigh_entry *
 681 mlxsw_sp_neigh_entry_lookup(struct mlxsw_sp *mlxsw_sp, struct neighbour *n)
 682 {
 683         struct mlxsw_sp_neigh_key key;
 684
 685         key.n = n;
 686         return rhashtable_lookup_fast(&mlxsw_sp->router.neigh_ht,
 687                                       &key, mlxsw_sp_neigh_ht_params);
 688 }
 689
 690 int mlxsw_sp_router_neigh_construct(struct net_device *dev,
 691                                     struct neighbour *n)
 692 {
 693         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
 694         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 695         struct mlxsw_sp_neigh_entry *neigh_entry;
 696         struct mlxsw_sp_rif *r;
 697         int err;
 698
 699         if (n->tbl != &arp_tbl)
 700                 return 0;
 701
 702         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
 703         if (neigh_entry)
 704                 return 0;
 705
 706         r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, n->dev);
 707         if (WARN_ON(!r))
 708                 return -EINVAL;
 709
 710         neigh_entry = mlxsw_sp_neigh_entry_create(n, r->rif);
 711         if (!neigh_entry)
 712                 return -ENOMEM;
 713         err = mlxsw_sp_neigh_entry_insert(mlxsw_sp, neigh_entry);
 714         if (err)
 715                 goto err_neigh_entry_insert;
 716         return 0;
 717
 718 err_neigh_entry_insert:
 719         mlxsw_sp_neigh_entry_destroy(neigh_entry);
 720         return err;
 721 }
 722
 723 void mlxsw_sp_router_neigh_destroy(struct net_device *dev,
 724                                    struct neighbour *n)
 725 {
 726         struct mlxsw_sp_port *mlxsw_sp_port = netdev_priv(dev);
 727         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 728         struct mlxsw_sp_neigh_entry *neigh_entry;
 729
 730         if (n->tbl != &arp_tbl)
 731                 return;
 732
 733         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
 734         if (!neigh_entry)
 735                 return;
 736         mlxsw_sp_neigh_entry_remove(mlxsw_sp, neigh_entry);
 737         mlxsw_sp_neigh_entry_destroy(neigh_entry);
 738 }
 739
 740 static void
 741 mlxsw_sp_router_neighs_update_interval_init(struct mlxsw_sp *mlxsw_sp)
 742 {
 743         unsigned long interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME);
 744
 745         mlxsw_sp->router.neighs_update.interval = jiffies_to_msecs(interval);
 746 }
 747
 748 static void mlxsw_sp_router_neigh_ent_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 749                                                    char *rauhtd_pl,
 750                                                    int ent_index)
 751 {
 752         struct net_device *dev;
 753         struct neighbour *n;
 754         __be32 dipn;
 755         u32 dip;
 756         u16 rif;
 757
 758         mlxsw_reg_rauhtd_ent_ipv4_unpack(rauhtd_pl, ent_index, &rif, &dip);
 759
 760         if (!mlxsw_sp->rifs[rif]) {
 761                 dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Incorrect RIF in neighbour entry\n");
 762                 return;
 763         }
 764
 765         dipn = htonl(dip);
 766         dev = mlxsw_sp->rifs[rif]->dev;
 767         n = neigh_lookup(&arp_tbl, &dipn, dev);
 768         if (!n)
 769                 return;
 770
 771         netdev_dbg(dev, "Updating neighbour with IP=%pI4h\n", &dip);
 772         neigh_event_send(n, NULL);
 773         neigh_release(n);
 774 }
 775
 776 static void mlxsw_sp_router_neigh_rec_ipv4_process(struct mlxsw_sp *mlxsw_sp,
 777                                                    char *rauhtd_pl,
 778                                                    int rec_index)
 779 {
 780         u8 num_entries;
 781         int i;
 782
 783         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 784                                                                 rec_index);
 785         /* Hardware starts counting at 0, so add 1. */
 786         num_entries++;
 787
 788         /* Each record consists of several neighbour entries. */
 789         for (i = 0; i < num_entries; i++) {
 790                 int ent_index;
 791
 792                 ent_index = rec_index * MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC + i;
 793                 mlxsw_sp_router_neigh_ent_ipv4_process(mlxsw_sp, rauhtd_pl,
 794                                                        ent_index);
 795         }
 796
 797 }
 798
 799 static void mlxsw_sp_router_neigh_rec_process(struct mlxsw_sp *mlxsw_sp,
 800                                               char *rauhtd_pl, int rec_index)
 801 {
 802         switch (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, rec_index)) {
 803         case MLXSW_REG_RAUHTD_TYPE_IPV4:
 804                 mlxsw_sp_router_neigh_rec_ipv4_process(mlxsw_sp, rauhtd_pl,
 805                                                        rec_index);
 806                 break;
 807         case MLXSW_REG_RAUHTD_TYPE_IPV6:
 808                 WARN_ON_ONCE(1);
 809                 break;
 810         }
 811 }
 812
 813 static bool mlxsw_sp_router_rauhtd_is_full(char *rauhtd_pl)
 814 {
 815         u8 num_rec, last_rec_index, num_entries;
 816
 817         num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 818         last_rec_index = num_rec - 1;
 819
 820         if (num_rec < MLXSW_REG_RAUHTD_REC_MAX_NUM)
 821                 return false;
 822         if (mlxsw_reg_rauhtd_rec_type_get(rauhtd_pl, last_rec_index) ==
 823             MLXSW_REG_RAUHTD_TYPE_IPV6)
 824                 return true;
 825
 826         num_entries = mlxsw_reg_rauhtd_ipv4_rec_num_entries_get(rauhtd_pl,
 827                                                                 last_rec_index);
 828         if (++num_entries == MLXSW_REG_RAUHTD_IPV4_ENT_PER_REC)
 829                 return true;
 830         return false;
 831 }
 832
 833 static int mlxsw_sp_router_neighs_update_rauhtd(struct mlxsw_sp *mlxsw_sp)
 834 {
 835         char *rauhtd_pl;
 836         u8 num_rec;
 837         int i, err;
 838
 839         rauhtd_pl = kmalloc(MLXSW_REG_RAUHTD_LEN, GFP_KERNEL);
 840         if (!rauhtd_pl)
 841                 return -ENOMEM;
 842
 843         /* Make sure the neighbour's netdev isn't removed in the
 844          * process.
 845          */
 846         rtnl_lock();
 847         do {
 848                 mlxsw_reg_rauhtd_pack(rauhtd_pl, MLXSW_REG_RAUHTD_TYPE_IPV4);
 849                 err = mlxsw_reg_query(mlxsw_sp->core, MLXSW_REG(rauhtd),
 850                                       rauhtd_pl);
 851                 if (err) {
 852                         dev_err_ratelimited(mlxsw_sp->bus_info->dev, "Failed to dump neighbour talbe\n");
 853                         break;
 854                 }
 855                 num_rec = mlxsw_reg_rauhtd_num_rec_get(rauhtd_pl);
 856                 for (i = 0; i < num_rec; i++)
 857                         mlxsw_sp_router_neigh_rec_process(mlxsw_sp, rauhtd_pl,
 858                                                           i);
 859         } while (mlxsw_sp_router_rauhtd_is_full(rauhtd_pl));
 860         rtnl_unlock();
 861
 862         kfree(rauhtd_pl);
 863         return err;
 864 }
 865
 866 static void mlxsw_sp_router_neighs_update_nh(struct mlxsw_sp *mlxsw_sp)
 867 {
 868         struct mlxsw_sp_neigh_entry *neigh_entry;
 869
 870         /* Take RTNL mutex here to prevent lists from changes */
 871         rtnl_lock();
 872         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 873                             nexthop_neighs_list_node) {
 874                 /* If this neigh have nexthops, make the kernel think this neigh
 875                  * is active regardless of the traffic.
 876                  */
 877                 if (!list_empty(&neigh_entry->nexthop_list))
 878                         neigh_event_send(neigh_entry->key.n, NULL);
 879         }
 880         rtnl_unlock();
 881 }
 882
 883 static void
 884 mlxsw_sp_router_neighs_update_work_schedule(struct mlxsw_sp *mlxsw_sp)
 885 {
 886         unsigned long interval = mlxsw_sp->router.neighs_update.interval;
 887
 888         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw,
 889                                msecs_to_jiffies(interval));
 890 }
 891
 892 static void mlxsw_sp_router_neighs_update_work(struct work_struct *work)
 893 {
 894         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 895                                                  router.neighs_update.dw.work);
 896         int err;
 897
 898         err = mlxsw_sp_router_neighs_update_rauhtd(mlxsw_sp);
 899         if (err)
 900                 dev_err(mlxsw_sp->bus_info->dev, "Could not update kernel for neigh activity");
 901
 902         mlxsw_sp_router_neighs_update_nh(mlxsw_sp);
 903
 904         mlxsw_sp_router_neighs_update_work_schedule(mlxsw_sp);
 905 }
 906
 907 static void mlxsw_sp_router_probe_unresolved_nexthops(struct work_struct *work)
 908 {
 909         struct mlxsw_sp_neigh_entry *neigh_entry;
 910         struct mlxsw_sp *mlxsw_sp = container_of(work, struct mlxsw_sp,
 911                                                  router.nexthop_probe_dw.work);
 912
 913         /* Iterate over nexthop neighbours, find those who are unresolved and
 914          * send arp on them. This solves the chicken-egg problem when
 915          * the nexthop wouldn't get offloaded until the neighbor is resolved
 916          * but it wouldn't get resolved ever in case traffic is flowing in HW
 917          * using different nexthop.
 918          *
 919          * Take RTNL mutex here to prevent lists from changes.
 920          */
 921         rtnl_lock();
 922         list_for_each_entry(neigh_entry, &mlxsw_sp->router.nexthop_neighs_list,
 923                             nexthop_neighs_list_node) {
 924                 if (!(neigh_entry->key.n->nud_state & NUD_VALID) &&
 925                     !list_empty(&neigh_entry->nexthop_list))
 926                         neigh_event_send(neigh_entry->key.n, NULL);
 927         }
 928         rtnl_unlock();
 929
 930         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw,
 931                                MLXSW_SP_UNRESOLVED_NH_PROBE_INTERVAL);
 932 }
 933
 934 static void
 935 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
 936                               struct mlxsw_sp_neigh_entry *neigh_entry,
 937                               bool removing);
 938
 939 static void mlxsw_sp_router_neigh_update_hw(struct work_struct *work)
 940 {
 941         struct mlxsw_sp_neigh_entry *neigh_entry =
 942                 container_of(work, struct mlxsw_sp_neigh_entry, dw.work);
 943         struct neighbour *n = neigh_entry->key.n;
 944         struct mlxsw_sp_port *mlxsw_sp_port = neigh_entry->mlxsw_sp_port;
 945         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 946         char rauht_pl[MLXSW_REG_RAUHT_LEN];
 947         struct net_device *dev;
 948         bool entry_connected;
 949         u8 nud_state;
 950         bool updating;
 951         bool removing;
 952         bool adding;
 953         u32 dip;
 954         int err;
 955
 956         read_lock_bh(&n->lock);
 957         dip = ntohl(*((__be32 *) n->primary_key));
 958         memcpy(neigh_entry->ha, n->ha, sizeof(neigh_entry->ha));
 959         nud_state = n->nud_state;
 960         dev = n->dev;
 961         read_unlock_bh(&n->lock);
 962
 963         entry_connected = nud_state & NUD_VALID;
 964         adding = (!neigh_entry->offloaded) && entry_connected;
 965         updating = neigh_entry->offloaded && entry_connected;
 966         removing = neigh_entry->offloaded && !entry_connected;
 967
 968         if (adding || updating) {
 969                 mlxsw_reg_rauht_pack4(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_ADD,
 970                                       neigh_entry->rif,
 971                                       neigh_entry->ha, dip);
 972                 err = mlxsw_reg_write(mlxsw_sp->core,
 973                                       MLXSW_REG(rauht), rauht_pl);
 974                 if (err) {
 975                         netdev_err(dev, "Could not add neigh %pI4h\n", &dip);
 976                         neigh_entry->offloaded = false;
 977                 } else {
 978                         neigh_entry->offloaded = true;
 979                 }
 980                 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, false);
 981         } else if (removing) {
 982                 mlxsw_reg_rauht_pack4(rauht_pl, MLXSW_REG_RAUHT_OP_WRITE_DELETE,
 983                                       neigh_entry->rif,
 984                                       neigh_entry->ha, dip);
 985                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rauht),
 986                                       rauht_pl);
 987                 if (err) {
 988                         netdev_err(dev, "Could not delete neigh %pI4h\n", &dip);
 989                         neigh_entry->offloaded = true;
 990                 } else {
 991                         neigh_entry->offloaded = false;
 992                 }
 993                 mlxsw_sp_nexthop_neigh_update(mlxsw_sp, neigh_entry, true);
 994         }
 995
 996         neigh_release(n);
 997         mlxsw_sp_port_dev_put(mlxsw_sp_port);
 998 }
 999
1000 int mlxsw_sp_router_netevent_event(struct notifier_block *unused,
1001                                    unsigned long event, void *ptr)
1002 {
1003         struct mlxsw_sp_neigh_entry *neigh_entry;
1004         struct mlxsw_sp_port *mlxsw_sp_port;
1005         struct mlxsw_sp *mlxsw_sp;
1006         unsigned long interval;
1007         struct net_device *dev;
1008         struct neigh_parms *p;
1009         struct neighbour *n;
1010         u32 dip;
1011
1012         switch (event) {
1013         case NETEVENT_DELAY_PROBE_TIME_UPDATE:
1014                 p = ptr;
1015
1016                 /* We don't care about changes in the default table. */
1017                 if (!p->dev || p->tbl != &arp_tbl)
1018                         return NOTIFY_DONE;
1019
1020                 /* We are in atomic context and can't take RTNL mutex,
1021                  * so use RCU variant to walk the device chain.
1022                  */
1023                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(p->dev);
1024                 if (!mlxsw_sp_port)
1025                         return NOTIFY_DONE;
1026
1027                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1028                 interval = jiffies_to_msecs(NEIGH_VAR(p, DELAY_PROBE_TIME));
1029                 mlxsw_sp->router.neighs_update.interval = interval;
1030
1031                 mlxsw_sp_port_dev_put(mlxsw_sp_port);
1032                 break;
1033         case NETEVENT_NEIGH_UPDATE:
1034                 n = ptr;
1035                 dev = n->dev;
1036
1037                 if (n->tbl != &arp_tbl)
1038                         return NOTIFY_DONE;
1039
1040                 mlxsw_sp_port = mlxsw_sp_port_lower_dev_hold(dev);
1041                 if (!mlxsw_sp_port)
1042                         return NOTIFY_DONE;
1043
1044                 mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1045                 dip = ntohl(*((__be32 *) n->primary_key));
1046                 neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1047                 if (WARN_ON(!neigh_entry)) {
1048                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1049                         return NOTIFY_DONE;
1050                 }
1051                 neigh_entry->mlxsw_sp_port = mlxsw_sp_port;
1052
1053                 /* Take a reference to ensure the neighbour won't be
1054                  * destructed until we drop the reference in delayed
1055                  * work.
1056                  */
1057                 neigh_clone(n);
1058                 if (!mlxsw_core_schedule_dw(&neigh_entry->dw, 0)) {
1059                         neigh_release(n);
1060                         mlxsw_sp_port_dev_put(mlxsw_sp_port);
1061                 }
1062                 break;
1063         }
1064
1065         return NOTIFY_DONE;
1066 }
1067
1068 static int mlxsw_sp_neigh_init(struct mlxsw_sp *mlxsw_sp)
1069 {
1070         int err;
1071
1072         err = rhashtable_init(&mlxsw_sp->router.neigh_ht,
1073                               &mlxsw_sp_neigh_ht_params);
1074         if (err)
1075                 return err;
1076
1077         /* Initialize the polling interval according to the default
1078          * table.
1079          */
1080         mlxsw_sp_router_neighs_update_interval_init(mlxsw_sp);
1081
1082         /* Create the delayed works for the activity_update */
1083         INIT_DELAYED_WORK(&mlxsw_sp->router.neighs_update.dw,
1084                           mlxsw_sp_router_neighs_update_work);
1085         INIT_DELAYED_WORK(&mlxsw_sp->router.nexthop_probe_dw,
1086                           mlxsw_sp_router_probe_unresolved_nexthops);
1087         mlxsw_core_schedule_dw(&mlxsw_sp->router.neighs_update.dw, 0);
1088         mlxsw_core_schedule_dw(&mlxsw_sp->router.nexthop_probe_dw, 0);
1089         return 0;
1090 }
1091
1092 static void mlxsw_sp_neigh_fini(struct mlxsw_sp *mlxsw_sp)
1093 {
1094         cancel_delayed_work_sync(&mlxsw_sp->router.neighs_update.dw);
1095         cancel_delayed_work_sync(&mlxsw_sp->router.nexthop_probe_dw);
1096         rhashtable_destroy(&mlxsw_sp->router.neigh_ht);
1097 }
1098
1099 struct mlxsw_sp_nexthop {
1100         struct list_head neigh_list_node; /* member of neigh entry list */
1101         struct mlxsw_sp_nexthop_group *nh_grp; /* pointer back to the group
1102                                                 * this belongs to
1103                                                 */
1104         u8 should_offload:1, /* set indicates this neigh is connected and
1105                               * should be put to KVD linear area of this group.
1106                               */
1107            offloaded:1, /* set in case the neigh is actually put into
1108                          * KVD linear area of this group.
1109                          */
1110            update:1; /* set indicates that MAC of this neigh should be
1111                       * updated in HW
1112                       */
1113         struct mlxsw_sp_neigh_entry *neigh_entry;
1114 };
1115
1116 struct mlxsw_sp_nexthop_group {
1117         struct list_head list; /* node in mlxsw->router.nexthop_group_list */
1118         struct list_head fib_list; /* list of fib entries that use this group */
1119         u8 adj_index_valid:1;
1120         u32 adj_index;
1121         u16 ecmp_size;
1122         u16 count;
1123         struct mlxsw_sp_nexthop nexthops[0];
1124 };
1125
1126 static int mlxsw_sp_adj_index_mass_update_vr(struct mlxsw_sp *mlxsw_sp,
1127                                              struct mlxsw_sp_vr *vr,
1128                                              u32 adj_index, u16 ecmp_size,
1129                                              u32 new_adj_index,
1130                                              u16 new_ecmp_size)
1131 {
1132         char raleu_pl[MLXSW_REG_RALEU_LEN];
1133
1134         mlxsw_reg_raleu_pack(raleu_pl,
1135                              (enum mlxsw_reg_ralxx_protocol) vr->proto, vr->id,
1136                              adj_index, ecmp_size, new_adj_index,
1137                              new_ecmp_size);
1138         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raleu), raleu_pl);
1139 }
1140
1141 static int mlxsw_sp_adj_index_mass_update(struct mlxsw_sp *mlxsw_sp,
1142                                           struct mlxsw_sp_nexthop_group *nh_grp,
1143                                           u32 old_adj_index, u16 old_ecmp_size)
1144 {
1145         struct mlxsw_sp_fib_entry *fib_entry;
1146         struct mlxsw_sp_vr *vr = NULL;
1147         int err;
1148
1149         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1150                 if (vr == fib_entry->vr)
1151                         continue;
1152                 vr = fib_entry->vr;
1153                 err = mlxsw_sp_adj_index_mass_update_vr(mlxsw_sp, vr,
1154                                                         old_adj_index,
1155                                                         old_ecmp_size,
1156                                                         nh_grp->adj_index,
1157                                                         nh_grp->ecmp_size);
1158                 if (err)
1159                         return err;
1160         }
1161         return 0;
1162 }
1163
1164 static int mlxsw_sp_nexthop_mac_update(struct mlxsw_sp *mlxsw_sp, u32 adj_index,
1165                                        struct mlxsw_sp_nexthop *nh)
1166 {
1167         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1168         char ratr_pl[MLXSW_REG_RATR_LEN];
1169
1170         mlxsw_reg_ratr_pack(ratr_pl, MLXSW_REG_RATR_OP_WRITE_WRITE_ENTRY,
1171                             true, adj_index, neigh_entry->rif);
1172         mlxsw_reg_ratr_eth_entry_pack(ratr_pl, neigh_entry->ha);
1173         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ratr), ratr_pl);
1174 }
1175
1176 static int
1177 mlxsw_sp_nexthop_group_mac_update(struct mlxsw_sp *mlxsw_sp,
1178                                   struct mlxsw_sp_nexthop_group *nh_grp,
1179                                   bool reallocate)
1180 {
1181         u32 adj_index = nh_grp->adj_index; /* base */
1182         struct mlxsw_sp_nexthop *nh;
1183         int i;
1184         int err;
1185
1186         for (i = 0; i < nh_grp->count; i++) {
1187                 nh = &nh_grp->nexthops[i];
1188
1189                 if (!nh->should_offload) {
1190                         nh->offloaded = 0;
1191                         continue;
1192                 }
1193
1194                 if (nh->update || reallocate) {
1195                         err = mlxsw_sp_nexthop_mac_update(mlxsw_sp,
1196                                                           adj_index, nh);
1197                         if (err)
1198                                 return err;
1199                         nh->update = 0;
1200                         nh->offloaded = 1;
1201                 }
1202                 adj_index++;
1203         }
1204         return 0;
1205 }
1206
1207 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1208                                      struct mlxsw_sp_fib_entry *fib_entry);
1209
1210 static int
1211 mlxsw_sp_nexthop_fib_entries_update(struct mlxsw_sp *mlxsw_sp,
1212                                     struct mlxsw_sp_nexthop_group *nh_grp)
1213 {
1214         struct mlxsw_sp_fib_entry *fib_entry;
1215         int err;
1216
1217         list_for_each_entry(fib_entry, &nh_grp->fib_list, nexthop_group_node) {
1218                 err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1219                 if (err)
1220                         return err;
1221         }
1222         return 0;
1223 }
1224
1225 static void
1226 mlxsw_sp_nexthop_group_refresh(struct mlxsw_sp *mlxsw_sp,
1227                                struct mlxsw_sp_nexthop_group *nh_grp)
1228 {
1229         struct mlxsw_sp_nexthop *nh;
1230         bool offload_change = false;
1231         u32 adj_index;
1232         u16 ecmp_size = 0;
1233         bool old_adj_index_valid;
1234         u32 old_adj_index;
1235         u16 old_ecmp_size;
1236         int ret;
1237         int i;
1238         int err;
1239
1240         for (i = 0; i < nh_grp->count; i++) {
1241                 nh = &nh_grp->nexthops[i];
1242
1243                 if (nh->should_offload ^ nh->offloaded) {
1244                         offload_change = true;
1245                         if (nh->should_offload)
1246                                 nh->update = 1;
1247                 }
1248                 if (nh->should_offload)
1249                         ecmp_size++;
1250         }
1251         if (!offload_change) {
1252                 /* Nothing was added or removed, so no need to reallocate. Just
1253                  * update MAC on existing adjacency indexes.
1254                  */
1255                 err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp,
1256                                                         false);
1257                 if (err) {
1258                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1259                         goto set_trap;
1260                 }
1261                 return;
1262         }
1263         if (!ecmp_size)
1264                 /* No neigh of this group is connected so we just set
1265                  * the trap and let everthing flow through kernel.
1266                  */
1267                 goto set_trap;
1268
1269         ret = mlxsw_sp_kvdl_alloc(mlxsw_sp, ecmp_size);
1270         if (ret < 0) {
1271                 /* We ran out of KVD linear space, just set the
1272                  * trap and let everything flow through kernel.
1273                  */
1274                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to allocate KVD linear area for nexthop group.\n");
1275                 goto set_trap;
1276         }
1277         adj_index = ret;
1278         old_adj_index_valid = nh_grp->adj_index_valid;
1279         old_adj_index = nh_grp->adj_index;
1280         old_ecmp_size = nh_grp->ecmp_size;
1281         nh_grp->adj_index_valid = 1;
1282         nh_grp->adj_index = adj_index;
1283         nh_grp->ecmp_size = ecmp_size;
1284         err = mlxsw_sp_nexthop_group_mac_update(mlxsw_sp, nh_grp, true);
1285         if (err) {
1286                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to update neigh MAC in adjacency table.\n");
1287                 goto set_trap;
1288         }
1289
1290         if (!old_adj_index_valid) {
1291                 /* The trap was set for fib entries, so we have to call
1292                  * fib entry update to unset it and use adjacency index.
1293                  */
1294                 err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1295                 if (err) {
1296                         dev_warn(mlxsw_sp->bus_info->dev, "Failed to add adjacency index to fib entries.\n");
1297                         goto set_trap;
1298                 }
1299                 return;
1300         }
1301
1302         err = mlxsw_sp_adj_index_mass_update(mlxsw_sp, nh_grp,
1303                                              old_adj_index, old_ecmp_size);
1304         mlxsw_sp_kvdl_free(mlxsw_sp, old_adj_index);
1305         if (err) {
1306                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to mass-update adjacency index for nexthop group.\n");
1307                 goto set_trap;
1308         }
1309         return;
1310
1311 set_trap:
1312         old_adj_index_valid = nh_grp->adj_index_valid;
1313         nh_grp->adj_index_valid = 0;
1314         for (i = 0; i < nh_grp->count; i++) {
1315                 nh = &nh_grp->nexthops[i];
1316                 nh->offloaded = 0;
1317         }
1318         err = mlxsw_sp_nexthop_fib_entries_update(mlxsw_sp, nh_grp);
1319         if (err)
1320                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set traps for fib entries.\n");
1321         if (old_adj_index_valid)
1322                 mlxsw_sp_kvdl_free(mlxsw_sp, nh_grp->adj_index);
1323 }
1324
1325 static void __mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp_nexthop *nh,
1326                                             bool removing)
1327 {
1328         if (!removing)
1329                 nh->should_offload = 1;
1330         else
1331                 nh->should_offload = 0;
1332         nh->update = 1;
1333 }
1334
1335 static void
1336 mlxsw_sp_nexthop_neigh_update(struct mlxsw_sp *mlxsw_sp,
1337                               struct mlxsw_sp_neigh_entry *neigh_entry,
1338                               bool removing)
1339 {
1340         struct mlxsw_sp_nexthop *nh;
1341
1342         /* Take RTNL mutex here to prevent lists from changes */
1343         rtnl_lock();
1344         list_for_each_entry(nh, &neigh_entry->nexthop_list,
1345                             neigh_list_node) {
1346                 __mlxsw_sp_nexthop_neigh_update(nh, removing);
1347                 mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh->nh_grp);
1348         }
1349         rtnl_unlock();
1350 }
1351
1352 static int mlxsw_sp_nexthop_init(struct mlxsw_sp *mlxsw_sp,
1353                                  struct mlxsw_sp_nexthop_group *nh_grp,
1354                                  struct mlxsw_sp_nexthop *nh,
1355                                  struct fib_nh *fib_nh)
1356 {
1357         struct mlxsw_sp_neigh_entry *neigh_entry;
1358         struct net_device *dev = fib_nh->nh_dev;
1359         struct neighbour *n;
1360         u8 nud_state;
1361
1362         /* Take a reference of neigh here ensuring that neigh would
1363          * not be detructed before the nexthop entry is finished.
1364          * The reference is taken either in neigh_lookup() or
1365          * in neith_create() in case n is not found.
1366          */
1367         n = neigh_lookup(&arp_tbl, &fib_nh->nh_gw, dev);
1368         if (!n) {
1369                 n = neigh_create(&arp_tbl, &fib_nh->nh_gw, dev);
1370                 if (IS_ERR(n))
1371                         return PTR_ERR(n);
1372                 neigh_event_send(n, NULL);
1373         }
1374         neigh_entry = mlxsw_sp_neigh_entry_lookup(mlxsw_sp, n);
1375         if (!neigh_entry) {
1376                 neigh_release(n);
1377                 return -EINVAL;
1378         }
1379
1380         /* If that is the first nexthop connected to that neigh, add to
1381          * nexthop_neighs_list
1382          */
1383         if (list_empty(&neigh_entry->nexthop_list))
1384                 list_add_tail(&neigh_entry->nexthop_neighs_list_node,
1385                               &mlxsw_sp->router.nexthop_neighs_list);
1386
1387         nh->nh_grp = nh_grp;
1388         nh->neigh_entry = neigh_entry;
1389         list_add_tail(&nh->neigh_list_node, &neigh_entry->nexthop_list);
1390         read_lock_bh(&n->lock);
1391         nud_state = n->nud_state;
1392         read_unlock_bh(&n->lock);
1393         __mlxsw_sp_nexthop_neigh_update(nh, !(nud_state & NUD_VALID));
1394
1395         return 0;
1396 }
1397
1398 static void mlxsw_sp_nexthop_fini(struct mlxsw_sp *mlxsw_sp,
1399                                   struct mlxsw_sp_nexthop *nh)
1400 {
1401         struct mlxsw_sp_neigh_entry *neigh_entry = nh->neigh_entry;
1402
1403         list_del(&nh->neigh_list_node);
1404
1405         /* If that is the last nexthop connected to that neigh, remove from
1406          * nexthop_neighs_list
1407          */
1408         if (list_empty(&nh->neigh_entry->nexthop_list))
1409                 list_del(&nh->neigh_entry->nexthop_neighs_list_node);
1410
1411         neigh_release(neigh_entry->key.n);
1412 }
1413
1414 static struct mlxsw_sp_nexthop_group *
1415 mlxsw_sp_nexthop_group_create(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1416 {
1417         struct mlxsw_sp_nexthop_group *nh_grp;
1418         struct mlxsw_sp_nexthop *nh;
1419         struct fib_nh *fib_nh;
1420         size_t alloc_size;
1421         int i;
1422         int err;
1423
1424         alloc_size = sizeof(*nh_grp) +
1425                      fi->fib_nhs * sizeof(struct mlxsw_sp_nexthop);
1426         nh_grp = kzalloc(alloc_size, GFP_KERNEL);
1427         if (!nh_grp)
1428                 return ERR_PTR(-ENOMEM);
1429         INIT_LIST_HEAD(&nh_grp->fib_list);
1430         nh_grp->count = fi->fib_nhs;
1431         for (i = 0; i < nh_grp->count; i++) {
1432                 nh = &nh_grp->nexthops[i];
1433                 fib_nh = &fi->fib_nh[i];
1434                 err = mlxsw_sp_nexthop_init(mlxsw_sp, nh_grp, nh, fib_nh);
1435                 if (err)
1436                         goto err_nexthop_init;
1437         }
1438         list_add_tail(&nh_grp->list, &mlxsw_sp->router.nexthop_group_list);
1439         mlxsw_sp_nexthop_group_refresh(mlxsw_sp, nh_grp);
1440         return nh_grp;
1441
1442 err_nexthop_init:
1443         for (i--; i >= 0; i--)
1444                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1445         kfree(nh_grp);
1446         return ERR_PTR(err);
1447 }
1448
1449 static void
1450 mlxsw_sp_nexthop_group_destroy(struct mlxsw_sp *mlxsw_sp,
1451                                struct mlxsw_sp_nexthop_group *nh_grp)
1452 {
1453         struct mlxsw_sp_nexthop *nh;
1454         int i;
1455
1456         list_del(&nh_grp->list);
1457         for (i = 0; i < nh_grp->count; i++) {
1458                 nh = &nh_grp->nexthops[i];
1459                 mlxsw_sp_nexthop_fini(mlxsw_sp, nh);
1460         }
1461         kfree(nh_grp);
1462 }
1463
1464 static bool mlxsw_sp_nexthop_match(struct mlxsw_sp_nexthop *nh,
1465                                    struct fib_info *fi)
1466 {
1467         int i;
1468
1469         for (i = 0; i < fi->fib_nhs; i++) {
1470                 struct fib_nh *fib_nh = &fi->fib_nh[i];
1471                 struct neighbour *n = nh->neigh_entry->key.n;
1472
1473                 if (memcmp(n->primary_key, &fib_nh->nh_gw,
1474                            sizeof(fib_nh->nh_gw)) == 0 &&
1475                     n->dev == fib_nh->nh_dev)
1476                         return true;
1477         }
1478         return false;
1479 }
1480
1481 static bool mlxsw_sp_nexthop_group_match(struct mlxsw_sp_nexthop_group *nh_grp,
1482                                          struct fib_info *fi)
1483 {
1484         int i;
1485
1486         if (nh_grp->count != fi->fib_nhs)
1487                 return false;
1488         for (i = 0; i < nh_grp->count; i++) {
1489                 struct mlxsw_sp_nexthop *nh = &nh_grp->nexthops[i];
1490
1491                 if (!mlxsw_sp_nexthop_match(nh, fi))
1492                         return false;
1493         }
1494         return true;
1495 }
1496
1497 static struct mlxsw_sp_nexthop_group *
1498 mlxsw_sp_nexthop_group_find(struct mlxsw_sp *mlxsw_sp, struct fib_info *fi)
1499 {
1500         struct mlxsw_sp_nexthop_group *nh_grp;
1501
1502         list_for_each_entry(nh_grp, &mlxsw_sp->router.nexthop_group_list,
1503                             list) {
1504                 if (mlxsw_sp_nexthop_group_match(nh_grp, fi))
1505                         return nh_grp;
1506         }
1507         return NULL;
1508 }
1509
1510 static int mlxsw_sp_nexthop_group_get(struct mlxsw_sp *mlxsw_sp,
1511                                       struct mlxsw_sp_fib_entry *fib_entry,
1512                                       struct fib_info *fi)
1513 {
1514         struct mlxsw_sp_nexthop_group *nh_grp;
1515
1516         nh_grp = mlxsw_sp_nexthop_group_find(mlxsw_sp, fi);
1517         if (!nh_grp) {
1518                 nh_grp = mlxsw_sp_nexthop_group_create(mlxsw_sp, fi);
1519                 if (IS_ERR(nh_grp))
1520                         return PTR_ERR(nh_grp);
1521         }
1522         list_add_tail(&fib_entry->nexthop_group_node, &nh_grp->fib_list);
1523         fib_entry->nh_group = nh_grp;
1524         return 0;
1525 }
1526
1527 static void mlxsw_sp_nexthop_group_put(struct mlxsw_sp *mlxsw_sp,
1528                                        struct mlxsw_sp_fib_entry *fib_entry)
1529 {
1530         struct mlxsw_sp_nexthop_group *nh_grp = fib_entry->nh_group;
1531
1532         list_del(&fib_entry->nexthop_group_node);
1533         if (!list_empty(&nh_grp->fib_list))
1534                 return;
1535         mlxsw_sp_nexthop_group_destroy(mlxsw_sp, nh_grp);
1536 }
1537
1538 static int mlxsw_sp_fib_entry_op4_remote(struct mlxsw_sp *mlxsw_sp,
1539                                          struct mlxsw_sp_fib_entry *fib_entry,
1540                                          enum mlxsw_reg_ralue_op op)
1541 {
1542         char ralue_pl[MLXSW_REG_RALUE_LEN];
1543         u32 *p_dip = (u32 *) fib_entry->key.addr;
1544         struct mlxsw_sp_vr *vr = fib_entry->vr;
1545         enum mlxsw_reg_ralue_trap_action trap_action;
1546         u16 trap_id = 0;
1547         u32 adjacency_index = 0;
1548         u16 ecmp_size = 0;
1549
1550         /* In case the nexthop group adjacency index is valid, use it
1551          * with provided ECMP size. Otherwise, setup trap and pass
1552          * traffic to kernel.
1553          */
1554         if (fib_entry->nh_group->adj_index_valid) {
1555                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_NOP;
1556                 adjacency_index = fib_entry->nh_group->adj_index;
1557                 ecmp_size = fib_entry->nh_group->ecmp_size;
1558         } else {
1559                 trap_action = MLXSW_REG_RALUE_TRAP_ACTION_TRAP;
1560                 trap_id = MLXSW_TRAP_ID_RTR_INGRESS0;
1561         }
1562
1563         mlxsw_reg_ralue_pack4(ralue_pl,
1564                               (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1565                               vr->id, fib_entry->key.prefix_len, *p_dip);
1566         mlxsw_reg_ralue_act_remote_pack(ralue_pl, trap_action, trap_id,
1567                                         adjacency_index, ecmp_size);
1568         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1569 }
1570
1571 static int mlxsw_sp_fib_entry_op4_local(struct mlxsw_sp *mlxsw_sp,
1572                                         struct mlxsw_sp_fib_entry *fib_entry,
1573                                         enum mlxsw_reg_ralue_op op)
1574 {
1575         char ralue_pl[MLXSW_REG_RALUE_LEN];
1576         u32 *p_dip = (u32 *) fib_entry->key.addr;
1577         struct mlxsw_sp_vr *vr = fib_entry->vr;
1578
1579         mlxsw_reg_ralue_pack4(ralue_pl,
1580                               (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1581                               vr->id, fib_entry->key.prefix_len, *p_dip);
1582         mlxsw_reg_ralue_act_local_pack(ralue_pl,
1583                                        MLXSW_REG_RALUE_TRAP_ACTION_NOP, 0,
1584                                        fib_entry->rif);
1585         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1586 }
1587
1588 static int mlxsw_sp_fib_entry_op4_trap(struct mlxsw_sp *mlxsw_sp,
1589                                        struct mlxsw_sp_fib_entry *fib_entry,
1590                                        enum mlxsw_reg_ralue_op op)
1591 {
1592         char ralue_pl[MLXSW_REG_RALUE_LEN];
1593         u32 *p_dip = (u32 *) fib_entry->key.addr;
1594         struct mlxsw_sp_vr *vr = fib_entry->vr;
1595
1596         mlxsw_reg_ralue_pack4(ralue_pl,
1597                               (enum mlxsw_reg_ralxx_protocol) vr->proto, op,
1598                               vr->id, fib_entry->key.prefix_len, *p_dip);
1599         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1600         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1601 }
1602
1603 static int mlxsw_sp_fib_entry_op4(struct mlxsw_sp *mlxsw_sp,
1604                                   struct mlxsw_sp_fib_entry *fib_entry,
1605                                   enum mlxsw_reg_ralue_op op)
1606 {
1607         switch (fib_entry->type) {
1608         case MLXSW_SP_FIB_ENTRY_TYPE_REMOTE:
1609                 return mlxsw_sp_fib_entry_op4_remote(mlxsw_sp, fib_entry, op);
1610         case MLXSW_SP_FIB_ENTRY_TYPE_LOCAL:
1611                 return mlxsw_sp_fib_entry_op4_local(mlxsw_sp, fib_entry, op);
1612         case MLXSW_SP_FIB_ENTRY_TYPE_TRAP:
1613                 return mlxsw_sp_fib_entry_op4_trap(mlxsw_sp, fib_entry, op);
1614         }
1615         return -EINVAL;
1616 }
1617
1618 static int mlxsw_sp_fib_entry_op(struct mlxsw_sp *mlxsw_sp,
1619                                  struct mlxsw_sp_fib_entry *fib_entry,
1620                                  enum mlxsw_reg_ralue_op op)
1621 {
1622         switch (fib_entry->vr->proto) {
1623         case MLXSW_SP_L3_PROTO_IPV4:
1624                 return mlxsw_sp_fib_entry_op4(mlxsw_sp, fib_entry, op);
1625         case MLXSW_SP_L3_PROTO_IPV6:
1626                 return -EINVAL;
1627         }
1628         return -EINVAL;
1629 }
1630
1631 static int mlxsw_sp_fib_entry_update(struct mlxsw_sp *mlxsw_sp,
1632                                      struct mlxsw_sp_fib_entry *fib_entry)
1633 {
1634         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1635                                      MLXSW_REG_RALUE_OP_WRITE_WRITE);
1636 }
1637
1638 static int mlxsw_sp_fib_entry_del(struct mlxsw_sp *mlxsw_sp,
1639                                   struct mlxsw_sp_fib_entry *fib_entry)
1640 {
1641         return mlxsw_sp_fib_entry_op(mlxsw_sp, fib_entry,
1642                                      MLXSW_REG_RALUE_OP_WRITE_DELETE);
1643 }
1644
1645 static int
1646 mlxsw_sp_router_fib4_entry_init(struct mlxsw_sp *mlxsw_sp,
1647                                 const struct fib_entry_notifier_info *fen_info,
1648                                 struct mlxsw_sp_fib_entry *fib_entry)
1649 {
1650         struct fib_info *fi = fen_info->fi;
1651         struct mlxsw_sp_rif *r = NULL;
1652         int nhsel;
1653         int err;
1654
1655         if (fen_info->type == RTN_LOCAL || fen_info->type == RTN_BROADCAST) {
1656                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1657                 return 0;
1658         }
1659         if (fen_info->type != RTN_UNICAST)
1660                 return -EINVAL;
1661
1662         for (nhsel = 0; nhsel < fi->fib_nhs; nhsel++) {
1663                 const struct fib_nh *nh = &fi->fib_nh[nhsel];
1664
1665                 if (!nh->nh_dev)
1666                         continue;
1667                 r = mlxsw_sp_rif_find_by_dev(mlxsw_sp, nh->nh_dev);
1668                 if (!r) {
1669                         /* In case router interface is not found for
1670                          * at least one of the nexthops, that means
1671                          * the nexthop points to some device unrelated
1672                          * to us. Set trap and pass the packets for
1673                          * this prefix to kernel.
1674                          */
1675                         break;
1676                 }
1677         }
1678
1679         if (!r) {
1680                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_TRAP;
1681                 return 0;
1682         }
1683
1684         if (fi->fib_scope != RT_SCOPE_UNIVERSE) {
1685                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_LOCAL;
1686                 fib_entry->rif = r->rif;
1687         } else {
1688                 fib_entry->type = MLXSW_SP_FIB_ENTRY_TYPE_REMOTE;
1689                 err = mlxsw_sp_nexthop_group_get(mlxsw_sp, fib_entry, fi);
1690                 if (err)
1691                         return err;
1692         }
1693         fib_info_offload_inc(fen_info->fi);
1694         return 0;
1695 }
1696
1697 static void
1698 mlxsw_sp_router_fib4_entry_fini(struct mlxsw_sp *mlxsw_sp,
1699                                 struct mlxsw_sp_fib_entry *fib_entry)
1700 {
1701         if (fib_entry->type != MLXSW_SP_FIB_ENTRY_TYPE_TRAP)
1702                 fib_info_offload_dec(fib_entry->fi);
1703         if (fib_entry->type == MLXSW_SP_FIB_ENTRY_TYPE_REMOTE)
1704                 mlxsw_sp_nexthop_group_put(mlxsw_sp, fib_entry);
1705 }
1706
1707 static struct mlxsw_sp_fib_entry *
1708 mlxsw_sp_fib_entry_get(struct mlxsw_sp *mlxsw_sp,
1709                        const struct fib_entry_notifier_info *fen_info)
1710 {
1711         struct mlxsw_sp_fib_entry *fib_entry;
1712         struct fib_info *fi = fen_info->fi;
1713         struct mlxsw_sp_vr *vr;
1714         int err;
1715
1716         vr = mlxsw_sp_vr_get(mlxsw_sp, fen_info->dst_len, fen_info->tb_id,
1717                              MLXSW_SP_L3_PROTO_IPV4);
1718         if (IS_ERR(vr))
1719                 return ERR_CAST(vr);
1720
1721         fib_entry = mlxsw_sp_fib_entry_lookup(vr->fib, &fen_info->dst,
1722                                               sizeof(fen_info->dst),
1723                                               fen_info->dst_len, fi->fib_dev);
1724         if (fib_entry) {
1725                 /* Already exists, just take a reference */
1726                 fib_entry->ref_count++;
1727                 return fib_entry;
1728         }
1729         fib_entry = mlxsw_sp_fib_entry_create(vr->fib, &fen_info->dst,
1730                                               sizeof(fen_info->dst),
1731                                               fen_info->dst_len, fi->fib_dev);
1732         if (!fib_entry) {
1733                 err = -ENOMEM;
1734                 goto err_fib_entry_create;
1735         }
1736         fib_entry->vr = vr;
1737         fib_entry->fi = fi;
1738         fib_entry->ref_count = 1;
1739
1740         err = mlxsw_sp_router_fib4_entry_init(mlxsw_sp, fen_info, fib_entry);
1741         if (err)
1742                 goto err_fib4_entry_init;
1743
1744         return fib_entry;
1745
1746 err_fib4_entry_init:
1747         mlxsw_sp_fib_entry_destroy(fib_entry);
1748 err_fib_entry_create:
1749         mlxsw_sp_vr_put(mlxsw_sp, vr);
1750
1751         return ERR_PTR(err);
1752 }
1753
1754 static struct mlxsw_sp_fib_entry *
1755 mlxsw_sp_fib_entry_find(struct mlxsw_sp *mlxsw_sp,
1756                         const struct fib_entry_notifier_info *fen_info)
1757 {
1758         struct mlxsw_sp_vr *vr;
1759
1760         vr = mlxsw_sp_vr_find(mlxsw_sp, fen_info->tb_id,
1761                               MLXSW_SP_L3_PROTO_IPV4);
1762         if (!vr)
1763                 return NULL;
1764
1765         return mlxsw_sp_fib_entry_lookup(vr->fib, &fen_info->dst,
1766                                          sizeof(fen_info->dst),
1767                                          fen_info->dst_len,
1768                                          fen_info->fi->fib_dev);
1769 }
1770
1771 static void mlxsw_sp_fib_entry_put(struct mlxsw_sp *mlxsw_sp,
1772                                    struct mlxsw_sp_fib_entry *fib_entry)
1773 {
1774         struct mlxsw_sp_vr *vr = fib_entry->vr;
1775
1776         if (--fib_entry->ref_count == 0) {
1777                 mlxsw_sp_router_fib4_entry_fini(mlxsw_sp, fib_entry);
1778                 mlxsw_sp_fib_entry_destroy(fib_entry);
1779         }
1780         mlxsw_sp_vr_put(mlxsw_sp, vr);
1781 }
1782
1783 static void mlxsw_sp_fib_entry_put_all(struct mlxsw_sp *mlxsw_sp,
1784                                        struct mlxsw_sp_fib_entry *fib_entry)
1785 {
1786         unsigned int last_ref_count;
1787
1788         do {
1789                 last_ref_count = fib_entry->ref_count;
1790                 mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
1791         } while (last_ref_count != 1);
1792 }
1793
1794 static int mlxsw_sp_router_fib4_add(struct mlxsw_sp *mlxsw_sp,
1795                                     struct fib_entry_notifier_info *fen_info)
1796 {
1797         struct mlxsw_sp_fib_entry *fib_entry;
1798         struct mlxsw_sp_vr *vr;
1799         int err;
1800
1801         if (mlxsw_sp->router.aborted)
1802                 return 0;
1803
1804         fib_entry = mlxsw_sp_fib_entry_get(mlxsw_sp, fen_info);
1805         if (IS_ERR(fib_entry)) {
1806                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to get FIB4 entry being added.\n");
1807                 return PTR_ERR(fib_entry);
1808         }
1809
1810         if (fib_entry->ref_count != 1)
1811                 return 0;
1812
1813         vr = fib_entry->vr;
1814         err = mlxsw_sp_fib_entry_insert(vr->fib, fib_entry);
1815         if (err) {
1816                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to insert FIB4 entry being added.\n");
1817                 goto err_fib_entry_insert;
1818         }
1819         err = mlxsw_sp_fib_entry_update(mlxsw_sp, fib_entry);
1820         if (err)
1821                 goto err_fib_entry_add;
1822         return 0;
1823
1824 err_fib_entry_add:
1825         mlxsw_sp_fib_entry_remove(vr->fib, fib_entry);
1826 err_fib_entry_insert:
1827         mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
1828         return err;
1829 }
1830
1831 static void mlxsw_sp_router_fib4_del(struct mlxsw_sp *mlxsw_sp,
1832                                      struct fib_entry_notifier_info *fen_info)
1833 {
1834         struct mlxsw_sp_fib_entry *fib_entry;
1835
1836         if (mlxsw_sp->router.aborted)
1837                 return;
1838
1839         fib_entry = mlxsw_sp_fib_entry_find(mlxsw_sp, fen_info);
1840         if (!fib_entry)
1841                 return;
1842
1843         if (fib_entry->ref_count == 1) {
1844                 mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
1845                 mlxsw_sp_fib_entry_remove(fib_entry->vr->fib, fib_entry);
1846         }
1847
1848         mlxsw_sp_fib_entry_put(mlxsw_sp, fib_entry);
1849 }
1850
1851 static int mlxsw_sp_router_set_abort_trap(struct mlxsw_sp *mlxsw_sp)
1852 {
1853         char ralta_pl[MLXSW_REG_RALTA_LEN];
1854         char ralst_pl[MLXSW_REG_RALST_LEN];
1855         char raltb_pl[MLXSW_REG_RALTB_LEN];
1856         char ralue_pl[MLXSW_REG_RALUE_LEN];
1857         int err;
1858
1859         mlxsw_reg_ralta_pack(ralta_pl, true, MLXSW_REG_RALXX_PROTOCOL_IPV4,
1860                              MLXSW_SP_LPM_TREE_MIN);
1861         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralta), ralta_pl);
1862         if (err)
1863                 return err;
1864
1865         mlxsw_reg_ralst_pack(ralst_pl, 0xff, MLXSW_SP_LPM_TREE_MIN);
1866         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralst), ralst_pl);
1867         if (err)
1868                 return err;
1869
1870         mlxsw_reg_raltb_pack(raltb_pl, 0, MLXSW_REG_RALXX_PROTOCOL_IPV4,
1871                              MLXSW_SP_LPM_TREE_MIN);
1872         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(raltb), raltb_pl);
1873         if (err)
1874                 return err;
1875
1876         mlxsw_reg_ralue_pack4(ralue_pl, MLXSW_SP_L3_PROTO_IPV4,
1877                               MLXSW_REG_RALUE_OP_WRITE_WRITE, 0, 0, 0);
1878         mlxsw_reg_ralue_act_ip2me_pack(ralue_pl);
1879         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ralue), ralue_pl);
1880 }
1881
1882 static void mlxsw_sp_router_fib_flush(struct mlxsw_sp *mlxsw_sp)
1883 {
1884         struct mlxsw_resources *resources;
1885         struct mlxsw_sp_fib_entry *fib_entry;
1886         struct mlxsw_sp_fib_entry *tmp;
1887         struct mlxsw_sp_vr *vr;
1888         int i;
1889
1890         resources = mlxsw_core_resources_get(mlxsw_sp->core);
1891         for (i = 0; i < resources->max_virtual_routers; i++) {
1892                 vr = &mlxsw_sp->router.vrs[i];
1893
1894                 if (!vr->used)
1895                         continue;
1896
1897                 list_for_each_entry_safe(fib_entry, tmp,
1898                                          &vr->fib->entry_list, list) {
1899                         bool do_break = &tmp->list == &vr->fib->entry_list;
1900
1901                         mlxsw_sp_fib_entry_del(mlxsw_sp, fib_entry);
1902                         mlxsw_sp_fib_entry_remove(fib_entry->vr->fib,
1903                                                   fib_entry);
1904                         mlxsw_sp_fib_entry_put_all(mlxsw_sp, fib_entry);
1905                         if (do_break)
1906                                 break;
1907                 }
1908         }
1909 }
1910
1911 static void mlxsw_sp_router_fib4_abort(struct mlxsw_sp *mlxsw_sp)
1912 {
1913         int err;
1914
1915         mlxsw_sp_router_fib_flush(mlxsw_sp);
1916         mlxsw_sp->router.aborted = true;
1917         err = mlxsw_sp_router_set_abort_trap(mlxsw_sp);
1918         if (err)
1919                 dev_warn(mlxsw_sp->bus_info->dev, "Failed to set abort trap.\n");
1920 }
1921
1922 static int __mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
1923 {
1924         struct mlxsw_resources *resources;
1925         char rgcr_pl[MLXSW_REG_RGCR_LEN];
1926         int err;
1927
1928         resources = mlxsw_core_resources_get(mlxsw_sp->core);
1929         if (!resources->max_rif_valid)
1930                 return -EIO;
1931
1932         mlxsw_sp->rifs = kcalloc(resources->max_rif,
1933                                  sizeof(struct mlxsw_sp_rif *), GFP_KERNEL);
1934         if (!mlxsw_sp->rifs)
1935                 return -ENOMEM;
1936
1937         mlxsw_reg_rgcr_pack(rgcr_pl, true);
1938         mlxsw_reg_rgcr_max_router_interfaces_set(rgcr_pl, resources->max_rif);
1939         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
1940         if (err)
1941                 goto err_rgcr_fail;
1942
1943         return 0;
1944
1945 err_rgcr_fail:
1946         kfree(mlxsw_sp->rifs);
1947         return err;
1948 }
1949
1950 static void __mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
1951 {
1952         struct mlxsw_resources *resources;
1953         char rgcr_pl[MLXSW_REG_RGCR_LEN];
1954         int i;
1955
1956         mlxsw_reg_rgcr_pack(rgcr_pl, false);
1957         mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(rgcr), rgcr_pl);
1958
1959         resources = mlxsw_core_resources_get(mlxsw_sp->core);
1960         for (i = 0; i < resources->max_rif; i++)
1961                 WARN_ON_ONCE(mlxsw_sp->rifs[i]);
1962
1963         kfree(mlxsw_sp->rifs);
1964 }
1965
1966 static int mlxsw_sp_router_fib_event(struct notifier_block *nb,
1967                                      unsigned long event, void *ptr)
1968 {
1969         struct mlxsw_sp *mlxsw_sp = container_of(nb, struct mlxsw_sp, fib_nb);
1970         struct fib_entry_notifier_info *fen_info = ptr;
1971         int err;
1972
1973         if (!net_eq(fen_info->info.net, &init_net))
1974                 return NOTIFY_DONE;
1975
1976         switch (event) {
1977         case FIB_EVENT_ENTRY_ADD:
1978                 err = mlxsw_sp_router_fib4_add(mlxsw_sp, fen_info);
1979                 if (err)
1980                         mlxsw_sp_router_fib4_abort(mlxsw_sp);
1981                 break;
1982         case FIB_EVENT_ENTRY_DEL:
1983                 mlxsw_sp_router_fib4_del(mlxsw_sp, fen_info);
1984                 break;
1985         case FIB_EVENT_RULE_ADD: /* fall through */
1986         case FIB_EVENT_RULE_DEL:
1987                 mlxsw_sp_router_fib4_abort(mlxsw_sp);
1988                 break;
1989         }
1990         return NOTIFY_DONE;
1991 }
1992
1993 int mlxsw_sp_router_init(struct mlxsw_sp *mlxsw_sp)
1994 {
1995         int err;
1996
1997         INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_neighs_list);
1998         INIT_LIST_HEAD(&mlxsw_sp->router.nexthop_group_list);
1999         err = __mlxsw_sp_router_init(mlxsw_sp);
2000         if (err)
2001                 return err;
2002
2003         mlxsw_sp_lpm_init(mlxsw_sp);
2004         err = mlxsw_sp_vrs_init(mlxsw_sp);
2005         if (err)
2006                 goto err_vrs_init;
2007
2008         err =  mlxsw_sp_neigh_init(mlxsw_sp);
2009         if (err)
2010                 goto err_neigh_init;
2011
2012         mlxsw_sp->fib_nb.notifier_call = mlxsw_sp_router_fib_event;
2013         register_fib_notifier(&mlxsw_sp->fib_nb);
2014         return 0;
2015
2016 err_neigh_init:
2017         mlxsw_sp_vrs_fini(mlxsw_sp);
2018 err_vrs_init:
2019         __mlxsw_sp_router_fini(mlxsw_sp);
2020         return err;
2021 }
2022
2023 void mlxsw_sp_router_fini(struct mlxsw_sp *mlxsw_sp)
2024 {
2025         unregister_fib_notifier(&mlxsw_sp->fib_nb);
2026         mlxsw_sp_neigh_fini(mlxsw_sp);
2027         mlxsw_sp_vrs_fini(mlxsw_sp);
2028         __mlxsw_sp_router_fini(mlxsw_sp);
2029 }