drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/types.h>
   6 #include <linux/dcbnl.h>
   7 #include <linux/if_ether.h>
   8 #include <linux/list.h>
   9 #include <linux/netlink.h>
  10
  11 #include "spectrum.h"
  12 #include "core.h"
  13 #include "port.h"
  14 #include "reg.h"
  15
  16 struct mlxsw_sp_sb_pr {
  17         enum mlxsw_reg_sbpr_mode mode;
  18         u32 size;
  19         u8 freeze_mode:1,
  20            freeze_size:1;
  21 };
  22
  23 struct mlxsw_cp_sb_occ {
  24         u32 cur;
  25         u32 max;
  26 };
  27
  28 struct mlxsw_sp_sb_cm {
  29         u32 min_buff;
  30         u32 max_buff;
  31         u16 pool_index;
  32         struct mlxsw_cp_sb_occ occ;
  33         u8 freeze_pool:1,
  34            freeze_thresh:1;
  35 };
  36
  37 #define MLXSW_SP_SB_INFI -1U
  38 #define MLXSW_SP_SB_REST -2U
  39
  40 struct mlxsw_sp_sb_pm {
  41         u32 min_buff;
  42         u32 max_buff;
  43         struct mlxsw_cp_sb_occ occ;
  44 };
  45
  46 struct mlxsw_sp_sb_mm {
  47         u32 min_buff;
  48         u32 max_buff;
  49         u16 pool_index;
  50 };
  51
  52 struct mlxsw_sp_sb_pool_des {
  53         enum mlxsw_reg_sbxx_dir dir;
  54         u8 pool;
  55 };
  56
  57 #define MLXSW_SP_SB_POOL_ING            0
  58 #define MLXSW_SP_SB_POOL_EGR            4
  59 #define MLXSW_SP_SB_POOL_EGR_MC         8
  60 #define MLXSW_SP_SB_POOL_ING_CPU        9
  61 #define MLXSW_SP_SB_POOL_EGR_CPU        10
  62
  63 static const struct mlxsw_sp_sb_pool_des mlxsw_sp1_sb_pool_dess[] = {
  64         {MLXSW_REG_SBXX_DIR_INGRESS, 0},
  65         {MLXSW_REG_SBXX_DIR_INGRESS, 1},
  66         {MLXSW_REG_SBXX_DIR_INGRESS, 2},
  67         {MLXSW_REG_SBXX_DIR_INGRESS, 3},
  68         {MLXSW_REG_SBXX_DIR_EGRESS, 0},
  69         {MLXSW_REG_SBXX_DIR_EGRESS, 1},
  70         {MLXSW_REG_SBXX_DIR_EGRESS, 2},
  71         {MLXSW_REG_SBXX_DIR_EGRESS, 3},
  72         {MLXSW_REG_SBXX_DIR_EGRESS, 15},
  73         {MLXSW_REG_SBXX_DIR_INGRESS, 4},
  74         {MLXSW_REG_SBXX_DIR_EGRESS, 4},
  75 };
  76
  77 static const struct mlxsw_sp_sb_pool_des mlxsw_sp2_sb_pool_dess[] = {
  78         {MLXSW_REG_SBXX_DIR_INGRESS, 0},
  79         {MLXSW_REG_SBXX_DIR_INGRESS, 1},
  80         {MLXSW_REG_SBXX_DIR_INGRESS, 2},
  81         {MLXSW_REG_SBXX_DIR_INGRESS, 3},
  82         {MLXSW_REG_SBXX_DIR_EGRESS, 0},
  83         {MLXSW_REG_SBXX_DIR_EGRESS, 1},
  84         {MLXSW_REG_SBXX_DIR_EGRESS, 2},
  85         {MLXSW_REG_SBXX_DIR_EGRESS, 3},
  86         {MLXSW_REG_SBXX_DIR_EGRESS, 15},
  87         {MLXSW_REG_SBXX_DIR_INGRESS, 4},
  88         {MLXSW_REG_SBXX_DIR_EGRESS, 4},
  89 };
  90
  91 #define MLXSW_SP_SB_ING_TC_COUNT 8
  92 #define MLXSW_SP_SB_EG_TC_COUNT 16
  93
  94 struct mlxsw_sp_sb_port {
  95         struct mlxsw_sp_sb_cm ing_cms[MLXSW_SP_SB_ING_TC_COUNT];
  96         struct mlxsw_sp_sb_cm eg_cms[MLXSW_SP_SB_EG_TC_COUNT];
  97         struct mlxsw_sp_sb_pm *pms;
  98 };
  99
 100 struct mlxsw_sp_sb {
 101         struct mlxsw_sp_sb_pr *prs;
 102         struct mlxsw_sp_sb_port *ports;
 103         u32 cell_size;
 104         u32 max_headroom_cells;
 105         u64 sb_size;
 106 };
 107
 108 struct mlxsw_sp_sb_vals {
 109         unsigned int pool_count;
 110         const struct mlxsw_sp_sb_pool_des *pool_dess;
 111         const struct mlxsw_sp_sb_pm *pms;
 112         const struct mlxsw_sp_sb_pm *pms_cpu;
 113         const struct mlxsw_sp_sb_pr *prs;
 114         const struct mlxsw_sp_sb_mm *mms;
 115         const struct mlxsw_sp_sb_cm *cms_ingress;
 116         const struct mlxsw_sp_sb_cm *cms_egress;
 117         const struct mlxsw_sp_sb_cm *cms_cpu;
 118         unsigned int mms_count;
 119         unsigned int cms_ingress_count;
 120         unsigned int cms_egress_count;
 121         unsigned int cms_cpu_count;
 122 };
 123
 124 struct mlxsw_sp_sb_ops {
 125         u32 (*int_buf_size_get)(int mtu, u32 speed);
 126 };
 127
 128 u32 mlxsw_sp_cells_bytes(const struct mlxsw_sp *mlxsw_sp, u32 cells)
 129 {
 130         return mlxsw_sp->sb->cell_size * cells;
 131 }
 132
 133 u32 mlxsw_sp_bytes_cells(const struct mlxsw_sp *mlxsw_sp, u32 bytes)
 134 {
 135         return DIV_ROUND_UP(bytes, mlxsw_sp->sb->cell_size);
 136 }
 137
 138 static u32 mlxsw_sp_port_headroom_8x_adjust(const struct mlxsw_sp_port *mlxsw_sp_port,
 139                                             u32 size_cells)
 140 {
 141         /* Ports with eight lanes use two headroom buffers between which the
 142          * configured headroom size is split. Therefore, multiply the calculated
 143          * headroom size by two.
 144          */
 145         return mlxsw_sp_port->mapping.width == 8 ? 2 * size_cells : size_cells;
 146 }
 147
 148 static struct mlxsw_sp_sb_pr *mlxsw_sp_sb_pr_get(struct mlxsw_sp *mlxsw_sp,
 149                                                  u16 pool_index)
 150 {
 151         return &mlxsw_sp->sb->prs[pool_index];
 152 }
 153
 154 static bool mlxsw_sp_sb_cm_exists(u8 pg_buff, enum mlxsw_reg_sbxx_dir dir)
 155 {
 156         if (dir == MLXSW_REG_SBXX_DIR_INGRESS)
 157                 return pg_buff < MLXSW_SP_SB_ING_TC_COUNT;
 158         else
 159                 return pg_buff < MLXSW_SP_SB_EG_TC_COUNT;
 160 }
 161
 162 static struct mlxsw_sp_sb_cm *mlxsw_sp_sb_cm_get(struct mlxsw_sp *mlxsw_sp,
 163                                                  u16 local_port, u8 pg_buff,
 164                                                  enum mlxsw_reg_sbxx_dir dir)
 165 {
 166         struct mlxsw_sp_sb_port *sb_port = &mlxsw_sp->sb->ports[local_port];
 167
 168         WARN_ON(!mlxsw_sp_sb_cm_exists(pg_buff, dir));
 169         if (dir == MLXSW_REG_SBXX_DIR_INGRESS)
 170                 return &sb_port->ing_cms[pg_buff];
 171         else
 172                 return &sb_port->eg_cms[pg_buff];
 173 }
 174
 175 static struct mlxsw_sp_sb_pm *mlxsw_sp_sb_pm_get(struct mlxsw_sp *mlxsw_sp,
 176                                                  u16 local_port, u16 pool_index)
 177 {
 178         return &mlxsw_sp->sb->ports[local_port].pms[pool_index];
 179 }
 180
 181 static int mlxsw_sp_sb_pr_write(struct mlxsw_sp *mlxsw_sp, u16 pool_index,
 182                                 enum mlxsw_reg_sbpr_mode mode,
 183                                 u32 size, bool infi_size)
 184 {
 185         const struct mlxsw_sp_sb_pool_des *des =
 186                 &mlxsw_sp->sb_vals->pool_dess[pool_index];
 187         char sbpr_pl[MLXSW_REG_SBPR_LEN];
 188         struct mlxsw_sp_sb_pr *pr;
 189         int err;
 190
 191         mlxsw_reg_sbpr_pack(sbpr_pl, des->pool, des->dir, mode,
 192                             size, infi_size);
 193         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbpr), sbpr_pl);
 194         if (err)
 195                 return err;
 196
 197         if (infi_size)
 198                 size = mlxsw_sp_bytes_cells(mlxsw_sp, mlxsw_sp->sb->sb_size);
 199         pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
 200         pr->mode = mode;
 201         pr->size = size;
 202         return 0;
 203 }
 204
 205 static int mlxsw_sp_sb_pr_desc_write(struct mlxsw_sp *mlxsw_sp,
 206                                      enum mlxsw_reg_sbxx_dir dir,
 207                                      enum mlxsw_reg_sbpr_mode mode,
 208                                      u32 size, bool infi_size)
 209 {
 210         char sbpr_pl[MLXSW_REG_SBPR_LEN];
 211
 212         /* The FW default descriptor buffer configuration uses only pool 14 for
 213          * descriptors.
 214          */
 215         mlxsw_reg_sbpr_pack(sbpr_pl, 14, dir, mode, size, infi_size);
 216         mlxsw_reg_sbpr_desc_set(sbpr_pl, true);
 217         return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbpr), sbpr_pl);
 218 }
 219
 220 static int mlxsw_sp_sb_cm_write(struct mlxsw_sp *mlxsw_sp, u16 local_port,
 221                                 u8 pg_buff, u32 min_buff, u32 max_buff,
 222                                 bool infi_max, u16 pool_index)
 223 {
 224         const struct mlxsw_sp_sb_pool_des *des =
 225                 &mlxsw_sp->sb_vals->pool_dess[pool_index];
 226         char sbcm_pl[MLXSW_REG_SBCM_LEN];
 227         struct mlxsw_sp_sb_cm *cm;
 228         int err;
 229
 230         mlxsw_reg_sbcm_pack(sbcm_pl, local_port, pg_buff, des->dir,
 231                             min_buff, max_buff, infi_max, des->pool);
 232         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbcm), sbcm_pl);
 233         if (err)
 234                 return err;
 235
 236         if (mlxsw_sp_sb_cm_exists(pg_buff, des->dir)) {
 237                 if (infi_max)
 238                         max_buff = mlxsw_sp_bytes_cells(mlxsw_sp,
 239                                                         mlxsw_sp->sb->sb_size);
 240
 241                 cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port, pg_buff,
 242                                         des->dir);
 243                 cm->min_buff = min_buff;
 244                 cm->max_buff = max_buff;
 245                 cm->pool_index = pool_index;
 246         }
 247         return 0;
 248 }
 249
 250 static int mlxsw_sp_sb_pm_write(struct mlxsw_sp *mlxsw_sp, u16 local_port,
 251                                 u16 pool_index, u32 min_buff, u32 max_buff)
 252 {
 253         const struct mlxsw_sp_sb_pool_des *des =
 254                 &mlxsw_sp->sb_vals->pool_dess[pool_index];
 255         char sbpm_pl[MLXSW_REG_SBPM_LEN];
 256         struct mlxsw_sp_sb_pm *pm;
 257         int err;
 258
 259         mlxsw_reg_sbpm_pack(sbpm_pl, local_port, des->pool, des->dir, false,
 260                             min_buff, max_buff);
 261         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbpm), sbpm_pl);
 262         if (err)
 263                 return err;
 264
 265         pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port, pool_index);
 266         pm->min_buff = min_buff;
 267         pm->max_buff = max_buff;
 268         return 0;
 269 }
 270
 271 static int mlxsw_sp_sb_pm_occ_clear(struct mlxsw_sp *mlxsw_sp, u16 local_port,
 272                                     u16 pool_index, struct list_head *bulk_list)
 273 {
 274         const struct mlxsw_sp_sb_pool_des *des =
 275                 &mlxsw_sp->sb_vals->pool_dess[pool_index];
 276         char sbpm_pl[MLXSW_REG_SBPM_LEN];
 277
 278         if (local_port == MLXSW_PORT_CPU_PORT &&
 279             des->dir == MLXSW_REG_SBXX_DIR_INGRESS)
 280                 return 0;
 281
 282         mlxsw_reg_sbpm_pack(sbpm_pl, local_port, des->pool, des->dir,
 283                             true, 0, 0);
 284         return mlxsw_reg_trans_query(mlxsw_sp->core, MLXSW_REG(sbpm), sbpm_pl,
 285                                      bulk_list, NULL, 0);
 286 }
 287
 288 static void mlxsw_sp_sb_pm_occ_query_cb(struct mlxsw_core *mlxsw_core,
 289                                         char *sbpm_pl, size_t sbpm_pl_len,
 290                                         unsigned long cb_priv)
 291 {
 292         struct mlxsw_sp_sb_pm *pm = (struct mlxsw_sp_sb_pm *) cb_priv;
 293
 294         mlxsw_reg_sbpm_unpack(sbpm_pl, &pm->occ.cur, &pm->occ.max);
 295 }
 296
 297 static int mlxsw_sp_sb_pm_occ_query(struct mlxsw_sp *mlxsw_sp, u16 local_port,
 298                                     u16 pool_index, struct list_head *bulk_list)
 299 {
 300         const struct mlxsw_sp_sb_pool_des *des =
 301                 &mlxsw_sp->sb_vals->pool_dess[pool_index];
 302         char sbpm_pl[MLXSW_REG_SBPM_LEN];
 303         struct mlxsw_sp_sb_pm *pm;
 304
 305         if (local_port == MLXSW_PORT_CPU_PORT &&
 306             des->dir == MLXSW_REG_SBXX_DIR_INGRESS)
 307                 return 0;
 308
 309         pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port, pool_index);
 310         mlxsw_reg_sbpm_pack(sbpm_pl, local_port, des->pool, des->dir,
 311                             false, 0, 0);
 312         return mlxsw_reg_trans_query(mlxsw_sp->core, MLXSW_REG(sbpm), sbpm_pl,
 313                                      bulk_list,
 314                                      mlxsw_sp_sb_pm_occ_query_cb,
 315                                      (unsigned long) pm);
 316 }
 317
 318 void mlxsw_sp_hdroom_prios_reset_buf_idx(struct mlxsw_sp_hdroom *hdroom)
 319 {
 320         int prio;
 321
 322         for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; prio++) {
 323                 switch (hdroom->mode) {
 324                 case MLXSW_SP_HDROOM_MODE_DCB:
 325                         hdroom->prios.prio[prio].buf_idx = hdroom->prios.prio[prio].ets_buf_idx;
 326                         break;
 327                 case MLXSW_SP_HDROOM_MODE_TC:
 328                         hdroom->prios.prio[prio].buf_idx = hdroom->prios.prio[prio].set_buf_idx;
 329                         break;
 330                 }
 331         }
 332 }
 333
 334 void mlxsw_sp_hdroom_bufs_reset_lossiness(struct mlxsw_sp_hdroom *hdroom)
 335 {
 336         int prio;
 337         int i;
 338
 339         for (i = 0; i < DCBX_MAX_BUFFERS; i++)
 340                 hdroom->bufs.buf[i].lossy = true;
 341
 342         for (prio = 0; prio < IEEE_8021Q_MAX_PRIORITIES; prio++) {
 343                 if (!hdroom->prios.prio[prio].lossy)
 344                         hdroom->bufs.buf[hdroom->prios.prio[prio].buf_idx].lossy = false;
 345         }
 346 }
 347
 348 static u16 mlxsw_sp_hdroom_buf_threshold_get(const struct mlxsw_sp *mlxsw_sp, int mtu)
 349 {
 350         return 2 * mlxsw_sp_bytes_cells(mlxsw_sp, mtu);
 351 }
 352
 353 static void mlxsw_sp_hdroom_buf_pack(char *pbmc_pl, int index, u16 size, u16 thres, bool lossy)
 354 {
 355         if (lossy)
 356                 mlxsw_reg_pbmc_lossy_buffer_pack(pbmc_pl, index, size);
 357         else
 358                 mlxsw_reg_pbmc_lossless_buffer_pack(pbmc_pl, index, size,
 359                                                     thres);
 360 }
 361
 362 static u16 mlxsw_sp_hdroom_buf_delay_get(const struct mlxsw_sp *mlxsw_sp,
 363                                          const struct mlxsw_sp_hdroom *hdroom)
 364 {
 365         u16 delay_cells;
 366
 367         delay_cells = mlxsw_sp_bytes_cells(mlxsw_sp, hdroom->delay_bytes);
 368
 369         /* In the worst case scenario the delay will be made up of packets that
 370          * are all of size CELL_SIZE + 1, which means each packet will require
 371          * almost twice its true size when buffered in the switch. We therefore
 372          * multiply this value by the "cell factor", which is close to 2.
 373          *
 374          * Another MTU is added in case the transmitting host already started
 375          * transmitting a maximum length frame when the PFC packet was received.
 376          */
 377         return 2 * delay_cells + mlxsw_sp_bytes_cells(mlxsw_sp, hdroom->mtu);
 378 }
 379
 380 static u32 mlxsw_sp_hdroom_int_buf_size_get(struct mlxsw_sp *mlxsw_sp, int mtu, u32 speed)
 381 {
 382         u32 buffsize = mlxsw_sp->sb_ops->int_buf_size_get(mtu, speed);
 383
 384         return mlxsw_sp_bytes_cells(mlxsw_sp, buffsize) + 1;
 385 }
 386
 387 static bool mlxsw_sp_hdroom_buf_is_used(const struct mlxsw_sp_hdroom *hdroom, int buf)
 388 {
 389         int prio;
 390
 391         for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; prio++) {
 392                 if (hdroom->prios.prio[prio].buf_idx == buf)
 393                         return true;
 394         }
 395         return false;
 396 }
 397
 398 void mlxsw_sp_hdroom_bufs_reset_sizes(struct mlxsw_sp_port *mlxsw_sp_port,
 399                                       struct mlxsw_sp_hdroom *hdroom)
 400 {
 401         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 402         u16 reserve_cells;
 403         int i;
 404
 405         /* Internal buffer. */
 406         reserve_cells = mlxsw_sp_hdroom_int_buf_size_get(mlxsw_sp, mlxsw_sp_port->max_mtu,
 407                                                          mlxsw_sp_port->max_speed);
 408         reserve_cells = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, reserve_cells);
 409         hdroom->int_buf.reserve_cells = reserve_cells;
 410
 411         if (hdroom->int_buf.enable)
 412                 hdroom->int_buf.size_cells = reserve_cells;
 413         else
 414                 hdroom->int_buf.size_cells = 0;
 415
 416         /* PG buffers. */
 417         for (i = 0; i < DCBX_MAX_BUFFERS; i++) {
 418                 struct mlxsw_sp_hdroom_buf *buf = &hdroom->bufs.buf[i];
 419                 u16 thres_cells;
 420                 u16 delay_cells;
 421
 422                 if (!mlxsw_sp_hdroom_buf_is_used(hdroom, i)) {
 423                         thres_cells = 0;
 424                         delay_cells = 0;
 425                 } else if (buf->lossy) {
 426                         thres_cells = mlxsw_sp_hdroom_buf_threshold_get(mlxsw_sp, hdroom->mtu);
 427                         delay_cells = 0;
 428                 } else {
 429                         thres_cells = mlxsw_sp_hdroom_buf_threshold_get(mlxsw_sp, hdroom->mtu);
 430                         delay_cells = mlxsw_sp_hdroom_buf_delay_get(mlxsw_sp, hdroom);
 431                 }
 432
 433                 thres_cells = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, thres_cells);
 434                 delay_cells = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, delay_cells);
 435
 436                 buf->thres_cells = thres_cells;
 437                 if (hdroom->mode == MLXSW_SP_HDROOM_MODE_DCB) {
 438                         buf->size_cells = thres_cells + delay_cells;
 439                 } else {
 440                         /* Do not allow going below the minimum size, even if
 441                          * the user requested it.
 442                          */
 443                         buf->size_cells = max(buf->set_size_cells, buf->thres_cells);
 444                 }
 445         }
 446 }
 447
 448 #define MLXSW_SP_PB_UNUSED 8
 449
 450 static int mlxsw_sp_hdroom_configure_buffers(struct mlxsw_sp_port *mlxsw_sp_port,
 451                                              const struct mlxsw_sp_hdroom *hdroom, bool force)
 452 {
 453         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 454         char pbmc_pl[MLXSW_REG_PBMC_LEN];
 455         bool dirty;
 456         int err;
 457         int i;
 458
 459         dirty = memcmp(&mlxsw_sp_port->hdroom->bufs, &hdroom->bufs, sizeof(hdroom->bufs));
 460         if (!dirty && !force)
 461                 return 0;
 462
 463         mlxsw_reg_pbmc_pack(pbmc_pl, mlxsw_sp_port->local_port, 0xffff, 0xffff / 2);
 464         for (i = 0; i < MLXSW_SP_PB_COUNT; i++) {
 465                 const struct mlxsw_sp_hdroom_buf *buf = &hdroom->bufs.buf[i];
 466
 467                 if (i == MLXSW_SP_PB_UNUSED)
 468                         continue;
 469
 470                 mlxsw_sp_hdroom_buf_pack(pbmc_pl, i, buf->size_cells, buf->thres_cells, buf->lossy);
 471         }
 472
 473         mlxsw_reg_pbmc_lossy_buffer_pack(pbmc_pl, MLXSW_REG_PBMC_PORT_SHARED_BUF_IDX, 0);
 474         err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pbmc), pbmc_pl);
 475         if (err)
 476                 return err;
 477
 478         mlxsw_sp_port->hdroom->bufs = hdroom->bufs;
 479         return 0;
 480 }
 481
 482 static int mlxsw_sp_hdroom_configure_priomap(struct mlxsw_sp_port *mlxsw_sp_port,
 483                                              const struct mlxsw_sp_hdroom *hdroom, bool force)
 484 {
 485         char pptb_pl[MLXSW_REG_PPTB_LEN];
 486         bool dirty;
 487         int prio;
 488         int err;
 489
 490         dirty = memcmp(&mlxsw_sp_port->hdroom->prios, &hdroom->prios, sizeof(hdroom->prios));
 491         if (!dirty && !force)
 492                 return 0;
 493
 494         mlxsw_reg_pptb_pack(pptb_pl, mlxsw_sp_port->local_port);
 495         for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; prio++)
 496                 mlxsw_reg_pptb_prio_to_buff_pack(pptb_pl, prio, hdroom->prios.prio[prio].buf_idx);
 497
 498         err = mlxsw_reg_write(mlxsw_sp_port->mlxsw_sp->core, MLXSW_REG(pptb), pptb_pl);
 499         if (err)
 500                 return err;
 501
 502         mlxsw_sp_port->hdroom->prios = hdroom->prios;
 503         return 0;
 504 }
 505
 506 static int mlxsw_sp_hdroom_configure_int_buf(struct mlxsw_sp_port *mlxsw_sp_port,
 507                                              const struct mlxsw_sp_hdroom *hdroom, bool force)
 508 {
 509         char sbib_pl[MLXSW_REG_SBIB_LEN];
 510         bool dirty;
 511         int err;
 512
 513         dirty = memcmp(&mlxsw_sp_port->hdroom->int_buf, &hdroom->int_buf, sizeof(hdroom->int_buf));
 514         if (!dirty && !force)
 515                 return 0;
 516
 517         mlxsw_reg_sbib_pack(sbib_pl, mlxsw_sp_port->local_port, hdroom->int_buf.size_cells);
 518         err = mlxsw_reg_write(mlxsw_sp_port->mlxsw_sp->core, MLXSW_REG(sbib), sbib_pl);
 519         if (err)
 520                 return err;
 521
 522         mlxsw_sp_port->hdroom->int_buf = hdroom->int_buf;
 523         return 0;
 524 }
 525
 526 static bool mlxsw_sp_hdroom_bufs_fit(struct mlxsw_sp *mlxsw_sp,
 527                                      const struct mlxsw_sp_hdroom *hdroom)
 528 {
 529         u32 taken_headroom_cells = 0;
 530         int i;
 531
 532         for (i = 0; i < MLXSW_SP_PB_COUNT; i++)
 533                 taken_headroom_cells += hdroom->bufs.buf[i].size_cells;
 534
 535         taken_headroom_cells += hdroom->int_buf.reserve_cells;
 536         return taken_headroom_cells <= mlxsw_sp->sb->max_headroom_cells;
 537 }
 538
 539 static int __mlxsw_sp_hdroom_configure(struct mlxsw_sp_port *mlxsw_sp_port,
 540                                        const struct mlxsw_sp_hdroom *hdroom, bool force)
 541 {
 542         struct mlxsw_sp_hdroom orig_hdroom;
 543         struct mlxsw_sp_hdroom tmp_hdroom;
 544         int err;
 545         int i;
 546
 547         /* Port buffers need to be configured in three steps. First, all buffers
 548          * with non-zero size are configured. Then, prio-to-buffer map is
 549          * updated, allowing traffic to flow to the now non-zero buffers.
 550          * Finally, zero-sized buffers are configured, because now no traffic
 551          * should be directed to them anymore. This way, in a non-congested
 552          * system, no packet drops are introduced by the reconfiguration.
 553          */
 554
 555         orig_hdroom = *mlxsw_sp_port->hdroom;
 556         tmp_hdroom = orig_hdroom;
 557         for (i = 0; i < MLXSW_SP_PB_COUNT; i++) {
 558                 if (hdroom->bufs.buf[i].size_cells)
 559                         tmp_hdroom.bufs.buf[i] = hdroom->bufs.buf[i];
 560         }
 561
 562         if (!mlxsw_sp_hdroom_bufs_fit(mlxsw_sp_port->mlxsw_sp, &tmp_hdroom) ||
 563             !mlxsw_sp_hdroom_bufs_fit(mlxsw_sp_port->mlxsw_sp, hdroom))
 564                 return -ENOBUFS;
 565
 566         err = mlxsw_sp_hdroom_configure_buffers(mlxsw_sp_port, &tmp_hdroom, force);
 567         if (err)
 568                 return err;
 569
 570         err = mlxsw_sp_hdroom_configure_priomap(mlxsw_sp_port, hdroom, force);
 571         if (err)
 572                 goto err_configure_priomap;
 573
 574         err = mlxsw_sp_hdroom_configure_buffers(mlxsw_sp_port, hdroom, false);
 575         if (err)
 576                 goto err_configure_buffers;
 577
 578         err = mlxsw_sp_hdroom_configure_int_buf(mlxsw_sp_port, hdroom, false);
 579         if (err)
 580                 goto err_configure_int_buf;
 581
 582         *mlxsw_sp_port->hdroom = *hdroom;
 583         return 0;
 584
 585 err_configure_int_buf:
 586         mlxsw_sp_hdroom_configure_buffers(mlxsw_sp_port, &tmp_hdroom, false);
 587 err_configure_buffers:
 588         mlxsw_sp_hdroom_configure_priomap(mlxsw_sp_port, &tmp_hdroom, false);
 589 err_configure_priomap:
 590         mlxsw_sp_hdroom_configure_buffers(mlxsw_sp_port, &orig_hdroom, false);
 591         return err;
 592 }
 593
 594 int mlxsw_sp_hdroom_configure(struct mlxsw_sp_port *mlxsw_sp_port,
 595                               const struct mlxsw_sp_hdroom *hdroom)
 596 {
 597         return __mlxsw_sp_hdroom_configure(mlxsw_sp_port, hdroom, false);
 598 }
 599
 600 static int mlxsw_sp_port_headroom_init(struct mlxsw_sp_port *mlxsw_sp_port)
 601 {
 602         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 603         struct mlxsw_sp_hdroom hdroom = {};
 604         u32 size9;
 605         int prio;
 606
 607         hdroom.mtu = mlxsw_sp_port->dev->mtu;
 608         hdroom.mode = MLXSW_SP_HDROOM_MODE_DCB;
 609         for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; prio++)
 610                 hdroom.prios.prio[prio].lossy = true;
 611
 612         mlxsw_sp_hdroom_bufs_reset_lossiness(&hdroom);
 613         mlxsw_sp_hdroom_bufs_reset_sizes(mlxsw_sp_port, &hdroom);
 614
 615         /* Buffer 9 is used for control traffic. */
 616         size9 = mlxsw_sp_port_headroom_8x_adjust(mlxsw_sp_port, mlxsw_sp_port->max_mtu);
 617         hdroom.bufs.buf[9].size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size9);
 618
 619         return __mlxsw_sp_hdroom_configure(mlxsw_sp_port, &hdroom, true);
 620 }
 621
 622 static int mlxsw_sp_sb_port_init(struct mlxsw_sp *mlxsw_sp,
 623                                  struct mlxsw_sp_sb_port *sb_port)
 624 {
 625         struct mlxsw_sp_sb_pm *pms;
 626
 627         pms = kcalloc(mlxsw_sp->sb_vals->pool_count, sizeof(*pms),
 628                       GFP_KERNEL);
 629         if (!pms)
 630                 return -ENOMEM;
 631         sb_port->pms = pms;
 632         return 0;
 633 }
 634
 635 static void mlxsw_sp_sb_port_fini(struct mlxsw_sp_sb_port *sb_port)
 636 {
 637         kfree(sb_port->pms);
 638 }
 639
 640 static int mlxsw_sp_sb_ports_init(struct mlxsw_sp *mlxsw_sp)
 641 {
 642         unsigned int max_ports = mlxsw_core_max_ports(mlxsw_sp->core);
 643         struct mlxsw_sp_sb_pr *prs;
 644         int i;
 645         int err;
 646
 647         mlxsw_sp->sb->ports = kcalloc(max_ports,
 648                                       sizeof(struct mlxsw_sp_sb_port),
 649                                       GFP_KERNEL);
 650         if (!mlxsw_sp->sb->ports)
 651                 return -ENOMEM;
 652
 653         prs = kcalloc(mlxsw_sp->sb_vals->pool_count, sizeof(*prs),
 654                       GFP_KERNEL);
 655         if (!prs) {
 656                 err = -ENOMEM;
 657                 goto err_alloc_prs;
 658         }
 659         mlxsw_sp->sb->prs = prs;
 660
 661         for (i = 0; i < max_ports; i++) {
 662                 err = mlxsw_sp_sb_port_init(mlxsw_sp, &mlxsw_sp->sb->ports[i]);
 663                 if (err)
 664                         goto err_sb_port_init;
 665         }
 666
 667         return 0;
 668
 669 err_sb_port_init:
 670         for (i--; i >= 0; i--)
 671                 mlxsw_sp_sb_port_fini(&mlxsw_sp->sb->ports[i]);
 672         kfree(mlxsw_sp->sb->prs);
 673 err_alloc_prs:
 674         kfree(mlxsw_sp->sb->ports);
 675         return err;
 676 }
 677
 678 static void mlxsw_sp_sb_ports_fini(struct mlxsw_sp *mlxsw_sp)
 679 {
 680         int max_ports = mlxsw_core_max_ports(mlxsw_sp->core);
 681         int i;
 682
 683         for (i = max_ports - 1; i >= 0; i--)
 684                 mlxsw_sp_sb_port_fini(&mlxsw_sp->sb->ports[i]);
 685         kfree(mlxsw_sp->sb->prs);
 686         kfree(mlxsw_sp->sb->ports);
 687 }
 688
 689 #define MLXSW_SP_SB_PR(_mode, _size)    \
 690         {                               \
 691                 .mode = _mode,          \
 692                 .size = _size,          \
 693         }
 694
 695 #define MLXSW_SP_SB_PR_EXT(_mode, _size, _freeze_mode, _freeze_size)    \
 696         {                                                               \
 697                 .mode = _mode,                                          \
 698                 .size = _size,                                          \
 699                 .freeze_mode = _freeze_mode,                            \
 700                 .freeze_size = _freeze_size,                            \
 701         }
 702
 703 #define MLXSW_SP1_SB_PR_CPU_SIZE        (256 * 1000)
 704
 705 /* Order according to mlxsw_sp1_sb_pool_dess */
 706 static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = {
 707         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST),
 708         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
 709         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
 710         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
 711         MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST,
 712                            true, false),
 713         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
 714         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
 715         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0),
 716         MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_STATIC, MLXSW_SP_SB_INFI,
 717                            true, true),
 718         MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,
 719                            MLXSW_SP1_SB_PR_CPU_SIZE, true, false),
 720         MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,
 721                            MLXSW_SP1_SB_PR_CPU_SIZE, true, false),
 722 };
 723
 724 #define MLXSW_SP2_SB_PR_CPU_SIZE        (256 * 1000)
 725
 726 /* Order according to mlxsw_sp2_sb_pool_dess */
 727 static const struct mlxsw_sp_sb_pr mlxsw_sp2_sb_prs[] = {
 728         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST),
 729         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
 730         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
 731         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
 732         MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST,
 733                            true, false),
 734         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
 735         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
 736         MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0),
 737         MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_STATIC, MLXSW_SP_SB_INFI,
 738                            true, true),
 739         MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,
 740                            MLXSW_SP2_SB_PR_CPU_SIZE, true, false),
 741         MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC,
 742                            MLXSW_SP2_SB_PR_CPU_SIZE, true, false),
 743 };
 744
 745 static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp,
 746                                 const struct mlxsw_sp_sb_pr *prs,
 747                                 const struct mlxsw_sp_sb_pool_des *pool_dess,
 748                                 size_t prs_len)
 749 {
 750         /* Round down, unlike mlxsw_sp_bytes_cells(). */
 751         u32 sb_cells = div_u64(mlxsw_sp->sb->sb_size, mlxsw_sp->sb->cell_size);
 752         u32 rest_cells[2] = {sb_cells, sb_cells};
 753         int i;
 754         int err;
 755
 756         /* Calculate how much space to give to the "REST" pools in either
 757          * direction.
 758          */
 759         for (i = 0; i < prs_len; i++) {
 760                 enum mlxsw_reg_sbxx_dir dir = pool_dess[i].dir;
 761                 u32 size = prs[i].size;
 762                 u32 size_cells;
 763
 764                 if (size == MLXSW_SP_SB_INFI || size == MLXSW_SP_SB_REST)
 765                         continue;
 766
 767                 size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size);
 768                 if (WARN_ON_ONCE(size_cells > rest_cells[dir]))
 769                         continue;
 770
 771                 rest_cells[dir] -= size_cells;
 772         }
 773
 774         for (i = 0; i < prs_len; i++) {
 775                 u32 size = prs[i].size;
 776                 u32 size_cells;
 777
 778                 if (size == MLXSW_SP_SB_INFI) {
 779                         err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
 780                                                    0, true);
 781                 } else if (size == MLXSW_SP_SB_REST) {
 782                         size_cells = rest_cells[pool_dess[i].dir];
 783                         err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
 784                                                    size_cells, false);
 785                 } else {
 786                         size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size);
 787                         err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode,
 788                                                    size_cells, false);
 789                 }
 790                 if (err)
 791                         return err;
 792         }
 793
 794         err = mlxsw_sp_sb_pr_desc_write(mlxsw_sp, MLXSW_REG_SBXX_DIR_INGRESS,
 795                                         MLXSW_REG_SBPR_MODE_DYNAMIC, 0, true);
 796         if (err)
 797                 return err;
 798
 799         err = mlxsw_sp_sb_pr_desc_write(mlxsw_sp, MLXSW_REG_SBXX_DIR_EGRESS,
 800                                         MLXSW_REG_SBPR_MODE_DYNAMIC, 0, true);
 801         if (err)
 802                 return err;
 803
 804         return 0;
 805 }
 806
 807 #define MLXSW_SP_SB_CM(_min_buff, _max_buff, _pool)     \
 808         {                                               \
 809                 .min_buff = _min_buff,                  \
 810                 .max_buff = _max_buff,                  \
 811                 .pool_index = _pool,                    \
 812         }
 813
 814 #define MLXSW_SP_SB_CM_ING(_min_buff, _max_buff)        \
 815         {                                               \
 816                 .min_buff = _min_buff,                  \
 817                 .max_buff = _max_buff,                  \
 818                 .pool_index = MLXSW_SP_SB_POOL_ING,     \
 819         }
 820
 821 #define MLXSW_SP_SB_CM_EGR(_min_buff, _max_buff)        \
 822         {                                               \
 823                 .min_buff = _min_buff,                  \
 824                 .max_buff = _max_buff,                  \
 825                 .pool_index = MLXSW_SP_SB_POOL_EGR,     \
 826         }
 827
 828 #define MLXSW_SP_SB_CM_EGR_MC(_min_buff, _max_buff)     \
 829         {                                               \
 830                 .min_buff = _min_buff,                  \
 831                 .max_buff = _max_buff,                  \
 832                 .pool_index = MLXSW_SP_SB_POOL_EGR_MC,  \
 833                 .freeze_pool = true,                    \
 834                 .freeze_thresh = true,                  \
 835         }
 836
 837 static const struct mlxsw_sp_sb_cm mlxsw_sp1_sb_cms_ingress[] = {
 838         MLXSW_SP_SB_CM_ING(10000, 8),
 839         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 840         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 841         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 842         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 843         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 844         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 845         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 846         MLXSW_SP_SB_CM_ING(0, 0), /* dummy, this PG does not exist */
 847         MLXSW_SP_SB_CM(10000, 8, MLXSW_SP_SB_POOL_ING_CPU),
 848 };
 849
 850 static const struct mlxsw_sp_sb_cm mlxsw_sp2_sb_cms_ingress[] = {
 851         MLXSW_SP_SB_CM_ING(0, 7),
 852         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 853         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 854         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 855         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 856         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 857         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 858         MLXSW_SP_SB_CM_ING(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
 859         MLXSW_SP_SB_CM_ING(0, 0), /* dummy, this PG does not exist */
 860         MLXSW_SP_SB_CM(10000, 8, MLXSW_SP_SB_POOL_ING_CPU),
 861 };
 862
 863 static const struct mlxsw_sp_sb_cm mlxsw_sp1_sb_cms_egress[] = {
 864         MLXSW_SP_SB_CM_EGR(1500, 9),
 865         MLXSW_SP_SB_CM_EGR(1500, 9),
 866         MLXSW_SP_SB_CM_EGR(1500, 9),
 867         MLXSW_SP_SB_CM_EGR(1500, 9),
 868         MLXSW_SP_SB_CM_EGR(1500, 9),
 869         MLXSW_SP_SB_CM_EGR(1500, 9),
 870         MLXSW_SP_SB_CM_EGR(1500, 9),
 871         MLXSW_SP_SB_CM_EGR(1500, 9),
 872         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 873         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 874         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 875         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 876         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 877         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 878         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 879         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 880         MLXSW_SP_SB_CM_EGR(1, 0xff),
 881 };
 882
 883 static const struct mlxsw_sp_sb_cm mlxsw_sp2_sb_cms_egress[] = {
 884         MLXSW_SP_SB_CM_EGR(0, 7),
 885         MLXSW_SP_SB_CM_EGR(0, 7),
 886         MLXSW_SP_SB_CM_EGR(0, 7),
 887         MLXSW_SP_SB_CM_EGR(0, 7),
 888         MLXSW_SP_SB_CM_EGR(0, 7),
 889         MLXSW_SP_SB_CM_EGR(0, 7),
 890         MLXSW_SP_SB_CM_EGR(0, 7),
 891         MLXSW_SP_SB_CM_EGR(0, 7),
 892         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 893         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 894         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 895         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 896         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 897         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 898         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 899         MLXSW_SP_SB_CM_EGR_MC(0, MLXSW_SP_SB_INFI),
 900         MLXSW_SP_SB_CM_EGR(1, 0xff),
 901 };
 902
 903 #define MLXSW_SP_CPU_PORT_SB_CM MLXSW_SP_SB_CM(0, 0, MLXSW_SP_SB_POOL_EGR_CPU)
 904
 905 static const struct mlxsw_sp_sb_cm mlxsw_sp_cpu_port_sb_cms[] = {
 906         MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 907         MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 908         MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 909         MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 910         MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 911         MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 912         MLXSW_SP_CPU_PORT_SB_CM,
 913         MLXSW_SP_SB_CM(1000, 8, MLXSW_SP_SB_POOL_EGR_CPU),
 914         MLXSW_SP_CPU_PORT_SB_CM,
 915         MLXSW_SP_CPU_PORT_SB_CM,
 916         MLXSW_SP_CPU_PORT_SB_CM,
 917         MLXSW_SP_CPU_PORT_SB_CM,
 918         MLXSW_SP_CPU_PORT_SB_CM,
 919         MLXSW_SP_CPU_PORT_SB_CM,
 920         MLXSW_SP_CPU_PORT_SB_CM,
 921         MLXSW_SP_CPU_PORT_SB_CM,
 922         MLXSW_SP_CPU_PORT_SB_CM,
 923         MLXSW_SP_CPU_PORT_SB_CM,
 924         MLXSW_SP_CPU_PORT_SB_CM,
 925         MLXSW_SP_CPU_PORT_SB_CM,
 926         MLXSW_SP_CPU_PORT_SB_CM,
 927         MLXSW_SP_CPU_PORT_SB_CM,
 928         MLXSW_SP_CPU_PORT_SB_CM,
 929         MLXSW_SP_CPU_PORT_SB_CM,
 930         MLXSW_SP_CPU_PORT_SB_CM,
 931         MLXSW_SP_CPU_PORT_SB_CM,
 932         MLXSW_SP_CPU_PORT_SB_CM,
 933         MLXSW_SP_CPU_PORT_SB_CM,
 934         MLXSW_SP_CPU_PORT_SB_CM,
 935         MLXSW_SP_CPU_PORT_SB_CM,
 936         MLXSW_SP_CPU_PORT_SB_CM,
 937         MLXSW_SP_CPU_PORT_SB_CM,
 938 };
 939
 940 static bool
 941 mlxsw_sp_sb_pool_is_static(struct mlxsw_sp *mlxsw_sp, u16 pool_index)
 942 {
 943         struct mlxsw_sp_sb_pr *pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
 944
 945         return pr->mode == MLXSW_REG_SBPR_MODE_STATIC;
 946 }
 947
 948 static int __mlxsw_sp_sb_cms_init(struct mlxsw_sp *mlxsw_sp, u16 local_port,
 949                                   enum mlxsw_reg_sbxx_dir dir,
 950                                   const struct mlxsw_sp_sb_cm *cms,
 951                                   size_t cms_len)
 952 {
 953         const struct mlxsw_sp_sb_vals *sb_vals = mlxsw_sp->sb_vals;
 954         int i;
 955         int err;
 956
 957         for (i = 0; i < cms_len; i++) {
 958                 const struct mlxsw_sp_sb_cm *cm;
 959                 u32 min_buff;
 960                 u32 max_buff;
 961
 962                 if (i == 8 && dir == MLXSW_REG_SBXX_DIR_INGRESS)
 963                         continue; /* PG number 8 does not exist, skip it */
 964                 cm = &cms[i];
 965                 if (WARN_ON(sb_vals->pool_dess[cm->pool_index].dir != dir))
 966                         continue;
 967
 968                 min_buff = mlxsw_sp_bytes_cells(mlxsw_sp, cm->min_buff);
 969                 max_buff = cm->max_buff;
 970                 if (max_buff == MLXSW_SP_SB_INFI) {
 971                         err = mlxsw_sp_sb_cm_write(mlxsw_sp, local_port, i,
 972                                                    min_buff, 0,
 973                                                    true, cm->pool_index);
 974                 } else {
 975                         if (mlxsw_sp_sb_pool_is_static(mlxsw_sp,
 976                                                        cm->pool_index))
 977                                 max_buff = mlxsw_sp_bytes_cells(mlxsw_sp,
 978                                                                 max_buff);
 979                         err = mlxsw_sp_sb_cm_write(mlxsw_sp, local_port, i,
 980                                                    min_buff, max_buff,
 981                                                    false, cm->pool_index);
 982                 }
 983                 if (err)
 984                         return err;
 985         }
 986         return 0;
 987 }
 988
 989 static int mlxsw_sp_port_sb_cms_init(struct mlxsw_sp_port *mlxsw_sp_port)
 990 {
 991         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
 992         int err;
 993
 994         err = __mlxsw_sp_sb_cms_init(mlxsw_sp,
 995                                      mlxsw_sp_port->local_port,
 996                                      MLXSW_REG_SBXX_DIR_INGRESS,
 997                                      mlxsw_sp->sb_vals->cms_ingress,
 998                                      mlxsw_sp->sb_vals->cms_ingress_count);
 999         if (err)
1000                 return err;
1001         return __mlxsw_sp_sb_cms_init(mlxsw_sp_port->mlxsw_sp,
1002                                       mlxsw_sp_port->local_port,
1003                                       MLXSW_REG_SBXX_DIR_EGRESS,
1004                                       mlxsw_sp->sb_vals->cms_egress,
1005                                       mlxsw_sp->sb_vals->cms_egress_count);
1006 }
1007
1008 static int mlxsw_sp_cpu_port_sb_cms_init(struct mlxsw_sp *mlxsw_sp)
1009 {
1010         return __mlxsw_sp_sb_cms_init(mlxsw_sp, 0, MLXSW_REG_SBXX_DIR_EGRESS,
1011                                       mlxsw_sp->sb_vals->cms_cpu,
1012                                       mlxsw_sp->sb_vals->cms_cpu_count);
1013 }
1014
1015 #define MLXSW_SP_SB_PM(_min_buff, _max_buff)    \
1016         {                                       \
1017                 .min_buff = _min_buff,          \
1018                 .max_buff = _max_buff,          \
1019         }
1020
1021 /* Order according to mlxsw_sp1_sb_pool_dess */
1022 static const struct mlxsw_sp_sb_pm mlxsw_sp1_sb_pms[] = {
1023         MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX),
1024         MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1025         MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1026         MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1027         MLXSW_SP_SB_PM(0, 7),
1028         MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1029         MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1030         MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1031         MLXSW_SP_SB_PM(10000, 90000),
1032         MLXSW_SP_SB_PM(0, 8),   /* 50% occupancy */
1033         MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1034 };
1035
1036 /* Order according to mlxsw_sp2_sb_pool_dess */
1037 static const struct mlxsw_sp_sb_pm mlxsw_sp2_sb_pms[] = {
1038         MLXSW_SP_SB_PM(0, 7),
1039         MLXSW_SP_SB_PM(0, 0),
1040         MLXSW_SP_SB_PM(0, 0),
1041         MLXSW_SP_SB_PM(0, 0),
1042         MLXSW_SP_SB_PM(0, 7),
1043         MLXSW_SP_SB_PM(0, 0),
1044         MLXSW_SP_SB_PM(0, 0),
1045         MLXSW_SP_SB_PM(0, 0),
1046         MLXSW_SP_SB_PM(10000, 90000),
1047         MLXSW_SP_SB_PM(0, 8),   /* 50% occupancy */
1048         MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN),
1049 };
1050
1051 /* Order according to mlxsw_sp*_sb_pool_dess */
1052 static const struct mlxsw_sp_sb_pm mlxsw_sp_cpu_port_sb_pms[] = {
1053         MLXSW_SP_SB_PM(0, 0),
1054         MLXSW_SP_SB_PM(0, 0),
1055         MLXSW_SP_SB_PM(0, 0),
1056         MLXSW_SP_SB_PM(0, 0),
1057         MLXSW_SP_SB_PM(0, 0),
1058         MLXSW_SP_SB_PM(0, 0),
1059         MLXSW_SP_SB_PM(0, 0),
1060         MLXSW_SP_SB_PM(0, 0),
1061         MLXSW_SP_SB_PM(0, 90000),
1062         MLXSW_SP_SB_PM(0, 0),
1063         MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX),
1064 };
1065
1066 static int mlxsw_sp_sb_pms_init(struct mlxsw_sp *mlxsw_sp, u16 local_port,
1067                                 const struct mlxsw_sp_sb_pm *pms,
1068                                 bool skip_ingress)
1069 {
1070         int i, err;
1071
1072         for (i = 0; i < mlxsw_sp->sb_vals->pool_count; i++) {
1073                 const struct mlxsw_sp_sb_pm *pm = &pms[i];
1074                 const struct mlxsw_sp_sb_pool_des *des;
1075                 u32 max_buff;
1076                 u32 min_buff;
1077
1078                 des = &mlxsw_sp->sb_vals->pool_dess[i];
1079                 if (skip_ingress && des->dir == MLXSW_REG_SBXX_DIR_INGRESS)
1080                         continue;
1081
1082                 min_buff = mlxsw_sp_bytes_cells(mlxsw_sp, pm->min_buff);
1083                 max_buff = pm->max_buff;
1084                 if (mlxsw_sp_sb_pool_is_static(mlxsw_sp, i))
1085                         max_buff = mlxsw_sp_bytes_cells(mlxsw_sp, max_buff);
1086                 err = mlxsw_sp_sb_pm_write(mlxsw_sp, local_port, i, min_buff,
1087                                            max_buff);
1088                 if (err)
1089                         return err;
1090         }
1091         return 0;
1092 }
1093
1094 static int mlxsw_sp_port_sb_pms_init(struct mlxsw_sp_port *mlxsw_sp_port)
1095 {
1096         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1097
1098         return mlxsw_sp_sb_pms_init(mlxsw_sp, mlxsw_sp_port->local_port,
1099                                     mlxsw_sp->sb_vals->pms, false);
1100 }
1101
1102 static int mlxsw_sp_cpu_port_sb_pms_init(struct mlxsw_sp *mlxsw_sp)
1103 {
1104         return mlxsw_sp_sb_pms_init(mlxsw_sp, 0, mlxsw_sp->sb_vals->pms_cpu,
1105                                     true);
1106 }
1107
1108 #define MLXSW_SP_SB_MM(_min_buff, _max_buff)            \
1109         {                                               \
1110                 .min_buff = _min_buff,                  \
1111                 .max_buff = _max_buff,                  \
1112                 .pool_index = MLXSW_SP_SB_POOL_EGR,     \
1113         }
1114
1115 static const struct mlxsw_sp_sb_mm mlxsw_sp_sb_mms[] = {
1116         MLXSW_SP_SB_MM(0, 6),
1117         MLXSW_SP_SB_MM(0, 6),
1118         MLXSW_SP_SB_MM(0, 6),
1119         MLXSW_SP_SB_MM(0, 6),
1120         MLXSW_SP_SB_MM(0, 6),
1121         MLXSW_SP_SB_MM(0, 6),
1122         MLXSW_SP_SB_MM(0, 6),
1123         MLXSW_SP_SB_MM(0, 6),
1124         MLXSW_SP_SB_MM(0, 6),
1125         MLXSW_SP_SB_MM(0, 6),
1126         MLXSW_SP_SB_MM(0, 6),
1127         MLXSW_SP_SB_MM(0, 6),
1128         MLXSW_SP_SB_MM(0, 6),
1129         MLXSW_SP_SB_MM(0, 6),
1130         MLXSW_SP_SB_MM(0, 6),
1131 };
1132
1133 static int mlxsw_sp_sb_mms_init(struct mlxsw_sp *mlxsw_sp)
1134 {
1135         char sbmm_pl[MLXSW_REG_SBMM_LEN];
1136         int i;
1137         int err;
1138
1139         for (i = 0; i < mlxsw_sp->sb_vals->mms_count; i++) {
1140                 const struct mlxsw_sp_sb_pool_des *des;
1141                 const struct mlxsw_sp_sb_mm *mc;
1142                 u32 min_buff;
1143
1144                 mc = &mlxsw_sp->sb_vals->mms[i];
1145                 des = &mlxsw_sp->sb_vals->pool_dess[mc->pool_index];
1146                 /* All pools used by sb_mm's are initialized using dynamic
1147                  * thresholds, therefore 'max_buff' isn't specified in cells.
1148                  */
1149                 min_buff = mlxsw_sp_bytes_cells(mlxsw_sp, mc->min_buff);
1150                 mlxsw_reg_sbmm_pack(sbmm_pl, i, min_buff, mc->max_buff,
1151                                     des->pool);
1152                 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sbmm), sbmm_pl);
1153                 if (err)
1154                         return err;
1155         }
1156         return 0;
1157 }
1158
1159 static void mlxsw_sp_pool_count(struct mlxsw_sp *mlxsw_sp,
1160                                 u16 *p_ingress_len, u16 *p_egress_len)
1161 {
1162         int i;
1163
1164         for (i = 0; i < mlxsw_sp->sb_vals->pool_count; ++i) {
1165                 if (mlxsw_sp->sb_vals->pool_dess[i].dir ==
1166                     MLXSW_REG_SBXX_DIR_INGRESS)
1167                         (*p_ingress_len)++;
1168                 else
1169                         (*p_egress_len)++;
1170         }
1171
1172         WARN(*p_egress_len == 0, "No egress pools\n");
1173 }
1174
1175 const struct mlxsw_sp_sb_vals mlxsw_sp1_sb_vals = {
1176         .pool_count = ARRAY_SIZE(mlxsw_sp1_sb_pool_dess),
1177         .pool_dess = mlxsw_sp1_sb_pool_dess,
1178         .pms = mlxsw_sp1_sb_pms,
1179         .pms_cpu = mlxsw_sp_cpu_port_sb_pms,
1180         .prs = mlxsw_sp1_sb_prs,
1181         .mms = mlxsw_sp_sb_mms,
1182         .cms_ingress = mlxsw_sp1_sb_cms_ingress,
1183         .cms_egress = mlxsw_sp1_sb_cms_egress,
1184         .cms_cpu = mlxsw_sp_cpu_port_sb_cms,
1185         .mms_count = ARRAY_SIZE(mlxsw_sp_sb_mms),
1186         .cms_ingress_count = ARRAY_SIZE(mlxsw_sp1_sb_cms_ingress),
1187         .cms_egress_count = ARRAY_SIZE(mlxsw_sp1_sb_cms_egress),
1188         .cms_cpu_count = ARRAY_SIZE(mlxsw_sp_cpu_port_sb_cms),
1189 };
1190
1191 const struct mlxsw_sp_sb_vals mlxsw_sp2_sb_vals = {
1192         .pool_count = ARRAY_SIZE(mlxsw_sp2_sb_pool_dess),
1193         .pool_dess = mlxsw_sp2_sb_pool_dess,
1194         .pms = mlxsw_sp2_sb_pms,
1195         .pms_cpu = mlxsw_sp_cpu_port_sb_pms,
1196         .prs = mlxsw_sp2_sb_prs,
1197         .mms = mlxsw_sp_sb_mms,
1198         .cms_ingress = mlxsw_sp2_sb_cms_ingress,
1199         .cms_egress = mlxsw_sp2_sb_cms_egress,
1200         .cms_cpu = mlxsw_sp_cpu_port_sb_cms,
1201         .mms_count = ARRAY_SIZE(mlxsw_sp_sb_mms),
1202         .cms_ingress_count = ARRAY_SIZE(mlxsw_sp2_sb_cms_ingress),
1203         .cms_egress_count = ARRAY_SIZE(mlxsw_sp2_sb_cms_egress),
1204         .cms_cpu_count = ARRAY_SIZE(mlxsw_sp_cpu_port_sb_cms),
1205 };
1206
1207 static u32 mlxsw_sp1_pb_int_buf_size_get(int mtu, u32 speed)
1208 {
1209         return mtu * 5 / 2;
1210 }
1211
1212 static u32 __mlxsw_sp_pb_int_buf_size_get(int mtu, u32 speed, u32 buffer_factor)
1213 {
1214         return 3 * mtu + buffer_factor * speed / 1000;
1215 }
1216
1217 #define MLXSW_SP2_SPAN_EG_MIRROR_BUFFER_FACTOR 38
1218
1219 static u32 mlxsw_sp2_pb_int_buf_size_get(int mtu, u32 speed)
1220 {
1221         int factor = MLXSW_SP2_SPAN_EG_MIRROR_BUFFER_FACTOR;
1222
1223         return __mlxsw_sp_pb_int_buf_size_get(mtu, speed, factor);
1224 }
1225
1226 #define MLXSW_SP3_SPAN_EG_MIRROR_BUFFER_FACTOR 50
1227
1228 static u32 mlxsw_sp3_pb_int_buf_size_get(int mtu, u32 speed)
1229 {
1230         int factor = MLXSW_SP3_SPAN_EG_MIRROR_BUFFER_FACTOR;
1231
1232         return __mlxsw_sp_pb_int_buf_size_get(mtu, speed, factor);
1233 }
1234
1235 const struct mlxsw_sp_sb_ops mlxsw_sp1_sb_ops = {
1236         .int_buf_size_get = mlxsw_sp1_pb_int_buf_size_get,
1237 };
1238
1239 const struct mlxsw_sp_sb_ops mlxsw_sp2_sb_ops = {
1240         .int_buf_size_get = mlxsw_sp2_pb_int_buf_size_get,
1241 };
1242
1243 const struct mlxsw_sp_sb_ops mlxsw_sp3_sb_ops = {
1244         .int_buf_size_get = mlxsw_sp3_pb_int_buf_size_get,
1245 };
1246
1247 int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp)
1248 {
1249         u32 max_headroom_size;
1250         u16 ing_pool_count = 0;
1251         u16 eg_pool_count = 0;
1252         int err;
1253
1254         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, CELL_SIZE))
1255                 return -EIO;
1256
1257         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, GUARANTEED_SHARED_BUFFER))
1258                 return -EIO;
1259
1260         if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_HEADROOM_SIZE))
1261                 return -EIO;
1262
1263         mlxsw_sp->sb = kzalloc(sizeof(*mlxsw_sp->sb), GFP_KERNEL);
1264         if (!mlxsw_sp->sb)
1265                 return -ENOMEM;
1266         mlxsw_sp->sb->cell_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, CELL_SIZE);
1267         mlxsw_sp->sb->sb_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1268                                                    GUARANTEED_SHARED_BUFFER);
1269         max_headroom_size = MLXSW_CORE_RES_GET(mlxsw_sp->core,
1270                                                MAX_HEADROOM_SIZE);
1271         /* Round down, because this limit must not be overstepped. */
1272         mlxsw_sp->sb->max_headroom_cells = max_headroom_size /
1273                                                 mlxsw_sp->sb->cell_size;
1274
1275         err = mlxsw_sp_sb_ports_init(mlxsw_sp);
1276         if (err)
1277                 goto err_sb_ports_init;
1278         err = mlxsw_sp_sb_prs_init(mlxsw_sp, mlxsw_sp->sb_vals->prs,
1279                                    mlxsw_sp->sb_vals->pool_dess,
1280                                    mlxsw_sp->sb_vals->pool_count);
1281         if (err)
1282                 goto err_sb_prs_init;
1283         err = mlxsw_sp_cpu_port_sb_cms_init(mlxsw_sp);
1284         if (err)
1285                 goto err_sb_cpu_port_sb_cms_init;
1286         err = mlxsw_sp_cpu_port_sb_pms_init(mlxsw_sp);
1287         if (err)
1288                 goto err_sb_cpu_port_pms_init;
1289         err = mlxsw_sp_sb_mms_init(mlxsw_sp);
1290         if (err)
1291                 goto err_sb_mms_init;
1292         mlxsw_sp_pool_count(mlxsw_sp, &ing_pool_count, &eg_pool_count);
1293         err = devl_sb_register(priv_to_devlink(mlxsw_sp->core), 0,
1294                                mlxsw_sp->sb->sb_size,
1295                                ing_pool_count,
1296                                eg_pool_count,
1297                                MLXSW_SP_SB_ING_TC_COUNT,
1298                                MLXSW_SP_SB_EG_TC_COUNT);
1299         if (err)
1300                 goto err_devlink_sb_register;
1301
1302         return 0;
1303
1304 err_devlink_sb_register:
1305 err_sb_mms_init:
1306 err_sb_cpu_port_pms_init:
1307 err_sb_cpu_port_sb_cms_init:
1308 err_sb_prs_init:
1309         mlxsw_sp_sb_ports_fini(mlxsw_sp);
1310 err_sb_ports_init:
1311         kfree(mlxsw_sp->sb);
1312         return err;
1313 }
1314
1315 void mlxsw_sp_buffers_fini(struct mlxsw_sp *mlxsw_sp)
1316 {
1317         devl_sb_unregister(priv_to_devlink(mlxsw_sp->core), 0);
1318         mlxsw_sp_sb_ports_fini(mlxsw_sp);
1319         kfree(mlxsw_sp->sb);
1320 }
1321
1322 int mlxsw_sp_port_buffers_init(struct mlxsw_sp_port *mlxsw_sp_port)
1323 {
1324         int err;
1325
1326         mlxsw_sp_port->hdroom = kzalloc(sizeof(*mlxsw_sp_port->hdroom), GFP_KERNEL);
1327         if (!mlxsw_sp_port->hdroom)
1328                 return -ENOMEM;
1329         mlxsw_sp_port->hdroom->mtu = mlxsw_sp_port->dev->mtu;
1330
1331         err = mlxsw_sp_port_headroom_init(mlxsw_sp_port);
1332         if (err)
1333                 goto err_headroom_init;
1334         err = mlxsw_sp_port_sb_cms_init(mlxsw_sp_port);
1335         if (err)
1336                 goto err_port_sb_cms_init;
1337         err = mlxsw_sp_port_sb_pms_init(mlxsw_sp_port);
1338         if (err)
1339                 goto err_port_sb_pms_init;
1340         return 0;
1341
1342 err_port_sb_pms_init:
1343 err_port_sb_cms_init:
1344 err_headroom_init:
1345         kfree(mlxsw_sp_port->hdroom);
1346         return err;
1347 }
1348
1349 void mlxsw_sp_port_buffers_fini(struct mlxsw_sp_port *mlxsw_sp_port)
1350 {
1351         kfree(mlxsw_sp_port->hdroom);
1352 }
1353
1354 int mlxsw_sp_sb_pool_get(struct mlxsw_core *mlxsw_core,
1355                          unsigned int sb_index, u16 pool_index,
1356                          struct devlink_sb_pool_info *pool_info)
1357 {
1358         struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1359         enum mlxsw_reg_sbxx_dir dir;
1360         struct mlxsw_sp_sb_pr *pr;
1361
1362         dir = mlxsw_sp->sb_vals->pool_dess[pool_index].dir;
1363         pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
1364         pool_info->pool_type = (enum devlink_sb_pool_type) dir;
1365         pool_info->size = mlxsw_sp_cells_bytes(mlxsw_sp, pr->size);
1366         pool_info->threshold_type = (enum devlink_sb_threshold_type) pr->mode;
1367         pool_info->cell_size = mlxsw_sp->sb->cell_size;
1368         return 0;
1369 }
1370
1371 int mlxsw_sp_sb_pool_set(struct mlxsw_core *mlxsw_core,
1372                          unsigned int sb_index, u16 pool_index, u32 size,
1373                          enum devlink_sb_threshold_type threshold_type,
1374                          struct netlink_ext_ack *extack)
1375 {
1376         struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1377         u32 pool_size = mlxsw_sp_bytes_cells(mlxsw_sp, size);
1378         const struct mlxsw_sp_sb_pr *pr;
1379         enum mlxsw_reg_sbpr_mode mode;
1380
1381         mode = (enum mlxsw_reg_sbpr_mode) threshold_type;
1382         pr = &mlxsw_sp->sb_vals->prs[pool_index];
1383
1384         if (size > MLXSW_CORE_RES_GET(mlxsw_sp->core,
1385                                       GUARANTEED_SHARED_BUFFER)) {
1386                 NL_SET_ERR_MSG_MOD(extack, "Exceeded shared buffer size");
1387                 return -EINVAL;
1388         }
1389
1390         if (pr->freeze_mode && pr->mode != mode) {
1391                 NL_SET_ERR_MSG_MOD(extack, "Changing this pool's threshold type is forbidden");
1392                 return -EINVAL;
1393         }
1394
1395         if (pr->freeze_size && pr->size != size) {
1396                 NL_SET_ERR_MSG_MOD(extack, "Changing this pool's size is forbidden");
1397                 return -EINVAL;
1398         }
1399
1400         return mlxsw_sp_sb_pr_write(mlxsw_sp, pool_index, mode,
1401                                     pool_size, false);
1402 }
1403
1404 #define MLXSW_SP_SB_THRESHOLD_TO_ALPHA_OFFSET (-2) /* 3->1, 16->14 */
1405
1406 static u32 mlxsw_sp_sb_threshold_out(struct mlxsw_sp *mlxsw_sp, u16 pool_index,
1407                                      u32 max_buff)
1408 {
1409         struct mlxsw_sp_sb_pr *pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
1410
1411         if (pr->mode == MLXSW_REG_SBPR_MODE_DYNAMIC)
1412                 return max_buff - MLXSW_SP_SB_THRESHOLD_TO_ALPHA_OFFSET;
1413         return mlxsw_sp_cells_bytes(mlxsw_sp, max_buff);
1414 }
1415
1416 static int mlxsw_sp_sb_threshold_in(struct mlxsw_sp *mlxsw_sp, u16 pool_index,
1417                                     u32 threshold, u32 *p_max_buff,
1418                                     struct netlink_ext_ack *extack)
1419 {
1420         struct mlxsw_sp_sb_pr *pr = mlxsw_sp_sb_pr_get(mlxsw_sp, pool_index);
1421
1422         if (pr->mode == MLXSW_REG_SBPR_MODE_DYNAMIC) {
1423                 int val;
1424
1425                 val = threshold + MLXSW_SP_SB_THRESHOLD_TO_ALPHA_OFFSET;
1426                 if (val < MLXSW_REG_SBXX_DYN_MAX_BUFF_MIN ||
1427                     val > MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX) {
1428                         NL_SET_ERR_MSG_MOD(extack, "Invalid dynamic threshold value");
1429                         return -EINVAL;
1430                 }
1431                 *p_max_buff = val;
1432         } else {
1433                 *p_max_buff = mlxsw_sp_bytes_cells(mlxsw_sp, threshold);
1434         }
1435         return 0;
1436 }
1437
1438 int mlxsw_sp_sb_port_pool_get(struct mlxsw_core_port *mlxsw_core_port,
1439                               unsigned int sb_index, u16 pool_index,
1440                               u32 *p_threshold)
1441 {
1442         struct mlxsw_sp_port *mlxsw_sp_port =
1443                         mlxsw_core_port_driver_priv(mlxsw_core_port);
1444         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1445         u16 local_port = mlxsw_sp_port->local_port;
1446         struct mlxsw_sp_sb_pm *pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port,
1447                                                        pool_index);
1448
1449         *p_threshold = mlxsw_sp_sb_threshold_out(mlxsw_sp, pool_index,
1450                                                  pm->max_buff);
1451         return 0;
1452 }
1453
1454 int mlxsw_sp_sb_port_pool_set(struct mlxsw_core_port *mlxsw_core_port,
1455                               unsigned int sb_index, u16 pool_index,
1456                               u32 threshold, struct netlink_ext_ack *extack)
1457 {
1458         struct mlxsw_sp_port *mlxsw_sp_port =
1459                         mlxsw_core_port_driver_priv(mlxsw_core_port);
1460         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1461         u16 local_port = mlxsw_sp_port->local_port;
1462         u32 max_buff;
1463         int err;
1464
1465         if (local_port == MLXSW_PORT_CPU_PORT) {
1466                 NL_SET_ERR_MSG_MOD(extack, "Changing CPU port's threshold is forbidden");
1467                 return -EINVAL;
1468         }
1469
1470         err = mlxsw_sp_sb_threshold_in(mlxsw_sp, pool_index,
1471                                        threshold, &max_buff, extack);
1472         if (err)
1473                 return err;
1474
1475         return mlxsw_sp_sb_pm_write(mlxsw_sp, local_port, pool_index,
1476                                     0, max_buff);
1477 }
1478
1479 int mlxsw_sp_sb_tc_pool_bind_get(struct mlxsw_core_port *mlxsw_core_port,
1480                                  unsigned int sb_index, u16 tc_index,
1481                                  enum devlink_sb_pool_type pool_type,
1482                                  u16 *p_pool_index, u32 *p_threshold)
1483 {
1484         struct mlxsw_sp_port *mlxsw_sp_port =
1485                         mlxsw_core_port_driver_priv(mlxsw_core_port);
1486         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1487         u16 local_port = mlxsw_sp_port->local_port;
1488         u8 pg_buff = tc_index;
1489         enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type;
1490         struct mlxsw_sp_sb_cm *cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port,
1491                                                        pg_buff, dir);
1492
1493         *p_threshold = mlxsw_sp_sb_threshold_out(mlxsw_sp, cm->pool_index,
1494                                                  cm->max_buff);
1495         *p_pool_index = cm->pool_index;
1496         return 0;
1497 }
1498
1499 int mlxsw_sp_sb_tc_pool_bind_set(struct mlxsw_core_port *mlxsw_core_port,
1500                                  unsigned int sb_index, u16 tc_index,
1501                                  enum devlink_sb_pool_type pool_type,
1502                                  u16 pool_index, u32 threshold,
1503                                  struct netlink_ext_ack *extack)
1504 {
1505         struct mlxsw_sp_port *mlxsw_sp_port =
1506                         mlxsw_core_port_driver_priv(mlxsw_core_port);
1507         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1508         u16 local_port = mlxsw_sp_port->local_port;
1509         const struct mlxsw_sp_sb_cm *cm;
1510         u8 pg_buff = tc_index;
1511         enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type;
1512         u32 max_buff;
1513         int err;
1514
1515         if (local_port == MLXSW_PORT_CPU_PORT) {
1516                 NL_SET_ERR_MSG_MOD(extack, "Changing CPU port's binding is forbidden");
1517                 return -EINVAL;
1518         }
1519
1520         if (dir != mlxsw_sp->sb_vals->pool_dess[pool_index].dir) {
1521                 NL_SET_ERR_MSG_MOD(extack, "Binding egress TC to ingress pool and vice versa is forbidden");
1522                 return -EINVAL;
1523         }
1524
1525         if (dir == MLXSW_REG_SBXX_DIR_INGRESS)
1526                 cm = &mlxsw_sp->sb_vals->cms_ingress[tc_index];
1527         else
1528                 cm = &mlxsw_sp->sb_vals->cms_egress[tc_index];
1529
1530         if (cm->freeze_pool && cm->pool_index != pool_index) {
1531                 NL_SET_ERR_MSG_MOD(extack, "Binding this TC to a different pool is forbidden");
1532                 return -EINVAL;
1533         }
1534
1535         if (cm->freeze_thresh && cm->max_buff != threshold) {
1536                 NL_SET_ERR_MSG_MOD(extack, "Changing this TC's threshold is forbidden");
1537                 return -EINVAL;
1538         }
1539
1540         err = mlxsw_sp_sb_threshold_in(mlxsw_sp, pool_index,
1541                                        threshold, &max_buff, extack);
1542         if (err)
1543                 return err;
1544
1545         return mlxsw_sp_sb_cm_write(mlxsw_sp, local_port, pg_buff,
1546                                     0, max_buff, false, pool_index);
1547 }
1548
1549 #define MASKED_COUNT_MAX \
1550         (MLXSW_REG_SBSR_REC_MAX_COUNT / \
1551          (MLXSW_SP_SB_ING_TC_COUNT + MLXSW_SP_SB_EG_TC_COUNT))
1552
1553 struct mlxsw_sp_sb_sr_occ_query_cb_ctx {
1554         u8 masked_count;
1555         u16 local_port_1;
1556 };
1557
1558 static void mlxsw_sp_sb_sr_occ_query_cb(struct mlxsw_core *mlxsw_core,
1559                                         char *sbsr_pl, size_t sbsr_pl_len,
1560                                         unsigned long cb_priv)
1561 {
1562         struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1563         struct mlxsw_sp_sb_sr_occ_query_cb_ctx cb_ctx;
1564         u8 masked_count;
1565         u16 local_port;
1566         int rec_index = 0;
1567         struct mlxsw_sp_sb_cm *cm;
1568         int i;
1569
1570         memcpy(&cb_ctx, &cb_priv, sizeof(cb_ctx));
1571
1572         masked_count = 0;
1573         for (local_port = cb_ctx.local_port_1;
1574              local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) {
1575                 if (!mlxsw_sp->ports[local_port])
1576                         continue;
1577                 if (local_port == MLXSW_PORT_CPU_PORT) {
1578                         /* Ingress quotas are not supported for the CPU port */
1579                         masked_count++;
1580                         continue;
1581                 }
1582                 for (i = 0; i < MLXSW_SP_SB_ING_TC_COUNT; i++) {
1583                         cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port, i,
1584                                                 MLXSW_REG_SBXX_DIR_INGRESS);
1585                         mlxsw_reg_sbsr_rec_unpack(sbsr_pl, rec_index++,
1586                                                   &cm->occ.cur, &cm->occ.max);
1587                 }
1588                 if (++masked_count == cb_ctx.masked_count)
1589                         break;
1590         }
1591         masked_count = 0;
1592         for (local_port = cb_ctx.local_port_1;
1593              local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) {
1594                 if (!mlxsw_sp->ports[local_port])
1595                         continue;
1596                 for (i = 0; i < MLXSW_SP_SB_EG_TC_COUNT; i++) {
1597                         cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port, i,
1598                                                 MLXSW_REG_SBXX_DIR_EGRESS);
1599                         mlxsw_reg_sbsr_rec_unpack(sbsr_pl, rec_index++,
1600                                                   &cm->occ.cur, &cm->occ.max);
1601                 }
1602                 if (++masked_count == cb_ctx.masked_count)
1603                         break;
1604         }
1605 }
1606
1607 int mlxsw_sp_sb_occ_snapshot(struct mlxsw_core *mlxsw_core,
1608                              unsigned int sb_index)
1609 {
1610         struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1611         u16 local_port, local_port_1, last_local_port;
1612         struct mlxsw_sp_sb_sr_occ_query_cb_ctx cb_ctx;
1613         u8 masked_count, current_page = 0;
1614         unsigned long cb_priv = 0;
1615         LIST_HEAD(bulk_list);
1616         char *sbsr_pl;
1617         int i;
1618         int err;
1619         int err2;
1620
1621         sbsr_pl = kmalloc(MLXSW_REG_SBSR_LEN, GFP_KERNEL);
1622         if (!sbsr_pl)
1623                 return -ENOMEM;
1624
1625         local_port = MLXSW_PORT_CPU_PORT;
1626 next_batch:
1627         local_port_1 = local_port;
1628         masked_count = 0;
1629         mlxsw_reg_sbsr_pack(sbsr_pl, false);
1630         mlxsw_reg_sbsr_port_page_set(sbsr_pl, current_page);
1631         last_local_port = current_page * MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE +
1632                           MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE - 1;
1633
1634         for (i = 0; i < MLXSW_SP_SB_ING_TC_COUNT; i++)
1635                 mlxsw_reg_sbsr_pg_buff_mask_set(sbsr_pl, i, 1);
1636         for (i = 0; i < MLXSW_SP_SB_EG_TC_COUNT; i++)
1637                 mlxsw_reg_sbsr_tclass_mask_set(sbsr_pl, i, 1);
1638         for (; local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) {
1639                 if (!mlxsw_sp->ports[local_port])
1640                         continue;
1641                 if (local_port > last_local_port) {
1642                         current_page++;
1643                         goto do_query;
1644                 }
1645                 if (local_port != MLXSW_PORT_CPU_PORT) {
1646                         /* Ingress quotas are not supported for the CPU port */
1647                         mlxsw_reg_sbsr_ingress_port_mask_set(sbsr_pl,
1648                                                              local_port, 1);
1649                 }
1650                 mlxsw_reg_sbsr_egress_port_mask_set(sbsr_pl, local_port, 1);
1651                 for (i = 0; i < mlxsw_sp->sb_vals->pool_count; i++) {
1652                         err = mlxsw_sp_sb_pm_occ_query(mlxsw_sp, local_port, i,
1653                                                        &bulk_list);
1654                         if (err)
1655                                 goto out;
1656                 }
1657                 if (++masked_count == MASKED_COUNT_MAX)
1658                         goto do_query;
1659         }
1660
1661 do_query:
1662         cb_ctx.masked_count = masked_count;
1663         cb_ctx.local_port_1 = local_port_1;
1664         memcpy(&cb_priv, &cb_ctx, sizeof(cb_ctx));
1665         err = mlxsw_reg_trans_query(mlxsw_core, MLXSW_REG(sbsr), sbsr_pl,
1666                                     &bulk_list, mlxsw_sp_sb_sr_occ_query_cb,
1667                                     cb_priv);
1668         if (err)
1669                 goto out;
1670         if (local_port < mlxsw_core_max_ports(mlxsw_core)) {
1671                 local_port++;
1672                 goto next_batch;
1673         }
1674
1675 out:
1676         err2 = mlxsw_reg_trans_bulk_wait(&bulk_list);
1677         if (!err)
1678                 err = err2;
1679         kfree(sbsr_pl);
1680         return err;
1681 }
1682
1683 int mlxsw_sp_sb_occ_max_clear(struct mlxsw_core *mlxsw_core,
1684                               unsigned int sb_index)
1685 {
1686         struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core);
1687         u16 local_port, last_local_port;
1688         LIST_HEAD(bulk_list);
1689         unsigned int masked_count;
1690         u8 current_page = 0;
1691         char *sbsr_pl;
1692         int i;
1693         int err;
1694         int err2;
1695
1696         sbsr_pl = kmalloc(MLXSW_REG_SBSR_LEN, GFP_KERNEL);
1697         if (!sbsr_pl)
1698                 return -ENOMEM;
1699
1700         local_port = MLXSW_PORT_CPU_PORT;
1701 next_batch:
1702         masked_count = 0;
1703         mlxsw_reg_sbsr_pack(sbsr_pl, true);
1704         mlxsw_reg_sbsr_port_page_set(sbsr_pl, current_page);
1705         last_local_port = current_page * MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE +
1706                           MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE - 1;
1707
1708         for (i = 0; i < MLXSW_SP_SB_ING_TC_COUNT; i++)
1709                 mlxsw_reg_sbsr_pg_buff_mask_set(sbsr_pl, i, 1);
1710         for (i = 0; i < MLXSW_SP_SB_EG_TC_COUNT; i++)
1711                 mlxsw_reg_sbsr_tclass_mask_set(sbsr_pl, i, 1);
1712         for (; local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) {
1713                 if (!mlxsw_sp->ports[local_port])
1714                         continue;
1715                 if (local_port > last_local_port) {
1716                         current_page++;
1717                         goto do_query;
1718                 }
1719                 if (local_port != MLXSW_PORT_CPU_PORT) {
1720                         /* Ingress quotas are not supported for the CPU port */
1721                         mlxsw_reg_sbsr_ingress_port_mask_set(sbsr_pl,
1722                                                              local_port, 1);
1723                 }
1724                 mlxsw_reg_sbsr_egress_port_mask_set(sbsr_pl, local_port, 1);
1725                 for (i = 0; i < mlxsw_sp->sb_vals->pool_count; i++) {
1726                         err = mlxsw_sp_sb_pm_occ_clear(mlxsw_sp, local_port, i,
1727                                                        &bulk_list);
1728                         if (err)
1729                                 goto out;
1730                 }
1731                 if (++masked_count == MASKED_COUNT_MAX)
1732                         goto do_query;
1733         }
1734
1735 do_query:
1736         err = mlxsw_reg_trans_query(mlxsw_core, MLXSW_REG(sbsr), sbsr_pl,
1737                                     &bulk_list, NULL, 0);
1738         if (err)
1739                 goto out;
1740         if (local_port < mlxsw_core_max_ports(mlxsw_core)) {
1741                 local_port++;
1742                 goto next_batch;
1743         }
1744
1745 out:
1746         err2 = mlxsw_reg_trans_bulk_wait(&bulk_list);
1747         if (!err)
1748                 err = err2;
1749         kfree(sbsr_pl);
1750         return err;
1751 }
1752
1753 int mlxsw_sp_sb_occ_port_pool_get(struct mlxsw_core_port *mlxsw_core_port,
1754                                   unsigned int sb_index, u16 pool_index,
1755                                   u32 *p_cur, u32 *p_max)
1756 {
1757         struct mlxsw_sp_port *mlxsw_sp_port =
1758                         mlxsw_core_port_driver_priv(mlxsw_core_port);
1759         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1760         u16 local_port = mlxsw_sp_port->local_port;
1761         struct mlxsw_sp_sb_pm *pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port,
1762                                                        pool_index);
1763
1764         *p_cur = mlxsw_sp_cells_bytes(mlxsw_sp, pm->occ.cur);
1765         *p_max = mlxsw_sp_cells_bytes(mlxsw_sp, pm->occ.max);
1766         return 0;
1767 }
1768
1769 int mlxsw_sp_sb_occ_tc_port_bind_get(struct mlxsw_core_port *mlxsw_core_port,
1770                                      unsigned int sb_index, u16 tc_index,
1771                                      enum devlink_sb_pool_type pool_type,
1772                                      u32 *p_cur, u32 *p_max)
1773 {
1774         struct mlxsw_sp_port *mlxsw_sp_port =
1775                         mlxsw_core_port_driver_priv(mlxsw_core_port);
1776         struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp;
1777         u16 local_port = mlxsw_sp_port->local_port;
1778         u8 pg_buff = tc_index;
1779         enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type;
1780         struct mlxsw_sp_sb_cm *cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port,
1781                                                        pg_buff, dir);
1782
1783         *p_cur = mlxsw_sp_cells_bytes(mlxsw_sp, cm->occ.cur);
1784         *p_max = mlxsw_sp_cells_bytes(mlxsw_sp, cm->occ.max);
1785         return 0;
1786 }