drivers/net/ethernet/aquantia/atlantic/aq_vec.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Atlantic Network Driver
   3  *
   4  * Copyright (C) 2014-2019 aQuantia Corporation
   5  * Copyright (C) 2019-2020 Marvell International Ltd.
   6  */
   7
   8 /* File aq_vec.c: Definition of common structure for vector of Rx and Tx rings.
   9  * Definition of functions for Rx and Tx rings. Friendly module for aq_nic.
  10  */
  11
  12 #include "aq_vec.h"
  13
  14 struct aq_vec_s {
  15         const struct aq_hw_ops *aq_hw_ops;
  16         struct aq_hw_s *aq_hw;
  17         struct aq_nic_s *aq_nic;
  18         unsigned int tx_rings;
  19         unsigned int rx_rings;
  20         struct aq_ring_param_s aq_ring_param;
  21         struct napi_struct napi;
  22         struct aq_ring_s ring[AQ_CFG_TCS_MAX][2];
  23 };
  24
  25 #define AQ_VEC_TX_ID 0
  26 #define AQ_VEC_RX_ID 1
  27
  28 static int aq_vec_poll(struct napi_struct *napi, int budget)
  29 {
  30         struct aq_vec_s *self = container_of(napi, struct aq_vec_s, napi);
  31         unsigned int sw_tail_old = 0U;
  32         struct aq_ring_s *ring = NULL;
  33         bool was_tx_cleaned = true;
  34         unsigned int i = 0U;
  35         int work_done = 0;
  36         int err = 0;
  37
  38         if (!self) {
  39                 err = -EINVAL;
  40         } else {
  41                 for (i = 0U; self->tx_rings > i; ++i) {
  42                         ring = self->ring[i];
  43                         u64_stats_update_begin(&ring[AQ_VEC_RX_ID].stats.rx.syncp);
  44                         ring[AQ_VEC_RX_ID].stats.rx.polls++;
  45                         u64_stats_update_end(&ring[AQ_VEC_RX_ID].stats.rx.syncp);
  46                         if (self->aq_hw_ops->hw_ring_tx_head_update) {
  47                                 err = self->aq_hw_ops->hw_ring_tx_head_update(
  48                                                         self->aq_hw,
  49                                                         &ring[AQ_VEC_TX_ID]);
  50                                 if (err < 0)
  51                                         goto err_exit;
  52                         }
  53
  54                         if (ring[AQ_VEC_TX_ID].sw_head !=
  55                             ring[AQ_VEC_TX_ID].hw_head) {
  56                                 was_tx_cleaned = aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
  57                                 aq_ring_update_queue_state(&ring[AQ_VEC_TX_ID]);
  58                         }
  59
  60                         err = self->aq_hw_ops->hw_ring_rx_receive(self->aq_hw,
  61                                             &ring[AQ_VEC_RX_ID]);
  62                         if (err < 0)
  63                                 goto err_exit;
  64
  65                         if (ring[AQ_VEC_RX_ID].sw_head !=
  66                                 ring[AQ_VEC_RX_ID].hw_head) {
  67                                 err = aq_ring_rx_clean(&ring[AQ_VEC_RX_ID],
  68                                                        napi,
  69                                                        &work_done,
  70                                                        budget - work_done);
  71                                 if (err < 0)
  72                                         goto err_exit;
  73
  74                                 sw_tail_old = ring[AQ_VEC_RX_ID].sw_tail;
  75
  76                                 err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]);
  77                                 if (err < 0)
  78                                         goto err_exit;
  79
  80                                 err = self->aq_hw_ops->hw_ring_rx_fill(
  81                                         self->aq_hw,
  82                                         &ring[AQ_VEC_RX_ID], sw_tail_old);
  83                                 if (err < 0)
  84                                         goto err_exit;
  85                         }
  86                 }
  87
  88 err_exit:
  89                 if (!was_tx_cleaned)
  90                         work_done = budget;
  91
  92                 if (work_done < budget) {
  93                         napi_complete_done(napi, work_done);
  94                         self->aq_hw_ops->hw_irq_enable(self->aq_hw,
  95                                         1U << self->aq_ring_param.vec_idx);
  96                 }
  97         }
  98
  99         return work_done;
 100 }
 101
 102 struct aq_vec_s *aq_vec_alloc(struct aq_nic_s *aq_nic, unsigned int idx,
 103                               struct aq_nic_cfg_s *aq_nic_cfg)
 104 {
 105         struct aq_vec_s *self = NULL;
 106
 107         self = kzalloc(sizeof(*self), GFP_KERNEL);
 108         if (!self)
 109                 goto err_exit;
 110
 111         self->aq_nic = aq_nic;
 112         self->aq_ring_param.vec_idx = idx;
 113         self->aq_ring_param.cpu =
 114                 idx + aq_nic_cfg->aq_rss.base_cpu_number;
 115
 116         cpumask_set_cpu(self->aq_ring_param.cpu,
 117                         &self->aq_ring_param.affinity_mask);
 118
 119         self->tx_rings = 0;
 120         self->rx_rings = 0;
 121
 122         netif_napi_add(aq_nic_get_ndev(aq_nic), &self->napi, aq_vec_poll);
 123
 124 err_exit:
 125         return self;
 126 }
 127
 128 int aq_vec_ring_alloc(struct aq_vec_s *self, struct aq_nic_s *aq_nic,
 129                       unsigned int idx, struct aq_nic_cfg_s *aq_nic_cfg)
 130 {
 131         struct aq_ring_s *ring = NULL;
 132         unsigned int i = 0U;
 133         int err = 0;
 134
 135         for (i = 0; i < aq_nic_cfg->tcs; ++i) {
 136                 const unsigned int idx_ring = AQ_NIC_CFG_TCVEC2RING(aq_nic_cfg,
 137                                                                     i, idx);
 138
 139                 ring = &self->ring[i][AQ_VEC_TX_ID];
 140                 err = aq_ring_tx_alloc(ring, aq_nic, idx_ring, aq_nic_cfg);
 141                 if (err)
 142                         goto err_exit;
 143
 144                 ++self->tx_rings;
 145
 146                 aq_nic_set_tx_ring(aq_nic, idx_ring, ring);
 147
 148                 ring = &self->ring[i][AQ_VEC_RX_ID];
 149                 if (xdp_rxq_info_reg(&ring->xdp_rxq,
 150                                      aq_nic->ndev, idx,
 151                                      self->napi.napi_id) < 0) {
 152                         err = -ENOMEM;
 153                         goto err_exit;
 154                 }
 155                 if (xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
 156                                                MEM_TYPE_PAGE_SHARED, NULL) < 0) {
 157                         xdp_rxq_info_unreg(&ring->xdp_rxq);
 158                         err = -ENOMEM;
 159                         goto err_exit;
 160                 }
 161
 162                 err = aq_ring_rx_alloc(ring, aq_nic, idx_ring, aq_nic_cfg);
 163                 if (err) {
 164                         xdp_rxq_info_unreg(&ring->xdp_rxq);
 165                         goto err_exit;
 166                 }
 167
 168                 ++self->rx_rings;
 169         }
 170
 171 err_exit:
 172         if (err < 0) {
 173                 aq_vec_ring_free(self);
 174                 self = NULL;
 175         }
 176
 177         return err;
 178 }
 179
 180 int aq_vec_init(struct aq_vec_s *self, const struct aq_hw_ops *aq_hw_ops,
 181                 struct aq_hw_s *aq_hw)
 182 {
 183         struct aq_ring_s *ring = NULL;
 184         unsigned int i = 0U;
 185         int err = 0;
 186
 187         self->aq_hw_ops = aq_hw_ops;
 188         self->aq_hw = aq_hw;
 189
 190         for (i = 0U; self->tx_rings > i; ++i) {
 191                 ring = self->ring[i];
 192                 err = aq_ring_init(&ring[AQ_VEC_TX_ID], ATL_RING_TX);
 193                 if (err < 0)
 194                         goto err_exit;
 195
 196                 err = self->aq_hw_ops->hw_ring_tx_init(self->aq_hw,
 197                                                        &ring[AQ_VEC_TX_ID],
 198                                                        &self->aq_ring_param);
 199                 if (err < 0)
 200                         goto err_exit;
 201
 202                 err = aq_ring_init(&ring[AQ_VEC_RX_ID], ATL_RING_RX);
 203                 if (err < 0)
 204                         goto err_exit;
 205
 206                 err = self->aq_hw_ops->hw_ring_rx_init(self->aq_hw,
 207                                                        &ring[AQ_VEC_RX_ID],
 208                                                        &self->aq_ring_param);
 209                 if (err < 0)
 210                         goto err_exit;
 211
 212                 err = aq_ring_rx_fill(&ring[AQ_VEC_RX_ID]);
 213                 if (err < 0)
 214                         goto err_exit;
 215
 216                 err = self->aq_hw_ops->hw_ring_rx_fill(self->aq_hw,
 217                                                        &ring[AQ_VEC_RX_ID], 0U);
 218                 if (err < 0)
 219                         goto err_exit;
 220         }
 221
 222 err_exit:
 223         return err;
 224 }
 225
 226 int aq_vec_start(struct aq_vec_s *self)
 227 {
 228         struct aq_ring_s *ring = NULL;
 229         unsigned int i = 0U;
 230         int err = 0;
 231
 232         for (i = 0U; self->tx_rings > i; ++i) {
 233                 ring = self->ring[i];
 234                 err = self->aq_hw_ops->hw_ring_tx_start(self->aq_hw,
 235                                                         &ring[AQ_VEC_TX_ID]);
 236                 if (err < 0)
 237                         goto err_exit;
 238
 239                 err = self->aq_hw_ops->hw_ring_rx_start(self->aq_hw,
 240                                                         &ring[AQ_VEC_RX_ID]);
 241                 if (err < 0)
 242                         goto err_exit;
 243         }
 244
 245         napi_enable(&self->napi);
 246
 247 err_exit:
 248         return err;
 249 }
 250
 251 void aq_vec_stop(struct aq_vec_s *self)
 252 {
 253         struct aq_ring_s *ring = NULL;
 254         unsigned int i = 0U;
 255
 256         for (i = 0U; self->tx_rings > i; ++i) {
 257                 ring = self->ring[i];
 258                 self->aq_hw_ops->hw_ring_tx_stop(self->aq_hw,
 259                                                  &ring[AQ_VEC_TX_ID]);
 260
 261                 self->aq_hw_ops->hw_ring_rx_stop(self->aq_hw,
 262                                                  &ring[AQ_VEC_RX_ID]);
 263         }
 264
 265         napi_disable(&self->napi);
 266 }
 267
 268 void aq_vec_deinit(struct aq_vec_s *self)
 269 {
 270         struct aq_ring_s *ring = NULL;
 271         unsigned int i = 0U;
 272
 273         if (!self)
 274                 goto err_exit;
 275
 276         for (i = 0U; self->tx_rings > i; ++i) {
 277                 ring = self->ring[i];
 278                 aq_ring_tx_clean(&ring[AQ_VEC_TX_ID]);
 279                 aq_ring_rx_deinit(&ring[AQ_VEC_RX_ID]);
 280         }
 281
 282 err_exit:;
 283 }
 284
 285 void aq_vec_free(struct aq_vec_s *self)
 286 {
 287         if (!self)
 288                 goto err_exit;
 289
 290         netif_napi_del(&self->napi);
 291
 292         kfree(self);
 293
 294 err_exit:;
 295 }
 296
 297 void aq_vec_ring_free(struct aq_vec_s *self)
 298 {
 299         struct aq_ring_s *ring = NULL;
 300         unsigned int i = 0U;
 301
 302         if (!self)
 303                 goto err_exit;
 304
 305         for (i = 0U; self->tx_rings > i; ++i) {
 306                 ring = self->ring[i];
 307                 aq_ring_free(&ring[AQ_VEC_TX_ID]);
 308                 if (i < self->rx_rings) {
 309                         xdp_rxq_info_unreg(&ring[AQ_VEC_RX_ID].xdp_rxq);
 310                         aq_ring_free(&ring[AQ_VEC_RX_ID]);
 311                 }
 312         }
 313
 314         self->tx_rings = 0;
 315         self->rx_rings = 0;
 316 err_exit:;
 317 }
 318
 319 irqreturn_t aq_vec_isr(int irq, void *private)
 320 {
 321         struct aq_vec_s *self = private;
 322         int err = 0;
 323
 324         if (!self) {
 325                 err = -EINVAL;
 326                 goto err_exit;
 327         }
 328         napi_schedule(&self->napi);
 329
 330 err_exit:
 331         return err >= 0 ? IRQ_HANDLED : IRQ_NONE;
 332 }
 333
 334 irqreturn_t aq_vec_isr_legacy(int irq, void *private)
 335 {
 336         struct aq_vec_s *self = private;
 337         u64 irq_mask = 0U;
 338         int err;
 339
 340         if (!self)
 341                 return IRQ_NONE;
 342         err = self->aq_hw_ops->hw_irq_read(self->aq_hw, &irq_mask);
 343         if (err < 0)
 344                 return IRQ_NONE;
 345
 346         if (irq_mask) {
 347                 self->aq_hw_ops->hw_irq_disable(self->aq_hw,
 348                               1U << self->aq_ring_param.vec_idx);
 349                 napi_schedule(&self->napi);
 350         } else {
 351                 self->aq_hw_ops->hw_irq_enable(self->aq_hw, 1U);
 352                 return IRQ_NONE;
 353         }
 354
 355         return IRQ_HANDLED;
 356 }
 357
 358 cpumask_t *aq_vec_get_affinity_mask(struct aq_vec_s *self)
 359 {
 360         return &self->aq_ring_param.affinity_mask;
 361 }
 362
 363 bool aq_vec_is_valid_tc(struct aq_vec_s *self, const unsigned int tc)
 364 {
 365         return tc < self->rx_rings && tc < self->tx_rings;
 366 }
 367
 368 unsigned int aq_vec_get_sw_stats(struct aq_vec_s *self, const unsigned int tc, u64 *data)
 369 {
 370         unsigned int count;
 371
 372         if (!aq_vec_is_valid_tc(self, tc))
 373                 return 0;
 374
 375         count = aq_ring_fill_stats_data(&self->ring[tc][AQ_VEC_RX_ID], data);
 376         count += aq_ring_fill_stats_data(&self->ring[tc][AQ_VEC_TX_ID], data + count);
 377
 378         return count;
 379 }