drivers/net/ethernet/marvell/octeon_ep/octep_rx.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Marvell Octeon EP (EndPoint) Ethernet Driver
   3  *
   4  * Copyright (C) 2020 Marvell.
   5  *
   6  */
   7
   8 #include <linux/pci.h>
   9 #include <linux/etherdevice.h>
  10 #include <linux/vmalloc.h>
  11
  12 #include "octep_config.h"
  13 #include "octep_main.h"
  14
  15 static void octep_oq_reset_indices(struct octep_oq *oq)
  16 {
  17         oq->host_read_idx = 0;
  18         oq->host_refill_idx = 0;
  19         oq->refill_count = 0;
  20         oq->last_pkt_count = 0;
  21         oq->pkts_pending = 0;
  22 }
  23
  24 /**
  25  * octep_oq_fill_ring_buffers() - fill initial receive buffers for Rx ring.
  26  *
  27  * @oq: Octeon Rx queue data structure.
  28  *
  29  * Return: 0, if successfully filled receive buffers for all descriptors.
  30  *         -1, if failed to allocate a buffer or failed to map for DMA.
  31  */
  32 static int octep_oq_fill_ring_buffers(struct octep_oq *oq)
  33 {
  34         struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
  35         struct page *page;
  36         u32 i;
  37
  38         for (i = 0; i < oq->max_count; i++) {
  39                 page = dev_alloc_page();
  40                 if (unlikely(!page)) {
  41                         dev_err(oq->dev, "Rx buffer alloc failed\n");
  42                         goto rx_buf_alloc_err;
  43                 }
  44                 desc_ring[i].buffer_ptr = dma_map_page(oq->dev, page, 0,
  45                                                        PAGE_SIZE,
  46                                                        DMA_FROM_DEVICE);
  47                 if (dma_mapping_error(oq->dev, desc_ring[i].buffer_ptr)) {
  48                         dev_err(oq->dev,
  49                                 "OQ-%d buffer alloc: DMA mapping error!\n",
  50                                 oq->q_no);
  51                         put_page(page);
  52                         goto dma_map_err;
  53                 }
  54                 oq->buff_info[i].page = page;
  55         }
  56
  57         return 0;
  58
  59 dma_map_err:
  60 rx_buf_alloc_err:
  61         while (i) {
  62                 i--;
  63                 dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr, PAGE_SIZE, DMA_FROM_DEVICE);
  64                 put_page(oq->buff_info[i].page);
  65                 oq->buff_info[i].page = NULL;
  66         }
  67
  68         return -1;
  69 }
  70
  71 /**
  72  * octep_oq_refill() - refill buffers for used Rx ring descriptors.
  73  *
  74  * @oct: Octeon device private data structure.
  75  * @oq: Octeon Rx queue data structure.
  76  *
  77  * Return: number of descriptors successfully refilled with receive buffers.
  78  */
  79 static int octep_oq_refill(struct octep_device *oct, struct octep_oq *oq)
  80 {
  81         struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
  82         struct page *page;
  83         u32 refill_idx, i;
  84
  85         refill_idx = oq->host_refill_idx;
  86         for (i = 0; i < oq->refill_count; i++) {
  87                 page = dev_alloc_page();
  88                 if (unlikely(!page)) {
  89                         dev_err(oq->dev, "refill: rx buffer alloc failed\n");
  90                         oq->stats.alloc_failures++;
  91                         break;
  92                 }
  93
  94                 desc_ring[refill_idx].buffer_ptr = dma_map_page(oq->dev, page, 0,
  95                                                                 PAGE_SIZE, DMA_FROM_DEVICE);
  96                 if (dma_mapping_error(oq->dev, desc_ring[refill_idx].buffer_ptr)) {
  97                         dev_err(oq->dev,
  98                                 "OQ-%d buffer refill: DMA mapping error!\n",
  99                                 oq->q_no);
 100                         put_page(page);
 101                         oq->stats.alloc_failures++;
 102                         break;
 103                 }
 104                 oq->buff_info[refill_idx].page = page;
 105                 refill_idx++;
 106                 if (refill_idx == oq->max_count)
 107                         refill_idx = 0;
 108         }
 109         oq->host_refill_idx = refill_idx;
 110         oq->refill_count -= i;
 111
 112         return i;
 113 }
 114
 115 /**
 116  * octep_setup_oq() - Setup a Rx queue.
 117  *
 118  * @oct: Octeon device private data structure.
 119  * @q_no: Rx queue number to be setup.
 120  *
 121  * Allocate resources for a Rx queue.
 122  */
 123 static int octep_setup_oq(struct octep_device *oct, int q_no)
 124 {
 125         struct octep_oq *oq;
 126         u32 desc_ring_size;
 127
 128         oq = vzalloc(sizeof(*oq));
 129         if (!oq)
 130                 goto create_oq_fail;
 131         oct->oq[q_no] = oq;
 132
 133         oq->octep_dev = oct;
 134         oq->netdev = oct->netdev;
 135         oq->dev = &oct->pdev->dev;
 136         oq->q_no = q_no;
 137         oq->max_count = CFG_GET_OQ_NUM_DESC(oct->conf);
 138         oq->ring_size_mask = oq->max_count - 1;
 139         oq->buffer_size = CFG_GET_OQ_BUF_SIZE(oct->conf);
 140         oq->max_single_buffer_size = oq->buffer_size - OCTEP_OQ_RESP_HW_SIZE;
 141
 142         /* When the hardware/firmware supports additional capabilities,
 143          * additional header is filled-in by Octeon after length field in
 144          * Rx packets. this header contains additional packet information.
 145          */
 146         if (oct->conf->fw_info.rx_ol_flags)
 147                 oq->max_single_buffer_size -= OCTEP_OQ_RESP_HW_EXT_SIZE;
 148
 149         oq->refill_threshold = CFG_GET_OQ_REFILL_THRESHOLD(oct->conf);
 150
 151         desc_ring_size = oq->max_count * OCTEP_OQ_DESC_SIZE;
 152         oq->desc_ring = dma_alloc_coherent(oq->dev, desc_ring_size,
 153                                            &oq->desc_ring_dma, GFP_KERNEL);
 154
 155         if (unlikely(!oq->desc_ring)) {
 156                 dev_err(oq->dev,
 157                         "Failed to allocate DMA memory for OQ-%d !!\n", q_no);
 158                 goto desc_dma_alloc_err;
 159         }
 160
 161         oq->buff_info = vcalloc(oq->max_count, OCTEP_OQ_RECVBUF_SIZE);
 162         if (unlikely(!oq->buff_info)) {
 163                 dev_err(&oct->pdev->dev,
 164                         "Failed to allocate buffer info for OQ-%d\n", q_no);
 165                 goto buf_list_err;
 166         }
 167
 168         if (octep_oq_fill_ring_buffers(oq))
 169                 goto oq_fill_buff_err;
 170
 171         octep_oq_reset_indices(oq);
 172         oct->hw_ops.setup_oq_regs(oct, q_no);
 173         oct->num_oqs++;
 174
 175         return 0;
 176
 177 oq_fill_buff_err:
 178         vfree(oq->buff_info);
 179         oq->buff_info = NULL;
 180 buf_list_err:
 181         dma_free_coherent(oq->dev, desc_ring_size,
 182                           oq->desc_ring, oq->desc_ring_dma);
 183         oq->desc_ring = NULL;
 184 desc_dma_alloc_err:
 185         vfree(oq);
 186         oct->oq[q_no] = NULL;
 187 create_oq_fail:
 188         return -1;
 189 }
 190
 191 /**
 192  * octep_oq_free_ring_buffers() - Free ring buffers.
 193  *
 194  * @oq: Octeon Rx queue data structure.
 195  *
 196  * Free receive buffers in unused Rx queue descriptors.
 197  */
 198 static void octep_oq_free_ring_buffers(struct octep_oq *oq)
 199 {
 200         struct octep_oq_desc_hw *desc_ring = oq->desc_ring;
 201         int  i;
 202
 203         if (!oq->desc_ring || !oq->buff_info)
 204                 return;
 205
 206         for (i = 0; i < oq->max_count; i++)  {
 207                 if (oq->buff_info[i].page) {
 208                         dma_unmap_page(oq->dev, desc_ring[i].buffer_ptr,
 209                                        PAGE_SIZE, DMA_FROM_DEVICE);
 210                         put_page(oq->buff_info[i].page);
 211                         oq->buff_info[i].page = NULL;
 212                         desc_ring[i].buffer_ptr = 0;
 213                 }
 214         }
 215         octep_oq_reset_indices(oq);
 216 }
 217
 218 /**
 219  * octep_free_oq() - Free Rx queue resources.
 220  *
 221  * @oq: Octeon Rx queue data structure.
 222  *
 223  * Free all resources of a Rx queue.
 224  */
 225 static int octep_free_oq(struct octep_oq *oq)
 226 {
 227         struct octep_device *oct = oq->octep_dev;
 228         int q_no = oq->q_no;
 229
 230         octep_oq_free_ring_buffers(oq);
 231
 232         vfree(oq->buff_info);
 233
 234         if (oq->desc_ring)
 235                 dma_free_coherent(oq->dev,
 236                                   oq->max_count * OCTEP_OQ_DESC_SIZE,
 237                                   oq->desc_ring, oq->desc_ring_dma);
 238
 239         vfree(oq);
 240         oct->oq[q_no] = NULL;
 241         oct->num_oqs--;
 242         return 0;
 243 }
 244
 245 /**
 246  * octep_setup_oqs() - setup resources for all Rx queues.
 247  *
 248  * @oct: Octeon device private data structure.
 249  */
 250 int octep_setup_oqs(struct octep_device *oct)
 251 {
 252         int i, retval = 0;
 253
 254         oct->num_oqs = 0;
 255         for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
 256                 retval = octep_setup_oq(oct, i);
 257                 if (retval) {
 258                         dev_err(&oct->pdev->dev,
 259                                 "Failed to setup OQ(RxQ)-%d.\n", i);
 260                         goto oq_setup_err;
 261                 }
 262                 dev_dbg(&oct->pdev->dev, "Successfully setup OQ(RxQ)-%d.\n", i);
 263         }
 264
 265         return 0;
 266
 267 oq_setup_err:
 268         while (i) {
 269                 i--;
 270                 octep_free_oq(oct->oq[i]);
 271         }
 272         return -1;
 273 }
 274
 275 /**
 276  * octep_oq_dbell_init() - Initialize Rx queue doorbell.
 277  *
 278  * @oct: Octeon device private data structure.
 279  *
 280  * Write number of descriptors to Rx queue doorbell register.
 281  */
 282 void octep_oq_dbell_init(struct octep_device *oct)
 283 {
 284         int i;
 285
 286         for (i = 0; i < oct->num_oqs; i++)
 287                 writel(oct->oq[i]->max_count, oct->oq[i]->pkts_credit_reg);
 288 }
 289
 290 /**
 291  * octep_free_oqs() - Free resources of all Rx queues.
 292  *
 293  * @oct: Octeon device private data structure.
 294  */
 295 void octep_free_oqs(struct octep_device *oct)
 296 {
 297         int i;
 298
 299         for (i = 0; i < CFG_GET_PORTS_ACTIVE_IO_RINGS(oct->conf); i++) {
 300                 if (!oct->oq[i])
 301                         continue;
 302                 octep_free_oq(oct->oq[i]);
 303                 dev_dbg(&oct->pdev->dev,
 304                         "Successfully freed OQ(RxQ)-%d.\n", i);
 305         }
 306 }
 307
 308 /**
 309  * octep_oq_check_hw_for_pkts() - Check for new Rx packets.
 310  *
 311  * @oct: Octeon device private data structure.
 312  * @oq: Octeon Rx queue data structure.
 313  *
 314  * Return: packets received after previous check.
 315  */
 316 static int octep_oq_check_hw_for_pkts(struct octep_device *oct,
 317                                       struct octep_oq *oq)
 318 {
 319         u32 pkt_count, new_pkts;
 320
 321         pkt_count = readl(oq->pkts_sent_reg);
 322         new_pkts = pkt_count - oq->last_pkt_count;
 323
 324         /* Clear the hardware packets counter register if the rx queue is
 325          * being processed continuously with-in a single interrupt and
 326          * reached half its max value.
 327          * this counter is not cleared every time read, to save write cycles.
 328          */
 329         if (unlikely(pkt_count > 0xF0000000U)) {
 330                 writel(pkt_count, oq->pkts_sent_reg);
 331                 pkt_count = readl(oq->pkts_sent_reg);
 332                 new_pkts += pkt_count;
 333         }
 334         oq->last_pkt_count = pkt_count;
 335         oq->pkts_pending += new_pkts;
 336         return new_pkts;
 337 }
 338
 339 /**
 340  * __octep_oq_process_rx() - Process hardware Rx queue and push to stack.
 341  *
 342  * @oct: Octeon device private data structure.
 343  * @oq: Octeon Rx queue data structure.
 344  * @pkts_to_process: number of packets to be processed.
 345  *
 346  * Process the new packets in Rx queue.
 347  * Packets larger than single Rx buffer arrive in consecutive descriptors.
 348  * But, count returned by the API only accounts full packets, not fragments.
 349  *
 350  * Return: number of packets processed and pushed to stack.
 351  */
 352 static int __octep_oq_process_rx(struct octep_device *oct,
 353                                  struct octep_oq *oq, u16 pkts_to_process)
 354 {
 355         struct octep_oq_resp_hw_ext *resp_hw_ext = NULL;
 356         netdev_features_t feat = oq->netdev->features;
 357         struct octep_rx_buffer *buff_info;
 358         struct octep_oq_resp_hw *resp_hw;
 359         u32 pkt, rx_bytes, desc_used;
 360         struct sk_buff *skb;
 361         u16 data_offset;
 362         u16 rx_ol_flags;
 363         u32 read_idx;
 364
 365         read_idx = oq->host_read_idx;
 366         rx_bytes = 0;
 367         desc_used = 0;
 368         for (pkt = 0; pkt < pkts_to_process; pkt++) {
 369                 buff_info = (struct octep_rx_buffer *)&oq->buff_info[read_idx];
 370                 dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr,
 371                                PAGE_SIZE, DMA_FROM_DEVICE);
 372                 resp_hw = page_address(buff_info->page);
 373                 buff_info->page = NULL;
 374
 375                 /* Swap the length field that is in Big-Endian to CPU */
 376                 buff_info->len = be64_to_cpu(resp_hw->length);
 377                 if (oct->conf->fw_info.rx_ol_flags) {
 378                         /* Extended response header is immediately after
 379                          * response header (resp_hw)
 380                          */
 381                         resp_hw_ext = (struct octep_oq_resp_hw_ext *)
 382                                       (resp_hw + 1);
 383                         buff_info->len -= OCTEP_OQ_RESP_HW_EXT_SIZE;
 384                         /* Packet Data is immediately after
 385                          * extended response header.
 386                          */
 387                         data_offset = OCTEP_OQ_RESP_HW_SIZE +
 388                                       OCTEP_OQ_RESP_HW_EXT_SIZE;
 389                         rx_ol_flags = resp_hw_ext->rx_ol_flags;
 390                 } else {
 391                         /* Data is immediately after
 392                          * Hardware Rx response header.
 393                          */
 394                         data_offset = OCTEP_OQ_RESP_HW_SIZE;
 395                         rx_ol_flags = 0;
 396                 }
 397                 rx_bytes += buff_info->len;
 398
 399                 if (buff_info->len <= oq->max_single_buffer_size) {
 400                         skb = build_skb((void *)resp_hw, PAGE_SIZE);
 401                         skb_reserve(skb, data_offset);
 402                         skb_put(skb, buff_info->len);
 403                         read_idx++;
 404                         desc_used++;
 405                         if (read_idx == oq->max_count)
 406                                 read_idx = 0;
 407                 } else {
 408                         struct skb_shared_info *shinfo;
 409                         u16 data_len;
 410
 411                         skb = build_skb((void *)resp_hw, PAGE_SIZE);
 412                         skb_reserve(skb, data_offset);
 413                         /* Head fragment includes response header(s);
 414                          * subsequent fragments contains only data.
 415                          */
 416                         skb_put(skb, oq->max_single_buffer_size);
 417                         read_idx++;
 418                         desc_used++;
 419                         if (read_idx == oq->max_count)
 420                                 read_idx = 0;
 421
 422                         shinfo = skb_shinfo(skb);
 423                         data_len = buff_info->len - oq->max_single_buffer_size;
 424                         while (data_len) {
 425                                 dma_unmap_page(oq->dev, oq->desc_ring[read_idx].buffer_ptr,
 426                                                PAGE_SIZE, DMA_FROM_DEVICE);
 427                                 buff_info = (struct octep_rx_buffer *)
 428                                             &oq->buff_info[read_idx];
 429                                 if (data_len < oq->buffer_size) {
 430                                         buff_info->len = data_len;
 431                                         data_len = 0;
 432                                 } else {
 433                                         buff_info->len = oq->buffer_size;
 434                                         data_len -= oq->buffer_size;
 435                                 }
 436
 437                                 skb_add_rx_frag(skb, shinfo->nr_frags,
 438                                                 buff_info->page, 0,
 439                                                 buff_info->len,
 440                                                 buff_info->len);
 441                                 buff_info->page = NULL;
 442                                 read_idx++;
 443                                 desc_used++;
 444                                 if (read_idx == oq->max_count)
 445                                         read_idx = 0;
 446                         }
 447                 }
 448
 449                 skb->dev = oq->netdev;
 450                 skb->protocol =  eth_type_trans(skb, skb->dev);
 451                 if (feat & NETIF_F_RXCSUM &&
 452                     OCTEP_RX_CSUM_VERIFIED(rx_ol_flags))
 453                         skb->ip_summed = CHECKSUM_UNNECESSARY;
 454                 else
 455                         skb->ip_summed = CHECKSUM_NONE;
 456                 napi_gro_receive(oq->napi, skb);
 457         }
 458
 459         oq->host_read_idx = read_idx;
 460         oq->refill_count += desc_used;
 461         oq->stats.packets += pkt;
 462         oq->stats.bytes += rx_bytes;
 463
 464         return pkt;
 465 }
 466
 467 /**
 468  * octep_oq_process_rx() - Process Rx queue.
 469  *
 470  * @oq: Octeon Rx queue data structure.
 471  * @budget: max number of packets can be processed in one invocation.
 472  *
 473  * Check for newly received packets and process them.
 474  * Keeps checking for new packets until budget is used or no new packets seen.
 475  *
 476  * Return: number of packets processed.
 477  */
 478 int octep_oq_process_rx(struct octep_oq *oq, int budget)
 479 {
 480         u32 pkts_available, pkts_processed, total_pkts_processed;
 481         struct octep_device *oct = oq->octep_dev;
 482
 483         pkts_available = 0;
 484         pkts_processed = 0;
 485         total_pkts_processed = 0;
 486         while (total_pkts_processed < budget) {
 487                  /* update pending count only when current one exhausted */
 488                 if (oq->pkts_pending == 0)
 489                         octep_oq_check_hw_for_pkts(oct, oq);
 490                 pkts_available = min(budget - total_pkts_processed,
 491                                      oq->pkts_pending);
 492                 if (!pkts_available)
 493                         break;
 494
 495                 pkts_processed = __octep_oq_process_rx(oct, oq,
 496                                                        pkts_available);
 497                 oq->pkts_pending -= pkts_processed;
 498                 total_pkts_processed += pkts_processed;
 499         }
 500
 501         if (oq->refill_count >= oq->refill_threshold) {
 502                 u32 desc_refilled = octep_oq_refill(oct, oq);
 503
 504                 /* flush pending writes before updating credits */
 505                 wmb();
 506                 writel(desc_refilled, oq->pkts_credit_reg);
 507         }
 508
 509         return total_pkts_processed;
 510 }