drivers/net/ethernet/cavium/liquidio/octeon_droq.c

   1 /**********************************************************************
   2  * Author: Cavium, Inc.
   3  *
   4  * Contact: support@cavium.com
   5  *          Please include "LiquidIO" in the subject.
   6  *
   7  * Copyright (c) 2003-2016 Cavium, Inc.
   8  *
   9  * This file is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License, Version 2, as
  11  * published by the Free Software Foundation.
  12  *
  13  * This file is distributed in the hope that it will be useful, but
  14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
  15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
  16  * NONINFRINGEMENT.  See the GNU General Public License for more details.
  17  ***********************************************************************/
  18 #include <linux/pci.h>
  19 #include <linux/netdevice.h>
  20 #include <linux/vmalloc.h>
  21 #include "liquidio_common.h"
  22 #include "octeon_droq.h"
  23 #include "octeon_iq.h"
  24 #include "response_manager.h"
  25 #include "octeon_device.h"
  26 #include "octeon_main.h"
  27 #include "octeon_network.h"
  28 #include "cn66xx_regs.h"
  29 #include "cn66xx_device.h"
  30 #include "cn23xx_pf_device.h"
  31 #include "cn23xx_vf_device.h"
  32
  33 struct niclist {
  34         struct list_head list;
  35         void *ptr;
  36 };
  37
  38 struct __dispatch {
  39         struct list_head list;
  40         struct octeon_recv_info *rinfo;
  41         octeon_dispatch_fn_t disp_fn;
  42 };
  43
  44 /** Get the argument that the user set when registering dispatch
  45  *  function for a given opcode/subcode.
  46  *  @param  octeon_dev - the octeon device pointer.
  47  *  @param  opcode     - the opcode for which the dispatch argument
  48  *                       is to be checked.
  49  *  @param  subcode    - the subcode for which the dispatch argument
  50  *                       is to be checked.
  51  *  @return  Success: void * (argument to the dispatch function)
  52  *  @return  Failure: NULL
  53  *
  54  */
  55 void *octeon_get_dispatch_arg(struct octeon_device *octeon_dev,
  56                               u16 opcode, u16 subcode)
  57 {
  58         int idx;
  59         struct list_head *dispatch;
  60         void *fn_arg = NULL;
  61         u16 combined_opcode = OPCODE_SUBCODE(opcode, subcode);
  62
  63         idx = combined_opcode & OCTEON_OPCODE_MASK;
  64
  65         spin_lock_bh(&octeon_dev->dispatch.lock);
  66
  67         if (octeon_dev->dispatch.count == 0) {
  68                 spin_unlock_bh(&octeon_dev->dispatch.lock);
  69                 return NULL;
  70         }
  71
  72         if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) {
  73                 fn_arg = octeon_dev->dispatch.dlist[idx].arg;
  74         } else {
  75                 list_for_each(dispatch,
  76                               &octeon_dev->dispatch.dlist[idx].list) {
  77                         if (((struct octeon_dispatch *)dispatch)->opcode ==
  78                             combined_opcode) {
  79                                 fn_arg = ((struct octeon_dispatch *)
  80                                           dispatch)->arg;
  81                                 break;
  82                         }
  83                 }
  84         }
  85
  86         spin_unlock_bh(&octeon_dev->dispatch.lock);
  87         return fn_arg;
  88 }
  89
  90 /** Check for packets on Droq. This function should be called with lock held.
  91  *  @param  droq - Droq on which count is checked.
  92  *  @return Returns packet count.
  93  */
  94 u32 octeon_droq_check_hw_for_pkts(struct octeon_droq *droq)
  95 {
  96         u32 pkt_count = 0;
  97         u32 last_count;
  98
  99         pkt_count = readl(droq->pkts_sent_reg);
 100
 101         last_count = pkt_count - droq->pkt_count;
 102         droq->pkt_count = pkt_count;
 103
 104         /* we shall write to cnts  at napi irq enable or end of droq tasklet */
 105         if (last_count)
 106                 atomic_add(last_count, &droq->pkts_pending);
 107
 108         return last_count;
 109 }
 110 EXPORT_SYMBOL_GPL(octeon_droq_check_hw_for_pkts);
 111
 112 static void octeon_droq_compute_max_packet_bufs(struct octeon_droq *droq)
 113 {
 114         u32 count = 0;
 115
 116         /* max_empty_descs is the max. no. of descs that can have no buffers.
 117          * If the empty desc count goes beyond this value, we cannot safely
 118          * read in a 64K packet sent by Octeon
 119          * (64K is max pkt size from Octeon)
 120          */
 121         droq->max_empty_descs = 0;
 122
 123         do {
 124                 droq->max_empty_descs++;
 125                 count += droq->buffer_size;
 126         } while (count < (64 * 1024));
 127
 128         droq->max_empty_descs = droq->max_count - droq->max_empty_descs;
 129 }
 130
 131 static void octeon_droq_reset_indices(struct octeon_droq *droq)
 132 {
 133         droq->read_idx = 0;
 134         droq->write_idx = 0;
 135         droq->refill_idx = 0;
 136         droq->refill_count = 0;
 137         atomic_set(&droq->pkts_pending, 0);
 138 }
 139
 140 static void
 141 octeon_droq_destroy_ring_buffers(struct octeon_device *oct,
 142                                  struct octeon_droq *droq)
 143 {
 144         u32 i;
 145         struct octeon_skb_page_info *pg_info;
 146
 147         for (i = 0; i < droq->max_count; i++) {
 148                 pg_info = &droq->recv_buf_list[i].pg_info;
 149                 if (!pg_info)
 150                         continue;
 151
 152                 if (pg_info->dma)
 153                         lio_unmap_ring(oct->pci_dev,
 154                                        (u64)pg_info->dma);
 155                 pg_info->dma = 0;
 156
 157                 if (pg_info->page)
 158                         recv_buffer_destroy(droq->recv_buf_list[i].buffer,
 159                                             pg_info);
 160
 161                 droq->recv_buf_list[i].buffer = NULL;
 162         }
 163
 164         octeon_droq_reset_indices(droq);
 165 }
 166
 167 static int
 168 octeon_droq_setup_ring_buffers(struct octeon_device *oct,
 169                                struct octeon_droq *droq)
 170 {
 171         u32 i;
 172         void *buf;
 173         struct octeon_droq_desc *desc_ring = droq->desc_ring;
 174
 175         for (i = 0; i < droq->max_count; i++) {
 176                 buf = recv_buffer_alloc(oct, &droq->recv_buf_list[i].pg_info);
 177
 178                 if (!buf) {
 179                         dev_err(&oct->pci_dev->dev, "%s buffer alloc failed\n",
 180                                 __func__);
 181                         droq->stats.rx_alloc_failure++;
 182                         return -ENOMEM;
 183                 }
 184
 185                 droq->recv_buf_list[i].buffer = buf;
 186                 droq->recv_buf_list[i].data = get_rbd(buf);
 187                 desc_ring[i].info_ptr = 0;
 188                 desc_ring[i].buffer_ptr =
 189                         lio_map_ring(droq->recv_buf_list[i].buffer);
 190         }
 191
 192         octeon_droq_reset_indices(droq);
 193
 194         octeon_droq_compute_max_packet_bufs(droq);
 195
 196         return 0;
 197 }
 198
 199 int octeon_delete_droq(struct octeon_device *oct, u32 q_no)
 200 {
 201         struct octeon_droq *droq = oct->droq[q_no];
 202
 203         dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
 204
 205         octeon_droq_destroy_ring_buffers(oct, droq);
 206         vfree(droq->recv_buf_list);
 207
 208         if (droq->desc_ring)
 209                 lio_dma_free(oct, (droq->max_count * OCT_DROQ_DESC_SIZE),
 210                              droq->desc_ring, droq->desc_ring_dma);
 211
 212         memset(droq, 0, OCT_DROQ_SIZE);
 213         oct->io_qmask.oq &= ~(1ULL << q_no);
 214         vfree(oct->droq[q_no]);
 215         oct->droq[q_no] = NULL;
 216         oct->num_oqs--;
 217
 218         return 0;
 219 }
 220 EXPORT_SYMBOL_GPL(octeon_delete_droq);
 221
 222 int octeon_init_droq(struct octeon_device *oct,
 223                      u32 q_no,
 224                      u32 num_descs,
 225                      u32 desc_size,
 226                      void *app_ctx)
 227 {
 228         struct octeon_droq *droq;
 229         u32 desc_ring_size = 0, c_num_descs = 0, c_buf_size = 0;
 230         u32 c_pkts_per_intr = 0, c_refill_threshold = 0;
 231         int numa_node = dev_to_node(&oct->pci_dev->dev);
 232
 233         dev_dbg(&oct->pci_dev->dev, "%s[%d]\n", __func__, q_no);
 234
 235         droq = oct->droq[q_no];
 236         memset(droq, 0, OCT_DROQ_SIZE);
 237
 238         droq->oct_dev = oct;
 239         droq->q_no = q_no;
 240         if (app_ctx)
 241                 droq->app_ctx = app_ctx;
 242         else
 243                 droq->app_ctx = (void *)(size_t)q_no;
 244
 245         c_num_descs = num_descs;
 246         c_buf_size = desc_size;
 247         if (OCTEON_CN6XXX(oct)) {
 248                 struct octeon_config *conf6x = CHIP_CONF(oct, cn6xxx);
 249
 250                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf6x);
 251                 c_refill_threshold =
 252                         (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf6x);
 253         } else if (OCTEON_CN23XX_PF(oct)) {
 254                 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_pf);
 255
 256                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
 257                 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
 258         } else if (OCTEON_CN23XX_VF(oct)) {
 259                 struct octeon_config *conf23 = CHIP_CONF(oct, cn23xx_vf);
 260
 261                 c_pkts_per_intr = (u32)CFG_GET_OQ_PKTS_PER_INTR(conf23);
 262                 c_refill_threshold = (u32)CFG_GET_OQ_REFILL_THRESHOLD(conf23);
 263         } else {
 264                 return 1;
 265         }
 266
 267         droq->max_count = c_num_descs;
 268         droq->buffer_size = c_buf_size;
 269
 270         desc_ring_size = droq->max_count * OCT_DROQ_DESC_SIZE;
 271         droq->desc_ring = lio_dma_alloc(oct, desc_ring_size,
 272                                         (dma_addr_t *)&droq->desc_ring_dma);
 273
 274         if (!droq->desc_ring) {
 275                 dev_err(&oct->pci_dev->dev,
 276                         "Output queue %d ring alloc failed\n", q_no);
 277                 return 1;
 278         }
 279
 280         dev_dbg(&oct->pci_dev->dev, "droq[%d]: desc_ring: virt: 0x%p, dma: %lx\n",
 281                 q_no, droq->desc_ring, droq->desc_ring_dma);
 282         dev_dbg(&oct->pci_dev->dev, "droq[%d]: num_desc: %d\n", q_no,
 283                 droq->max_count);
 284
 285         droq->recv_buf_list = vzalloc_node(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE),
 286                                            numa_node);
 287         if (!droq->recv_buf_list)
 288                 droq->recv_buf_list = vzalloc(array_size(droq->max_count, OCT_DROQ_RECVBUF_SIZE));
 289         if (!droq->recv_buf_list) {
 290                 dev_err(&oct->pci_dev->dev, "Output queue recv buf list alloc failed\n");
 291                 goto init_droq_fail;
 292         }
 293
 294         if (octeon_droq_setup_ring_buffers(oct, droq))
 295                 goto init_droq_fail;
 296
 297         droq->pkts_per_intr = c_pkts_per_intr;
 298         droq->refill_threshold = c_refill_threshold;
 299
 300         dev_dbg(&oct->pci_dev->dev, "DROQ INIT: max_empty_descs: %d\n",
 301                 droq->max_empty_descs);
 302
 303         INIT_LIST_HEAD(&droq->dispatch_list);
 304
 305         /* For 56xx Pass1, this function won't be called, so no checks. */
 306         oct->fn_list.setup_oq_regs(oct, q_no);
 307
 308         oct->io_qmask.oq |= BIT_ULL(q_no);
 309
 310         return 0;
 311
 312 init_droq_fail:
 313         octeon_delete_droq(oct, q_no);
 314         return 1;
 315 }
 316
 317 /* octeon_create_recv_info
 318  * Parameters:
 319  *  octeon_dev - pointer to the octeon device structure
 320  *  droq       - droq in which the packet arrived.
 321  *  buf_cnt    - no. of buffers used by the packet.
 322  *  idx        - index in the descriptor for the first buffer in the packet.
 323  * Description:
 324  *  Allocates a recv_info_t and copies the buffer addresses for packet data
 325  *  into the recv_pkt space which starts at an 8B offset from recv_info_t.
 326  *  Flags the descriptors for refill later. If available descriptors go
 327  *  below the threshold to receive a 64K pkt, new buffers are first allocated
 328  *  before the recv_pkt_t is created.
 329  *  This routine will be called in interrupt context.
 330  * Returns:
 331  *  Success: Pointer to recv_info_t
 332  *  Failure: NULL.
 333  */
 334 static inline struct octeon_recv_info *octeon_create_recv_info(
 335                 struct octeon_device *octeon_dev,
 336                 struct octeon_droq *droq,
 337                 u32 buf_cnt,
 338                 u32 idx)
 339 {
 340         struct octeon_droq_info *info;
 341         struct octeon_recv_pkt *recv_pkt;
 342         struct octeon_recv_info *recv_info;
 343         u32 i, bytes_left;
 344         struct octeon_skb_page_info *pg_info;
 345
 346         info = (struct octeon_droq_info *)droq->recv_buf_list[idx].data;
 347
 348         recv_info = octeon_alloc_recv_info(sizeof(struct __dispatch));
 349         if (!recv_info)
 350                 return NULL;
 351
 352         recv_pkt = recv_info->recv_pkt;
 353         recv_pkt->rh = info->rh;
 354         recv_pkt->length = (u32)info->length;
 355         recv_pkt->buffer_count = (u16)buf_cnt;
 356         recv_pkt->octeon_id = (u16)octeon_dev->octeon_id;
 357
 358         i = 0;
 359         bytes_left = (u32)info->length;
 360
 361         while (buf_cnt) {
 362                 {
 363                         pg_info = &droq->recv_buf_list[idx].pg_info;
 364
 365                         lio_unmap_ring(octeon_dev->pci_dev,
 366                                        (u64)pg_info->dma);
 367                         pg_info->page = NULL;
 368                         pg_info->dma = 0;
 369                 }
 370
 371                 recv_pkt->buffer_size[i] =
 372                         (bytes_left >=
 373                          droq->buffer_size) ? droq->buffer_size : bytes_left;
 374
 375                 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer;
 376                 droq->recv_buf_list[idx].buffer = NULL;
 377
 378                 idx = incr_index(idx, 1, droq->max_count);
 379                 bytes_left -= droq->buffer_size;
 380                 i++;
 381                 buf_cnt--;
 382         }
 383
 384         return recv_info;
 385 }
 386
 387 /* If we were not able to refill all buffers, try to move around
 388  * the buffers that were not dispatched.
 389  */
 390 static inline u32
 391 octeon_droq_refill_pullup_descs(struct octeon_droq *droq,
 392                                 struct octeon_droq_desc *desc_ring)
 393 {
 394         u32 desc_refilled = 0;
 395
 396         u32 refill_index = droq->refill_idx;
 397
 398         while (refill_index != droq->read_idx) {
 399                 if (droq->recv_buf_list[refill_index].buffer) {
 400                         droq->recv_buf_list[droq->refill_idx].buffer =
 401                                 droq->recv_buf_list[refill_index].buffer;
 402                         droq->recv_buf_list[droq->refill_idx].data =
 403                                 droq->recv_buf_list[refill_index].data;
 404                         desc_ring[droq->refill_idx].buffer_ptr =
 405                                 desc_ring[refill_index].buffer_ptr;
 406                         droq->recv_buf_list[refill_index].buffer = NULL;
 407                         desc_ring[refill_index].buffer_ptr = 0;
 408                         do {
 409                                 droq->refill_idx = incr_index(droq->refill_idx,
 410                                                               1,
 411                                                               droq->max_count);
 412                                 desc_refilled++;
 413                                 droq->refill_count--;
 414                         } while (droq->recv_buf_list[droq->refill_idx].buffer);
 415                 }
 416                 refill_index = incr_index(refill_index, 1, droq->max_count);
 417         }                       /* while */
 418         return desc_refilled;
 419 }
 420
 421 /* octeon_droq_refill
 422  * Parameters:
 423  *  droq       - droq in which descriptors require new buffers.
 424  * Description:
 425  *  Called during normal DROQ processing in interrupt mode or by the poll
 426  *  thread to refill the descriptors from which buffers were dispatched
 427  *  to upper layers. Attempts to allocate new buffers. If that fails, moves
 428  *  up buffers (that were not dispatched) to form a contiguous ring.
 429  * Returns:
 430  *  No of descriptors refilled.
 431  */
 432 static u32
 433 octeon_droq_refill(struct octeon_device *octeon_dev, struct octeon_droq *droq)
 434 {
 435         struct octeon_droq_desc *desc_ring;
 436         void *buf = NULL;
 437         u8 *data;
 438         u32 desc_refilled = 0;
 439         struct octeon_skb_page_info *pg_info;
 440
 441         desc_ring = droq->desc_ring;
 442
 443         while (droq->refill_count && (desc_refilled < droq->max_count)) {
 444                 /* If a valid buffer exists (happens if there is no dispatch),
 445                  * reuse the buffer, else allocate.
 446                  */
 447                 if (!droq->recv_buf_list[droq->refill_idx].buffer) {
 448                         pg_info =
 449                                 &droq->recv_buf_list[droq->refill_idx].pg_info;
 450                         /* Either recycle the existing pages or go for
 451                          * new page alloc
 452                          */
 453                         if (pg_info->page)
 454                                 buf = recv_buffer_reuse(octeon_dev, pg_info);
 455                         else
 456                                 buf = recv_buffer_alloc(octeon_dev, pg_info);
 457                         /* If a buffer could not be allocated, no point in
 458                          * continuing
 459                          */
 460                         if (!buf) {
 461                                 droq->stats.rx_alloc_failure++;
 462                                 break;
 463                         }
 464                         droq->recv_buf_list[droq->refill_idx].buffer =
 465                                 buf;
 466                         data = get_rbd(buf);
 467                 } else {
 468                         data = get_rbd(droq->recv_buf_list
 469                                        [droq->refill_idx].buffer);
 470                 }
 471
 472                 droq->recv_buf_list[droq->refill_idx].data = data;
 473
 474                 desc_ring[droq->refill_idx].buffer_ptr =
 475                         lio_map_ring(droq->recv_buf_list[
 476                                      droq->refill_idx].buffer);
 477
 478                 droq->refill_idx = incr_index(droq->refill_idx, 1,
 479                                               droq->max_count);
 480                 desc_refilled++;
 481                 droq->refill_count--;
 482         }
 483
 484         if (droq->refill_count)
 485                 desc_refilled +=
 486                         octeon_droq_refill_pullup_descs(droq, desc_ring);
 487
 488         /* if droq->refill_count
 489          * The refill count would not change in pass two. We only moved buffers
 490          * to close the gap in the ring, but we would still have the same no. of
 491          * buffers to refill.
 492          */
 493         return desc_refilled;
 494 }
 495
 496 /** check if we can allocate packets to get out of oom.
 497  *  @param  droq - Droq being checked.
 498  *  @return 1 if fails to refill minimum
 499  */
 500 int octeon_retry_droq_refill(struct octeon_droq *droq)
 501 {
 502         struct octeon_device *oct = droq->oct_dev;
 503         int desc_refilled, reschedule = 1;
 504         u32 pkts_credit;
 505
 506         pkts_credit = readl(droq->pkts_credit_reg);
 507         desc_refilled = octeon_droq_refill(oct, droq);
 508         if (desc_refilled) {
 509                 /* Flush the droq descriptor data to memory to be sure
 510                  * that when we update the credits the data in memory
 511                  * is accurate.
 512                  */
 513                 wmb();
 514                 writel(desc_refilled, droq->pkts_credit_reg);
 515
 516                 if (pkts_credit + desc_refilled >= CN23XX_SLI_DEF_BP)
 517                         reschedule = 0;
 518         }
 519
 520         return reschedule;
 521 }
 522
 523 static inline u32
 524 octeon_droq_get_bufcount(u32 buf_size, u32 total_len)
 525 {
 526         return DIV_ROUND_UP(total_len, buf_size);
 527 }
 528
 529 static int
 530 octeon_droq_dispatch_pkt(struct octeon_device *oct,
 531                          struct octeon_droq *droq,
 532                          union octeon_rh *rh,
 533                          struct octeon_droq_info *info)
 534 {
 535         u32 cnt;
 536         octeon_dispatch_fn_t disp_fn;
 537         struct octeon_recv_info *rinfo;
 538
 539         cnt = octeon_droq_get_bufcount(droq->buffer_size, (u32)info->length);
 540
 541         disp_fn = octeon_get_dispatch(oct, (u16)rh->r.opcode,
 542                                       (u16)rh->r.subcode);
 543         if (disp_fn) {
 544                 rinfo = octeon_create_recv_info(oct, droq, cnt, droq->read_idx);
 545                 if (rinfo) {
 546                         struct __dispatch *rdisp = rinfo->rsvd;
 547
 548                         rdisp->rinfo = rinfo;
 549                         rdisp->disp_fn = disp_fn;
 550                         rinfo->recv_pkt->rh = *rh;
 551                         list_add_tail(&rdisp->list,
 552                                       &droq->dispatch_list);
 553                 } else {
 554                         droq->stats.dropped_nomem++;
 555                 }
 556         } else {
 557                 dev_err(&oct->pci_dev->dev, "DROQ: No dispatch function (opcode %u/%u)\n",
 558                         (unsigned int)rh->r.opcode,
 559                         (unsigned int)rh->r.subcode);
 560                 droq->stats.dropped_nodispatch++;
 561         }
 562
 563         return cnt;
 564 }
 565
 566 static inline void octeon_droq_drop_packets(struct octeon_device *oct,
 567                                             struct octeon_droq *droq,
 568                                             u32 cnt)
 569 {
 570         u32 i = 0, buf_cnt;
 571         struct octeon_droq_info *info;
 572
 573         for (i = 0; i < cnt; i++) {
 574                 info = (struct octeon_droq_info *)
 575                         droq->recv_buf_list[droq->read_idx].data;
 576                 octeon_swap_8B_data((u64 *)info, 2);
 577
 578                 if (info->length) {
 579                         info->length += OCTNET_FRM_LENGTH_SIZE;
 580                         droq->stats.bytes_received += info->length;
 581                         buf_cnt = octeon_droq_get_bufcount(droq->buffer_size,
 582                                                            (u32)info->length);
 583                 } else {
 584                         dev_err(&oct->pci_dev->dev, "DROQ: In drop: pkt with len 0\n");
 585                         buf_cnt = 1;
 586                 }
 587
 588                 droq->read_idx = incr_index(droq->read_idx, buf_cnt,
 589                                             droq->max_count);
 590                 droq->refill_count += buf_cnt;
 591         }
 592 }
 593
 594 static u32
 595 octeon_droq_fast_process_packets(struct octeon_device *oct,
 596                                  struct octeon_droq *droq,
 597                                  u32 pkts_to_process)
 598 {
 599         u32 pkt, total_len = 0, pkt_count, retval;
 600         struct octeon_droq_info *info;
 601         union octeon_rh *rh;
 602
 603         pkt_count = pkts_to_process;
 604
 605         for (pkt = 0; pkt < pkt_count; pkt++) {
 606                 u32 pkt_len = 0;
 607                 struct sk_buff *nicbuf = NULL;
 608                 struct octeon_skb_page_info *pg_info;
 609                 void *buf;
 610
 611                 info = (struct octeon_droq_info *)
 612                         droq->recv_buf_list[droq->read_idx].data;
 613                 octeon_swap_8B_data((u64 *)info, 2);
 614
 615                 if (!info->length) {
 616                         dev_err(&oct->pci_dev->dev,
 617                                 "DROQ[%d] idx: %d len:0, pkt_cnt: %d\n",
 618                                 droq->q_no, droq->read_idx, pkt_count);
 619                         print_hex_dump_bytes("", DUMP_PREFIX_ADDRESS,
 620                                              (u8 *)info,
 621                                              OCT_DROQ_INFO_SIZE);
 622                         break;
 623                 }
 624
 625                 /* Len of resp hdr in included in the received data len. */
 626                 rh = &info->rh;
 627
 628                 info->length += OCTNET_FRM_LENGTH_SIZE;
 629                 rh->r_dh.len += (ROUNDUP8(OCT_DROQ_INFO_SIZE) / sizeof(u64));
 630                 total_len += (u32)info->length;
 631                 if (opcode_slow_path(rh)) {
 632                         u32 buf_cnt;
 633
 634                         buf_cnt = octeon_droq_dispatch_pkt(oct, droq, rh, info);
 635                         droq->read_idx = incr_index(droq->read_idx,
 636                                                     buf_cnt, droq->max_count);
 637                         droq->refill_count += buf_cnt;
 638                 } else {
 639                         if (info->length <= droq->buffer_size) {
 640                                 pkt_len = (u32)info->length;
 641                                 nicbuf = droq->recv_buf_list[
 642                                         droq->read_idx].buffer;
 643                                 pg_info = &droq->recv_buf_list[
 644                                         droq->read_idx].pg_info;
 645                                 if (recv_buffer_recycle(oct, pg_info))
 646                                         pg_info->page = NULL;
 647                                 droq->recv_buf_list[droq->read_idx].buffer =
 648                                         NULL;
 649
 650                                 droq->read_idx = incr_index(droq->read_idx, 1,
 651                                                             droq->max_count);
 652                                 droq->refill_count++;
 653                         } else {
 654                                 nicbuf = octeon_fast_packet_alloc((u32)
 655                                                                   info->length);
 656                                 pkt_len = 0;
 657                                 /* nicbuf allocation can fail. We'll handle it
 658                                  * inside the loop.
 659                                  */
 660                                 while (pkt_len < info->length) {
 661                                         int cpy_len, idx = droq->read_idx;
 662
 663                                         cpy_len = ((pkt_len + droq->buffer_size)
 664                                                    > info->length) ?
 665                                                 ((u32)info->length - pkt_len) :
 666                                                 droq->buffer_size;
 667
 668                                         if (nicbuf) {
 669                                                 octeon_fast_packet_next(droq,
 670                                                                         nicbuf,
 671                                                                         cpy_len,
 672                                                                         idx);
 673                                                 buf = droq->recv_buf_list[
 674                                                         idx].buffer;
 675                                                 recv_buffer_fast_free(buf);
 676                                                 droq->recv_buf_list[idx].buffer
 677                                                         = NULL;
 678                                         } else {
 679                                                 droq->stats.rx_alloc_failure++;
 680                                         }
 681
 682                                         pkt_len += cpy_len;
 683                                         droq->read_idx =
 684                                                 incr_index(droq->read_idx, 1,
 685                                                            droq->max_count);
 686                                         droq->refill_count++;
 687                                 }
 688                         }
 689
 690                         if (nicbuf) {
 691                                 if (droq->ops.fptr) {
 692                                         droq->ops.fptr(oct->octeon_id,
 693                                                        nicbuf, pkt_len,
 694                                                        rh, &droq->napi,
 695                                                        droq->ops.farg);
 696                                 } else {
 697                                         recv_buffer_free(nicbuf);
 698                                 }
 699                         }
 700                 }
 701
 702                 if (droq->refill_count >= droq->refill_threshold) {
 703                         int desc_refilled = octeon_droq_refill(oct, droq);
 704
 705                         if (desc_refilled) {
 706                                 /* Flush the droq descriptor data to memory to
 707                                  * be sure that when we update the credits the
 708                                  * data in memory is accurate.
 709                                  */
 710                                 wmb();
 711                                 writel(desc_refilled, droq->pkts_credit_reg);
 712                         }
 713                 }
 714         }                       /* for (each packet)... */
 715
 716         /* Increment refill_count by the number of buffers processed. */
 717         droq->stats.pkts_received += pkt;
 718         droq->stats.bytes_received += total_len;
 719
 720         retval = pkt;
 721         if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) {
 722                 octeon_droq_drop_packets(oct, droq, (pkts_to_process - pkt));
 723
 724                 droq->stats.dropped_toomany += (pkts_to_process - pkt);
 725                 retval = pkts_to_process;
 726         }
 727
 728         atomic_sub(retval, &droq->pkts_pending);
 729
 730         if (droq->refill_count >= droq->refill_threshold &&
 731             readl(droq->pkts_credit_reg) < CN23XX_SLI_DEF_BP) {
 732                 octeon_droq_check_hw_for_pkts(droq);
 733
 734                 /* Make sure there are no pkts_pending */
 735                 if (!atomic_read(&droq->pkts_pending))
 736                         octeon_schedule_rxq_oom_work(oct, droq);
 737         }
 738
 739         return retval;
 740 }
 741
 742 int
 743 octeon_droq_process_packets(struct octeon_device *oct,
 744                             struct octeon_droq *droq,
 745                             u32 budget)
 746 {
 747         u32 pkt_count = 0;
 748         struct list_head *tmp, *tmp2;
 749
 750         octeon_droq_check_hw_for_pkts(droq);
 751         pkt_count = atomic_read(&droq->pkts_pending);
 752
 753         if (!pkt_count)
 754                 return 0;
 755
 756         if (pkt_count > budget)
 757                 pkt_count = budget;
 758
 759         octeon_droq_fast_process_packets(oct, droq, pkt_count);
 760
 761         list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
 762                 struct __dispatch *rdisp = (struct __dispatch *)tmp;
 763
 764                 list_del(tmp);
 765                 rdisp->disp_fn(rdisp->rinfo,
 766                                octeon_get_dispatch_arg
 767                                (oct,
 768                                 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
 769                                 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
 770         }
 771
 772         /* If there are packets pending. schedule tasklet again */
 773         if (atomic_read(&droq->pkts_pending))
 774                 return 1;
 775
 776         return 0;
 777 }
 778 EXPORT_SYMBOL_GPL(octeon_droq_process_packets);
 779
 780 /*
 781  * Utility function to poll for packets. check_hw_for_packets must be
 782  * called before calling this routine.
 783  */
 784
 785 int
 786 octeon_droq_process_poll_pkts(struct octeon_device *oct,
 787                               struct octeon_droq *droq, u32 budget)
 788 {
 789         struct list_head *tmp, *tmp2;
 790         u32 pkts_available = 0, pkts_processed = 0;
 791         u32 total_pkts_processed = 0;
 792
 793         if (budget > droq->max_count)
 794                 budget = droq->max_count;
 795
 796         while (total_pkts_processed < budget) {
 797                 octeon_droq_check_hw_for_pkts(droq);
 798
 799                 pkts_available = min((budget - total_pkts_processed),
 800                                      (u32)(atomic_read(&droq->pkts_pending)));
 801
 802                 if (pkts_available == 0)
 803                         break;
 804
 805                 pkts_processed =
 806                         octeon_droq_fast_process_packets(oct, droq,
 807                                                          pkts_available);
 808
 809                 total_pkts_processed += pkts_processed;
 810         }
 811
 812         list_for_each_safe(tmp, tmp2, &droq->dispatch_list) {
 813                 struct __dispatch *rdisp = (struct __dispatch *)tmp;
 814
 815                 list_del(tmp);
 816                 rdisp->disp_fn(rdisp->rinfo,
 817                                octeon_get_dispatch_arg
 818                                (oct,
 819                                 (u16)rdisp->rinfo->recv_pkt->rh.r.opcode,
 820                                 (u16)rdisp->rinfo->recv_pkt->rh.r.subcode));
 821         }
 822
 823         return total_pkts_processed;
 824 }
 825
 826 /* Enable Pkt Interrupt */
 827 int
 828 octeon_enable_irq(struct octeon_device *oct, u32 q_no)
 829 {
 830         switch (oct->chip_id) {
 831         case OCTEON_CN66XX:
 832         case OCTEON_CN68XX: {
 833                 struct octeon_cn6xxx *cn6xxx =
 834                         (struct octeon_cn6xxx *)oct->chip;
 835                 unsigned long flags;
 836                 u32 value;
 837
 838                 spin_lock_irqsave
 839                         (&cn6xxx->lock_for_droq_int_enb_reg, flags);
 840                 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB);
 841                 value |= (1 << q_no);
 842                 octeon_write_csr(oct, CN6XXX_SLI_PKT_TIME_INT_ENB, value);
 843                 value = octeon_read_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB);
 844                 value |= (1 << q_no);
 845                 octeon_write_csr(oct, CN6XXX_SLI_PKT_CNT_INT_ENB, value);
 846
 847                 /* don't bother flushing the enables */
 848
 849                 spin_unlock_irqrestore
 850                         (&cn6xxx->lock_for_droq_int_enb_reg, flags);
 851         }
 852                 break;
 853         case OCTEON_CN23XX_PF_VID:
 854                 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
 855                 break;
 856
 857         case OCTEON_CN23XX_VF_VID:
 858                 lio_enable_irq(oct->droq[q_no], oct->instr_queue[q_no]);
 859                 break;
 860         default:
 861                 dev_err(&oct->pci_dev->dev, "%s Unknown Chip\n", __func__);
 862                 return 1;
 863         }
 864
 865         return 0;
 866 }
 867
 868 int octeon_register_droq_ops(struct octeon_device *oct, u32 q_no,
 869                              struct octeon_droq_ops *ops)
 870 {
 871         struct octeon_config *oct_cfg = NULL;
 872         struct octeon_droq *droq;
 873
 874         oct_cfg = octeon_get_conf(oct);
 875
 876         if (!oct_cfg)
 877                 return -EINVAL;
 878
 879         if (!(ops)) {
 880                 dev_err(&oct->pci_dev->dev, "%s: droq_ops pointer is NULL\n",
 881                         __func__);
 882                 return -EINVAL;
 883         }
 884
 885         if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
 886                 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
 887                         __func__, q_no, (oct->num_oqs - 1));
 888                 return -EINVAL;
 889         }
 890
 891         droq = oct->droq[q_no];
 892         memcpy(&droq->ops, ops, sizeof(struct octeon_droq_ops));
 893
 894         return 0;
 895 }
 896
 897 int octeon_unregister_droq_ops(struct octeon_device *oct, u32 q_no)
 898 {
 899         struct octeon_config *oct_cfg = NULL;
 900         struct octeon_droq *droq;
 901
 902         oct_cfg = octeon_get_conf(oct);
 903
 904         if (!oct_cfg)
 905                 return -EINVAL;
 906
 907         if (q_no >= CFG_GET_OQ_MAX_Q(oct_cfg)) {
 908                 dev_err(&oct->pci_dev->dev, "%s: droq id (%d) exceeds MAX (%d)\n",
 909                         __func__, q_no, oct->num_oqs - 1);
 910                 return -EINVAL;
 911         }
 912
 913         droq = oct->droq[q_no];
 914
 915         if (!droq) {
 916                 dev_info(&oct->pci_dev->dev,
 917                          "Droq id (%d) not available.\n", q_no);
 918                 return 0;
 919         }
 920
 921         droq->ops.fptr = NULL;
 922         droq->ops.farg = NULL;
 923         droq->ops.drop_on_max = 0;
 924
 925         return 0;
 926 }
 927 EXPORT_SYMBOL_GPL(octeon_unregister_droq_ops);
 928
 929 int octeon_create_droq(struct octeon_device *oct,
 930                        u32 q_no, u32 num_descs,
 931                        u32 desc_size, void *app_ctx)
 932 {
 933         struct octeon_droq *droq;
 934         int numa_node = dev_to_node(&oct->pci_dev->dev);
 935
 936         if (oct->droq[q_no]) {
 937                 dev_dbg(&oct->pci_dev->dev, "Droq already in use. Cannot create droq %d again\n",
 938                         q_no);
 939                 return 1;
 940         }
 941
 942         /* Allocate the DS for the new droq. */
 943         droq = vmalloc_node(sizeof(*droq), numa_node);
 944         if (!droq)
 945                 droq = vmalloc(sizeof(*droq));
 946         if (!droq)
 947                 return -1;
 948
 949         memset(droq, 0, sizeof(struct octeon_droq));
 950
 951         /*Disable the pkt o/p for this Q  */
 952         octeon_set_droq_pkt_op(oct, q_no, 0);
 953         oct->droq[q_no] = droq;
 954
 955         /* Initialize the Droq */
 956         if (octeon_init_droq(oct, q_no, num_descs, desc_size, app_ctx)) {
 957                 vfree(oct->droq[q_no]);
 958                 oct->droq[q_no] = NULL;
 959                 return -1;
 960         }
 961
 962         oct->num_oqs++;
 963
 964         dev_dbg(&oct->pci_dev->dev, "%s: Total number of OQ: %d\n", __func__,
 965                 oct->num_oqs);
 966
 967         /* Global Droq register settings */
 968
 969         /* As of now not required, as setting are done for all 32 Droqs at
 970          * the same time.
 971          */
 972         return 0;
 973 }