drivers/net/ethernet/huawei/hinic/hinic_hw_qp.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Huawei HiNIC PCI Express Linux driver
   4  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   5  */
   6
   7 #include <linux/kernel.h>
   8 #include <linux/types.h>
   9 #include <linux/pci.h>
  10 #include <linux/device.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/vmalloc.h>
  13 #include <linux/errno.h>
  14 #include <linux/sizes.h>
  15 #include <linux/atomic.h>
  16 #include <linux/skbuff.h>
  17 #include <linux/io.h>
  18 #include <asm/barrier.h>
  19 #include <asm/byteorder.h>
  20
  21 #include "hinic_common.h"
  22 #include "hinic_hw_if.h"
  23 #include "hinic_hw_wqe.h"
  24 #include "hinic_hw_wq.h"
  25 #include "hinic_hw_qp_ctxt.h"
  26 #include "hinic_hw_qp.h"
  27 #include "hinic_hw_io.h"
  28
  29 #define SQ_DB_OFF               SZ_2K
  30
  31 /* The number of cache line to prefetch Until threshold state */
  32 #define WQ_PREFETCH_MAX         2
  33 /* The number of cache line to prefetch After threshold state */
  34 #define WQ_PREFETCH_MIN         1
  35 /* Threshold state */
  36 #define WQ_PREFETCH_THRESHOLD   256
  37
  38 /* sizes of the SQ/RQ ctxt */
  39 #define Q_CTXT_SIZE             48
  40 #define CTXT_RSVD               240
  41
  42 #define SQ_CTXT_OFFSET(max_sqs, max_rqs, q_id)  \
  43                 (((max_rqs) + (max_sqs)) * CTXT_RSVD + (q_id) * Q_CTXT_SIZE)
  44
  45 #define RQ_CTXT_OFFSET(max_sqs, max_rqs, q_id)  \
  46                 (((max_rqs) + (max_sqs)) * CTXT_RSVD + \
  47                  (max_sqs + (q_id)) * Q_CTXT_SIZE)
  48
  49 #define SIZE_16BYTES(size)              (ALIGN(size, 16) >> 4)
  50 #define SIZE_8BYTES(size)               (ALIGN(size, 8) >> 3)
  51 #define SECT_SIZE_FROM_8BYTES(size)     ((size) << 3)
  52
  53 #define SQ_DB_PI_HI_SHIFT       8
  54 #define SQ_DB_PI_HI(prod_idx)   ((prod_idx) >> SQ_DB_PI_HI_SHIFT)
  55
  56 #define SQ_DB_PI_LOW_MASK       0xFF
  57 #define SQ_DB_PI_LOW(prod_idx)  ((prod_idx) & SQ_DB_PI_LOW_MASK)
  58
  59 #define SQ_DB_ADDR(sq, pi)      ((u64 *)((sq)->db_base) + SQ_DB_PI_LOW(pi))
  60
  61 #define SQ_MASKED_IDX(sq, idx)  ((idx) & (sq)->wq->mask)
  62 #define RQ_MASKED_IDX(rq, idx)  ((idx) & (rq)->wq->mask)
  63
  64 enum sq_wqe_type {
  65         SQ_NORMAL_WQE = 0,
  66 };
  67
  68 enum rq_completion_fmt {
  69         RQ_COMPLETE_SGE = 1
  70 };
  71
  72 void hinic_qp_prepare_header(struct hinic_qp_ctxt_header *qp_ctxt_hdr,
  73                              enum hinic_qp_ctxt_type ctxt_type,
  74                              u16 num_queues, u16 max_queues)
  75 {
  76         u16 max_sqs = max_queues;
  77         u16 max_rqs = max_queues;
  78
  79         qp_ctxt_hdr->num_queues = num_queues;
  80         qp_ctxt_hdr->queue_type = ctxt_type;
  81
  82         if (ctxt_type == HINIC_QP_CTXT_TYPE_SQ)
  83                 qp_ctxt_hdr->addr_offset = SQ_CTXT_OFFSET(max_sqs, max_rqs, 0);
  84         else
  85                 qp_ctxt_hdr->addr_offset = RQ_CTXT_OFFSET(max_sqs, max_rqs, 0);
  86
  87         qp_ctxt_hdr->addr_offset = SIZE_16BYTES(qp_ctxt_hdr->addr_offset);
  88
  89         hinic_cpu_to_be32(qp_ctxt_hdr, sizeof(*qp_ctxt_hdr));
  90 }
  91
  92 void hinic_sq_prepare_ctxt(struct hinic_sq_ctxt *sq_ctxt,
  93                            struct hinic_sq *sq, u16 global_qid)
  94 {
  95         u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo;
  96         u64 wq_page_addr, wq_page_pfn, wq_block_pfn;
  97         u16 pi_start, ci_start;
  98         struct hinic_wq *wq;
  99
 100         wq = sq->wq;
 101         ci_start = atomic_read(&wq->cons_idx);
 102         pi_start = atomic_read(&wq->prod_idx);
 103
 104         /* Read the first page paddr from the WQ page paddr ptrs */
 105         wq_page_addr = be64_to_cpu(*wq->block_vaddr);
 106
 107         wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr);
 108         wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
 109         wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
 110
 111         /* If only one page, use 0-level CLA */
 112         if (wq->num_q_pages == 1)
 113                 wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq_page_addr);
 114         else
 115                 wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr);
 116
 117         wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
 118         wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
 119
 120         sq_ctxt->ceq_attr = HINIC_SQ_CTXT_CEQ_ATTR_SET(global_qid,
 121                                                        GLOBAL_SQ_ID) |
 122                             HINIC_SQ_CTXT_CEQ_ATTR_SET(0, EN);
 123
 124         sq_ctxt->ci_wrapped = HINIC_SQ_CTXT_CI_SET(ci_start, IDX) |
 125                               HINIC_SQ_CTXT_CI_SET(1, WRAPPED);
 126
 127         sq_ctxt->wq_hi_pfn_pi =
 128                         HINIC_SQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi, HI_PFN) |
 129                         HINIC_SQ_CTXT_WQ_PAGE_SET(pi_start, PI);
 130
 131         sq_ctxt->wq_lo_pfn = wq_page_pfn_lo;
 132
 133         sq_ctxt->pref_cache =
 134                 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |
 135                 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |
 136                 HINIC_SQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);
 137
 138         sq_ctxt->pref_wrapped = 1;
 139
 140         sq_ctxt->pref_wq_hi_pfn_ci =
 141                 HINIC_SQ_CTXT_PREF_SET(ci_start, CI) |
 142                 HINIC_SQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN);
 143
 144         sq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo;
 145
 146         sq_ctxt->wq_block_hi_pfn =
 147                 HINIC_SQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN);
 148
 149         sq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo;
 150
 151         hinic_cpu_to_be32(sq_ctxt, sizeof(*sq_ctxt));
 152 }
 153
 154 void hinic_rq_prepare_ctxt(struct hinic_rq_ctxt *rq_ctxt,
 155                            struct hinic_rq *rq, u16 global_qid)
 156 {
 157         u32 wq_page_pfn_hi, wq_page_pfn_lo, wq_block_pfn_hi, wq_block_pfn_lo;
 158         u64 wq_page_addr, wq_page_pfn, wq_block_pfn;
 159         u16 pi_start, ci_start;
 160         struct hinic_wq *wq;
 161
 162         wq = rq->wq;
 163         ci_start = atomic_read(&wq->cons_idx);
 164         pi_start = atomic_read(&wq->prod_idx);
 165
 166         /* Read the first page paddr from the WQ page paddr ptrs */
 167         wq_page_addr = be64_to_cpu(*wq->block_vaddr);
 168
 169         wq_page_pfn = HINIC_WQ_PAGE_PFN(wq_page_addr);
 170         wq_page_pfn_hi = upper_32_bits(wq_page_pfn);
 171         wq_page_pfn_lo = lower_32_bits(wq_page_pfn);
 172
 173         wq_block_pfn = HINIC_WQ_BLOCK_PFN(wq->block_paddr);
 174         wq_block_pfn_hi = upper_32_bits(wq_block_pfn);
 175         wq_block_pfn_lo = lower_32_bits(wq_block_pfn);
 176
 177         rq_ctxt->ceq_attr = HINIC_RQ_CTXT_CEQ_ATTR_SET(0, EN) |
 178                             HINIC_RQ_CTXT_CEQ_ATTR_SET(1, WRAPPED);
 179
 180         rq_ctxt->pi_intr_attr = HINIC_RQ_CTXT_PI_SET(pi_start, IDX) |
 181                                 HINIC_RQ_CTXT_PI_SET(rq->msix_entry, INTR);
 182
 183         rq_ctxt->wq_hi_pfn_ci = HINIC_RQ_CTXT_WQ_PAGE_SET(wq_page_pfn_hi,
 184                                                           HI_PFN) |
 185                                 HINIC_RQ_CTXT_WQ_PAGE_SET(ci_start, CI);
 186
 187         rq_ctxt->wq_lo_pfn = wq_page_pfn_lo;
 188
 189         rq_ctxt->pref_cache =
 190                 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MIN, CACHE_MIN) |
 191                 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_MAX, CACHE_MAX) |
 192                 HINIC_RQ_CTXT_PREF_SET(WQ_PREFETCH_THRESHOLD, CACHE_THRESHOLD);
 193
 194         rq_ctxt->pref_wrapped = 1;
 195
 196         rq_ctxt->pref_wq_hi_pfn_ci =
 197                 HINIC_RQ_CTXT_PREF_SET(wq_page_pfn_hi, WQ_HI_PFN) |
 198                 HINIC_RQ_CTXT_PREF_SET(ci_start, CI);
 199
 200         rq_ctxt->pref_wq_lo_pfn = wq_page_pfn_lo;
 201
 202         rq_ctxt->pi_paddr_hi = upper_32_bits(rq->pi_dma_addr);
 203         rq_ctxt->pi_paddr_lo = lower_32_bits(rq->pi_dma_addr);
 204
 205         rq_ctxt->wq_block_hi_pfn =
 206                 HINIC_RQ_CTXT_WQ_BLOCK_SET(wq_block_pfn_hi, HI_PFN);
 207
 208         rq_ctxt->wq_block_lo_pfn = wq_block_pfn_lo;
 209
 210         hinic_cpu_to_be32(rq_ctxt, sizeof(*rq_ctxt));
 211 }
 212
 213 /**
 214  * alloc_sq_skb_arr - allocate sq array for saved skb
 215  * @sq: HW Send Queue
 216  *
 217  * Return 0 - Success, negative - Failure
 218  **/
 219 static int alloc_sq_skb_arr(struct hinic_sq *sq)
 220 {
 221         struct hinic_wq *wq = sq->wq;
 222         size_t skb_arr_size;
 223
 224         skb_arr_size = wq->q_depth * sizeof(*sq->saved_skb);
 225         sq->saved_skb = vzalloc(skb_arr_size);
 226         if (!sq->saved_skb)
 227                 return -ENOMEM;
 228
 229         return 0;
 230 }
 231
 232 /**
 233  * free_sq_skb_arr - free sq array for saved skb
 234  * @sq: HW Send Queue
 235  **/
 236 static void free_sq_skb_arr(struct hinic_sq *sq)
 237 {
 238         vfree(sq->saved_skb);
 239 }
 240
 241 /**
 242  * alloc_rq_skb_arr - allocate rq array for saved skb
 243  * @rq: HW Receive Queue
 244  *
 245  * Return 0 - Success, negative - Failure
 246  **/
 247 static int alloc_rq_skb_arr(struct hinic_rq *rq)
 248 {
 249         struct hinic_wq *wq = rq->wq;
 250         size_t skb_arr_size;
 251
 252         skb_arr_size = wq->q_depth * sizeof(*rq->saved_skb);
 253         rq->saved_skb = vzalloc(skb_arr_size);
 254         if (!rq->saved_skb)
 255                 return -ENOMEM;
 256
 257         return 0;
 258 }
 259
 260 /**
 261  * free_rq_skb_arr - free rq array for saved skb
 262  * @rq: HW Receive Queue
 263  **/
 264 static void free_rq_skb_arr(struct hinic_rq *rq)
 265 {
 266         vfree(rq->saved_skb);
 267 }
 268
 269 /**
 270  * hinic_init_sq - Initialize HW Send Queue
 271  * @sq: HW Send Queue
 272  * @hwif: HW Interface for accessing HW
 273  * @wq: Work Queue for the data of the SQ
 274  * @entry: msix entry for sq
 275  * @ci_addr: address for reading the current HW consumer index
 276  * @ci_dma_addr: dma address for reading the current HW consumer index
 277  * @db_base: doorbell base address
 278  *
 279  * Return 0 - Success, negative - Failure
 280  **/
 281 int hinic_init_sq(struct hinic_sq *sq, struct hinic_hwif *hwif,
 282                   struct hinic_wq *wq, struct msix_entry *entry,
 283                   void *ci_addr, dma_addr_t ci_dma_addr,
 284                   void __iomem *db_base)
 285 {
 286         sq->hwif = hwif;
 287
 288         sq->wq = wq;
 289
 290         sq->irq = entry->vector;
 291         sq->msix_entry = entry->entry;
 292
 293         sq->hw_ci_addr = ci_addr;
 294         sq->hw_ci_dma_addr = ci_dma_addr;
 295
 296         sq->db_base = db_base + SQ_DB_OFF;
 297
 298         return alloc_sq_skb_arr(sq);
 299 }
 300
 301 /**
 302  * hinic_clean_sq - Clean HW Send Queue's Resources
 303  * @sq: Send Queue
 304  **/
 305 void hinic_clean_sq(struct hinic_sq *sq)
 306 {
 307         free_sq_skb_arr(sq);
 308 }
 309
 310 /**
 311  * alloc_rq_cqe - allocate rq completion queue elements
 312  * @rq: HW Receive Queue
 313  *
 314  * Return 0 - Success, negative - Failure
 315  **/
 316 static int alloc_rq_cqe(struct hinic_rq *rq)
 317 {
 318         struct hinic_hwif *hwif = rq->hwif;
 319         struct pci_dev *pdev = hwif->pdev;
 320         size_t cqe_dma_size, cqe_size;
 321         struct hinic_wq *wq = rq->wq;
 322         int j, i;
 323
 324         cqe_size = wq->q_depth * sizeof(*rq->cqe);
 325         rq->cqe = vzalloc(cqe_size);
 326         if (!rq->cqe)
 327                 return -ENOMEM;
 328
 329         cqe_dma_size = wq->q_depth * sizeof(*rq->cqe_dma);
 330         rq->cqe_dma = vzalloc(cqe_dma_size);
 331         if (!rq->cqe_dma)
 332                 goto err_cqe_dma_arr_alloc;
 333
 334         for (i = 0; i < wq->q_depth; i++) {
 335                 rq->cqe[i] = dma_alloc_coherent(&pdev->dev,
 336                                                 sizeof(*rq->cqe[i]),
 337                                                 &rq->cqe_dma[i], GFP_KERNEL);
 338                 if (!rq->cqe[i])
 339                         goto err_cqe_alloc;
 340         }
 341
 342         return 0;
 343
 344 err_cqe_alloc:
 345         for (j = 0; j < i; j++)
 346                 dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[j]), rq->cqe[j],
 347                                   rq->cqe_dma[j]);
 348
 349         vfree(rq->cqe_dma);
 350
 351 err_cqe_dma_arr_alloc:
 352         vfree(rq->cqe);
 353         return -ENOMEM;
 354 }
 355
 356 /**
 357  * free_rq_cqe - free rq completion queue elements
 358  * @rq: HW Receive Queue
 359  **/
 360 static void free_rq_cqe(struct hinic_rq *rq)
 361 {
 362         struct hinic_hwif *hwif = rq->hwif;
 363         struct pci_dev *pdev = hwif->pdev;
 364         struct hinic_wq *wq = rq->wq;
 365         int i;
 366
 367         for (i = 0; i < wq->q_depth; i++)
 368                 dma_free_coherent(&pdev->dev, sizeof(*rq->cqe[i]), rq->cqe[i],
 369                                   rq->cqe_dma[i]);
 370
 371         vfree(rq->cqe_dma);
 372         vfree(rq->cqe);
 373 }
 374
 375 /**
 376  * hinic_init_rq - Initialize HW Receive Queue
 377  * @rq: HW Receive Queue
 378  * @hwif: HW Interface for accessing HW
 379  * @wq: Work Queue for the data of the RQ
 380  * @entry: msix entry for rq
 381  *
 382  * Return 0 - Success, negative - Failure
 383  **/
 384 int hinic_init_rq(struct hinic_rq *rq, struct hinic_hwif *hwif,
 385                   struct hinic_wq *wq, struct msix_entry *entry)
 386 {
 387         struct pci_dev *pdev = hwif->pdev;
 388         size_t pi_size;
 389         int err;
 390
 391         rq->hwif = hwif;
 392
 393         rq->wq = wq;
 394
 395         rq->irq = entry->vector;
 396         rq->msix_entry = entry->entry;
 397
 398         rq->buf_sz = HINIC_RX_BUF_SZ;
 399
 400         err = alloc_rq_skb_arr(rq);
 401         if (err) {
 402                 dev_err(&pdev->dev, "Failed to allocate rq priv data\n");
 403                 return err;
 404         }
 405
 406         err = alloc_rq_cqe(rq);
 407         if (err) {
 408                 dev_err(&pdev->dev, "Failed to allocate rq cqe\n");
 409                 goto err_alloc_rq_cqe;
 410         }
 411
 412         /* HW requirements: Must be at least 32 bit */
 413         pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32));
 414         rq->pi_virt_addr = dma_alloc_coherent(&pdev->dev, pi_size,
 415                                               &rq->pi_dma_addr, GFP_KERNEL);
 416         if (!rq->pi_virt_addr) {
 417                 err = -ENOMEM;
 418                 goto err_pi_virt;
 419         }
 420
 421         return 0;
 422
 423 err_pi_virt:
 424         free_rq_cqe(rq);
 425
 426 err_alloc_rq_cqe:
 427         free_rq_skb_arr(rq);
 428         return err;
 429 }
 430
 431 /**
 432  * hinic_clean_rq - Clean HW Receive Queue's Resources
 433  * @rq: HW Receive Queue
 434  **/
 435 void hinic_clean_rq(struct hinic_rq *rq)
 436 {
 437         struct hinic_hwif *hwif = rq->hwif;
 438         struct pci_dev *pdev = hwif->pdev;
 439         size_t pi_size;
 440
 441         pi_size = ALIGN(sizeof(*rq->pi_virt_addr), sizeof(u32));
 442         dma_free_coherent(&pdev->dev, pi_size, rq->pi_virt_addr,
 443                           rq->pi_dma_addr);
 444
 445         free_rq_cqe(rq);
 446         free_rq_skb_arr(rq);
 447 }
 448
 449 /**
 450  * hinic_get_sq_free_wqebbs - return number of free wqebbs for use
 451  * @sq: send queue
 452  *
 453  * Return number of free wqebbs
 454  **/
 455 int hinic_get_sq_free_wqebbs(struct hinic_sq *sq)
 456 {
 457         struct hinic_wq *wq = sq->wq;
 458
 459         return atomic_read(&wq->delta) - 1;
 460 }
 461
 462 /**
 463  * hinic_get_rq_free_wqebbs - return number of free wqebbs for use
 464  * @rq: recv queue
 465  *
 466  * Return number of free wqebbs
 467  **/
 468 int hinic_get_rq_free_wqebbs(struct hinic_rq *rq)
 469 {
 470         struct hinic_wq *wq = rq->wq;
 471
 472         return atomic_read(&wq->delta) - 1;
 473 }
 474
 475 static void sq_prepare_ctrl(struct hinic_sq_ctrl *ctrl, int nr_descs)
 476 {
 477         u32 ctrl_size, task_size, bufdesc_size;
 478
 479         ctrl_size = SIZE_8BYTES(sizeof(struct hinic_sq_ctrl));
 480         task_size = SIZE_8BYTES(sizeof(struct hinic_sq_task));
 481         bufdesc_size = nr_descs * sizeof(struct hinic_sq_bufdesc);
 482         bufdesc_size = SIZE_8BYTES(bufdesc_size);
 483
 484         ctrl->ctrl_info = HINIC_SQ_CTRL_SET(bufdesc_size, BUFDESC_SECT_LEN) |
 485                           HINIC_SQ_CTRL_SET(task_size, TASKSECT_LEN)        |
 486                           HINIC_SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT)     |
 487                           HINIC_SQ_CTRL_SET(ctrl_size, LEN);
 488
 489         ctrl->queue_info = HINIC_SQ_CTRL_SET(HINIC_MSS_DEFAULT,
 490                                              QUEUE_INFO_MSS) |
 491                            HINIC_SQ_CTRL_SET(1, QUEUE_INFO_UC);
 492 }
 493
 494 static void sq_prepare_task(struct hinic_sq_task *task)
 495 {
 496         task->pkt_info0 = 0;
 497         task->pkt_info1 = 0;
 498         task->pkt_info2 = 0;
 499
 500         task->ufo_v6_identify = 0;
 501
 502         task->pkt_info4 = HINIC_SQ_TASK_INFO4_SET(HINIC_L2TYPE_ETH, L2TYPE);
 503
 504         task->zero_pad = 0;
 505 }
 506
 507 void hinic_task_set_l2hdr(struct hinic_sq_task *task, u32 len)
 508 {
 509         task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(len, L2HDR_LEN);
 510 }
 511
 512 void hinic_task_set_outter_l3(struct hinic_sq_task *task,
 513                               enum hinic_l3_offload_type l3_type,
 514                               u32 network_len)
 515 {
 516         task->pkt_info2 |= HINIC_SQ_TASK_INFO2_SET(l3_type, OUTER_L3TYPE) |
 517                            HINIC_SQ_TASK_INFO2_SET(network_len, OUTER_L3LEN);
 518 }
 519
 520 void hinic_task_set_inner_l3(struct hinic_sq_task *task,
 521                              enum hinic_l3_offload_type l3_type,
 522                              u32 network_len)
 523 {
 524         task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l3_type, INNER_L3TYPE);
 525         task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(network_len, INNER_L3LEN);
 526 }
 527
 528 void hinic_task_set_tunnel_l4(struct hinic_sq_task *task,
 529                               enum hinic_l4_tunnel_type l4_type,
 530                               u32 tunnel_len)
 531 {
 532         task->pkt_info2 |= HINIC_SQ_TASK_INFO2_SET(l4_type, TUNNEL_L4TYPE) |
 533                            HINIC_SQ_TASK_INFO2_SET(tunnel_len, TUNNEL_L4LEN);
 534 }
 535
 536 void hinic_set_cs_inner_l4(struct hinic_sq_task *task, u32 *queue_info,
 537                            enum hinic_l4_offload_type l4_offload,
 538                            u32 l4_len, u32 offset)
 539 {
 540         u32 tcp_udp_cs = 0, sctp = 0;
 541         u32 mss = HINIC_MSS_DEFAULT;
 542
 543         if (l4_offload == TCP_OFFLOAD_ENABLE ||
 544             l4_offload == UDP_OFFLOAD_ENABLE)
 545                 tcp_udp_cs = 1;
 546         else if (l4_offload == SCTP_OFFLOAD_ENABLE)
 547                 sctp = 1;
 548
 549         task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l4_offload, L4_OFFLOAD);
 550         task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(l4_len, INNER_L4LEN);
 551
 552         *queue_info |= HINIC_SQ_CTRL_SET(offset, QUEUE_INFO_PLDOFF) |
 553                        HINIC_SQ_CTRL_SET(tcp_udp_cs, QUEUE_INFO_TCPUDP_CS) |
 554                        HINIC_SQ_CTRL_SET(sctp, QUEUE_INFO_SCTP);
 555
 556         *queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS);
 557         *queue_info |= HINIC_SQ_CTRL_SET(mss, QUEUE_INFO_MSS);
 558 }
 559
 560 void hinic_set_tso_inner_l4(struct hinic_sq_task *task, u32 *queue_info,
 561                             enum hinic_l4_offload_type l4_offload,
 562                             u32 l4_len, u32 offset, u32 ip_ident, u32 mss)
 563 {
 564         u32 tso = 0, ufo = 0;
 565
 566         if (l4_offload == TCP_OFFLOAD_ENABLE)
 567                 tso = 1;
 568         else if (l4_offload == UDP_OFFLOAD_ENABLE)
 569                 ufo = 1;
 570
 571         task->ufo_v6_identify = ip_ident;
 572
 573         task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(l4_offload, L4_OFFLOAD);
 574         task->pkt_info0 |= HINIC_SQ_TASK_INFO0_SET(tso || ufo, TSO_FLAG);
 575         task->pkt_info1 |= HINIC_SQ_TASK_INFO1_SET(l4_len, INNER_L4LEN);
 576
 577         *queue_info |= HINIC_SQ_CTRL_SET(offset, QUEUE_INFO_PLDOFF) |
 578                        HINIC_SQ_CTRL_SET(tso, QUEUE_INFO_TSO) |
 579                        HINIC_SQ_CTRL_SET(ufo, QUEUE_INFO_UFO) |
 580                        HINIC_SQ_CTRL_SET(!!l4_offload, QUEUE_INFO_TCPUDP_CS);
 581
 582         /* set MSS value */
 583         *queue_info = HINIC_SQ_CTRL_CLEAR(*queue_info, QUEUE_INFO_MSS);
 584         *queue_info |= HINIC_SQ_CTRL_SET(mss, QUEUE_INFO_MSS);
 585 }
 586
 587 /**
 588  * hinic_sq_prepare_wqe - prepare wqe before insert to the queue
 589  * @sq: send queue
 590  * @sq_wqe: wqe to prepare
 591  * @sges: sges for use by the wqe for send for buf addresses
 592  * @nr_sges: number of sges
 593  **/
 594 void hinic_sq_prepare_wqe(struct hinic_sq *sq, struct hinic_sq_wqe *sq_wqe,
 595                           struct hinic_sge *sges, int nr_sges)
 596 {
 597         int i;
 598
 599         sq_prepare_ctrl(&sq_wqe->ctrl, nr_sges);
 600
 601         sq_prepare_task(&sq_wqe->task);
 602
 603         for (i = 0; i < nr_sges; i++)
 604                 sq_wqe->buf_descs[i].sge = sges[i];
 605 }
 606
 607 /**
 608  * sq_prepare_db - prepare doorbell to write
 609  * @sq: send queue
 610  * @prod_idx: pi value for the doorbell
 611  * @cos: cos of the doorbell
 612  *
 613  * Return db value
 614  **/
 615 static u32 sq_prepare_db(struct hinic_sq *sq, u16 prod_idx, unsigned int cos)
 616 {
 617         struct hinic_qp *qp = container_of(sq, struct hinic_qp, sq);
 618         u8 hi_prod_idx = SQ_DB_PI_HI(SQ_MASKED_IDX(sq, prod_idx));
 619
 620         /* Data should be written to HW in Big Endian Format */
 621         return cpu_to_be32(HINIC_SQ_DB_INFO_SET(hi_prod_idx, PI_HI)     |
 622                            HINIC_SQ_DB_INFO_SET(HINIC_DB_SQ_TYPE, TYPE) |
 623                            HINIC_SQ_DB_INFO_SET(HINIC_DATA_PATH, PATH)  |
 624                            HINIC_SQ_DB_INFO_SET(cos, COS)               |
 625                            HINIC_SQ_DB_INFO_SET(qp->q_id, QID));
 626 }
 627
 628 /**
 629  * hinic_sq_write_db- write doorbell
 630  * @sq: send queue
 631  * @prod_idx: pi value for the doorbell
 632  * @wqe_size: wqe size
 633  * @cos: cos of the wqe
 634  **/
 635 void hinic_sq_write_db(struct hinic_sq *sq, u16 prod_idx, unsigned int wqe_size,
 636                        unsigned int cos)
 637 {
 638         struct hinic_wq *wq = sq->wq;
 639
 640         /* increment prod_idx to the next */
 641         prod_idx += ALIGN(wqe_size, wq->wqebb_size) / wq->wqebb_size;
 642         prod_idx = SQ_MASKED_IDX(sq, prod_idx);
 643
 644         wmb();  /* Write all before the doorbell */
 645
 646         writel(sq_prepare_db(sq, prod_idx, cos), SQ_DB_ADDR(sq, prod_idx));
 647 }
 648
 649 /**
 650  * hinic_sq_get_wqe - get wqe ptr in the current pi and update the pi
 651  * @sq: sq to get wqe from
 652  * @wqe_size: wqe size
 653  * @prod_idx: returned pi
 654  *
 655  * Return wqe pointer
 656  **/
 657 struct hinic_sq_wqe *hinic_sq_get_wqe(struct hinic_sq *sq,
 658                                       unsigned int wqe_size, u16 *prod_idx)
 659 {
 660         struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(sq->wq, wqe_size,
 661                                                     prod_idx);
 662
 663         if (IS_ERR(hw_wqe))
 664                 return NULL;
 665
 666         return &hw_wqe->sq_wqe;
 667 }
 668
 669 /**
 670  * hinic_sq_return_wqe - return the wqe to the sq
 671  * @sq: send queue
 672  * @wqe_size: the size of the wqe
 673  **/
 674 void hinic_sq_return_wqe(struct hinic_sq *sq, unsigned int wqe_size)
 675 {
 676         hinic_return_wqe(sq->wq, wqe_size);
 677 }
 678
 679 /**
 680  * hinic_sq_write_wqe - write the wqe to the sq
 681  * @sq: send queue
 682  * @prod_idx: pi of the wqe
 683  * @sq_wqe: the wqe to write
 684  * @skb: skb to save
 685  * @wqe_size: the size of the wqe
 686  **/
 687 void hinic_sq_write_wqe(struct hinic_sq *sq, u16 prod_idx,
 688                         struct hinic_sq_wqe *sq_wqe,
 689                         struct sk_buff *skb, unsigned int wqe_size)
 690 {
 691         struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)sq_wqe;
 692
 693         sq->saved_skb[prod_idx] = skb;
 694
 695         /* The data in the HW should be in Big Endian Format */
 696         hinic_cpu_to_be32(sq_wqe, wqe_size);
 697
 698         hinic_write_wqe(sq->wq, hw_wqe, wqe_size);
 699 }
 700
 701 /**
 702  * hinic_sq_read_wqebb - read wqe ptr in the current ci and update the ci, the
 703  * wqe only have one wqebb
 704  * @sq: send queue
 705  * @skb: return skb that was saved
 706  * @wqe_size: the wqe size ptr
 707  * @cons_idx: consumer index of the wqe
 708  *
 709  * Return wqe in ci position
 710  **/
 711 struct hinic_sq_wqe *hinic_sq_read_wqebb(struct hinic_sq *sq,
 712                                          struct sk_buff **skb,
 713                                          unsigned int *wqe_size, u16 *cons_idx)
 714 {
 715         struct hinic_hw_wqe *hw_wqe;
 716         struct hinic_sq_wqe *sq_wqe;
 717         struct hinic_sq_ctrl *ctrl;
 718         unsigned int buf_sect_len;
 719         u32 ctrl_info;
 720
 721         /* read the ctrl section for getting wqe size */
 722         hw_wqe = hinic_read_wqe(sq->wq, sizeof(*ctrl), cons_idx);
 723         if (IS_ERR(hw_wqe))
 724                 return NULL;
 725
 726         *skb = sq->saved_skb[*cons_idx];
 727
 728         sq_wqe = &hw_wqe->sq_wqe;
 729         ctrl = &sq_wqe->ctrl;
 730         ctrl_info = be32_to_cpu(ctrl->ctrl_info);
 731         buf_sect_len = HINIC_SQ_CTRL_GET(ctrl_info, BUFDESC_SECT_LEN);
 732
 733         *wqe_size = sizeof(*ctrl) + sizeof(sq_wqe->task);
 734         *wqe_size += SECT_SIZE_FROM_8BYTES(buf_sect_len);
 735         *wqe_size = ALIGN(*wqe_size, sq->wq->wqebb_size);
 736
 737         return &hw_wqe->sq_wqe;
 738 }
 739
 740 /**
 741  * hinic_sq_read_wqe - read wqe ptr in the current ci and update the ci
 742  * @sq: send queue
 743  * @skb: return skb that was saved
 744  * @wqe_size: the size of the wqe
 745  * @cons_idx: consumer index of the wqe
 746  *
 747  * Return wqe in ci position
 748  **/
 749 struct hinic_sq_wqe *hinic_sq_read_wqe(struct hinic_sq *sq,
 750                                        struct sk_buff **skb,
 751                                        unsigned int wqe_size, u16 *cons_idx)
 752 {
 753         struct hinic_hw_wqe *hw_wqe;
 754
 755         hw_wqe = hinic_read_wqe(sq->wq, wqe_size, cons_idx);
 756         *skb = sq->saved_skb[*cons_idx];
 757
 758         return &hw_wqe->sq_wqe;
 759 }
 760
 761 /**
 762  * hinic_sq_put_wqe - release the ci for new wqes
 763  * @sq: send queue
 764  * @wqe_size: the size of the wqe
 765  **/
 766 void hinic_sq_put_wqe(struct hinic_sq *sq, unsigned int wqe_size)
 767 {
 768         hinic_put_wqe(sq->wq, wqe_size);
 769 }
 770
 771 /**
 772  * hinic_sq_get_sges - get sges from the wqe
 773  * @sq_wqe: wqe to get the sges from its buffer addresses
 774  * @sges: returned sges
 775  * @nr_sges: number sges to return
 776  **/
 777 void hinic_sq_get_sges(struct hinic_sq_wqe *sq_wqe, struct hinic_sge *sges,
 778                        int nr_sges)
 779 {
 780         int i;
 781
 782         for (i = 0; i < nr_sges && i < HINIC_MAX_SQ_BUFDESCS; i++) {
 783                 sges[i] = sq_wqe->buf_descs[i].sge;
 784                 hinic_be32_to_cpu(&sges[i], sizeof(sges[i]));
 785         }
 786 }
 787
 788 /**
 789  * hinic_rq_get_wqe - get wqe ptr in the current pi and update the pi
 790  * @rq: rq to get wqe from
 791  * @wqe_size: wqe size
 792  * @prod_idx: returned pi
 793  *
 794  * Return wqe pointer
 795  **/
 796 struct hinic_rq_wqe *hinic_rq_get_wqe(struct hinic_rq *rq,
 797                                       unsigned int wqe_size, u16 *prod_idx)
 798 {
 799         struct hinic_hw_wqe *hw_wqe = hinic_get_wqe(rq->wq, wqe_size,
 800                                                     prod_idx);
 801
 802         if (IS_ERR(hw_wqe))
 803                 return NULL;
 804
 805         return &hw_wqe->rq_wqe;
 806 }
 807
 808 /**
 809  * hinic_rq_write_wqe - write the wqe to the rq
 810  * @rq: recv queue
 811  * @prod_idx: pi of the wqe
 812  * @rq_wqe: the wqe to write
 813  * @skb: skb to save
 814  **/
 815 void hinic_rq_write_wqe(struct hinic_rq *rq, u16 prod_idx,
 816                         struct hinic_rq_wqe *rq_wqe, struct sk_buff *skb)
 817 {
 818         struct hinic_hw_wqe *hw_wqe = (struct hinic_hw_wqe *)rq_wqe;
 819
 820         rq->saved_skb[prod_idx] = skb;
 821
 822         /* The data in the HW should be in Big Endian Format */
 823         hinic_cpu_to_be32(rq_wqe, sizeof(*rq_wqe));
 824
 825         hinic_write_wqe(rq->wq, hw_wqe, sizeof(*rq_wqe));
 826 }
 827
 828 /**
 829  * hinic_rq_read_wqe - read wqe ptr in the current ci and update the ci
 830  * @rq: recv queue
 831  * @wqe_size: the size of the wqe
 832  * @skb: return saved skb
 833  * @cons_idx: consumer index of the wqe
 834  *
 835  * Return wqe in ci position
 836  **/
 837 struct hinic_rq_wqe *hinic_rq_read_wqe(struct hinic_rq *rq,
 838                                        unsigned int wqe_size,
 839                                        struct sk_buff **skb, u16 *cons_idx)
 840 {
 841         struct hinic_hw_wqe *hw_wqe;
 842         struct hinic_rq_cqe *cqe;
 843         int rx_done;
 844         u32 status;
 845
 846         hw_wqe = hinic_read_wqe(rq->wq, wqe_size, cons_idx);
 847         if (IS_ERR(hw_wqe))
 848                 return NULL;
 849
 850         cqe = rq->cqe[*cons_idx];
 851
 852         status = be32_to_cpu(cqe->status);
 853
 854         rx_done = HINIC_RQ_CQE_STATUS_GET(status, RXDONE);
 855         if (!rx_done)
 856                 return NULL;
 857
 858         *skb = rq->saved_skb[*cons_idx];
 859
 860         return &hw_wqe->rq_wqe;
 861 }
 862
 863 /**
 864  * hinic_rq_read_next_wqe - increment ci and read the wqe in ci position
 865  * @rq: recv queue
 866  * @wqe_size: the size of the wqe
 867  * @skb: return saved skb
 868  * @cons_idx: consumer index in the wq
 869  *
 870  * Return wqe in incremented ci position
 871  **/
 872 struct hinic_rq_wqe *hinic_rq_read_next_wqe(struct hinic_rq *rq,
 873                                             unsigned int wqe_size,
 874                                             struct sk_buff **skb,
 875                                             u16 *cons_idx)
 876 {
 877         struct hinic_wq *wq = rq->wq;
 878         struct hinic_hw_wqe *hw_wqe;
 879         unsigned int num_wqebbs;
 880
 881         wqe_size = ALIGN(wqe_size, wq->wqebb_size);
 882         num_wqebbs = wqe_size / wq->wqebb_size;
 883
 884         *cons_idx = RQ_MASKED_IDX(rq, *cons_idx + num_wqebbs);
 885
 886         *skb = rq->saved_skb[*cons_idx];
 887
 888         hw_wqe = hinic_read_wqe_direct(wq, *cons_idx);
 889
 890         return &hw_wqe->rq_wqe;
 891 }
 892
 893 /**
 894  * hinic_rq_put_wqe - release the ci for new wqes
 895  * @rq: recv queue
 896  * @cons_idx: consumer index of the wqe
 897  * @wqe_size: the size of the wqe
 898  **/
 899 void hinic_rq_put_wqe(struct hinic_rq *rq, u16 cons_idx,
 900                       unsigned int wqe_size)
 901 {
 902         struct hinic_rq_cqe *cqe = rq->cqe[cons_idx];
 903         u32 status = be32_to_cpu(cqe->status);
 904
 905         status = HINIC_RQ_CQE_STATUS_CLEAR(status, RXDONE);
 906
 907         /* Rx WQE size is 1 WQEBB, no wq shadow*/
 908         cqe->status = cpu_to_be32(status);
 909
 910         wmb();          /* clear done flag */
 911
 912         hinic_put_wqe(rq->wq, wqe_size);
 913 }
 914
 915 /**
 916  * hinic_rq_get_sge - get sge from the wqe
 917  * @rq: recv queue
 918  * @rq_wqe: wqe to get the sge from its buf address
 919  * @cons_idx: consumer index
 920  * @sge: returned sge
 921  **/
 922 void hinic_rq_get_sge(struct hinic_rq *rq, struct hinic_rq_wqe *rq_wqe,
 923                       u16 cons_idx, struct hinic_sge *sge)
 924 {
 925         struct hinic_rq_cqe *cqe = rq->cqe[cons_idx];
 926         u32 len = be32_to_cpu(cqe->len);
 927
 928         sge->hi_addr = be32_to_cpu(rq_wqe->buf_desc.hi_addr);
 929         sge->lo_addr = be32_to_cpu(rq_wqe->buf_desc.lo_addr);
 930         sge->len = HINIC_RQ_CQE_SGE_GET(len, LEN);
 931 }
 932
 933 /**
 934  * hinic_rq_prepare_wqe - prepare wqe before insert to the queue
 935  * @rq: recv queue
 936  * @prod_idx: pi value
 937  * @rq_wqe: the wqe
 938  * @sge: sge for use by the wqe for recv buf address
 939  **/
 940 void hinic_rq_prepare_wqe(struct hinic_rq *rq, u16 prod_idx,
 941                           struct hinic_rq_wqe *rq_wqe, struct hinic_sge *sge)
 942 {
 943         struct hinic_rq_cqe_sect *cqe_sect = &rq_wqe->cqe_sect;
 944         struct hinic_rq_bufdesc *buf_desc = &rq_wqe->buf_desc;
 945         struct hinic_rq_cqe *cqe = rq->cqe[prod_idx];
 946         struct hinic_rq_ctrl *ctrl = &rq_wqe->ctrl;
 947         dma_addr_t cqe_dma = rq->cqe_dma[prod_idx];
 948
 949         ctrl->ctrl_info =
 950                 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*ctrl)), LEN) |
 951                 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*cqe_sect)),
 952                                   COMPLETE_LEN)                    |
 953                 HINIC_RQ_CTRL_SET(SIZE_8BYTES(sizeof(*buf_desc)),
 954                                   BUFDESC_SECT_LEN)                |
 955                 HINIC_RQ_CTRL_SET(RQ_COMPLETE_SGE, COMPLETE_FORMAT);
 956
 957         hinic_set_sge(&cqe_sect->sge, cqe_dma, sizeof(*cqe));
 958
 959         buf_desc->hi_addr = sge->hi_addr;
 960         buf_desc->lo_addr = sge->lo_addr;
 961 }
 962
 963 /**
 964  * hinic_rq_update - update pi of the rq
 965  * @rq: recv queue
 966  * @prod_idx: pi value
 967  **/
 968 void hinic_rq_update(struct hinic_rq *rq, u16 prod_idx)
 969 {
 970         *rq->pi_virt_addr = cpu_to_be16(RQ_MASKED_IDX(rq, prod_idx + 1));
 971 }