drivers/infiniband/hw/qedr/verbs.c

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/dma-mapping.h>
  33 #include <linux/crc32.h>
  34 #include <net/ip.h>
  35 #include <net/ipv6.h>
  36 #include <net/udp.h>
  37 #include <linux/iommu.h>
  38
  39 #include <rdma/ib_verbs.h>
  40 #include <rdma/ib_user_verbs.h>
  41 #include <rdma/iw_cm.h>
  42 #include <rdma/ib_umem.h>
  43 #include <rdma/ib_addr.h>
  44 #include <rdma/ib_cache.h>
  45
  46 #include <linux/qed/common_hsi.h>
  47 #include "qedr_hsi_rdma.h"
  48 #include <linux/qed/qed_if.h>
  49 #include "qedr.h"
  50 #include "verbs.h"
  51 #include <rdma/qedr-abi.h>
  52 #include "qedr_roce_cm.h"
  53
  54 #define QEDR_SRQ_WQE_ELEM_SIZE  sizeof(union rdma_srq_elm)
  55 #define RDMA_MAX_SGE_PER_SRQ    (4)
  56 #define RDMA_MAX_SRQ_WQE_SIZE   (RDMA_MAX_SGE_PER_SRQ + 1)
  57
  58 #define DB_ADDR_SHIFT(addr)             ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  59
  60 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
  61                                         size_t len)
  62 {
  63         size_t min_len = min_t(size_t, len, udata->outlen);
  64
  65         return ib_copy_to_udata(udata, src, min_len);
  66 }
  67
  68 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
  69 {
  70         if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
  71                 return -EINVAL;
  72
  73         *pkey = QEDR_ROCE_PKEY_DEFAULT;
  74         return 0;
  75 }
  76
  77 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
  78                       int index, union ib_gid *sgid)
  79 {
  80         struct qedr_dev *dev = get_qedr_dev(ibdev);
  81
  82         memset(sgid->raw, 0, sizeof(sgid->raw));
  83         ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
  84
  85         DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
  86                  sgid->global.interface_id, sgid->global.subnet_prefix);
  87
  88         return 0;
  89 }
  90
  91 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
  92 {
  93         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
  94         struct qedr_device_attr *qattr = &dev->attr;
  95         struct qedr_srq *srq = get_qedr_srq(ibsrq);
  96
  97         srq_attr->srq_limit = srq->srq_limit;
  98         srq_attr->max_wr = qattr->max_srq_wr;
  99         srq_attr->max_sge = qattr->max_sge;
 100
 101         return 0;
 102 }
 103
 104 int qedr_query_device(struct ib_device *ibdev,
 105                       struct ib_device_attr *attr, struct ib_udata *udata)
 106 {
 107         struct qedr_dev *dev = get_qedr_dev(ibdev);
 108         struct qedr_device_attr *qattr = &dev->attr;
 109
 110         if (!dev->rdma_ctx) {
 111                 DP_ERR(dev,
 112                        "qedr_query_device called with invalid params rdma_ctx=%p\n",
 113                        dev->rdma_ctx);
 114                 return -EINVAL;
 115         }
 116
 117         memset(attr, 0, sizeof(*attr));
 118
 119         attr->fw_ver = qattr->fw_ver;
 120         attr->sys_image_guid = qattr->sys_image_guid;
 121         attr->max_mr_size = qattr->max_mr_size;
 122         attr->page_size_cap = qattr->page_size_caps;
 123         attr->vendor_id = qattr->vendor_id;
 124         attr->vendor_part_id = qattr->vendor_part_id;
 125         attr->hw_ver = qattr->hw_ver;
 126         attr->max_qp = qattr->max_qp;
 127         attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 128         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 129             IB_DEVICE_RC_RNR_NAK_GEN |
 130             IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
 131
 132         attr->max_send_sge = qattr->max_sge;
 133         attr->max_recv_sge = qattr->max_sge;
 134         attr->max_sge_rd = qattr->max_sge;
 135         attr->max_cq = qattr->max_cq;
 136         attr->max_cqe = qattr->max_cqe;
 137         attr->max_mr = qattr->max_mr;
 138         attr->max_mw = qattr->max_mw;
 139         attr->max_pd = qattr->max_pd;
 140         attr->atomic_cap = dev->atomic_cap;
 141         attr->max_fmr = qattr->max_fmr;
 142         attr->max_map_per_fmr = 16;
 143         attr->max_qp_init_rd_atom =
 144             1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 145         attr->max_qp_rd_atom =
 146             min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 147                 attr->max_qp_init_rd_atom);
 148
 149         attr->max_srq = qattr->max_srq;
 150         attr->max_srq_sge = qattr->max_srq_sge;
 151         attr->max_srq_wr = qattr->max_srq_wr;
 152
 153         attr->local_ca_ack_delay = qattr->dev_ack_delay;
 154         attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 155         attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
 156         attr->max_ah = qattr->max_ah;
 157
 158         return 0;
 159 }
 160
 161 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
 162                                             u8 *ib_width)
 163 {
 164         switch (speed) {
 165         case 1000:
 166                 *ib_speed = IB_SPEED_SDR;
 167                 *ib_width = IB_WIDTH_1X;
 168                 break;
 169         case 10000:
 170                 *ib_speed = IB_SPEED_QDR;
 171                 *ib_width = IB_WIDTH_1X;
 172                 break;
 173
 174         case 20000:
 175                 *ib_speed = IB_SPEED_DDR;
 176                 *ib_width = IB_WIDTH_4X;
 177                 break;
 178
 179         case 25000:
 180                 *ib_speed = IB_SPEED_EDR;
 181                 *ib_width = IB_WIDTH_1X;
 182                 break;
 183
 184         case 40000:
 185                 *ib_speed = IB_SPEED_QDR;
 186                 *ib_width = IB_WIDTH_4X;
 187                 break;
 188
 189         case 50000:
 190                 *ib_speed = IB_SPEED_HDR;
 191                 *ib_width = IB_WIDTH_1X;
 192                 break;
 193
 194         case 100000:
 195                 *ib_speed = IB_SPEED_EDR;
 196                 *ib_width = IB_WIDTH_4X;
 197                 break;
 198
 199         default:
 200                 /* Unsupported */
 201                 *ib_speed = IB_SPEED_SDR;
 202                 *ib_width = IB_WIDTH_1X;
 203         }
 204 }
 205
 206 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
 207 {
 208         struct qedr_dev *dev;
 209         struct qed_rdma_port *rdma_port;
 210
 211         dev = get_qedr_dev(ibdev);
 212         if (port > 1) {
 213                 DP_ERR(dev, "invalid_port=0x%x\n", port);
 214                 return -EINVAL;
 215         }
 216
 217         if (!dev->rdma_ctx) {
 218                 DP_ERR(dev, "rdma_ctx is NULL\n");
 219                 return -EINVAL;
 220         }
 221
 222         rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 223
 224         /* *attr being zeroed by the caller, avoid zeroing it here */
 225         if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 226                 attr->state = IB_PORT_ACTIVE;
 227                 attr->phys_state = 5;
 228         } else {
 229                 attr->state = IB_PORT_DOWN;
 230                 attr->phys_state = 3;
 231         }
 232         attr->max_mtu = IB_MTU_4096;
 233         attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 234         attr->lid = 0;
 235         attr->lmc = 0;
 236         attr->sm_lid = 0;
 237         attr->sm_sl = 0;
 238         attr->ip_gids = true;
 239         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
 240                 attr->gid_tbl_len = 1;
 241                 attr->pkey_tbl_len = 1;
 242         } else {
 243                 attr->gid_tbl_len = QEDR_MAX_SGID;
 244                 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 245         }
 246         attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 247         attr->qkey_viol_cntr = 0;
 248         get_link_speed_and_width(rdma_port->link_speed,
 249                                  &attr->active_speed, &attr->active_width);
 250         attr->max_msg_sz = rdma_port->max_msg_size;
 251         attr->max_vl_num = 4;
 252
 253         return 0;
 254 }
 255
 256 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
 257                      struct ib_port_modify *props)
 258 {
 259         struct qedr_dev *dev;
 260
 261         dev = get_qedr_dev(ibdev);
 262         if (port > 1) {
 263                 DP_ERR(dev, "invalid_port=0x%x\n", port);
 264                 return -EINVAL;
 265         }
 266
 267         return 0;
 268 }
 269
 270 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
 271                          unsigned long len)
 272 {
 273         struct qedr_mm *mm;
 274
 275         mm = kzalloc(sizeof(*mm), GFP_KERNEL);
 276         if (!mm)
 277                 return -ENOMEM;
 278
 279         mm->key.phy_addr = phy_addr;
 280         /* This function might be called with a length which is not a multiple
 281          * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
 282          * forces this granularity by increasing the requested size if needed.
 283          * When qedr_mmap is called, it will search the list with the updated
 284          * length as a key. To prevent search failures, the length is rounded up
 285          * in advance to PAGE_SIZE.
 286          */
 287         mm->key.len = roundup(len, PAGE_SIZE);
 288         INIT_LIST_HEAD(&mm->entry);
 289
 290         mutex_lock(&uctx->mm_list_lock);
 291         list_add(&mm->entry, &uctx->mm_head);
 292         mutex_unlock(&uctx->mm_list_lock);
 293
 294         DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 295                  "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
 296                  (unsigned long long)mm->key.phy_addr,
 297                  (unsigned long)mm->key.len, uctx);
 298
 299         return 0;
 300 }
 301
 302 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
 303                              unsigned long len)
 304 {
 305         bool found = false;
 306         struct qedr_mm *mm;
 307
 308         mutex_lock(&uctx->mm_list_lock);
 309         list_for_each_entry(mm, &uctx->mm_head, entry) {
 310                 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
 311                         continue;
 312
 313                 found = true;
 314                 break;
 315         }
 316         mutex_unlock(&uctx->mm_list_lock);
 317         DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 318                  "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
 319                  mm->key.phy_addr, mm->key.len, uctx, found);
 320
 321         return found;
 322 }
 323
 324 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
 325                                         struct ib_udata *udata)
 326 {
 327         int rc;
 328         struct qedr_ucontext *ctx;
 329         struct qedr_alloc_ucontext_resp uresp;
 330         struct qedr_dev *dev = get_qedr_dev(ibdev);
 331         struct qed_rdma_add_user_out_params oparams;
 332
 333         if (!udata)
 334                 return ERR_PTR(-EFAULT);
 335
 336         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 337         if (!ctx)
 338                 return ERR_PTR(-ENOMEM);
 339
 340         rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 341         if (rc) {
 342                 DP_ERR(dev,
 343                        "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 344                        rc);
 345                 goto err;
 346         }
 347
 348         ctx->dpi = oparams.dpi;
 349         ctx->dpi_addr = oparams.dpi_addr;
 350         ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 351         ctx->dpi_size = oparams.dpi_size;
 352         INIT_LIST_HEAD(&ctx->mm_head);
 353         mutex_init(&ctx->mm_list_lock);
 354
 355         memset(&uresp, 0, sizeof(uresp));
 356
 357         uresp.dpm_enabled = dev->user_dpm_enabled;
 358         uresp.wids_enabled = 1;
 359         uresp.wid_count = oparams.wid_count;
 360         uresp.db_pa = ctx->dpi_phys_addr;
 361         uresp.db_size = ctx->dpi_size;
 362         uresp.max_send_wr = dev->attr.max_sqe;
 363         uresp.max_recv_wr = dev->attr.max_rqe;
 364         uresp.max_srq_wr = dev->attr.max_srq_wr;
 365         uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 366         uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 367         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 368         uresp.max_cqes = QEDR_MAX_CQES;
 369
 370         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 371         if (rc)
 372                 goto err;
 373
 374         ctx->dev = dev;
 375
 376         rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
 377         if (rc)
 378                 goto err;
 379
 380         DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 381                  &ctx->ibucontext);
 382         return &ctx->ibucontext;
 383
 384 err:
 385         kfree(ctx);
 386         return ERR_PTR(rc);
 387 }
 388
 389 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 390 {
 391         struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 392         struct qedr_mm *mm, *tmp;
 393         int status = 0;
 394
 395         DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 396                  uctx);
 397         uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
 398
 399         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 400                 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 401                          "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
 402                          mm->key.phy_addr, mm->key.len, uctx);
 403                 list_del(&mm->entry);
 404                 kfree(mm);
 405         }
 406
 407         kfree(uctx);
 408         return status;
 409 }
 410
 411 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 412 {
 413         struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
 414         struct qedr_dev *dev = get_qedr_dev(context->device);
 415         unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT;
 416         unsigned long len = (vma->vm_end - vma->vm_start);
 417         unsigned long dpi_start;
 418
 419         dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
 420
 421         DP_DEBUG(dev, QEDR_MSG_INIT,
 422                  "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n",
 423                  (void *)vma->vm_start, (void *)vma->vm_end,
 424                  (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size);
 425
 426         if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
 427                 DP_ERR(dev,
 428                        "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n",
 429                        (void *)vma->vm_start, (void *)vma->vm_end);
 430                 return -EINVAL;
 431         }
 432
 433         if (!qedr_search_mmap(ucontext, phys_addr, len)) {
 434                 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n",
 435                        vma->vm_pgoff);
 436                 return -EINVAL;
 437         }
 438
 439         if (phys_addr < dpi_start ||
 440             ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) {
 441                 DP_ERR(dev,
 442                        "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n",
 443                        (void *)phys_addr, (void *)dpi_start,
 444                        ucontext->dpi_size);
 445                 return -EINVAL;
 446         }
 447
 448         if (vma->vm_flags & VM_READ) {
 449                 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n");
 450                 return -EINVAL;
 451         }
 452
 453         vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 454         return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
 455                                   vma->vm_page_prot);
 456 }
 457
 458 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
 459                             struct ib_ucontext *context, struct ib_udata *udata)
 460 {
 461         struct qedr_dev *dev = get_qedr_dev(ibdev);
 462         struct qedr_pd *pd;
 463         u16 pd_id;
 464         int rc;
 465
 466         DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 467                  (udata && context) ? "User Lib" : "Kernel");
 468
 469         if (!dev->rdma_ctx) {
 470                 DP_ERR(dev, "invalid RDMA context\n");
 471                 return ERR_PTR(-EINVAL);
 472         }
 473
 474         pd = kzalloc(sizeof(*pd), GFP_KERNEL);
 475         if (!pd)
 476                 return ERR_PTR(-ENOMEM);
 477
 478         rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 479         if (rc)
 480                 goto err;
 481
 482         pd->pd_id = pd_id;
 483
 484         if (udata && context) {
 485                 struct qedr_alloc_pd_uresp uresp = {
 486                         .pd_id = pd_id,
 487                 };
 488
 489                 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 490                 if (rc) {
 491                         DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 492                         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 493                         goto err;
 494                 }
 495
 496                 pd->uctx = get_qedr_ucontext(context);
 497                 pd->uctx->pd = pd;
 498         }
 499
 500         return &pd->ibpd;
 501
 502 err:
 503         kfree(pd);
 504         return ERR_PTR(rc);
 505 }
 506
 507 int qedr_dealloc_pd(struct ib_pd *ibpd)
 508 {
 509         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 510         struct qedr_pd *pd = get_qedr_pd(ibpd);
 511
 512         if (!pd) {
 513                 pr_err("Invalid PD received in dealloc_pd\n");
 514                 return -EINVAL;
 515         }
 516
 517         DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 518         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 519
 520         kfree(pd);
 521
 522         return 0;
 523 }
 524
 525 static void qedr_free_pbl(struct qedr_dev *dev,
 526                           struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 527 {
 528         struct pci_dev *pdev = dev->pdev;
 529         int i;
 530
 531         for (i = 0; i < pbl_info->num_pbls; i++) {
 532                 if (!pbl[i].va)
 533                         continue;
 534                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 535                                   pbl[i].va, pbl[i].pa);
 536         }
 537
 538         kfree(pbl);
 539 }
 540
 541 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 542 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 543
 544 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 545 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 546 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 547
 548 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 549                                            struct qedr_pbl_info *pbl_info,
 550                                            gfp_t flags)
 551 {
 552         struct pci_dev *pdev = dev->pdev;
 553         struct qedr_pbl *pbl_table;
 554         dma_addr_t *pbl_main_tbl;
 555         dma_addr_t pa;
 556         void *va;
 557         int i;
 558
 559         pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 560         if (!pbl_table)
 561                 return ERR_PTR(-ENOMEM);
 562
 563         for (i = 0; i < pbl_info->num_pbls; i++) {
 564                 va = dma_zalloc_coherent(&pdev->dev, pbl_info->pbl_size,
 565                                          &pa, flags);
 566                 if (!va)
 567                         goto err;
 568
 569                 pbl_table[i].va = va;
 570                 pbl_table[i].pa = pa;
 571         }
 572
 573         /* Two-Layer PBLs, if we have more than one pbl we need to initialize
 574          * the first one with physical pointers to all of the rest
 575          */
 576         pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 577         for (i = 0; i < pbl_info->num_pbls - 1; i++)
 578                 pbl_main_tbl[i] = pbl_table[i + 1].pa;
 579
 580         return pbl_table;
 581
 582 err:
 583         for (i--; i >= 0; i--)
 584                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 585                                   pbl_table[i].va, pbl_table[i].pa);
 586
 587         qedr_free_pbl(dev, pbl_info, pbl_table);
 588
 589         return ERR_PTR(-ENOMEM);
 590 }
 591
 592 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 593                                 struct qedr_pbl_info *pbl_info,
 594                                 u32 num_pbes, int two_layer_capable)
 595 {
 596         u32 pbl_capacity;
 597         u32 pbl_size;
 598         u32 num_pbls;
 599
 600         if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 601                 if (num_pbes > MAX_PBES_TWO_LAYER) {
 602                         DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 603                                num_pbes);
 604                         return -EINVAL;
 605                 }
 606
 607                 /* calculate required pbl page size */
 608                 pbl_size = MIN_FW_PBL_PAGE_SIZE;
 609                 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 610                                NUM_PBES_ON_PAGE(pbl_size);
 611
 612                 while (pbl_capacity < num_pbes) {
 613                         pbl_size *= 2;
 614                         pbl_capacity = pbl_size / sizeof(u64);
 615                         pbl_capacity = pbl_capacity * pbl_capacity;
 616                 }
 617
 618                 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 619                 num_pbls++;     /* One for the layer0 ( points to the pbls) */
 620                 pbl_info->two_layered = true;
 621         } else {
 622                 /* One layered PBL */
 623                 num_pbls = 1;
 624                 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 625                                  roundup_pow_of_two((num_pbes * sizeof(u64))));
 626                 pbl_info->two_layered = false;
 627         }
 628
 629         pbl_info->num_pbls = num_pbls;
 630         pbl_info->pbl_size = pbl_size;
 631         pbl_info->num_pbes = num_pbes;
 632
 633         DP_DEBUG(dev, QEDR_MSG_MR,
 634                  "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 635                  pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 636
 637         return 0;
 638 }
 639
 640 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 641                                struct qedr_pbl *pbl,
 642                                struct qedr_pbl_info *pbl_info, u32 pg_shift)
 643 {
 644         int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
 645         u32 fw_pg_cnt, fw_pg_per_umem_pg;
 646         struct qedr_pbl *pbl_tbl;
 647         struct scatterlist *sg;
 648         struct regpair *pbe;
 649         u64 pg_addr;
 650         int entry;
 651
 652         if (!pbl_info->num_pbes)
 653                 return;
 654
 655         /* If we have a two layered pbl, the first pbl points to the rest
 656          * of the pbls and the first entry lays on the second pbl in the table
 657          */
 658         if (pbl_info->two_layered)
 659                 pbl_tbl = &pbl[1];
 660         else
 661                 pbl_tbl = pbl;
 662
 663         pbe = (struct regpair *)pbl_tbl->va;
 664         if (!pbe) {
 665                 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 666                 return;
 667         }
 668
 669         pbe_cnt = 0;
 670
 671         shift = umem->page_shift;
 672
 673         fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift);
 674
 675         for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
 676                 pages = sg_dma_len(sg) >> shift;
 677                 pg_addr = sg_dma_address(sg);
 678                 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
 679                         for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
 680                                 pbe->lo = cpu_to_le32(pg_addr);
 681                                 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 682
 683                                 pg_addr += BIT(pg_shift);
 684                                 pbe_cnt++;
 685                                 total_num_pbes++;
 686                                 pbe++;
 687
 688                                 if (total_num_pbes == pbl_info->num_pbes)
 689                                         return;
 690
 691                                 /* If the given pbl is full storing the pbes,
 692                                  * move to next pbl.
 693                                  */
 694                                 if (pbe_cnt ==
 695                                     (pbl_info->pbl_size / sizeof(u64))) {
 696                                         pbl_tbl++;
 697                                         pbe = (struct regpair *)pbl_tbl->va;
 698                                         pbe_cnt = 0;
 699                                 }
 700
 701                                 fw_pg_cnt++;
 702                         }
 703                 }
 704         }
 705 }
 706
 707 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 708                               struct qedr_cq *cq, struct ib_udata *udata)
 709 {
 710         struct qedr_create_cq_uresp uresp;
 711         int rc;
 712
 713         memset(&uresp, 0, sizeof(uresp));
 714
 715         uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 716         uresp.icid = cq->icid;
 717
 718         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 719         if (rc)
 720                 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 721
 722         return rc;
 723 }
 724
 725 static void consume_cqe(struct qedr_cq *cq)
 726 {
 727         if (cq->latest_cqe == cq->toggle_cqe)
 728                 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 729
 730         cq->latest_cqe = qed_chain_consume(&cq->pbl);
 731 }
 732
 733 static inline int qedr_align_cq_entries(int entries)
 734 {
 735         u64 size, aligned_size;
 736
 737         /* We allocate an extra entry that we don't report to the FW. */
 738         size = (entries + 1) * QEDR_CQE_SIZE;
 739         aligned_size = ALIGN(size, PAGE_SIZE);
 740
 741         return aligned_size / QEDR_CQE_SIZE;
 742 }
 743
 744 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
 745                                        struct qedr_dev *dev,
 746                                        struct qedr_userq *q,
 747                                        u64 buf_addr, size_t buf_len,
 748                                        int access, int dmasync,
 749                                        int alloc_and_init)
 750 {
 751         u32 fw_pages;
 752         int rc;
 753
 754         q->buf_addr = buf_addr;
 755         q->buf_len = buf_len;
 756         q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
 757         if (IS_ERR(q->umem)) {
 758                 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 759                        PTR_ERR(q->umem));
 760                 return PTR_ERR(q->umem);
 761         }
 762
 763         fw_pages = ib_umem_page_count(q->umem) <<
 764             (q->umem->page_shift - FW_PAGE_SHIFT);
 765
 766         rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 767         if (rc)
 768                 goto err0;
 769
 770         if (alloc_and_init) {
 771                 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 772                 if (IS_ERR(q->pbl_tbl)) {
 773                         rc = PTR_ERR(q->pbl_tbl);
 774                         goto err0;
 775                 }
 776                 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 777                                    FW_PAGE_SHIFT);
 778         } else {
 779                 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
 780                 if (!q->pbl_tbl) {
 781                         rc = -ENOMEM;
 782                         goto err0;
 783                 }
 784         }
 785
 786         return 0;
 787
 788 err0:
 789         ib_umem_release(q->umem);
 790         q->umem = NULL;
 791
 792         return rc;
 793 }
 794
 795 static inline void qedr_init_cq_params(struct qedr_cq *cq,
 796                                        struct qedr_ucontext *ctx,
 797                                        struct qedr_dev *dev, int vector,
 798                                        int chain_entries, int page_cnt,
 799                                        u64 pbl_ptr,
 800                                        struct qed_rdma_create_cq_in_params
 801                                        *params)
 802 {
 803         memset(params, 0, sizeof(*params));
 804         params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 805         params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 806         params->cnq_id = vector;
 807         params->cq_size = chain_entries - 1;
 808         params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 809         params->pbl_num_pages = page_cnt;
 810         params->pbl_ptr = pbl_ptr;
 811         params->pbl_two_level = 0;
 812 }
 813
 814 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 815 {
 816         cq->db.data.agg_flags = flags;
 817         cq->db.data.value = cpu_to_le32(cons);
 818         writeq(cq->db.raw, cq->db_addr);
 819
 820         /* Make sure write would stick */
 821         mmiowb();
 822 }
 823
 824 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 825 {
 826         struct qedr_cq *cq = get_qedr_cq(ibcq);
 827         unsigned long sflags;
 828         struct qedr_dev *dev;
 829
 830         dev = get_qedr_dev(ibcq->device);
 831
 832         if (cq->destroyed) {
 833                 DP_ERR(dev,
 834                        "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 835                        cq, cq->icid);
 836                 return -EINVAL;
 837         }
 838
 839
 840         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 841                 return 0;
 842
 843         spin_lock_irqsave(&cq->cq_lock, sflags);
 844
 845         cq->arm_flags = 0;
 846
 847         if (flags & IB_CQ_SOLICITED)
 848                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 849
 850         if (flags & IB_CQ_NEXT_COMP)
 851                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 852
 853         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 854
 855         spin_unlock_irqrestore(&cq->cq_lock, sflags);
 856
 857         return 0;
 858 }
 859
 860 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
 861                              const struct ib_cq_init_attr *attr,
 862                              struct ib_ucontext *ib_ctx, struct ib_udata *udata)
 863 {
 864         struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
 865         struct qed_rdma_destroy_cq_out_params destroy_oparams;
 866         struct qed_rdma_destroy_cq_in_params destroy_iparams;
 867         struct qedr_dev *dev = get_qedr_dev(ibdev);
 868         struct qed_rdma_create_cq_in_params params;
 869         struct qedr_create_cq_ureq ureq;
 870         int vector = attr->comp_vector;
 871         int entries = attr->cqe;
 872         struct qedr_cq *cq;
 873         int chain_entries;
 874         int page_cnt;
 875         u64 pbl_ptr;
 876         u16 icid;
 877         int rc;
 878
 879         DP_DEBUG(dev, QEDR_MSG_INIT,
 880                  "create_cq: called from %s. entries=%d, vector=%d\n",
 881                  udata ? "User Lib" : "Kernel", entries, vector);
 882
 883         if (entries > QEDR_MAX_CQES) {
 884                 DP_ERR(dev,
 885                        "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 886                        entries, QEDR_MAX_CQES);
 887                 return ERR_PTR(-EINVAL);
 888         }
 889
 890         chain_entries = qedr_align_cq_entries(entries);
 891         chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 892
 893         cq = kzalloc(sizeof(*cq), GFP_KERNEL);
 894         if (!cq)
 895                 return ERR_PTR(-ENOMEM);
 896
 897         if (udata) {
 898                 memset(&ureq, 0, sizeof(ureq));
 899                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
 900                         DP_ERR(dev,
 901                                "create cq: problem copying data from user space\n");
 902                         goto err0;
 903                 }
 904
 905                 if (!ureq.len) {
 906                         DP_ERR(dev,
 907                                "create cq: cannot create a cq with 0 entries\n");
 908                         goto err0;
 909                 }
 910
 911                 cq->cq_type = QEDR_CQ_TYPE_USER;
 912
 913                 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
 914                                           ureq.len, IB_ACCESS_LOCAL_WRITE,
 915                                           1, 1);
 916                 if (rc)
 917                         goto err0;
 918
 919                 pbl_ptr = cq->q.pbl_tbl->pa;
 920                 page_cnt = cq->q.pbl_info.num_pbes;
 921
 922                 cq->ibcq.cqe = chain_entries;
 923         } else {
 924                 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 925
 926                 rc = dev->ops->common->chain_alloc(dev->cdev,
 927                                                    QED_CHAIN_USE_TO_CONSUME,
 928                                                    QED_CHAIN_MODE_PBL,
 929                                                    QED_CHAIN_CNT_TYPE_U32,
 930                                                    chain_entries,
 931                                                    sizeof(union rdma_cqe),
 932                                                    &cq->pbl, NULL);
 933                 if (rc)
 934                         goto err1;
 935
 936                 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 937                 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 938                 cq->ibcq.cqe = cq->pbl.capacity;
 939         }
 940
 941         qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 942                             pbl_ptr, &params);
 943
 944         rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 945         if (rc)
 946                 goto err2;
 947
 948         cq->icid = icid;
 949         cq->sig = QEDR_CQ_MAGIC_NUMBER;
 950         spin_lock_init(&cq->cq_lock);
 951
 952         if (ib_ctx) {
 953                 rc = qedr_copy_cq_uresp(dev, cq, udata);
 954                 if (rc)
 955                         goto err3;
 956         } else {
 957                 /* Generate doorbell address. */
 958                 cq->db_addr = dev->db_addr +
 959                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 960                 cq->db.data.icid = cq->icid;
 961                 cq->db.data.params = DB_AGG_CMD_SET <<
 962                     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
 963
 964                 /* point to the very last element, passing it we will toggle */
 965                 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
 966                 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 967                 cq->latest_cqe = NULL;
 968                 consume_cqe(cq);
 969                 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
 970         }
 971
 972         DP_DEBUG(dev, QEDR_MSG_CQ,
 973                  "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
 974                  cq->icid, cq, params.cq_size);
 975
 976         return &cq->ibcq;
 977
 978 err3:
 979         destroy_iparams.icid = cq->icid;
 980         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
 981                                   &destroy_oparams);
 982 err2:
 983         if (udata)
 984                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
 985         else
 986                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
 987 err1:
 988         if (udata)
 989                 ib_umem_release(cq->q.umem);
 990 err0:
 991         kfree(cq);
 992         return ERR_PTR(-EINVAL);
 993 }
 994
 995 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
 996 {
 997         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
 998         struct qedr_cq *cq = get_qedr_cq(ibcq);
 999
1000         DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1001
1002         return 0;
1003 }
1004
1005 #define QEDR_DESTROY_CQ_MAX_ITERATIONS          (10)
1006 #define QEDR_DESTROY_CQ_ITER_DURATION           (10)
1007
1008 int qedr_destroy_cq(struct ib_cq *ibcq)
1009 {
1010         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1011         struct qed_rdma_destroy_cq_out_params oparams;
1012         struct qed_rdma_destroy_cq_in_params iparams;
1013         struct qedr_cq *cq = get_qedr_cq(ibcq);
1014         int iter;
1015         int rc;
1016
1017         DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1018
1019         cq->destroyed = 1;
1020
1021         /* GSIs CQs are handled by driver, so they don't exist in the FW */
1022         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
1023                 goto done;
1024
1025         iparams.icid = cq->icid;
1026         rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1027         if (rc)
1028                 return rc;
1029
1030         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1031
1032         if (ibcq->uobject && ibcq->uobject->context) {
1033                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1034                 ib_umem_release(cq->q.umem);
1035         }
1036
1037         /* We don't want the IRQ handler to handle a non-existing CQ so we
1038          * wait until all CNQ interrupts, if any, are received. This will always
1039          * happen and will always happen very fast. If not, then a serious error
1040          * has occured. That is why we can use a long delay.
1041          * We spin for a short time so we don’t lose time on context switching
1042          * in case all the completions are handled in that span. Otherwise
1043          * we sleep for a while and check again. Since the CNQ may be
1044          * associated with (only) the current CPU we use msleep to allow the
1045          * current CPU to be freed.
1046          * The CNQ notification is increased in qedr_irq_handler().
1047          */
1048         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1049         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1050                 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1051                 iter--;
1052         }
1053
1054         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1055         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1056                 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1057                 iter--;
1058         }
1059
1060         if (oparams.num_cq_notif != cq->cnq_notif)
1061                 goto err;
1062
1063         /* Note that we don't need to have explicit code to wait for the
1064          * completion of the event handler because it is invoked from the EQ.
1065          * Since the destroy CQ ramrod has also been received on the EQ we can
1066          * be certain that there's no event handler in process.
1067          */
1068 done:
1069         cq->sig = ~cq->sig;
1070
1071         kfree(cq);
1072
1073         return 0;
1074
1075 err:
1076         DP_ERR(dev,
1077                "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1078                cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1079
1080         return -EINVAL;
1081 }
1082
1083 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1084                                           struct ib_qp_attr *attr,
1085                                           int attr_mask,
1086                                           struct qed_rdma_modify_qp_in_params
1087                                           *qp_params)
1088 {
1089         const struct ib_gid_attr *gid_attr;
1090         enum rdma_network_type nw_type;
1091         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1092         u32 ipv4_addr;
1093         int i;
1094
1095         gid_attr = grh->sgid_attr;
1096         qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr->ndev);
1097
1098         nw_type = rdma_gid_attr_network_type(gid_attr);
1099         switch (nw_type) {
1100         case RDMA_NETWORK_IPV6:
1101                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1102                        sizeof(qp_params->sgid));
1103                 memcpy(&qp_params->dgid.bytes[0],
1104                        &grh->dgid,
1105                        sizeof(qp_params->dgid));
1106                 qp_params->roce_mode = ROCE_V2_IPV6;
1107                 SET_FIELD(qp_params->modify_flags,
1108                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1109                 break;
1110         case RDMA_NETWORK_IB:
1111                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1112                        sizeof(qp_params->sgid));
1113                 memcpy(&qp_params->dgid.bytes[0],
1114                        &grh->dgid,
1115                        sizeof(qp_params->dgid));
1116                 qp_params->roce_mode = ROCE_V1;
1117                 break;
1118         case RDMA_NETWORK_IPV4:
1119                 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1120                 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1121                 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1122                 qp_params->sgid.ipv4_addr = ipv4_addr;
1123                 ipv4_addr =
1124                     qedr_get_ipv4_from_gid(grh->dgid.raw);
1125                 qp_params->dgid.ipv4_addr = ipv4_addr;
1126                 SET_FIELD(qp_params->modify_flags,
1127                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1128                 qp_params->roce_mode = ROCE_V2_IPV4;
1129                 break;
1130         }
1131
1132         for (i = 0; i < 4; i++) {
1133                 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1134                 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1135         }
1136
1137         if (qp_params->vlan_id >= VLAN_CFI_MASK)
1138                 qp_params->vlan_id = 0;
1139
1140         return 0;
1141 }
1142
1143 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1144                                struct ib_qp_init_attr *attrs)
1145 {
1146         struct qedr_device_attr *qattr = &dev->attr;
1147
1148         /* QP0... attrs->qp_type == IB_QPT_GSI */
1149         if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1150                 DP_DEBUG(dev, QEDR_MSG_QP,
1151                          "create qp: unsupported qp type=0x%x requested\n",
1152                          attrs->qp_type);
1153                 return -EINVAL;
1154         }
1155
1156         if (attrs->cap.max_send_wr > qattr->max_sqe) {
1157                 DP_ERR(dev,
1158                        "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1159                        attrs->cap.max_send_wr, qattr->max_sqe);
1160                 return -EINVAL;
1161         }
1162
1163         if (attrs->cap.max_inline_data > qattr->max_inline) {
1164                 DP_ERR(dev,
1165                        "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1166                        attrs->cap.max_inline_data, qattr->max_inline);
1167                 return -EINVAL;
1168         }
1169
1170         if (attrs->cap.max_send_sge > qattr->max_sge) {
1171                 DP_ERR(dev,
1172                        "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1173                        attrs->cap.max_send_sge, qattr->max_sge);
1174                 return -EINVAL;
1175         }
1176
1177         if (attrs->cap.max_recv_sge > qattr->max_sge) {
1178                 DP_ERR(dev,
1179                        "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1180                        attrs->cap.max_recv_sge, qattr->max_sge);
1181                 return -EINVAL;
1182         }
1183
1184         /* Unprivileged user space cannot create special QP */
1185         if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1186                 DP_ERR(dev,
1187                        "create qp: userspace can't create special QPs of type=0x%x\n",
1188                        attrs->qp_type);
1189                 return -EINVAL;
1190         }
1191
1192         return 0;
1193 }
1194
1195 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1196                                struct qedr_srq *srq, struct ib_udata *udata)
1197 {
1198         struct qedr_create_srq_uresp uresp = {};
1199         int rc;
1200
1201         uresp.srq_id = srq->srq_id;
1202
1203         rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1204         if (rc)
1205                 DP_ERR(dev, "create srq: problem copying data to user space\n");
1206
1207         return rc;
1208 }
1209
1210 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1211                                struct qedr_create_qp_uresp *uresp,
1212                                struct qedr_qp *qp)
1213 {
1214         /* iWARP requires two doorbells per RQ. */
1215         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1216                 uresp->rq_db_offset =
1217                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1218                 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1219         } else {
1220                 uresp->rq_db_offset =
1221                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1222         }
1223
1224         uresp->rq_icid = qp->icid;
1225 }
1226
1227 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1228                                struct qedr_create_qp_uresp *uresp,
1229                                struct qedr_qp *qp)
1230 {
1231         uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1232
1233         /* iWARP uses the same cid for rq and sq */
1234         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1235                 uresp->sq_icid = qp->icid;
1236         else
1237                 uresp->sq_icid = qp->icid + 1;
1238 }
1239
1240 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1241                               struct qedr_qp *qp, struct ib_udata *udata)
1242 {
1243         struct qedr_create_qp_uresp uresp;
1244         int rc;
1245
1246         memset(&uresp, 0, sizeof(uresp));
1247         qedr_copy_sq_uresp(dev, &uresp, qp);
1248         qedr_copy_rq_uresp(dev, &uresp, qp);
1249
1250         uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1251         uresp.qp_id = qp->qp_id;
1252
1253         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1254         if (rc)
1255                 DP_ERR(dev,
1256                        "create qp: failed a copy to user space with qp icid=0x%x.\n",
1257                        qp->icid);
1258
1259         return rc;
1260 }
1261
1262 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1263                                       struct qedr_qp *qp,
1264                                       struct qedr_pd *pd,
1265                                       struct ib_qp_init_attr *attrs)
1266 {
1267         spin_lock_init(&qp->q_lock);
1268         atomic_set(&qp->refcnt, 1);
1269         qp->pd = pd;
1270         qp->qp_type = attrs->qp_type;
1271         qp->max_inline_data = attrs->cap.max_inline_data;
1272         qp->sq.max_sges = attrs->cap.max_send_sge;
1273         qp->state = QED_ROCE_QP_STATE_RESET;
1274         qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1275         qp->sq_cq = get_qedr_cq(attrs->send_cq);
1276         qp->dev = dev;
1277
1278         if (attrs->srq) {
1279                 qp->srq = get_qedr_srq(attrs->srq);
1280         } else {
1281                 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1282                 qp->rq.max_sges = attrs->cap.max_recv_sge;
1283                 DP_DEBUG(dev, QEDR_MSG_QP,
1284                          "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1285                          qp->rq.max_sges, qp->rq_cq->icid);
1286         }
1287
1288         DP_DEBUG(dev, QEDR_MSG_QP,
1289                  "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1290                  pd->pd_id, qp->qp_type, qp->max_inline_data,
1291                  qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1292         DP_DEBUG(dev, QEDR_MSG_QP,
1293                  "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1294                  qp->sq.max_sges, qp->sq_cq->icid);
1295 }
1296
1297 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1298 {
1299         qp->sq.db = dev->db_addr +
1300                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1301         qp->sq.db_data.data.icid = qp->icid + 1;
1302         if (!qp->srq) {
1303                 qp->rq.db = dev->db_addr +
1304                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1305                 qp->rq.db_data.data.icid = qp->icid;
1306         }
1307 }
1308
1309 static int qedr_check_srq_params(struct ib_pd *ibpd, struct qedr_dev *dev,
1310                                  struct ib_srq_init_attr *attrs,
1311                                  struct ib_udata *udata)
1312 {
1313         struct qedr_device_attr *qattr = &dev->attr;
1314
1315         if (attrs->attr.max_wr > qattr->max_srq_wr) {
1316                 DP_ERR(dev,
1317                        "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1318                        attrs->attr.max_wr, qattr->max_srq_wr);
1319                 return -EINVAL;
1320         }
1321
1322         if (attrs->attr.max_sge > qattr->max_sge) {
1323                 DP_ERR(dev,
1324                        "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1325                        attrs->attr.max_sge, qattr->max_sge);
1326                 return -EINVAL;
1327         }
1328
1329         return 0;
1330 }
1331
1332 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1333 {
1334         qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1335         ib_umem_release(srq->usrq.umem);
1336         ib_umem_release(srq->prod_umem);
1337 }
1338
1339 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1340 {
1341         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1342         struct qedr_dev *dev = srq->dev;
1343
1344         dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1345
1346         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1347                           hw_srq->virt_prod_pair_addr,
1348                           hw_srq->phy_prod_pair_addr);
1349 }
1350
1351 static int qedr_init_srq_user_params(struct ib_ucontext *ib_ctx,
1352                                      struct qedr_srq *srq,
1353                                      struct qedr_create_srq_ureq *ureq,
1354                                      int access, int dmasync)
1355 {
1356         struct scatterlist *sg;
1357         int rc;
1358
1359         rc = qedr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr,
1360                                   ureq->srq_len, access, dmasync, 1);
1361         if (rc)
1362                 return rc;
1363
1364         srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr,
1365                                      sizeof(struct rdma_srq_producers),
1366                                      access, dmasync);
1367         if (IS_ERR(srq->prod_umem)) {
1368                 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1369                 ib_umem_release(srq->usrq.umem);
1370                 DP_ERR(srq->dev,
1371                        "create srq: failed ib_umem_get for producer, got %ld\n",
1372                        PTR_ERR(srq->prod_umem));
1373                 return PTR_ERR(srq->prod_umem);
1374         }
1375
1376         sg = srq->prod_umem->sg_head.sgl;
1377         srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1378
1379         return 0;
1380 }
1381
1382 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1383                                         struct qedr_dev *dev,
1384                                         struct ib_srq_init_attr *init_attr)
1385 {
1386         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1387         dma_addr_t phy_prod_pair_addr;
1388         u32 num_elems;
1389         void *va;
1390         int rc;
1391
1392         va = dma_alloc_coherent(&dev->pdev->dev,
1393                                 sizeof(struct rdma_srq_producers),
1394                                 &phy_prod_pair_addr, GFP_KERNEL);
1395         if (!va) {
1396                 DP_ERR(dev,
1397                        "create srq: failed to allocate dma memory for producer\n");
1398                 return -ENOMEM;
1399         }
1400
1401         hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1402         hw_srq->virt_prod_pair_addr = va;
1403
1404         num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1405         rc = dev->ops->common->chain_alloc(dev->cdev,
1406                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1407                                            QED_CHAIN_MODE_PBL,
1408                                            QED_CHAIN_CNT_TYPE_U32,
1409                                            num_elems,
1410                                            QEDR_SRQ_WQE_ELEM_SIZE,
1411                                            &hw_srq->pbl, NULL);
1412         if (rc)
1413                 goto err0;
1414
1415         hw_srq->num_elems = num_elems;
1416
1417         return 0;
1418
1419 err0:
1420         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1421                           va, phy_prod_pair_addr);
1422         return rc;
1423 }
1424
1425 static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr,
1426                         void *ptr, u32 id);
1427 static void qedr_idr_remove(struct qedr_dev *dev,
1428                             struct qedr_idr *qidr, u32 id);
1429
1430 struct ib_srq *qedr_create_srq(struct ib_pd *ibpd,
1431                                struct ib_srq_init_attr *init_attr,
1432                                struct ib_udata *udata)
1433 {
1434         struct qed_rdma_destroy_srq_in_params destroy_in_params;
1435         struct qed_rdma_create_srq_in_params in_params = {};
1436         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1437         struct qed_rdma_create_srq_out_params out_params;
1438         struct qedr_pd *pd = get_qedr_pd(ibpd);
1439         struct qedr_create_srq_ureq ureq = {};
1440         u64 pbl_base_addr, phy_prod_pair_addr;
1441         struct ib_ucontext *ib_ctx = NULL;
1442         struct qedr_srq_hwq_info *hw_srq;
1443         struct qedr_ucontext *ctx = NULL;
1444         u32 page_cnt, page_size;
1445         struct qedr_srq *srq;
1446         int rc = 0;
1447
1448         DP_DEBUG(dev, QEDR_MSG_QP,
1449                  "create SRQ called from %s (pd %p)\n",
1450                  (udata) ? "User lib" : "kernel", pd);
1451
1452         rc = qedr_check_srq_params(ibpd, dev, init_attr, udata);
1453         if (rc)
1454                 return ERR_PTR(-EINVAL);
1455
1456         srq = kzalloc(sizeof(*srq), GFP_KERNEL);
1457         if (!srq)
1458                 return ERR_PTR(-ENOMEM);
1459
1460         srq->dev = dev;
1461         hw_srq = &srq->hw_srq;
1462         spin_lock_init(&srq->lock);
1463
1464         hw_srq->max_wr = init_attr->attr.max_wr;
1465         hw_srq->max_sges = init_attr->attr.max_sge;
1466
1467         if (udata && ibpd->uobject && ibpd->uobject->context) {
1468                 ib_ctx = ibpd->uobject->context;
1469                 ctx = get_qedr_ucontext(ib_ctx);
1470
1471                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1472                         DP_ERR(dev,
1473                                "create srq: problem copying data from user space\n");
1474                         goto err0;
1475                 }
1476
1477                 rc = qedr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0);
1478                 if (rc)
1479                         goto err0;
1480
1481                 page_cnt = srq->usrq.pbl_info.num_pbes;
1482                 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1483                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1484                 page_size = BIT(srq->usrq.umem->page_shift);
1485         } else {
1486                 struct qed_chain *pbl;
1487
1488                 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1489                 if (rc)
1490                         goto err0;
1491
1492                 pbl = &hw_srq->pbl;
1493                 page_cnt = qed_chain_get_page_cnt(pbl);
1494                 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1495                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1496                 page_size = QED_CHAIN_PAGE_SIZE;
1497         }
1498
1499         in_params.pd_id = pd->pd_id;
1500         in_params.pbl_base_addr = pbl_base_addr;
1501         in_params.prod_pair_addr = phy_prod_pair_addr;
1502         in_params.num_pages = page_cnt;
1503         in_params.page_size = page_size;
1504
1505         rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1506         if (rc)
1507                 goto err1;
1508
1509         srq->srq_id = out_params.srq_id;
1510
1511         if (udata) {
1512                 rc = qedr_copy_srq_uresp(dev, srq, udata);
1513                 if (rc)
1514                         goto err2;
1515         }
1516
1517         rc = qedr_idr_add(dev, &dev->srqidr, srq, srq->srq_id);
1518         if (rc)
1519                 goto err2;
1520
1521         DP_DEBUG(dev, QEDR_MSG_SRQ,
1522                  "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1523         return &srq->ibsrq;
1524
1525 err2:
1526         destroy_in_params.srq_id = srq->srq_id;
1527
1528         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1529 err1:
1530         if (udata)
1531                 qedr_free_srq_user_params(srq);
1532         else
1533                 qedr_free_srq_kernel_params(srq);
1534 err0:
1535         kfree(srq);
1536
1537         return ERR_PTR(-EFAULT);
1538 }
1539
1540 int qedr_destroy_srq(struct ib_srq *ibsrq)
1541 {
1542         struct qed_rdma_destroy_srq_in_params in_params = {};
1543         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1544         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1545
1546         qedr_idr_remove(dev, &dev->srqidr, srq->srq_id);
1547         in_params.srq_id = srq->srq_id;
1548         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1549
1550         if (ibsrq->pd->uobject)
1551                 qedr_free_srq_user_params(srq);
1552         else
1553                 qedr_free_srq_kernel_params(srq);
1554
1555         DP_DEBUG(dev, QEDR_MSG_SRQ,
1556                  "destroy srq: destroyed srq with srq_id=0x%0x\n",
1557                  srq->srq_id);
1558         kfree(srq);
1559
1560         return 0;
1561 }
1562
1563 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1564                     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1565 {
1566         struct qed_rdma_modify_srq_in_params in_params = {};
1567         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1568         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1569         int rc;
1570
1571         if (attr_mask & IB_SRQ_MAX_WR) {
1572                 DP_ERR(dev,
1573                        "modify srq: invalid attribute mask=0x%x specified for %p\n",
1574                        attr_mask, srq);
1575                 return -EINVAL;
1576         }
1577
1578         if (attr_mask & IB_SRQ_LIMIT) {
1579                 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1580                         DP_ERR(dev,
1581                                "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1582                                attr->srq_limit, srq->hw_srq.max_wr);
1583                         return -EINVAL;
1584                 }
1585
1586                 in_params.srq_id = srq->srq_id;
1587                 in_params.wqe_limit = attr->srq_limit;
1588                 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1589                 if (rc)
1590                         return rc;
1591         }
1592
1593         srq->srq_limit = attr->srq_limit;
1594
1595         DP_DEBUG(dev, QEDR_MSG_SRQ,
1596                  "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1597
1598         return 0;
1599 }
1600
1601 static inline void
1602 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1603                               struct qedr_pd *pd,
1604                               struct qedr_qp *qp,
1605                               struct ib_qp_init_attr *attrs,
1606                               bool fmr_and_reserved_lkey,
1607                               struct qed_rdma_create_qp_in_params *params)
1608 {
1609         /* QP handle to be written in an async event */
1610         params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1611         params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1612
1613         params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1614         params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1615         params->pd = pd->pd_id;
1616         params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1617         params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1618         params->stats_queue = 0;
1619         params->srq_id = 0;
1620         params->use_srq = false;
1621
1622         if (!qp->srq) {
1623                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1624
1625         } else {
1626                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1627                 params->srq_id = qp->srq->srq_id;
1628                 params->use_srq = true;
1629         }
1630 }
1631
1632 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1633 {
1634         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1635                  "qp=%p. "
1636                  "sq_addr=0x%llx, "
1637                  "sq_len=%zd, "
1638                  "rq_addr=0x%llx, "
1639                  "rq_len=%zd"
1640                  "\n",
1641                  qp,
1642                  qp->usq.buf_addr,
1643                  qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1644 }
1645
1646 static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr,
1647                         void *ptr, u32 id)
1648 {
1649         int rc;
1650
1651         idr_preload(GFP_KERNEL);
1652         spin_lock_irq(&qidr->idr_lock);
1653
1654         rc = idr_alloc(&qidr->idr, ptr, id, id + 1, GFP_ATOMIC);
1655
1656         spin_unlock_irq(&qidr->idr_lock);
1657         idr_preload_end();
1658
1659         return rc < 0 ? rc : 0;
1660 }
1661
1662 static void qedr_idr_remove(struct qedr_dev *dev, struct qedr_idr *qidr, u32 id)
1663 {
1664         spin_lock_irq(&qidr->idr_lock);
1665         idr_remove(&qidr->idr, id);
1666         spin_unlock_irq(&qidr->idr_lock);
1667 }
1668
1669 static inline void
1670 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1671                             struct qedr_qp *qp,
1672                             struct qed_rdma_create_qp_out_params *out_params)
1673 {
1674         qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1675         qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1676
1677         qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1678                            &qp->usq.pbl_info, FW_PAGE_SHIFT);
1679         if (!qp->srq) {
1680                 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1681                 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1682         }
1683
1684         qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1685                            &qp->urq.pbl_info, FW_PAGE_SHIFT);
1686 }
1687
1688 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1689 {
1690         if (qp->usq.umem)
1691                 ib_umem_release(qp->usq.umem);
1692         qp->usq.umem = NULL;
1693
1694         if (qp->urq.umem)
1695                 ib_umem_release(qp->urq.umem);
1696         qp->urq.umem = NULL;
1697
1698         if (rdma_protocol_roce(&dev->ibdev, 1)) {
1699                 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1700                 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1701         } else {
1702                 kfree(qp->usq.pbl_tbl);
1703                 kfree(qp->urq.pbl_tbl);
1704         }
1705 }
1706
1707 static int qedr_create_user_qp(struct qedr_dev *dev,
1708                                struct qedr_qp *qp,
1709                                struct ib_pd *ibpd,
1710                                struct ib_udata *udata,
1711                                struct ib_qp_init_attr *attrs)
1712 {
1713         struct qed_rdma_create_qp_in_params in_params;
1714         struct qed_rdma_create_qp_out_params out_params;
1715         struct qedr_pd *pd = get_qedr_pd(ibpd);
1716         struct ib_ucontext *ib_ctx = NULL;
1717         struct qedr_create_qp_ureq ureq;
1718         int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1719         int rc = -EINVAL;
1720
1721         ib_ctx = ibpd->uobject->context;
1722
1723         memset(&ureq, 0, sizeof(ureq));
1724         rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1725         if (rc) {
1726                 DP_ERR(dev, "Problem copying data from user space\n");
1727                 return rc;
1728         }
1729
1730         /* SQ - read access only (0), dma sync not required (0) */
1731         rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
1732                                   ureq.sq_len, 0, 0, alloc_and_init);
1733         if (rc)
1734                 return rc;
1735
1736         if (!qp->srq) {
1737                 /* RQ - read access only (0), dma sync not required (0) */
1738                 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
1739                                           ureq.rq_len, 0, 0, alloc_and_init);
1740                 if (rc)
1741                         return rc;
1742         }
1743
1744         memset(&in_params, 0, sizeof(in_params));
1745         qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1746         in_params.qp_handle_lo = ureq.qp_handle_lo;
1747         in_params.qp_handle_hi = ureq.qp_handle_hi;
1748         in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1749         in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1750         if (!qp->srq) {
1751                 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1752                 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1753         }
1754
1755         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1756                                               &in_params, &out_params);
1757
1758         if (!qp->qed_qp) {
1759                 rc = -ENOMEM;
1760                 goto err1;
1761         }
1762
1763         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1764                 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1765
1766         qp->qp_id = out_params.qp_id;
1767         qp->icid = out_params.icid;
1768
1769         rc = qedr_copy_qp_uresp(dev, qp, udata);
1770         if (rc)
1771                 goto err;
1772
1773         qedr_qp_user_print(dev, qp);
1774
1775         return 0;
1776 err:
1777         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1778         if (rc)
1779                 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1780
1781 err1:
1782         qedr_cleanup_user(dev, qp);
1783         return rc;
1784 }
1785
1786 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1787 {
1788         qp->sq.db = dev->db_addr +
1789             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1790         qp->sq.db_data.data.icid = qp->icid;
1791
1792         qp->rq.db = dev->db_addr +
1793                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1794         qp->rq.db_data.data.icid = qp->icid;
1795         qp->rq.iwarp_db2 = dev->db_addr +
1796                            DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1797         qp->rq.iwarp_db2_data.data.icid = qp->icid;
1798         qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1799 }
1800
1801 static int
1802 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1803                            struct qedr_qp *qp,
1804                            struct qed_rdma_create_qp_in_params *in_params,
1805                            u32 n_sq_elems, u32 n_rq_elems)
1806 {
1807         struct qed_rdma_create_qp_out_params out_params;
1808         int rc;
1809
1810         rc = dev->ops->common->chain_alloc(dev->cdev,
1811                                            QED_CHAIN_USE_TO_PRODUCE,
1812                                            QED_CHAIN_MODE_PBL,
1813                                            QED_CHAIN_CNT_TYPE_U32,
1814                                            n_sq_elems,
1815                                            QEDR_SQE_ELEMENT_SIZE,
1816                                            &qp->sq.pbl, NULL);
1817
1818         if (rc)
1819                 return rc;
1820
1821         in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1822         in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1823
1824         rc = dev->ops->common->chain_alloc(dev->cdev,
1825                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1826                                            QED_CHAIN_MODE_PBL,
1827                                            QED_CHAIN_CNT_TYPE_U32,
1828                                            n_rq_elems,
1829                                            QEDR_RQE_ELEMENT_SIZE,
1830                                            &qp->rq.pbl, NULL);
1831         if (rc)
1832                 return rc;
1833
1834         in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1835         in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1836
1837         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1838                                               in_params, &out_params);
1839
1840         if (!qp->qed_qp)
1841                 return -EINVAL;
1842
1843         qp->qp_id = out_params.qp_id;
1844         qp->icid = out_params.icid;
1845
1846         qedr_set_roce_db_info(dev, qp);
1847         return rc;
1848 }
1849
1850 static int
1851 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1852                             struct qedr_qp *qp,
1853                             struct qed_rdma_create_qp_in_params *in_params,
1854                             u32 n_sq_elems, u32 n_rq_elems)
1855 {
1856         struct qed_rdma_create_qp_out_params out_params;
1857         struct qed_chain_ext_pbl ext_pbl;
1858         int rc;
1859
1860         in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1861                                                      QEDR_SQE_ELEMENT_SIZE,
1862                                                      QED_CHAIN_MODE_PBL);
1863         in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1864                                                      QEDR_RQE_ELEMENT_SIZE,
1865                                                      QED_CHAIN_MODE_PBL);
1866
1867         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1868                                               in_params, &out_params);
1869
1870         if (!qp->qed_qp)
1871                 return -EINVAL;
1872
1873         /* Now we allocate the chain */
1874         ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1875         ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1876
1877         rc = dev->ops->common->chain_alloc(dev->cdev,
1878                                            QED_CHAIN_USE_TO_PRODUCE,
1879                                            QED_CHAIN_MODE_PBL,
1880                                            QED_CHAIN_CNT_TYPE_U32,
1881                                            n_sq_elems,
1882                                            QEDR_SQE_ELEMENT_SIZE,
1883                                            &qp->sq.pbl, &ext_pbl);
1884
1885         if (rc)
1886                 goto err;
1887
1888         ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1889         ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1890
1891         rc = dev->ops->common->chain_alloc(dev->cdev,
1892                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1893                                            QED_CHAIN_MODE_PBL,
1894                                            QED_CHAIN_CNT_TYPE_U32,
1895                                            n_rq_elems,
1896                                            QEDR_RQE_ELEMENT_SIZE,
1897                                            &qp->rq.pbl, &ext_pbl);
1898
1899         if (rc)
1900                 goto err;
1901
1902         qp->qp_id = out_params.qp_id;
1903         qp->icid = out_params.icid;
1904
1905         qedr_set_iwarp_db_info(dev, qp);
1906         return rc;
1907
1908 err:
1909         dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1910
1911         return rc;
1912 }
1913
1914 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1915 {
1916         dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1917         kfree(qp->wqe_wr_id);
1918
1919         dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1920         kfree(qp->rqe_wr_id);
1921 }
1922
1923 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1924                                  struct qedr_qp *qp,
1925                                  struct ib_pd *ibpd,
1926                                  struct ib_qp_init_attr *attrs)
1927 {
1928         struct qed_rdma_create_qp_in_params in_params;
1929         struct qedr_pd *pd = get_qedr_pd(ibpd);
1930         int rc = -EINVAL;
1931         u32 n_rq_elems;
1932         u32 n_sq_elems;
1933         u32 n_sq_entries;
1934
1935         memset(&in_params, 0, sizeof(in_params));
1936
1937         /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1938          * the ring. The ring should allow at least a single WR, even if the
1939          * user requested none, due to allocation issues.
1940          * We should add an extra WR since the prod and cons indices of
1941          * wqe_wr_id are managed in such a way that the WQ is considered full
1942          * when (prod+1)%max_wr==cons. We currently don't do that because we
1943          * double the number of entries due an iSER issue that pushes far more
1944          * WRs than indicated. If we decline its ib_post_send() then we get
1945          * error prints in the dmesg we'd like to avoid.
1946          */
1947         qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1948                               dev->attr.max_sqe);
1949
1950         qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1951                                 GFP_KERNEL);
1952         if (!qp->wqe_wr_id) {
1953                 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1954                 return -ENOMEM;
1955         }
1956
1957         /* QP handle to be written in CQE */
1958         in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1959         in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1960
1961         /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1962          * the ring. There ring should allow at least a single WR, even if the
1963          * user requested none, due to allocation issues.
1964          */
1965         qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1966
1967         /* Allocate driver internal RQ array */
1968         qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1969                                 GFP_KERNEL);
1970         if (!qp->rqe_wr_id) {
1971                 DP_ERR(dev,
1972                        "create qp: failed RQ shadow memory allocation\n");
1973                 kfree(qp->wqe_wr_id);
1974                 return -ENOMEM;
1975         }
1976
1977         qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1978
1979         n_sq_entries = attrs->cap.max_send_wr;
1980         n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1981         n_sq_entries = max_t(u32, n_sq_entries, 1);
1982         n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1983
1984         n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1985
1986         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1987                 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1988                                                  n_sq_elems, n_rq_elems);
1989         else
1990                 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1991                                                 n_sq_elems, n_rq_elems);
1992         if (rc)
1993                 qedr_cleanup_kernel(dev, qp);
1994
1995         return rc;
1996 }
1997
1998 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1999                              struct ib_qp_init_attr *attrs,
2000                              struct ib_udata *udata)
2001 {
2002         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2003         struct qedr_pd *pd = get_qedr_pd(ibpd);
2004         struct qedr_qp *qp;
2005         struct ib_qp *ibqp;
2006         int rc = 0;
2007
2008         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2009                  udata ? "user library" : "kernel", pd);
2010
2011         rc = qedr_check_qp_attrs(ibpd, dev, attrs);
2012         if (rc)
2013                 return ERR_PTR(rc);
2014
2015         DP_DEBUG(dev, QEDR_MSG_QP,
2016                  "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2017                  udata ? "user library" : "kernel", attrs->event_handler, pd,
2018                  get_qedr_cq(attrs->send_cq),
2019                  get_qedr_cq(attrs->send_cq)->icid,
2020                  get_qedr_cq(attrs->recv_cq),
2021                  attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2022
2023         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2024         if (!qp) {
2025                 DP_ERR(dev, "create qp: failed allocating memory\n");
2026                 return ERR_PTR(-ENOMEM);
2027         }
2028
2029         qedr_set_common_qp_params(dev, qp, pd, attrs);
2030
2031         if (attrs->qp_type == IB_QPT_GSI) {
2032                 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
2033                 if (IS_ERR(ibqp))
2034                         kfree(qp);
2035                 return ibqp;
2036         }
2037
2038         if (udata)
2039                 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2040         else
2041                 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2042
2043         if (rc)
2044                 goto err;
2045
2046         qp->ibqp.qp_num = qp->qp_id;
2047
2048         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2049                 rc = qedr_idr_add(dev, &dev->qpidr, qp, qp->qp_id);
2050                 if (rc)
2051                         goto err;
2052         }
2053
2054         return &qp->ibqp;
2055
2056 err:
2057         kfree(qp);
2058
2059         return ERR_PTR(-EFAULT);
2060 }
2061
2062 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2063 {
2064         switch (qp_state) {
2065         case QED_ROCE_QP_STATE_RESET:
2066                 return IB_QPS_RESET;
2067         case QED_ROCE_QP_STATE_INIT:
2068                 return IB_QPS_INIT;
2069         case QED_ROCE_QP_STATE_RTR:
2070                 return IB_QPS_RTR;
2071         case QED_ROCE_QP_STATE_RTS:
2072                 return IB_QPS_RTS;
2073         case QED_ROCE_QP_STATE_SQD:
2074                 return IB_QPS_SQD;
2075         case QED_ROCE_QP_STATE_ERR:
2076                 return IB_QPS_ERR;
2077         case QED_ROCE_QP_STATE_SQE:
2078                 return IB_QPS_SQE;
2079         }
2080         return IB_QPS_ERR;
2081 }
2082
2083 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2084                                         enum ib_qp_state qp_state)
2085 {
2086         switch (qp_state) {
2087         case IB_QPS_RESET:
2088                 return QED_ROCE_QP_STATE_RESET;
2089         case IB_QPS_INIT:
2090                 return QED_ROCE_QP_STATE_INIT;
2091         case IB_QPS_RTR:
2092                 return QED_ROCE_QP_STATE_RTR;
2093         case IB_QPS_RTS:
2094                 return QED_ROCE_QP_STATE_RTS;
2095         case IB_QPS_SQD:
2096                 return QED_ROCE_QP_STATE_SQD;
2097         case IB_QPS_ERR:
2098                 return QED_ROCE_QP_STATE_ERR;
2099         default:
2100                 return QED_ROCE_QP_STATE_ERR;
2101         }
2102 }
2103
2104 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2105 {
2106         qed_chain_reset(&qph->pbl);
2107         qph->prod = 0;
2108         qph->cons = 0;
2109         qph->wqe_cons = 0;
2110         qph->db_data.data.value = cpu_to_le16(0);
2111 }
2112
2113 static int qedr_update_qp_state(struct qedr_dev *dev,
2114                                 struct qedr_qp *qp,
2115                                 enum qed_roce_qp_state cur_state,
2116                                 enum qed_roce_qp_state new_state)
2117 {
2118         int status = 0;
2119
2120         if (new_state == cur_state)
2121                 return 0;
2122
2123         switch (cur_state) {
2124         case QED_ROCE_QP_STATE_RESET:
2125                 switch (new_state) {
2126                 case QED_ROCE_QP_STATE_INIT:
2127                         qp->prev_wqe_size = 0;
2128                         qedr_reset_qp_hwq_info(&qp->sq);
2129                         qedr_reset_qp_hwq_info(&qp->rq);
2130                         break;
2131                 default:
2132                         status = -EINVAL;
2133                         break;
2134                 };
2135                 break;
2136         case QED_ROCE_QP_STATE_INIT:
2137                 switch (new_state) {
2138                 case QED_ROCE_QP_STATE_RTR:
2139                         /* Update doorbell (in case post_recv was
2140                          * done before move to RTR)
2141                          */
2142
2143                         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2144                                 writel(qp->rq.db_data.raw, qp->rq.db);
2145                                 /* Make sure write takes effect */
2146                                 mmiowb();
2147                         }
2148                         break;
2149                 case QED_ROCE_QP_STATE_ERR:
2150                         break;
2151                 default:
2152                         /* Invalid state change. */
2153                         status = -EINVAL;
2154                         break;
2155                 };
2156                 break;
2157         case QED_ROCE_QP_STATE_RTR:
2158                 /* RTR->XXX */
2159                 switch (new_state) {
2160                 case QED_ROCE_QP_STATE_RTS:
2161                         break;
2162                 case QED_ROCE_QP_STATE_ERR:
2163                         break;
2164                 default:
2165                         /* Invalid state change. */
2166                         status = -EINVAL;
2167                         break;
2168                 };
2169                 break;
2170         case QED_ROCE_QP_STATE_RTS:
2171                 /* RTS->XXX */
2172                 switch (new_state) {
2173                 case QED_ROCE_QP_STATE_SQD:
2174                         break;
2175                 case QED_ROCE_QP_STATE_ERR:
2176                         break;
2177                 default:
2178                         /* Invalid state change. */
2179                         status = -EINVAL;
2180                         break;
2181                 };
2182                 break;
2183         case QED_ROCE_QP_STATE_SQD:
2184                 /* SQD->XXX */
2185                 switch (new_state) {
2186                 case QED_ROCE_QP_STATE_RTS:
2187                 case QED_ROCE_QP_STATE_ERR:
2188                         break;
2189                 default:
2190                         /* Invalid state change. */
2191                         status = -EINVAL;
2192                         break;
2193                 };
2194                 break;
2195         case QED_ROCE_QP_STATE_ERR:
2196                 /* ERR->XXX */
2197                 switch (new_state) {
2198                 case QED_ROCE_QP_STATE_RESET:
2199                         if ((qp->rq.prod != qp->rq.cons) ||
2200                             (qp->sq.prod != qp->sq.cons)) {
2201                                 DP_NOTICE(dev,
2202                                           "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2203                                           qp->rq.prod, qp->rq.cons, qp->sq.prod,
2204                                           qp->sq.cons);
2205                                 status = -EINVAL;
2206                         }
2207                         break;
2208                 default:
2209                         status = -EINVAL;
2210                         break;
2211                 };
2212                 break;
2213         default:
2214                 status = -EINVAL;
2215                 break;
2216         };
2217
2218         return status;
2219 }
2220
2221 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2222                    int attr_mask, struct ib_udata *udata)
2223 {
2224         struct qedr_qp *qp = get_qedr_qp(ibqp);
2225         struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2226         struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2227         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2228         enum ib_qp_state old_qp_state, new_qp_state;
2229         enum qed_roce_qp_state cur_state;
2230         int rc = 0;
2231
2232         DP_DEBUG(dev, QEDR_MSG_QP,
2233                  "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2234                  attr->qp_state);
2235
2236         old_qp_state = qedr_get_ibqp_state(qp->state);
2237         if (attr_mask & IB_QP_STATE)
2238                 new_qp_state = attr->qp_state;
2239         else
2240                 new_qp_state = old_qp_state;
2241
2242         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2243                 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2244                                         ibqp->qp_type, attr_mask,
2245                                         IB_LINK_LAYER_ETHERNET)) {
2246                         DP_ERR(dev,
2247                                "modify qp: invalid attribute mask=0x%x specified for\n"
2248                                "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2249                                attr_mask, qp->qp_id, ibqp->qp_type,
2250                                old_qp_state, new_qp_state);
2251                         rc = -EINVAL;
2252                         goto err;
2253                 }
2254         }
2255
2256         /* Translate the masks... */
2257         if (attr_mask & IB_QP_STATE) {
2258                 SET_FIELD(qp_params.modify_flags,
2259                           QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2260                 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2261         }
2262
2263         if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2264                 qp_params.sqd_async = true;
2265
2266         if (attr_mask & IB_QP_PKEY_INDEX) {
2267                 SET_FIELD(qp_params.modify_flags,
2268                           QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2269                 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2270                         rc = -EINVAL;
2271                         goto err;
2272                 }
2273
2274                 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2275         }
2276
2277         if (attr_mask & IB_QP_QKEY)
2278                 qp->qkey = attr->qkey;
2279
2280         if (attr_mask & IB_QP_ACCESS_FLAGS) {
2281                 SET_FIELD(qp_params.modify_flags,
2282                           QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2283                 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2284                                                   IB_ACCESS_REMOTE_READ;
2285                 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2286                                                    IB_ACCESS_REMOTE_WRITE;
2287                 qp_params.incoming_atomic_en = attr->qp_access_flags &
2288                                                IB_ACCESS_REMOTE_ATOMIC;
2289         }
2290
2291         if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2292                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2293                         return -EINVAL;
2294
2295                 if (attr_mask & IB_QP_PATH_MTU) {
2296                         if (attr->path_mtu < IB_MTU_256 ||
2297                             attr->path_mtu > IB_MTU_4096) {
2298                                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2299                                 rc = -EINVAL;
2300                                 goto err;
2301                         }
2302                         qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2303                                       ib_mtu_enum_to_int(iboe_get_mtu
2304                                                          (dev->ndev->mtu)));
2305                 }
2306
2307                 if (!qp->mtu) {
2308                         qp->mtu =
2309                         ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2310                         pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2311                 }
2312
2313                 SET_FIELD(qp_params.modify_flags,
2314                           QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2315
2316                 qp_params.traffic_class_tos = grh->traffic_class;
2317                 qp_params.flow_label = grh->flow_label;
2318                 qp_params.hop_limit_ttl = grh->hop_limit;
2319
2320                 qp->sgid_idx = grh->sgid_index;
2321
2322                 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2323                 if (rc) {
2324                         DP_ERR(dev,
2325                                "modify qp: problems with GID index %d (rc=%d)\n",
2326                                grh->sgid_index, rc);
2327                         return rc;
2328                 }
2329
2330                 rc = qedr_get_dmac(dev, &attr->ah_attr,
2331                                    qp_params.remote_mac_addr);
2332                 if (rc)
2333                         return rc;
2334
2335                 qp_params.use_local_mac = true;
2336                 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2337
2338                 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2339                          qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2340                          qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2341                 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2342                          qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2343                          qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2344                 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2345                          qp_params.remote_mac_addr);
2346
2347                 qp_params.mtu = qp->mtu;
2348                 qp_params.lb_indication = false;
2349         }
2350
2351         if (!qp_params.mtu) {
2352                 /* Stay with current MTU */
2353                 if (qp->mtu)
2354                         qp_params.mtu = qp->mtu;
2355                 else
2356                         qp_params.mtu =
2357                             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2358         }
2359
2360         if (attr_mask & IB_QP_TIMEOUT) {
2361                 SET_FIELD(qp_params.modify_flags,
2362                           QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2363
2364                 /* The received timeout value is an exponent used like this:
2365                  *    "12.7.34 LOCAL ACK TIMEOUT
2366                  *    Value representing the transport (ACK) timeout for use by
2367                  *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2368                  * The FW expects timeout in msec so we need to divide the usec
2369                  * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2370                  * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2371                  * The value of zero means infinite so we use a 'max_t' to make
2372                  * sure that sub 1 msec values will be configured as 1 msec.
2373                  */
2374                 if (attr->timeout)
2375                         qp_params.ack_timeout =
2376                                         1 << max_t(int, attr->timeout - 8, 0);
2377                 else
2378                         qp_params.ack_timeout = 0;
2379         }
2380
2381         if (attr_mask & IB_QP_RETRY_CNT) {
2382                 SET_FIELD(qp_params.modify_flags,
2383                           QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2384                 qp_params.retry_cnt = attr->retry_cnt;
2385         }
2386
2387         if (attr_mask & IB_QP_RNR_RETRY) {
2388                 SET_FIELD(qp_params.modify_flags,
2389                           QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2390                 qp_params.rnr_retry_cnt = attr->rnr_retry;
2391         }
2392
2393         if (attr_mask & IB_QP_RQ_PSN) {
2394                 SET_FIELD(qp_params.modify_flags,
2395                           QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2396                 qp_params.rq_psn = attr->rq_psn;
2397                 qp->rq_psn = attr->rq_psn;
2398         }
2399
2400         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2401                 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2402                         rc = -EINVAL;
2403                         DP_ERR(dev,
2404                                "unsupported max_rd_atomic=%d, supported=%d\n",
2405                                attr->max_rd_atomic,
2406                                dev->attr.max_qp_req_rd_atomic_resc);
2407                         goto err;
2408                 }
2409
2410                 SET_FIELD(qp_params.modify_flags,
2411                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2412                 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2413         }
2414
2415         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2416                 SET_FIELD(qp_params.modify_flags,
2417                           QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2418                 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2419         }
2420
2421         if (attr_mask & IB_QP_SQ_PSN) {
2422                 SET_FIELD(qp_params.modify_flags,
2423                           QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2424                 qp_params.sq_psn = attr->sq_psn;
2425                 qp->sq_psn = attr->sq_psn;
2426         }
2427
2428         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2429                 if (attr->max_dest_rd_atomic >
2430                     dev->attr.max_qp_resp_rd_atomic_resc) {
2431                         DP_ERR(dev,
2432                                "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2433                                attr->max_dest_rd_atomic,
2434                                dev->attr.max_qp_resp_rd_atomic_resc);
2435
2436                         rc = -EINVAL;
2437                         goto err;
2438                 }
2439
2440                 SET_FIELD(qp_params.modify_flags,
2441                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2442                 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2443         }
2444
2445         if (attr_mask & IB_QP_DEST_QPN) {
2446                 SET_FIELD(qp_params.modify_flags,
2447                           QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2448
2449                 qp_params.dest_qp = attr->dest_qp_num;
2450                 qp->dest_qp_num = attr->dest_qp_num;
2451         }
2452
2453         cur_state = qp->state;
2454
2455         /* Update the QP state before the actual ramrod to prevent a race with
2456          * fast path. Modifying the QP state to error will cause the device to
2457          * flush the CQEs and while polling the flushed CQEs will considered as
2458          * a potential issue if the QP isn't in error state.
2459          */
2460         if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2461             !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2462                 qp->state = QED_ROCE_QP_STATE_ERR;
2463
2464         if (qp->qp_type != IB_QPT_GSI)
2465                 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2466                                               qp->qed_qp, &qp_params);
2467
2468         if (attr_mask & IB_QP_STATE) {
2469                 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2470                         rc = qedr_update_qp_state(dev, qp, cur_state,
2471                                                   qp_params.new_state);
2472                 qp->state = qp_params.new_state;
2473         }
2474
2475 err:
2476         return rc;
2477 }
2478
2479 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2480 {
2481         int ib_qp_acc_flags = 0;
2482
2483         if (params->incoming_rdma_write_en)
2484                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2485         if (params->incoming_rdma_read_en)
2486                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2487         if (params->incoming_atomic_en)
2488                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2489         ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2490         return ib_qp_acc_flags;
2491 }
2492
2493 int qedr_query_qp(struct ib_qp *ibqp,
2494                   struct ib_qp_attr *qp_attr,
2495                   int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2496 {
2497         struct qed_rdma_query_qp_out_params params;
2498         struct qedr_qp *qp = get_qedr_qp(ibqp);
2499         struct qedr_dev *dev = qp->dev;
2500         int rc = 0;
2501
2502         memset(&params, 0, sizeof(params));
2503         memset(qp_attr, 0, sizeof(*qp_attr));
2504         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2505
2506         if (qp->qp_type != IB_QPT_GSI) {
2507                 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2508                 if (rc)
2509                         goto err;
2510                 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2511         } else {
2512                 qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS);
2513         }
2514
2515         qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2516         qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2517         qp_attr->path_mig_state = IB_MIG_MIGRATED;
2518         qp_attr->rq_psn = params.rq_psn;
2519         qp_attr->sq_psn = params.sq_psn;
2520         qp_attr->dest_qp_num = params.dest_qp;
2521
2522         qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2523
2524         qp_attr->cap.max_send_wr = qp->sq.max_wr;
2525         qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2526         qp_attr->cap.max_send_sge = qp->sq.max_sges;
2527         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2528         qp_attr->cap.max_inline_data = dev->attr.max_inline;
2529         qp_init_attr->cap = qp_attr->cap;
2530
2531         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2532         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2533                         params.flow_label, qp->sgid_idx,
2534                         params.hop_limit_ttl, params.traffic_class_tos);
2535         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2536         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2537         rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2538         qp_attr->timeout = params.timeout;
2539         qp_attr->rnr_retry = params.rnr_retry;
2540         qp_attr->retry_cnt = params.retry_cnt;
2541         qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2542         qp_attr->pkey_index = params.pkey_index;
2543         qp_attr->port_num = 1;
2544         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2545         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2546         qp_attr->alt_pkey_index = 0;
2547         qp_attr->alt_port_num = 0;
2548         qp_attr->alt_timeout = 0;
2549         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2550
2551         qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2552         qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2553         qp_attr->max_rd_atomic = params.max_rd_atomic;
2554         qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2555
2556         DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2557                  qp_attr->cap.max_inline_data);
2558
2559 err:
2560         return rc;
2561 }
2562
2563 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2564 {
2565         int rc = 0;
2566
2567         if (qp->qp_type != IB_QPT_GSI) {
2568                 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2569                 if (rc)
2570                         return rc;
2571         }
2572
2573         if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2574                 qedr_cleanup_user(dev, qp);
2575         else
2576                 qedr_cleanup_kernel(dev, qp);
2577
2578         return 0;
2579 }
2580
2581 int qedr_destroy_qp(struct ib_qp *ibqp)
2582 {
2583         struct qedr_qp *qp = get_qedr_qp(ibqp);
2584         struct qedr_dev *dev = qp->dev;
2585         struct ib_qp_attr attr;
2586         int attr_mask = 0;
2587         int rc = 0;
2588
2589         DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2590                  qp, qp->qp_type);
2591
2592         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2593                 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2594                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
2595                     (qp->state != QED_ROCE_QP_STATE_INIT)) {
2596
2597                         attr.qp_state = IB_QPS_ERR;
2598                         attr_mask |= IB_QP_STATE;
2599
2600                         /* Change the QP state to ERROR */
2601                         qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2602                 }
2603         } else {
2604                 /* Wait for the connect/accept to complete */
2605                 if (qp->ep) {
2606                         int wait_count = 1;
2607
2608                         while (qp->ep->during_connect) {
2609                                 DP_DEBUG(dev, QEDR_MSG_QP,
2610                                          "Still in during connect/accept\n");
2611
2612                                 msleep(100);
2613                                 if (wait_count++ > 200) {
2614                                         DP_NOTICE(dev,
2615                                                   "during connect timeout\n");
2616                                         break;
2617                                 }
2618                         }
2619                 }
2620         }
2621
2622         if (qp->qp_type == IB_QPT_GSI)
2623                 qedr_destroy_gsi_qp(dev);
2624
2625         qedr_free_qp_resources(dev, qp);
2626
2627         if (atomic_dec_and_test(&qp->refcnt) &&
2628             rdma_protocol_iwarp(&dev->ibdev, 1)) {
2629                 qedr_idr_remove(dev, &dev->qpidr, qp->qp_id);
2630                 kfree(qp);
2631         }
2632         return rc;
2633 }
2634
2635 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
2636                              struct ib_udata *udata)
2637 {
2638         struct qedr_ah *ah;
2639
2640         ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2641         if (!ah)
2642                 return ERR_PTR(-ENOMEM);
2643
2644         rdma_copy_ah_attr(&ah->attr, attr);
2645
2646         return &ah->ibah;
2647 }
2648
2649 int qedr_destroy_ah(struct ib_ah *ibah)
2650 {
2651         struct qedr_ah *ah = get_qedr_ah(ibah);
2652
2653         rdma_destroy_ah_attr(&ah->attr);
2654         kfree(ah);
2655         return 0;
2656 }
2657
2658 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2659 {
2660         struct qedr_pbl *pbl, *tmp;
2661
2662         if (info->pbl_table)
2663                 list_add_tail(&info->pbl_table->list_entry,
2664                               &info->free_pbl_list);
2665
2666         if (!list_empty(&info->inuse_pbl_list))
2667                 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2668
2669         list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2670                 list_del(&pbl->list_entry);
2671                 qedr_free_pbl(dev, &info->pbl_info, pbl);
2672         }
2673 }
2674
2675 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2676                         size_t page_list_len, bool two_layered)
2677 {
2678         struct qedr_pbl *tmp;
2679         int rc;
2680
2681         INIT_LIST_HEAD(&info->free_pbl_list);
2682         INIT_LIST_HEAD(&info->inuse_pbl_list);
2683
2684         rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2685                                   page_list_len, two_layered);
2686         if (rc)
2687                 goto done;
2688
2689         info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2690         if (IS_ERR(info->pbl_table)) {
2691                 rc = PTR_ERR(info->pbl_table);
2692                 goto done;
2693         }
2694
2695         DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2696                  &info->pbl_table->pa);
2697
2698         /* in usual case we use 2 PBLs, so we add one to free
2699          * list and allocating another one
2700          */
2701         tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2702         if (IS_ERR(tmp)) {
2703                 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2704                 goto done;
2705         }
2706
2707         list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2708
2709         DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2710
2711 done:
2712         if (rc)
2713                 free_mr_info(dev, info);
2714
2715         return rc;
2716 }
2717
2718 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2719                                u64 usr_addr, int acc, struct ib_udata *udata)
2720 {
2721         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2722         struct qedr_mr *mr;
2723         struct qedr_pd *pd;
2724         int rc = -ENOMEM;
2725
2726         pd = get_qedr_pd(ibpd);
2727         DP_DEBUG(dev, QEDR_MSG_MR,
2728                  "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2729                  pd->pd_id, start, len, usr_addr, acc);
2730
2731         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2732                 return ERR_PTR(-EINVAL);
2733
2734         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2735         if (!mr)
2736                 return ERR_PTR(rc);
2737
2738         mr->type = QEDR_MR_USER;
2739
2740         mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2741         if (IS_ERR(mr->umem)) {
2742                 rc = -EFAULT;
2743                 goto err0;
2744         }
2745
2746         rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2747         if (rc)
2748                 goto err1;
2749
2750         qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2751                            &mr->info.pbl_info, mr->umem->page_shift);
2752
2753         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2754         if (rc) {
2755                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2756                 goto err1;
2757         }
2758
2759         /* Index only, 18 bit long, lkey = itid << 8 | key */
2760         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2761         mr->hw_mr.key = 0;
2762         mr->hw_mr.pd = pd->pd_id;
2763         mr->hw_mr.local_read = 1;
2764         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2765         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2766         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2767         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2768         mr->hw_mr.mw_bind = false;
2769         mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2770         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2771         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2772         mr->hw_mr.page_size_log = mr->umem->page_shift;
2773         mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2774         mr->hw_mr.length = len;
2775         mr->hw_mr.vaddr = usr_addr;
2776         mr->hw_mr.zbva = false;
2777         mr->hw_mr.phy_mr = false;
2778         mr->hw_mr.dma_mr = false;
2779
2780         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2781         if (rc) {
2782                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2783                 goto err2;
2784         }
2785
2786         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2787         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2788             mr->hw_mr.remote_atomic)
2789                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2790
2791         DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2792                  mr->ibmr.lkey);
2793         return &mr->ibmr;
2794
2795 err2:
2796         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2797 err1:
2798         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2799 err0:
2800         kfree(mr);
2801         return ERR_PTR(rc);
2802 }
2803
2804 int qedr_dereg_mr(struct ib_mr *ib_mr)
2805 {
2806         struct qedr_mr *mr = get_qedr_mr(ib_mr);
2807         struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2808         int rc = 0;
2809
2810         rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2811         if (rc)
2812                 return rc;
2813
2814         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2815
2816         if (mr->type != QEDR_MR_DMA)
2817                 free_mr_info(dev, &mr->info);
2818
2819         /* it could be user registered memory. */
2820         if (mr->umem)
2821                 ib_umem_release(mr->umem);
2822
2823         kfree(mr);
2824
2825         return rc;
2826 }
2827
2828 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2829                                        int max_page_list_len)
2830 {
2831         struct qedr_pd *pd = get_qedr_pd(ibpd);
2832         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2833         struct qedr_mr *mr;
2834         int rc = -ENOMEM;
2835
2836         DP_DEBUG(dev, QEDR_MSG_MR,
2837                  "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2838                  max_page_list_len);
2839
2840         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2841         if (!mr)
2842                 return ERR_PTR(rc);
2843
2844         mr->dev = dev;
2845         mr->type = QEDR_MR_FRMR;
2846
2847         rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2848         if (rc)
2849                 goto err0;
2850
2851         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2852         if (rc) {
2853                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2854                 goto err0;
2855         }
2856
2857         /* Index only, 18 bit long, lkey = itid << 8 | key */
2858         mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2859         mr->hw_mr.key = 0;
2860         mr->hw_mr.pd = pd->pd_id;
2861         mr->hw_mr.local_read = 1;
2862         mr->hw_mr.local_write = 0;
2863         mr->hw_mr.remote_read = 0;
2864         mr->hw_mr.remote_write = 0;
2865         mr->hw_mr.remote_atomic = 0;
2866         mr->hw_mr.mw_bind = false;
2867         mr->hw_mr.pbl_ptr = 0;
2868         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2869         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2870         mr->hw_mr.fbo = 0;
2871         mr->hw_mr.length = 0;
2872         mr->hw_mr.vaddr = 0;
2873         mr->hw_mr.zbva = false;
2874         mr->hw_mr.phy_mr = true;
2875         mr->hw_mr.dma_mr = false;
2876
2877         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2878         if (rc) {
2879                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2880                 goto err1;
2881         }
2882
2883         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2884         mr->ibmr.rkey = mr->ibmr.lkey;
2885
2886         DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2887         return mr;
2888
2889 err1:
2890         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2891 err0:
2892         kfree(mr);
2893         return ERR_PTR(rc);
2894 }
2895
2896 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2897                             enum ib_mr_type mr_type, u32 max_num_sg)
2898 {
2899         struct qedr_mr *mr;
2900
2901         if (mr_type != IB_MR_TYPE_MEM_REG)
2902                 return ERR_PTR(-EINVAL);
2903
2904         mr = __qedr_alloc_mr(ibpd, max_num_sg);
2905
2906         if (IS_ERR(mr))
2907                 return ERR_PTR(-EINVAL);
2908
2909         return &mr->ibmr;
2910 }
2911
2912 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2913 {
2914         struct qedr_mr *mr = get_qedr_mr(ibmr);
2915         struct qedr_pbl *pbl_table;
2916         struct regpair *pbe;
2917         u32 pbes_in_page;
2918
2919         if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2920                 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
2921                 return -ENOMEM;
2922         }
2923
2924         DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2925                  mr->npages, addr);
2926
2927         pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2928         pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2929         pbe = (struct regpair *)pbl_table->va;
2930         pbe +=  mr->npages % pbes_in_page;
2931         pbe->lo = cpu_to_le32((u32)addr);
2932         pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2933
2934         mr->npages++;
2935
2936         return 0;
2937 }
2938
2939 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2940 {
2941         int work = info->completed - info->completed_handled - 1;
2942
2943         DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2944         while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2945                 struct qedr_pbl *pbl;
2946
2947                 /* Free all the page list that are possible to be freed
2948                  * (all the ones that were invalidated), under the assumption
2949                  * that if an FMR was completed successfully that means that
2950                  * if there was an invalidate operation before it also ended
2951                  */
2952                 pbl = list_first_entry(&info->inuse_pbl_list,
2953                                        struct qedr_pbl, list_entry);
2954                 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2955                 info->completed_handled++;
2956         }
2957 }
2958
2959 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2960                    int sg_nents, unsigned int *sg_offset)
2961 {
2962         struct qedr_mr *mr = get_qedr_mr(ibmr);
2963
2964         mr->npages = 0;
2965
2966         handle_completed_mrs(mr->dev, &mr->info);
2967         return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2968 }
2969
2970 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2971 {
2972         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2973         struct qedr_pd *pd = get_qedr_pd(ibpd);
2974         struct qedr_mr *mr;
2975         int rc;
2976
2977         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2978         if (!mr)
2979                 return ERR_PTR(-ENOMEM);
2980
2981         mr->type = QEDR_MR_DMA;
2982
2983         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2984         if (rc) {
2985                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2986                 goto err1;
2987         }
2988
2989         /* index only, 18 bit long, lkey = itid << 8 | key */
2990         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2991         mr->hw_mr.pd = pd->pd_id;
2992         mr->hw_mr.local_read = 1;
2993         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2994         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2995         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2996         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2997         mr->hw_mr.dma_mr = true;
2998
2999         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3000         if (rc) {
3001                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3002                 goto err2;
3003         }
3004
3005         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3006         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3007             mr->hw_mr.remote_atomic)
3008                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3009
3010         DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3011         return &mr->ibmr;
3012
3013 err2:
3014         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3015 err1:
3016         kfree(mr);
3017         return ERR_PTR(rc);
3018 }
3019
3020 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3021 {
3022         return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3023 }
3024
3025 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3026 {
3027         int i, len = 0;
3028
3029         for (i = 0; i < num_sge; i++)
3030                 len += sg_list[i].length;
3031
3032         return len;
3033 }
3034
3035 static void swap_wqe_data64(u64 *p)
3036 {
3037         int i;
3038
3039         for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3040                 *p = cpu_to_be64(cpu_to_le64(*p));
3041 }
3042
3043 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3044                                        struct qedr_qp *qp, u8 *wqe_size,
3045                                        const struct ib_send_wr *wr,
3046                                        const struct ib_send_wr **bad_wr,
3047                                        u8 *bits, u8 bit)
3048 {
3049         u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3050         char *seg_prt, *wqe;
3051         int i, seg_siz;
3052
3053         if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3054                 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3055                 *bad_wr = wr;
3056                 return 0;
3057         }
3058
3059         if (!data_size)
3060                 return data_size;
3061
3062         *bits |= bit;
3063
3064         seg_prt = NULL;
3065         wqe = NULL;
3066         seg_siz = 0;
3067
3068         /* Copy data inline */
3069         for (i = 0; i < wr->num_sge; i++) {
3070                 u32 len = wr->sg_list[i].length;
3071                 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3072
3073                 while (len > 0) {
3074                         u32 cur;
3075
3076                         /* New segment required */
3077                         if (!seg_siz) {
3078                                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3079                                 seg_prt = wqe;
3080                                 seg_siz = sizeof(struct rdma_sq_common_wqe);
3081                                 (*wqe_size)++;
3082                         }
3083
3084                         /* Calculate currently allowed length */
3085                         cur = min_t(u32, len, seg_siz);
3086                         memcpy(seg_prt, src, cur);
3087
3088                         /* Update segment variables */
3089                         seg_prt += cur;
3090                         seg_siz -= cur;
3091
3092                         /* Update sge variables */
3093                         src += cur;
3094                         len -= cur;
3095
3096                         /* Swap fully-completed segments */
3097                         if (!seg_siz)
3098                                 swap_wqe_data64((u64 *)wqe);
3099                 }
3100         }
3101
3102         /* swap last not completed segment */
3103         if (seg_siz)
3104                 swap_wqe_data64((u64 *)wqe);
3105
3106         return data_size;
3107 }
3108
3109 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)                 \
3110         do {                                                    \
3111                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3112                 (sge)->length = cpu_to_le32(vlength);           \
3113                 (sge)->flags = cpu_to_le32(vflags);             \
3114         } while (0)
3115
3116 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)                       \
3117         do {                                                    \
3118                 DMA_REGPAIR_LE(hdr->wr_id, vwr_id);             \
3119                 (hdr)->num_sges = num_sge;                      \
3120         } while (0)
3121
3122 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)                 \
3123         do {                                                    \
3124                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3125                 (sge)->length = cpu_to_le32(vlength);           \
3126                 (sge)->l_key = cpu_to_le32(vlkey);              \
3127         } while (0)
3128
3129 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3130                                 const struct ib_send_wr *wr)
3131 {
3132         u32 data_size = 0;
3133         int i;
3134
3135         for (i = 0; i < wr->num_sge; i++) {
3136                 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3137
3138                 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3139                 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3140                 sge->length = cpu_to_le32(wr->sg_list[i].length);
3141                 data_size += wr->sg_list[i].length;
3142         }
3143
3144         if (wqe_size)
3145                 *wqe_size += wr->num_sge;
3146
3147         return data_size;
3148 }
3149
3150 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3151                                      struct qedr_qp *qp,
3152                                      struct rdma_sq_rdma_wqe_1st *rwqe,
3153                                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
3154                                      const struct ib_send_wr *wr,
3155                                      const struct ib_send_wr **bad_wr)
3156 {
3157         rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3158         DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3159
3160         if (wr->send_flags & IB_SEND_INLINE &&
3161             (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3162              wr->opcode == IB_WR_RDMA_WRITE)) {
3163                 u8 flags = 0;
3164
3165                 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3166                 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3167                                                    bad_wr, &rwqe->flags, flags);
3168         }
3169
3170         return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3171 }
3172
3173 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3174                                      struct qedr_qp *qp,
3175                                      struct rdma_sq_send_wqe_1st *swqe,
3176                                      struct rdma_sq_send_wqe_2st *swqe2,
3177                                      const struct ib_send_wr *wr,
3178                                      const struct ib_send_wr **bad_wr)
3179 {
3180         memset(swqe2, 0, sizeof(*swqe2));
3181         if (wr->send_flags & IB_SEND_INLINE) {
3182                 u8 flags = 0;
3183
3184                 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3185                 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3186                                                    bad_wr, &swqe->flags, flags);
3187         }
3188
3189         return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3190 }
3191
3192 static int qedr_prepare_reg(struct qedr_qp *qp,
3193                             struct rdma_sq_fmr_wqe_1st *fwqe1,
3194                             const struct ib_reg_wr *wr)
3195 {
3196         struct qedr_mr *mr = get_qedr_mr(wr->mr);
3197         struct rdma_sq_fmr_wqe_2nd *fwqe2;
3198
3199         fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3200         fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3201         fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3202         fwqe1->l_key = wr->key;
3203
3204         fwqe2->access_ctrl = 0;
3205
3206         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3207                    !!(wr->access & IB_ACCESS_REMOTE_READ));
3208         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3209                    !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3210         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3211                    !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3212         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3213         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3214                    !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3215         fwqe2->fmr_ctrl = 0;
3216
3217         SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3218                    ilog2(mr->ibmr.page_size) - 12);
3219
3220         fwqe2->length_hi = 0;
3221         fwqe2->length_lo = mr->ibmr.length;
3222         fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3223         fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3224
3225         qp->wqe_wr_id[qp->sq.prod].mr = mr;
3226
3227         return 0;
3228 }
3229
3230 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3231 {
3232         switch (opcode) {
3233         case IB_WR_RDMA_WRITE:
3234         case IB_WR_RDMA_WRITE_WITH_IMM:
3235                 return IB_WC_RDMA_WRITE;
3236         case IB_WR_SEND_WITH_IMM:
3237         case IB_WR_SEND:
3238         case IB_WR_SEND_WITH_INV:
3239                 return IB_WC_SEND;
3240         case IB_WR_RDMA_READ:
3241         case IB_WR_RDMA_READ_WITH_INV:
3242                 return IB_WC_RDMA_READ;
3243         case IB_WR_ATOMIC_CMP_AND_SWP:
3244                 return IB_WC_COMP_SWAP;
3245         case IB_WR_ATOMIC_FETCH_AND_ADD:
3246                 return IB_WC_FETCH_ADD;
3247         case IB_WR_REG_MR:
3248                 return IB_WC_REG_MR;
3249         case IB_WR_LOCAL_INV:
3250                 return IB_WC_LOCAL_INV;
3251         default:
3252                 return IB_WC_SEND;
3253         }
3254 }
3255
3256 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3257                                       const struct ib_send_wr *wr)
3258 {
3259         int wq_is_full, err_wr, pbl_is_full;
3260         struct qedr_dev *dev = qp->dev;
3261
3262         /* prevent SQ overflow and/or processing of a bad WR */
3263         err_wr = wr->num_sge > qp->sq.max_sges;
3264         wq_is_full = qedr_wq_is_full(&qp->sq);
3265         pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3266                       QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3267         if (wq_is_full || err_wr || pbl_is_full) {
3268                 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3269                         DP_ERR(dev,
3270                                "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3271                                qp);
3272                         qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3273                 }
3274
3275                 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3276                         DP_ERR(dev,
3277                                "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3278                                qp);
3279                         qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3280                 }
3281
3282                 if (pbl_is_full &&
3283                     !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3284                         DP_ERR(dev,
3285                                "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3286                                qp);
3287                         qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3288                 }
3289                 return false;
3290         }
3291         return true;
3292 }
3293
3294 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3295                             const struct ib_send_wr **bad_wr)
3296 {
3297         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3298         struct qedr_qp *qp = get_qedr_qp(ibqp);
3299         struct rdma_sq_atomic_wqe_1st *awqe1;
3300         struct rdma_sq_atomic_wqe_2nd *awqe2;
3301         struct rdma_sq_atomic_wqe_3rd *awqe3;
3302         struct rdma_sq_send_wqe_2st *swqe2;
3303         struct rdma_sq_local_inv_wqe *iwqe;
3304         struct rdma_sq_rdma_wqe_2nd *rwqe2;
3305         struct rdma_sq_send_wqe_1st *swqe;
3306         struct rdma_sq_rdma_wqe_1st *rwqe;
3307         struct rdma_sq_fmr_wqe_1st *fwqe1;
3308         struct rdma_sq_common_wqe *wqe;
3309         u32 length;
3310         int rc = 0;
3311         bool comp;
3312
3313         if (!qedr_can_post_send(qp, wr)) {
3314                 *bad_wr = wr;
3315                 return -ENOMEM;
3316         }
3317
3318         wqe = qed_chain_produce(&qp->sq.pbl);
3319         qp->wqe_wr_id[qp->sq.prod].signaled =
3320                 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3321
3322         wqe->flags = 0;
3323         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3324                    !!(wr->send_flags & IB_SEND_SOLICITED));
3325         comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3326         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3327         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3328                    !!(wr->send_flags & IB_SEND_FENCE));
3329         wqe->prev_wqe_size = qp->prev_wqe_size;
3330
3331         qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3332
3333         switch (wr->opcode) {
3334         case IB_WR_SEND_WITH_IMM:
3335                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3336                         rc = -EINVAL;
3337                         *bad_wr = wr;
3338                         break;
3339                 }
3340                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3341                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3342                 swqe->wqe_size = 2;
3343                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3344
3345                 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3346                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3347                                                    wr, bad_wr);
3348                 swqe->length = cpu_to_le32(length);
3349                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3350                 qp->prev_wqe_size = swqe->wqe_size;
3351                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3352                 break;
3353         case IB_WR_SEND:
3354                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3355                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3356
3357                 swqe->wqe_size = 2;
3358                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3359                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3360                                                    wr, bad_wr);
3361                 swqe->length = cpu_to_le32(length);
3362                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3363                 qp->prev_wqe_size = swqe->wqe_size;
3364                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3365                 break;
3366         case IB_WR_SEND_WITH_INV:
3367                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3368                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3369                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3370                 swqe->wqe_size = 2;
3371                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3372                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3373                                                    wr, bad_wr);
3374                 swqe->length = cpu_to_le32(length);
3375                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3376                 qp->prev_wqe_size = swqe->wqe_size;
3377                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3378                 break;
3379
3380         case IB_WR_RDMA_WRITE_WITH_IMM:
3381                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3382                         rc = -EINVAL;
3383                         *bad_wr = wr;
3384                         break;
3385                 }
3386                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3387                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3388
3389                 rwqe->wqe_size = 2;
3390                 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3391                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3392                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3393                                                    wr, bad_wr);
3394                 rwqe->length = cpu_to_le32(length);
3395                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3396                 qp->prev_wqe_size = rwqe->wqe_size;
3397                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3398                 break;
3399         case IB_WR_RDMA_WRITE:
3400                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3401                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3402
3403                 rwqe->wqe_size = 2;
3404                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3405                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3406                                                    wr, bad_wr);
3407                 rwqe->length = cpu_to_le32(length);
3408                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3409                 qp->prev_wqe_size = rwqe->wqe_size;
3410                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3411                 break;
3412         case IB_WR_RDMA_READ_WITH_INV:
3413                 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3414                 /* fallthrough -- same is identical to RDMA READ */
3415
3416         case IB_WR_RDMA_READ:
3417                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3418                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3419
3420                 rwqe->wqe_size = 2;
3421                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3422                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3423                                                    wr, bad_wr);
3424                 rwqe->length = cpu_to_le32(length);
3425                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3426                 qp->prev_wqe_size = rwqe->wqe_size;
3427                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3428                 break;
3429
3430         case IB_WR_ATOMIC_CMP_AND_SWP:
3431         case IB_WR_ATOMIC_FETCH_AND_ADD:
3432                 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3433                 awqe1->wqe_size = 4;
3434
3435                 awqe2 = qed_chain_produce(&qp->sq.pbl);
3436                 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3437                 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3438
3439                 awqe3 = qed_chain_produce(&qp->sq.pbl);
3440
3441                 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3442                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3443                         DMA_REGPAIR_LE(awqe3->swap_data,
3444                                        atomic_wr(wr)->compare_add);
3445                 } else {
3446                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3447                         DMA_REGPAIR_LE(awqe3->swap_data,
3448                                        atomic_wr(wr)->swap);
3449                         DMA_REGPAIR_LE(awqe3->cmp_data,
3450                                        atomic_wr(wr)->compare_add);
3451                 }
3452
3453                 qedr_prepare_sq_sges(qp, NULL, wr);
3454
3455                 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3456                 qp->prev_wqe_size = awqe1->wqe_size;
3457                 break;
3458
3459         case IB_WR_LOCAL_INV:
3460                 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3461                 iwqe->wqe_size = 1;
3462
3463                 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3464                 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3465                 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3466                 qp->prev_wqe_size = iwqe->wqe_size;
3467                 break;
3468         case IB_WR_REG_MR:
3469                 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3470                 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3471                 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3472                 fwqe1->wqe_size = 2;
3473
3474                 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3475                 if (rc) {
3476                         DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3477                         *bad_wr = wr;
3478                         break;
3479                 }
3480
3481                 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3482                 qp->prev_wqe_size = fwqe1->wqe_size;
3483                 break;
3484         default:
3485                 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3486                 rc = -EINVAL;
3487                 *bad_wr = wr;
3488                 break;
3489         }
3490
3491         if (*bad_wr) {
3492                 u16 value;
3493
3494                 /* Restore prod to its position before
3495                  * this WR was processed
3496                  */
3497                 value = le16_to_cpu(qp->sq.db_data.data.value);
3498                 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3499
3500                 /* Restore prev_wqe_size */
3501                 qp->prev_wqe_size = wqe->prev_wqe_size;
3502                 rc = -EINVAL;
3503                 DP_ERR(dev, "POST SEND FAILED\n");
3504         }
3505
3506         return rc;
3507 }
3508
3509 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3510                    const struct ib_send_wr **bad_wr)
3511 {
3512         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3513         struct qedr_qp *qp = get_qedr_qp(ibqp);
3514         unsigned long flags;
3515         int rc = 0;
3516
3517         *bad_wr = NULL;
3518
3519         if (qp->qp_type == IB_QPT_GSI)
3520                 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3521
3522         spin_lock_irqsave(&qp->q_lock, flags);
3523
3524         if (rdma_protocol_roce(&dev->ibdev, 1)) {
3525                 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3526                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
3527                     (qp->state != QED_ROCE_QP_STATE_SQD)) {
3528                         spin_unlock_irqrestore(&qp->q_lock, flags);
3529                         *bad_wr = wr;
3530                         DP_DEBUG(dev, QEDR_MSG_CQ,
3531                                  "QP in wrong state! QP icid=0x%x state %d\n",
3532                                  qp->icid, qp->state);
3533                         return -EINVAL;
3534                 }
3535         }
3536
3537         while (wr) {
3538                 rc = __qedr_post_send(ibqp, wr, bad_wr);
3539                 if (rc)
3540                         break;
3541
3542                 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3543
3544                 qedr_inc_sw_prod(&qp->sq);
3545
3546                 qp->sq.db_data.data.value++;
3547
3548                 wr = wr->next;
3549         }
3550
3551         /* Trigger doorbell
3552          * If there was a failure in the first WR then it will be triggered in
3553          * vane. However this is not harmful (as long as the producer value is
3554          * unchanged). For performance reasons we avoid checking for this
3555          * redundant doorbell.
3556          *
3557          * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3558          * soon as we give the doorbell, we could get a completion
3559          * for this wr, therefore we need to make sure that the
3560          * memory is updated before giving the doorbell.
3561          * During qedr_poll_cq, rmb is called before accessing the
3562          * cqe. This covers for the smp_rmb as well.
3563          */
3564         smp_wmb();
3565         writel(qp->sq.db_data.raw, qp->sq.db);
3566
3567         /* Make sure write sticks */
3568         mmiowb();
3569
3570         spin_unlock_irqrestore(&qp->q_lock, flags);
3571
3572         return rc;
3573 }
3574
3575 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3576 {
3577         u32 used;
3578
3579         /* Calculate number of elements used based on producer
3580          * count and consumer count and subtract it from max
3581          * work request supported so that we get elements left.
3582          */
3583         used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3584
3585         return hw_srq->max_wr - used;
3586 }
3587
3588 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3589                        const struct ib_recv_wr **bad_wr)
3590 {
3591         struct qedr_srq *srq = get_qedr_srq(ibsrq);
3592         struct qedr_srq_hwq_info *hw_srq;
3593         struct qedr_dev *dev = srq->dev;
3594         struct qed_chain *pbl;
3595         unsigned long flags;
3596         int status = 0;
3597         u32 num_sge;
3598
3599         spin_lock_irqsave(&srq->lock, flags);
3600
3601         hw_srq = &srq->hw_srq;
3602         pbl = &srq->hw_srq.pbl;
3603         while (wr) {
3604                 struct rdma_srq_wqe_header *hdr;
3605                 int i;
3606
3607                 if (!qedr_srq_elem_left(hw_srq) ||
3608                     wr->num_sge > srq->hw_srq.max_sges) {
3609                         DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3610                                hw_srq->wr_prod_cnt,
3611                                atomic_read(&hw_srq->wr_cons_cnt),
3612                                wr->num_sge, srq->hw_srq.max_sges);
3613                         status = -ENOMEM;
3614                         *bad_wr = wr;
3615                         break;
3616                 }
3617
3618                 hdr = qed_chain_produce(pbl);
3619                 num_sge = wr->num_sge;
3620                 /* Set number of sge and work request id in header */
3621                 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3622
3623                 srq->hw_srq.wr_prod_cnt++;
3624                 hw_srq->wqe_prod++;
3625                 hw_srq->sge_prod++;
3626
3627                 DP_DEBUG(dev, QEDR_MSG_SRQ,
3628                          "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3629                          wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3630
3631                 for (i = 0; i < wr->num_sge; i++) {
3632                         struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3633
3634                         /* Set SGE length, lkey and address */
3635                         SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3636                                     wr->sg_list[i].length, wr->sg_list[i].lkey);
3637
3638                         DP_DEBUG(dev, QEDR_MSG_SRQ,
3639                                  "[%d]: len %d key %x addr %x:%x\n",
3640                                  i, srq_sge->length, srq_sge->l_key,
3641                                  srq_sge->addr.hi, srq_sge->addr.lo);
3642                         hw_srq->sge_prod++;
3643                 }
3644
3645                 /* Update WQE and SGE information before
3646                  * updating producer.
3647                  */
3648                 dma_wmb();
3649
3650                 /* SRQ producer is 8 bytes. Need to update SGE producer index
3651                  * in first 4 bytes and need to update WQE producer in
3652                  * next 4 bytes.
3653                  */
3654                 srq->hw_srq.virt_prod_pair_addr->sge_prod = hw_srq->sge_prod;
3655                 /* Make sure sge producer is updated first */
3656                 dma_wmb();
3657                 srq->hw_srq.virt_prod_pair_addr->wqe_prod = hw_srq->wqe_prod;
3658
3659                 wr = wr->next;
3660         }
3661
3662         DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3663                  qed_chain_get_elem_left(pbl));
3664         spin_unlock_irqrestore(&srq->lock, flags);
3665
3666         return status;
3667 }
3668
3669 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3670                    const struct ib_recv_wr **bad_wr)
3671 {
3672         struct qedr_qp *qp = get_qedr_qp(ibqp);
3673         struct qedr_dev *dev = qp->dev;
3674         unsigned long flags;
3675         int status = 0;
3676
3677         if (qp->qp_type == IB_QPT_GSI)
3678                 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3679
3680         spin_lock_irqsave(&qp->q_lock, flags);
3681
3682         if (qp->state == QED_ROCE_QP_STATE_RESET) {
3683                 spin_unlock_irqrestore(&qp->q_lock, flags);
3684                 *bad_wr = wr;
3685                 return -EINVAL;
3686         }
3687
3688         while (wr) {
3689                 int i;
3690
3691                 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3692                     QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3693                     wr->num_sge > qp->rq.max_sges) {
3694                         DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3695                                qed_chain_get_elem_left_u32(&qp->rq.pbl),
3696                                QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3697                                qp->rq.max_sges);
3698                         status = -ENOMEM;
3699                         *bad_wr = wr;
3700                         break;
3701                 }
3702                 for (i = 0; i < wr->num_sge; i++) {
3703                         u32 flags = 0;
3704                         struct rdma_rq_sge *rqe =
3705                             qed_chain_produce(&qp->rq.pbl);
3706
3707                         /* First one must include the number
3708                          * of SGE in the list
3709                          */
3710                         if (!i)
3711                                 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3712                                           wr->num_sge);
3713
3714                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3715                                   wr->sg_list[i].lkey);
3716
3717                         RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3718                                    wr->sg_list[i].length, flags);
3719                 }
3720
3721                 /* Special case of no sges. FW requires between 1-4 sges...
3722                  * in this case we need to post 1 sge with length zero. this is
3723                  * because rdma write with immediate consumes an RQ.
3724                  */
3725                 if (!wr->num_sge) {
3726                         u32 flags = 0;
3727                         struct rdma_rq_sge *rqe =
3728                             qed_chain_produce(&qp->rq.pbl);
3729
3730                         /* First one must include the number
3731                          * of SGE in the list
3732                          */
3733                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3734                         SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3735
3736                         RQ_SGE_SET(rqe, 0, 0, flags);
3737                         i = 1;
3738                 }
3739
3740                 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3741                 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3742
3743                 qedr_inc_sw_prod(&qp->rq);
3744
3745                 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3746                  * soon as we give the doorbell, we could get a completion
3747                  * for this wr, therefore we need to make sure that the
3748                  * memory is update before giving the doorbell.
3749                  * During qedr_poll_cq, rmb is called before accessing the
3750                  * cqe. This covers for the smp_rmb as well.
3751                  */
3752                 smp_wmb();
3753
3754                 qp->rq.db_data.data.value++;
3755
3756                 writel(qp->rq.db_data.raw, qp->rq.db);
3757
3758                 /* Make sure write sticks */
3759                 mmiowb();
3760
3761                 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3762                         writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3763                         mmiowb();       /* for second doorbell */
3764                 }
3765
3766                 wr = wr->next;
3767         }
3768
3769         spin_unlock_irqrestore(&qp->q_lock, flags);
3770
3771         return status;
3772 }
3773
3774 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3775 {
3776         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3777
3778         return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3779                 cq->pbl_toggle;
3780 }
3781
3782 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3783 {
3784         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3785         struct qedr_qp *qp;
3786
3787         qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3788                                                    resp_cqe->qp_handle.lo,
3789                                                    u64);
3790         return qp;
3791 }
3792
3793 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3794 {
3795         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3796
3797         return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3798 }
3799
3800 /* Return latest CQE (needs processing) */
3801 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3802 {
3803         return cq->latest_cqe;
3804 }
3805
3806 /* In fmr we need to increase the number of fmr completed counter for the fmr
3807  * algorithm determining whether we can free a pbl or not.
3808  * we need to perform this whether the work request was signaled or not. for
3809  * this purpose we call this function from the condition that checks if a wr
3810  * should be skipped, to make sure we don't miss it ( possibly this fmr
3811  * operation was not signalted)
3812  */
3813 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3814 {
3815         if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3816                 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3817 }
3818
3819 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3820                        struct qedr_cq *cq, int num_entries,
3821                        struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3822                        int force)
3823 {
3824         u16 cnt = 0;
3825
3826         while (num_entries && qp->sq.wqe_cons != hw_cons) {
3827                 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3828                         qedr_chk_if_fmr(qp);
3829                         /* skip WC */
3830                         goto next_cqe;
3831                 }
3832
3833                 /* fill WC */
3834                 wc->status = status;
3835                 wc->vendor_err = 0;
3836                 wc->wc_flags = 0;
3837                 wc->src_qp = qp->id;
3838                 wc->qp = &qp->ibqp;
3839
3840                 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3841                 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3842
3843                 switch (wc->opcode) {
3844                 case IB_WC_RDMA_WRITE:
3845                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3846                         break;
3847                 case IB_WC_COMP_SWAP:
3848                 case IB_WC_FETCH_ADD:
3849                         wc->byte_len = 8;
3850                         break;
3851                 case IB_WC_REG_MR:
3852                         qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3853                         break;
3854                 case IB_WC_RDMA_READ:
3855                 case IB_WC_SEND:
3856                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3857                         break;
3858                 default:
3859                         break;
3860                 }
3861
3862                 num_entries--;
3863                 wc++;
3864                 cnt++;
3865 next_cqe:
3866                 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3867                         qed_chain_consume(&qp->sq.pbl);
3868                 qedr_inc_sw_cons(&qp->sq);
3869         }
3870
3871         return cnt;
3872 }
3873
3874 static int qedr_poll_cq_req(struct qedr_dev *dev,
3875                             struct qedr_qp *qp, struct qedr_cq *cq,
3876                             int num_entries, struct ib_wc *wc,
3877                             struct rdma_cqe_requester *req)
3878 {
3879         int cnt = 0;
3880
3881         switch (req->status) {
3882         case RDMA_CQE_REQ_STS_OK:
3883                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3884                                   IB_WC_SUCCESS, 0);
3885                 break;
3886         case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3887                 if (qp->state != QED_ROCE_QP_STATE_ERR)
3888                         DP_DEBUG(dev, QEDR_MSG_CQ,
3889                                  "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3890                                  cq->icid, qp->icid);
3891                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3892                                   IB_WC_WR_FLUSH_ERR, 1);
3893                 break;
3894         default:
3895                 /* process all WQE before the cosumer */
3896                 qp->state = QED_ROCE_QP_STATE_ERR;
3897                 cnt = process_req(dev, qp, cq, num_entries, wc,
3898                                   req->sq_cons - 1, IB_WC_SUCCESS, 0);
3899                 wc += cnt;
3900                 /* if we have extra WC fill it with actual error info */
3901                 if (cnt < num_entries) {
3902                         enum ib_wc_status wc_status;
3903
3904                         switch (req->status) {
3905                         case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3906                                 DP_ERR(dev,
3907                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3908                                        cq->icid, qp->icid);
3909                                 wc_status = IB_WC_BAD_RESP_ERR;
3910                                 break;
3911                         case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3912                                 DP_ERR(dev,
3913                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3914                                        cq->icid, qp->icid);
3915                                 wc_status = IB_WC_LOC_LEN_ERR;
3916                                 break;
3917                         case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3918                                 DP_ERR(dev,
3919                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3920                                        cq->icid, qp->icid);
3921                                 wc_status = IB_WC_LOC_QP_OP_ERR;
3922                                 break;
3923                         case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3924                                 DP_ERR(dev,
3925                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3926                                        cq->icid, qp->icid);
3927                                 wc_status = IB_WC_LOC_PROT_ERR;
3928                                 break;
3929                         case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3930                                 DP_ERR(dev,
3931                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3932                                        cq->icid, qp->icid);
3933                                 wc_status = IB_WC_MW_BIND_ERR;
3934                                 break;
3935                         case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3936                                 DP_ERR(dev,
3937                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3938                                        cq->icid, qp->icid);
3939                                 wc_status = IB_WC_REM_INV_REQ_ERR;
3940                                 break;
3941                         case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3942                                 DP_ERR(dev,
3943                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3944                                        cq->icid, qp->icid);
3945                                 wc_status = IB_WC_REM_ACCESS_ERR;
3946                                 break;
3947                         case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3948                                 DP_ERR(dev,
3949                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3950                                        cq->icid, qp->icid);
3951                                 wc_status = IB_WC_REM_OP_ERR;
3952                                 break;
3953                         case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3954                                 DP_ERR(dev,
3955                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3956                                        cq->icid, qp->icid);
3957                                 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3958                                 break;
3959                         case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3960                                 DP_ERR(dev,
3961                                        "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3962                                        cq->icid, qp->icid);
3963                                 wc_status = IB_WC_RETRY_EXC_ERR;
3964                                 break;
3965                         default:
3966                                 DP_ERR(dev,
3967                                        "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3968                                        cq->icid, qp->icid);
3969                                 wc_status = IB_WC_GENERAL_ERR;
3970                         }
3971                         cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3972                                            wc_status, 1);
3973                 }
3974         }
3975
3976         return cnt;
3977 }
3978
3979 static inline int qedr_cqe_resp_status_to_ib(u8 status)
3980 {
3981         switch (status) {
3982         case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3983                 return IB_WC_LOC_ACCESS_ERR;
3984         case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3985                 return IB_WC_LOC_LEN_ERR;
3986         case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3987                 return IB_WC_LOC_QP_OP_ERR;
3988         case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3989                 return IB_WC_LOC_PROT_ERR;
3990         case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3991                 return IB_WC_MW_BIND_ERR;
3992         case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3993                 return IB_WC_REM_INV_RD_REQ_ERR;
3994         case RDMA_CQE_RESP_STS_OK:
3995                 return IB_WC_SUCCESS;
3996         default:
3997                 return IB_WC_GENERAL_ERR;
3998         }
3999 }
4000
4001 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4002                                           struct ib_wc *wc)
4003 {
4004         wc->status = IB_WC_SUCCESS;
4005         wc->byte_len = le32_to_cpu(resp->length);
4006
4007         if (resp->flags & QEDR_RESP_IMM) {
4008                 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4009                 wc->wc_flags |= IB_WC_WITH_IMM;
4010
4011                 if (resp->flags & QEDR_RESP_RDMA)
4012                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4013
4014                 if (resp->flags & QEDR_RESP_INV)
4015                         return -EINVAL;
4016
4017         } else if (resp->flags & QEDR_RESP_INV) {
4018                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4019                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4020
4021                 if (resp->flags & QEDR_RESP_RDMA)
4022                         return -EINVAL;
4023
4024         } else if (resp->flags & QEDR_RESP_RDMA) {
4025                 return -EINVAL;
4026         }
4027
4028         return 0;
4029 }
4030
4031 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4032                                struct qedr_cq *cq, struct ib_wc *wc,
4033                                struct rdma_cqe_responder *resp, u64 wr_id)
4034 {
4035         /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4036         wc->opcode = IB_WC_RECV;
4037         wc->wc_flags = 0;
4038
4039         if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4040                 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4041                         DP_ERR(dev,
4042                                "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4043                                cq, cq->icid, resp->flags);
4044
4045         } else {
4046                 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4047                 if (wc->status == IB_WC_GENERAL_ERR)
4048                         DP_ERR(dev,
4049                                "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4050                                cq, cq->icid, resp->status);
4051         }
4052
4053         /* Fill the rest of the WC */
4054         wc->vendor_err = 0;
4055         wc->src_qp = qp->id;
4056         wc->qp = &qp->ibqp;
4057         wc->wr_id = wr_id;
4058 }
4059
4060 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4061                                 struct qedr_cq *cq, struct ib_wc *wc,
4062                                 struct rdma_cqe_responder *resp)
4063 {
4064         struct qedr_srq *srq = qp->srq;
4065         u64 wr_id;
4066
4067         wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4068                          le32_to_cpu(resp->srq_wr_id.lo), u64);
4069
4070         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4071                 wc->status = IB_WC_WR_FLUSH_ERR;
4072                 wc->vendor_err = 0;
4073                 wc->wr_id = wr_id;
4074                 wc->byte_len = 0;
4075                 wc->src_qp = qp->id;
4076                 wc->qp = &qp->ibqp;
4077                 wc->wr_id = wr_id;
4078         } else {
4079                 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4080         }
4081         atomic_inc(&srq->hw_srq.wr_cons_cnt);
4082
4083         return 1;
4084 }
4085 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4086                             struct qedr_cq *cq, struct ib_wc *wc,
4087                             struct rdma_cqe_responder *resp)
4088 {
4089         u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4090
4091         __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4092
4093         while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4094                 qed_chain_consume(&qp->rq.pbl);
4095         qedr_inc_sw_cons(&qp->rq);
4096
4097         return 1;
4098 }
4099
4100 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4101                               int num_entries, struct ib_wc *wc, u16 hw_cons)
4102 {
4103         u16 cnt = 0;
4104
4105         while (num_entries && qp->rq.wqe_cons != hw_cons) {
4106                 /* fill WC */
4107                 wc->status = IB_WC_WR_FLUSH_ERR;
4108                 wc->vendor_err = 0;
4109                 wc->wc_flags = 0;
4110                 wc->src_qp = qp->id;
4111                 wc->byte_len = 0;
4112                 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4113                 wc->qp = &qp->ibqp;
4114                 num_entries--;
4115                 wc++;
4116                 cnt++;
4117                 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4118                         qed_chain_consume(&qp->rq.pbl);
4119                 qedr_inc_sw_cons(&qp->rq);
4120         }
4121
4122         return cnt;
4123 }
4124
4125 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4126                                  struct rdma_cqe_responder *resp, int *update)
4127 {
4128         if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4129                 consume_cqe(cq);
4130                 *update |= 1;
4131         }
4132 }
4133
4134 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4135                                  struct qedr_cq *cq, int num_entries,
4136                                  struct ib_wc *wc,
4137                                  struct rdma_cqe_responder *resp)
4138 {
4139         int cnt;
4140
4141         cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4142         consume_cqe(cq);
4143
4144         return cnt;
4145 }
4146
4147 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4148                              struct qedr_cq *cq, int num_entries,
4149                              struct ib_wc *wc, struct rdma_cqe_responder *resp,
4150                              int *update)
4151 {
4152         int cnt;
4153
4154         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4155                 cnt = process_resp_flush(qp, cq, num_entries, wc,
4156                                          resp->rq_cons_or_srq_id);
4157                 try_consume_resp_cqe(cq, qp, resp, update);
4158         } else {
4159                 cnt = process_resp_one(dev, qp, cq, wc, resp);
4160                 consume_cqe(cq);
4161                 *update |= 1;
4162         }
4163
4164         return cnt;
4165 }
4166
4167 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4168                                 struct rdma_cqe_requester *req, int *update)
4169 {
4170         if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4171                 consume_cqe(cq);
4172                 *update |= 1;
4173         }
4174 }
4175
4176 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4177 {
4178         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4179         struct qedr_cq *cq = get_qedr_cq(ibcq);
4180         union rdma_cqe *cqe;
4181         u32 old_cons, new_cons;
4182         unsigned long flags;
4183         int update = 0;
4184         int done = 0;
4185
4186         if (cq->destroyed) {
4187                 DP_ERR(dev,
4188                        "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4189                        cq, cq->icid);
4190                 return 0;
4191         }
4192
4193         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4194                 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4195
4196         spin_lock_irqsave(&cq->cq_lock, flags);
4197         cqe = cq->latest_cqe;
4198         old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4199         while (num_entries && is_valid_cqe(cq, cqe)) {
4200                 struct qedr_qp *qp;
4201                 int cnt = 0;
4202
4203                 /* prevent speculative reads of any field of CQE */
4204                 rmb();
4205
4206                 qp = cqe_get_qp(cqe);
4207                 if (!qp) {
4208                         WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4209                         break;
4210                 }
4211
4212                 wc->qp = &qp->ibqp;
4213
4214                 switch (cqe_get_type(cqe)) {
4215                 case RDMA_CQE_TYPE_REQUESTER:
4216                         cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4217                                                &cqe->req);
4218                         try_consume_req_cqe(cq, qp, &cqe->req, &update);
4219                         break;
4220                 case RDMA_CQE_TYPE_RESPONDER_RQ:
4221                         cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4222                                                 &cqe->resp, &update);
4223                         break;
4224                 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4225                         cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4226                                                     wc, &cqe->resp);
4227                         update = 1;
4228                         break;
4229                 case RDMA_CQE_TYPE_INVALID:
4230                 default:
4231                         DP_ERR(dev, "Error: invalid CQE type = %d\n",
4232                                cqe_get_type(cqe));
4233                 }
4234                 num_entries -= cnt;
4235                 wc += cnt;
4236                 done += cnt;
4237
4238                 cqe = get_cqe(cq);
4239         }
4240         new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4241
4242         cq->cq_cons += new_cons - old_cons;
4243
4244         if (update)
4245                 /* doorbell notifies abount latest VALID entry,
4246                  * but chain already point to the next INVALID one
4247                  */
4248                 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4249
4250         spin_unlock_irqrestore(&cq->cq_lock, flags);
4251         return done;
4252 }
4253
4254 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4255                      u8 port_num,
4256                      const struct ib_wc *in_wc,
4257                      const struct ib_grh *in_grh,
4258                      const struct ib_mad_hdr *mad_hdr,
4259                      size_t in_mad_size, struct ib_mad_hdr *out_mad,
4260                      size_t *out_mad_size, u16 *out_mad_pkey_index)
4261 {
4262         struct qedr_dev *dev = get_qedr_dev(ibdev);
4263
4264         DP_DEBUG(dev, QEDR_MSG_GSI,
4265                  "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
4266                  mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
4267                  mad_hdr->class_specific, mad_hdr->class_version,
4268                  mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
4269         return IB_MAD_RESULT_SUCCESS;
4270 }