drivers/infiniband/hw/qedr/verbs.c

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/dma-mapping.h>
  33 #include <linux/crc32.h>
  34 #include <net/ip.h>
  35 #include <net/ipv6.h>
  36 #include <net/udp.h>
  37 #include <linux/iommu.h>
  38
  39 #include <rdma/ib_verbs.h>
  40 #include <rdma/ib_user_verbs.h>
  41 #include <rdma/iw_cm.h>
  42 #include <rdma/ib_umem.h>
  43 #include <rdma/ib_addr.h>
  44 #include <rdma/ib_cache.h>
  45
  46 #include <linux/qed/common_hsi.h>
  47 #include "qedr_hsi_rdma.h"
  48 #include <linux/qed/qed_if.h>
  49 #include "qedr.h"
  50 #include "verbs.h"
  51 #include <rdma/qedr-abi.h>
  52 #include "qedr_roce_cm.h"
  53
  54 #define QEDR_SRQ_WQE_ELEM_SIZE  sizeof(union rdma_srq_elm)
  55 #define RDMA_MAX_SGE_PER_SRQ    (4)
  56 #define RDMA_MAX_SRQ_WQE_SIZE   (RDMA_MAX_SGE_PER_SRQ + 1)
  57
  58 #define DB_ADDR_SHIFT(addr)             ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  59
  60 static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
  61                                         size_t len)
  62 {
  63         size_t min_len = min_t(size_t, len, udata->outlen);
  64
  65         return ib_copy_to_udata(udata, src, min_len);
  66 }
  67
  68 int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
  69 {
  70         if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
  71                 return -EINVAL;
  72
  73         *pkey = QEDR_ROCE_PKEY_DEFAULT;
  74         return 0;
  75 }
  76
  77 int qedr_iw_query_gid(struct ib_device *ibdev, u8 port,
  78                       int index, union ib_gid *sgid)
  79 {
  80         struct qedr_dev *dev = get_qedr_dev(ibdev);
  81
  82         memset(sgid->raw, 0, sizeof(sgid->raw));
  83         ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
  84
  85         DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
  86                  sgid->global.interface_id, sgid->global.subnet_prefix);
  87
  88         return 0;
  89 }
  90
  91 int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
  92 {
  93         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
  94         struct qedr_device_attr *qattr = &dev->attr;
  95         struct qedr_srq *srq = get_qedr_srq(ibsrq);
  96
  97         srq_attr->srq_limit = srq->srq_limit;
  98         srq_attr->max_wr = qattr->max_srq_wr;
  99         srq_attr->max_sge = qattr->max_sge;
 100
 101         return 0;
 102 }
 103
 104 int qedr_query_device(struct ib_device *ibdev,
 105                       struct ib_device_attr *attr, struct ib_udata *udata)
 106 {
 107         struct qedr_dev *dev = get_qedr_dev(ibdev);
 108         struct qedr_device_attr *qattr = &dev->attr;
 109
 110         if (!dev->rdma_ctx) {
 111                 DP_ERR(dev,
 112                        "qedr_query_device called with invalid params rdma_ctx=%p\n",
 113                        dev->rdma_ctx);
 114                 return -EINVAL;
 115         }
 116
 117         memset(attr, 0, sizeof(*attr));
 118
 119         attr->fw_ver = qattr->fw_ver;
 120         attr->sys_image_guid = qattr->sys_image_guid;
 121         attr->max_mr_size = qattr->max_mr_size;
 122         attr->page_size_cap = qattr->page_size_caps;
 123         attr->vendor_id = qattr->vendor_id;
 124         attr->vendor_part_id = qattr->vendor_part_id;
 125         attr->hw_ver = qattr->hw_ver;
 126         attr->max_qp = qattr->max_qp;
 127         attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 128         attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 129             IB_DEVICE_RC_RNR_NAK_GEN |
 130             IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
 131
 132         attr->max_send_sge = qattr->max_sge;
 133         attr->max_recv_sge = qattr->max_sge;
 134         attr->max_sge_rd = qattr->max_sge;
 135         attr->max_cq = qattr->max_cq;
 136         attr->max_cqe = qattr->max_cqe;
 137         attr->max_mr = qattr->max_mr;
 138         attr->max_mw = qattr->max_mw;
 139         attr->max_pd = qattr->max_pd;
 140         attr->atomic_cap = dev->atomic_cap;
 141         attr->max_fmr = qattr->max_fmr;
 142         attr->max_map_per_fmr = 16;
 143         attr->max_qp_init_rd_atom =
 144             1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 145         attr->max_qp_rd_atom =
 146             min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 147                 attr->max_qp_init_rd_atom);
 148
 149         attr->max_srq = qattr->max_srq;
 150         attr->max_srq_sge = qattr->max_srq_sge;
 151         attr->max_srq_wr = qattr->max_srq_wr;
 152
 153         attr->local_ca_ack_delay = qattr->dev_ack_delay;
 154         attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 155         attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
 156         attr->max_ah = qattr->max_ah;
 157
 158         return 0;
 159 }
 160
 161 static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
 162                                             u8 *ib_width)
 163 {
 164         switch (speed) {
 165         case 1000:
 166                 *ib_speed = IB_SPEED_SDR;
 167                 *ib_width = IB_WIDTH_1X;
 168                 break;
 169         case 10000:
 170                 *ib_speed = IB_SPEED_QDR;
 171                 *ib_width = IB_WIDTH_1X;
 172                 break;
 173
 174         case 20000:
 175                 *ib_speed = IB_SPEED_DDR;
 176                 *ib_width = IB_WIDTH_4X;
 177                 break;
 178
 179         case 25000:
 180                 *ib_speed = IB_SPEED_EDR;
 181                 *ib_width = IB_WIDTH_1X;
 182                 break;
 183
 184         case 40000:
 185                 *ib_speed = IB_SPEED_QDR;
 186                 *ib_width = IB_WIDTH_4X;
 187                 break;
 188
 189         case 50000:
 190                 *ib_speed = IB_SPEED_HDR;
 191                 *ib_width = IB_WIDTH_1X;
 192                 break;
 193
 194         case 100000:
 195                 *ib_speed = IB_SPEED_EDR;
 196                 *ib_width = IB_WIDTH_4X;
 197                 break;
 198
 199         default:
 200                 /* Unsupported */
 201                 *ib_speed = IB_SPEED_SDR;
 202                 *ib_width = IB_WIDTH_1X;
 203         }
 204 }
 205
 206 int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
 207 {
 208         struct qedr_dev *dev;
 209         struct qed_rdma_port *rdma_port;
 210
 211         dev = get_qedr_dev(ibdev);
 212         if (port > 1) {
 213                 DP_ERR(dev, "invalid_port=0x%x\n", port);
 214                 return -EINVAL;
 215         }
 216
 217         if (!dev->rdma_ctx) {
 218                 DP_ERR(dev, "rdma_ctx is NULL\n");
 219                 return -EINVAL;
 220         }
 221
 222         rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 223
 224         /* *attr being zeroed by the caller, avoid zeroing it here */
 225         if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 226                 attr->state = IB_PORT_ACTIVE;
 227                 attr->phys_state = 5;
 228         } else {
 229                 attr->state = IB_PORT_DOWN;
 230                 attr->phys_state = 3;
 231         }
 232         attr->max_mtu = IB_MTU_4096;
 233         attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 234         attr->lid = 0;
 235         attr->lmc = 0;
 236         attr->sm_lid = 0;
 237         attr->sm_sl = 0;
 238         attr->ip_gids = true;
 239         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
 240                 attr->gid_tbl_len = 1;
 241                 attr->pkey_tbl_len = 1;
 242         } else {
 243                 attr->gid_tbl_len = QEDR_MAX_SGID;
 244                 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 245         }
 246         attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 247         attr->qkey_viol_cntr = 0;
 248         get_link_speed_and_width(rdma_port->link_speed,
 249                                  &attr->active_speed, &attr->active_width);
 250         attr->max_msg_sz = rdma_port->max_msg_size;
 251         attr->max_vl_num = 4;
 252
 253         return 0;
 254 }
 255
 256 int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
 257                      struct ib_port_modify *props)
 258 {
 259         struct qedr_dev *dev;
 260
 261         dev = get_qedr_dev(ibdev);
 262         if (port > 1) {
 263                 DP_ERR(dev, "invalid_port=0x%x\n", port);
 264                 return -EINVAL;
 265         }
 266
 267         return 0;
 268 }
 269
 270 static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
 271                          unsigned long len)
 272 {
 273         struct qedr_mm *mm;
 274
 275         mm = kzalloc(sizeof(*mm), GFP_KERNEL);
 276         if (!mm)
 277                 return -ENOMEM;
 278
 279         mm->key.phy_addr = phy_addr;
 280         /* This function might be called with a length which is not a multiple
 281          * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
 282          * forces this granularity by increasing the requested size if needed.
 283          * When qedr_mmap is called, it will search the list with the updated
 284          * length as a key. To prevent search failures, the length is rounded up
 285          * in advance to PAGE_SIZE.
 286          */
 287         mm->key.len = roundup(len, PAGE_SIZE);
 288         INIT_LIST_HEAD(&mm->entry);
 289
 290         mutex_lock(&uctx->mm_list_lock);
 291         list_add(&mm->entry, &uctx->mm_head);
 292         mutex_unlock(&uctx->mm_list_lock);
 293
 294         DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 295                  "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
 296                  (unsigned long long)mm->key.phy_addr,
 297                  (unsigned long)mm->key.len, uctx);
 298
 299         return 0;
 300 }
 301
 302 static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
 303                              unsigned long len)
 304 {
 305         bool found = false;
 306         struct qedr_mm *mm;
 307
 308         mutex_lock(&uctx->mm_list_lock);
 309         list_for_each_entry(mm, &uctx->mm_head, entry) {
 310                 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
 311                         continue;
 312
 313                 found = true;
 314                 break;
 315         }
 316         mutex_unlock(&uctx->mm_list_lock);
 317         DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 318                  "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
 319                  mm->key.phy_addr, mm->key.len, uctx, found);
 320
 321         return found;
 322 }
 323
 324 struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
 325                                         struct ib_udata *udata)
 326 {
 327         int rc;
 328         struct qedr_ucontext *ctx;
 329         struct qedr_alloc_ucontext_resp uresp;
 330         struct qedr_dev *dev = get_qedr_dev(ibdev);
 331         struct qed_rdma_add_user_out_params oparams;
 332
 333         if (!udata)
 334                 return ERR_PTR(-EFAULT);
 335
 336         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
 337         if (!ctx)
 338                 return ERR_PTR(-ENOMEM);
 339
 340         rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 341         if (rc) {
 342                 DP_ERR(dev,
 343                        "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 344                        rc);
 345                 goto err;
 346         }
 347
 348         ctx->dpi = oparams.dpi;
 349         ctx->dpi_addr = oparams.dpi_addr;
 350         ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 351         ctx->dpi_size = oparams.dpi_size;
 352         INIT_LIST_HEAD(&ctx->mm_head);
 353         mutex_init(&ctx->mm_list_lock);
 354
 355         memset(&uresp, 0, sizeof(uresp));
 356
 357         uresp.dpm_enabled = dev->user_dpm_enabled;
 358         uresp.wids_enabled = 1;
 359         uresp.wid_count = oparams.wid_count;
 360         uresp.db_pa = ctx->dpi_phys_addr;
 361         uresp.db_size = ctx->dpi_size;
 362         uresp.max_send_wr = dev->attr.max_sqe;
 363         uresp.max_recv_wr = dev->attr.max_rqe;
 364         uresp.max_srq_wr = dev->attr.max_srq_wr;
 365         uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 366         uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 367         uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 368         uresp.max_cqes = QEDR_MAX_CQES;
 369
 370         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 371         if (rc)
 372                 goto err;
 373
 374         ctx->dev = dev;
 375
 376         rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
 377         if (rc)
 378                 goto err;
 379
 380         DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 381                  &ctx->ibucontext);
 382         return &ctx->ibucontext;
 383
 384 err:
 385         kfree(ctx);
 386         return ERR_PTR(rc);
 387 }
 388
 389 int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 390 {
 391         struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 392         struct qedr_mm *mm, *tmp;
 393         int status = 0;
 394
 395         DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 396                  uctx);
 397         uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
 398
 399         list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
 400                 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
 401                          "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
 402                          mm->key.phy_addr, mm->key.len, uctx);
 403                 list_del(&mm->entry);
 404                 kfree(mm);
 405         }
 406
 407         kfree(uctx);
 408         return status;
 409 }
 410
 411 int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
 412 {
 413         struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
 414         struct qedr_dev *dev = get_qedr_dev(context->device);
 415         unsigned long phys_addr = vma->vm_pgoff << PAGE_SHIFT;
 416         unsigned long len = (vma->vm_end - vma->vm_start);
 417         unsigned long dpi_start;
 418
 419         dpi_start = dev->db_phys_addr + (ucontext->dpi * ucontext->dpi_size);
 420
 421         DP_DEBUG(dev, QEDR_MSG_INIT,
 422                  "mmap invoked with vm_start=0x%pK, vm_end=0x%pK,vm_pgoff=0x%pK; dpi_start=0x%pK dpi_size=0x%x\n",
 423                  (void *)vma->vm_start, (void *)vma->vm_end,
 424                  (void *)vma->vm_pgoff, (void *)dpi_start, ucontext->dpi_size);
 425
 426         if ((vma->vm_start & (PAGE_SIZE - 1)) || (len & (PAGE_SIZE - 1))) {
 427                 DP_ERR(dev,
 428                        "failed mmap, addresses must be page aligned: start=0x%pK, end=0x%pK\n",
 429                        (void *)vma->vm_start, (void *)vma->vm_end);
 430                 return -EINVAL;
 431         }
 432
 433         if (!qedr_search_mmap(ucontext, phys_addr, len)) {
 434                 DP_ERR(dev, "failed mmap, vm_pgoff=0x%lx is not authorized\n",
 435                        vma->vm_pgoff);
 436                 return -EINVAL;
 437         }
 438
 439         if (phys_addr < dpi_start ||
 440             ((phys_addr + len) > (dpi_start + ucontext->dpi_size))) {
 441                 DP_ERR(dev,
 442                        "failed mmap, pages are outside of dpi; page address=0x%pK, dpi_start=0x%pK, dpi_size=0x%x\n",
 443                        (void *)phys_addr, (void *)dpi_start,
 444                        ucontext->dpi_size);
 445                 return -EINVAL;
 446         }
 447
 448         if (vma->vm_flags & VM_READ) {
 449                 DP_ERR(dev, "failed mmap, cannot map doorbell bar for read\n");
 450                 return -EINVAL;
 451         }
 452
 453         vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
 454         return io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, len,
 455                                   vma->vm_page_prot);
 456 }
 457
 458 struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
 459                             struct ib_ucontext *context, struct ib_udata *udata)
 460 {
 461         struct qedr_dev *dev = get_qedr_dev(ibdev);
 462         struct qedr_pd *pd;
 463         u16 pd_id;
 464         int rc;
 465
 466         DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 467                  (udata && context) ? "User Lib" : "Kernel");
 468
 469         if (!dev->rdma_ctx) {
 470                 DP_ERR(dev, "invalid RDMA context\n");
 471                 return ERR_PTR(-EINVAL);
 472         }
 473
 474         pd = kzalloc(sizeof(*pd), GFP_KERNEL);
 475         if (!pd)
 476                 return ERR_PTR(-ENOMEM);
 477
 478         rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 479         if (rc)
 480                 goto err;
 481
 482         pd->pd_id = pd_id;
 483
 484         if (udata && context) {
 485                 struct qedr_alloc_pd_uresp uresp = {
 486                         .pd_id = pd_id,
 487                 };
 488
 489                 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 490                 if (rc) {
 491                         DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 492                         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 493                         goto err;
 494                 }
 495
 496                 pd->uctx = get_qedr_ucontext(context);
 497                 pd->uctx->pd = pd;
 498         }
 499
 500         return &pd->ibpd;
 501
 502 err:
 503         kfree(pd);
 504         return ERR_PTR(rc);
 505 }
 506
 507 int qedr_dealloc_pd(struct ib_pd *ibpd)
 508 {
 509         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 510         struct qedr_pd *pd = get_qedr_pd(ibpd);
 511
 512         if (!pd) {
 513                 pr_err("Invalid PD received in dealloc_pd\n");
 514                 return -EINVAL;
 515         }
 516
 517         DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 518         dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 519
 520         kfree(pd);
 521
 522         return 0;
 523 }
 524
 525 static void qedr_free_pbl(struct qedr_dev *dev,
 526                           struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 527 {
 528         struct pci_dev *pdev = dev->pdev;
 529         int i;
 530
 531         for (i = 0; i < pbl_info->num_pbls; i++) {
 532                 if (!pbl[i].va)
 533                         continue;
 534                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 535                                   pbl[i].va, pbl[i].pa);
 536         }
 537
 538         kfree(pbl);
 539 }
 540
 541 #define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 542 #define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 543
 544 #define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 545 #define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 546 #define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 547
 548 static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 549                                            struct qedr_pbl_info *pbl_info,
 550                                            gfp_t flags)
 551 {
 552         struct pci_dev *pdev = dev->pdev;
 553         struct qedr_pbl *pbl_table;
 554         dma_addr_t *pbl_main_tbl;
 555         dma_addr_t pa;
 556         void *va;
 557         int i;
 558
 559         pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 560         if (!pbl_table)
 561                 return ERR_PTR(-ENOMEM);
 562
 563         for (i = 0; i < pbl_info->num_pbls; i++) {
 564                 va = dma_zalloc_coherent(&pdev->dev, pbl_info->pbl_size,
 565                                          &pa, flags);
 566                 if (!va)
 567                         goto err;
 568
 569                 pbl_table[i].va = va;
 570                 pbl_table[i].pa = pa;
 571         }
 572
 573         /* Two-Layer PBLs, if we have more than one pbl we need to initialize
 574          * the first one with physical pointers to all of the rest
 575          */
 576         pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 577         for (i = 0; i < pbl_info->num_pbls - 1; i++)
 578                 pbl_main_tbl[i] = pbl_table[i + 1].pa;
 579
 580         return pbl_table;
 581
 582 err:
 583         for (i--; i >= 0; i--)
 584                 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 585                                   pbl_table[i].va, pbl_table[i].pa);
 586
 587         qedr_free_pbl(dev, pbl_info, pbl_table);
 588
 589         return ERR_PTR(-ENOMEM);
 590 }
 591
 592 static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 593                                 struct qedr_pbl_info *pbl_info,
 594                                 u32 num_pbes, int two_layer_capable)
 595 {
 596         u32 pbl_capacity;
 597         u32 pbl_size;
 598         u32 num_pbls;
 599
 600         if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 601                 if (num_pbes > MAX_PBES_TWO_LAYER) {
 602                         DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 603                                num_pbes);
 604                         return -EINVAL;
 605                 }
 606
 607                 /* calculate required pbl page size */
 608                 pbl_size = MIN_FW_PBL_PAGE_SIZE;
 609                 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 610                                NUM_PBES_ON_PAGE(pbl_size);
 611
 612                 while (pbl_capacity < num_pbes) {
 613                         pbl_size *= 2;
 614                         pbl_capacity = pbl_size / sizeof(u64);
 615                         pbl_capacity = pbl_capacity * pbl_capacity;
 616                 }
 617
 618                 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 619                 num_pbls++;     /* One for the layer0 ( points to the pbls) */
 620                 pbl_info->two_layered = true;
 621         } else {
 622                 /* One layered PBL */
 623                 num_pbls = 1;
 624                 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 625                                  roundup_pow_of_two((num_pbes * sizeof(u64))));
 626                 pbl_info->two_layered = false;
 627         }
 628
 629         pbl_info->num_pbls = num_pbls;
 630         pbl_info->pbl_size = pbl_size;
 631         pbl_info->num_pbes = num_pbes;
 632
 633         DP_DEBUG(dev, QEDR_MSG_MR,
 634                  "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 635                  pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 636
 637         return 0;
 638 }
 639
 640 static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 641                                struct qedr_pbl *pbl,
 642                                struct qedr_pbl_info *pbl_info, u32 pg_shift)
 643 {
 644         int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
 645         u32 fw_pg_cnt, fw_pg_per_umem_pg;
 646         struct qedr_pbl *pbl_tbl;
 647         struct scatterlist *sg;
 648         struct regpair *pbe;
 649         u64 pg_addr;
 650         int entry;
 651
 652         if (!pbl_info->num_pbes)
 653                 return;
 654
 655         /* If we have a two layered pbl, the first pbl points to the rest
 656          * of the pbls and the first entry lays on the second pbl in the table
 657          */
 658         if (pbl_info->two_layered)
 659                 pbl_tbl = &pbl[1];
 660         else
 661                 pbl_tbl = pbl;
 662
 663         pbe = (struct regpair *)pbl_tbl->va;
 664         if (!pbe) {
 665                 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 666                 return;
 667         }
 668
 669         pbe_cnt = 0;
 670
 671         shift = umem->page_shift;
 672
 673         fw_pg_per_umem_pg = BIT(umem->page_shift - pg_shift);
 674
 675         for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
 676                 pages = sg_dma_len(sg) >> shift;
 677                 pg_addr = sg_dma_address(sg);
 678                 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
 679                         for (fw_pg_cnt = 0; fw_pg_cnt < fw_pg_per_umem_pg;) {
 680                                 pbe->lo = cpu_to_le32(pg_addr);
 681                                 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 682
 683                                 pg_addr += BIT(pg_shift);
 684                                 pbe_cnt++;
 685                                 total_num_pbes++;
 686                                 pbe++;
 687
 688                                 if (total_num_pbes == pbl_info->num_pbes)
 689                                         return;
 690
 691                                 /* If the given pbl is full storing the pbes,
 692                                  * move to next pbl.
 693                                  */
 694                                 if (pbe_cnt ==
 695                                     (pbl_info->pbl_size / sizeof(u64))) {
 696                                         pbl_tbl++;
 697                                         pbe = (struct regpair *)pbl_tbl->va;
 698                                         pbe_cnt = 0;
 699                                 }
 700
 701                                 fw_pg_cnt++;
 702                         }
 703                 }
 704         }
 705 }
 706
 707 static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 708                               struct qedr_cq *cq, struct ib_udata *udata)
 709 {
 710         struct qedr_create_cq_uresp uresp;
 711         int rc;
 712
 713         memset(&uresp, 0, sizeof(uresp));
 714
 715         uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 716         uresp.icid = cq->icid;
 717
 718         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 719         if (rc)
 720                 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 721
 722         return rc;
 723 }
 724
 725 static void consume_cqe(struct qedr_cq *cq)
 726 {
 727         if (cq->latest_cqe == cq->toggle_cqe)
 728                 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 729
 730         cq->latest_cqe = qed_chain_consume(&cq->pbl);
 731 }
 732
 733 static inline int qedr_align_cq_entries(int entries)
 734 {
 735         u64 size, aligned_size;
 736
 737         /* We allocate an extra entry that we don't report to the FW. */
 738         size = (entries + 1) * QEDR_CQE_SIZE;
 739         aligned_size = ALIGN(size, PAGE_SIZE);
 740
 741         return aligned_size / QEDR_CQE_SIZE;
 742 }
 743
 744 static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
 745                                        struct qedr_dev *dev,
 746                                        struct qedr_userq *q,
 747                                        u64 buf_addr, size_t buf_len,
 748                                        int access, int dmasync,
 749                                        int alloc_and_init)
 750 {
 751         u32 fw_pages;
 752         int rc;
 753
 754         q->buf_addr = buf_addr;
 755         q->buf_len = buf_len;
 756         q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
 757         if (IS_ERR(q->umem)) {
 758                 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 759                        PTR_ERR(q->umem));
 760                 return PTR_ERR(q->umem);
 761         }
 762
 763         fw_pages = ib_umem_page_count(q->umem) <<
 764             (q->umem->page_shift - FW_PAGE_SHIFT);
 765
 766         rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 767         if (rc)
 768                 goto err0;
 769
 770         if (alloc_and_init) {
 771                 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 772                 if (IS_ERR(q->pbl_tbl)) {
 773                         rc = PTR_ERR(q->pbl_tbl);
 774                         goto err0;
 775                 }
 776                 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 777                                    FW_PAGE_SHIFT);
 778         } else {
 779                 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
 780                 if (!q->pbl_tbl) {
 781                         rc = -ENOMEM;
 782                         goto err0;
 783                 }
 784         }
 785
 786         return 0;
 787
 788 err0:
 789         ib_umem_release(q->umem);
 790         q->umem = NULL;
 791
 792         return rc;
 793 }
 794
 795 static inline void qedr_init_cq_params(struct qedr_cq *cq,
 796                                        struct qedr_ucontext *ctx,
 797                                        struct qedr_dev *dev, int vector,
 798                                        int chain_entries, int page_cnt,
 799                                        u64 pbl_ptr,
 800                                        struct qed_rdma_create_cq_in_params
 801                                        *params)
 802 {
 803         memset(params, 0, sizeof(*params));
 804         params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 805         params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 806         params->cnq_id = vector;
 807         params->cq_size = chain_entries - 1;
 808         params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 809         params->pbl_num_pages = page_cnt;
 810         params->pbl_ptr = pbl_ptr;
 811         params->pbl_two_level = 0;
 812 }
 813
 814 static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 815 {
 816         cq->db.data.agg_flags = flags;
 817         cq->db.data.value = cpu_to_le32(cons);
 818         writeq(cq->db.raw, cq->db_addr);
 819
 820         /* Make sure write would stick */
 821         mmiowb();
 822 }
 823
 824 int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 825 {
 826         struct qedr_cq *cq = get_qedr_cq(ibcq);
 827         unsigned long sflags;
 828         struct qedr_dev *dev;
 829
 830         dev = get_qedr_dev(ibcq->device);
 831
 832         if (cq->destroyed) {
 833                 DP_ERR(dev,
 834                        "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 835                        cq, cq->icid);
 836                 return -EINVAL;
 837         }
 838
 839
 840         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 841                 return 0;
 842
 843         spin_lock_irqsave(&cq->cq_lock, sflags);
 844
 845         cq->arm_flags = 0;
 846
 847         if (flags & IB_CQ_SOLICITED)
 848                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 849
 850         if (flags & IB_CQ_NEXT_COMP)
 851                 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 852
 853         doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 854
 855         spin_unlock_irqrestore(&cq->cq_lock, sflags);
 856
 857         return 0;
 858 }
 859
 860 struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
 861                              const struct ib_cq_init_attr *attr,
 862                              struct ib_ucontext *ib_ctx, struct ib_udata *udata)
 863 {
 864         struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
 865         struct qed_rdma_destroy_cq_out_params destroy_oparams;
 866         struct qed_rdma_destroy_cq_in_params destroy_iparams;
 867         struct qedr_dev *dev = get_qedr_dev(ibdev);
 868         struct qed_rdma_create_cq_in_params params;
 869         struct qedr_create_cq_ureq ureq;
 870         int vector = attr->comp_vector;
 871         int entries = attr->cqe;
 872         struct qedr_cq *cq;
 873         int chain_entries;
 874         int page_cnt;
 875         u64 pbl_ptr;
 876         u16 icid;
 877         int rc;
 878
 879         DP_DEBUG(dev, QEDR_MSG_INIT,
 880                  "create_cq: called from %s. entries=%d, vector=%d\n",
 881                  udata ? "User Lib" : "Kernel", entries, vector);
 882
 883         if (entries > QEDR_MAX_CQES) {
 884                 DP_ERR(dev,
 885                        "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 886                        entries, QEDR_MAX_CQES);
 887                 return ERR_PTR(-EINVAL);
 888         }
 889
 890         chain_entries = qedr_align_cq_entries(entries);
 891         chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 892
 893         cq = kzalloc(sizeof(*cq), GFP_KERNEL);
 894         if (!cq)
 895                 return ERR_PTR(-ENOMEM);
 896
 897         if (udata) {
 898                 memset(&ureq, 0, sizeof(ureq));
 899                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
 900                         DP_ERR(dev,
 901                                "create cq: problem copying data from user space\n");
 902                         goto err0;
 903                 }
 904
 905                 if (!ureq.len) {
 906                         DP_ERR(dev,
 907                                "create cq: cannot create a cq with 0 entries\n");
 908                         goto err0;
 909                 }
 910
 911                 cq->cq_type = QEDR_CQ_TYPE_USER;
 912
 913                 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
 914                                           ureq.len, IB_ACCESS_LOCAL_WRITE,
 915                                           1, 1);
 916                 if (rc)
 917                         goto err0;
 918
 919                 pbl_ptr = cq->q.pbl_tbl->pa;
 920                 page_cnt = cq->q.pbl_info.num_pbes;
 921
 922                 cq->ibcq.cqe = chain_entries;
 923         } else {
 924                 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 925
 926                 rc = dev->ops->common->chain_alloc(dev->cdev,
 927                                                    QED_CHAIN_USE_TO_CONSUME,
 928                                                    QED_CHAIN_MODE_PBL,
 929                                                    QED_CHAIN_CNT_TYPE_U32,
 930                                                    chain_entries,
 931                                                    sizeof(union rdma_cqe),
 932                                                    &cq->pbl, NULL);
 933                 if (rc)
 934                         goto err1;
 935
 936                 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 937                 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 938                 cq->ibcq.cqe = cq->pbl.capacity;
 939         }
 940
 941         qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 942                             pbl_ptr, &params);
 943
 944         rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 945         if (rc)
 946                 goto err2;
 947
 948         cq->icid = icid;
 949         cq->sig = QEDR_CQ_MAGIC_NUMBER;
 950         spin_lock_init(&cq->cq_lock);
 951
 952         if (ib_ctx) {
 953                 rc = qedr_copy_cq_uresp(dev, cq, udata);
 954                 if (rc)
 955                         goto err3;
 956         } else {
 957                 /* Generate doorbell address. */
 958                 cq->db_addr = dev->db_addr +
 959                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 960                 cq->db.data.icid = cq->icid;
 961                 cq->db.data.params = DB_AGG_CMD_SET <<
 962                     RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
 963
 964                 /* point to the very last element, passing it we will toggle */
 965                 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
 966                 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 967                 cq->latest_cqe = NULL;
 968                 consume_cqe(cq);
 969                 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
 970         }
 971
 972         DP_DEBUG(dev, QEDR_MSG_CQ,
 973                  "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
 974                  cq->icid, cq, params.cq_size);
 975
 976         return &cq->ibcq;
 977
 978 err3:
 979         destroy_iparams.icid = cq->icid;
 980         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
 981                                   &destroy_oparams);
 982 err2:
 983         if (udata)
 984                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
 985         else
 986                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
 987 err1:
 988         if (udata)
 989                 ib_umem_release(cq->q.umem);
 990 err0:
 991         kfree(cq);
 992         return ERR_PTR(-EINVAL);
 993 }
 994
 995 int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
 996 {
 997         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
 998         struct qedr_cq *cq = get_qedr_cq(ibcq);
 999
1000         DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
1001
1002         return 0;
1003 }
1004
1005 #define QEDR_DESTROY_CQ_MAX_ITERATIONS          (10)
1006 #define QEDR_DESTROY_CQ_ITER_DURATION           (10)
1007
1008 int qedr_destroy_cq(struct ib_cq *ibcq)
1009 {
1010         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1011         struct qed_rdma_destroy_cq_out_params oparams;
1012         struct qed_rdma_destroy_cq_in_params iparams;
1013         struct qedr_cq *cq = get_qedr_cq(ibcq);
1014         int iter;
1015         int rc;
1016
1017         DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1018
1019         cq->destroyed = 1;
1020
1021         /* GSIs CQs are handled by driver, so they don't exist in the FW */
1022         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
1023                 goto done;
1024
1025         iparams.icid = cq->icid;
1026         rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1027         if (rc)
1028                 return rc;
1029
1030         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1031
1032         if (ibcq->uobject && ibcq->uobject->context) {
1033                 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1034                 ib_umem_release(cq->q.umem);
1035         }
1036
1037         /* We don't want the IRQ handler to handle a non-existing CQ so we
1038          * wait until all CNQ interrupts, if any, are received. This will always
1039          * happen and will always happen very fast. If not, then a serious error
1040          * has occured. That is why we can use a long delay.
1041          * We spin for a short time so we don’t lose time on context switching
1042          * in case all the completions are handled in that span. Otherwise
1043          * we sleep for a while and check again. Since the CNQ may be
1044          * associated with (only) the current CPU we use msleep to allow the
1045          * current CPU to be freed.
1046          * The CNQ notification is increased in qedr_irq_handler().
1047          */
1048         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1049         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1050                 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1051                 iter--;
1052         }
1053
1054         iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1055         while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1056                 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1057                 iter--;
1058         }
1059
1060         if (oparams.num_cq_notif != cq->cnq_notif)
1061                 goto err;
1062
1063         /* Note that we don't need to have explicit code to wait for the
1064          * completion of the event handler because it is invoked from the EQ.
1065          * Since the destroy CQ ramrod has also been received on the EQ we can
1066          * be certain that there's no event handler in process.
1067          */
1068 done:
1069         cq->sig = ~cq->sig;
1070
1071         kfree(cq);
1072
1073         return 0;
1074
1075 err:
1076         DP_ERR(dev,
1077                "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n",
1078                cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif);
1079
1080         return -EINVAL;
1081 }
1082
1083 static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1084                                           struct ib_qp_attr *attr,
1085                                           int attr_mask,
1086                                           struct qed_rdma_modify_qp_in_params
1087                                           *qp_params)
1088 {
1089         const struct ib_gid_attr *gid_attr;
1090         enum rdma_network_type nw_type;
1091         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1092         u32 ipv4_addr;
1093         int i;
1094
1095         gid_attr = grh->sgid_attr;
1096         qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr->ndev);
1097
1098         nw_type = rdma_gid_attr_network_type(gid_attr);
1099         switch (nw_type) {
1100         case RDMA_NETWORK_IPV6:
1101                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1102                        sizeof(qp_params->sgid));
1103                 memcpy(&qp_params->dgid.bytes[0],
1104                        &grh->dgid,
1105                        sizeof(qp_params->dgid));
1106                 qp_params->roce_mode = ROCE_V2_IPV6;
1107                 SET_FIELD(qp_params->modify_flags,
1108                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1109                 break;
1110         case RDMA_NETWORK_IB:
1111                 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1112                        sizeof(qp_params->sgid));
1113                 memcpy(&qp_params->dgid.bytes[0],
1114                        &grh->dgid,
1115                        sizeof(qp_params->dgid));
1116                 qp_params->roce_mode = ROCE_V1;
1117                 break;
1118         case RDMA_NETWORK_IPV4:
1119                 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1120                 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1121                 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1122                 qp_params->sgid.ipv4_addr = ipv4_addr;
1123                 ipv4_addr =
1124                     qedr_get_ipv4_from_gid(grh->dgid.raw);
1125                 qp_params->dgid.ipv4_addr = ipv4_addr;
1126                 SET_FIELD(qp_params->modify_flags,
1127                           QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1128                 qp_params->roce_mode = ROCE_V2_IPV4;
1129                 break;
1130         }
1131
1132         for (i = 0; i < 4; i++) {
1133                 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1134                 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1135         }
1136
1137         if (qp_params->vlan_id >= VLAN_CFI_MASK)
1138                 qp_params->vlan_id = 0;
1139
1140         return 0;
1141 }
1142
1143 static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1144                                struct ib_qp_init_attr *attrs)
1145 {
1146         struct qedr_device_attr *qattr = &dev->attr;
1147
1148         /* QP0... attrs->qp_type == IB_QPT_GSI */
1149         if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1150                 DP_DEBUG(dev, QEDR_MSG_QP,
1151                          "create qp: unsupported qp type=0x%x requested\n",
1152                          attrs->qp_type);
1153                 return -EINVAL;
1154         }
1155
1156         if (attrs->cap.max_send_wr > qattr->max_sqe) {
1157                 DP_ERR(dev,
1158                        "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1159                        attrs->cap.max_send_wr, qattr->max_sqe);
1160                 return -EINVAL;
1161         }
1162
1163         if (attrs->cap.max_inline_data > qattr->max_inline) {
1164                 DP_ERR(dev,
1165                        "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1166                        attrs->cap.max_inline_data, qattr->max_inline);
1167                 return -EINVAL;
1168         }
1169
1170         if (attrs->cap.max_send_sge > qattr->max_sge) {
1171                 DP_ERR(dev,
1172                        "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1173                        attrs->cap.max_send_sge, qattr->max_sge);
1174                 return -EINVAL;
1175         }
1176
1177         if (attrs->cap.max_recv_sge > qattr->max_sge) {
1178                 DP_ERR(dev,
1179                        "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1180                        attrs->cap.max_recv_sge, qattr->max_sge);
1181                 return -EINVAL;
1182         }
1183
1184         /* Unprivileged user space cannot create special QP */
1185         if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1186                 DP_ERR(dev,
1187                        "create qp: userspace can't create special QPs of type=0x%x\n",
1188                        attrs->qp_type);
1189                 return -EINVAL;
1190         }
1191
1192         return 0;
1193 }
1194
1195 static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1196                                struct qedr_srq *srq, struct ib_udata *udata)
1197 {
1198         struct qedr_create_srq_uresp uresp = {};
1199         int rc;
1200
1201         uresp.srq_id = srq->srq_id;
1202
1203         rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1204         if (rc)
1205                 DP_ERR(dev, "create srq: problem copying data to user space\n");
1206
1207         return rc;
1208 }
1209
1210 static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1211                                struct qedr_create_qp_uresp *uresp,
1212                                struct qedr_qp *qp)
1213 {
1214         /* iWARP requires two doorbells per RQ. */
1215         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1216                 uresp->rq_db_offset =
1217                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1218                 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1219         } else {
1220                 uresp->rq_db_offset =
1221                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1222         }
1223
1224         uresp->rq_icid = qp->icid;
1225 }
1226
1227 static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1228                                struct qedr_create_qp_uresp *uresp,
1229                                struct qedr_qp *qp)
1230 {
1231         uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1232
1233         /* iWARP uses the same cid for rq and sq */
1234         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1235                 uresp->sq_icid = qp->icid;
1236         else
1237                 uresp->sq_icid = qp->icid + 1;
1238 }
1239
1240 static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1241                               struct qedr_qp *qp, struct ib_udata *udata)
1242 {
1243         struct qedr_create_qp_uresp uresp;
1244         int rc;
1245
1246         memset(&uresp, 0, sizeof(uresp));
1247         qedr_copy_sq_uresp(dev, &uresp, qp);
1248         qedr_copy_rq_uresp(dev, &uresp, qp);
1249
1250         uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1251         uresp.qp_id = qp->qp_id;
1252
1253         rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1254         if (rc)
1255                 DP_ERR(dev,
1256                        "create qp: failed a copy to user space with qp icid=0x%x.\n",
1257                        qp->icid);
1258
1259         return rc;
1260 }
1261
1262 static void qedr_set_common_qp_params(struct qedr_dev *dev,
1263                                       struct qedr_qp *qp,
1264                                       struct qedr_pd *pd,
1265                                       struct ib_qp_init_attr *attrs)
1266 {
1267         spin_lock_init(&qp->q_lock);
1268         atomic_set(&qp->refcnt, 1);
1269         qp->pd = pd;
1270         qp->qp_type = attrs->qp_type;
1271         qp->max_inline_data = attrs->cap.max_inline_data;
1272         qp->sq.max_sges = attrs->cap.max_send_sge;
1273         qp->state = QED_ROCE_QP_STATE_RESET;
1274         qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1275         qp->sq_cq = get_qedr_cq(attrs->send_cq);
1276         qp->dev = dev;
1277
1278         if (attrs->srq) {
1279                 qp->srq = get_qedr_srq(attrs->srq);
1280         } else {
1281                 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1282                 qp->rq.max_sges = attrs->cap.max_recv_sge;
1283                 DP_DEBUG(dev, QEDR_MSG_QP,
1284                          "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1285                          qp->rq.max_sges, qp->rq_cq->icid);
1286         }
1287
1288         DP_DEBUG(dev, QEDR_MSG_QP,
1289                  "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1290                  pd->pd_id, qp->qp_type, qp->max_inline_data,
1291                  qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1292         DP_DEBUG(dev, QEDR_MSG_QP,
1293                  "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1294                  qp->sq.max_sges, qp->sq_cq->icid);
1295 }
1296
1297 static void qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1298 {
1299         qp->sq.db = dev->db_addr +
1300                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1301         qp->sq.db_data.data.icid = qp->icid + 1;
1302         if (!qp->srq) {
1303                 qp->rq.db = dev->db_addr +
1304                             DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1305                 qp->rq.db_data.data.icid = qp->icid;
1306         }
1307 }
1308
1309 static int qedr_check_srq_params(struct ib_pd *ibpd, struct qedr_dev *dev,
1310                                  struct ib_srq_init_attr *attrs,
1311                                  struct ib_udata *udata)
1312 {
1313         struct qedr_device_attr *qattr = &dev->attr;
1314
1315         if (attrs->attr.max_wr > qattr->max_srq_wr) {
1316                 DP_ERR(dev,
1317                        "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1318                        attrs->attr.max_wr, qattr->max_srq_wr);
1319                 return -EINVAL;
1320         }
1321
1322         if (attrs->attr.max_sge > qattr->max_sge) {
1323                 DP_ERR(dev,
1324                        "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1325                        attrs->attr.max_sge, qattr->max_sge);
1326                 return -EINVAL;
1327         }
1328
1329         return 0;
1330 }
1331
1332 static void qedr_free_srq_user_params(struct qedr_srq *srq)
1333 {
1334         qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1335         ib_umem_release(srq->usrq.umem);
1336         ib_umem_release(srq->prod_umem);
1337 }
1338
1339 static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1340 {
1341         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1342         struct qedr_dev *dev = srq->dev;
1343
1344         dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1345
1346         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1347                           hw_srq->virt_prod_pair_addr,
1348                           hw_srq->phy_prod_pair_addr);
1349 }
1350
1351 static int qedr_init_srq_user_params(struct ib_ucontext *ib_ctx,
1352                                      struct qedr_srq *srq,
1353                                      struct qedr_create_srq_ureq *ureq,
1354                                      int access, int dmasync)
1355 {
1356         struct scatterlist *sg;
1357         int rc;
1358
1359         rc = qedr_init_user_queue(ib_ctx, srq->dev, &srq->usrq, ureq->srq_addr,
1360                                   ureq->srq_len, access, dmasync, 1);
1361         if (rc)
1362                 return rc;
1363
1364         srq->prod_umem = ib_umem_get(ib_ctx, ureq->prod_pair_addr,
1365                                      sizeof(struct rdma_srq_producers),
1366                                      access, dmasync);
1367         if (IS_ERR(srq->prod_umem)) {
1368                 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1369                 ib_umem_release(srq->usrq.umem);
1370                 DP_ERR(srq->dev,
1371                        "create srq: failed ib_umem_get for producer, got %ld\n",
1372                        PTR_ERR(srq->prod_umem));
1373                 return PTR_ERR(srq->prod_umem);
1374         }
1375
1376         sg = srq->prod_umem->sg_head.sgl;
1377         srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1378
1379         return 0;
1380 }
1381
1382 static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1383                                         struct qedr_dev *dev,
1384                                         struct ib_srq_init_attr *init_attr)
1385 {
1386         struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1387         dma_addr_t phy_prod_pair_addr;
1388         u32 num_elems;
1389         void *va;
1390         int rc;
1391
1392         va = dma_alloc_coherent(&dev->pdev->dev,
1393                                 sizeof(struct rdma_srq_producers),
1394                                 &phy_prod_pair_addr, GFP_KERNEL);
1395         if (!va) {
1396                 DP_ERR(dev,
1397                        "create srq: failed to allocate dma memory for producer\n");
1398                 return -ENOMEM;
1399         }
1400
1401         hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1402         hw_srq->virt_prod_pair_addr = va;
1403
1404         num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1405         rc = dev->ops->common->chain_alloc(dev->cdev,
1406                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1407                                            QED_CHAIN_MODE_PBL,
1408                                            QED_CHAIN_CNT_TYPE_U32,
1409                                            num_elems,
1410                                            QEDR_SRQ_WQE_ELEM_SIZE,
1411                                            &hw_srq->pbl, NULL);
1412         if (rc)
1413                 goto err0;
1414
1415         hw_srq->num_elems = num_elems;
1416
1417         return 0;
1418
1419 err0:
1420         dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1421                           va, phy_prod_pair_addr);
1422         return rc;
1423 }
1424
1425 static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr,
1426                         void *ptr, u32 id);
1427 static void qedr_idr_remove(struct qedr_dev *dev,
1428                             struct qedr_idr *qidr, u32 id);
1429
1430 struct ib_srq *qedr_create_srq(struct ib_pd *ibpd,
1431                                struct ib_srq_init_attr *init_attr,
1432                                struct ib_udata *udata)
1433 {
1434         struct qed_rdma_destroy_srq_in_params destroy_in_params;
1435         struct qed_rdma_create_srq_in_params in_params = {};
1436         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1437         struct qed_rdma_create_srq_out_params out_params;
1438         struct qedr_pd *pd = get_qedr_pd(ibpd);
1439         struct qedr_create_srq_ureq ureq = {};
1440         u64 pbl_base_addr, phy_prod_pair_addr;
1441         struct ib_ucontext *ib_ctx = NULL;
1442         struct qedr_srq_hwq_info *hw_srq;
1443         struct qedr_ucontext *ctx = NULL;
1444         u32 page_cnt, page_size;
1445         struct qedr_srq *srq;
1446         int rc = 0;
1447
1448         DP_DEBUG(dev, QEDR_MSG_QP,
1449                  "create SRQ called from %s (pd %p)\n",
1450                  (udata) ? "User lib" : "kernel", pd);
1451
1452         rc = qedr_check_srq_params(ibpd, dev, init_attr, udata);
1453         if (rc)
1454                 return ERR_PTR(-EINVAL);
1455
1456         srq = kzalloc(sizeof(*srq), GFP_KERNEL);
1457         if (!srq)
1458                 return ERR_PTR(-ENOMEM);
1459
1460         srq->dev = dev;
1461         hw_srq = &srq->hw_srq;
1462         spin_lock_init(&srq->lock);
1463
1464         hw_srq->max_wr = init_attr->attr.max_wr;
1465         hw_srq->max_sges = init_attr->attr.max_sge;
1466
1467         if (udata && ibpd->uobject && ibpd->uobject->context) {
1468                 ib_ctx = ibpd->uobject->context;
1469                 ctx = get_qedr_ucontext(ib_ctx);
1470
1471                 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1472                         DP_ERR(dev,
1473                                "create srq: problem copying data from user space\n");
1474                         goto err0;
1475                 }
1476
1477                 rc = qedr_init_srq_user_params(ib_ctx, srq, &ureq, 0, 0);
1478                 if (rc)
1479                         goto err0;
1480
1481                 page_cnt = srq->usrq.pbl_info.num_pbes;
1482                 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1483                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1484                 page_size = BIT(srq->usrq.umem->page_shift);
1485         } else {
1486                 struct qed_chain *pbl;
1487
1488                 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1489                 if (rc)
1490                         goto err0;
1491
1492                 pbl = &hw_srq->pbl;
1493                 page_cnt = qed_chain_get_page_cnt(pbl);
1494                 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1495                 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1496                 page_size = QED_CHAIN_PAGE_SIZE;
1497         }
1498
1499         in_params.pd_id = pd->pd_id;
1500         in_params.pbl_base_addr = pbl_base_addr;
1501         in_params.prod_pair_addr = phy_prod_pair_addr;
1502         in_params.num_pages = page_cnt;
1503         in_params.page_size = page_size;
1504
1505         rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1506         if (rc)
1507                 goto err1;
1508
1509         srq->srq_id = out_params.srq_id;
1510
1511         if (udata) {
1512                 rc = qedr_copy_srq_uresp(dev, srq, udata);
1513                 if (rc)
1514                         goto err2;
1515         }
1516
1517         rc = qedr_idr_add(dev, &dev->srqidr, srq, srq->srq_id);
1518         if (rc)
1519                 goto err2;
1520
1521         DP_DEBUG(dev, QEDR_MSG_SRQ,
1522                  "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1523         return &srq->ibsrq;
1524
1525 err2:
1526         destroy_in_params.srq_id = srq->srq_id;
1527
1528         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1529 err1:
1530         if (udata)
1531                 qedr_free_srq_user_params(srq);
1532         else
1533                 qedr_free_srq_kernel_params(srq);
1534 err0:
1535         kfree(srq);
1536
1537         return ERR_PTR(-EFAULT);
1538 }
1539
1540 int qedr_destroy_srq(struct ib_srq *ibsrq)
1541 {
1542         struct qed_rdma_destroy_srq_in_params in_params = {};
1543         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1544         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1545
1546         qedr_idr_remove(dev, &dev->srqidr, srq->srq_id);
1547         in_params.srq_id = srq->srq_id;
1548         dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1549
1550         if (ibsrq->pd->uobject)
1551                 qedr_free_srq_user_params(srq);
1552         else
1553                 qedr_free_srq_kernel_params(srq);
1554
1555         DP_DEBUG(dev, QEDR_MSG_SRQ,
1556                  "destroy srq: destroyed srq with srq_id=0x%0x\n",
1557                  srq->srq_id);
1558         kfree(srq);
1559
1560         return 0;
1561 }
1562
1563 int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1564                     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1565 {
1566         struct qed_rdma_modify_srq_in_params in_params = {};
1567         struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1568         struct qedr_srq *srq = get_qedr_srq(ibsrq);
1569         int rc;
1570
1571         if (attr_mask & IB_SRQ_MAX_WR) {
1572                 DP_ERR(dev,
1573                        "modify srq: invalid attribute mask=0x%x specified for %p\n",
1574                        attr_mask, srq);
1575                 return -EINVAL;
1576         }
1577
1578         if (attr_mask & IB_SRQ_LIMIT) {
1579                 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1580                         DP_ERR(dev,
1581                                "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1582                                attr->srq_limit, srq->hw_srq.max_wr);
1583                         return -EINVAL;
1584                 }
1585
1586                 in_params.srq_id = srq->srq_id;
1587                 in_params.wqe_limit = attr->srq_limit;
1588                 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1589                 if (rc)
1590                         return rc;
1591         }
1592
1593         srq->srq_limit = attr->srq_limit;
1594
1595         DP_DEBUG(dev, QEDR_MSG_SRQ,
1596                  "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1597
1598         return 0;
1599 }
1600
1601 static inline void
1602 qedr_init_common_qp_in_params(struct qedr_dev *dev,
1603                               struct qedr_pd *pd,
1604                               struct qedr_qp *qp,
1605                               struct ib_qp_init_attr *attrs,
1606                               bool fmr_and_reserved_lkey,
1607                               struct qed_rdma_create_qp_in_params *params)
1608 {
1609         /* QP handle to be written in an async event */
1610         params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1611         params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1612
1613         params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1614         params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1615         params->pd = pd->pd_id;
1616         params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1617         params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1618         params->stats_queue = 0;
1619         params->srq_id = 0;
1620         params->use_srq = false;
1621
1622         if (!qp->srq) {
1623                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1624
1625         } else {
1626                 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1627                 params->srq_id = qp->srq->srq_id;
1628                 params->use_srq = true;
1629         }
1630 }
1631
1632 static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1633 {
1634         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1635                  "qp=%p. "
1636                  "sq_addr=0x%llx, "
1637                  "sq_len=%zd, "
1638                  "rq_addr=0x%llx, "
1639                  "rq_len=%zd"
1640                  "\n",
1641                  qp,
1642                  qp->usq.buf_addr,
1643                  qp->usq.buf_len, qp->urq.buf_addr, qp->urq.buf_len);
1644 }
1645
1646 static int qedr_idr_add(struct qedr_dev *dev, struct qedr_idr *qidr,
1647                         void *ptr, u32 id)
1648 {
1649         int rc;
1650
1651         idr_preload(GFP_KERNEL);
1652         spin_lock_irq(&qidr->idr_lock);
1653
1654         rc = idr_alloc(&qidr->idr, ptr, id, id + 1, GFP_ATOMIC);
1655
1656         spin_unlock_irq(&qidr->idr_lock);
1657         idr_preload_end();
1658
1659         return rc < 0 ? rc : 0;
1660 }
1661
1662 static void qedr_idr_remove(struct qedr_dev *dev, struct qedr_idr *qidr, u32 id)
1663 {
1664         spin_lock_irq(&qidr->idr_lock);
1665         idr_remove(&qidr->idr, id);
1666         spin_unlock_irq(&qidr->idr_lock);
1667 }
1668
1669 static inline void
1670 qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1671                             struct qedr_qp *qp,
1672                             struct qed_rdma_create_qp_out_params *out_params)
1673 {
1674         qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1675         qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1676
1677         qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1678                            &qp->usq.pbl_info, FW_PAGE_SHIFT);
1679         if (!qp->srq) {
1680                 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1681                 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1682         }
1683
1684         qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1685                            &qp->urq.pbl_info, FW_PAGE_SHIFT);
1686 }
1687
1688 static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp)
1689 {
1690         if (qp->usq.umem)
1691                 ib_umem_release(qp->usq.umem);
1692         qp->usq.umem = NULL;
1693
1694         if (qp->urq.umem)
1695                 ib_umem_release(qp->urq.umem);
1696         qp->urq.umem = NULL;
1697
1698         if (rdma_protocol_roce(&dev->ibdev, 1)) {
1699                 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1700                 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1701         } else {
1702                 kfree(qp->usq.pbl_tbl);
1703                 kfree(qp->urq.pbl_tbl);
1704         }
1705 }
1706
1707 static int qedr_create_user_qp(struct qedr_dev *dev,
1708                                struct qedr_qp *qp,
1709                                struct ib_pd *ibpd,
1710                                struct ib_udata *udata,
1711                                struct ib_qp_init_attr *attrs)
1712 {
1713         struct qed_rdma_create_qp_in_params in_params;
1714         struct qed_rdma_create_qp_out_params out_params;
1715         struct qedr_pd *pd = get_qedr_pd(ibpd);
1716         struct ib_ucontext *ib_ctx = NULL;
1717         struct qedr_create_qp_ureq ureq;
1718         int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1719         int rc = -EINVAL;
1720
1721         ib_ctx = ibpd->uobject->context;
1722
1723         memset(&ureq, 0, sizeof(ureq));
1724         rc = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1725         if (rc) {
1726                 DP_ERR(dev, "Problem copying data from user space\n");
1727                 return rc;
1728         }
1729
1730         /* SQ - read access only (0), dma sync not required (0) */
1731         rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq.sq_addr,
1732                                   ureq.sq_len, 0, 0, alloc_and_init);
1733         if (rc)
1734                 return rc;
1735
1736         if (!qp->srq) {
1737                 /* RQ - read access only (0), dma sync not required (0) */
1738                 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq.rq_addr,
1739                                           ureq.rq_len, 0, 0, alloc_and_init);
1740                 if (rc)
1741                         return rc;
1742         }
1743
1744         memset(&in_params, 0, sizeof(in_params));
1745         qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1746         in_params.qp_handle_lo = ureq.qp_handle_lo;
1747         in_params.qp_handle_hi = ureq.qp_handle_hi;
1748         in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1749         in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1750         if (!qp->srq) {
1751                 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1752                 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1753         }
1754
1755         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1756                                               &in_params, &out_params);
1757
1758         if (!qp->qed_qp) {
1759                 rc = -ENOMEM;
1760                 goto err1;
1761         }
1762
1763         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1764                 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1765
1766         qp->qp_id = out_params.qp_id;
1767         qp->icid = out_params.icid;
1768
1769         rc = qedr_copy_qp_uresp(dev, qp, udata);
1770         if (rc)
1771                 goto err;
1772
1773         qedr_qp_user_print(dev, qp);
1774
1775         return 0;
1776 err:
1777         rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1778         if (rc)
1779                 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1780
1781 err1:
1782         qedr_cleanup_user(dev, qp);
1783         return rc;
1784 }
1785
1786 static void qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1787 {
1788         qp->sq.db = dev->db_addr +
1789             DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1790         qp->sq.db_data.data.icid = qp->icid;
1791
1792         qp->rq.db = dev->db_addr +
1793                     DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1794         qp->rq.db_data.data.icid = qp->icid;
1795         qp->rq.iwarp_db2 = dev->db_addr +
1796                            DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1797         qp->rq.iwarp_db2_data.data.icid = qp->icid;
1798         qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
1799 }
1800
1801 static int
1802 qedr_roce_create_kernel_qp(struct qedr_dev *dev,
1803                            struct qedr_qp *qp,
1804                            struct qed_rdma_create_qp_in_params *in_params,
1805                            u32 n_sq_elems, u32 n_rq_elems)
1806 {
1807         struct qed_rdma_create_qp_out_params out_params;
1808         int rc;
1809
1810         rc = dev->ops->common->chain_alloc(dev->cdev,
1811                                            QED_CHAIN_USE_TO_PRODUCE,
1812                                            QED_CHAIN_MODE_PBL,
1813                                            QED_CHAIN_CNT_TYPE_U32,
1814                                            n_sq_elems,
1815                                            QEDR_SQE_ELEMENT_SIZE,
1816                                            &qp->sq.pbl, NULL);
1817
1818         if (rc)
1819                 return rc;
1820
1821         in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1822         in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1823
1824         rc = dev->ops->common->chain_alloc(dev->cdev,
1825                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1826                                            QED_CHAIN_MODE_PBL,
1827                                            QED_CHAIN_CNT_TYPE_U32,
1828                                            n_rq_elems,
1829                                            QEDR_RQE_ELEMENT_SIZE,
1830                                            &qp->rq.pbl, NULL);
1831         if (rc)
1832                 return rc;
1833
1834         in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1835         in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1836
1837         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1838                                               in_params, &out_params);
1839
1840         if (!qp->qed_qp)
1841                 return -EINVAL;
1842
1843         qp->qp_id = out_params.qp_id;
1844         qp->icid = out_params.icid;
1845
1846         qedr_set_roce_db_info(dev, qp);
1847         return rc;
1848 }
1849
1850 static int
1851 qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
1852                             struct qedr_qp *qp,
1853                             struct qed_rdma_create_qp_in_params *in_params,
1854                             u32 n_sq_elems, u32 n_rq_elems)
1855 {
1856         struct qed_rdma_create_qp_out_params out_params;
1857         struct qed_chain_ext_pbl ext_pbl;
1858         int rc;
1859
1860         in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
1861                                                      QEDR_SQE_ELEMENT_SIZE,
1862                                                      QED_CHAIN_MODE_PBL);
1863         in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
1864                                                      QEDR_RQE_ELEMENT_SIZE,
1865                                                      QED_CHAIN_MODE_PBL);
1866
1867         qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1868                                               in_params, &out_params);
1869
1870         if (!qp->qed_qp)
1871                 return -EINVAL;
1872
1873         /* Now we allocate the chain */
1874         ext_pbl.p_pbl_virt = out_params.sq_pbl_virt;
1875         ext_pbl.p_pbl_phys = out_params.sq_pbl_phys;
1876
1877         rc = dev->ops->common->chain_alloc(dev->cdev,
1878                                            QED_CHAIN_USE_TO_PRODUCE,
1879                                            QED_CHAIN_MODE_PBL,
1880                                            QED_CHAIN_CNT_TYPE_U32,
1881                                            n_sq_elems,
1882                                            QEDR_SQE_ELEMENT_SIZE,
1883                                            &qp->sq.pbl, &ext_pbl);
1884
1885         if (rc)
1886                 goto err;
1887
1888         ext_pbl.p_pbl_virt = out_params.rq_pbl_virt;
1889         ext_pbl.p_pbl_phys = out_params.rq_pbl_phys;
1890
1891         rc = dev->ops->common->chain_alloc(dev->cdev,
1892                                            QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1893                                            QED_CHAIN_MODE_PBL,
1894                                            QED_CHAIN_CNT_TYPE_U32,
1895                                            n_rq_elems,
1896                                            QEDR_RQE_ELEMENT_SIZE,
1897                                            &qp->rq.pbl, &ext_pbl);
1898
1899         if (rc)
1900                 goto err;
1901
1902         qp->qp_id = out_params.qp_id;
1903         qp->icid = out_params.icid;
1904
1905         qedr_set_iwarp_db_info(dev, qp);
1906         return rc;
1907
1908 err:
1909         dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1910
1911         return rc;
1912 }
1913
1914 static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
1915 {
1916         dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1917         kfree(qp->wqe_wr_id);
1918
1919         dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1920         kfree(qp->rqe_wr_id);
1921 }
1922
1923 static int qedr_create_kernel_qp(struct qedr_dev *dev,
1924                                  struct qedr_qp *qp,
1925                                  struct ib_pd *ibpd,
1926                                  struct ib_qp_init_attr *attrs)
1927 {
1928         struct qed_rdma_create_qp_in_params in_params;
1929         struct qedr_pd *pd = get_qedr_pd(ibpd);
1930         int rc = -EINVAL;
1931         u32 n_rq_elems;
1932         u32 n_sq_elems;
1933         u32 n_sq_entries;
1934
1935         memset(&in_params, 0, sizeof(in_params));
1936
1937         /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1938          * the ring. The ring should allow at least a single WR, even if the
1939          * user requested none, due to allocation issues.
1940          * We should add an extra WR since the prod and cons indices of
1941          * wqe_wr_id are managed in such a way that the WQ is considered full
1942          * when (prod+1)%max_wr==cons. We currently don't do that because we
1943          * double the number of entries due an iSER issue that pushes far more
1944          * WRs than indicated. If we decline its ib_post_send() then we get
1945          * error prints in the dmesg we'd like to avoid.
1946          */
1947         qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
1948                               dev->attr.max_sqe);
1949
1950         qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1951                                 GFP_KERNEL);
1952         if (!qp->wqe_wr_id) {
1953                 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
1954                 return -ENOMEM;
1955         }
1956
1957         /* QP handle to be written in CQE */
1958         in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
1959         in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
1960
1961         /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1962          * the ring. There ring should allow at least a single WR, even if the
1963          * user requested none, due to allocation issues.
1964          */
1965         qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
1966
1967         /* Allocate driver internal RQ array */
1968         qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1969                                 GFP_KERNEL);
1970         if (!qp->rqe_wr_id) {
1971                 DP_ERR(dev,
1972                        "create qp: failed RQ shadow memory allocation\n");
1973                 kfree(qp->wqe_wr_id);
1974                 return -ENOMEM;
1975         }
1976
1977         qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
1978
1979         n_sq_entries = attrs->cap.max_send_wr;
1980         n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1981         n_sq_entries = max_t(u32, n_sq_entries, 1);
1982         n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1983
1984         n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1985
1986         if (rdma_protocol_iwarp(&dev->ibdev, 1))
1987                 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
1988                                                  n_sq_elems, n_rq_elems);
1989         else
1990                 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
1991                                                 n_sq_elems, n_rq_elems);
1992         if (rc)
1993                 qedr_cleanup_kernel(dev, qp);
1994
1995         return rc;
1996 }
1997
1998 struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1999                              struct ib_qp_init_attr *attrs,
2000                              struct ib_udata *udata)
2001 {
2002         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2003         struct qedr_pd *pd = get_qedr_pd(ibpd);
2004         struct qedr_qp *qp;
2005         struct ib_qp *ibqp;
2006         int rc = 0;
2007
2008         DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2009                  udata ? "user library" : "kernel", pd);
2010
2011         rc = qedr_check_qp_attrs(ibpd, dev, attrs);
2012         if (rc)
2013                 return ERR_PTR(rc);
2014
2015         DP_DEBUG(dev, QEDR_MSG_QP,
2016                  "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2017                  udata ? "user library" : "kernel", attrs->event_handler, pd,
2018                  get_qedr_cq(attrs->send_cq),
2019                  get_qedr_cq(attrs->send_cq)->icid,
2020                  get_qedr_cq(attrs->recv_cq),
2021                  attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2022
2023         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
2024         if (!qp) {
2025                 DP_ERR(dev, "create qp: failed allocating memory\n");
2026                 return ERR_PTR(-ENOMEM);
2027         }
2028
2029         qedr_set_common_qp_params(dev, qp, pd, attrs);
2030
2031         if (attrs->qp_type == IB_QPT_GSI) {
2032                 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
2033                 if (IS_ERR(ibqp))
2034                         kfree(qp);
2035                 return ibqp;
2036         }
2037
2038         if (udata)
2039                 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2040         else
2041                 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2042
2043         if (rc)
2044                 goto err;
2045
2046         qp->ibqp.qp_num = qp->qp_id;
2047
2048         if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2049                 rc = qedr_idr_add(dev, &dev->qpidr, qp, qp->qp_id);
2050                 if (rc)
2051                         goto err;
2052         }
2053
2054         return &qp->ibqp;
2055
2056 err:
2057         kfree(qp);
2058
2059         return ERR_PTR(-EFAULT);
2060 }
2061
2062 static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2063 {
2064         switch (qp_state) {
2065         case QED_ROCE_QP_STATE_RESET:
2066                 return IB_QPS_RESET;
2067         case QED_ROCE_QP_STATE_INIT:
2068                 return IB_QPS_INIT;
2069         case QED_ROCE_QP_STATE_RTR:
2070                 return IB_QPS_RTR;
2071         case QED_ROCE_QP_STATE_RTS:
2072                 return IB_QPS_RTS;
2073         case QED_ROCE_QP_STATE_SQD:
2074                 return IB_QPS_SQD;
2075         case QED_ROCE_QP_STATE_ERR:
2076                 return IB_QPS_ERR;
2077         case QED_ROCE_QP_STATE_SQE:
2078                 return IB_QPS_SQE;
2079         }
2080         return IB_QPS_ERR;
2081 }
2082
2083 static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2084                                         enum ib_qp_state qp_state)
2085 {
2086         switch (qp_state) {
2087         case IB_QPS_RESET:
2088                 return QED_ROCE_QP_STATE_RESET;
2089         case IB_QPS_INIT:
2090                 return QED_ROCE_QP_STATE_INIT;
2091         case IB_QPS_RTR:
2092                 return QED_ROCE_QP_STATE_RTR;
2093         case IB_QPS_RTS:
2094                 return QED_ROCE_QP_STATE_RTS;
2095         case IB_QPS_SQD:
2096                 return QED_ROCE_QP_STATE_SQD;
2097         case IB_QPS_ERR:
2098                 return QED_ROCE_QP_STATE_ERR;
2099         default:
2100                 return QED_ROCE_QP_STATE_ERR;
2101         }
2102 }
2103
2104 static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
2105 {
2106         qed_chain_reset(&qph->pbl);
2107         qph->prod = 0;
2108         qph->cons = 0;
2109         qph->wqe_cons = 0;
2110         qph->db_data.data.value = cpu_to_le16(0);
2111 }
2112
2113 static int qedr_update_qp_state(struct qedr_dev *dev,
2114                                 struct qedr_qp *qp,
2115                                 enum qed_roce_qp_state cur_state,
2116                                 enum qed_roce_qp_state new_state)
2117 {
2118         int status = 0;
2119
2120         if (new_state == cur_state)
2121                 return 0;
2122
2123         switch (cur_state) {
2124         case QED_ROCE_QP_STATE_RESET:
2125                 switch (new_state) {
2126                 case QED_ROCE_QP_STATE_INIT:
2127                         qp->prev_wqe_size = 0;
2128                         qedr_reset_qp_hwq_info(&qp->sq);
2129                         qedr_reset_qp_hwq_info(&qp->rq);
2130                         break;
2131                 default:
2132                         status = -EINVAL;
2133                         break;
2134                 };
2135                 break;
2136         case QED_ROCE_QP_STATE_INIT:
2137                 switch (new_state) {
2138                 case QED_ROCE_QP_STATE_RTR:
2139                         /* Update doorbell (in case post_recv was
2140                          * done before move to RTR)
2141                          */
2142
2143                         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2144                                 writel(qp->rq.db_data.raw, qp->rq.db);
2145                                 /* Make sure write takes effect */
2146                                 mmiowb();
2147                         }
2148                         break;
2149                 case QED_ROCE_QP_STATE_ERR:
2150                         break;
2151                 default:
2152                         /* Invalid state change. */
2153                         status = -EINVAL;
2154                         break;
2155                 };
2156                 break;
2157         case QED_ROCE_QP_STATE_RTR:
2158                 /* RTR->XXX */
2159                 switch (new_state) {
2160                 case QED_ROCE_QP_STATE_RTS:
2161                         break;
2162                 case QED_ROCE_QP_STATE_ERR:
2163                         break;
2164                 default:
2165                         /* Invalid state change. */
2166                         status = -EINVAL;
2167                         break;
2168                 };
2169                 break;
2170         case QED_ROCE_QP_STATE_RTS:
2171                 /* RTS->XXX */
2172                 switch (new_state) {
2173                 case QED_ROCE_QP_STATE_SQD:
2174                         break;
2175                 case QED_ROCE_QP_STATE_ERR:
2176                         break;
2177                 default:
2178                         /* Invalid state change. */
2179                         status = -EINVAL;
2180                         break;
2181                 };
2182                 break;
2183         case QED_ROCE_QP_STATE_SQD:
2184                 /* SQD->XXX */
2185                 switch (new_state) {
2186                 case QED_ROCE_QP_STATE_RTS:
2187                 case QED_ROCE_QP_STATE_ERR:
2188                         break;
2189                 default:
2190                         /* Invalid state change. */
2191                         status = -EINVAL;
2192                         break;
2193                 };
2194                 break;
2195         case QED_ROCE_QP_STATE_ERR:
2196                 /* ERR->XXX */
2197                 switch (new_state) {
2198                 case QED_ROCE_QP_STATE_RESET:
2199                         if ((qp->rq.prod != qp->rq.cons) ||
2200                             (qp->sq.prod != qp->sq.cons)) {
2201                                 DP_NOTICE(dev,
2202                                           "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2203                                           qp->rq.prod, qp->rq.cons, qp->sq.prod,
2204                                           qp->sq.cons);
2205                                 status = -EINVAL;
2206                         }
2207                         break;
2208                 default:
2209                         status = -EINVAL;
2210                         break;
2211                 };
2212                 break;
2213         default:
2214                 status = -EINVAL;
2215                 break;
2216         };
2217
2218         return status;
2219 }
2220
2221 int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2222                    int attr_mask, struct ib_udata *udata)
2223 {
2224         struct qedr_qp *qp = get_qedr_qp(ibqp);
2225         struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2226         struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2227         const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2228         enum ib_qp_state old_qp_state, new_qp_state;
2229         enum qed_roce_qp_state cur_state;
2230         int rc = 0;
2231
2232         DP_DEBUG(dev, QEDR_MSG_QP,
2233                  "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2234                  attr->qp_state);
2235
2236         old_qp_state = qedr_get_ibqp_state(qp->state);
2237         if (attr_mask & IB_QP_STATE)
2238                 new_qp_state = attr->qp_state;
2239         else
2240                 new_qp_state = old_qp_state;
2241
2242         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2243                 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2244                                         ibqp->qp_type, attr_mask,
2245                                         IB_LINK_LAYER_ETHERNET)) {
2246                         DP_ERR(dev,
2247                                "modify qp: invalid attribute mask=0x%x specified for\n"
2248                                "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2249                                attr_mask, qp->qp_id, ibqp->qp_type,
2250                                old_qp_state, new_qp_state);
2251                         rc = -EINVAL;
2252                         goto err;
2253                 }
2254         }
2255
2256         /* Translate the masks... */
2257         if (attr_mask & IB_QP_STATE) {
2258                 SET_FIELD(qp_params.modify_flags,
2259                           QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2260                 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2261         }
2262
2263         if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2264                 qp_params.sqd_async = true;
2265
2266         if (attr_mask & IB_QP_PKEY_INDEX) {
2267                 SET_FIELD(qp_params.modify_flags,
2268                           QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2269                 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2270                         rc = -EINVAL;
2271                         goto err;
2272                 }
2273
2274                 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2275         }
2276
2277         if (attr_mask & IB_QP_QKEY)
2278                 qp->qkey = attr->qkey;
2279
2280         if (attr_mask & IB_QP_ACCESS_FLAGS) {
2281                 SET_FIELD(qp_params.modify_flags,
2282                           QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2283                 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2284                                                   IB_ACCESS_REMOTE_READ;
2285                 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2286                                                    IB_ACCESS_REMOTE_WRITE;
2287                 qp_params.incoming_atomic_en = attr->qp_access_flags &
2288                                                IB_ACCESS_REMOTE_ATOMIC;
2289         }
2290
2291         if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2292                 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2293                         return -EINVAL;
2294
2295                 if (attr_mask & IB_QP_PATH_MTU) {
2296                         if (attr->path_mtu < IB_MTU_256 ||
2297                             attr->path_mtu > IB_MTU_4096) {
2298                                 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2299                                 rc = -EINVAL;
2300                                 goto err;
2301                         }
2302                         qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2303                                       ib_mtu_enum_to_int(iboe_get_mtu
2304                                                          (dev->ndev->mtu)));
2305                 }
2306
2307                 if (!qp->mtu) {
2308                         qp->mtu =
2309                         ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2310                         pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2311                 }
2312
2313                 SET_FIELD(qp_params.modify_flags,
2314                           QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2315
2316                 qp_params.traffic_class_tos = grh->traffic_class;
2317                 qp_params.flow_label = grh->flow_label;
2318                 qp_params.hop_limit_ttl = grh->hop_limit;
2319
2320                 qp->sgid_idx = grh->sgid_index;
2321
2322                 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2323                 if (rc) {
2324                         DP_ERR(dev,
2325                                "modify qp: problems with GID index %d (rc=%d)\n",
2326                                grh->sgid_index, rc);
2327                         return rc;
2328                 }
2329
2330                 rc = qedr_get_dmac(dev, &attr->ah_attr,
2331                                    qp_params.remote_mac_addr);
2332                 if (rc)
2333                         return rc;
2334
2335                 qp_params.use_local_mac = true;
2336                 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2337
2338                 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2339                          qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2340                          qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2341                 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2342                          qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2343                          qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2344                 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2345                          qp_params.remote_mac_addr);
2346
2347                 qp_params.mtu = qp->mtu;
2348                 qp_params.lb_indication = false;
2349         }
2350
2351         if (!qp_params.mtu) {
2352                 /* Stay with current MTU */
2353                 if (qp->mtu)
2354                         qp_params.mtu = qp->mtu;
2355                 else
2356                         qp_params.mtu =
2357                             ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2358         }
2359
2360         if (attr_mask & IB_QP_TIMEOUT) {
2361                 SET_FIELD(qp_params.modify_flags,
2362                           QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2363
2364                 /* The received timeout value is an exponent used like this:
2365                  *    "12.7.34 LOCAL ACK TIMEOUT
2366                  *    Value representing the transport (ACK) timeout for use by
2367                  *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2368                  * The FW expects timeout in msec so we need to divide the usec
2369                  * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2370                  * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2371                  * The value of zero means infinite so we use a 'max_t' to make
2372                  * sure that sub 1 msec values will be configured as 1 msec.
2373                  */
2374                 if (attr->timeout)
2375                         qp_params.ack_timeout =
2376                                         1 << max_t(int, attr->timeout - 8, 0);
2377                 else
2378                         qp_params.ack_timeout = 0;
2379
2380                 qp->timeout = attr->timeout;
2381         }
2382
2383         if (attr_mask & IB_QP_RETRY_CNT) {
2384                 SET_FIELD(qp_params.modify_flags,
2385                           QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2386                 qp_params.retry_cnt = attr->retry_cnt;
2387         }
2388
2389         if (attr_mask & IB_QP_RNR_RETRY) {
2390                 SET_FIELD(qp_params.modify_flags,
2391                           QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2392                 qp_params.rnr_retry_cnt = attr->rnr_retry;
2393         }
2394
2395         if (attr_mask & IB_QP_RQ_PSN) {
2396                 SET_FIELD(qp_params.modify_flags,
2397                           QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2398                 qp_params.rq_psn = attr->rq_psn;
2399                 qp->rq_psn = attr->rq_psn;
2400         }
2401
2402         if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2403                 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2404                         rc = -EINVAL;
2405                         DP_ERR(dev,
2406                                "unsupported max_rd_atomic=%d, supported=%d\n",
2407                                attr->max_rd_atomic,
2408                                dev->attr.max_qp_req_rd_atomic_resc);
2409                         goto err;
2410                 }
2411
2412                 SET_FIELD(qp_params.modify_flags,
2413                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2414                 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2415         }
2416
2417         if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2418                 SET_FIELD(qp_params.modify_flags,
2419                           QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2420                 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2421         }
2422
2423         if (attr_mask & IB_QP_SQ_PSN) {
2424                 SET_FIELD(qp_params.modify_flags,
2425                           QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2426                 qp_params.sq_psn = attr->sq_psn;
2427                 qp->sq_psn = attr->sq_psn;
2428         }
2429
2430         if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2431                 if (attr->max_dest_rd_atomic >
2432                     dev->attr.max_qp_resp_rd_atomic_resc) {
2433                         DP_ERR(dev,
2434                                "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2435                                attr->max_dest_rd_atomic,
2436                                dev->attr.max_qp_resp_rd_atomic_resc);
2437
2438                         rc = -EINVAL;
2439                         goto err;
2440                 }
2441
2442                 SET_FIELD(qp_params.modify_flags,
2443                           QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2444                 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2445         }
2446
2447         if (attr_mask & IB_QP_DEST_QPN) {
2448                 SET_FIELD(qp_params.modify_flags,
2449                           QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2450
2451                 qp_params.dest_qp = attr->dest_qp_num;
2452                 qp->dest_qp_num = attr->dest_qp_num;
2453         }
2454
2455         cur_state = qp->state;
2456
2457         /* Update the QP state before the actual ramrod to prevent a race with
2458          * fast path. Modifying the QP state to error will cause the device to
2459          * flush the CQEs and while polling the flushed CQEs will considered as
2460          * a potential issue if the QP isn't in error state.
2461          */
2462         if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2463             !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2464                 qp->state = QED_ROCE_QP_STATE_ERR;
2465
2466         if (qp->qp_type != IB_QPT_GSI)
2467                 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2468                                               qp->qed_qp, &qp_params);
2469
2470         if (attr_mask & IB_QP_STATE) {
2471                 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2472                         rc = qedr_update_qp_state(dev, qp, cur_state,
2473                                                   qp_params.new_state);
2474                 qp->state = qp_params.new_state;
2475         }
2476
2477 err:
2478         return rc;
2479 }
2480
2481 static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2482 {
2483         int ib_qp_acc_flags = 0;
2484
2485         if (params->incoming_rdma_write_en)
2486                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2487         if (params->incoming_rdma_read_en)
2488                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2489         if (params->incoming_atomic_en)
2490                 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2491         ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2492         return ib_qp_acc_flags;
2493 }
2494
2495 int qedr_query_qp(struct ib_qp *ibqp,
2496                   struct ib_qp_attr *qp_attr,
2497                   int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2498 {
2499         struct qed_rdma_query_qp_out_params params;
2500         struct qedr_qp *qp = get_qedr_qp(ibqp);
2501         struct qedr_dev *dev = qp->dev;
2502         int rc = 0;
2503
2504         memset(&params, 0, sizeof(params));
2505         memset(qp_attr, 0, sizeof(*qp_attr));
2506         memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2507
2508         if (qp->qp_type != IB_QPT_GSI) {
2509                 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2510                 if (rc)
2511                         goto err;
2512                 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2513         } else {
2514                 qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS);
2515         }
2516
2517         qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2518         qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2519         qp_attr->path_mig_state = IB_MIG_MIGRATED;
2520         qp_attr->rq_psn = params.rq_psn;
2521         qp_attr->sq_psn = params.sq_psn;
2522         qp_attr->dest_qp_num = params.dest_qp;
2523
2524         qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2525
2526         qp_attr->cap.max_send_wr = qp->sq.max_wr;
2527         qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2528         qp_attr->cap.max_send_sge = qp->sq.max_sges;
2529         qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2530         qp_attr->cap.max_inline_data = dev->attr.max_inline;
2531         qp_init_attr->cap = qp_attr->cap;
2532
2533         qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2534         rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2535                         params.flow_label, qp->sgid_idx,
2536                         params.hop_limit_ttl, params.traffic_class_tos);
2537         rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2538         rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2539         rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2540         qp_attr->timeout = qp->timeout;
2541         qp_attr->rnr_retry = params.rnr_retry;
2542         qp_attr->retry_cnt = params.retry_cnt;
2543         qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2544         qp_attr->pkey_index = params.pkey_index;
2545         qp_attr->port_num = 1;
2546         rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2547         rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2548         qp_attr->alt_pkey_index = 0;
2549         qp_attr->alt_port_num = 0;
2550         qp_attr->alt_timeout = 0;
2551         memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2552
2553         qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2554         qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2555         qp_attr->max_rd_atomic = params.max_rd_atomic;
2556         qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2557
2558         DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2559                  qp_attr->cap.max_inline_data);
2560
2561 err:
2562         return rc;
2563 }
2564
2565 static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp)
2566 {
2567         int rc = 0;
2568
2569         if (qp->qp_type != IB_QPT_GSI) {
2570                 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2571                 if (rc)
2572                         return rc;
2573         }
2574
2575         if (qp->ibqp.uobject && qp->ibqp.uobject->context)
2576                 qedr_cleanup_user(dev, qp);
2577         else
2578                 qedr_cleanup_kernel(dev, qp);
2579
2580         return 0;
2581 }
2582
2583 int qedr_destroy_qp(struct ib_qp *ibqp)
2584 {
2585         struct qedr_qp *qp = get_qedr_qp(ibqp);
2586         struct qedr_dev *dev = qp->dev;
2587         struct ib_qp_attr attr;
2588         int attr_mask = 0;
2589         int rc = 0;
2590
2591         DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2592                  qp, qp->qp_type);
2593
2594         if (rdma_protocol_roce(&dev->ibdev, 1)) {
2595                 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2596                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
2597                     (qp->state != QED_ROCE_QP_STATE_INIT)) {
2598
2599                         attr.qp_state = IB_QPS_ERR;
2600                         attr_mask |= IB_QP_STATE;
2601
2602                         /* Change the QP state to ERROR */
2603                         qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2604                 }
2605         } else {
2606                 /* Wait for the connect/accept to complete */
2607                 if (qp->ep) {
2608                         int wait_count = 1;
2609
2610                         while (qp->ep->during_connect) {
2611                                 DP_DEBUG(dev, QEDR_MSG_QP,
2612                                          "Still in during connect/accept\n");
2613
2614                                 msleep(100);
2615                                 if (wait_count++ > 200) {
2616                                         DP_NOTICE(dev,
2617                                                   "during connect timeout\n");
2618                                         break;
2619                                 }
2620                         }
2621                 }
2622         }
2623
2624         if (qp->qp_type == IB_QPT_GSI)
2625                 qedr_destroy_gsi_qp(dev);
2626
2627         qedr_free_qp_resources(dev, qp);
2628
2629         if (atomic_dec_and_test(&qp->refcnt) &&
2630             rdma_protocol_iwarp(&dev->ibdev, 1)) {
2631                 qedr_idr_remove(dev, &dev->qpidr, qp->qp_id);
2632                 kfree(qp);
2633         }
2634         return rc;
2635 }
2636
2637 struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct rdma_ah_attr *attr,
2638                              struct ib_udata *udata)
2639 {
2640         struct qedr_ah *ah;
2641
2642         ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2643         if (!ah)
2644                 return ERR_PTR(-ENOMEM);
2645
2646         rdma_copy_ah_attr(&ah->attr, attr);
2647
2648         return &ah->ibah;
2649 }
2650
2651 int qedr_destroy_ah(struct ib_ah *ibah)
2652 {
2653         struct qedr_ah *ah = get_qedr_ah(ibah);
2654
2655         rdma_destroy_ah_attr(&ah->attr);
2656         kfree(ah);
2657         return 0;
2658 }
2659
2660 static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2661 {
2662         struct qedr_pbl *pbl, *tmp;
2663
2664         if (info->pbl_table)
2665                 list_add_tail(&info->pbl_table->list_entry,
2666                               &info->free_pbl_list);
2667
2668         if (!list_empty(&info->inuse_pbl_list))
2669                 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2670
2671         list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2672                 list_del(&pbl->list_entry);
2673                 qedr_free_pbl(dev, &info->pbl_info, pbl);
2674         }
2675 }
2676
2677 static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2678                         size_t page_list_len, bool two_layered)
2679 {
2680         struct qedr_pbl *tmp;
2681         int rc;
2682
2683         INIT_LIST_HEAD(&info->free_pbl_list);
2684         INIT_LIST_HEAD(&info->inuse_pbl_list);
2685
2686         rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2687                                   page_list_len, two_layered);
2688         if (rc)
2689                 goto done;
2690
2691         info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2692         if (IS_ERR(info->pbl_table)) {
2693                 rc = PTR_ERR(info->pbl_table);
2694                 goto done;
2695         }
2696
2697         DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2698                  &info->pbl_table->pa);
2699
2700         /* in usual case we use 2 PBLs, so we add one to free
2701          * list and allocating another one
2702          */
2703         tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2704         if (IS_ERR(tmp)) {
2705                 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2706                 goto done;
2707         }
2708
2709         list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2710
2711         DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2712
2713 done:
2714         if (rc)
2715                 free_mr_info(dev, info);
2716
2717         return rc;
2718 }
2719
2720 struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2721                                u64 usr_addr, int acc, struct ib_udata *udata)
2722 {
2723         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2724         struct qedr_mr *mr;
2725         struct qedr_pd *pd;
2726         int rc = -ENOMEM;
2727
2728         pd = get_qedr_pd(ibpd);
2729         DP_DEBUG(dev, QEDR_MSG_MR,
2730                  "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2731                  pd->pd_id, start, len, usr_addr, acc);
2732
2733         if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2734                 return ERR_PTR(-EINVAL);
2735
2736         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2737         if (!mr)
2738                 return ERR_PTR(rc);
2739
2740         mr->type = QEDR_MR_USER;
2741
2742         mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2743         if (IS_ERR(mr->umem)) {
2744                 rc = -EFAULT;
2745                 goto err0;
2746         }
2747
2748         rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2749         if (rc)
2750                 goto err1;
2751
2752         qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2753                            &mr->info.pbl_info, mr->umem->page_shift);
2754
2755         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2756         if (rc) {
2757                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2758                 goto err1;
2759         }
2760
2761         /* Index only, 18 bit long, lkey = itid << 8 | key */
2762         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2763         mr->hw_mr.key = 0;
2764         mr->hw_mr.pd = pd->pd_id;
2765         mr->hw_mr.local_read = 1;
2766         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2767         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2768         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2769         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2770         mr->hw_mr.mw_bind = false;
2771         mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2772         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2773         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2774         mr->hw_mr.page_size_log = mr->umem->page_shift;
2775         mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2776         mr->hw_mr.length = len;
2777         mr->hw_mr.vaddr = usr_addr;
2778         mr->hw_mr.zbva = false;
2779         mr->hw_mr.phy_mr = false;
2780         mr->hw_mr.dma_mr = false;
2781
2782         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2783         if (rc) {
2784                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2785                 goto err2;
2786         }
2787
2788         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2789         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2790             mr->hw_mr.remote_atomic)
2791                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2792
2793         DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2794                  mr->ibmr.lkey);
2795         return &mr->ibmr;
2796
2797 err2:
2798         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2799 err1:
2800         qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2801 err0:
2802         kfree(mr);
2803         return ERR_PTR(rc);
2804 }
2805
2806 int qedr_dereg_mr(struct ib_mr *ib_mr)
2807 {
2808         struct qedr_mr *mr = get_qedr_mr(ib_mr);
2809         struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2810         int rc = 0;
2811
2812         rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2813         if (rc)
2814                 return rc;
2815
2816         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2817
2818         if (mr->type != QEDR_MR_DMA)
2819                 free_mr_info(dev, &mr->info);
2820
2821         /* it could be user registered memory. */
2822         if (mr->umem)
2823                 ib_umem_release(mr->umem);
2824
2825         kfree(mr);
2826
2827         return rc;
2828 }
2829
2830 static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2831                                        int max_page_list_len)
2832 {
2833         struct qedr_pd *pd = get_qedr_pd(ibpd);
2834         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2835         struct qedr_mr *mr;
2836         int rc = -ENOMEM;
2837
2838         DP_DEBUG(dev, QEDR_MSG_MR,
2839                  "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2840                  max_page_list_len);
2841
2842         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2843         if (!mr)
2844                 return ERR_PTR(rc);
2845
2846         mr->dev = dev;
2847         mr->type = QEDR_MR_FRMR;
2848
2849         rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2850         if (rc)
2851                 goto err0;
2852
2853         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2854         if (rc) {
2855                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2856                 goto err0;
2857         }
2858
2859         /* Index only, 18 bit long, lkey = itid << 8 | key */
2860         mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2861         mr->hw_mr.key = 0;
2862         mr->hw_mr.pd = pd->pd_id;
2863         mr->hw_mr.local_read = 1;
2864         mr->hw_mr.local_write = 0;
2865         mr->hw_mr.remote_read = 0;
2866         mr->hw_mr.remote_write = 0;
2867         mr->hw_mr.remote_atomic = 0;
2868         mr->hw_mr.mw_bind = false;
2869         mr->hw_mr.pbl_ptr = 0;
2870         mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2871         mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2872         mr->hw_mr.fbo = 0;
2873         mr->hw_mr.length = 0;
2874         mr->hw_mr.vaddr = 0;
2875         mr->hw_mr.zbva = false;
2876         mr->hw_mr.phy_mr = true;
2877         mr->hw_mr.dma_mr = false;
2878
2879         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2880         if (rc) {
2881                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2882                 goto err1;
2883         }
2884
2885         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2886         mr->ibmr.rkey = mr->ibmr.lkey;
2887
2888         DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2889         return mr;
2890
2891 err1:
2892         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2893 err0:
2894         kfree(mr);
2895         return ERR_PTR(rc);
2896 }
2897
2898 struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2899                             enum ib_mr_type mr_type, u32 max_num_sg)
2900 {
2901         struct qedr_mr *mr;
2902
2903         if (mr_type != IB_MR_TYPE_MEM_REG)
2904                 return ERR_PTR(-EINVAL);
2905
2906         mr = __qedr_alloc_mr(ibpd, max_num_sg);
2907
2908         if (IS_ERR(mr))
2909                 return ERR_PTR(-EINVAL);
2910
2911         return &mr->ibmr;
2912 }
2913
2914 static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2915 {
2916         struct qedr_mr *mr = get_qedr_mr(ibmr);
2917         struct qedr_pbl *pbl_table;
2918         struct regpair *pbe;
2919         u32 pbes_in_page;
2920
2921         if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2922                 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
2923                 return -ENOMEM;
2924         }
2925
2926         DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2927                  mr->npages, addr);
2928
2929         pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2930         pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2931         pbe = (struct regpair *)pbl_table->va;
2932         pbe +=  mr->npages % pbes_in_page;
2933         pbe->lo = cpu_to_le32((u32)addr);
2934         pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2935
2936         mr->npages++;
2937
2938         return 0;
2939 }
2940
2941 static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2942 {
2943         int work = info->completed - info->completed_handled - 1;
2944
2945         DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2946         while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2947                 struct qedr_pbl *pbl;
2948
2949                 /* Free all the page list that are possible to be freed
2950                  * (all the ones that were invalidated), under the assumption
2951                  * that if an FMR was completed successfully that means that
2952                  * if there was an invalidate operation before it also ended
2953                  */
2954                 pbl = list_first_entry(&info->inuse_pbl_list,
2955                                        struct qedr_pbl, list_entry);
2956                 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2957                 info->completed_handled++;
2958         }
2959 }
2960
2961 int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2962                    int sg_nents, unsigned int *sg_offset)
2963 {
2964         struct qedr_mr *mr = get_qedr_mr(ibmr);
2965
2966         mr->npages = 0;
2967
2968         handle_completed_mrs(mr->dev, &mr->info);
2969         return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2970 }
2971
2972 struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2973 {
2974         struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2975         struct qedr_pd *pd = get_qedr_pd(ibpd);
2976         struct qedr_mr *mr;
2977         int rc;
2978
2979         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2980         if (!mr)
2981                 return ERR_PTR(-ENOMEM);
2982
2983         mr->type = QEDR_MR_DMA;
2984
2985         rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2986         if (rc) {
2987                 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2988                 goto err1;
2989         }
2990
2991         /* index only, 18 bit long, lkey = itid << 8 | key */
2992         mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2993         mr->hw_mr.pd = pd->pd_id;
2994         mr->hw_mr.local_read = 1;
2995         mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2996         mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2997         mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2998         mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2999         mr->hw_mr.dma_mr = true;
3000
3001         rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3002         if (rc) {
3003                 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3004                 goto err2;
3005         }
3006
3007         mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3008         if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3009             mr->hw_mr.remote_atomic)
3010                 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3011
3012         DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3013         return &mr->ibmr;
3014
3015 err2:
3016         dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3017 err1:
3018         kfree(mr);
3019         return ERR_PTR(rc);
3020 }
3021
3022 static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3023 {
3024         return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3025 }
3026
3027 static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3028 {
3029         int i, len = 0;
3030
3031         for (i = 0; i < num_sge; i++)
3032                 len += sg_list[i].length;
3033
3034         return len;
3035 }
3036
3037 static void swap_wqe_data64(u64 *p)
3038 {
3039         int i;
3040
3041         for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3042                 *p = cpu_to_be64(cpu_to_le64(*p));
3043 }
3044
3045 static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3046                                        struct qedr_qp *qp, u8 *wqe_size,
3047                                        const struct ib_send_wr *wr,
3048                                        const struct ib_send_wr **bad_wr,
3049                                        u8 *bits, u8 bit)
3050 {
3051         u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3052         char *seg_prt, *wqe;
3053         int i, seg_siz;
3054
3055         if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3056                 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3057                 *bad_wr = wr;
3058                 return 0;
3059         }
3060
3061         if (!data_size)
3062                 return data_size;
3063
3064         *bits |= bit;
3065
3066         seg_prt = NULL;
3067         wqe = NULL;
3068         seg_siz = 0;
3069
3070         /* Copy data inline */
3071         for (i = 0; i < wr->num_sge; i++) {
3072                 u32 len = wr->sg_list[i].length;
3073                 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3074
3075                 while (len > 0) {
3076                         u32 cur;
3077
3078                         /* New segment required */
3079                         if (!seg_siz) {
3080                                 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3081                                 seg_prt = wqe;
3082                                 seg_siz = sizeof(struct rdma_sq_common_wqe);
3083                                 (*wqe_size)++;
3084                         }
3085
3086                         /* Calculate currently allowed length */
3087                         cur = min_t(u32, len, seg_siz);
3088                         memcpy(seg_prt, src, cur);
3089
3090                         /* Update segment variables */
3091                         seg_prt += cur;
3092                         seg_siz -= cur;
3093
3094                         /* Update sge variables */
3095                         src += cur;
3096                         len -= cur;
3097
3098                         /* Swap fully-completed segments */
3099                         if (!seg_siz)
3100                                 swap_wqe_data64((u64 *)wqe);
3101                 }
3102         }
3103
3104         /* swap last not completed segment */
3105         if (seg_siz)
3106                 swap_wqe_data64((u64 *)wqe);
3107
3108         return data_size;
3109 }
3110
3111 #define RQ_SGE_SET(sge, vaddr, vlength, vflags)                 \
3112         do {                                                    \
3113                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3114                 (sge)->length = cpu_to_le32(vlength);           \
3115                 (sge)->flags = cpu_to_le32(vflags);             \
3116         } while (0)
3117
3118 #define SRQ_HDR_SET(hdr, vwr_id, num_sge)                       \
3119         do {                                                    \
3120                 DMA_REGPAIR_LE(hdr->wr_id, vwr_id);             \
3121                 (hdr)->num_sges = num_sge;                      \
3122         } while (0)
3123
3124 #define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)                 \
3125         do {                                                    \
3126                 DMA_REGPAIR_LE(sge->addr, vaddr);               \
3127                 (sge)->length = cpu_to_le32(vlength);           \
3128                 (sge)->l_key = cpu_to_le32(vlkey);              \
3129         } while (0)
3130
3131 static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3132                                 const struct ib_send_wr *wr)
3133 {
3134         u32 data_size = 0;
3135         int i;
3136
3137         for (i = 0; i < wr->num_sge; i++) {
3138                 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3139
3140                 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3141                 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3142                 sge->length = cpu_to_le32(wr->sg_list[i].length);
3143                 data_size += wr->sg_list[i].length;
3144         }
3145
3146         if (wqe_size)
3147                 *wqe_size += wr->num_sge;
3148
3149         return data_size;
3150 }
3151
3152 static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3153                                      struct qedr_qp *qp,
3154                                      struct rdma_sq_rdma_wqe_1st *rwqe,
3155                                      struct rdma_sq_rdma_wqe_2nd *rwqe2,
3156                                      const struct ib_send_wr *wr,
3157                                      const struct ib_send_wr **bad_wr)
3158 {
3159         rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3160         DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3161
3162         if (wr->send_flags & IB_SEND_INLINE &&
3163             (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3164              wr->opcode == IB_WR_RDMA_WRITE)) {
3165                 u8 flags = 0;
3166
3167                 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3168                 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3169                                                    bad_wr, &rwqe->flags, flags);
3170         }
3171
3172         return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3173 }
3174
3175 static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3176                                      struct qedr_qp *qp,
3177                                      struct rdma_sq_send_wqe_1st *swqe,
3178                                      struct rdma_sq_send_wqe_2st *swqe2,
3179                                      const struct ib_send_wr *wr,
3180                                      const struct ib_send_wr **bad_wr)
3181 {
3182         memset(swqe2, 0, sizeof(*swqe2));
3183         if (wr->send_flags & IB_SEND_INLINE) {
3184                 u8 flags = 0;
3185
3186                 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3187                 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3188                                                    bad_wr, &swqe->flags, flags);
3189         }
3190
3191         return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3192 }
3193
3194 static int qedr_prepare_reg(struct qedr_qp *qp,
3195                             struct rdma_sq_fmr_wqe_1st *fwqe1,
3196                             const struct ib_reg_wr *wr)
3197 {
3198         struct qedr_mr *mr = get_qedr_mr(wr->mr);
3199         struct rdma_sq_fmr_wqe_2nd *fwqe2;
3200
3201         fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3202         fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3203         fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3204         fwqe1->l_key = wr->key;
3205
3206         fwqe2->access_ctrl = 0;
3207
3208         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3209                    !!(wr->access & IB_ACCESS_REMOTE_READ));
3210         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3211                    !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3212         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3213                    !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3214         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3215         SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3216                    !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3217         fwqe2->fmr_ctrl = 0;
3218
3219         SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3220                    ilog2(mr->ibmr.page_size) - 12);
3221
3222         fwqe2->length_hi = 0;
3223         fwqe2->length_lo = mr->ibmr.length;
3224         fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3225         fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3226
3227         qp->wqe_wr_id[qp->sq.prod].mr = mr;
3228
3229         return 0;
3230 }
3231
3232 static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3233 {
3234         switch (opcode) {
3235         case IB_WR_RDMA_WRITE:
3236         case IB_WR_RDMA_WRITE_WITH_IMM:
3237                 return IB_WC_RDMA_WRITE;
3238         case IB_WR_SEND_WITH_IMM:
3239         case IB_WR_SEND:
3240         case IB_WR_SEND_WITH_INV:
3241                 return IB_WC_SEND;
3242         case IB_WR_RDMA_READ:
3243         case IB_WR_RDMA_READ_WITH_INV:
3244                 return IB_WC_RDMA_READ;
3245         case IB_WR_ATOMIC_CMP_AND_SWP:
3246                 return IB_WC_COMP_SWAP;
3247         case IB_WR_ATOMIC_FETCH_AND_ADD:
3248                 return IB_WC_FETCH_ADD;
3249         case IB_WR_REG_MR:
3250                 return IB_WC_REG_MR;
3251         case IB_WR_LOCAL_INV:
3252                 return IB_WC_LOCAL_INV;
3253         default:
3254                 return IB_WC_SEND;
3255         }
3256 }
3257
3258 static inline bool qedr_can_post_send(struct qedr_qp *qp,
3259                                       const struct ib_send_wr *wr)
3260 {
3261         int wq_is_full, err_wr, pbl_is_full;
3262         struct qedr_dev *dev = qp->dev;
3263
3264         /* prevent SQ overflow and/or processing of a bad WR */
3265         err_wr = wr->num_sge > qp->sq.max_sges;
3266         wq_is_full = qedr_wq_is_full(&qp->sq);
3267         pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3268                       QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3269         if (wq_is_full || err_wr || pbl_is_full) {
3270                 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3271                         DP_ERR(dev,
3272                                "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3273                                qp);
3274                         qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3275                 }
3276
3277                 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3278                         DP_ERR(dev,
3279                                "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3280                                qp);
3281                         qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3282                 }
3283
3284                 if (pbl_is_full &&
3285                     !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3286                         DP_ERR(dev,
3287                                "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3288                                qp);
3289                         qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3290                 }
3291                 return false;
3292         }
3293         return true;
3294 }
3295
3296 static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3297                             const struct ib_send_wr **bad_wr)
3298 {
3299         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3300         struct qedr_qp *qp = get_qedr_qp(ibqp);
3301         struct rdma_sq_atomic_wqe_1st *awqe1;
3302         struct rdma_sq_atomic_wqe_2nd *awqe2;
3303         struct rdma_sq_atomic_wqe_3rd *awqe3;
3304         struct rdma_sq_send_wqe_2st *swqe2;
3305         struct rdma_sq_local_inv_wqe *iwqe;
3306         struct rdma_sq_rdma_wqe_2nd *rwqe2;
3307         struct rdma_sq_send_wqe_1st *swqe;
3308         struct rdma_sq_rdma_wqe_1st *rwqe;
3309         struct rdma_sq_fmr_wqe_1st *fwqe1;
3310         struct rdma_sq_common_wqe *wqe;
3311         u32 length;
3312         int rc = 0;
3313         bool comp;
3314
3315         if (!qedr_can_post_send(qp, wr)) {
3316                 *bad_wr = wr;
3317                 return -ENOMEM;
3318         }
3319
3320         wqe = qed_chain_produce(&qp->sq.pbl);
3321         qp->wqe_wr_id[qp->sq.prod].signaled =
3322                 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3323
3324         wqe->flags = 0;
3325         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3326                    !!(wr->send_flags & IB_SEND_SOLICITED));
3327         comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3328         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3329         SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3330                    !!(wr->send_flags & IB_SEND_FENCE));
3331         wqe->prev_wqe_size = qp->prev_wqe_size;
3332
3333         qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3334
3335         switch (wr->opcode) {
3336         case IB_WR_SEND_WITH_IMM:
3337                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3338                         rc = -EINVAL;
3339                         *bad_wr = wr;
3340                         break;
3341                 }
3342                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3343                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3344                 swqe->wqe_size = 2;
3345                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3346
3347                 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3348                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3349                                                    wr, bad_wr);
3350                 swqe->length = cpu_to_le32(length);
3351                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3352                 qp->prev_wqe_size = swqe->wqe_size;
3353                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3354                 break;
3355         case IB_WR_SEND:
3356                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3357                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3358
3359                 swqe->wqe_size = 2;
3360                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3361                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3362                                                    wr, bad_wr);
3363                 swqe->length = cpu_to_le32(length);
3364                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3365                 qp->prev_wqe_size = swqe->wqe_size;
3366                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3367                 break;
3368         case IB_WR_SEND_WITH_INV:
3369                 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3370                 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3371                 swqe2 = qed_chain_produce(&qp->sq.pbl);
3372                 swqe->wqe_size = 2;
3373                 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3374                 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3375                                                    wr, bad_wr);
3376                 swqe->length = cpu_to_le32(length);
3377                 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3378                 qp->prev_wqe_size = swqe->wqe_size;
3379                 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3380                 break;
3381
3382         case IB_WR_RDMA_WRITE_WITH_IMM:
3383                 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3384                         rc = -EINVAL;
3385                         *bad_wr = wr;
3386                         break;
3387                 }
3388                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3389                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3390
3391                 rwqe->wqe_size = 2;
3392                 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3393                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3394                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3395                                                    wr, bad_wr);
3396                 rwqe->length = cpu_to_le32(length);
3397                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3398                 qp->prev_wqe_size = rwqe->wqe_size;
3399                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3400                 break;
3401         case IB_WR_RDMA_WRITE:
3402                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3403                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3404
3405                 rwqe->wqe_size = 2;
3406                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3407                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3408                                                    wr, bad_wr);
3409                 rwqe->length = cpu_to_le32(length);
3410                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3411                 qp->prev_wqe_size = rwqe->wqe_size;
3412                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3413                 break;
3414         case IB_WR_RDMA_READ_WITH_INV:
3415                 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3416                 /* fallthrough -- same is identical to RDMA READ */
3417
3418         case IB_WR_RDMA_READ:
3419                 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3420                 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3421
3422                 rwqe->wqe_size = 2;
3423                 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3424                 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3425                                                    wr, bad_wr);
3426                 rwqe->length = cpu_to_le32(length);
3427                 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3428                 qp->prev_wqe_size = rwqe->wqe_size;
3429                 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3430                 break;
3431
3432         case IB_WR_ATOMIC_CMP_AND_SWP:
3433         case IB_WR_ATOMIC_FETCH_AND_ADD:
3434                 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3435                 awqe1->wqe_size = 4;
3436
3437                 awqe2 = qed_chain_produce(&qp->sq.pbl);
3438                 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3439                 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3440
3441                 awqe3 = qed_chain_produce(&qp->sq.pbl);
3442
3443                 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3444                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3445                         DMA_REGPAIR_LE(awqe3->swap_data,
3446                                        atomic_wr(wr)->compare_add);
3447                 } else {
3448                         wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3449                         DMA_REGPAIR_LE(awqe3->swap_data,
3450                                        atomic_wr(wr)->swap);
3451                         DMA_REGPAIR_LE(awqe3->cmp_data,
3452                                        atomic_wr(wr)->compare_add);
3453                 }
3454
3455                 qedr_prepare_sq_sges(qp, NULL, wr);
3456
3457                 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3458                 qp->prev_wqe_size = awqe1->wqe_size;
3459                 break;
3460
3461         case IB_WR_LOCAL_INV:
3462                 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3463                 iwqe->wqe_size = 1;
3464
3465                 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3466                 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3467                 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3468                 qp->prev_wqe_size = iwqe->wqe_size;
3469                 break;
3470         case IB_WR_REG_MR:
3471                 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3472                 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3473                 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3474                 fwqe1->wqe_size = 2;
3475
3476                 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3477                 if (rc) {
3478                         DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3479                         *bad_wr = wr;
3480                         break;
3481                 }
3482
3483                 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3484                 qp->prev_wqe_size = fwqe1->wqe_size;
3485                 break;
3486         default:
3487                 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3488                 rc = -EINVAL;
3489                 *bad_wr = wr;
3490                 break;
3491         }
3492
3493         if (*bad_wr) {
3494                 u16 value;
3495
3496                 /* Restore prod to its position before
3497                  * this WR was processed
3498                  */
3499                 value = le16_to_cpu(qp->sq.db_data.data.value);
3500                 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3501
3502                 /* Restore prev_wqe_size */
3503                 qp->prev_wqe_size = wqe->prev_wqe_size;
3504                 rc = -EINVAL;
3505                 DP_ERR(dev, "POST SEND FAILED\n");
3506         }
3507
3508         return rc;
3509 }
3510
3511 int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3512                    const struct ib_send_wr **bad_wr)
3513 {
3514         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3515         struct qedr_qp *qp = get_qedr_qp(ibqp);
3516         unsigned long flags;
3517         int rc = 0;
3518
3519         *bad_wr = NULL;
3520
3521         if (qp->qp_type == IB_QPT_GSI)
3522                 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3523
3524         spin_lock_irqsave(&qp->q_lock, flags);
3525
3526         if (rdma_protocol_roce(&dev->ibdev, 1)) {
3527                 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3528                     (qp->state != QED_ROCE_QP_STATE_ERR) &&
3529                     (qp->state != QED_ROCE_QP_STATE_SQD)) {
3530                         spin_unlock_irqrestore(&qp->q_lock, flags);
3531                         *bad_wr = wr;
3532                         DP_DEBUG(dev, QEDR_MSG_CQ,
3533                                  "QP in wrong state! QP icid=0x%x state %d\n",
3534                                  qp->icid, qp->state);
3535                         return -EINVAL;
3536                 }
3537         }
3538
3539         while (wr) {
3540                 rc = __qedr_post_send(ibqp, wr, bad_wr);
3541                 if (rc)
3542                         break;
3543
3544                 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3545
3546                 qedr_inc_sw_prod(&qp->sq);
3547
3548                 qp->sq.db_data.data.value++;
3549
3550                 wr = wr->next;
3551         }
3552
3553         /* Trigger doorbell
3554          * If there was a failure in the first WR then it will be triggered in
3555          * vane. However this is not harmful (as long as the producer value is
3556          * unchanged). For performance reasons we avoid checking for this
3557          * redundant doorbell.
3558          *
3559          * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3560          * soon as we give the doorbell, we could get a completion
3561          * for this wr, therefore we need to make sure that the
3562          * memory is updated before giving the doorbell.
3563          * During qedr_poll_cq, rmb is called before accessing the
3564          * cqe. This covers for the smp_rmb as well.
3565          */
3566         smp_wmb();
3567         writel(qp->sq.db_data.raw, qp->sq.db);
3568
3569         /* Make sure write sticks */
3570         mmiowb();
3571
3572         spin_unlock_irqrestore(&qp->q_lock, flags);
3573
3574         return rc;
3575 }
3576
3577 static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3578 {
3579         u32 used;
3580
3581         /* Calculate number of elements used based on producer
3582          * count and consumer count and subtract it from max
3583          * work request supported so that we get elements left.
3584          */
3585         used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3586
3587         return hw_srq->max_wr - used;
3588 }
3589
3590 int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3591                        const struct ib_recv_wr **bad_wr)
3592 {
3593         struct qedr_srq *srq = get_qedr_srq(ibsrq);
3594         struct qedr_srq_hwq_info *hw_srq;
3595         struct qedr_dev *dev = srq->dev;
3596         struct qed_chain *pbl;
3597         unsigned long flags;
3598         int status = 0;
3599         u32 num_sge;
3600
3601         spin_lock_irqsave(&srq->lock, flags);
3602
3603         hw_srq = &srq->hw_srq;
3604         pbl = &srq->hw_srq.pbl;
3605         while (wr) {
3606                 struct rdma_srq_wqe_header *hdr;
3607                 int i;
3608
3609                 if (!qedr_srq_elem_left(hw_srq) ||
3610                     wr->num_sge > srq->hw_srq.max_sges) {
3611                         DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3612                                hw_srq->wr_prod_cnt,
3613                                atomic_read(&hw_srq->wr_cons_cnt),
3614                                wr->num_sge, srq->hw_srq.max_sges);
3615                         status = -ENOMEM;
3616                         *bad_wr = wr;
3617                         break;
3618                 }
3619
3620                 hdr = qed_chain_produce(pbl);
3621                 num_sge = wr->num_sge;
3622                 /* Set number of sge and work request id in header */
3623                 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3624
3625                 srq->hw_srq.wr_prod_cnt++;
3626                 hw_srq->wqe_prod++;
3627                 hw_srq->sge_prod++;
3628
3629                 DP_DEBUG(dev, QEDR_MSG_SRQ,
3630                          "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3631                          wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3632
3633                 for (i = 0; i < wr->num_sge; i++) {
3634                         struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3635
3636                         /* Set SGE length, lkey and address */
3637                         SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3638                                     wr->sg_list[i].length, wr->sg_list[i].lkey);
3639
3640                         DP_DEBUG(dev, QEDR_MSG_SRQ,
3641                                  "[%d]: len %d key %x addr %x:%x\n",
3642                                  i, srq_sge->length, srq_sge->l_key,
3643                                  srq_sge->addr.hi, srq_sge->addr.lo);
3644                         hw_srq->sge_prod++;
3645                 }
3646
3647                 /* Update WQE and SGE information before
3648                  * updating producer.
3649                  */
3650                 dma_wmb();
3651
3652                 /* SRQ producer is 8 bytes. Need to update SGE producer index
3653                  * in first 4 bytes and need to update WQE producer in
3654                  * next 4 bytes.
3655                  */
3656                 srq->hw_srq.virt_prod_pair_addr->sge_prod = hw_srq->sge_prod;
3657                 /* Make sure sge producer is updated first */
3658                 dma_wmb();
3659                 srq->hw_srq.virt_prod_pair_addr->wqe_prod = hw_srq->wqe_prod;
3660
3661                 wr = wr->next;
3662         }
3663
3664         DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3665                  qed_chain_get_elem_left(pbl));
3666         spin_unlock_irqrestore(&srq->lock, flags);
3667
3668         return status;
3669 }
3670
3671 int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3672                    const struct ib_recv_wr **bad_wr)
3673 {
3674         struct qedr_qp *qp = get_qedr_qp(ibqp);
3675         struct qedr_dev *dev = qp->dev;
3676         unsigned long flags;
3677         int status = 0;
3678
3679         if (qp->qp_type == IB_QPT_GSI)
3680                 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3681
3682         spin_lock_irqsave(&qp->q_lock, flags);
3683
3684         if (qp->state == QED_ROCE_QP_STATE_RESET) {
3685                 spin_unlock_irqrestore(&qp->q_lock, flags);
3686                 *bad_wr = wr;
3687                 return -EINVAL;
3688         }
3689
3690         while (wr) {
3691                 int i;
3692
3693                 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3694                     QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3695                     wr->num_sge > qp->rq.max_sges) {
3696                         DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3697                                qed_chain_get_elem_left_u32(&qp->rq.pbl),
3698                                QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3699                                qp->rq.max_sges);
3700                         status = -ENOMEM;
3701                         *bad_wr = wr;
3702                         break;
3703                 }
3704                 for (i = 0; i < wr->num_sge; i++) {
3705                         u32 flags = 0;
3706                         struct rdma_rq_sge *rqe =
3707                             qed_chain_produce(&qp->rq.pbl);
3708
3709                         /* First one must include the number
3710                          * of SGE in the list
3711                          */
3712                         if (!i)
3713                                 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3714                                           wr->num_sge);
3715
3716                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3717                                   wr->sg_list[i].lkey);
3718
3719                         RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3720                                    wr->sg_list[i].length, flags);
3721                 }
3722
3723                 /* Special case of no sges. FW requires between 1-4 sges...
3724                  * in this case we need to post 1 sge with length zero. this is
3725                  * because rdma write with immediate consumes an RQ.
3726                  */
3727                 if (!wr->num_sge) {
3728                         u32 flags = 0;
3729                         struct rdma_rq_sge *rqe =
3730                             qed_chain_produce(&qp->rq.pbl);
3731
3732                         /* First one must include the number
3733                          * of SGE in the list
3734                          */
3735                         SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3736                         SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3737
3738                         RQ_SGE_SET(rqe, 0, 0, flags);
3739                         i = 1;
3740                 }
3741
3742                 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3743                 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3744
3745                 qedr_inc_sw_prod(&qp->rq);
3746
3747                 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3748                  * soon as we give the doorbell, we could get a completion
3749                  * for this wr, therefore we need to make sure that the
3750                  * memory is update before giving the doorbell.
3751                  * During qedr_poll_cq, rmb is called before accessing the
3752                  * cqe. This covers for the smp_rmb as well.
3753                  */
3754                 smp_wmb();
3755
3756                 qp->rq.db_data.data.value++;
3757
3758                 writel(qp->rq.db_data.raw, qp->rq.db);
3759
3760                 /* Make sure write sticks */
3761                 mmiowb();
3762
3763                 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3764                         writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3765                         mmiowb();       /* for second doorbell */
3766                 }
3767
3768                 wr = wr->next;
3769         }
3770
3771         spin_unlock_irqrestore(&qp->q_lock, flags);
3772
3773         return status;
3774 }
3775
3776 static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3777 {
3778         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3779
3780         return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3781                 cq->pbl_toggle;
3782 }
3783
3784 static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3785 {
3786         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3787         struct qedr_qp *qp;
3788
3789         qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3790                                                    resp_cqe->qp_handle.lo,
3791                                                    u64);
3792         return qp;
3793 }
3794
3795 static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3796 {
3797         struct rdma_cqe_requester *resp_cqe = &cqe->req;
3798
3799         return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3800 }
3801
3802 /* Return latest CQE (needs processing) */
3803 static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3804 {
3805         return cq->latest_cqe;
3806 }
3807
3808 /* In fmr we need to increase the number of fmr completed counter for the fmr
3809  * algorithm determining whether we can free a pbl or not.
3810  * we need to perform this whether the work request was signaled or not. for
3811  * this purpose we call this function from the condition that checks if a wr
3812  * should be skipped, to make sure we don't miss it ( possibly this fmr
3813  * operation was not signalted)
3814  */
3815 static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3816 {
3817         if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3818                 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3819 }
3820
3821 static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3822                        struct qedr_cq *cq, int num_entries,
3823                        struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3824                        int force)
3825 {
3826         u16 cnt = 0;
3827
3828         while (num_entries && qp->sq.wqe_cons != hw_cons) {
3829                 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3830                         qedr_chk_if_fmr(qp);
3831                         /* skip WC */
3832                         goto next_cqe;
3833                 }
3834
3835                 /* fill WC */
3836                 wc->status = status;
3837                 wc->vendor_err = 0;
3838                 wc->wc_flags = 0;
3839                 wc->src_qp = qp->id;
3840                 wc->qp = &qp->ibqp;
3841
3842                 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3843                 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3844
3845                 switch (wc->opcode) {
3846                 case IB_WC_RDMA_WRITE:
3847                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3848                         break;
3849                 case IB_WC_COMP_SWAP:
3850                 case IB_WC_FETCH_ADD:
3851                         wc->byte_len = 8;
3852                         break;
3853                 case IB_WC_REG_MR:
3854                         qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3855                         break;
3856                 case IB_WC_RDMA_READ:
3857                 case IB_WC_SEND:
3858                         wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3859                         break;
3860                 default:
3861                         break;
3862                 }
3863
3864                 num_entries--;
3865                 wc++;
3866                 cnt++;
3867 next_cqe:
3868                 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3869                         qed_chain_consume(&qp->sq.pbl);
3870                 qedr_inc_sw_cons(&qp->sq);
3871         }
3872
3873         return cnt;
3874 }
3875
3876 static int qedr_poll_cq_req(struct qedr_dev *dev,
3877                             struct qedr_qp *qp, struct qedr_cq *cq,
3878                             int num_entries, struct ib_wc *wc,
3879                             struct rdma_cqe_requester *req)
3880 {
3881         int cnt = 0;
3882
3883         switch (req->status) {
3884         case RDMA_CQE_REQ_STS_OK:
3885                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3886                                   IB_WC_SUCCESS, 0);
3887                 break;
3888         case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3889                 if (qp->state != QED_ROCE_QP_STATE_ERR)
3890                         DP_DEBUG(dev, QEDR_MSG_CQ,
3891                                  "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3892                                  cq->icid, qp->icid);
3893                 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3894                                   IB_WC_WR_FLUSH_ERR, 1);
3895                 break;
3896         default:
3897                 /* process all WQE before the cosumer */
3898                 qp->state = QED_ROCE_QP_STATE_ERR;
3899                 cnt = process_req(dev, qp, cq, num_entries, wc,
3900                                   req->sq_cons - 1, IB_WC_SUCCESS, 0);
3901                 wc += cnt;
3902                 /* if we have extra WC fill it with actual error info */
3903                 if (cnt < num_entries) {
3904                         enum ib_wc_status wc_status;
3905
3906                         switch (req->status) {
3907                         case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3908                                 DP_ERR(dev,
3909                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3910                                        cq->icid, qp->icid);
3911                                 wc_status = IB_WC_BAD_RESP_ERR;
3912                                 break;
3913                         case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3914                                 DP_ERR(dev,
3915                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3916                                        cq->icid, qp->icid);
3917                                 wc_status = IB_WC_LOC_LEN_ERR;
3918                                 break;
3919                         case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3920                                 DP_ERR(dev,
3921                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3922                                        cq->icid, qp->icid);
3923                                 wc_status = IB_WC_LOC_QP_OP_ERR;
3924                                 break;
3925                         case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3926                                 DP_ERR(dev,
3927                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3928                                        cq->icid, qp->icid);
3929                                 wc_status = IB_WC_LOC_PROT_ERR;
3930                                 break;
3931                         case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3932                                 DP_ERR(dev,
3933                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3934                                        cq->icid, qp->icid);
3935                                 wc_status = IB_WC_MW_BIND_ERR;
3936                                 break;
3937                         case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3938                                 DP_ERR(dev,
3939                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3940                                        cq->icid, qp->icid);
3941                                 wc_status = IB_WC_REM_INV_REQ_ERR;
3942                                 break;
3943                         case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3944                                 DP_ERR(dev,
3945                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3946                                        cq->icid, qp->icid);
3947                                 wc_status = IB_WC_REM_ACCESS_ERR;
3948                                 break;
3949                         case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3950                                 DP_ERR(dev,
3951                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3952                                        cq->icid, qp->icid);
3953                                 wc_status = IB_WC_REM_OP_ERR;
3954                                 break;
3955                         case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3956                                 DP_ERR(dev,
3957                                        "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3958                                        cq->icid, qp->icid);
3959                                 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3960                                 break;
3961                         case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3962                                 DP_ERR(dev,
3963                                        "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3964                                        cq->icid, qp->icid);
3965                                 wc_status = IB_WC_RETRY_EXC_ERR;
3966                                 break;
3967                         default:
3968                                 DP_ERR(dev,
3969                                        "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3970                                        cq->icid, qp->icid);
3971                                 wc_status = IB_WC_GENERAL_ERR;
3972                         }
3973                         cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3974                                            wc_status, 1);
3975                 }
3976         }
3977
3978         return cnt;
3979 }
3980
3981 static inline int qedr_cqe_resp_status_to_ib(u8 status)
3982 {
3983         switch (status) {
3984         case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3985                 return IB_WC_LOC_ACCESS_ERR;
3986         case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3987                 return IB_WC_LOC_LEN_ERR;
3988         case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3989                 return IB_WC_LOC_QP_OP_ERR;
3990         case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3991                 return IB_WC_LOC_PROT_ERR;
3992         case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3993                 return IB_WC_MW_BIND_ERR;
3994         case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3995                 return IB_WC_REM_INV_RD_REQ_ERR;
3996         case RDMA_CQE_RESP_STS_OK:
3997                 return IB_WC_SUCCESS;
3998         default:
3999                 return IB_WC_GENERAL_ERR;
4000         }
4001 }
4002
4003 static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4004                                           struct ib_wc *wc)
4005 {
4006         wc->status = IB_WC_SUCCESS;
4007         wc->byte_len = le32_to_cpu(resp->length);
4008
4009         if (resp->flags & QEDR_RESP_IMM) {
4010                 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4011                 wc->wc_flags |= IB_WC_WITH_IMM;
4012
4013                 if (resp->flags & QEDR_RESP_RDMA)
4014                         wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4015
4016                 if (resp->flags & QEDR_RESP_INV)
4017                         return -EINVAL;
4018
4019         } else if (resp->flags & QEDR_RESP_INV) {
4020                 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4021                 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4022
4023                 if (resp->flags & QEDR_RESP_RDMA)
4024                         return -EINVAL;
4025
4026         } else if (resp->flags & QEDR_RESP_RDMA) {
4027                 return -EINVAL;
4028         }
4029
4030         return 0;
4031 }
4032
4033 static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4034                                struct qedr_cq *cq, struct ib_wc *wc,
4035                                struct rdma_cqe_responder *resp, u64 wr_id)
4036 {
4037         /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4038         wc->opcode = IB_WC_RECV;
4039         wc->wc_flags = 0;
4040
4041         if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4042                 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4043                         DP_ERR(dev,
4044                                "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4045                                cq, cq->icid, resp->flags);
4046
4047         } else {
4048                 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4049                 if (wc->status == IB_WC_GENERAL_ERR)
4050                         DP_ERR(dev,
4051                                "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4052                                cq, cq->icid, resp->status);
4053         }
4054
4055         /* Fill the rest of the WC */
4056         wc->vendor_err = 0;
4057         wc->src_qp = qp->id;
4058         wc->qp = &qp->ibqp;
4059         wc->wr_id = wr_id;
4060 }
4061
4062 static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4063                                 struct qedr_cq *cq, struct ib_wc *wc,
4064                                 struct rdma_cqe_responder *resp)
4065 {
4066         struct qedr_srq *srq = qp->srq;
4067         u64 wr_id;
4068
4069         wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4070                          le32_to_cpu(resp->srq_wr_id.lo), u64);
4071
4072         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4073                 wc->status = IB_WC_WR_FLUSH_ERR;
4074                 wc->vendor_err = 0;
4075                 wc->wr_id = wr_id;
4076                 wc->byte_len = 0;
4077                 wc->src_qp = qp->id;
4078                 wc->qp = &qp->ibqp;
4079                 wc->wr_id = wr_id;
4080         } else {
4081                 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4082         }
4083         atomic_inc(&srq->hw_srq.wr_cons_cnt);
4084
4085         return 1;
4086 }
4087 static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4088                             struct qedr_cq *cq, struct ib_wc *wc,
4089                             struct rdma_cqe_responder *resp)
4090 {
4091         u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4092
4093         __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4094
4095         while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4096                 qed_chain_consume(&qp->rq.pbl);
4097         qedr_inc_sw_cons(&qp->rq);
4098
4099         return 1;
4100 }
4101
4102 static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4103                               int num_entries, struct ib_wc *wc, u16 hw_cons)
4104 {
4105         u16 cnt = 0;
4106
4107         while (num_entries && qp->rq.wqe_cons != hw_cons) {
4108                 /* fill WC */
4109                 wc->status = IB_WC_WR_FLUSH_ERR;
4110                 wc->vendor_err = 0;
4111                 wc->wc_flags = 0;
4112                 wc->src_qp = qp->id;
4113                 wc->byte_len = 0;
4114                 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4115                 wc->qp = &qp->ibqp;
4116                 num_entries--;
4117                 wc++;
4118                 cnt++;
4119                 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4120                         qed_chain_consume(&qp->rq.pbl);
4121                 qedr_inc_sw_cons(&qp->rq);
4122         }
4123
4124         return cnt;
4125 }
4126
4127 static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4128                                  struct rdma_cqe_responder *resp, int *update)
4129 {
4130         if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4131                 consume_cqe(cq);
4132                 *update |= 1;
4133         }
4134 }
4135
4136 static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4137                                  struct qedr_cq *cq, int num_entries,
4138                                  struct ib_wc *wc,
4139                                  struct rdma_cqe_responder *resp)
4140 {
4141         int cnt;
4142
4143         cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4144         consume_cqe(cq);
4145
4146         return cnt;
4147 }
4148
4149 static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4150                              struct qedr_cq *cq, int num_entries,
4151                              struct ib_wc *wc, struct rdma_cqe_responder *resp,
4152                              int *update)
4153 {
4154         int cnt;
4155
4156         if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4157                 cnt = process_resp_flush(qp, cq, num_entries, wc,
4158                                          resp->rq_cons_or_srq_id);
4159                 try_consume_resp_cqe(cq, qp, resp, update);
4160         } else {
4161                 cnt = process_resp_one(dev, qp, cq, wc, resp);
4162                 consume_cqe(cq);
4163                 *update |= 1;
4164         }
4165
4166         return cnt;
4167 }
4168
4169 static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4170                                 struct rdma_cqe_requester *req, int *update)
4171 {
4172         if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4173                 consume_cqe(cq);
4174                 *update |= 1;
4175         }
4176 }
4177
4178 int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4179 {
4180         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4181         struct qedr_cq *cq = get_qedr_cq(ibcq);
4182         union rdma_cqe *cqe;
4183         u32 old_cons, new_cons;
4184         unsigned long flags;
4185         int update = 0;
4186         int done = 0;
4187
4188         if (cq->destroyed) {
4189                 DP_ERR(dev,
4190                        "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4191                        cq, cq->icid);
4192                 return 0;
4193         }
4194
4195         if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4196                 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4197
4198         spin_lock_irqsave(&cq->cq_lock, flags);
4199         cqe = cq->latest_cqe;
4200         old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4201         while (num_entries && is_valid_cqe(cq, cqe)) {
4202                 struct qedr_qp *qp;
4203                 int cnt = 0;
4204
4205                 /* prevent speculative reads of any field of CQE */
4206                 rmb();
4207
4208                 qp = cqe_get_qp(cqe);
4209                 if (!qp) {
4210                         WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4211                         break;
4212                 }
4213
4214                 wc->qp = &qp->ibqp;
4215
4216                 switch (cqe_get_type(cqe)) {
4217                 case RDMA_CQE_TYPE_REQUESTER:
4218                         cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4219                                                &cqe->req);
4220                         try_consume_req_cqe(cq, qp, &cqe->req, &update);
4221                         break;
4222                 case RDMA_CQE_TYPE_RESPONDER_RQ:
4223                         cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4224                                                 &cqe->resp, &update);
4225                         break;
4226                 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4227                         cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4228                                                     wc, &cqe->resp);
4229                         update = 1;
4230                         break;
4231                 case RDMA_CQE_TYPE_INVALID:
4232                 default:
4233                         DP_ERR(dev, "Error: invalid CQE type = %d\n",
4234                                cqe_get_type(cqe));
4235                 }
4236                 num_entries -= cnt;
4237                 wc += cnt;
4238                 done += cnt;
4239
4240                 cqe = get_cqe(cq);
4241         }
4242         new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4243
4244         cq->cq_cons += new_cons - old_cons;
4245
4246         if (update)
4247                 /* doorbell notifies abount latest VALID entry,
4248                  * but chain already point to the next INVALID one
4249                  */
4250                 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4251
4252         spin_unlock_irqrestore(&cq->cq_lock, flags);
4253         return done;
4254 }
4255
4256 int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4257                      u8 port_num,
4258                      const struct ib_wc *in_wc,
4259                      const struct ib_grh *in_grh,
4260                      const struct ib_mad_hdr *mad_hdr,
4261                      size_t in_mad_size, struct ib_mad_hdr *out_mad,
4262                      size_t *out_mad_size, u16 *out_mad_pkey_index)
4263 {
4264         struct qedr_dev *dev = get_qedr_dev(ibdev);
4265
4266         DP_DEBUG(dev, QEDR_MSG_GSI,
4267                  "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
4268                  mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
4269                  mad_hdr->class_specific, mad_hdr->class_version,
4270                  mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
4271         return IB_MAD_RESULT_SUCCESS;
4272 }