drivers/infiniband/hw/qedr/qedr_cm.c

   1 /* QLogic qedr NIC Driver
   2  * Copyright (c) 2015-2016  QLogic Corporation
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and /or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32 #include <linux/dma-mapping.h>
  33 #include <linux/crc32.h>
  34 #include <linux/iommu.h>
  35 #include <net/ip.h>
  36 #include <net/ipv6.h>
  37 #include <net/udp.h>
  38
  39 #include <rdma/ib_verbs.h>
  40 #include <rdma/ib_user_verbs.h>
  41 #include <rdma/iw_cm.h>
  42 #include <rdma/ib_umem.h>
  43 #include <rdma/ib_addr.h>
  44 #include <rdma/ib_cache.h>
  45
  46 #include "qedr_hsi.h"
  47 #include <linux/qed/qed_if.h>
  48 #include <linux/qed/qed_roce_if.h>
  49 #include "qedr.h"
  50 #include "qedr_hsi.h"
  51 #include "verbs.h"
  52 #include <rdma/qedr-abi.h>
  53 #include "qedr_hsi.h"
  54 #include "qedr_cm.h"
  55
  56 void qedr_inc_sw_gsi_cons(struct qedr_qp_hwq_info *info)
  57 {
  58         info->gsi_cons = (info->gsi_cons + 1) % info->max_wr;
  59 }
  60
  61 void qedr_store_gsi_qp_cq(struct qedr_dev *dev, struct qedr_qp *qp,
  62                           struct ib_qp_init_attr *attrs)
  63 {
  64         dev->gsi_qp_created = 1;
  65         dev->gsi_sqcq = get_qedr_cq(attrs->send_cq);
  66         dev->gsi_rqcq = get_qedr_cq(attrs->recv_cq);
  67         dev->gsi_qp = qp;
  68 }
  69
  70 void qedr_ll2_tx_cb(void *_qdev, struct qed_roce_ll2_packet *pkt)
  71 {
  72         struct qedr_dev *dev = (struct qedr_dev *)_qdev;
  73         struct qedr_cq *cq = dev->gsi_sqcq;
  74         struct qedr_qp *qp = dev->gsi_qp;
  75         unsigned long flags;
  76
  77         DP_DEBUG(dev, QEDR_MSG_GSI,
  78                  "LL2 TX CB: gsi_sqcq=%p, gsi_rqcq=%p, gsi_cons=%d, ibcq_comp=%s\n",
  79                  dev->gsi_sqcq, dev->gsi_rqcq, qp->sq.gsi_cons,
  80                  cq->ibcq.comp_handler ? "Yes" : "No");
  81
  82         dma_free_coherent(&dev->pdev->dev, pkt->header.len, pkt->header.vaddr,
  83                           pkt->header.baddr);
  84         kfree(pkt);
  85
  86         spin_lock_irqsave(&qp->q_lock, flags);
  87         qedr_inc_sw_gsi_cons(&qp->sq);
  88         spin_unlock_irqrestore(&qp->q_lock, flags);
  89
  90         if (cq->ibcq.comp_handler) {
  91                 spin_lock_irqsave(&cq->comp_handler_lock, flags);
  92                 (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context);
  93                 spin_unlock_irqrestore(&cq->comp_handler_lock, flags);
  94         }
  95 }
  96
  97 void qedr_ll2_rx_cb(void *_dev, struct qed_roce_ll2_packet *pkt,
  98                     struct qed_roce_ll2_rx_params *params)
  99 {
 100         struct qedr_dev *dev = (struct qedr_dev *)_dev;
 101         struct qedr_cq *cq = dev->gsi_rqcq;
 102         struct qedr_qp *qp = dev->gsi_qp;
 103         unsigned long flags;
 104
 105         spin_lock_irqsave(&qp->q_lock, flags);
 106
 107         qp->rqe_wr_id[qp->rq.gsi_cons].rc = params->rc;
 108         qp->rqe_wr_id[qp->rq.gsi_cons].vlan_id = params->vlan_id;
 109         qp->rqe_wr_id[qp->rq.gsi_cons].sg_list[0].length = pkt->payload[0].len;
 110         ether_addr_copy(qp->rqe_wr_id[qp->rq.gsi_cons].smac, params->smac);
 111
 112         qedr_inc_sw_gsi_cons(&qp->rq);
 113
 114         spin_unlock_irqrestore(&qp->q_lock, flags);
 115
 116         if (cq->ibcq.comp_handler) {
 117                 spin_lock_irqsave(&cq->comp_handler_lock, flags);
 118                 (*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context);
 119                 spin_unlock_irqrestore(&cq->comp_handler_lock, flags);
 120         }
 121 }
 122
 123 static void qedr_destroy_gsi_cq(struct qedr_dev *dev,
 124                                 struct ib_qp_init_attr *attrs)
 125 {
 126         struct qed_rdma_destroy_cq_in_params iparams;
 127         struct qed_rdma_destroy_cq_out_params oparams;
 128         struct qedr_cq *cq;
 129
 130         cq = get_qedr_cq(attrs->send_cq);
 131         iparams.icid = cq->icid;
 132         dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
 133         dev->ops->common->chain_free(dev->cdev, &cq->pbl);
 134
 135         cq = get_qedr_cq(attrs->recv_cq);
 136         /* if a dedicated recv_cq was used, delete it too */
 137         if (iparams.icid != cq->icid) {
 138                 iparams.icid = cq->icid;
 139                 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
 140                 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
 141         }
 142 }
 143
 144 static inline int qedr_check_gsi_qp_attrs(struct qedr_dev *dev,
 145                                           struct ib_qp_init_attr *attrs)
 146 {
 147         if (attrs->cap.max_recv_sge > QEDR_GSI_MAX_RECV_SGE) {
 148                 DP_ERR(dev,
 149                        " create gsi qp: failed. max_recv_sge is larger the max %d>%d\n",
 150                        attrs->cap.max_recv_sge, QEDR_GSI_MAX_RECV_SGE);
 151                 return -EINVAL;
 152         }
 153
 154         if (attrs->cap.max_recv_wr > QEDR_GSI_MAX_RECV_WR) {
 155                 DP_ERR(dev,
 156                        " create gsi qp: failed. max_recv_wr is too large %d>%d\n",
 157                        attrs->cap.max_recv_wr, QEDR_GSI_MAX_RECV_WR);
 158                 return -EINVAL;
 159         }
 160
 161         if (attrs->cap.max_send_wr > QEDR_GSI_MAX_SEND_WR) {
 162                 DP_ERR(dev,
 163                        " create gsi qp: failed. max_send_wr is too large %d>%d\n",
 164                        attrs->cap.max_send_wr, QEDR_GSI_MAX_SEND_WR);
 165                 return -EINVAL;
 166         }
 167
 168         return 0;
 169 }
 170
 171 struct ib_qp *qedr_create_gsi_qp(struct qedr_dev *dev,
 172                                  struct ib_qp_init_attr *attrs,
 173                                  struct qedr_qp *qp)
 174 {
 175         struct qed_roce_ll2_params ll2_params;
 176         int rc;
 177
 178         rc = qedr_check_gsi_qp_attrs(dev, attrs);
 179         if (rc)
 180                 return ERR_PTR(rc);
 181
 182         /* configure and start LL2 */
 183         memset(&ll2_params, 0, sizeof(ll2_params));
 184         ll2_params.max_tx_buffers = attrs->cap.max_send_wr;
 185         ll2_params.max_rx_buffers = attrs->cap.max_recv_wr;
 186         ll2_params.cbs.tx_cb = qedr_ll2_tx_cb;
 187         ll2_params.cbs.rx_cb = qedr_ll2_rx_cb;
 188         ll2_params.cb_cookie = (void *)dev;
 189         ll2_params.mtu = dev->ndev->mtu;
 190         ether_addr_copy(ll2_params.mac_address, dev->ndev->dev_addr);
 191         rc = dev->ops->roce_ll2_start(dev->cdev, &ll2_params);
 192         if (rc) {
 193                 DP_ERR(dev, "create gsi qp: failed on ll2 start. rc=%d\n", rc);
 194                 return ERR_PTR(rc);
 195         }
 196
 197         /* create QP */
 198         qp->ibqp.qp_num = 1;
 199         qp->rq.max_wr = attrs->cap.max_recv_wr;
 200         qp->sq.max_wr = attrs->cap.max_send_wr;
 201
 202         qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
 203                                 GFP_KERNEL);
 204         if (!qp->rqe_wr_id)
 205                 goto err;
 206         qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
 207                                 GFP_KERNEL);
 208         if (!qp->wqe_wr_id)
 209                 goto err;
 210
 211         qedr_store_gsi_qp_cq(dev, qp, attrs);
 212         ether_addr_copy(dev->gsi_ll2_mac_address, dev->ndev->dev_addr);
 213
 214         /* the GSI CQ is handled by the driver so remove it from the FW */
 215         qedr_destroy_gsi_cq(dev, attrs);
 216         dev->gsi_rqcq->cq_type = QEDR_CQ_TYPE_GSI;
 217         dev->gsi_rqcq->cq_type = QEDR_CQ_TYPE_GSI;
 218
 219         DP_DEBUG(dev, QEDR_MSG_GSI, "created GSI QP %p\n", qp);
 220
 221         return &qp->ibqp;
 222
 223 err:
 224         kfree(qp->rqe_wr_id);
 225
 226         rc = dev->ops->roce_ll2_stop(dev->cdev);
 227         if (rc)
 228                 DP_ERR(dev, "create gsi qp: failed destroy on create\n");
 229
 230         return ERR_PTR(-ENOMEM);
 231 }
 232
 233 int qedr_destroy_gsi_qp(struct qedr_dev *dev)
 234 {
 235         int rc;
 236
 237         rc = dev->ops->roce_ll2_stop(dev->cdev);
 238         if (rc)
 239                 DP_ERR(dev, "destroy gsi qp: failed (rc=%d)\n", rc);
 240         else
 241                 DP_DEBUG(dev, QEDR_MSG_GSI, "destroy gsi qp: success\n");
 242
 243         return rc;
 244 }
 245
 246 #define QEDR_MAX_UD_HEADER_SIZE (100)
 247 #define QEDR_GSI_QPN            (1)
 248 static inline int qedr_gsi_build_header(struct qedr_dev *dev,
 249                                         struct qedr_qp *qp,
 250                                         struct ib_send_wr *swr,
 251                                         struct ib_ud_header *udh,
 252                                         int *roce_mode)
 253 {
 254         bool has_vlan = false, has_grh_ipv6 = true;
 255         struct ib_ah_attr *ah_attr = &get_qedr_ah(ud_wr(swr)->ah)->attr;
 256         struct ib_global_route *grh = &ah_attr->grh;
 257         union ib_gid sgid;
 258         int send_size = 0;
 259         u16 vlan_id = 0;
 260         u16 ether_type;
 261         struct ib_gid_attr sgid_attr;
 262         int rc;
 263         int ip_ver = 0;
 264
 265         bool has_udp = false;
 266         int i;
 267
 268         send_size = 0;
 269         for (i = 0; i < swr->num_sge; ++i)
 270                 send_size += swr->sg_list[i].length;
 271
 272         rc = ib_get_cached_gid(qp->ibqp.device, ah_attr->port_num,
 273                                grh->sgid_index, &sgid, &sgid_attr);
 274         if (rc) {
 275                 DP_ERR(dev,
 276                        "gsi post send: failed to get cached GID (port=%d, ix=%d)\n",
 277                        ah_attr->port_num, grh->sgid_index);
 278                 return rc;
 279         }
 280
 281         vlan_id = rdma_vlan_dev_vlan_id(sgid_attr.ndev);
 282         if (vlan_id < VLAN_CFI_MASK)
 283                 has_vlan = true;
 284         if (sgid_attr.ndev)
 285                 dev_put(sgid_attr.ndev);
 286
 287         if (!memcmp(&sgid, &zgid, sizeof(sgid))) {
 288                 DP_ERR(dev, "gsi post send: GID not found GID index %d\n",
 289                        ah_attr->grh.sgid_index);
 290                 return -ENOENT;
 291         }
 292
 293         has_udp = (sgid_attr.gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP);
 294         if (!has_udp) {
 295                 /* RoCE v1 */
 296                 ether_type = ETH_P_ROCE;
 297                 *roce_mode = ROCE_V1;
 298         } else if (ipv6_addr_v4mapped((struct in6_addr *)&sgid)) {
 299                 /* RoCE v2 IPv4 */
 300                 ip_ver = 4;
 301                 ether_type = ETH_P_IP;
 302                 has_grh_ipv6 = false;
 303                 *roce_mode = ROCE_V2_IPV4;
 304         } else {
 305                 /* RoCE v2 IPv6 */
 306                 ip_ver = 6;
 307                 ether_type = ETH_P_IPV6;
 308                 *roce_mode = ROCE_V2_IPV6;
 309         }
 310
 311         rc = ib_ud_header_init(send_size, false, true, has_vlan,
 312                                has_grh_ipv6, ip_ver, has_udp, 0, udh);
 313         if (rc) {
 314                 DP_ERR(dev, "gsi post send: failed to init header\n");
 315                 return rc;
 316         }
 317
 318         /* ENET + VLAN headers */
 319         ether_addr_copy(udh->eth.dmac_h, ah_attr->dmac);
 320         ether_addr_copy(udh->eth.smac_h, dev->ndev->dev_addr);
 321         if (has_vlan) {
 322                 udh->eth.type = htons(ETH_P_8021Q);
 323                 udh->vlan.tag = htons(vlan_id);
 324                 udh->vlan.type = htons(ether_type);
 325         } else {
 326                 udh->eth.type = htons(ether_type);
 327         }
 328
 329         /* BTH */
 330         udh->bth.solicited_event = !!(swr->send_flags & IB_SEND_SOLICITED);
 331         udh->bth.pkey = QEDR_ROCE_PKEY_DEFAULT;
 332         udh->bth.destination_qpn = htonl(ud_wr(swr)->remote_qpn);
 333         udh->bth.psn = htonl((qp->sq_psn++) & ((1 << 24) - 1));
 334         udh->bth.opcode = IB_OPCODE_UD_SEND_ONLY;
 335
 336         /* DETH */
 337         udh->deth.qkey = htonl(0x80010000);
 338         udh->deth.source_qpn = htonl(QEDR_GSI_QPN);
 339
 340         if (has_grh_ipv6) {
 341                 /* GRH / IPv6 header */
 342                 udh->grh.traffic_class = grh->traffic_class;
 343                 udh->grh.flow_label = grh->flow_label;
 344                 udh->grh.hop_limit = grh->hop_limit;
 345                 udh->grh.destination_gid = grh->dgid;
 346                 memcpy(&udh->grh.source_gid.raw, &sgid.raw,
 347                        sizeof(udh->grh.source_gid.raw));
 348         } else {
 349                 /* IPv4 header */
 350                 u32 ipv4_addr;
 351
 352                 udh->ip4.protocol = IPPROTO_UDP;
 353                 udh->ip4.tos = htonl(ah_attr->grh.flow_label);
 354                 udh->ip4.frag_off = htons(IP_DF);
 355                 udh->ip4.ttl = ah_attr->grh.hop_limit;
 356
 357                 ipv4_addr = qedr_get_ipv4_from_gid(sgid.raw);
 358                 udh->ip4.saddr = ipv4_addr;
 359                 ipv4_addr = qedr_get_ipv4_from_gid(ah_attr->grh.dgid.raw);
 360                 udh->ip4.daddr = ipv4_addr;
 361                 /* note: checksum is calculated by the device */
 362         }
 363
 364         /* UDP */
 365         if (has_udp) {
 366                 udh->udp.sport = htons(QEDR_ROCE_V2_UDP_SPORT);
 367                 udh->udp.dport = htons(ROCE_V2_UDP_DPORT);
 368                 udh->udp.csum = 0;
 369                 /* UDP length is untouched hence is zero */
 370         }
 371         return 0;
 372 }
 373
 374 static inline int qedr_gsi_build_packet(struct qedr_dev *dev,
 375                                         struct qedr_qp *qp,
 376                                         struct ib_send_wr *swr,
 377                                         struct qed_roce_ll2_packet **p_packet)
 378 {
 379         u8 ud_header_buffer[QEDR_MAX_UD_HEADER_SIZE];
 380         struct qed_roce_ll2_packet *packet;
 381         struct pci_dev *pdev = dev->pdev;
 382         int roce_mode, header_size;
 383         struct ib_ud_header udh;
 384         int i, rc;
 385
 386         *p_packet = NULL;
 387
 388         rc = qedr_gsi_build_header(dev, qp, swr, &udh, &roce_mode);
 389         if (rc)
 390                 return rc;
 391
 392         header_size = ib_ud_header_pack(&udh, &ud_header_buffer);
 393
 394         packet = kzalloc(sizeof(*packet), GFP_ATOMIC);
 395         if (!packet)
 396                 return -ENOMEM;
 397
 398         packet->header.vaddr = dma_alloc_coherent(&pdev->dev, header_size,
 399                                                   &packet->header.baddr,
 400                                                   GFP_ATOMIC);
 401         if (!packet->header.vaddr) {
 402                 kfree(packet);
 403                 return -ENOMEM;
 404         }
 405
 406         if (ether_addr_equal(udh.eth.smac_h, udh.eth.dmac_h))
 407                 packet->tx_dest = QED_ROCE_LL2_TX_DEST_LB;
 408         else
 409                 packet->tx_dest = QED_ROCE_LL2_TX_DEST_NW;
 410
 411         packet->roce_mode = roce_mode;
 412         memcpy(packet->header.vaddr, ud_header_buffer, header_size);
 413         packet->header.len = header_size;
 414         packet->n_seg = swr->num_sge;
 415         for (i = 0; i < packet->n_seg; i++) {
 416                 packet->payload[i].baddr = swr->sg_list[i].addr;
 417                 packet->payload[i].len = swr->sg_list[i].length;
 418         }
 419
 420         *p_packet = packet;
 421
 422         return 0;
 423 }
 424
 425 int qedr_gsi_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 426                        struct ib_send_wr **bad_wr)
 427 {
 428         struct qed_roce_ll2_packet *pkt = NULL;
 429         struct qedr_qp *qp = get_qedr_qp(ibqp);
 430         struct qed_roce_ll2_tx_params params;
 431         struct qedr_dev *dev = qp->dev;
 432         unsigned long flags;
 433         int rc;
 434
 435         if (qp->state != QED_ROCE_QP_STATE_RTS) {
 436                 *bad_wr = wr;
 437                 DP_ERR(dev,
 438                        "gsi post recv: failed to post rx buffer. state is %d and not QED_ROCE_QP_STATE_RTS\n",
 439                        qp->state);
 440                 return -EINVAL;
 441         }
 442
 443         if (wr->num_sge > RDMA_MAX_SGE_PER_SQ_WQE) {
 444                 DP_ERR(dev, "gsi post send: num_sge is too large (%d>%d)\n",
 445                        wr->num_sge, RDMA_MAX_SGE_PER_SQ_WQE);
 446                 rc = -EINVAL;
 447                 goto err;
 448         }
 449
 450         if (wr->opcode != IB_WR_SEND) {
 451                 DP_ERR(dev,
 452                        "gsi post send: failed due to unsupported opcode %d\n",
 453                        wr->opcode);
 454                 rc = -EINVAL;
 455                 goto err;
 456         }
 457
 458         memset(&params, 0, sizeof(params));
 459
 460         spin_lock_irqsave(&qp->q_lock, flags);
 461
 462         rc = qedr_gsi_build_packet(dev, qp, wr, &pkt);
 463         if (rc) {
 464                 spin_unlock_irqrestore(&qp->q_lock, flags);
 465                 goto err;
 466         }
 467
 468         rc = dev->ops->roce_ll2_tx(dev->cdev, pkt, &params);
 469         if (!rc) {
 470                 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
 471                 qedr_inc_sw_prod(&qp->sq);
 472                 DP_DEBUG(qp->dev, QEDR_MSG_GSI,
 473                          "gsi post send: opcode=%d, in_irq=%ld, irqs_disabled=%d, wr_id=%llx\n",
 474                          wr->opcode, in_irq(), irqs_disabled(), wr->wr_id);
 475         } else {
 476                 if (rc == QED_ROCE_TX_HEAD_FAILURE) {
 477                         /* TX failed while posting header - release resources */
 478                         dma_free_coherent(&dev->pdev->dev, pkt->header.len,
 479                                           pkt->header.vaddr, pkt->header.baddr);
 480                         kfree(pkt);
 481                 } else if (rc == QED_ROCE_TX_FRAG_FAILURE) {
 482                         /* NTD since TX failed while posting a fragment. We will
 483                          * release the resources on TX callback
 484                          */
 485                 }
 486
 487                 DP_ERR(dev, "gsi post send: failed to transmit (rc=%d)\n", rc);
 488                 rc = -EAGAIN;
 489                 *bad_wr = wr;
 490         }
 491
 492         spin_unlock_irqrestore(&qp->q_lock, flags);
 493
 494         if (wr->next) {
 495                 DP_ERR(dev,
 496                        "gsi post send: failed second WR. Only one WR may be passed at a time\n");
 497                 *bad_wr = wr->next;
 498                 rc = -EINVAL;
 499         }
 500
 501         return rc;
 502
 503 err:
 504         *bad_wr = wr;
 505         return rc;
 506 }
 507
 508 int qedr_gsi_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 509                        struct ib_recv_wr **bad_wr)
 510 {
 511         struct qedr_dev *dev = get_qedr_dev(ibqp->device);
 512         struct qedr_qp *qp = get_qedr_qp(ibqp);
 513         struct qed_roce_ll2_buffer buf;
 514         unsigned long flags;
 515         int status = 0;
 516         int rc;
 517
 518         if ((qp->state != QED_ROCE_QP_STATE_RTR) &&
 519             (qp->state != QED_ROCE_QP_STATE_RTS)) {
 520                 *bad_wr = wr;
 521                 DP_ERR(dev,
 522                        "gsi post recv: failed to post rx buffer. state is %d and not QED_ROCE_QP_STATE_RTR/S\n",
 523                        qp->state);
 524                 return -EINVAL;
 525         }
 526
 527         memset(&buf, 0, sizeof(buf));
 528
 529         spin_lock_irqsave(&qp->q_lock, flags);
 530
 531         while (wr) {
 532                 if (wr->num_sge > QEDR_GSI_MAX_RECV_SGE) {
 533                         DP_ERR(dev,
 534                                "gsi post recv: failed to post rx buffer. too many sges %d>%d\n",
 535                                wr->num_sge, QEDR_GSI_MAX_RECV_SGE);
 536                         goto err;
 537                 }
 538
 539                 buf.baddr = wr->sg_list[0].addr;
 540                 buf.len = wr->sg_list[0].length;
 541
 542                 rc = dev->ops->roce_ll2_post_rx_buffer(dev->cdev, &buf, 0, 1);
 543                 if (rc) {
 544                         DP_ERR(dev,
 545                                "gsi post recv: failed to post rx buffer (rc=%d)\n",
 546                                rc);
 547                         goto err;
 548                 }
 549
 550                 memset(&qp->rqe_wr_id[qp->rq.prod], 0,
 551                        sizeof(qp->rqe_wr_id[qp->rq.prod]));
 552                 qp->rqe_wr_id[qp->rq.prod].sg_list[0] = wr->sg_list[0];
 553                 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
 554
 555                 qedr_inc_sw_prod(&qp->rq);
 556
 557                 wr = wr->next;
 558         }
 559
 560         spin_unlock_irqrestore(&qp->q_lock, flags);
 561
 562         return status;
 563 err:
 564         spin_unlock_irqrestore(&qp->q_lock, flags);
 565         *bad_wr = wr;
 566         return -ENOMEM;
 567 }
 568
 569 int qedr_gsi_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
 570 {
 571         struct qedr_dev *dev = get_qedr_dev(ibcq->device);
 572         struct qedr_cq *cq = get_qedr_cq(ibcq);
 573         struct qedr_qp *qp = dev->gsi_qp;
 574         unsigned long flags;
 575         int i = 0;
 576
 577         spin_lock_irqsave(&cq->cq_lock, flags);
 578
 579         while (i < num_entries && qp->rq.cons != qp->rq.gsi_cons) {
 580                 memset(&wc[i], 0, sizeof(*wc));
 581
 582                 wc[i].qp = &qp->ibqp;
 583                 wc[i].wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
 584                 wc[i].opcode = IB_WC_RECV;
 585                 wc[i].pkey_index = 0;
 586                 wc[i].status = (qp->rqe_wr_id[qp->rq.cons].rc) ?
 587                     IB_WC_GENERAL_ERR : IB_WC_SUCCESS;
 588                 /* 0 - currently only one recv sg is supported */
 589                 wc[i].byte_len = qp->rqe_wr_id[qp->rq.cons].sg_list[0].length;
 590                 wc[i].wc_flags |= IB_WC_GRH | IB_WC_IP_CSUM_OK;
 591                 ether_addr_copy(wc[i].smac, qp->rqe_wr_id[qp->rq.cons].smac);
 592                 wc[i].wc_flags |= IB_WC_WITH_SMAC;
 593                 if (qp->rqe_wr_id[qp->rq.cons].vlan_id) {
 594                         wc[i].wc_flags |= IB_WC_WITH_VLAN;
 595                         wc[i].vlan_id = qp->rqe_wr_id[qp->rq.cons].vlan_id;
 596                 }
 597
 598                 qedr_inc_sw_cons(&qp->rq);
 599                 i++;
 600         }
 601
 602         while (i < num_entries && qp->sq.cons != qp->sq.gsi_cons) {
 603                 memset(&wc[i], 0, sizeof(*wc));
 604
 605                 wc[i].qp = &qp->ibqp;
 606                 wc[i].wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
 607                 wc[i].opcode = IB_WC_SEND;
 608                 wc[i].status = IB_WC_SUCCESS;
 609
 610                 qedr_inc_sw_cons(&qp->sq);
 611                 i++;
 612         }
 613
 614         spin_unlock_irqrestore(&cq->cq_lock, flags);
 615
 616         DP_DEBUG(dev, QEDR_MSG_GSI,
 617                  "gsi poll_cq: requested entries=%d, actual=%d, qp->rq.cons=%d, qp->rq.gsi_cons=%x, qp->sq.cons=%d, qp->sq.gsi_cons=%d, qp_num=%d\n",
 618                  num_entries, i, qp->rq.cons, qp->rq.gsi_cons, qp->sq.cons,
 619                  qp->sq.gsi_cons, qp->ibqp.qp_num);
 620
 621         return i;
 622 }