drivers/infiniband/sw/siw/siw_verbs.c

   1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
   2
   3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
   4 /* Copyright (c) 2008-2019, IBM Corporation */
   5
   6 #include <linux/errno.h>
   7 #include <linux/types.h>
   8 #include <linux/uaccess.h>
   9 #include <linux/vmalloc.h>
  10 #include <linux/xarray.h>
  11
  12 #include <rdma/iw_cm.h>
  13 #include <rdma/ib_verbs.h>
  14 #include <rdma/ib_user_verbs.h>
  15 #include <rdma/uverbs_ioctl.h>
  16
  17 #include "siw.h"
  18 #include "siw_verbs.h"
  19 #include "siw_mem.h"
  20
  21 static int ib_qp_state_to_siw_qp_state[IB_QPS_ERR + 1] = {
  22         [IB_QPS_RESET] = SIW_QP_STATE_IDLE,
  23         [IB_QPS_INIT] = SIW_QP_STATE_IDLE,
  24         [IB_QPS_RTR] = SIW_QP_STATE_RTR,
  25         [IB_QPS_RTS] = SIW_QP_STATE_RTS,
  26         [IB_QPS_SQD] = SIW_QP_STATE_CLOSING,
  27         [IB_QPS_SQE] = SIW_QP_STATE_TERMINATE,
  28         [IB_QPS_ERR] = SIW_QP_STATE_ERROR
  29 };
  30
  31 static char ib_qp_state_to_string[IB_QPS_ERR + 1][sizeof("RESET")] = {
  32         [IB_QPS_RESET] = "RESET", [IB_QPS_INIT] = "INIT", [IB_QPS_RTR] = "RTR",
  33         [IB_QPS_RTS] = "RTS",     [IB_QPS_SQD] = "SQD",   [IB_QPS_SQE] = "SQE",
  34         [IB_QPS_ERR] = "ERR"
  35 };
  36
  37 void siw_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
  38 {
  39         struct siw_user_mmap_entry *entry = to_siw_mmap_entry(rdma_entry);
  40
  41         kfree(entry);
  42 }
  43
  44 int siw_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
  45 {
  46         struct siw_ucontext *uctx = to_siw_ctx(ctx);
  47         size_t size = vma->vm_end - vma->vm_start;
  48         struct rdma_user_mmap_entry *rdma_entry;
  49         struct siw_user_mmap_entry *entry;
  50         int rv = -EINVAL;
  51
  52         /*
  53          * Must be page aligned
  54          */
  55         if (vma->vm_start & (PAGE_SIZE - 1)) {
  56                 pr_warn("siw: mmap not page aligned\n");
  57                 return -EINVAL;
  58         }
  59         rdma_entry = rdma_user_mmap_entry_get(&uctx->base_ucontext, vma);
  60         if (!rdma_entry) {
  61                 siw_dbg(&uctx->sdev->base_dev, "mmap lookup failed: %lu, %#zx\n",
  62                         vma->vm_pgoff, size);
  63                 return -EINVAL;
  64         }
  65         entry = to_siw_mmap_entry(rdma_entry);
  66
  67         rv = remap_vmalloc_range(vma, entry->address, 0);
  68         if (rv) {
  69                 pr_warn("remap_vmalloc_range failed: %lu, %zu\n", vma->vm_pgoff,
  70                         size);
  71                 goto out;
  72         }
  73 out:
  74         rdma_user_mmap_entry_put(rdma_entry);
  75
  76         return rv;
  77 }
  78
  79 int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata)
  80 {
  81         struct siw_device *sdev = to_siw_dev(base_ctx->device);
  82         struct siw_ucontext *ctx = to_siw_ctx(base_ctx);
  83         struct siw_uresp_alloc_ctx uresp = {};
  84         int rv;
  85
  86         if (atomic_inc_return(&sdev->num_ctx) > SIW_MAX_CONTEXT) {
  87                 rv = -ENOMEM;
  88                 goto err_out;
  89         }
  90         ctx->sdev = sdev;
  91
  92         uresp.dev_id = sdev->vendor_part_id;
  93
  94         if (udata->outlen < sizeof(uresp)) {
  95                 rv = -EINVAL;
  96                 goto err_out;
  97         }
  98         rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
  99         if (rv)
 100                 goto err_out;
 101
 102         siw_dbg(base_ctx->device, "success. now %d context(s)\n",
 103                 atomic_read(&sdev->num_ctx));
 104
 105         return 0;
 106
 107 err_out:
 108         atomic_dec(&sdev->num_ctx);
 109         siw_dbg(base_ctx->device, "failure %d. now %d context(s)\n", rv,
 110                 atomic_read(&sdev->num_ctx));
 111
 112         return rv;
 113 }
 114
 115 void siw_dealloc_ucontext(struct ib_ucontext *base_ctx)
 116 {
 117         struct siw_ucontext *uctx = to_siw_ctx(base_ctx);
 118
 119         atomic_dec(&uctx->sdev->num_ctx);
 120 }
 121
 122 int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
 123                      struct ib_udata *udata)
 124 {
 125         struct siw_device *sdev = to_siw_dev(base_dev);
 126
 127         if (udata->inlen || udata->outlen)
 128                 return -EINVAL;
 129
 130         memset(attr, 0, sizeof(*attr));
 131
 132         /* Revisit atomic caps if RFC 7306 gets supported */
 133         attr->atomic_cap = 0;
 134         attr->device_cap_flags =
 135                 IB_DEVICE_MEM_MGT_EXTENSIONS | IB_DEVICE_ALLOW_USER_UNREG;
 136         attr->max_cq = sdev->attrs.max_cq;
 137         attr->max_cqe = sdev->attrs.max_cqe;
 138         attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
 139         attr->max_mr = sdev->attrs.max_mr;
 140         attr->max_mw = sdev->attrs.max_mw;
 141         attr->max_mr_size = ~0ull;
 142         attr->max_pd = sdev->attrs.max_pd;
 143         attr->max_qp = sdev->attrs.max_qp;
 144         attr->max_qp_init_rd_atom = sdev->attrs.max_ird;
 145         attr->max_qp_rd_atom = sdev->attrs.max_ord;
 146         attr->max_qp_wr = sdev->attrs.max_qp_wr;
 147         attr->max_recv_sge = sdev->attrs.max_sge;
 148         attr->max_res_rd_atom = sdev->attrs.max_qp * sdev->attrs.max_ird;
 149         attr->max_send_sge = sdev->attrs.max_sge;
 150         attr->max_sge_rd = sdev->attrs.max_sge_rd;
 151         attr->max_srq = sdev->attrs.max_srq;
 152         attr->max_srq_sge = sdev->attrs.max_srq_sge;
 153         attr->max_srq_wr = sdev->attrs.max_srq_wr;
 154         attr->page_size_cap = PAGE_SIZE;
 155         attr->vendor_id = SIW_VENDOR_ID;
 156         attr->vendor_part_id = sdev->vendor_part_id;
 157
 158         memcpy(&attr->sys_image_guid, sdev->netdev->dev_addr, 6);
 159
 160         return 0;
 161 }
 162
 163 int siw_query_port(struct ib_device *base_dev, u8 port,
 164                    struct ib_port_attr *attr)
 165 {
 166         struct siw_device *sdev = to_siw_dev(base_dev);
 167         int rv;
 168
 169         memset(attr, 0, sizeof(*attr));
 170
 171         rv = ib_get_eth_speed(base_dev, port, &attr->active_speed,
 172                          &attr->active_width);
 173         attr->gid_tbl_len = 1;
 174         attr->max_msg_sz = -1;
 175         attr->max_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 176         attr->active_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 177         attr->phys_state = sdev->state == IB_PORT_ACTIVE ?
 178                 IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
 179         attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
 180         attr->state = sdev->state;
 181         /*
 182          * All zero
 183          *
 184          * attr->lid = 0;
 185          * attr->bad_pkey_cntr = 0;
 186          * attr->qkey_viol_cntr = 0;
 187          * attr->sm_lid = 0;
 188          * attr->lmc = 0;
 189          * attr->max_vl_num = 0;
 190          * attr->sm_sl = 0;
 191          * attr->subnet_timeout = 0;
 192          * attr->init_type_repy = 0;
 193          */
 194         return rv;
 195 }
 196
 197 int siw_get_port_immutable(struct ib_device *base_dev, u8 port,
 198                            struct ib_port_immutable *port_immutable)
 199 {
 200         struct ib_port_attr attr;
 201         int rv = siw_query_port(base_dev, port, &attr);
 202
 203         if (rv)
 204                 return rv;
 205
 206         port_immutable->gid_tbl_len = attr.gid_tbl_len;
 207         port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
 208
 209         return 0;
 210 }
 211
 212 int siw_query_gid(struct ib_device *base_dev, u8 port, int idx,
 213                   union ib_gid *gid)
 214 {
 215         struct siw_device *sdev = to_siw_dev(base_dev);
 216
 217         /* subnet_prefix == interface_id == 0; */
 218         memset(gid, 0, sizeof(*gid));
 219         memcpy(&gid->raw[0], sdev->netdev->dev_addr, 6);
 220
 221         return 0;
 222 }
 223
 224 int siw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 225 {
 226         struct siw_device *sdev = to_siw_dev(pd->device);
 227
 228         if (atomic_inc_return(&sdev->num_pd) > SIW_MAX_PD) {
 229                 atomic_dec(&sdev->num_pd);
 230                 return -ENOMEM;
 231         }
 232         siw_dbg_pd(pd, "now %d PD's(s)\n", atomic_read(&sdev->num_pd));
 233
 234         return 0;
 235 }
 236
 237 int siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
 238 {
 239         struct siw_device *sdev = to_siw_dev(pd->device);
 240
 241         siw_dbg_pd(pd, "free PD\n");
 242         atomic_dec(&sdev->num_pd);
 243         return 0;
 244 }
 245
 246 void siw_qp_get_ref(struct ib_qp *base_qp)
 247 {
 248         siw_qp_get(to_siw_qp(base_qp));
 249 }
 250
 251 void siw_qp_put_ref(struct ib_qp *base_qp)
 252 {
 253         siw_qp_put(to_siw_qp(base_qp));
 254 }
 255
 256 static struct rdma_user_mmap_entry *
 257 siw_mmap_entry_insert(struct siw_ucontext *uctx,
 258                       void *address, size_t length,
 259                       u64 *offset)
 260 {
 261         struct siw_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 262         int rv;
 263
 264         *offset = SIW_INVAL_UOBJ_KEY;
 265         if (!entry)
 266                 return NULL;
 267
 268         entry->address = address;
 269
 270         rv = rdma_user_mmap_entry_insert(&uctx->base_ucontext,
 271                                          &entry->rdma_entry,
 272                                          length);
 273         if (rv) {
 274                 kfree(entry);
 275                 return NULL;
 276         }
 277
 278         *offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
 279
 280         return &entry->rdma_entry;
 281 }
 282
 283 /*
 284  * siw_create_qp()
 285  *
 286  * Create QP of requested size on given device.
 287  *
 288  * @pd:         Protection Domain
 289  * @attrs:      Initial QP attributes.
 290  * @udata:      used to provide QP ID, SQ and RQ size back to user.
 291  */
 292
 293 struct ib_qp *siw_create_qp(struct ib_pd *pd,
 294                             struct ib_qp_init_attr *attrs,
 295                             struct ib_udata *udata)
 296 {
 297         struct siw_qp *qp = NULL;
 298         struct ib_device *base_dev = pd->device;
 299         struct siw_device *sdev = to_siw_dev(base_dev);
 300         struct siw_ucontext *uctx =
 301                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
 302                                           base_ucontext);
 303         unsigned long flags;
 304         int num_sqe, num_rqe, rv = 0;
 305         size_t length;
 306
 307         siw_dbg(base_dev, "create new QP\n");
 308
 309         if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
 310                 siw_dbg(base_dev, "too many QP's\n");
 311                 rv = -ENOMEM;
 312                 goto err_out;
 313         }
 314         if (attrs->qp_type != IB_QPT_RC) {
 315                 siw_dbg(base_dev, "only RC QP's supported\n");
 316                 rv = -EOPNOTSUPP;
 317                 goto err_out;
 318         }
 319         if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||
 320             (attrs->cap.max_recv_wr > SIW_MAX_QP_WR) ||
 321             (attrs->cap.max_send_sge > SIW_MAX_SGE) ||
 322             (attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
 323                 siw_dbg(base_dev, "QP size error\n");
 324                 rv = -EINVAL;
 325                 goto err_out;
 326         }
 327         if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
 328                 siw_dbg(base_dev, "max inline send: %d > %d\n",
 329                         attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
 330                 rv = -EINVAL;
 331                 goto err_out;
 332         }
 333         /*
 334          * NOTE: we allow for zero element SQ and RQ WQE's SGL's
 335          * but not for a QP unable to hold any WQE (SQ + RQ)
 336          */
 337         if (attrs->cap.max_send_wr + attrs->cap.max_recv_wr == 0) {
 338                 siw_dbg(base_dev, "QP must have send or receive queue\n");
 339                 rv = -EINVAL;
 340                 goto err_out;
 341         }
 342
 343         if (!attrs->send_cq || (!attrs->recv_cq && !attrs->srq)) {
 344                 siw_dbg(base_dev, "send CQ or receive CQ invalid\n");
 345                 rv = -EINVAL;
 346                 goto err_out;
 347         }
 348         qp = kzalloc(sizeof(*qp), GFP_KERNEL);
 349         if (!qp) {
 350                 rv = -ENOMEM;
 351                 goto err_out;
 352         }
 353         init_rwsem(&qp->state_lock);
 354         spin_lock_init(&qp->sq_lock);
 355         spin_lock_init(&qp->rq_lock);
 356         spin_lock_init(&qp->orq_lock);
 357
 358         rv = siw_qp_add(sdev, qp);
 359         if (rv)
 360                 goto err_out;
 361
 362         num_sqe = attrs->cap.max_send_wr;
 363         num_rqe = attrs->cap.max_recv_wr;
 364
 365         /* All queue indices are derived from modulo operations
 366          * on a free running 'get' (consumer) and 'put' (producer)
 367          * unsigned counter. Having queue sizes at power of two
 368          * avoids handling counter wrap around.
 369          */
 370         if (num_sqe)
 371                 num_sqe = roundup_pow_of_two(num_sqe);
 372         else {
 373                 /* Zero sized SQ is not supported */
 374                 rv = -EINVAL;
 375                 goto err_out_xa;
 376         }
 377         if (num_rqe)
 378                 num_rqe = roundup_pow_of_two(num_rqe);
 379
 380         if (udata)
 381                 qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe));
 382         else
 383                 qp->sendq = vzalloc(num_sqe * sizeof(struct siw_sqe));
 384
 385         if (qp->sendq == NULL) {
 386                 rv = -ENOMEM;
 387                 goto err_out_xa;
 388         }
 389         if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
 390                 if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
 391                         qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
 392                 else {
 393                         rv = -EINVAL;
 394                         goto err_out_xa;
 395                 }
 396         }
 397         qp->pd = pd;
 398         qp->scq = to_siw_cq(attrs->send_cq);
 399         qp->rcq = to_siw_cq(attrs->recv_cq);
 400
 401         if (attrs->srq) {
 402                 /*
 403                  * SRQ support.
 404                  * Verbs 6.3.7: ignore RQ size, if SRQ present
 405                  * Verbs 6.3.5: do not check PD of SRQ against PD of QP
 406                  */
 407                 qp->srq = to_siw_srq(attrs->srq);
 408                 qp->attrs.rq_size = 0;
 409                 siw_dbg(base_dev, "QP [%u]: SRQ attached\n",
 410                         qp->base_qp.qp_num);
 411         } else if (num_rqe) {
 412                 if (udata)
 413                         qp->recvq =
 414                                 vmalloc_user(num_rqe * sizeof(struct siw_rqe));
 415                 else
 416                         qp->recvq = vzalloc(num_rqe * sizeof(struct siw_rqe));
 417
 418                 if (qp->recvq == NULL) {
 419                         rv = -ENOMEM;
 420                         goto err_out_xa;
 421                 }
 422                 qp->attrs.rq_size = num_rqe;
 423         }
 424         qp->attrs.sq_size = num_sqe;
 425         qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
 426         qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
 427
 428         /* Make those two tunables fixed for now. */
 429         qp->tx_ctx.gso_seg_limit = 1;
 430         qp->tx_ctx.zcopy_tx = zcopy_tx;
 431
 432         qp->attrs.state = SIW_QP_STATE_IDLE;
 433
 434         if (udata) {
 435                 struct siw_uresp_create_qp uresp = {};
 436
 437                 uresp.num_sqe = num_sqe;
 438                 uresp.num_rqe = num_rqe;
 439                 uresp.qp_id = qp_id(qp);
 440
 441                 if (qp->sendq) {
 442                         length = num_sqe * sizeof(struct siw_sqe);
 443                         qp->sq_entry =
 444                                 siw_mmap_entry_insert(uctx, qp->sendq,
 445                                                       length, &uresp.sq_key);
 446                         if (!qp->sq_entry) {
 447                                 rv = -ENOMEM;
 448                                 goto err_out_xa;
 449                         }
 450                 }
 451
 452                 if (qp->recvq) {
 453                         length = num_rqe * sizeof(struct siw_rqe);
 454                         qp->rq_entry =
 455                                 siw_mmap_entry_insert(uctx, qp->recvq,
 456                                                       length, &uresp.rq_key);
 457                         if (!qp->rq_entry) {
 458                                 uresp.sq_key = SIW_INVAL_UOBJ_KEY;
 459                                 rv = -ENOMEM;
 460                                 goto err_out_xa;
 461                         }
 462                 }
 463
 464                 if (udata->outlen < sizeof(uresp)) {
 465                         rv = -EINVAL;
 466                         goto err_out_xa;
 467                 }
 468                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 469                 if (rv)
 470                         goto err_out_xa;
 471         }
 472         qp->tx_cpu = siw_get_tx_cpu(sdev);
 473         if (qp->tx_cpu < 0) {
 474                 rv = -EINVAL;
 475                 goto err_out_xa;
 476         }
 477         INIT_LIST_HEAD(&qp->devq);
 478         spin_lock_irqsave(&sdev->lock, flags);
 479         list_add_tail(&qp->devq, &sdev->qp_list);
 480         spin_unlock_irqrestore(&sdev->lock, flags);
 481
 482         return &qp->base_qp;
 483
 484 err_out_xa:
 485         xa_erase(&sdev->qp_xa, qp_id(qp));
 486 err_out:
 487         if (qp) {
 488                 if (uctx) {
 489                         rdma_user_mmap_entry_remove(qp->sq_entry);
 490                         rdma_user_mmap_entry_remove(qp->rq_entry);
 491                 }
 492                 vfree(qp->sendq);
 493                 vfree(qp->recvq);
 494                 kfree(qp);
 495         }
 496         atomic_dec(&sdev->num_qp);
 497
 498         return ERR_PTR(rv);
 499 }
 500
 501 /*
 502  * Minimum siw_query_qp() verb interface.
 503  *
 504  * @qp_attr_mask is not used but all available information is provided
 505  */
 506 int siw_query_qp(struct ib_qp *base_qp, struct ib_qp_attr *qp_attr,
 507                  int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
 508 {
 509         struct siw_qp *qp;
 510         struct siw_device *sdev;
 511
 512         if (base_qp && qp_attr && qp_init_attr) {
 513                 qp = to_siw_qp(base_qp);
 514                 sdev = to_siw_dev(base_qp->device);
 515         } else {
 516                 return -EINVAL;
 517         }
 518         qp_attr->cap.max_inline_data = SIW_MAX_INLINE;
 519         qp_attr->cap.max_send_wr = qp->attrs.sq_size;
 520         qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges;
 521         qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
 522         qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges;
 523         qp_attr->path_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
 524         qp_attr->max_rd_atomic = qp->attrs.irq_size;
 525         qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
 526
 527         qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
 528                                    IB_ACCESS_REMOTE_WRITE |
 529                                    IB_ACCESS_REMOTE_READ;
 530
 531         qp_init_attr->qp_type = base_qp->qp_type;
 532         qp_init_attr->send_cq = base_qp->send_cq;
 533         qp_init_attr->recv_cq = base_qp->recv_cq;
 534         qp_init_attr->srq = base_qp->srq;
 535
 536         qp_init_attr->cap = qp_attr->cap;
 537
 538         return 0;
 539 }
 540
 541 int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr,
 542                         int attr_mask, struct ib_udata *udata)
 543 {
 544         struct siw_qp_attrs new_attrs;
 545         enum siw_qp_attr_mask siw_attr_mask = 0;
 546         struct siw_qp *qp = to_siw_qp(base_qp);
 547         int rv = 0;
 548
 549         if (!attr_mask)
 550                 return 0;
 551
 552         memset(&new_attrs, 0, sizeof(new_attrs));
 553
 554         if (attr_mask & IB_QP_ACCESS_FLAGS) {
 555                 siw_attr_mask = SIW_QP_ATTR_ACCESS_FLAGS;
 556
 557                 if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
 558                         new_attrs.flags |= SIW_RDMA_READ_ENABLED;
 559                 if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
 560                         new_attrs.flags |= SIW_RDMA_WRITE_ENABLED;
 561                 if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
 562                         new_attrs.flags |= SIW_RDMA_BIND_ENABLED;
 563         }
 564         if (attr_mask & IB_QP_STATE) {
 565                 siw_dbg_qp(qp, "desired IB QP state: %s\n",
 566                            ib_qp_state_to_string[attr->qp_state]);
 567
 568                 new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
 569
 570                 if (new_attrs.state > SIW_QP_STATE_RTS)
 571                         qp->tx_ctx.tx_suspend = 1;
 572
 573                 siw_attr_mask |= SIW_QP_ATTR_STATE;
 574         }
 575         if (!siw_attr_mask)
 576                 goto out;
 577
 578         down_write(&qp->state_lock);
 579
 580         rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask);
 581
 582         up_write(&qp->state_lock);
 583 out:
 584         return rv;
 585 }
 586
 587 int siw_destroy_qp(struct ib_qp *base_qp, struct ib_udata *udata)
 588 {
 589         struct siw_qp *qp = to_siw_qp(base_qp);
 590         struct siw_ucontext *uctx =
 591                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
 592                                           base_ucontext);
 593         struct siw_qp_attrs qp_attrs;
 594
 595         siw_dbg_qp(qp, "state %d\n", qp->attrs.state);
 596
 597         /*
 598          * Mark QP as in process of destruction to prevent from
 599          * any async callbacks to RDMA core
 600          */
 601         qp->attrs.flags |= SIW_QP_IN_DESTROY;
 602         qp->rx_stream.rx_suspend = 1;
 603
 604         if (uctx) {
 605                 rdma_user_mmap_entry_remove(qp->sq_entry);
 606                 rdma_user_mmap_entry_remove(qp->rq_entry);
 607         }
 608
 609         down_write(&qp->state_lock);
 610
 611         qp_attrs.state = SIW_QP_STATE_ERROR;
 612         siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
 613
 614         if (qp->cep) {
 615                 siw_cep_put(qp->cep);
 616                 qp->cep = NULL;
 617         }
 618         up_write(&qp->state_lock);
 619
 620         kfree(qp->tx_ctx.mpa_crc_hd);
 621         kfree(qp->rx_stream.mpa_crc_hd);
 622
 623         qp->scq = qp->rcq = NULL;
 624
 625         siw_qp_put(qp);
 626
 627         return 0;
 628 }
 629
 630 /*
 631  * siw_copy_inline_sgl()
 632  *
 633  * Prepare sgl of inlined data for sending. For userland callers
 634  * function checks if given buffer addresses and len's are within
 635  * process context bounds.
 636  * Data from all provided sge's are copied together into the wqe,
 637  * referenced by a single sge.
 638  */
 639 static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
 640                                struct siw_sqe *sqe)
 641 {
 642         struct ib_sge *core_sge = core_wr->sg_list;
 643         void *kbuf = &sqe->sge[1];
 644         int num_sge = core_wr->num_sge, bytes = 0;
 645
 646         sqe->sge[0].laddr = (uintptr_t)kbuf;
 647         sqe->sge[0].lkey = 0;
 648
 649         while (num_sge--) {
 650                 if (!core_sge->length) {
 651                         core_sge++;
 652                         continue;
 653                 }
 654                 bytes += core_sge->length;
 655                 if (bytes > SIW_MAX_INLINE) {
 656                         bytes = -EINVAL;
 657                         break;
 658                 }
 659                 memcpy(kbuf, (void *)(uintptr_t)core_sge->addr,
 660                        core_sge->length);
 661
 662                 kbuf += core_sge->length;
 663                 core_sge++;
 664         }
 665         sqe->sge[0].length = bytes > 0 ? bytes : 0;
 666         sqe->num_sge = bytes > 0 ? 1 : 0;
 667
 668         return bytes;
 669 }
 670
 671 /* Complete SQ WR's without processing */
 672 static int siw_sq_flush_wr(struct siw_qp *qp, const struct ib_send_wr *wr,
 673                            const struct ib_send_wr **bad_wr)
 674 {
 675         int rv = 0;
 676
 677         while (wr) {
 678                 struct siw_sqe sqe = {};
 679
 680                 switch (wr->opcode) {
 681                 case IB_WR_RDMA_WRITE:
 682                         sqe.opcode = SIW_OP_WRITE;
 683                         break;
 684                 case IB_WR_RDMA_READ:
 685                         sqe.opcode = SIW_OP_READ;
 686                         break;
 687                 case IB_WR_RDMA_READ_WITH_INV:
 688                         sqe.opcode = SIW_OP_READ_LOCAL_INV;
 689                         break;
 690                 case IB_WR_SEND:
 691                         sqe.opcode = SIW_OP_SEND;
 692                         break;
 693                 case IB_WR_SEND_WITH_IMM:
 694                         sqe.opcode = SIW_OP_SEND_WITH_IMM;
 695                         break;
 696                 case IB_WR_SEND_WITH_INV:
 697                         sqe.opcode = SIW_OP_SEND_REMOTE_INV;
 698                         break;
 699                 case IB_WR_LOCAL_INV:
 700                         sqe.opcode = SIW_OP_INVAL_STAG;
 701                         break;
 702                 case IB_WR_REG_MR:
 703                         sqe.opcode = SIW_OP_REG_MR;
 704                         break;
 705                 default:
 706                         rv = -EINVAL;
 707                         break;
 708                 }
 709                 if (!rv) {
 710                         sqe.id = wr->wr_id;
 711                         rv = siw_sqe_complete(qp, &sqe, 0,
 712                                               SIW_WC_WR_FLUSH_ERR);
 713                 }
 714                 if (rv) {
 715                         if (bad_wr)
 716                                 *bad_wr = wr;
 717                         break;
 718                 }
 719                 wr = wr->next;
 720         }
 721         return rv;
 722 }
 723
 724 /* Complete RQ WR's without processing */
 725 static int siw_rq_flush_wr(struct siw_qp *qp, const struct ib_recv_wr *wr,
 726                            const struct ib_recv_wr **bad_wr)
 727 {
 728         struct siw_rqe rqe = {};
 729         int rv = 0;
 730
 731         while (wr) {
 732                 rqe.id = wr->wr_id;
 733                 rv = siw_rqe_complete(qp, &rqe, 0, 0, SIW_WC_WR_FLUSH_ERR);
 734                 if (rv) {
 735                         if (bad_wr)
 736                                 *bad_wr = wr;
 737                         break;
 738                 }
 739                 wr = wr->next;
 740         }
 741         return rv;
 742 }
 743
 744 /*
 745  * siw_post_send()
 746  *
 747  * Post a list of S-WR's to a SQ.
 748  *
 749  * @base_qp:    Base QP contained in siw QP
 750  * @wr:         Null terminated list of user WR's
 751  * @bad_wr:     Points to failing WR in case of synchronous failure.
 752  */
 753 int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr,
 754                   const struct ib_send_wr **bad_wr)
 755 {
 756         struct siw_qp *qp = to_siw_qp(base_qp);
 757         struct siw_wqe *wqe = tx_wqe(qp);
 758
 759         unsigned long flags;
 760         int rv = 0;
 761
 762         if (wr && !rdma_is_kernel_res(&qp->base_qp.res)) {
 763                 siw_dbg_qp(qp, "wr must be empty for user mapped sq\n");
 764                 *bad_wr = wr;
 765                 return -EINVAL;
 766         }
 767
 768         /*
 769          * Try to acquire QP state lock. Must be non-blocking
 770          * to accommodate kernel clients needs.
 771          */
 772         if (!down_read_trylock(&qp->state_lock)) {
 773                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
 774                         /*
 775                          * ERROR state is final, so we can be sure
 776                          * this state will not change as long as the QP
 777                          * exists.
 778                          *
 779                          * This handles an ib_drain_sq() call with
 780                          * a concurrent request to set the QP state
 781                          * to ERROR.
 782                          */
 783                         rv = siw_sq_flush_wr(qp, wr, bad_wr);
 784                 } else {
 785                         siw_dbg_qp(qp, "QP locked, state %d\n",
 786                                    qp->attrs.state);
 787                         *bad_wr = wr;
 788                         rv = -ENOTCONN;
 789                 }
 790                 return rv;
 791         }
 792         if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) {
 793                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
 794                         /*
 795                          * Immediately flush this WR to CQ, if QP
 796                          * is in ERROR state. SQ is guaranteed to
 797                          * be empty, so WR complets in-order.
 798                          *
 799                          * Typically triggered by ib_drain_sq().
 800                          */
 801                         rv = siw_sq_flush_wr(qp, wr, bad_wr);
 802                 } else {
 803                         siw_dbg_qp(qp, "QP out of state %d\n",
 804                                    qp->attrs.state);
 805                         *bad_wr = wr;
 806                         rv = -ENOTCONN;
 807                 }
 808                 up_read(&qp->state_lock);
 809                 return rv;
 810         }
 811         spin_lock_irqsave(&qp->sq_lock, flags);
 812
 813         while (wr) {
 814                 u32 idx = qp->sq_put % qp->attrs.sq_size;
 815                 struct siw_sqe *sqe = &qp->sendq[idx];
 816
 817                 if (sqe->flags) {
 818                         siw_dbg_qp(qp, "sq full\n");
 819                         rv = -ENOMEM;
 820                         break;
 821                 }
 822                 if (wr->num_sge > qp->attrs.sq_max_sges) {
 823                         siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
 824                         rv = -EINVAL;
 825                         break;
 826                 }
 827                 sqe->id = wr->wr_id;
 828
 829                 if ((wr->send_flags & IB_SEND_SIGNALED) ||
 830                     (qp->attrs.flags & SIW_SIGNAL_ALL_WR))
 831                         sqe->flags |= SIW_WQE_SIGNALLED;
 832
 833                 if (wr->send_flags & IB_SEND_FENCE)
 834                         sqe->flags |= SIW_WQE_READ_FENCE;
 835
 836                 switch (wr->opcode) {
 837                 case IB_WR_SEND:
 838                 case IB_WR_SEND_WITH_INV:
 839                         if (wr->send_flags & IB_SEND_SOLICITED)
 840                                 sqe->flags |= SIW_WQE_SOLICITED;
 841
 842                         if (!(wr->send_flags & IB_SEND_INLINE)) {
 843                                 siw_copy_sgl(wr->sg_list, sqe->sge,
 844                                              wr->num_sge);
 845                                 sqe->num_sge = wr->num_sge;
 846                         } else {
 847                                 rv = siw_copy_inline_sgl(wr, sqe);
 848                                 if (rv <= 0) {
 849                                         rv = -EINVAL;
 850                                         break;
 851                                 }
 852                                 sqe->flags |= SIW_WQE_INLINE;
 853                                 sqe->num_sge = 1;
 854                         }
 855                         if (wr->opcode == IB_WR_SEND)
 856                                 sqe->opcode = SIW_OP_SEND;
 857                         else {
 858                                 sqe->opcode = SIW_OP_SEND_REMOTE_INV;
 859                                 sqe->rkey = wr->ex.invalidate_rkey;
 860                         }
 861                         break;
 862
 863                 case IB_WR_RDMA_READ_WITH_INV:
 864                 case IB_WR_RDMA_READ:
 865                         /*
 866                          * iWarp restricts RREAD sink to SGL containing
 867                          * 1 SGE only. we could relax to SGL with multiple
 868                          * elements referring the SAME ltag or even sending
 869                          * a private per-rreq tag referring to a checked
 870                          * local sgl with MULTIPLE ltag's.
 871                          */
 872                         if (unlikely(wr->num_sge != 1)) {
 873                                 rv = -EINVAL;
 874                                 break;
 875                         }
 876                         siw_copy_sgl(wr->sg_list, &sqe->sge[0], 1);
 877                         /*
 878                          * NOTE: zero length RREAD is allowed!
 879                          */
 880                         sqe->raddr = rdma_wr(wr)->remote_addr;
 881                         sqe->rkey = rdma_wr(wr)->rkey;
 882                         sqe->num_sge = 1;
 883
 884                         if (wr->opcode == IB_WR_RDMA_READ)
 885                                 sqe->opcode = SIW_OP_READ;
 886                         else
 887                                 sqe->opcode = SIW_OP_READ_LOCAL_INV;
 888                         break;
 889
 890                 case IB_WR_RDMA_WRITE:
 891                         if (!(wr->send_flags & IB_SEND_INLINE)) {
 892                                 siw_copy_sgl(wr->sg_list, &sqe->sge[0],
 893                                              wr->num_sge);
 894                                 sqe->num_sge = wr->num_sge;
 895                         } else {
 896                                 rv = siw_copy_inline_sgl(wr, sqe);
 897                                 if (unlikely(rv < 0)) {
 898                                         rv = -EINVAL;
 899                                         break;
 900                                 }
 901                                 sqe->flags |= SIW_WQE_INLINE;
 902                                 sqe->num_sge = 1;
 903                         }
 904                         sqe->raddr = rdma_wr(wr)->remote_addr;
 905                         sqe->rkey = rdma_wr(wr)->rkey;
 906                         sqe->opcode = SIW_OP_WRITE;
 907                         break;
 908
 909                 case IB_WR_REG_MR:
 910                         sqe->base_mr = (uintptr_t)reg_wr(wr)->mr;
 911                         sqe->rkey = reg_wr(wr)->key;
 912                         sqe->access = reg_wr(wr)->access & IWARP_ACCESS_MASK;
 913                         sqe->opcode = SIW_OP_REG_MR;
 914                         break;
 915
 916                 case IB_WR_LOCAL_INV:
 917                         sqe->rkey = wr->ex.invalidate_rkey;
 918                         sqe->opcode = SIW_OP_INVAL_STAG;
 919                         break;
 920
 921                 default:
 922                         siw_dbg_qp(qp, "ib wr type %d unsupported\n",
 923                                    wr->opcode);
 924                         rv = -EINVAL;
 925                         break;
 926                 }
 927                 siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%pK\n",
 928                            sqe->opcode, sqe->flags,
 929                            (void *)(uintptr_t)sqe->id);
 930
 931                 if (unlikely(rv < 0))
 932                         break;
 933
 934                 /* make SQE only valid after completely written */
 935                 smp_wmb();
 936                 sqe->flags |= SIW_WQE_VALID;
 937
 938                 qp->sq_put++;
 939                 wr = wr->next;
 940         }
 941
 942         /*
 943          * Send directly if SQ processing is not in progress.
 944          * Eventual immediate errors (rv < 0) do not affect the involved
 945          * RI resources (Verbs, 8.3.1) and thus do not prevent from SQ
 946          * processing, if new work is already pending. But rv must be passed
 947          * to caller.
 948          */
 949         if (wqe->wr_status != SIW_WR_IDLE) {
 950                 spin_unlock_irqrestore(&qp->sq_lock, flags);
 951                 goto skip_direct_sending;
 952         }
 953         rv = siw_activate_tx(qp);
 954         spin_unlock_irqrestore(&qp->sq_lock, flags);
 955
 956         if (rv <= 0)
 957                 goto skip_direct_sending;
 958
 959         if (rdma_is_kernel_res(&qp->base_qp.res)) {
 960                 rv = siw_sq_start(qp);
 961         } else {
 962                 qp->tx_ctx.in_syscall = 1;
 963
 964                 if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend))
 965                         siw_qp_cm_drop(qp, 0);
 966
 967                 qp->tx_ctx.in_syscall = 0;
 968         }
 969 skip_direct_sending:
 970
 971         up_read(&qp->state_lock);
 972
 973         if (rv >= 0)
 974                 return 0;
 975         /*
 976          * Immediate error
 977          */
 978         siw_dbg_qp(qp, "error %d\n", rv);
 979
 980         *bad_wr = wr;
 981         return rv;
 982 }
 983
 984 /*
 985  * siw_post_receive()
 986  *
 987  * Post a list of R-WR's to a RQ.
 988  *
 989  * @base_qp:    Base QP contained in siw QP
 990  * @wr:         Null terminated list of user WR's
 991  * @bad_wr:     Points to failing WR in case of synchronous failure.
 992  */
 993 int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr,
 994                      const struct ib_recv_wr **bad_wr)
 995 {
 996         struct siw_qp *qp = to_siw_qp(base_qp);
 997         unsigned long flags;
 998         int rv = 0;
 999
1000         if (qp->srq || qp->attrs.rq_size == 0) {
1001                 *bad_wr = wr;
1002                 return -EINVAL;
1003         }
1004         if (!rdma_is_kernel_res(&qp->base_qp.res)) {
1005                 siw_dbg_qp(qp, "no kernel post_recv for user mapped rq\n");
1006                 *bad_wr = wr;
1007                 return -EINVAL;
1008         }
1009
1010         /*
1011          * Try to acquire QP state lock. Must be non-blocking
1012          * to accommodate kernel clients needs.
1013          */
1014         if (!down_read_trylock(&qp->state_lock)) {
1015                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1016                         /*
1017                          * ERROR state is final, so we can be sure
1018                          * this state will not change as long as the QP
1019                          * exists.
1020                          *
1021                          * This handles an ib_drain_rq() call with
1022                          * a concurrent request to set the QP state
1023                          * to ERROR.
1024                          */
1025                         rv = siw_rq_flush_wr(qp, wr, bad_wr);
1026                 } else {
1027                         siw_dbg_qp(qp, "QP locked, state %d\n",
1028                                    qp->attrs.state);
1029                         *bad_wr = wr;
1030                         rv = -ENOTCONN;
1031                 }
1032                 return rv;
1033         }
1034         if (qp->attrs.state > SIW_QP_STATE_RTS) {
1035                 if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1036                         /*
1037                          * Immediately flush this WR to CQ, if QP
1038                          * is in ERROR state. RQ is guaranteed to
1039                          * be empty, so WR complets in-order.
1040                          *
1041                          * Typically triggered by ib_drain_rq().
1042                          */
1043                         rv = siw_rq_flush_wr(qp, wr, bad_wr);
1044                 } else {
1045                         siw_dbg_qp(qp, "QP out of state %d\n",
1046                                    qp->attrs.state);
1047                         *bad_wr = wr;
1048                         rv = -ENOTCONN;
1049                 }
1050                 up_read(&qp->state_lock);
1051                 return rv;
1052         }
1053         /*
1054          * Serialize potentially multiple producers.
1055          * Not needed for single threaded consumer side.
1056          */
1057         spin_lock_irqsave(&qp->rq_lock, flags);
1058
1059         while (wr) {
1060                 u32 idx = qp->rq_put % qp->attrs.rq_size;
1061                 struct siw_rqe *rqe = &qp->recvq[idx];
1062
1063                 if (rqe->flags) {
1064                         siw_dbg_qp(qp, "RQ full\n");
1065                         rv = -ENOMEM;
1066                         break;
1067                 }
1068                 if (wr->num_sge > qp->attrs.rq_max_sges) {
1069                         siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
1070                         rv = -EINVAL;
1071                         break;
1072                 }
1073                 rqe->id = wr->wr_id;
1074                 rqe->num_sge = wr->num_sge;
1075                 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1076
1077                 /* make sure RQE is completely written before valid */
1078                 smp_wmb();
1079
1080                 rqe->flags = SIW_WQE_VALID;
1081
1082                 qp->rq_put++;
1083                 wr = wr->next;
1084         }
1085         spin_unlock_irqrestore(&qp->rq_lock, flags);
1086
1087         up_read(&qp->state_lock);
1088
1089         if (rv < 0) {
1090                 siw_dbg_qp(qp, "error %d\n", rv);
1091                 *bad_wr = wr;
1092         }
1093         return rv > 0 ? 0 : rv;
1094 }
1095
1096 int siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata)
1097 {
1098         struct siw_cq *cq = to_siw_cq(base_cq);
1099         struct siw_device *sdev = to_siw_dev(base_cq->device);
1100         struct siw_ucontext *ctx =
1101                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1102                                           base_ucontext);
1103
1104         siw_dbg_cq(cq, "free CQ resources\n");
1105
1106         siw_cq_flush(cq);
1107
1108         if (ctx)
1109                 rdma_user_mmap_entry_remove(cq->cq_entry);
1110
1111         atomic_dec(&sdev->num_cq);
1112
1113         vfree(cq->queue);
1114         return 0;
1115 }
1116
1117 /*
1118  * siw_create_cq()
1119  *
1120  * Populate CQ of requested size
1121  *
1122  * @base_cq: CQ as allocated by RDMA midlayer
1123  * @attr: Initial CQ attributes
1124  * @udata: relates to user context
1125  */
1126
1127 int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr,
1128                   struct ib_udata *udata)
1129 {
1130         struct siw_device *sdev = to_siw_dev(base_cq->device);
1131         struct siw_cq *cq = to_siw_cq(base_cq);
1132         int rv, size = attr->cqe;
1133
1134         if (atomic_inc_return(&sdev->num_cq) > SIW_MAX_CQ) {
1135                 siw_dbg(base_cq->device, "too many CQ's\n");
1136                 rv = -ENOMEM;
1137                 goto err_out;
1138         }
1139         if (size < 1 || size > sdev->attrs.max_cqe) {
1140                 siw_dbg(base_cq->device, "CQ size error: %d\n", size);
1141                 rv = -EINVAL;
1142                 goto err_out;
1143         }
1144         size = roundup_pow_of_two(size);
1145         cq->base_cq.cqe = size;
1146         cq->num_cqe = size;
1147
1148         if (udata)
1149                 cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) +
1150                                          sizeof(struct siw_cq_ctrl));
1151         else
1152                 cq->queue = vzalloc(size * sizeof(struct siw_cqe) +
1153                                     sizeof(struct siw_cq_ctrl));
1154
1155         if (cq->queue == NULL) {
1156                 rv = -ENOMEM;
1157                 goto err_out;
1158         }
1159         get_random_bytes(&cq->id, 4);
1160         siw_dbg(base_cq->device, "new CQ [%u]\n", cq->id);
1161
1162         spin_lock_init(&cq->lock);
1163
1164         cq->notify = (struct siw_cq_ctrl *)&cq->queue[size];
1165
1166         if (udata) {
1167                 struct siw_uresp_create_cq uresp = {};
1168                 struct siw_ucontext *ctx =
1169                         rdma_udata_to_drv_context(udata, struct siw_ucontext,
1170                                                   base_ucontext);
1171                 size_t length = size * sizeof(struct siw_cqe) +
1172                         sizeof(struct siw_cq_ctrl);
1173
1174                 cq->cq_entry =
1175                         siw_mmap_entry_insert(ctx, cq->queue,
1176                                               length, &uresp.cq_key);
1177                 if (!cq->cq_entry) {
1178                         rv = -ENOMEM;
1179                         goto err_out;
1180                 }
1181
1182                 uresp.cq_id = cq->id;
1183                 uresp.num_cqe = size;
1184
1185                 if (udata->outlen < sizeof(uresp)) {
1186                         rv = -EINVAL;
1187                         goto err_out;
1188                 }
1189                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1190                 if (rv)
1191                         goto err_out;
1192         }
1193         return 0;
1194
1195 err_out:
1196         siw_dbg(base_cq->device, "CQ creation failed: %d", rv);
1197
1198         if (cq && cq->queue) {
1199                 struct siw_ucontext *ctx =
1200                         rdma_udata_to_drv_context(udata, struct siw_ucontext,
1201                                                   base_ucontext);
1202                 if (ctx)
1203                         rdma_user_mmap_entry_remove(cq->cq_entry);
1204                 vfree(cq->queue);
1205         }
1206         atomic_dec(&sdev->num_cq);
1207
1208         return rv;
1209 }
1210
1211 /*
1212  * siw_poll_cq()
1213  *
1214  * Reap CQ entries if available and copy work completion status into
1215  * array of WC's provided by caller. Returns number of reaped CQE's.
1216  *
1217  * @base_cq:    Base CQ contained in siw CQ.
1218  * @num_cqe:    Maximum number of CQE's to reap.
1219  * @wc:         Array of work completions to be filled by siw.
1220  */
1221 int siw_poll_cq(struct ib_cq *base_cq, int num_cqe, struct ib_wc *wc)
1222 {
1223         struct siw_cq *cq = to_siw_cq(base_cq);
1224         int i;
1225
1226         for (i = 0; i < num_cqe; i++) {
1227                 if (!siw_reap_cqe(cq, wc))
1228                         break;
1229                 wc++;
1230         }
1231         return i;
1232 }
1233
1234 /*
1235  * siw_req_notify_cq()
1236  *
1237  * Request notification for new CQE's added to that CQ.
1238  * Defined flags:
1239  * o SIW_CQ_NOTIFY_SOLICITED lets siw trigger a notification
1240  *   event if a WQE with notification flag set enters the CQ
1241  * o SIW_CQ_NOTIFY_NEXT_COMP lets siw trigger a notification
1242  *   event if a WQE enters the CQ.
1243  * o IB_CQ_REPORT_MISSED_EVENTS: return value will provide the
1244  *   number of not reaped CQE's regardless of its notification
1245  *   type and current or new CQ notification settings.
1246  *
1247  * @base_cq:    Base CQ contained in siw CQ.
1248  * @flags:      Requested notification flags.
1249  */
1250 int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags)
1251 {
1252         struct siw_cq *cq = to_siw_cq(base_cq);
1253
1254         siw_dbg_cq(cq, "flags: 0x%02x\n", flags);
1255
1256         if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
1257                 /*
1258                  * Enable CQ event for next solicited completion.
1259                  * and make it visible to all associated producers.
1260                  */
1261                 smp_store_mb(cq->notify->flags, SIW_NOTIFY_SOLICITED);
1262         else
1263                 /*
1264                  * Enable CQ event for any signalled completion.
1265                  * and make it visible to all associated producers.
1266                  */
1267                 smp_store_mb(cq->notify->flags, SIW_NOTIFY_ALL);
1268
1269         if (flags & IB_CQ_REPORT_MISSED_EVENTS)
1270                 return cq->cq_put - cq->cq_get;
1271
1272         return 0;
1273 }
1274
1275 /*
1276  * siw_dereg_mr()
1277  *
1278  * Release Memory Region.
1279  *
1280  * @base_mr: Base MR contained in siw MR.
1281  * @udata: points to user context, unused.
1282  */
1283 int siw_dereg_mr(struct ib_mr *base_mr, struct ib_udata *udata)
1284 {
1285         struct siw_mr *mr = to_siw_mr(base_mr);
1286         struct siw_device *sdev = to_siw_dev(base_mr->device);
1287
1288         siw_dbg_mem(mr->mem, "deregister MR\n");
1289
1290         atomic_dec(&sdev->num_mr);
1291
1292         siw_mr_drop_mem(mr);
1293         kfree_rcu(mr, rcu);
1294
1295         return 0;
1296 }
1297
1298 /*
1299  * siw_reg_user_mr()
1300  *
1301  * Register Memory Region.
1302  *
1303  * @pd:         Protection Domain
1304  * @start:      starting address of MR (virtual address)
1305  * @len:        len of MR
1306  * @rnic_va:    not used by siw
1307  * @rights:     MR access rights
1308  * @udata:      user buffer to communicate STag and Key.
1309  */
1310 struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1311                               u64 rnic_va, int rights, struct ib_udata *udata)
1312 {
1313         struct siw_mr *mr = NULL;
1314         struct siw_umem *umem = NULL;
1315         struct siw_ureq_reg_mr ureq;
1316         struct siw_device *sdev = to_siw_dev(pd->device);
1317
1318         unsigned long mem_limit = rlimit(RLIMIT_MEMLOCK);
1319         int rv;
1320
1321         siw_dbg_pd(pd, "start: 0x%pK, va: 0x%pK, len: %llu\n",
1322                    (void *)(uintptr_t)start, (void *)(uintptr_t)rnic_va,
1323                    (unsigned long long)len);
1324
1325         if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1326                 siw_dbg_pd(pd, "too many mr's\n");
1327                 rv = -ENOMEM;
1328                 goto err_out;
1329         }
1330         if (!len) {
1331                 rv = -EINVAL;
1332                 goto err_out;
1333         }
1334         if (mem_limit != RLIM_INFINITY) {
1335                 unsigned long num_pages =
1336                         (PAGE_ALIGN(len + (start & ~PAGE_MASK))) >> PAGE_SHIFT;
1337                 mem_limit >>= PAGE_SHIFT;
1338
1339                 if (num_pages > mem_limit - current->mm->locked_vm) {
1340                         siw_dbg_pd(pd, "pages req %lu, max %lu, lock %lu\n",
1341                                    num_pages, mem_limit,
1342                                    current->mm->locked_vm);
1343                         rv = -ENOMEM;
1344                         goto err_out;
1345                 }
1346         }
1347         umem = siw_umem_get(start, len, ib_access_writable(rights));
1348         if (IS_ERR(umem)) {
1349                 rv = PTR_ERR(umem);
1350                 siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
1351                 umem = NULL;
1352                 goto err_out;
1353         }
1354         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1355         if (!mr) {
1356                 rv = -ENOMEM;
1357                 goto err_out;
1358         }
1359         rv = siw_mr_add_mem(mr, pd, umem, start, len, rights);
1360         if (rv)
1361                 goto err_out;
1362
1363         if (udata) {
1364                 struct siw_uresp_reg_mr uresp = {};
1365                 struct siw_mem *mem = mr->mem;
1366
1367                 if (udata->inlen < sizeof(ureq)) {
1368                         rv = -EINVAL;
1369                         goto err_out;
1370                 }
1371                 rv = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1372                 if (rv)
1373                         goto err_out;
1374
1375                 mr->base_mr.lkey |= ureq.stag_key;
1376                 mr->base_mr.rkey |= ureq.stag_key;
1377                 mem->stag |= ureq.stag_key;
1378                 uresp.stag = mem->stag;
1379
1380                 if (udata->outlen < sizeof(uresp)) {
1381                         rv = -EINVAL;
1382                         goto err_out;
1383                 }
1384                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1385                 if (rv)
1386                         goto err_out;
1387         }
1388         mr->mem->stag_valid = 1;
1389
1390         return &mr->base_mr;
1391
1392 err_out:
1393         atomic_dec(&sdev->num_mr);
1394         if (mr) {
1395                 if (mr->mem)
1396                         siw_mr_drop_mem(mr);
1397                 kfree_rcu(mr, rcu);
1398         } else {
1399                 if (umem)
1400                         siw_umem_release(umem, false);
1401         }
1402         return ERR_PTR(rv);
1403 }
1404
1405 struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1406                            u32 max_sge)
1407 {
1408         struct siw_device *sdev = to_siw_dev(pd->device);
1409         struct siw_mr *mr = NULL;
1410         struct siw_pbl *pbl = NULL;
1411         int rv;
1412
1413         if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1414                 siw_dbg_pd(pd, "too many mr's\n");
1415                 rv = -ENOMEM;
1416                 goto err_out;
1417         }
1418         if (mr_type != IB_MR_TYPE_MEM_REG) {
1419                 siw_dbg_pd(pd, "mr type %d unsupported\n", mr_type);
1420                 rv = -EOPNOTSUPP;
1421                 goto err_out;
1422         }
1423         if (max_sge > SIW_MAX_SGE_PBL) {
1424                 siw_dbg_pd(pd, "too many sge's: %d\n", max_sge);
1425                 rv = -ENOMEM;
1426                 goto err_out;
1427         }
1428         pbl = siw_pbl_alloc(max_sge);
1429         if (IS_ERR(pbl)) {
1430                 rv = PTR_ERR(pbl);
1431                 siw_dbg_pd(pd, "pbl allocation failed: %d\n", rv);
1432                 pbl = NULL;
1433                 goto err_out;
1434         }
1435         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1436         if (!mr) {
1437                 rv = -ENOMEM;
1438                 goto err_out;
1439         }
1440         rv = siw_mr_add_mem(mr, pd, pbl, 0, max_sge * PAGE_SIZE, 0);
1441         if (rv)
1442                 goto err_out;
1443
1444         mr->mem->is_pbl = 1;
1445
1446         siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1447
1448         return &mr->base_mr;
1449
1450 err_out:
1451         atomic_dec(&sdev->num_mr);
1452
1453         if (!mr) {
1454                 kfree(pbl);
1455         } else {
1456                 if (mr->mem)
1457                         siw_mr_drop_mem(mr);
1458                 kfree_rcu(mr, rcu);
1459         }
1460         siw_dbg_pd(pd, "failed: %d\n", rv);
1461
1462         return ERR_PTR(rv);
1463 }
1464
1465 /* Just used to count number of pages being mapped */
1466 static int siw_set_pbl_page(struct ib_mr *base_mr, u64 buf_addr)
1467 {
1468         return 0;
1469 }
1470
1471 int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
1472                   unsigned int *sg_off)
1473 {
1474         struct scatterlist *slp;
1475         struct siw_mr *mr = to_siw_mr(base_mr);
1476         struct siw_mem *mem = mr->mem;
1477         struct siw_pbl *pbl = mem->pbl;
1478         struct siw_pble *pble;
1479         unsigned long pbl_size;
1480         int i, rv;
1481
1482         if (!pbl) {
1483                 siw_dbg_mem(mem, "no PBL allocated\n");
1484                 return -EINVAL;
1485         }
1486         pble = pbl->pbe;
1487
1488         if (pbl->max_buf < num_sle) {
1489                 siw_dbg_mem(mem, "too many SGE's: %d > %d\n",
1490                             num_sle, pbl->max_buf);
1491                 return -ENOMEM;
1492         }
1493         for_each_sg(sl, slp, num_sle, i) {
1494                 if (sg_dma_len(slp) == 0) {
1495                         siw_dbg_mem(mem, "empty SGE\n");
1496                         return -EINVAL;
1497                 }
1498                 if (i == 0) {
1499                         pble->addr = sg_dma_address(slp);
1500                         pble->size = sg_dma_len(slp);
1501                         pble->pbl_off = 0;
1502                         pbl_size = pble->size;
1503                         pbl->num_buf = 1;
1504                 } else {
1505                         /* Merge PBL entries if adjacent */
1506                         if (pble->addr + pble->size == sg_dma_address(slp)) {
1507                                 pble->size += sg_dma_len(slp);
1508                         } else {
1509                                 pble++;
1510                                 pbl->num_buf++;
1511                                 pble->addr = sg_dma_address(slp);
1512                                 pble->size = sg_dma_len(slp);
1513                                 pble->pbl_off = pbl_size;
1514                         }
1515                         pbl_size += sg_dma_len(slp);
1516                 }
1517                 siw_dbg_mem(mem,
1518                         "sge[%d], size %u, addr 0x%p, total %lu\n",
1519                         i, pble->size, (void *)(uintptr_t)pble->addr,
1520                         pbl_size);
1521         }
1522         rv = ib_sg_to_pages(base_mr, sl, num_sle, sg_off, siw_set_pbl_page);
1523         if (rv > 0) {
1524                 mem->len = base_mr->length;
1525                 mem->va = base_mr->iova;
1526                 siw_dbg_mem(mem,
1527                         "%llu bytes, start 0x%pK, %u SLE to %u entries\n",
1528                         mem->len, (void *)(uintptr_t)mem->va, num_sle,
1529                         pbl->num_buf);
1530         }
1531         return rv;
1532 }
1533
1534 /*
1535  * siw_get_dma_mr()
1536  *
1537  * Create a (empty) DMA memory region, where no umem is attached.
1538  */
1539 struct ib_mr *siw_get_dma_mr(struct ib_pd *pd, int rights)
1540 {
1541         struct siw_device *sdev = to_siw_dev(pd->device);
1542         struct siw_mr *mr = NULL;
1543         int rv;
1544
1545         if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1546                 siw_dbg_pd(pd, "too many mr's\n");
1547                 rv = -ENOMEM;
1548                 goto err_out;
1549         }
1550         mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1551         if (!mr) {
1552                 rv = -ENOMEM;
1553                 goto err_out;
1554         }
1555         rv = siw_mr_add_mem(mr, pd, NULL, 0, ULONG_MAX, rights);
1556         if (rv)
1557                 goto err_out;
1558
1559         mr->mem->stag_valid = 1;
1560
1561         siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1562
1563         return &mr->base_mr;
1564
1565 err_out:
1566         if (rv)
1567                 kfree(mr);
1568
1569         atomic_dec(&sdev->num_mr);
1570
1571         return ERR_PTR(rv);
1572 }
1573
1574 /*
1575  * siw_create_srq()
1576  *
1577  * Create Shared Receive Queue of attributes @init_attrs
1578  * within protection domain given by @pd.
1579  *
1580  * @base_srq:   Base SRQ contained in siw SRQ.
1581  * @init_attrs: SRQ init attributes.
1582  * @udata:      points to user context
1583  */
1584 int siw_create_srq(struct ib_srq *base_srq,
1585                    struct ib_srq_init_attr *init_attrs, struct ib_udata *udata)
1586 {
1587         struct siw_srq *srq = to_siw_srq(base_srq);
1588         struct ib_srq_attr *attrs = &init_attrs->attr;
1589         struct siw_device *sdev = to_siw_dev(base_srq->device);
1590         struct siw_ucontext *ctx =
1591                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1592                                           base_ucontext);
1593         int rv;
1594
1595         if (atomic_inc_return(&sdev->num_srq) > SIW_MAX_SRQ) {
1596                 siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
1597                 rv = -ENOMEM;
1598                 goto err_out;
1599         }
1600         if (attrs->max_wr == 0 || attrs->max_wr > SIW_MAX_SRQ_WR ||
1601             attrs->max_sge > SIW_MAX_SGE || attrs->srq_limit > attrs->max_wr) {
1602                 rv = -EINVAL;
1603                 goto err_out;
1604         }
1605         srq->max_sge = attrs->max_sge;
1606         srq->num_rqe = roundup_pow_of_two(attrs->max_wr);
1607         srq->limit = attrs->srq_limit;
1608         if (srq->limit)
1609                 srq->armed = true;
1610
1611         srq->is_kernel_res = !udata;
1612
1613         if (udata)
1614                 srq->recvq =
1615                         vmalloc_user(srq->num_rqe * sizeof(struct siw_rqe));
1616         else
1617                 srq->recvq = vzalloc(srq->num_rqe * sizeof(struct siw_rqe));
1618
1619         if (srq->recvq == NULL) {
1620                 rv = -ENOMEM;
1621                 goto err_out;
1622         }
1623         if (udata) {
1624                 struct siw_uresp_create_srq uresp = {};
1625                 size_t length = srq->num_rqe * sizeof(struct siw_rqe);
1626
1627                 srq->srq_entry =
1628                         siw_mmap_entry_insert(ctx, srq->recvq,
1629                                               length, &uresp.srq_key);
1630                 if (!srq->srq_entry) {
1631                         rv = -ENOMEM;
1632                         goto err_out;
1633                 }
1634
1635                 uresp.num_rqe = srq->num_rqe;
1636
1637                 if (udata->outlen < sizeof(uresp)) {
1638                         rv = -EINVAL;
1639                         goto err_out;
1640                 }
1641                 rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1642                 if (rv)
1643                         goto err_out;
1644         }
1645         spin_lock_init(&srq->lock);
1646
1647         siw_dbg_pd(base_srq->pd, "[SRQ]: success\n");
1648
1649         return 0;
1650
1651 err_out:
1652         if (srq->recvq) {
1653                 if (ctx)
1654                         rdma_user_mmap_entry_remove(srq->srq_entry);
1655                 vfree(srq->recvq);
1656         }
1657         atomic_dec(&sdev->num_srq);
1658
1659         return rv;
1660 }
1661
1662 /*
1663  * siw_modify_srq()
1664  *
1665  * Modify SRQ. The caller may resize SRQ and/or set/reset notification
1666  * limit and (re)arm IB_EVENT_SRQ_LIMIT_REACHED notification.
1667  *
1668  * NOTE: it is unclear if RDMA core allows for changing the MAX_SGE
1669  * parameter. siw_modify_srq() does not check the attrs->max_sge param.
1670  */
1671 int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs,
1672                    enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1673 {
1674         struct siw_srq *srq = to_siw_srq(base_srq);
1675         unsigned long flags;
1676         int rv = 0;
1677
1678         spin_lock_irqsave(&srq->lock, flags);
1679
1680         if (attr_mask & IB_SRQ_MAX_WR) {
1681                 /* resize request not yet supported */
1682                 rv = -EOPNOTSUPP;
1683                 goto out;
1684         }
1685         if (attr_mask & IB_SRQ_LIMIT) {
1686                 if (attrs->srq_limit) {
1687                         if (unlikely(attrs->srq_limit > srq->num_rqe)) {
1688                                 rv = -EINVAL;
1689                                 goto out;
1690                         }
1691                         srq->armed = true;
1692                 } else {
1693                         srq->armed = false;
1694                 }
1695                 srq->limit = attrs->srq_limit;
1696         }
1697 out:
1698         spin_unlock_irqrestore(&srq->lock, flags);
1699
1700         return rv;
1701 }
1702
1703 /*
1704  * siw_query_srq()
1705  *
1706  * Query SRQ attributes.
1707  */
1708 int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs)
1709 {
1710         struct siw_srq *srq = to_siw_srq(base_srq);
1711         unsigned long flags;
1712
1713         spin_lock_irqsave(&srq->lock, flags);
1714
1715         attrs->max_wr = srq->num_rqe;
1716         attrs->max_sge = srq->max_sge;
1717         attrs->srq_limit = srq->limit;
1718
1719         spin_unlock_irqrestore(&srq->lock, flags);
1720
1721         return 0;
1722 }
1723
1724 /*
1725  * siw_destroy_srq()
1726  *
1727  * Destroy SRQ.
1728  * It is assumed that the SRQ is not referenced by any
1729  * QP anymore - the code trusts the RDMA core environment to keep track
1730  * of QP references.
1731  */
1732 int siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata)
1733 {
1734         struct siw_srq *srq = to_siw_srq(base_srq);
1735         struct siw_device *sdev = to_siw_dev(base_srq->device);
1736         struct siw_ucontext *ctx =
1737                 rdma_udata_to_drv_context(udata, struct siw_ucontext,
1738                                           base_ucontext);
1739
1740         if (ctx)
1741                 rdma_user_mmap_entry_remove(srq->srq_entry);
1742         vfree(srq->recvq);
1743         atomic_dec(&sdev->num_srq);
1744         return 0;
1745 }
1746
1747 /*
1748  * siw_post_srq_recv()
1749  *
1750  * Post a list of receive queue elements to SRQ.
1751  * NOTE: The function does not check or lock a certain SRQ state
1752  *       during the post operation. The code simply trusts the
1753  *       RDMA core environment.
1754  *
1755  * @base_srq:   Base SRQ contained in siw SRQ
1756  * @wr:         List of R-WR's
1757  * @bad_wr:     Updated to failing WR if posting fails.
1758  */
1759 int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr,
1760                       const struct ib_recv_wr **bad_wr)
1761 {
1762         struct siw_srq *srq = to_siw_srq(base_srq);
1763         unsigned long flags;
1764         int rv = 0;
1765
1766         if (unlikely(!srq->is_kernel_res)) {
1767                 siw_dbg_pd(base_srq->pd,
1768                            "[SRQ]: no kernel post_recv for mapped srq\n");
1769                 rv = -EINVAL;
1770                 goto out;
1771         }
1772         /*
1773          * Serialize potentially multiple producers.
1774          * Also needed to serialize potentially multiple
1775          * consumers.
1776          */
1777         spin_lock_irqsave(&srq->lock, flags);
1778
1779         while (wr) {
1780                 u32 idx = srq->rq_put % srq->num_rqe;
1781                 struct siw_rqe *rqe = &srq->recvq[idx];
1782
1783                 if (rqe->flags) {
1784                         siw_dbg_pd(base_srq->pd, "SRQ full\n");
1785                         rv = -ENOMEM;
1786                         break;
1787                 }
1788                 if (unlikely(wr->num_sge > srq->max_sge)) {
1789                         siw_dbg_pd(base_srq->pd,
1790                                    "[SRQ]: too many sge's: %d\n", wr->num_sge);
1791                         rv = -EINVAL;
1792                         break;
1793                 }
1794                 rqe->id = wr->wr_id;
1795                 rqe->num_sge = wr->num_sge;
1796                 siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1797
1798                 /* Make sure S-RQE is completely written before valid */
1799                 smp_wmb();
1800
1801                 rqe->flags = SIW_WQE_VALID;
1802
1803                 srq->rq_put++;
1804                 wr = wr->next;
1805         }
1806         spin_unlock_irqrestore(&srq->lock, flags);
1807 out:
1808         if (unlikely(rv < 0)) {
1809                 siw_dbg_pd(base_srq->pd, "[SRQ]: error %d\n", rv);
1810                 *bad_wr = wr;
1811         }
1812         return rv;
1813 }
1814
1815 void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype)
1816 {
1817         struct ib_event event;
1818         struct ib_qp *base_qp = &qp->base_qp;
1819
1820         /*
1821          * Do not report asynchronous errors on QP which gets
1822          * destroyed via verbs interface (siw_destroy_qp())
1823          */
1824         if (qp->attrs.flags & SIW_QP_IN_DESTROY)
1825                 return;
1826
1827         event.event = etype;
1828         event.device = base_qp->device;
1829         event.element.qp = base_qp;
1830
1831         if (base_qp->event_handler) {
1832                 siw_dbg_qp(qp, "reporting event %d\n", etype);
1833                 base_qp->event_handler(&event, base_qp->qp_context);
1834         }
1835 }
1836
1837 void siw_cq_event(struct siw_cq *cq, enum ib_event_type etype)
1838 {
1839         struct ib_event event;
1840         struct ib_cq *base_cq = &cq->base_cq;
1841
1842         event.event = etype;
1843         event.device = base_cq->device;
1844         event.element.cq = base_cq;
1845
1846         if (base_cq->event_handler) {
1847                 siw_dbg_cq(cq, "reporting CQ event %d\n", etype);
1848                 base_cq->event_handler(&event, base_cq->cq_context);
1849         }
1850 }
1851
1852 void siw_srq_event(struct siw_srq *srq, enum ib_event_type etype)
1853 {
1854         struct ib_event event;
1855         struct ib_srq *base_srq = &srq->base_srq;
1856
1857         event.event = etype;
1858         event.device = base_srq->device;
1859         event.element.srq = base_srq;
1860
1861         if (base_srq->event_handler) {
1862                 siw_dbg_pd(srq->base_srq.pd,
1863                            "reporting SRQ event %d\n", etype);
1864                 base_srq->event_handler(&event, base_srq->srq_context);
1865         }
1866 }
1867
1868 void siw_port_event(struct siw_device *sdev, u8 port, enum ib_event_type etype)
1869 {
1870         struct ib_event event;
1871
1872         event.event = etype;
1873         event.device = &sdev->base_dev;
1874         event.element.port_num = port;
1875
1876         siw_dbg(&sdev->base_dev, "reporting port event %d\n", etype);
1877
1878         ib_dispatch_event(&event);
1879 }