drivers/infiniband/sw/rdmavt/srq.c

   1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
   2 /*
   3  * Copyright(c) 2016 Intel Corporation.
   4  */
   5
   6 #include <linux/err.h>
   7 #include <linux/slab.h>
   8 #include <linux/vmalloc.h>
   9 #include <rdma/uverbs_ioctl.h>
  10
  11 #include "srq.h"
  12 #include "vt.h"
  13 #include "qp.h"
  14 /**
  15  * rvt_driver_srq_init - init srq resources on a per driver basis
  16  * @rdi: rvt dev structure
  17  *
  18  * Do any initialization needed when a driver registers with rdmavt.
  19  */
  20 void rvt_driver_srq_init(struct rvt_dev_info *rdi)
  21 {
  22         spin_lock_init(&rdi->n_srqs_lock);
  23         rdi->n_srqs_allocated = 0;
  24 }
  25
  26 /**
  27  * rvt_create_srq - create a shared receive queue
  28  * @ibsrq: the protection domain of the SRQ to create
  29  * @srq_init_attr: the attributes of the SRQ
  30  * @udata: data from libibverbs when creating a user SRQ
  31  *
  32  * Return: 0 on success
  33  */
  34 int rvt_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *srq_init_attr,
  35                    struct ib_udata *udata)
  36 {
  37         struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
  38         struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
  39         u32 sz;
  40         int ret;
  41
  42         if (srq_init_attr->srq_type != IB_SRQT_BASIC)
  43                 return -EOPNOTSUPP;
  44
  45         if (srq_init_attr->attr.max_sge == 0 ||
  46             srq_init_attr->attr.max_sge > dev->dparms.props.max_srq_sge ||
  47             srq_init_attr->attr.max_wr == 0 ||
  48             srq_init_attr->attr.max_wr > dev->dparms.props.max_srq_wr)
  49                 return -EINVAL;
  50
  51         /*
  52          * Need to use vmalloc() if we want to support large #s of entries.
  53          */
  54         srq->rq.size = srq_init_attr->attr.max_wr + 1;
  55         srq->rq.max_sge = srq_init_attr->attr.max_sge;
  56         sz = sizeof(struct ib_sge) * srq->rq.max_sge +
  57                 sizeof(struct rvt_rwqe);
  58         if (rvt_alloc_rq(&srq->rq, srq->rq.size * sz,
  59                          dev->dparms.node, udata)) {
  60                 ret = -ENOMEM;
  61                 goto bail_srq;
  62         }
  63
  64         /*
  65          * Return the address of the RWQ as the offset to mmap.
  66          * See rvt_mmap() for details.
  67          */
  68         if (udata && udata->outlen >= sizeof(__u64)) {
  69                 u32 s = sizeof(struct rvt_rwq) + srq->rq.size * sz;
  70
  71                 srq->ip = rvt_create_mmap_info(dev, s, udata, srq->rq.wq);
  72                 if (IS_ERR(srq->ip)) {
  73                         ret = PTR_ERR(srq->ip);
  74                         goto bail_wq;
  75                 }
  76
  77                 ret = ib_copy_to_udata(udata, &srq->ip->offset,
  78                                        sizeof(srq->ip->offset));
  79                 if (ret)
  80                         goto bail_ip;
  81         }
  82
  83         /*
  84          * ib_create_srq() will initialize srq->ibsrq.
  85          */
  86         spin_lock_init(&srq->rq.lock);
  87         srq->limit = srq_init_attr->attr.srq_limit;
  88
  89         spin_lock(&dev->n_srqs_lock);
  90         if (dev->n_srqs_allocated == dev->dparms.props.max_srq) {
  91                 spin_unlock(&dev->n_srqs_lock);
  92                 ret = -ENOMEM;
  93                 goto bail_ip;
  94         }
  95
  96         dev->n_srqs_allocated++;
  97         spin_unlock(&dev->n_srqs_lock);
  98
  99         if (srq->ip) {
 100                 spin_lock_irq(&dev->pending_lock);
 101                 list_add(&srq->ip->pending_mmaps, &dev->pending_mmaps);
 102                 spin_unlock_irq(&dev->pending_lock);
 103         }
 104
 105         return 0;
 106
 107 bail_ip:
 108         kfree(srq->ip);
 109 bail_wq:
 110         rvt_free_rq(&srq->rq);
 111 bail_srq:
 112         return ret;
 113 }
 114
 115 /**
 116  * rvt_modify_srq - modify a shared receive queue
 117  * @ibsrq: the SRQ to modify
 118  * @attr: the new attributes of the SRQ
 119  * @attr_mask: indicates which attributes to modify
 120  * @udata: user data for libibverbs.so
 121  *
 122  * Return: 0 on success
 123  */
 124 int rvt_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
 125                    enum ib_srq_attr_mask attr_mask,
 126                    struct ib_udata *udata)
 127 {
 128         struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
 129         struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
 130         struct rvt_rq tmp_rq = {};
 131         int ret = 0;
 132
 133         if (attr_mask & IB_SRQ_MAX_WR) {
 134                 struct rvt_krwq *okwq = NULL;
 135                 struct rvt_rwq *owq = NULL;
 136                 struct rvt_rwqe *p;
 137                 u32 sz, size, n, head, tail;
 138
 139                 /* Check that the requested sizes are below the limits. */
 140                 if ((attr->max_wr > dev->dparms.props.max_srq_wr) ||
 141                     ((attr_mask & IB_SRQ_LIMIT) ?
 142                      attr->srq_limit : srq->limit) > attr->max_wr)
 143                         return -EINVAL;
 144                 sz = sizeof(struct rvt_rwqe) +
 145                         srq->rq.max_sge * sizeof(struct ib_sge);
 146                 size = attr->max_wr + 1;
 147                 if (rvt_alloc_rq(&tmp_rq, size * sz, dev->dparms.node,
 148                                  udata))
 149                         return -ENOMEM;
 150                 /* Check that we can write the offset to mmap. */
 151                 if (udata && udata->inlen >= sizeof(__u64)) {
 152                         __u64 offset_addr;
 153                         __u64 offset = 0;
 154
 155                         ret = ib_copy_from_udata(&offset_addr, udata,
 156                                                  sizeof(offset_addr));
 157                         if (ret)
 158                                 goto bail_free;
 159                         udata->outbuf = (void __user *)
 160                                         (unsigned long)offset_addr;
 161                         ret = ib_copy_to_udata(udata, &offset,
 162                                                sizeof(offset));
 163                         if (ret)
 164                                 goto bail_free;
 165                 }
 166
 167                 spin_lock_irq(&srq->rq.kwq->c_lock);
 168                 /*
 169                  * validate head and tail pointer values and compute
 170                  * the number of remaining WQEs.
 171                  */
 172                 if (udata) {
 173                         owq = srq->rq.wq;
 174                         head = RDMA_READ_UAPI_ATOMIC(owq->head);
 175                         tail = RDMA_READ_UAPI_ATOMIC(owq->tail);
 176                 } else {
 177                         okwq = srq->rq.kwq;
 178                         head = okwq->head;
 179                         tail = okwq->tail;
 180                 }
 181                 if (head >= srq->rq.size || tail >= srq->rq.size) {
 182                         ret = -EINVAL;
 183                         goto bail_unlock;
 184                 }
 185                 n = head;
 186                 if (n < tail)
 187                         n += srq->rq.size - tail;
 188                 else
 189                         n -= tail;
 190                 if (size <= n) {
 191                         ret = -EINVAL;
 192                         goto bail_unlock;
 193                 }
 194                 n = 0;
 195                 p = tmp_rq.kwq->curr_wq;
 196                 while (tail != head) {
 197                         struct rvt_rwqe *wqe;
 198                         int i;
 199
 200                         wqe = rvt_get_rwqe_ptr(&srq->rq, tail);
 201                         p->wr_id = wqe->wr_id;
 202                         p->num_sge = wqe->num_sge;
 203                         for (i = 0; i < wqe->num_sge; i++)
 204                                 p->sg_list[i] = wqe->sg_list[i];
 205                         n++;
 206                         p = (struct rvt_rwqe *)((char *)p + sz);
 207                         if (++tail >= srq->rq.size)
 208                                 tail = 0;
 209                 }
 210                 srq->rq.kwq = tmp_rq.kwq;
 211                 if (udata) {
 212                         srq->rq.wq = tmp_rq.wq;
 213                         RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->head, n);
 214                         RDMA_WRITE_UAPI_ATOMIC(tmp_rq.wq->tail, 0);
 215                 } else {
 216                         tmp_rq.kwq->head = n;
 217                         tmp_rq.kwq->tail = 0;
 218                 }
 219                 srq->rq.size = size;
 220                 if (attr_mask & IB_SRQ_LIMIT)
 221                         srq->limit = attr->srq_limit;
 222                 spin_unlock_irq(&srq->rq.kwq->c_lock);
 223
 224                 vfree(owq);
 225                 kvfree(okwq);
 226
 227                 if (srq->ip) {
 228                         struct rvt_mmap_info *ip = srq->ip;
 229                         struct rvt_dev_info *dev = ib_to_rvt(srq->ibsrq.device);
 230                         u32 s = sizeof(struct rvt_rwq) + size * sz;
 231
 232                         rvt_update_mmap_info(dev, ip, s, tmp_rq.wq);
 233
 234                         /*
 235                          * Return the offset to mmap.
 236                          * See rvt_mmap() for details.
 237                          */
 238                         if (udata && udata->inlen >= sizeof(__u64)) {
 239                                 ret = ib_copy_to_udata(udata, &ip->offset,
 240                                                        sizeof(ip->offset));
 241                                 if (ret)
 242                                         return ret;
 243                         }
 244
 245                         /*
 246                          * Put user mapping info onto the pending list
 247                          * unless it already is on the list.
 248                          */
 249                         spin_lock_irq(&dev->pending_lock);
 250                         if (list_empty(&ip->pending_mmaps))
 251                                 list_add(&ip->pending_mmaps,
 252                                          &dev->pending_mmaps);
 253                         spin_unlock_irq(&dev->pending_lock);
 254                 }
 255         } else if (attr_mask & IB_SRQ_LIMIT) {
 256                 spin_lock_irq(&srq->rq.kwq->c_lock);
 257                 if (attr->srq_limit >= srq->rq.size)
 258                         ret = -EINVAL;
 259                 else
 260                         srq->limit = attr->srq_limit;
 261                 spin_unlock_irq(&srq->rq.kwq->c_lock);
 262         }
 263         return ret;
 264
 265 bail_unlock:
 266         spin_unlock_irq(&srq->rq.kwq->c_lock);
 267 bail_free:
 268         rvt_free_rq(&tmp_rq);
 269         return ret;
 270 }
 271
 272 /**
 273  * rvt_query_srq - query srq data
 274  * @ibsrq: srq to query
 275  * @attr: return info in attr
 276  *
 277  * Return: always 0
 278  */
 279 int rvt_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr)
 280 {
 281         struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
 282
 283         attr->max_wr = srq->rq.size - 1;
 284         attr->max_sge = srq->rq.max_sge;
 285         attr->srq_limit = srq->limit;
 286         return 0;
 287 }
 288
 289 /**
 290  * rvt_destroy_srq - destory an srq
 291  * @ibsrq: srq object to destroy
 292  * @udata: user data for libibverbs.so
 293  */
 294 int rvt_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
 295 {
 296         struct rvt_srq *srq = ibsrq_to_rvtsrq(ibsrq);
 297         struct rvt_dev_info *dev = ib_to_rvt(ibsrq->device);
 298
 299         spin_lock(&dev->n_srqs_lock);
 300         dev->n_srqs_allocated--;
 301         spin_unlock(&dev->n_srqs_lock);
 302         if (srq->ip)
 303                 kref_put(&srq->ip->ref, rvt_release_mmap_info);
 304         kvfree(srq->rq.kwq);
 305         return 0;
 306 }