drivers/infiniband/core/uverbs_ioctl.c

   1 /*
   2  * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <rdma/rdma_user_ioctl.h>
  34 #include <rdma/uverbs_ioctl.h>
  35 #include "rdma_core.h"
  36 #include "uverbs.h"
  37
  38 struct bundle_alloc_head {
  39         struct bundle_alloc_head *next;
  40         u8 data[];
  41 };
  42
  43 struct bundle_priv {
  44         /* Must be first */
  45         struct bundle_alloc_head alloc_head;
  46         struct bundle_alloc_head *allocated_mem;
  47         size_t internal_avail;
  48         size_t internal_used;
  49
  50         struct radix_tree_root *radix;
  51         const struct uverbs_api_ioctl_method *method_elm;
  52         void __rcu **radix_slots;
  53         unsigned long radix_slots_len;
  54         u32 method_key;
  55
  56         struct ib_uverbs_attr __user *user_attrs;
  57         struct ib_uverbs_attr *uattrs;
  58
  59         DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
  60         DECLARE_BITMAP(spec_finalize, UVERBS_API_ATTR_BKEY_LEN);
  61         DECLARE_BITMAP(uobj_hw_obj_valid, UVERBS_API_ATTR_BKEY_LEN);
  62
  63         /*
  64          * Must be last. bundle ends in a flex array which overlaps
  65          * internal_buffer.
  66          */
  67         struct uverbs_attr_bundle bundle;
  68         u64 internal_buffer[32];
  69 };
  70
  71 /*
  72  * Each method has an absolute minimum amount of memory it needs to allocate,
  73  * precompute that amount and determine if the onstack memory can be used or
  74  * if allocation is need.
  75  */
  76 void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
  77                               unsigned int num_attrs)
  78 {
  79         struct bundle_priv *pbundle;
  80         size_t bundle_size =
  81                 offsetof(struct bundle_priv, internal_buffer) +
  82                 sizeof(*pbundle->bundle.attrs) * method_elm->key_bitmap_len +
  83                 sizeof(*pbundle->uattrs) * num_attrs;
  84
  85         method_elm->use_stack = bundle_size <= sizeof(*pbundle);
  86         method_elm->bundle_size =
  87                 ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));
  88
  89         /* Do not want order-2 allocations for this. */
  90         WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
  91 }
  92
  93 /**
  94  * _uverbs_alloc() - Quickly allocate memory for use with a bundle
  95  * @bundle: The bundle
  96  * @size: Number of bytes to allocate
  97  * @flags: Allocator flags
  98  *
  99  * The bundle allocator is intended for allocations that are connected with
 100  * processing the system call related to the bundle. The allocated memory is
 101  * always freed once the system call completes, and cannot be freed any other
 102  * way.
 103  *
 104  * This tries to use a small pool of pre-allocated memory for performance.
 105  */
 106 __malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
 107                              gfp_t flags)
 108 {
 109         struct bundle_priv *pbundle =
 110                 container_of(bundle, struct bundle_priv, bundle);
 111         size_t new_used;
 112         void *res;
 113
 114         if (check_add_overflow(size, pbundle->internal_used, &new_used))
 115                 return ERR_PTR(-EOVERFLOW);
 116
 117         if (new_used > pbundle->internal_avail) {
 118                 struct bundle_alloc_head *buf;
 119
 120                 buf = kvmalloc(struct_size(buf, data, size), flags);
 121                 if (!buf)
 122                         return ERR_PTR(-ENOMEM);
 123                 buf->next = pbundle->allocated_mem;
 124                 pbundle->allocated_mem = buf;
 125                 return buf->data;
 126         }
 127
 128         res = (void *)pbundle->internal_buffer + pbundle->internal_used;
 129         pbundle->internal_used =
 130                 ALIGN(new_used, sizeof(*pbundle->internal_buffer));
 131         if (want_init_on_alloc(flags))
 132                 memset(res, 0, size);
 133         return res;
 134 }
 135 EXPORT_SYMBOL(_uverbs_alloc);
 136
 137 static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
 138                                    u16 len)
 139 {
 140         if (uattr->len > sizeof_field(struct ib_uverbs_attr, data))
 141                 return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,
 142                                             uattr->len - len);
 143
 144         return !memchr_inv((const void *)&uattr->data + len,
 145                            0, uattr->len - len);
 146 }
 147
 148 static int uverbs_set_output(const struct uverbs_attr_bundle *bundle,
 149                              const struct uverbs_attr *attr)
 150 {
 151         struct bundle_priv *pbundle =
 152                 container_of(bundle, struct bundle_priv, bundle);
 153         u16 flags;
 154
 155         flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
 156                 UVERBS_ATTR_F_VALID_OUTPUT;
 157         if (put_user(flags,
 158                      &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
 159                 return -EFAULT;
 160         return 0;
 161 }
 162
 163 static int uverbs_process_idrs_array(struct bundle_priv *pbundle,
 164                                      const struct uverbs_api_attr *attr_uapi,
 165                                      struct uverbs_objs_arr_attr *attr,
 166                                      struct ib_uverbs_attr *uattr,
 167                                      u32 attr_bkey)
 168 {
 169         const struct uverbs_attr_spec *spec = &attr_uapi->spec;
 170         size_t array_len;
 171         u32 *idr_vals;
 172         int ret = 0;
 173         size_t i;
 174
 175         if (uattr->attr_data.reserved)
 176                 return -EINVAL;
 177
 178         if (uattr->len % sizeof(u32))
 179                 return -EINVAL;
 180
 181         array_len = uattr->len / sizeof(u32);
 182         if (array_len < spec->u2.objs_arr.min_len ||
 183             array_len > spec->u2.objs_arr.max_len)
 184                 return -EINVAL;
 185
 186         attr->uobjects =
 187                 uverbs_alloc(&pbundle->bundle,
 188                              array_size(array_len, sizeof(*attr->uobjects)));
 189         if (IS_ERR(attr->uobjects))
 190                 return PTR_ERR(attr->uobjects);
 191
 192         /*
 193          * Since idr is 4B and *uobjects is >= 4B, we can use attr->uobjects
 194          * to store idrs array and avoid additional memory allocation. The
 195          * idrs array is offset to the end of the uobjects array so we will be
 196          * able to read idr and replace with a pointer.
 197          */
 198         idr_vals = (u32 *)(attr->uobjects + array_len) - array_len;
 199
 200         if (uattr->len > sizeof(uattr->data)) {
 201                 ret = copy_from_user(idr_vals, u64_to_user_ptr(uattr->data),
 202                                      uattr->len);
 203                 if (ret)
 204                         return -EFAULT;
 205         } else {
 206                 memcpy(idr_vals, &uattr->data, uattr->len);
 207         }
 208
 209         for (i = 0; i != array_len; i++) {
 210                 attr->uobjects[i] = uverbs_get_uobject_from_file(
 211                         spec->u2.objs_arr.obj_type, spec->u2.objs_arr.access,
 212                         idr_vals[i], &pbundle->bundle);
 213                 if (IS_ERR(attr->uobjects[i])) {
 214                         ret = PTR_ERR(attr->uobjects[i]);
 215                         break;
 216                 }
 217         }
 218
 219         attr->len = i;
 220         __set_bit(attr_bkey, pbundle->spec_finalize);
 221         return ret;
 222 }
 223
 224 static void uverbs_free_idrs_array(const struct uverbs_api_attr *attr_uapi,
 225                                    struct uverbs_objs_arr_attr *attr,
 226                                    bool commit,
 227                                    struct uverbs_attr_bundle *attrs)
 228 {
 229         const struct uverbs_attr_spec *spec = &attr_uapi->spec;
 230         size_t i;
 231
 232         for (i = 0; i != attr->len; i++)
 233                 uverbs_finalize_object(attr->uobjects[i],
 234                                        spec->u2.objs_arr.access, false, commit,
 235                                        attrs);
 236 }
 237
 238 static int uverbs_process_attr(struct bundle_priv *pbundle,
 239                                const struct uverbs_api_attr *attr_uapi,
 240                                struct ib_uverbs_attr *uattr, u32 attr_bkey)
 241 {
 242         const struct uverbs_attr_spec *spec = &attr_uapi->spec;
 243         struct uverbs_attr *e = &pbundle->bundle.attrs[attr_bkey];
 244         const struct uverbs_attr_spec *val_spec = spec;
 245         struct uverbs_obj_attr *o_attr;
 246
 247         switch (spec->type) {
 248         case UVERBS_ATTR_TYPE_ENUM_IN:
 249                 if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)
 250                         return -EOPNOTSUPP;
 251
 252                 if (uattr->attr_data.enum_data.reserved)
 253                         return -EINVAL;
 254
 255                 val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];
 256
 257                 /* Currently we only support PTR_IN based enums */
 258                 if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)
 259                         return -EOPNOTSUPP;
 260
 261                 e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;
 262                 fallthrough;
 263         case UVERBS_ATTR_TYPE_PTR_IN:
 264                 /* Ensure that any data provided by userspace beyond the known
 265                  * struct is zero. Userspace that knows how to use some future
 266                  * longer struct will fail here if used with an old kernel and
 267                  * non-zero content, making ABI compat/discovery simpler.
 268                  */
 269                 if (uattr->len > val_spec->u.ptr.len &&
 270                     val_spec->zero_trailing &&
 271                     !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len))
 272                         return -EOPNOTSUPP;
 273
 274                 fallthrough;
 275         case UVERBS_ATTR_TYPE_PTR_OUT:
 276                 if (uattr->len < val_spec->u.ptr.min_len ||
 277                     (!val_spec->zero_trailing &&
 278                      uattr->len > val_spec->u.ptr.len))
 279                         return -EINVAL;
 280
 281                 if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&
 282                     uattr->attr_data.reserved)
 283                         return -EINVAL;
 284
 285                 e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;
 286                 e->ptr_attr.len = uattr->len;
 287
 288                 if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {
 289                         void *p;
 290
 291                         p = uverbs_alloc(&pbundle->bundle, uattr->len);
 292                         if (IS_ERR(p))
 293                                 return PTR_ERR(p);
 294
 295                         e->ptr_attr.ptr = p;
 296
 297                         if (copy_from_user(p, u64_to_user_ptr(uattr->data),
 298                                            uattr->len))
 299                                 return -EFAULT;
 300                 } else {
 301                         e->ptr_attr.data = uattr->data;
 302                 }
 303                 break;
 304
 305         case UVERBS_ATTR_TYPE_IDR:
 306         case UVERBS_ATTR_TYPE_FD:
 307                 if (uattr->attr_data.reserved)
 308                         return -EINVAL;
 309
 310                 if (uattr->len != 0)
 311                         return -EINVAL;
 312
 313                 o_attr = &e->obj_attr;
 314                 o_attr->attr_elm = attr_uapi;
 315
 316                 /*
 317                  * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and
 318                  * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64
 319                  * here without caring about truncation as we know that the
 320                  * IDR implementation today rejects negative IDs
 321                  */
 322                 o_attr->uobject = uverbs_get_uobject_from_file(
 323                         spec->u.obj.obj_type, spec->u.obj.access,
 324                         uattr->data_s64, &pbundle->bundle);
 325                 if (IS_ERR(o_attr->uobject))
 326                         return PTR_ERR(o_attr->uobject);
 327                 __set_bit(attr_bkey, pbundle->uobj_finalize);
 328
 329                 if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
 330                         unsigned int uattr_idx = uattr - pbundle->uattrs;
 331                         s64 id = o_attr->uobject->id;
 332
 333                         /* Copy the allocated id to the user-space */
 334                         if (put_user(id, &pbundle->user_attrs[uattr_idx].data))
 335                                 return -EFAULT;
 336                 }
 337
 338                 break;
 339
 340         case UVERBS_ATTR_TYPE_RAW_FD:
 341                 if (uattr->attr_data.reserved || uattr->len != 0 ||
 342                     uattr->data_s64 < INT_MIN || uattr->data_s64 > INT_MAX)
 343                         return -EINVAL;
 344                 /* _uverbs_get_const_signed() is the accessor */
 345                 e->ptr_attr.data = uattr->data_s64;
 346                 break;
 347
 348         case UVERBS_ATTR_TYPE_IDRS_ARRAY:
 349                 return uverbs_process_idrs_array(pbundle, attr_uapi,
 350                                                  &e->objs_arr_attr, uattr,
 351                                                  attr_bkey);
 352         default:
 353                 return -EOPNOTSUPP;
 354         }
 355
 356         return 0;
 357 }
 358
 359 /*
 360  * We search the radix tree with the method prefix and now we want to fast
 361  * search the suffix bits to get a particular attribute pointer. It is not
 362  * totally clear to me if this breaks the radix tree encasulation or not, but
 363  * it uses the iter data to determine if the method iter points at the same
 364  * chunk that will store the attribute, if so it just derefs it directly. By
 365  * construction in most kernel configs the method and attrs will all fit in a
 366  * single radix chunk, so in most cases this will have no search. Other cases
 367  * this falls back to a full search.
 368  */
 369 static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle,
 370                                              u32 attr_key)
 371 {
 372         void __rcu **slot;
 373
 374         if (likely(attr_key < pbundle->radix_slots_len)) {
 375                 void *entry;
 376
 377                 slot = pbundle->radix_slots + attr_key;
 378                 entry = rcu_dereference_raw(*slot);
 379                 if (likely(!radix_tree_is_internal_node(entry) && entry))
 380                         return slot;
 381         }
 382
 383         return radix_tree_lookup_slot(pbundle->radix,
 384                                       pbundle->method_key | attr_key);
 385 }
 386
 387 static int uverbs_set_attr(struct bundle_priv *pbundle,
 388                            struct ib_uverbs_attr *uattr)
 389 {
 390         u32 attr_key = uapi_key_attr(uattr->attr_id);
 391         u32 attr_bkey = uapi_bkey_attr(attr_key);
 392         const struct uverbs_api_attr *attr;
 393         void __rcu **slot;
 394         int ret;
 395
 396         slot = uapi_get_attr_for_method(pbundle, attr_key);
 397         if (!slot) {
 398                 /*
 399                  * Kernel does not support the attribute but user-space says it
 400                  * is mandatory
 401                  */
 402                 if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
 403                         return -EPROTONOSUPPORT;
 404                 return 0;
 405         }
 406         attr = rcu_dereference_protected(*slot, true);
 407
 408         /* Reject duplicate attributes from user-space */
 409         if (test_bit(attr_bkey, pbundle->bundle.attr_present))
 410                 return -EINVAL;
 411
 412         ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey);
 413         if (ret)
 414                 return ret;
 415
 416         __set_bit(attr_bkey, pbundle->bundle.attr_present);
 417
 418         return 0;
 419 }
 420
 421 static int ib_uverbs_run_method(struct bundle_priv *pbundle,
 422                                 unsigned int num_attrs)
 423 {
 424         int (*handler)(struct uverbs_attr_bundle *attrs);
 425         size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
 426         unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
 427         unsigned int i;
 428         int ret;
 429
 430         /* See uverbs_disassociate_api() */
 431         handler = srcu_dereference(
 432                 pbundle->method_elm->handler,
 433                 &pbundle->bundle.ufile->device->disassociate_srcu);
 434         if (!handler)
 435                 return -EIO;
 436
 437         pbundle->uattrs = uverbs_alloc(&pbundle->bundle, uattrs_size);
 438         if (IS_ERR(pbundle->uattrs))
 439                 return PTR_ERR(pbundle->uattrs);
 440         if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size))
 441                 return -EFAULT;
 442
 443         for (i = 0; i != num_attrs; i++) {
 444                 ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]);
 445                 if (unlikely(ret))
 446                         return ret;
 447         }
 448
 449         /* User space did not provide all the mandatory attributes */
 450         if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
 451                                     pbundle->bundle.attr_present,
 452                                     pbundle->method_elm->key_bitmap_len)))
 453                 return -EINVAL;
 454
 455         if (pbundle->method_elm->has_udata)
 456                 uverbs_fill_udata(&pbundle->bundle,
 457                                   &pbundle->bundle.driver_udata,
 458                                   UVERBS_ATTR_UHW_IN, UVERBS_ATTR_UHW_OUT);
 459         else
 460                 pbundle->bundle.driver_udata = (struct ib_udata){};
 461
 462         if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
 463                 struct uverbs_obj_attr *destroy_attr =
 464                         &pbundle->bundle.attrs[destroy_bkey].obj_attr;
 465
 466                 ret = uobj_destroy(destroy_attr->uobject, &pbundle->bundle);
 467                 if (ret)
 468                         return ret;
 469                 __clear_bit(destroy_bkey, pbundle->uobj_finalize);
 470
 471                 ret = handler(&pbundle->bundle);
 472                 uobj_put_destroy(destroy_attr->uobject);
 473         } else {
 474                 ret = handler(&pbundle->bundle);
 475         }
 476
 477         /*
 478          * Until the drivers are revised to use the bundle directly we have to
 479          * assume that the driver wrote to its UHW_OUT and flag userspace
 480          * appropriately.
 481          */
 482         if (!ret && pbundle->method_elm->has_udata) {
 483                 const struct uverbs_attr *attr =
 484                         uverbs_attr_get(&pbundle->bundle, UVERBS_ATTR_UHW_OUT);
 485
 486                 if (!IS_ERR(attr))
 487                         ret = uverbs_set_output(&pbundle->bundle, attr);
 488         }
 489
 490         /*
 491          * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
 492          * not invoke the method because the request is not supported.  No
 493          * other cases should return this code.
 494          */
 495         if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))
 496                 return -EINVAL;
 497
 498         return ret;
 499 }
 500
 501 static void bundle_destroy(struct bundle_priv *pbundle, bool commit)
 502 {
 503         unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
 504         struct bundle_alloc_head *memblock;
 505         unsigned int i;
 506
 507         /* fast path for simple uobjects */
 508         i = -1;
 509         while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len,
 510                                   i + 1)) < key_bitmap_len) {
 511                 struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
 512
 513                 uverbs_finalize_object(
 514                         attr->obj_attr.uobject,
 515                         attr->obj_attr.attr_elm->spec.u.obj.access,
 516                         test_bit(i, pbundle->uobj_hw_obj_valid),
 517                         commit,
 518                         &pbundle->bundle);
 519         }
 520
 521         i = -1;
 522         while ((i = find_next_bit(pbundle->spec_finalize, key_bitmap_len,
 523                                   i + 1)) < key_bitmap_len) {
 524                 struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
 525                 const struct uverbs_api_attr *attr_uapi;
 526                 void __rcu **slot;
 527
 528                 slot = uapi_get_attr_for_method(
 529                         pbundle,
 530                         pbundle->method_key | uapi_bkey_to_key_attr(i));
 531                 if (WARN_ON(!slot))
 532                         continue;
 533
 534                 attr_uapi = rcu_dereference_protected(*slot, true);
 535
 536                 if (attr_uapi->spec.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
 537                         uverbs_free_idrs_array(attr_uapi, &attr->objs_arr_attr,
 538                                                commit, &pbundle->bundle);
 539                 }
 540         }
 541
 542         for (memblock = pbundle->allocated_mem; memblock;) {
 543                 struct bundle_alloc_head *tmp = memblock;
 544
 545                 memblock = memblock->next;
 546                 kvfree(tmp);
 547         }
 548 }
 549
 550 static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
 551                                struct ib_uverbs_ioctl_hdr *hdr,
 552                                struct ib_uverbs_attr __user *user_attrs)
 553 {
 554         const struct uverbs_api_ioctl_method *method_elm;
 555         struct uverbs_api *uapi = ufile->device->uapi;
 556         struct radix_tree_iter attrs_iter;
 557         struct bundle_priv *pbundle;
 558         struct bundle_priv onstack;
 559         void __rcu **slot;
 560         int ret;
 561
 562         if (unlikely(hdr->driver_id != uapi->driver_id))
 563                 return -EINVAL;
 564
 565         slot = radix_tree_iter_lookup(
 566                 &uapi->radix, &attrs_iter,
 567                 uapi_key_obj(hdr->object_id) |
 568                         uapi_key_ioctl_method(hdr->method_id));
 569         if (unlikely(!slot))
 570                 return -EPROTONOSUPPORT;
 571         method_elm = rcu_dereference_protected(*slot, true);
 572
 573         if (!method_elm->use_stack) {
 574                 pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);
 575                 if (!pbundle)
 576                         return -ENOMEM;
 577                 pbundle->internal_avail =
 578                         method_elm->bundle_size -
 579                         offsetof(struct bundle_priv, internal_buffer);
 580                 pbundle->alloc_head.next = NULL;
 581                 pbundle->allocated_mem = &pbundle->alloc_head;
 582         } else {
 583                 pbundle = &onstack;
 584                 pbundle->internal_avail = sizeof(pbundle->internal_buffer);
 585                 pbundle->allocated_mem = NULL;
 586         }
 587
 588         /* Space for the pbundle->bundle.attrs flex array */
 589         pbundle->method_elm = method_elm;
 590         pbundle->method_key = attrs_iter.index;
 591         pbundle->bundle.ufile = ufile;
 592         pbundle->bundle.context = NULL; /* only valid if bundle has uobject */
 593         pbundle->radix = &uapi->radix;
 594         pbundle->radix_slots = slot;
 595         pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);
 596         pbundle->user_attrs = user_attrs;
 597
 598         pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
 599                                                sizeof(*pbundle->bundle.attrs),
 600                                        sizeof(*pbundle->internal_buffer));
 601         memset(pbundle->bundle.attr_present, 0,
 602                sizeof(pbundle->bundle.attr_present));
 603         memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
 604         memset(pbundle->spec_finalize, 0, sizeof(pbundle->spec_finalize));
 605         memset(pbundle->uobj_hw_obj_valid, 0,
 606                sizeof(pbundle->uobj_hw_obj_valid));
 607
 608         ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);
 609         bundle_destroy(pbundle, ret == 0);
 610         return ret;
 611 }
 612
 613 long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
 614 {
 615         struct ib_uverbs_file *file = filp->private_data;
 616         struct ib_uverbs_ioctl_hdr __user *user_hdr =
 617                 (struct ib_uverbs_ioctl_hdr __user *)arg;
 618         struct ib_uverbs_ioctl_hdr hdr;
 619         int srcu_key;
 620         int err;
 621
 622         if (unlikely(cmd != RDMA_VERBS_IOCTL))
 623                 return -ENOIOCTLCMD;
 624
 625         err = copy_from_user(&hdr, user_hdr, sizeof(hdr));
 626         if (err)
 627                 return -EFAULT;
 628
 629         if (hdr.length > PAGE_SIZE ||
 630             hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))
 631                 return -EINVAL;
 632
 633         if (hdr.reserved1 || hdr.reserved2)
 634                 return -EPROTONOSUPPORT;
 635
 636         srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
 637         err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs);
 638         srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
 639         return err;
 640 }
 641
 642 int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
 643                        size_t idx, u64 allowed_bits)
 644 {
 645         const struct uverbs_attr *attr;
 646         u64 flags;
 647
 648         attr = uverbs_attr_get(attrs_bundle, idx);
 649         /* Missing attribute means 0 flags */
 650         if (IS_ERR(attr)) {
 651                 *to = 0;
 652                 return 0;
 653         }
 654
 655         /*
 656          * New userspace code should use 8 bytes to pass flags, but we
 657          * transparently support old userspaces that were using 4 bytes as
 658          * well.
 659          */
 660         if (attr->ptr_attr.len == 8)
 661                 flags = attr->ptr_attr.data;
 662         else if (attr->ptr_attr.len == 4)
 663                 flags = *(u32 *)&attr->ptr_attr.data;
 664         else
 665                 return -EINVAL;
 666
 667         if (flags & ~allowed_bits)
 668                 return -EINVAL;
 669
 670         *to = flags;
 671         return 0;
 672 }
 673 EXPORT_SYMBOL(uverbs_get_flags64);
 674
 675 int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
 676                        size_t idx, u64 allowed_bits)
 677 {
 678         u64 flags;
 679         int ret;
 680
 681         ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
 682         if (ret)
 683                 return ret;
 684
 685         if (flags > U32_MAX)
 686                 return -EINVAL;
 687         *to = flags;
 688
 689         return 0;
 690 }
 691 EXPORT_SYMBOL(uverbs_get_flags32);
 692
 693 /*
 694  * Fill a ib_udata struct (core or uhw) using the given attribute IDs.
 695  * This is primarily used to convert the UVERBS_ATTR_UHW() into the
 696  * ib_udata format used by the drivers.
 697  */
 698 void uverbs_fill_udata(struct uverbs_attr_bundle *bundle,
 699                        struct ib_udata *udata, unsigned int attr_in,
 700                        unsigned int attr_out)
 701 {
 702         struct bundle_priv *pbundle =
 703                 container_of(bundle, struct bundle_priv, bundle);
 704         const struct uverbs_attr *in =
 705                 uverbs_attr_get(&pbundle->bundle, attr_in);
 706         const struct uverbs_attr *out =
 707                 uverbs_attr_get(&pbundle->bundle, attr_out);
 708
 709         if (!IS_ERR(in)) {
 710                 udata->inlen = in->ptr_attr.len;
 711                 if (uverbs_attr_ptr_is_inline(in))
 712                         udata->inbuf =
 713                                 &pbundle->user_attrs[in->ptr_attr.uattr_idx]
 714                                          .data;
 715                 else
 716                         udata->inbuf = u64_to_user_ptr(in->ptr_attr.data);
 717         } else {
 718                 udata->inbuf = NULL;
 719                 udata->inlen = 0;
 720         }
 721
 722         if (!IS_ERR(out)) {
 723                 udata->outbuf = u64_to_user_ptr(out->ptr_attr.data);
 724                 udata->outlen = out->ptr_attr.len;
 725         } else {
 726                 udata->outbuf = NULL;
 727                 udata->outlen = 0;
 728         }
 729 }
 730
 731 int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
 732                    const void *from, size_t size)
 733 {
 734         const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
 735         size_t min_size;
 736
 737         if (IS_ERR(attr))
 738                 return PTR_ERR(attr);
 739
 740         min_size = min_t(size_t, attr->ptr_attr.len, size);
 741         if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
 742                 return -EFAULT;
 743
 744         return uverbs_set_output(bundle, attr);
 745 }
 746 EXPORT_SYMBOL(uverbs_copy_to);
 747
 748
 749 /*
 750  * This is only used if the caller has directly used copy_to_use to write the
 751  * data.  It signals to user space that the buffer is filled in.
 752  */
 753 int uverbs_output_written(const struct uverbs_attr_bundle *bundle, size_t idx)
 754 {
 755         const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
 756
 757         if (IS_ERR(attr))
 758                 return PTR_ERR(attr);
 759
 760         return uverbs_set_output(bundle, attr);
 761 }
 762
 763 int _uverbs_get_const_signed(s64 *to,
 764                              const struct uverbs_attr_bundle *attrs_bundle,
 765                              size_t idx, s64 lower_bound, u64 upper_bound,
 766                              s64  *def_val)
 767 {
 768         const struct uverbs_attr *attr;
 769
 770         attr = uverbs_attr_get(attrs_bundle, idx);
 771         if (IS_ERR(attr)) {
 772                 if ((PTR_ERR(attr) != -ENOENT) || !def_val)
 773                         return PTR_ERR(attr);
 774
 775                 *to = *def_val;
 776         } else {
 777                 *to = attr->ptr_attr.data;
 778         }
 779
 780         if (*to < lower_bound || (*to > 0 && (u64)*to > upper_bound))
 781                 return -EINVAL;
 782
 783         return 0;
 784 }
 785 EXPORT_SYMBOL(_uverbs_get_const_signed);
 786
 787 int _uverbs_get_const_unsigned(u64 *to,
 788                                const struct uverbs_attr_bundle *attrs_bundle,
 789                                size_t idx, u64 upper_bound, u64 *def_val)
 790 {
 791         const struct uverbs_attr *attr;
 792
 793         attr = uverbs_attr_get(attrs_bundle, idx);
 794         if (IS_ERR(attr)) {
 795                 if ((PTR_ERR(attr) != -ENOENT) || !def_val)
 796                         return PTR_ERR(attr);
 797
 798                 *to = *def_val;
 799         } else {
 800                 *to = attr->ptr_attr.data;
 801         }
 802
 803         if (*to > upper_bound)
 804                 return -EINVAL;
 805
 806         return 0;
 807 }
 808 EXPORT_SYMBOL(_uverbs_get_const_unsigned);
 809
 810 int uverbs_copy_to_struct_or_zero(const struct uverbs_attr_bundle *bundle,
 811                                   size_t idx, const void *from, size_t size)
 812 {
 813         const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
 814
 815         if (IS_ERR(attr))
 816                 return PTR_ERR(attr);
 817
 818         if (size < attr->ptr_attr.len) {
 819                 if (clear_user(u64_to_user_ptr(attr->ptr_attr.data) + size,
 820                                attr->ptr_attr.len - size))
 821                         return -EFAULT;
 822         }
 823         return uverbs_copy_to(bundle, idx, from, size);
 824 }
 825 EXPORT_SYMBOL(uverbs_copy_to_struct_or_zero);
 826
 827 /* Once called an abort will call through to the type's destroy_hw() */
 828 void uverbs_finalize_uobj_create(const struct uverbs_attr_bundle *bundle,
 829                                  u16 idx)
 830 {
 831         struct bundle_priv *pbundle =
 832                 container_of(bundle, struct bundle_priv, bundle);
 833
 834         __set_bit(uapi_bkey_attr(uapi_key_attr(idx)),
 835                   pbundle->uobj_hw_obj_valid);
 836 }
 837 EXPORT_SYMBOL(uverbs_finalize_uobj_create);