2 * Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <rdma/rdma_user_ioctl.h>
34 #include <rdma/uverbs_ioctl.h>
35 #include "rdma_core.h"
38 struct bundle_alloc_head {
39 struct bundle_alloc_head *next;
45 struct bundle_alloc_head alloc_head;
46 struct bundle_alloc_head *allocated_mem;
47 size_t internal_avail;
50 struct radix_tree_root *radix;
51 const struct uverbs_api_ioctl_method *method_elm;
52 void __rcu **radix_slots;
53 unsigned long radix_slots_len;
56 struct ib_uverbs_attr __user *user_attrs;
57 struct ib_uverbs_attr *uattrs;
59 DECLARE_BITMAP(uobj_finalize, UVERBS_API_ATTR_BKEY_LEN);
62 * Must be last. bundle ends in a flex array which overlaps
65 struct uverbs_attr_bundle bundle;
66 u64 internal_buffer[32];
70 * Each method has an absolute minimum amount of memory it needs to allocate,
71 * precompute that amount and determine if the onstack memory can be used or
72 * if allocation is need.
74 void uapi_compute_bundle_size(struct uverbs_api_ioctl_method *method_elm,
75 unsigned int num_attrs)
77 struct bundle_priv *pbundle;
79 offsetof(struct bundle_priv, internal_buffer) +
80 sizeof(*pbundle->bundle.attrs) * method_elm->key_bitmap_len +
81 sizeof(*pbundle->uattrs) * num_attrs;
83 method_elm->use_stack = bundle_size <= sizeof(*pbundle);
84 method_elm->bundle_size =
85 ALIGN(bundle_size + 256, sizeof(*pbundle->internal_buffer));
87 /* Do not want order-2 allocations for this. */
88 WARN_ON_ONCE(method_elm->bundle_size > PAGE_SIZE);
92 * uverbs_alloc() - Quickly allocate memory for use with a bundle
94 * @size: Number of bytes to allocate
95 * @flags: Allocator flags
97 * The bundle allocator is intended for allocations that are connected with
98 * processing the system call related to the bundle. The allocated memory is
99 * always freed once the system call completes, and cannot be freed any other
102 * This tries to use a small pool of pre-allocated memory for performance.
104 __malloc void *_uverbs_alloc(struct uverbs_attr_bundle *bundle, size_t size,
107 struct bundle_priv *pbundle =
108 container_of(bundle, struct bundle_priv, bundle);
112 if (check_add_overflow(size, pbundle->internal_used, &new_used))
113 return ERR_PTR(-EOVERFLOW);
115 if (new_used > pbundle->internal_avail) {
116 struct bundle_alloc_head *buf;
118 buf = kvmalloc(struct_size(buf, data, size), flags);
120 return ERR_PTR(-ENOMEM);
121 buf->next = pbundle->allocated_mem;
122 pbundle->allocated_mem = buf;
126 res = (void *)pbundle->internal_buffer + pbundle->internal_used;
127 pbundle->internal_used =
128 ALIGN(new_used, sizeof(*pbundle->internal_buffer));
129 if (flags & __GFP_ZERO)
130 memset(res, 0, size);
133 EXPORT_SYMBOL(_uverbs_alloc);
135 static bool uverbs_is_attr_cleared(const struct ib_uverbs_attr *uattr,
138 if (uattr->len > sizeof(((struct ib_uverbs_attr *)0)->data))
139 return ib_is_buffer_cleared(u64_to_user_ptr(uattr->data) + len,
142 return !memchr_inv((const void *)&uattr->data + len,
143 0, uattr->len - len);
146 static int uverbs_process_attr(struct bundle_priv *pbundle,
147 const struct uverbs_api_attr *attr_uapi,
148 struct ib_uverbs_attr *uattr, u32 attr_bkey)
150 const struct uverbs_attr_spec *spec = &attr_uapi->spec;
151 struct uverbs_attr *e = &pbundle->bundle.attrs[attr_bkey];
152 const struct uverbs_attr_spec *val_spec = spec;
153 struct uverbs_obj_attr *o_attr;
155 switch (spec->type) {
156 case UVERBS_ATTR_TYPE_ENUM_IN:
157 if (uattr->attr_data.enum_data.elem_id >= spec->u.enum_def.num_elems)
160 if (uattr->attr_data.enum_data.reserved)
163 val_spec = &spec->u2.enum_def.ids[uattr->attr_data.enum_data.elem_id];
165 /* Currently we only support PTR_IN based enums */
166 if (val_spec->type != UVERBS_ATTR_TYPE_PTR_IN)
169 e->ptr_attr.enum_id = uattr->attr_data.enum_data.elem_id;
171 case UVERBS_ATTR_TYPE_PTR_IN:
172 /* Ensure that any data provided by userspace beyond the known
173 * struct is zero. Userspace that knows how to use some future
174 * longer struct will fail here if used with an old kernel and
175 * non-zero content, making ABI compat/discovery simpler.
177 if (uattr->len > val_spec->u.ptr.len &&
178 val_spec->zero_trailing &&
179 !uverbs_is_attr_cleared(uattr, val_spec->u.ptr.len))
183 case UVERBS_ATTR_TYPE_PTR_OUT:
184 if (uattr->len < val_spec->u.ptr.min_len ||
185 (!val_spec->zero_trailing &&
186 uattr->len > val_spec->u.ptr.len))
189 if (spec->type != UVERBS_ATTR_TYPE_ENUM_IN &&
190 uattr->attr_data.reserved)
193 e->ptr_attr.uattr_idx = uattr - pbundle->uattrs;
194 e->ptr_attr.len = uattr->len;
196 if (val_spec->alloc_and_copy && !uverbs_attr_ptr_is_inline(e)) {
199 p = uverbs_alloc(&pbundle->bundle, uattr->len);
205 if (copy_from_user(p, u64_to_user_ptr(uattr->data),
209 e->ptr_attr.data = uattr->data;
213 case UVERBS_ATTR_TYPE_IDR:
214 case UVERBS_ATTR_TYPE_FD:
215 if (uattr->attr_data.reserved)
221 o_attr = &e->obj_attr;
222 o_attr->attr_elm = attr_uapi;
225 * The type of uattr->data is u64 for UVERBS_ATTR_TYPE_IDR and
226 * s64 for UVERBS_ATTR_TYPE_FD. We can cast the u64 to s64
227 * here without caring about truncation as we know that the
228 * IDR implementation today rejects negative IDs
230 o_attr->uobject = uverbs_get_uobject_from_file(
231 spec->u.obj.obj_type,
232 pbundle->bundle.ufile,
235 if (IS_ERR(o_attr->uobject))
236 return PTR_ERR(o_attr->uobject);
237 __set_bit(attr_bkey, pbundle->uobj_finalize);
239 if (spec->u.obj.access == UVERBS_ACCESS_NEW) {
240 unsigned int uattr_idx = uattr - pbundle->uattrs;
241 s64 id = o_attr->uobject->id;
243 /* Copy the allocated id to the user-space */
244 if (put_user(id, &pbundle->user_attrs[uattr_idx].data))
257 * We search the radix tree with the method prefix and now we want to fast
258 * search the suffix bits to get a particular attribute pointer. It is not
259 * totally clear to me if this breaks the radix tree encasulation or not, but
260 * it uses the iter data to determine if the method iter points at the same
261 * chunk that will store the attribute, if so it just derefs it directly. By
262 * construction in most kernel configs the method and attrs will all fit in a
263 * single radix chunk, so in most cases this will have no search. Other cases
264 * this falls back to a full search.
266 static void __rcu **uapi_get_attr_for_method(struct bundle_priv *pbundle,
271 if (likely(attr_key < pbundle->radix_slots_len)) {
274 slot = pbundle->radix_slots + attr_key;
275 entry = rcu_dereference_raw(*slot);
276 if (likely(!radix_tree_is_internal_node(entry) && entry))
280 return radix_tree_lookup_slot(pbundle->radix,
281 pbundle->method_key | attr_key);
284 static int uverbs_set_attr(struct bundle_priv *pbundle,
285 struct ib_uverbs_attr *uattr)
287 u32 attr_key = uapi_key_attr(uattr->attr_id);
288 u32 attr_bkey = uapi_bkey_attr(attr_key);
289 const struct uverbs_api_attr *attr;
293 slot = uapi_get_attr_for_method(pbundle, attr_key);
296 * Kernel does not support the attribute but user-space says it
299 if (uattr->flags & UVERBS_ATTR_F_MANDATORY)
300 return -EPROTONOSUPPORT;
303 attr = srcu_dereference(
304 *slot, &pbundle->bundle.ufile->device->disassociate_srcu);
306 /* Reject duplicate attributes from user-space */
307 if (test_bit(attr_bkey, pbundle->bundle.attr_present))
310 ret = uverbs_process_attr(pbundle, attr, uattr, attr_bkey);
314 __set_bit(attr_bkey, pbundle->bundle.attr_present);
319 static int ib_uverbs_run_method(struct bundle_priv *pbundle,
320 unsigned int num_attrs)
322 int (*handler)(struct ib_uverbs_file *ufile,
323 struct uverbs_attr_bundle *ctx);
324 size_t uattrs_size = array_size(sizeof(*pbundle->uattrs), num_attrs);
325 unsigned int destroy_bkey = pbundle->method_elm->destroy_bkey;
329 /* See uverbs_disassociate_api() */
330 handler = srcu_dereference(
331 pbundle->method_elm->handler,
332 &pbundle->bundle.ufile->device->disassociate_srcu);
336 pbundle->uattrs = uverbs_alloc(&pbundle->bundle, uattrs_size);
337 if (IS_ERR(pbundle->uattrs))
338 return PTR_ERR(pbundle->uattrs);
339 if (copy_from_user(pbundle->uattrs, pbundle->user_attrs, uattrs_size))
342 for (i = 0; i != num_attrs; i++) {
343 ret = uverbs_set_attr(pbundle, &pbundle->uattrs[i]);
348 /* User space did not provide all the mandatory attributes */
349 if (unlikely(!bitmap_subset(pbundle->method_elm->attr_mandatory,
350 pbundle->bundle.attr_present,
351 pbundle->method_elm->key_bitmap_len)))
354 if (destroy_bkey != UVERBS_API_ATTR_BKEY_LEN) {
355 struct uverbs_obj_attr *destroy_attr =
356 &pbundle->bundle.attrs[destroy_bkey].obj_attr;
358 ret = uobj_destroy(destroy_attr->uobject);
361 __clear_bit(destroy_bkey, pbundle->uobj_finalize);
363 ret = handler(pbundle->bundle.ufile, &pbundle->bundle);
364 uobj_put_destroy(destroy_attr->uobject);
366 ret = handler(pbundle->bundle.ufile, &pbundle->bundle);
370 * EPROTONOSUPPORT is ONLY to be returned if the ioctl framework can
371 * not invoke the method because the request is not supported. No
372 * other cases should return this code.
374 if (WARN_ON_ONCE(ret == -EPROTONOSUPPORT))
380 static int bundle_destroy(struct bundle_priv *pbundle, bool commit)
382 unsigned int key_bitmap_len = pbundle->method_elm->key_bitmap_len;
383 struct bundle_alloc_head *memblock;
388 while ((i = find_next_bit(pbundle->uobj_finalize, key_bitmap_len,
389 i + 1)) < key_bitmap_len) {
390 struct uverbs_attr *attr = &pbundle->bundle.attrs[i];
393 current_ret = uverbs_finalize_object(
394 attr->obj_attr.uobject,
395 attr->obj_attr.attr_elm->spec.u.obj.access, commit);
400 for (memblock = pbundle->allocated_mem; memblock;) {
401 struct bundle_alloc_head *tmp = memblock;
403 memblock = memblock->next;
410 static int ib_uverbs_cmd_verbs(struct ib_uverbs_file *ufile,
411 struct ib_uverbs_ioctl_hdr *hdr,
412 struct ib_uverbs_attr __user *user_attrs)
414 const struct uverbs_api_ioctl_method *method_elm;
415 struct uverbs_api *uapi = ufile->device->uapi;
416 struct radix_tree_iter attrs_iter;
417 struct bundle_priv *pbundle;
418 struct bundle_priv onstack;
423 if (unlikely(hdr->driver_id != uapi->driver_id))
426 slot = radix_tree_iter_lookup(
427 &uapi->radix, &attrs_iter,
428 uapi_key_obj(hdr->object_id) |
429 uapi_key_ioctl_method(hdr->method_id));
431 return -EPROTONOSUPPORT;
432 method_elm = srcu_dereference(*slot, &ufile->device->disassociate_srcu);
434 if (!method_elm->use_stack) {
435 pbundle = kmalloc(method_elm->bundle_size, GFP_KERNEL);
438 pbundle->internal_avail =
439 method_elm->bundle_size -
440 offsetof(struct bundle_priv, internal_buffer);
441 pbundle->alloc_head.next = NULL;
442 pbundle->allocated_mem = &pbundle->alloc_head;
445 pbundle->internal_avail = sizeof(pbundle->internal_buffer);
446 pbundle->allocated_mem = NULL;
449 /* Space for the pbundle->bundle.attrs flex array */
450 pbundle->method_elm = method_elm;
451 pbundle->method_key = attrs_iter.index;
452 pbundle->bundle.ufile = ufile;
453 pbundle->radix = &uapi->radix;
454 pbundle->radix_slots = slot;
455 pbundle->radix_slots_len = radix_tree_chunk_size(&attrs_iter);
456 pbundle->user_attrs = user_attrs;
458 pbundle->internal_used = ALIGN(pbundle->method_elm->key_bitmap_len *
459 sizeof(*pbundle->bundle.attrs),
460 sizeof(*pbundle->internal_buffer));
461 memset(pbundle->bundle.attr_present, 0,
462 sizeof(pbundle->bundle.attr_present));
463 memset(pbundle->uobj_finalize, 0, sizeof(pbundle->uobj_finalize));
465 ret = ib_uverbs_run_method(pbundle, hdr->num_attrs);
466 destroy_ret = bundle_destroy(pbundle, ret == 0);
467 if (unlikely(destroy_ret && !ret))
473 long ib_uverbs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
475 struct ib_uverbs_file *file = filp->private_data;
476 struct ib_uverbs_ioctl_hdr __user *user_hdr =
477 (struct ib_uverbs_ioctl_hdr __user *)arg;
478 struct ib_uverbs_ioctl_hdr hdr;
482 if (unlikely(cmd != RDMA_VERBS_IOCTL))
485 err = copy_from_user(&hdr, user_hdr, sizeof(hdr));
489 if (hdr.length > PAGE_SIZE ||
490 hdr.length != struct_size(&hdr, attrs, hdr.num_attrs))
493 if (hdr.reserved1 || hdr.reserved2)
494 return -EPROTONOSUPPORT;
496 srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
497 err = ib_uverbs_cmd_verbs(file, &hdr, user_hdr->attrs);
498 srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
502 int uverbs_get_flags64(u64 *to, const struct uverbs_attr_bundle *attrs_bundle,
503 size_t idx, u64 allowed_bits)
505 const struct uverbs_attr *attr;
508 attr = uverbs_attr_get(attrs_bundle, idx);
509 /* Missing attribute means 0 flags */
516 * New userspace code should use 8 bytes to pass flags, but we
517 * transparently support old userspaces that were using 4 bytes as
520 if (attr->ptr_attr.len == 8)
521 flags = attr->ptr_attr.data;
522 else if (attr->ptr_attr.len == 4)
523 flags = *(u32 *)&attr->ptr_attr.data;
527 if (flags & ~allowed_bits)
533 EXPORT_SYMBOL(uverbs_get_flags64);
535 int uverbs_get_flags32(u32 *to, const struct uverbs_attr_bundle *attrs_bundle,
536 size_t idx, u64 allowed_bits)
541 ret = uverbs_get_flags64(&flags, attrs_bundle, idx, allowed_bits);
551 EXPORT_SYMBOL(uverbs_get_flags32);
554 * This is for ease of conversion. The purpose is to convert all drivers to
555 * use uverbs_attr_bundle instead of ib_udata. Assume attr == 0 is input and
556 * attr == 1 is output.
558 void create_udata(struct uverbs_attr_bundle *bundle, struct ib_udata *udata)
560 struct bundle_priv *pbundle =
561 container_of(bundle, struct bundle_priv, bundle);
562 const struct uverbs_attr *uhw_in =
563 uverbs_attr_get(bundle, UVERBS_ATTR_UHW_IN);
564 const struct uverbs_attr *uhw_out =
565 uverbs_attr_get(bundle, UVERBS_ATTR_UHW_OUT);
567 if (!IS_ERR(uhw_in)) {
568 udata->inlen = uhw_in->ptr_attr.len;
569 if (uverbs_attr_ptr_is_inline(uhw_in))
571 &pbundle->user_attrs[uhw_in->ptr_attr.uattr_idx]
574 udata->inbuf = u64_to_user_ptr(uhw_in->ptr_attr.data);
580 if (!IS_ERR(uhw_out)) {
581 udata->outbuf = u64_to_user_ptr(uhw_out->ptr_attr.data);
582 udata->outlen = uhw_out->ptr_attr.len;
584 udata->outbuf = NULL;
589 int uverbs_copy_to(const struct uverbs_attr_bundle *bundle, size_t idx,
590 const void *from, size_t size)
592 struct bundle_priv *pbundle =
593 container_of(bundle, struct bundle_priv, bundle);
594 const struct uverbs_attr *attr = uverbs_attr_get(bundle, idx);
599 return PTR_ERR(attr);
601 min_size = min_t(size_t, attr->ptr_attr.len, size);
602 if (copy_to_user(u64_to_user_ptr(attr->ptr_attr.data), from, min_size))
605 flags = pbundle->uattrs[attr->ptr_attr.uattr_idx].flags |
606 UVERBS_ATTR_F_VALID_OUTPUT;
608 &pbundle->user_attrs[attr->ptr_attr.uattr_idx].flags))
613 EXPORT_SYMBOL(uverbs_copy_to);