drivers/infiniband/core/uverbs_main.c

   1 /*
   2  * Copyright (c) 2005 Topspin Communications.  All rights reserved.
   3  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
   4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
   5  * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
   6  * Copyright (c) 2005 PathScale, Inc. All rights reserved.
   7  *
   8  * This software is available to you under a choice of one of two
   9  * licenses.  You may choose to be licensed under the terms of the GNU
  10  * General Public License (GPL) Version 2, available from the file
  11  * COPYING in the main directory of this source tree, or the
  12  * OpenIB.org BSD license below:
  13  *
  14  *     Redistribution and use in source and binary forms, with or
  15  *     without modification, are permitted provided that the following
  16  *     conditions are met:
  17  *
  18  *      - Redistributions of source code must retain the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer.
  21  *
  22  *      - Redistributions in binary form must reproduce the above
  23  *        copyright notice, this list of conditions and the following
  24  *        disclaimer in the documentation and/or other materials
  25  *        provided with the distribution.
  26  *
  27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34  * SOFTWARE.
  35  */
  36
  37 #include <linux/module.h>
  38 #include <linux/init.h>
  39 #include <linux/device.h>
  40 #include <linux/err.h>
  41 #include <linux/fs.h>
  42 #include <linux/poll.h>
  43 #include <linux/sched.h>
  44 #include <linux/file.h>
  45 #include <linux/cdev.h>
  46 #include <linux/anon_inodes.h>
  47 #include <linux/slab.h>
  48 #include <linux/sched/mm.h>
  49
  50 #include <linux/uaccess.h>
  51
  52 #include <rdma/ib.h>
  53 #include <rdma/uverbs_std_types.h>
  54 #include <rdma/rdma_netlink.h>
  55
  56 #include "uverbs.h"
  57 #include "core_priv.h"
  58 #include "rdma_core.h"
  59
  60 MODULE_AUTHOR("Roland Dreier");
  61 MODULE_DESCRIPTION("InfiniBand userspace verbs access");
  62 MODULE_LICENSE("Dual BSD/GPL");
  63
  64 enum {
  65         IB_UVERBS_MAJOR       = 231,
  66         IB_UVERBS_BASE_MINOR  = 192,
  67         IB_UVERBS_MAX_DEVICES = RDMA_MAX_PORTS,
  68         IB_UVERBS_NUM_FIXED_MINOR = 32,
  69         IB_UVERBS_NUM_DYNAMIC_MINOR = IB_UVERBS_MAX_DEVICES - IB_UVERBS_NUM_FIXED_MINOR,
  70 };
  71
  72 #define IB_UVERBS_BASE_DEV      MKDEV(IB_UVERBS_MAJOR, IB_UVERBS_BASE_MINOR)
  73
  74 static dev_t dynamic_uverbs_dev;
  75 static struct class *uverbs_class;
  76
  77 static DEFINE_IDA(uverbs_ida);
  78 static int ib_uverbs_add_one(struct ib_device *device);
  79 static void ib_uverbs_remove_one(struct ib_device *device, void *client_data);
  80
  81 /*
  82  * Must be called with the ufile->device->disassociate_srcu held, and the lock
  83  * must be held until use of the ucontext is finished.
  84  */
  85 struct ib_ucontext *ib_uverbs_get_ucontext_file(struct ib_uverbs_file *ufile)
  86 {
  87         /*
  88          * We do not hold the hw_destroy_rwsem lock for this flow, instead
  89          * srcu is used. It does not matter if someone races this with
  90          * get_context, we get NULL or valid ucontext.
  91          */
  92         struct ib_ucontext *ucontext = smp_load_acquire(&ufile->ucontext);
  93
  94         if (!srcu_dereference(ufile->device->ib_dev,
  95                               &ufile->device->disassociate_srcu))
  96                 return ERR_PTR(-EIO);
  97
  98         if (!ucontext)
  99                 return ERR_PTR(-EINVAL);
 100
 101         return ucontext;
 102 }
 103 EXPORT_SYMBOL(ib_uverbs_get_ucontext_file);
 104
 105 int uverbs_dealloc_mw(struct ib_mw *mw)
 106 {
 107         struct ib_pd *pd = mw->pd;
 108         int ret;
 109
 110         ret = mw->device->ops.dealloc_mw(mw);
 111         if (ret)
 112                 return ret;
 113
 114         atomic_dec(&pd->usecnt);
 115         kfree(mw);
 116         return ret;
 117 }
 118
 119 static void ib_uverbs_release_dev(struct device *device)
 120 {
 121         struct ib_uverbs_device *dev =
 122                         container_of(device, struct ib_uverbs_device, dev);
 123
 124         uverbs_destroy_api(dev->uapi);
 125         cleanup_srcu_struct(&dev->disassociate_srcu);
 126         mutex_destroy(&dev->lists_mutex);
 127         mutex_destroy(&dev->xrcd_tree_mutex);
 128         kfree(dev);
 129 }
 130
 131 void ib_uverbs_release_ucq(struct ib_uverbs_completion_event_file *ev_file,
 132                            struct ib_ucq_object *uobj)
 133 {
 134         struct ib_uverbs_event *evt, *tmp;
 135
 136         if (ev_file) {
 137                 spin_lock_irq(&ev_file->ev_queue.lock);
 138                 list_for_each_entry_safe(evt, tmp, &uobj->comp_list, obj_list) {
 139                         list_del(&evt->list);
 140                         kfree(evt);
 141                 }
 142                 spin_unlock_irq(&ev_file->ev_queue.lock);
 143
 144                 uverbs_uobject_put(&ev_file->uobj);
 145         }
 146
 147         ib_uverbs_release_uevent(&uobj->uevent);
 148 }
 149
 150 void ib_uverbs_release_uevent(struct ib_uevent_object *uobj)
 151 {
 152         struct ib_uverbs_async_event_file *async_file = uobj->event_file;
 153         struct ib_uverbs_event *evt, *tmp;
 154
 155         if (!async_file)
 156                 return;
 157
 158         spin_lock_irq(&async_file->ev_queue.lock);
 159         list_for_each_entry_safe(evt, tmp, &uobj->event_list, obj_list) {
 160                 list_del(&evt->list);
 161                 kfree(evt);
 162         }
 163         spin_unlock_irq(&async_file->ev_queue.lock);
 164         uverbs_uobject_put(&async_file->uobj);
 165 }
 166
 167 void ib_uverbs_detach_umcast(struct ib_qp *qp,
 168                              struct ib_uqp_object *uobj)
 169 {
 170         struct ib_uverbs_mcast_entry *mcast, *tmp;
 171
 172         list_for_each_entry_safe(mcast, tmp, &uobj->mcast_list, list) {
 173                 ib_detach_mcast(qp, &mcast->gid, mcast->lid);
 174                 list_del(&mcast->list);
 175                 kfree(mcast);
 176         }
 177 }
 178
 179 static void ib_uverbs_comp_dev(struct ib_uverbs_device *dev)
 180 {
 181         complete(&dev->comp);
 182 }
 183
 184 void ib_uverbs_release_file(struct kref *ref)
 185 {
 186         struct ib_uverbs_file *file =
 187                 container_of(ref, struct ib_uverbs_file, ref);
 188         struct ib_device *ib_dev;
 189         int srcu_key;
 190
 191         release_ufile_idr_uobject(file);
 192
 193         srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
 194         ib_dev = srcu_dereference(file->device->ib_dev,
 195                                   &file->device->disassociate_srcu);
 196         if (ib_dev && !ib_dev->ops.disassociate_ucontext)
 197                 module_put(ib_dev->ops.owner);
 198         srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
 199
 200         if (atomic_dec_and_test(&file->device->refcount))
 201                 ib_uverbs_comp_dev(file->device);
 202
 203         if (file->default_async_file)
 204                 uverbs_uobject_put(&file->default_async_file->uobj);
 205         put_device(&file->device->dev);
 206
 207         if (file->disassociate_page)
 208                 __free_pages(file->disassociate_page, 0);
 209         mutex_destroy(&file->umap_lock);
 210         mutex_destroy(&file->ucontext_lock);
 211         kfree(file);
 212 }
 213
 214 static ssize_t ib_uverbs_event_read(struct ib_uverbs_event_queue *ev_queue,
 215                                     struct file *filp, char __user *buf,
 216                                     size_t count, loff_t *pos,
 217                                     size_t eventsz)
 218 {
 219         struct ib_uverbs_event *event;
 220         int ret = 0;
 221
 222         spin_lock_irq(&ev_queue->lock);
 223
 224         while (list_empty(&ev_queue->event_list)) {
 225                 if (ev_queue->is_closed) {
 226                         spin_unlock_irq(&ev_queue->lock);
 227                         return -EIO;
 228                 }
 229
 230                 spin_unlock_irq(&ev_queue->lock);
 231                 if (filp->f_flags & O_NONBLOCK)
 232                         return -EAGAIN;
 233
 234                 if (wait_event_interruptible(ev_queue->poll_wait,
 235                                              (!list_empty(&ev_queue->event_list) ||
 236                                               ev_queue->is_closed)))
 237                         return -ERESTARTSYS;
 238
 239                 spin_lock_irq(&ev_queue->lock);
 240         }
 241
 242         event = list_entry(ev_queue->event_list.next, struct ib_uverbs_event, list);
 243
 244         if (eventsz > count) {
 245                 ret   = -EINVAL;
 246                 event = NULL;
 247         } else {
 248                 list_del(ev_queue->event_list.next);
 249                 if (event->counter) {
 250                         ++(*event->counter);
 251                         list_del(&event->obj_list);
 252                 }
 253         }
 254
 255         spin_unlock_irq(&ev_queue->lock);
 256
 257         if (event) {
 258                 if (copy_to_user(buf, event, eventsz))
 259                         ret = -EFAULT;
 260                 else
 261                         ret = eventsz;
 262         }
 263
 264         kfree(event);
 265
 266         return ret;
 267 }
 268
 269 static ssize_t ib_uverbs_async_event_read(struct file *filp, char __user *buf,
 270                                           size_t count, loff_t *pos)
 271 {
 272         struct ib_uverbs_async_event_file *file = filp->private_data;
 273
 274         return ib_uverbs_event_read(&file->ev_queue, filp, buf, count, pos,
 275                                     sizeof(struct ib_uverbs_async_event_desc));
 276 }
 277
 278 static ssize_t ib_uverbs_comp_event_read(struct file *filp, char __user *buf,
 279                                          size_t count, loff_t *pos)
 280 {
 281         struct ib_uverbs_completion_event_file *comp_ev_file =
 282                 filp->private_data;
 283
 284         return ib_uverbs_event_read(&comp_ev_file->ev_queue, filp, buf, count,
 285                                     pos,
 286                                     sizeof(struct ib_uverbs_comp_event_desc));
 287 }
 288
 289 static __poll_t ib_uverbs_event_poll(struct ib_uverbs_event_queue *ev_queue,
 290                                          struct file *filp,
 291                                          struct poll_table_struct *wait)
 292 {
 293         __poll_t pollflags = 0;
 294
 295         poll_wait(filp, &ev_queue->poll_wait, wait);
 296
 297         spin_lock_irq(&ev_queue->lock);
 298         if (!list_empty(&ev_queue->event_list))
 299                 pollflags = EPOLLIN | EPOLLRDNORM;
 300         else if (ev_queue->is_closed)
 301                 pollflags = EPOLLERR;
 302         spin_unlock_irq(&ev_queue->lock);
 303
 304         return pollflags;
 305 }
 306
 307 static __poll_t ib_uverbs_async_event_poll(struct file *filp,
 308                                                struct poll_table_struct *wait)
 309 {
 310         struct ib_uverbs_async_event_file *file = filp->private_data;
 311
 312         return ib_uverbs_event_poll(&file->ev_queue, filp, wait);
 313 }
 314
 315 static __poll_t ib_uverbs_comp_event_poll(struct file *filp,
 316                                               struct poll_table_struct *wait)
 317 {
 318         struct ib_uverbs_completion_event_file *comp_ev_file =
 319                 filp->private_data;
 320
 321         return ib_uverbs_event_poll(&comp_ev_file->ev_queue, filp, wait);
 322 }
 323
 324 static int ib_uverbs_async_event_fasync(int fd, struct file *filp, int on)
 325 {
 326         struct ib_uverbs_async_event_file *file = filp->private_data;
 327
 328         return fasync_helper(fd, filp, on, &file->ev_queue.async_queue);
 329 }
 330
 331 static int ib_uverbs_comp_event_fasync(int fd, struct file *filp, int on)
 332 {
 333         struct ib_uverbs_completion_event_file *comp_ev_file =
 334                 filp->private_data;
 335
 336         return fasync_helper(fd, filp, on, &comp_ev_file->ev_queue.async_queue);
 337 }
 338
 339 const struct file_operations uverbs_event_fops = {
 340         .owner   = THIS_MODULE,
 341         .read    = ib_uverbs_comp_event_read,
 342         .poll    = ib_uverbs_comp_event_poll,
 343         .release = uverbs_uobject_fd_release,
 344         .fasync  = ib_uverbs_comp_event_fasync,
 345         .llseek  = no_llseek,
 346 };
 347
 348 const struct file_operations uverbs_async_event_fops = {
 349         .owner   = THIS_MODULE,
 350         .read    = ib_uverbs_async_event_read,
 351         .poll    = ib_uverbs_async_event_poll,
 352         .release = uverbs_async_event_release,
 353         .fasync  = ib_uverbs_async_event_fasync,
 354         .llseek  = no_llseek,
 355 };
 356
 357 void ib_uverbs_comp_handler(struct ib_cq *cq, void *cq_context)
 358 {
 359         struct ib_uverbs_event_queue   *ev_queue = cq_context;
 360         struct ib_ucq_object           *uobj;
 361         struct ib_uverbs_event         *entry;
 362         unsigned long                   flags;
 363
 364         if (!ev_queue)
 365                 return;
 366
 367         spin_lock_irqsave(&ev_queue->lock, flags);
 368         if (ev_queue->is_closed) {
 369                 spin_unlock_irqrestore(&ev_queue->lock, flags);
 370                 return;
 371         }
 372
 373         entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
 374         if (!entry) {
 375                 spin_unlock_irqrestore(&ev_queue->lock, flags);
 376                 return;
 377         }
 378
 379         uobj = cq->uobject;
 380
 381         entry->desc.comp.cq_handle = cq->uobject->uevent.uobject.user_handle;
 382         entry->counter             = &uobj->comp_events_reported;
 383
 384         list_add_tail(&entry->list, &ev_queue->event_list);
 385         list_add_tail(&entry->obj_list, &uobj->comp_list);
 386         spin_unlock_irqrestore(&ev_queue->lock, flags);
 387
 388         wake_up_interruptible(&ev_queue->poll_wait);
 389         kill_fasync(&ev_queue->async_queue, SIGIO, POLL_IN);
 390 }
 391
 392 void ib_uverbs_async_handler(struct ib_uverbs_async_event_file *async_file,
 393                              __u64 element, __u64 event,
 394                              struct list_head *obj_list, u32 *counter)
 395 {
 396         struct ib_uverbs_event *entry;
 397         unsigned long flags;
 398
 399         if (!async_file)
 400                 return;
 401
 402         spin_lock_irqsave(&async_file->ev_queue.lock, flags);
 403         if (async_file->ev_queue.is_closed) {
 404                 spin_unlock_irqrestore(&async_file->ev_queue.lock, flags);
 405                 return;
 406         }
 407
 408         entry = kmalloc(sizeof(*entry), GFP_ATOMIC);
 409         if (!entry) {
 410                 spin_unlock_irqrestore(&async_file->ev_queue.lock, flags);
 411                 return;
 412         }
 413
 414         entry->desc.async.element = element;
 415         entry->desc.async.event_type = event;
 416         entry->desc.async.reserved = 0;
 417         entry->counter = counter;
 418
 419         list_add_tail(&entry->list, &async_file->ev_queue.event_list);
 420         if (obj_list)
 421                 list_add_tail(&entry->obj_list, obj_list);
 422         spin_unlock_irqrestore(&async_file->ev_queue.lock, flags);
 423
 424         wake_up_interruptible(&async_file->ev_queue.poll_wait);
 425         kill_fasync(&async_file->ev_queue.async_queue, SIGIO, POLL_IN);
 426 }
 427
 428 static void uverbs_uobj_event(struct ib_uevent_object *eobj,
 429                               struct ib_event *event)
 430 {
 431         ib_uverbs_async_handler(eobj->event_file,
 432                                 eobj->uobject.user_handle, event->event,
 433                                 &eobj->event_list, &eobj->events_reported);
 434 }
 435
 436 void ib_uverbs_cq_event_handler(struct ib_event *event, void *context_ptr)
 437 {
 438         uverbs_uobj_event(&event->element.cq->uobject->uevent, event);
 439 }
 440
 441 void ib_uverbs_qp_event_handler(struct ib_event *event, void *context_ptr)
 442 {
 443         /* for XRC target qp's, check that qp is live */
 444         if (!event->element.qp->uobject)
 445                 return;
 446
 447         uverbs_uobj_event(&event->element.qp->uobject->uevent, event);
 448 }
 449
 450 void ib_uverbs_wq_event_handler(struct ib_event *event, void *context_ptr)
 451 {
 452         uverbs_uobj_event(&event->element.wq->uobject->uevent, event);
 453 }
 454
 455 void ib_uverbs_srq_event_handler(struct ib_event *event, void *context_ptr)
 456 {
 457         uverbs_uobj_event(&event->element.srq->uobject->uevent, event);
 458 }
 459
 460 static void ib_uverbs_event_handler(struct ib_event_handler *handler,
 461                                     struct ib_event *event)
 462 {
 463         ib_uverbs_async_handler(
 464                 container_of(handler, struct ib_uverbs_async_event_file,
 465                              event_handler),
 466                 event->element.port_num, event->event, NULL, NULL);
 467 }
 468
 469 void ib_uverbs_init_event_queue(struct ib_uverbs_event_queue *ev_queue)
 470 {
 471         spin_lock_init(&ev_queue->lock);
 472         INIT_LIST_HEAD(&ev_queue->event_list);
 473         init_waitqueue_head(&ev_queue->poll_wait);
 474         ev_queue->is_closed   = 0;
 475         ev_queue->async_queue = NULL;
 476 }
 477
 478 void ib_uverbs_init_async_event_file(
 479         struct ib_uverbs_async_event_file *async_file)
 480 {
 481         struct ib_uverbs_file *uverbs_file = async_file->uobj.ufile;
 482         struct ib_device *ib_dev = async_file->uobj.context->device;
 483
 484         ib_uverbs_init_event_queue(&async_file->ev_queue);
 485
 486         /* The first async_event_file becomes the default one for the file. */
 487         mutex_lock(&uverbs_file->ucontext_lock);
 488         if (!uverbs_file->default_async_file) {
 489                 /* Pairs with the put in ib_uverbs_release_file */
 490                 uverbs_uobject_get(&async_file->uobj);
 491                 smp_store_release(&uverbs_file->default_async_file, async_file);
 492         }
 493         mutex_unlock(&uverbs_file->ucontext_lock);
 494
 495         INIT_IB_EVENT_HANDLER(&async_file->event_handler, ib_dev,
 496                               ib_uverbs_event_handler);
 497         ib_register_event_handler(&async_file->event_handler);
 498 }
 499
 500 static ssize_t verify_hdr(struct ib_uverbs_cmd_hdr *hdr,
 501                           struct ib_uverbs_ex_cmd_hdr *ex_hdr, size_t count,
 502                           const struct uverbs_api_write_method *method_elm)
 503 {
 504         if (method_elm->is_ex) {
 505                 count -= sizeof(*hdr) + sizeof(*ex_hdr);
 506
 507                 if ((hdr->in_words + ex_hdr->provider_in_words) * 8 != count)
 508                         return -EINVAL;
 509
 510                 if (hdr->in_words * 8 < method_elm->req_size)
 511                         return -ENOSPC;
 512
 513                 if (ex_hdr->cmd_hdr_reserved)
 514                         return -EINVAL;
 515
 516                 if (ex_hdr->response) {
 517                         if (!hdr->out_words && !ex_hdr->provider_out_words)
 518                                 return -EINVAL;
 519
 520                         if (hdr->out_words * 8 < method_elm->resp_size)
 521                                 return -ENOSPC;
 522
 523                         if (!access_ok(u64_to_user_ptr(ex_hdr->response),
 524                                        (hdr->out_words + ex_hdr->provider_out_words) * 8))
 525                                 return -EFAULT;
 526                 } else {
 527                         if (hdr->out_words || ex_hdr->provider_out_words)
 528                                 return -EINVAL;
 529                 }
 530
 531                 return 0;
 532         }
 533
 534         /* not extended command */
 535         if (hdr->in_words * 4 != count)
 536                 return -EINVAL;
 537
 538         if (count < method_elm->req_size + sizeof(*hdr)) {
 539                 /*
 540                  * rdma-core v18 and v19 have a bug where they send DESTROY_CQ
 541                  * with a 16 byte write instead of 24. Old kernels didn't
 542                  * check the size so they allowed this. Now that the size is
 543                  * checked provide a compatibility work around to not break
 544                  * those userspaces.
 545                  */
 546                 if (hdr->command == IB_USER_VERBS_CMD_DESTROY_CQ &&
 547                     count == 16) {
 548                         hdr->in_words = 6;
 549                         return 0;
 550                 }
 551                 return -ENOSPC;
 552         }
 553         if (hdr->out_words * 4 < method_elm->resp_size)
 554                 return -ENOSPC;
 555
 556         return 0;
 557 }
 558
 559 static ssize_t ib_uverbs_write(struct file *filp, const char __user *buf,
 560                              size_t count, loff_t *pos)
 561 {
 562         struct ib_uverbs_file *file = filp->private_data;
 563         const struct uverbs_api_write_method *method_elm;
 564         struct uverbs_api *uapi = file->device->uapi;
 565         struct ib_uverbs_ex_cmd_hdr ex_hdr;
 566         struct ib_uverbs_cmd_hdr hdr;
 567         struct uverbs_attr_bundle bundle;
 568         int srcu_key;
 569         ssize_t ret;
 570
 571         if (!ib_safe_file_access(filp)) {
 572                 pr_err_once("uverbs_write: process %d (%s) changed security contexts after opening file descriptor, this is not allowed.\n",
 573                             task_tgid_vnr(current), current->comm);
 574                 return -EACCES;
 575         }
 576
 577         if (count < sizeof(hdr))
 578                 return -EINVAL;
 579
 580         if (copy_from_user(&hdr, buf, sizeof(hdr)))
 581                 return -EFAULT;
 582
 583         method_elm = uapi_get_method(uapi, hdr.command);
 584         if (IS_ERR(method_elm))
 585                 return PTR_ERR(method_elm);
 586
 587         if (method_elm->is_ex) {
 588                 if (count < (sizeof(hdr) + sizeof(ex_hdr)))
 589                         return -EINVAL;
 590                 if (copy_from_user(&ex_hdr, buf + sizeof(hdr), sizeof(ex_hdr)))
 591                         return -EFAULT;
 592         }
 593
 594         ret = verify_hdr(&hdr, &ex_hdr, count, method_elm);
 595         if (ret)
 596                 return ret;
 597
 598         srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
 599
 600         buf += sizeof(hdr);
 601
 602         memset(bundle.attr_present, 0, sizeof(bundle.attr_present));
 603         bundle.ufile = file;
 604         bundle.context = NULL; /* only valid if bundle has uobject */
 605         bundle.uobject = NULL;
 606         if (!method_elm->is_ex) {
 607                 size_t in_len = hdr.in_words * 4 - sizeof(hdr);
 608                 size_t out_len = hdr.out_words * 4;
 609                 u64 response = 0;
 610
 611                 if (method_elm->has_udata) {
 612                         bundle.driver_udata.inlen =
 613                                 in_len - method_elm->req_size;
 614                         in_len = method_elm->req_size;
 615                         if (bundle.driver_udata.inlen)
 616                                 bundle.driver_udata.inbuf = buf + in_len;
 617                         else
 618                                 bundle.driver_udata.inbuf = NULL;
 619                 } else {
 620                         memset(&bundle.driver_udata, 0,
 621                                sizeof(bundle.driver_udata));
 622                 }
 623
 624                 if (method_elm->has_resp) {
 625                         /*
 626                          * The macros check that if has_resp is set
 627                          * then the command request structure starts
 628                          * with a '__aligned u64 response' member.
 629                          */
 630                         ret = get_user(response, (const u64 __user *)buf);
 631                         if (ret)
 632                                 goto out_unlock;
 633
 634                         if (method_elm->has_udata) {
 635                                 bundle.driver_udata.outlen =
 636                                         out_len - method_elm->resp_size;
 637                                 out_len = method_elm->resp_size;
 638                                 if (bundle.driver_udata.outlen)
 639                                         bundle.driver_udata.outbuf =
 640                                                 u64_to_user_ptr(response +
 641                                                                 out_len);
 642                                 else
 643                                         bundle.driver_udata.outbuf = NULL;
 644                         }
 645                 } else {
 646                         bundle.driver_udata.outlen = 0;
 647                         bundle.driver_udata.outbuf = NULL;
 648                 }
 649
 650                 ib_uverbs_init_udata_buf_or_null(
 651                         &bundle.ucore, buf, u64_to_user_ptr(response),
 652                         in_len, out_len);
 653         } else {
 654                 buf += sizeof(ex_hdr);
 655
 656                 ib_uverbs_init_udata_buf_or_null(&bundle.ucore, buf,
 657                                         u64_to_user_ptr(ex_hdr.response),
 658                                         hdr.in_words * 8, hdr.out_words * 8);
 659
 660                 ib_uverbs_init_udata_buf_or_null(
 661                         &bundle.driver_udata, buf + bundle.ucore.inlen,
 662                         u64_to_user_ptr(ex_hdr.response) + bundle.ucore.outlen,
 663                         ex_hdr.provider_in_words * 8,
 664                         ex_hdr.provider_out_words * 8);
 665
 666         }
 667
 668         ret = method_elm->handler(&bundle);
 669         if (bundle.uobject)
 670                 uverbs_finalize_object(bundle.uobject, UVERBS_ACCESS_NEW, true,
 671                                        !ret, &bundle);
 672 out_unlock:
 673         srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
 674         return (ret) ? : count;
 675 }
 676
 677 static const struct vm_operations_struct rdma_umap_ops;
 678
 679 static int ib_uverbs_mmap(struct file *filp, struct vm_area_struct *vma)
 680 {
 681         struct ib_uverbs_file *file = filp->private_data;
 682         struct ib_ucontext *ucontext;
 683         int ret = 0;
 684         int srcu_key;
 685
 686         srcu_key = srcu_read_lock(&file->device->disassociate_srcu);
 687         ucontext = ib_uverbs_get_ucontext_file(file);
 688         if (IS_ERR(ucontext)) {
 689                 ret = PTR_ERR(ucontext);
 690                 goto out;
 691         }
 692         vma->vm_ops = &rdma_umap_ops;
 693         ret = ucontext->device->ops.mmap(ucontext, vma);
 694 out:
 695         srcu_read_unlock(&file->device->disassociate_srcu, srcu_key);
 696         return ret;
 697 }
 698
 699 /*
 700  * The VMA has been dup'd, initialize the vm_private_data with a new tracking
 701  * struct
 702  */
 703 static void rdma_umap_open(struct vm_area_struct *vma)
 704 {
 705         struct ib_uverbs_file *ufile = vma->vm_file->private_data;
 706         struct rdma_umap_priv *opriv = vma->vm_private_data;
 707         struct rdma_umap_priv *priv;
 708
 709         if (!opriv)
 710                 return;
 711
 712         /* We are racing with disassociation */
 713         if (!down_read_trylock(&ufile->hw_destroy_rwsem))
 714                 goto out_zap;
 715         /*
 716          * Disassociation already completed, the VMA should already be zapped.
 717          */
 718         if (!ufile->ucontext)
 719                 goto out_unlock;
 720
 721         priv = kzalloc(sizeof(*priv), GFP_KERNEL);
 722         if (!priv)
 723                 goto out_unlock;
 724         rdma_umap_priv_init(priv, vma, opriv->entry);
 725
 726         up_read(&ufile->hw_destroy_rwsem);
 727         return;
 728
 729 out_unlock:
 730         up_read(&ufile->hw_destroy_rwsem);
 731 out_zap:
 732         /*
 733          * We can't allow the VMA to be created with the actual IO pages, that
 734          * would break our API contract, and it can't be stopped at this
 735          * point, so zap it.
 736          */
 737         vma->vm_private_data = NULL;
 738         zap_vma_ptes(vma, vma->vm_start, vma->vm_end - vma->vm_start);
 739 }
 740
 741 static void rdma_umap_close(struct vm_area_struct *vma)
 742 {
 743         struct ib_uverbs_file *ufile = vma->vm_file->private_data;
 744         struct rdma_umap_priv *priv = vma->vm_private_data;
 745
 746         if (!priv)
 747                 return;
 748
 749         /*
 750          * The vma holds a reference on the struct file that created it, which
 751          * in turn means that the ib_uverbs_file is guaranteed to exist at
 752          * this point.
 753          */
 754         mutex_lock(&ufile->umap_lock);
 755         if (priv->entry)
 756                 rdma_user_mmap_entry_put(priv->entry);
 757
 758         list_del(&priv->list);
 759         mutex_unlock(&ufile->umap_lock);
 760         kfree(priv);
 761 }
 762
 763 /*
 764  * Once the zap_vma_ptes has been called touches to the VMA will come here and
 765  * we return a dummy writable zero page for all the pfns.
 766  */
 767 static vm_fault_t rdma_umap_fault(struct vm_fault *vmf)
 768 {
 769         struct ib_uverbs_file *ufile = vmf->vma->vm_file->private_data;
 770         struct rdma_umap_priv *priv = vmf->vma->vm_private_data;
 771         vm_fault_t ret = 0;
 772
 773         if (!priv)
 774                 return VM_FAULT_SIGBUS;
 775
 776         /* Read only pages can just use the system zero page. */
 777         if (!(vmf->vma->vm_flags & (VM_WRITE | VM_MAYWRITE))) {
 778                 vmf->page = ZERO_PAGE(vmf->address);
 779                 get_page(vmf->page);
 780                 return 0;
 781         }
 782
 783         mutex_lock(&ufile->umap_lock);
 784         if (!ufile->disassociate_page)
 785                 ufile->disassociate_page =
 786                         alloc_pages(vmf->gfp_mask | __GFP_ZERO, 0);
 787
 788         if (ufile->disassociate_page) {
 789                 /*
 790                  * This VMA is forced to always be shared so this doesn't have
 791                  * to worry about COW.
 792                  */
 793                 vmf->page = ufile->disassociate_page;
 794                 get_page(vmf->page);
 795         } else {
 796                 ret = VM_FAULT_SIGBUS;
 797         }
 798         mutex_unlock(&ufile->umap_lock);
 799
 800         return ret;
 801 }
 802
 803 static const struct vm_operations_struct rdma_umap_ops = {
 804         .open = rdma_umap_open,
 805         .close = rdma_umap_close,
 806         .fault = rdma_umap_fault,
 807 };
 808
 809 void uverbs_user_mmap_disassociate(struct ib_uverbs_file *ufile)
 810 {
 811         struct rdma_umap_priv *priv, *next_priv;
 812
 813         lockdep_assert_held(&ufile->hw_destroy_rwsem);
 814
 815         while (1) {
 816                 struct mm_struct *mm = NULL;
 817
 818                 /* Get an arbitrary mm pointer that hasn't been cleaned yet */
 819                 mutex_lock(&ufile->umap_lock);
 820                 while (!list_empty(&ufile->umaps)) {
 821                         int ret;
 822
 823                         priv = list_first_entry(&ufile->umaps,
 824                                                 struct rdma_umap_priv, list);
 825                         mm = priv->vma->vm_mm;
 826                         ret = mmget_not_zero(mm);
 827                         if (!ret) {
 828                                 list_del_init(&priv->list);
 829                                 if (priv->entry) {
 830                                         rdma_user_mmap_entry_put(priv->entry);
 831                                         priv->entry = NULL;
 832                                 }
 833                                 mm = NULL;
 834                                 continue;
 835                         }
 836                         break;
 837                 }
 838                 mutex_unlock(&ufile->umap_lock);
 839                 if (!mm)
 840                         return;
 841
 842                 /*
 843                  * The umap_lock is nested under mmap_lock since it used within
 844                  * the vma_ops callbacks, so we have to clean the list one mm
 845                  * at a time to get the lock ordering right. Typically there
 846                  * will only be one mm, so no big deal.
 847                  */
 848                 mmap_read_lock(mm);
 849                 mutex_lock(&ufile->umap_lock);
 850                 list_for_each_entry_safe (priv, next_priv, &ufile->umaps,
 851                                           list) {
 852                         struct vm_area_struct *vma = priv->vma;
 853
 854                         if (vma->vm_mm != mm)
 855                                 continue;
 856                         list_del_init(&priv->list);
 857
 858                         zap_vma_ptes(vma, vma->vm_start,
 859                                      vma->vm_end - vma->vm_start);
 860
 861                         if (priv->entry) {
 862                                 rdma_user_mmap_entry_put(priv->entry);
 863                                 priv->entry = NULL;
 864                         }
 865                 }
 866                 mutex_unlock(&ufile->umap_lock);
 867                 mmap_read_unlock(mm);
 868                 mmput(mm);
 869         }
 870 }
 871
 872 /*
 873  * ib_uverbs_open() does not need the BKL:
 874  *
 875  *  - the ib_uverbs_device structures are properly reference counted and
 876  *    everything else is purely local to the file being created, so
 877  *    races against other open calls are not a problem;
 878  *  - there is no ioctl method to race against;
 879  *  - the open method will either immediately run -ENXIO, or all
 880  *    required initialization will be done.
 881  */
 882 static int ib_uverbs_open(struct inode *inode, struct file *filp)
 883 {
 884         struct ib_uverbs_device *dev;
 885         struct ib_uverbs_file *file;
 886         struct ib_device *ib_dev;
 887         int ret;
 888         int module_dependent;
 889         int srcu_key;
 890
 891         dev = container_of(inode->i_cdev, struct ib_uverbs_device, cdev);
 892         if (!atomic_inc_not_zero(&dev->refcount))
 893                 return -ENXIO;
 894
 895         get_device(&dev->dev);
 896         srcu_key = srcu_read_lock(&dev->disassociate_srcu);
 897         mutex_lock(&dev->lists_mutex);
 898         ib_dev = srcu_dereference(dev->ib_dev,
 899                                   &dev->disassociate_srcu);
 900         if (!ib_dev) {
 901                 ret = -EIO;
 902                 goto err;
 903         }
 904
 905         if (!rdma_dev_access_netns(ib_dev, current->nsproxy->net_ns)) {
 906                 ret = -EPERM;
 907                 goto err;
 908         }
 909
 910         /* In case IB device supports disassociate ucontext, there is no hard
 911          * dependency between uverbs device and its low level device.
 912          */
 913         module_dependent = !(ib_dev->ops.disassociate_ucontext);
 914
 915         if (module_dependent) {
 916                 if (!try_module_get(ib_dev->ops.owner)) {
 917                         ret = -ENODEV;
 918                         goto err;
 919                 }
 920         }
 921
 922         file = kzalloc(sizeof(*file), GFP_KERNEL);
 923         if (!file) {
 924                 ret = -ENOMEM;
 925                 if (module_dependent)
 926                         goto err_module;
 927
 928                 goto err;
 929         }
 930
 931         file->device     = dev;
 932         kref_init(&file->ref);
 933         mutex_init(&file->ucontext_lock);
 934
 935         spin_lock_init(&file->uobjects_lock);
 936         INIT_LIST_HEAD(&file->uobjects);
 937         init_rwsem(&file->hw_destroy_rwsem);
 938         mutex_init(&file->umap_lock);
 939         INIT_LIST_HEAD(&file->umaps);
 940
 941         filp->private_data = file;
 942         list_add_tail(&file->list, &dev->uverbs_file_list);
 943         mutex_unlock(&dev->lists_mutex);
 944         srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
 945
 946         setup_ufile_idr_uobject(file);
 947
 948         return stream_open(inode, filp);
 949
 950 err_module:
 951         module_put(ib_dev->ops.owner);
 952
 953 err:
 954         mutex_unlock(&dev->lists_mutex);
 955         srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
 956         if (atomic_dec_and_test(&dev->refcount))
 957                 ib_uverbs_comp_dev(dev);
 958
 959         put_device(&dev->dev);
 960         return ret;
 961 }
 962
 963 static int ib_uverbs_close(struct inode *inode, struct file *filp)
 964 {
 965         struct ib_uverbs_file *file = filp->private_data;
 966
 967         uverbs_destroy_ufile_hw(file, RDMA_REMOVE_CLOSE);
 968
 969         mutex_lock(&file->device->lists_mutex);
 970         list_del_init(&file->list);
 971         mutex_unlock(&file->device->lists_mutex);
 972
 973         kref_put(&file->ref, ib_uverbs_release_file);
 974
 975         return 0;
 976 }
 977
 978 static const struct file_operations uverbs_fops = {
 979         .owner   = THIS_MODULE,
 980         .write   = ib_uverbs_write,
 981         .open    = ib_uverbs_open,
 982         .release = ib_uverbs_close,
 983         .llseek  = no_llseek,
 984         .unlocked_ioctl = ib_uverbs_ioctl,
 985         .compat_ioctl = compat_ptr_ioctl,
 986 };
 987
 988 static const struct file_operations uverbs_mmap_fops = {
 989         .owner   = THIS_MODULE,
 990         .write   = ib_uverbs_write,
 991         .mmap    = ib_uverbs_mmap,
 992         .open    = ib_uverbs_open,
 993         .release = ib_uverbs_close,
 994         .llseek  = no_llseek,
 995         .unlocked_ioctl = ib_uverbs_ioctl,
 996         .compat_ioctl = compat_ptr_ioctl,
 997 };
 998
 999 static int ib_uverbs_get_nl_info(struct ib_device *ibdev, void *client_data,
1000                                  struct ib_client_nl_info *res)
1001 {
1002         struct ib_uverbs_device *uverbs_dev = client_data;
1003         int ret;
1004
1005         if (res->port != -1)
1006                 return -EINVAL;
1007
1008         res->abi = ibdev->ops.uverbs_abi_ver;
1009         res->cdev = &uverbs_dev->dev;
1010
1011         /*
1012          * To support DRIVER_ID binding in userspace some of the driver need
1013          * upgrading to expose their PCI dependent revision information
1014          * through get_context instead of relying on modalias matching. When
1015          * the drivers are fixed they can drop this flag.
1016          */
1017         if (!ibdev->ops.uverbs_no_driver_id_binding) {
1018                 ret = nla_put_u32(res->nl_msg, RDMA_NLDEV_ATTR_UVERBS_DRIVER_ID,
1019                                   ibdev->ops.driver_id);
1020                 if (ret)
1021                         return ret;
1022         }
1023         return 0;
1024 }
1025
1026 static struct ib_client uverbs_client = {
1027         .name   = "uverbs",
1028         .no_kverbs_req = true,
1029         .add    = ib_uverbs_add_one,
1030         .remove = ib_uverbs_remove_one,
1031         .get_nl_info = ib_uverbs_get_nl_info,
1032 };
1033 MODULE_ALIAS_RDMA_CLIENT("uverbs");
1034
1035 static ssize_t ibdev_show(struct device *device, struct device_attribute *attr,
1036                           char *buf)
1037 {
1038         struct ib_uverbs_device *dev =
1039                         container_of(device, struct ib_uverbs_device, dev);
1040         int ret = -ENODEV;
1041         int srcu_key;
1042         struct ib_device *ib_dev;
1043
1044         srcu_key = srcu_read_lock(&dev->disassociate_srcu);
1045         ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
1046         if (ib_dev)
1047                 ret = sprintf(buf, "%s\n", dev_name(&ib_dev->dev));
1048         srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
1049
1050         return ret;
1051 }
1052 static DEVICE_ATTR_RO(ibdev);
1053
1054 static ssize_t abi_version_show(struct device *device,
1055                                 struct device_attribute *attr, char *buf)
1056 {
1057         struct ib_uverbs_device *dev =
1058                         container_of(device, struct ib_uverbs_device, dev);
1059         int ret = -ENODEV;
1060         int srcu_key;
1061         struct ib_device *ib_dev;
1062
1063         srcu_key = srcu_read_lock(&dev->disassociate_srcu);
1064         ib_dev = srcu_dereference(dev->ib_dev, &dev->disassociate_srcu);
1065         if (ib_dev)
1066                 ret = sprintf(buf, "%u\n", ib_dev->ops.uverbs_abi_ver);
1067         srcu_read_unlock(&dev->disassociate_srcu, srcu_key);
1068
1069         return ret;
1070 }
1071 static DEVICE_ATTR_RO(abi_version);
1072
1073 static struct attribute *ib_dev_attrs[] = {
1074         &dev_attr_abi_version.attr,
1075         &dev_attr_ibdev.attr,
1076         NULL,
1077 };
1078
1079 static const struct attribute_group dev_attr_group = {
1080         .attrs = ib_dev_attrs,
1081 };
1082
1083 static CLASS_ATTR_STRING(abi_version, S_IRUGO,
1084                          __stringify(IB_USER_VERBS_ABI_VERSION));
1085
1086 static int ib_uverbs_create_uapi(struct ib_device *device,
1087                                  struct ib_uverbs_device *uverbs_dev)
1088 {
1089         struct uverbs_api *uapi;
1090
1091         uapi = uverbs_alloc_api(device);
1092         if (IS_ERR(uapi))
1093                 return PTR_ERR(uapi);
1094
1095         uverbs_dev->uapi = uapi;
1096         return 0;
1097 }
1098
1099 static int ib_uverbs_add_one(struct ib_device *device)
1100 {
1101         int devnum;
1102         dev_t base;
1103         struct ib_uverbs_device *uverbs_dev;
1104         int ret;
1105
1106         if (!device->ops.alloc_ucontext)
1107                 return -EOPNOTSUPP;
1108
1109         uverbs_dev = kzalloc(sizeof(*uverbs_dev), GFP_KERNEL);
1110         if (!uverbs_dev)
1111                 return -ENOMEM;
1112
1113         ret = init_srcu_struct(&uverbs_dev->disassociate_srcu);
1114         if (ret) {
1115                 kfree(uverbs_dev);
1116                 return -ENOMEM;
1117         }
1118
1119         device_initialize(&uverbs_dev->dev);
1120         uverbs_dev->dev.class = uverbs_class;
1121         uverbs_dev->dev.parent = device->dev.parent;
1122         uverbs_dev->dev.release = ib_uverbs_release_dev;
1123         uverbs_dev->groups[0] = &dev_attr_group;
1124         uverbs_dev->dev.groups = uverbs_dev->groups;
1125         atomic_set(&uverbs_dev->refcount, 1);
1126         init_completion(&uverbs_dev->comp);
1127         uverbs_dev->xrcd_tree = RB_ROOT;
1128         mutex_init(&uverbs_dev->xrcd_tree_mutex);
1129         mutex_init(&uverbs_dev->lists_mutex);
1130         INIT_LIST_HEAD(&uverbs_dev->uverbs_file_list);
1131         rcu_assign_pointer(uverbs_dev->ib_dev, device);
1132         uverbs_dev->num_comp_vectors = device->num_comp_vectors;
1133
1134         devnum = ida_alloc_max(&uverbs_ida, IB_UVERBS_MAX_DEVICES - 1,
1135                                GFP_KERNEL);
1136         if (devnum < 0) {
1137                 ret = -ENOMEM;
1138                 goto err;
1139         }
1140         uverbs_dev->devnum = devnum;
1141         if (devnum >= IB_UVERBS_NUM_FIXED_MINOR)
1142                 base = dynamic_uverbs_dev + devnum - IB_UVERBS_NUM_FIXED_MINOR;
1143         else
1144                 base = IB_UVERBS_BASE_DEV + devnum;
1145
1146         ret = ib_uverbs_create_uapi(device, uverbs_dev);
1147         if (ret)
1148                 goto err_uapi;
1149
1150         uverbs_dev->dev.devt = base;
1151         dev_set_name(&uverbs_dev->dev, "uverbs%d", uverbs_dev->devnum);
1152
1153         cdev_init(&uverbs_dev->cdev,
1154                   device->ops.mmap ? &uverbs_mmap_fops : &uverbs_fops);
1155         uverbs_dev->cdev.owner = THIS_MODULE;
1156
1157         ret = cdev_device_add(&uverbs_dev->cdev, &uverbs_dev->dev);
1158         if (ret)
1159                 goto err_uapi;
1160
1161         ib_set_client_data(device, &uverbs_client, uverbs_dev);
1162         return 0;
1163
1164 err_uapi:
1165         ida_free(&uverbs_ida, devnum);
1166 err:
1167         if (atomic_dec_and_test(&uverbs_dev->refcount))
1168                 ib_uverbs_comp_dev(uverbs_dev);
1169         wait_for_completion(&uverbs_dev->comp);
1170         put_device(&uverbs_dev->dev);
1171         return ret;
1172 }
1173
1174 static void ib_uverbs_free_hw_resources(struct ib_uverbs_device *uverbs_dev,
1175                                         struct ib_device *ib_dev)
1176 {
1177         struct ib_uverbs_file *file;
1178
1179         /* Pending running commands to terminate */
1180         uverbs_disassociate_api_pre(uverbs_dev);
1181
1182         mutex_lock(&uverbs_dev->lists_mutex);
1183         while (!list_empty(&uverbs_dev->uverbs_file_list)) {
1184                 file = list_first_entry(&uverbs_dev->uverbs_file_list,
1185                                         struct ib_uverbs_file, list);
1186                 list_del_init(&file->list);
1187                 kref_get(&file->ref);
1188
1189                 /* We must release the mutex before going ahead and calling
1190                  * uverbs_cleanup_ufile, as it might end up indirectly calling
1191                  * uverbs_close, for example due to freeing the resources (e.g
1192                  * mmput).
1193                  */
1194                 mutex_unlock(&uverbs_dev->lists_mutex);
1195
1196                 uverbs_destroy_ufile_hw(file, RDMA_REMOVE_DRIVER_REMOVE);
1197                 kref_put(&file->ref, ib_uverbs_release_file);
1198
1199                 mutex_lock(&uverbs_dev->lists_mutex);
1200         }
1201         mutex_unlock(&uverbs_dev->lists_mutex);
1202
1203         uverbs_disassociate_api(uverbs_dev->uapi);
1204 }
1205
1206 static void ib_uverbs_remove_one(struct ib_device *device, void *client_data)
1207 {
1208         struct ib_uverbs_device *uverbs_dev = client_data;
1209         int wait_clients = 1;
1210
1211         cdev_device_del(&uverbs_dev->cdev, &uverbs_dev->dev);
1212         ida_free(&uverbs_ida, uverbs_dev->devnum);
1213
1214         if (device->ops.disassociate_ucontext) {
1215                 /* We disassociate HW resources and immediately return.
1216                  * Userspace will see a EIO errno for all future access.
1217                  * Upon returning, ib_device may be freed internally and is not
1218                  * valid any more.
1219                  * uverbs_device is still available until all clients close
1220                  * their files, then the uverbs device ref count will be zero
1221                  * and its resources will be freed.
1222                  * Note: At this point no more files can be opened since the
1223                  * cdev was deleted, however active clients can still issue
1224                  * commands and close their open files.
1225                  */
1226                 ib_uverbs_free_hw_resources(uverbs_dev, device);
1227                 wait_clients = 0;
1228         }
1229
1230         if (atomic_dec_and_test(&uverbs_dev->refcount))
1231                 ib_uverbs_comp_dev(uverbs_dev);
1232         if (wait_clients)
1233                 wait_for_completion(&uverbs_dev->comp);
1234
1235         put_device(&uverbs_dev->dev);
1236 }
1237
1238 static char *uverbs_devnode(struct device *dev, umode_t *mode)
1239 {
1240         if (mode)
1241                 *mode = 0666;
1242         return kasprintf(GFP_KERNEL, "infiniband/%s", dev_name(dev));
1243 }
1244
1245 static int __init ib_uverbs_init(void)
1246 {
1247         int ret;
1248
1249         ret = register_chrdev_region(IB_UVERBS_BASE_DEV,
1250                                      IB_UVERBS_NUM_FIXED_MINOR,
1251                                      "infiniband_verbs");
1252         if (ret) {
1253                 pr_err("user_verbs: couldn't register device number\n");
1254                 goto out;
1255         }
1256
1257         ret = alloc_chrdev_region(&dynamic_uverbs_dev, 0,
1258                                   IB_UVERBS_NUM_DYNAMIC_MINOR,
1259                                   "infiniband_verbs");
1260         if (ret) {
1261                 pr_err("couldn't register dynamic device number\n");
1262                 goto out_alloc;
1263         }
1264
1265         uverbs_class = class_create(THIS_MODULE, "infiniband_verbs");
1266         if (IS_ERR(uverbs_class)) {
1267                 ret = PTR_ERR(uverbs_class);
1268                 pr_err("user_verbs: couldn't create class infiniband_verbs\n");
1269                 goto out_chrdev;
1270         }
1271
1272         uverbs_class->devnode = uverbs_devnode;
1273
1274         ret = class_create_file(uverbs_class, &class_attr_abi_version.attr);
1275         if (ret) {
1276                 pr_err("user_verbs: couldn't create abi_version attribute\n");
1277                 goto out_class;
1278         }
1279
1280         ret = ib_register_client(&uverbs_client);
1281         if (ret) {
1282                 pr_err("user_verbs: couldn't register client\n");
1283                 goto out_class;
1284         }
1285
1286         return 0;
1287
1288 out_class:
1289         class_destroy(uverbs_class);
1290
1291 out_chrdev:
1292         unregister_chrdev_region(dynamic_uverbs_dev,
1293                                  IB_UVERBS_NUM_DYNAMIC_MINOR);
1294
1295 out_alloc:
1296         unregister_chrdev_region(IB_UVERBS_BASE_DEV,
1297                                  IB_UVERBS_NUM_FIXED_MINOR);
1298
1299 out:
1300         return ret;
1301 }
1302
1303 static void __exit ib_uverbs_cleanup(void)
1304 {
1305         ib_unregister_client(&uverbs_client);
1306         class_destroy(uverbs_class);
1307         unregister_chrdev_region(IB_UVERBS_BASE_DEV,
1308                                  IB_UVERBS_NUM_FIXED_MINOR);
1309         unregister_chrdev_region(dynamic_uverbs_dev,
1310                                  IB_UVERBS_NUM_DYNAMIC_MINOR);
1311         mmu_notifier_synchronize();
1312 }
1313
1314 module_init(ib_uverbs_init);
1315 module_exit(ib_uverbs_cleanup);