drivers/block/rnbd/rnbd-clt-sysfs.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * RDMA Network Block Driver
   4  *
   5  * Copyright (c) 2014 - 2018 ProfitBricks GmbH. All rights reserved.
   6  * Copyright (c) 2018 - 2019 1&1 IONOS Cloud GmbH. All rights reserved.
   7  * Copyright (c) 2019 - 2020 1&1 IONOS SE. All rights reserved.
   8  */
   9
  10 #undef pr_fmt
  11 #define pr_fmt(fmt) KBUILD_MODNAME " L" __stringify(__LINE__) ": " fmt
  12
  13 #include <linux/types.h>
  14 #include <linux/ctype.h>
  15 #include <linux/parser.h>
  16 #include <linux/module.h>
  17 #include <linux/in6.h>
  18 #include <linux/fs.h>
  19 #include <linux/uaccess.h>
  20 #include <linux/device.h>
  21 #include <rdma/ib.h>
  22 #include <rdma/rdma_cm.h>
  23
  24 #include "rnbd-clt.h"
  25
  26 static struct device *rnbd_dev;
  27 static struct class *rnbd_dev_class;
  28 static struct kobject *rnbd_devs_kobj;
  29
  30 enum {
  31         RNBD_OPT_ERR            = 0,
  32         RNBD_OPT_DEST_PORT      = 1 << 0,
  33         RNBD_OPT_PATH           = 1 << 1,
  34         RNBD_OPT_DEV_PATH       = 1 << 2,
  35         RNBD_OPT_ACCESS_MODE    = 1 << 3,
  36         RNBD_OPT_SESSNAME       = 1 << 6,
  37         RNBD_OPT_NR_POLL_QUEUES = 1 << 7,
  38 };
  39
  40 static const unsigned int rnbd_opt_mandatory[] = {
  41         RNBD_OPT_DEV_PATH,
  42         RNBD_OPT_SESSNAME,
  43 };
  44
  45 static const match_table_t rnbd_opt_tokens = {
  46         {RNBD_OPT_PATH,                 "path=%s"               },
  47         {RNBD_OPT_DEV_PATH,             "device_path=%s"        },
  48         {RNBD_OPT_DEST_PORT,            "dest_port=%d"          },
  49         {RNBD_OPT_ACCESS_MODE,          "access_mode=%s"        },
  50         {RNBD_OPT_SESSNAME,             "sessname=%s"           },
  51         {RNBD_OPT_NR_POLL_QUEUES,       "nr_poll_queues=%d"     },
  52         {RNBD_OPT_ERR,                  NULL                    },
  53 };
  54
  55 struct rnbd_map_options {
  56         char *sessname;
  57         struct rtrs_addr *paths;
  58         size_t *path_cnt;
  59         char *pathname;
  60         u16 *dest_port;
  61         enum rnbd_access_mode *access_mode;
  62         u32 *nr_poll_queues;
  63 };
  64
  65 static int rnbd_clt_parse_map_options(const char *buf, size_t max_path_cnt,
  66                                        struct rnbd_map_options *opt)
  67 {
  68         char *options, *sep_opt;
  69         char *p;
  70         substring_t args[MAX_OPT_ARGS];
  71         int opt_mask = 0;
  72         int token;
  73         int ret = -EINVAL;
  74         int nr_poll_queues = 0;
  75         int dest_port = 0;
  76         int p_cnt = 0;
  77         int i;
  78
  79         options = kstrdup(buf, GFP_KERNEL);
  80         if (!options)
  81                 return -ENOMEM;
  82
  83         sep_opt = strstrip(options);
  84         while ((p = strsep(&sep_opt, " ")) != NULL) {
  85                 if (!*p)
  86                         continue;
  87
  88                 token = match_token(p, rnbd_opt_tokens, args);
  89                 opt_mask |= token;
  90
  91                 switch (token) {
  92                 case RNBD_OPT_SESSNAME:
  93                         p = match_strdup(args);
  94                         if (!p) {
  95                                 ret = -ENOMEM;
  96                                 goto out;
  97                         }
  98                         if (strlen(p) > NAME_MAX) {
  99                                 pr_err("map_device: sessname too long\n");
 100                                 ret = -EINVAL;
 101                                 kfree(p);
 102                                 goto out;
 103                         }
 104                         strscpy(opt->sessname, p, NAME_MAX);
 105                         kfree(p);
 106                         break;
 107
 108                 case RNBD_OPT_PATH:
 109                         if (p_cnt >= max_path_cnt) {
 110                                 pr_err("map_device: too many (> %zu) paths provided\n",
 111                                        max_path_cnt);
 112                                 ret = -ENOMEM;
 113                                 goto out;
 114                         }
 115                         p = match_strdup(args);
 116                         if (!p) {
 117                                 ret = -ENOMEM;
 118                                 goto out;
 119                         }
 120
 121                         ret = rtrs_addr_to_sockaddr(p, strlen(p),
 122                                                     *opt->dest_port,
 123                                                     &opt->paths[p_cnt]);
 124                         if (ret) {
 125                                 pr_err("Can't parse path %s: %d\n", p, ret);
 126                                 kfree(p);
 127                                 goto out;
 128                         }
 129
 130                         p_cnt++;
 131
 132                         kfree(p);
 133                         break;
 134
 135                 case RNBD_OPT_DEV_PATH:
 136                         p = match_strdup(args);
 137                         if (!p) {
 138                                 ret = -ENOMEM;
 139                                 goto out;
 140                         }
 141                         if (strlen(p) > NAME_MAX) {
 142                                 pr_err("map_device: Device path too long\n");
 143                                 ret = -EINVAL;
 144                                 kfree(p);
 145                                 goto out;
 146                         }
 147                         strscpy(opt->pathname, p, NAME_MAX);
 148                         kfree(p);
 149                         break;
 150
 151                 case RNBD_OPT_DEST_PORT:
 152                         if (match_int(args, &dest_port) || dest_port < 0 ||
 153                             dest_port > 65535) {
 154                                 pr_err("bad destination port number parameter '%d'\n",
 155                                        dest_port);
 156                                 ret = -EINVAL;
 157                                 goto out;
 158                         }
 159                         *opt->dest_port = dest_port;
 160                         break;
 161
 162                 case RNBD_OPT_ACCESS_MODE:
 163                         p = match_strdup(args);
 164                         if (!p) {
 165                                 ret = -ENOMEM;
 166                                 goto out;
 167                         }
 168
 169                         if (!strcmp(p, "ro")) {
 170                                 *opt->access_mode = RNBD_ACCESS_RO;
 171                         } else if (!strcmp(p, "rw")) {
 172                                 *opt->access_mode = RNBD_ACCESS_RW;
 173                         } else if (!strcmp(p, "migration")) {
 174                                 *opt->access_mode = RNBD_ACCESS_MIGRATION;
 175                         } else {
 176                                 pr_err("map_device: Invalid access_mode: '%s'\n",
 177                                        p);
 178                                 ret = -EINVAL;
 179                                 kfree(p);
 180                                 goto out;
 181                         }
 182
 183                         kfree(p);
 184                         break;
 185
 186                 case RNBD_OPT_NR_POLL_QUEUES:
 187                         if (match_int(args, &nr_poll_queues) || nr_poll_queues < -1 ||
 188                             nr_poll_queues > (int)nr_cpu_ids) {
 189                                 pr_err("bad nr_poll_queues parameter '%d'\n",
 190                                        nr_poll_queues);
 191                                 ret = -EINVAL;
 192                                 goto out;
 193                         }
 194                         if (nr_poll_queues == -1)
 195                                 nr_poll_queues = nr_cpu_ids;
 196                         *opt->nr_poll_queues = nr_poll_queues;
 197                         break;
 198
 199                 default:
 200                         pr_err("map_device: Unknown parameter or missing value '%s'\n",
 201                                p);
 202                         ret = -EINVAL;
 203                         goto out;
 204                 }
 205         }
 206
 207         for (i = 0; i < ARRAY_SIZE(rnbd_opt_mandatory); i++) {
 208                 if ((opt_mask & rnbd_opt_mandatory[i])) {
 209                         ret = 0;
 210                 } else {
 211                         pr_err("map_device: Parameters missing\n");
 212                         ret = -EINVAL;
 213                         break;
 214                 }
 215         }
 216
 217 out:
 218         *opt->path_cnt = p_cnt;
 219         kfree(options);
 220         return ret;
 221 }
 222
 223 static ssize_t state_show(struct kobject *kobj,
 224                           struct kobj_attribute *attr, char *page)
 225 {
 226         struct rnbd_clt_dev *dev;
 227
 228         dev = container_of(kobj, struct rnbd_clt_dev, kobj);
 229
 230         switch (dev->dev_state) {
 231         case DEV_STATE_INIT:
 232                 return sysfs_emit(page, "init\n");
 233         case DEV_STATE_MAPPED:
 234                 /* TODO fix cli tool before changing to proper state */
 235                 return sysfs_emit(page, "open\n");
 236         case DEV_STATE_MAPPED_DISCONNECTED:
 237                 /* TODO fix cli tool before changing to proper state */
 238                 return sysfs_emit(page, "closed\n");
 239         case DEV_STATE_UNMAPPED:
 240                 return sysfs_emit(page, "unmapped\n");
 241         default:
 242                 return sysfs_emit(page, "unknown\n");
 243         }
 244 }
 245
 246 static struct kobj_attribute rnbd_clt_state_attr = __ATTR_RO(state);
 247
 248 static ssize_t nr_poll_queues_show(struct kobject *kobj,
 249                                    struct kobj_attribute *attr, char *page)
 250 {
 251         struct rnbd_clt_dev *dev;
 252
 253         dev = container_of(kobj, struct rnbd_clt_dev, kobj);
 254
 255         return sysfs_emit(page, "%d\n", dev->nr_poll_queues);
 256 }
 257
 258 static struct kobj_attribute rnbd_clt_nr_poll_queues =
 259         __ATTR_RO(nr_poll_queues);
 260
 261 static ssize_t mapping_path_show(struct kobject *kobj,
 262                                  struct kobj_attribute *attr, char *page)
 263 {
 264         struct rnbd_clt_dev *dev;
 265
 266         dev = container_of(kobj, struct rnbd_clt_dev, kobj);
 267
 268         return sysfs_emit(page, "%s\n", dev->pathname);
 269 }
 270
 271 static struct kobj_attribute rnbd_clt_mapping_path_attr =
 272         __ATTR_RO(mapping_path);
 273
 274 static ssize_t access_mode_show(struct kobject *kobj,
 275                                 struct kobj_attribute *attr, char *page)
 276 {
 277         struct rnbd_clt_dev *dev;
 278
 279         dev = container_of(kobj, struct rnbd_clt_dev, kobj);
 280
 281         return sysfs_emit(page, "%s\n", rnbd_access_mode_str(dev->access_mode));
 282 }
 283
 284 static struct kobj_attribute rnbd_clt_access_mode =
 285         __ATTR_RO(access_mode);
 286
 287 static ssize_t rnbd_clt_unmap_dev_show(struct kobject *kobj,
 288                                         struct kobj_attribute *attr, char *page)
 289 {
 290         return sysfs_emit(page, "Usage: echo <normal|force> > %s\n",
 291                           attr->attr.name);
 292 }
 293
 294 static ssize_t rnbd_clt_unmap_dev_store(struct kobject *kobj,
 295                                          struct kobj_attribute *attr,
 296                                          const char *buf, size_t count)
 297 {
 298         struct rnbd_clt_dev *dev;
 299         char *opt, *options;
 300         bool force;
 301         int err;
 302
 303         opt = kstrdup(buf, GFP_KERNEL);
 304         if (!opt)
 305                 return -ENOMEM;
 306
 307         options = strstrip(opt);
 308         dev = container_of(kobj, struct rnbd_clt_dev, kobj);
 309         if (sysfs_streq(options, "normal")) {
 310                 force = false;
 311         } else if (sysfs_streq(options, "force")) {
 312                 force = true;
 313         } else {
 314                 rnbd_clt_err(dev,
 315                               "unmap_device: Invalid value: %s\n",
 316                               options);
 317                 err = -EINVAL;
 318                 goto out;
 319         }
 320
 321         rnbd_clt_info(dev, "Unmapping device, option: %s.\n",
 322                        force ? "force" : "normal");
 323
 324         /*
 325          * We take explicit module reference only for one reason: do not
 326          * race with lockless rnbd_destroy_sessions().
 327          */
 328         if (!try_module_get(THIS_MODULE)) {
 329                 err = -ENODEV;
 330                 goto out;
 331         }
 332         err = rnbd_clt_unmap_device(dev, force, &attr->attr);
 333         if (err) {
 334                 if (err != -EALREADY)
 335                         rnbd_clt_err(dev, "unmap_device: %d\n",  err);
 336                 goto module_put;
 337         }
 338
 339         /*
 340          * Here device can be vanished!
 341          */
 342
 343         err = count;
 344
 345 module_put:
 346         module_put(THIS_MODULE);
 347 out:
 348         kfree(opt);
 349
 350         return err;
 351 }
 352
 353 static struct kobj_attribute rnbd_clt_unmap_device_attr =
 354         __ATTR(unmap_device, 0644, rnbd_clt_unmap_dev_show,
 355                rnbd_clt_unmap_dev_store);
 356
 357 static ssize_t rnbd_clt_resize_dev_show(struct kobject *kobj,
 358                                          struct kobj_attribute *attr,
 359                                          char *page)
 360 {
 361         return sysfs_emit(page, "Usage: echo <new size in sectors> > %s\n",
 362                           attr->attr.name);
 363 }
 364
 365 static ssize_t rnbd_clt_resize_dev_store(struct kobject *kobj,
 366                                           struct kobj_attribute *attr,
 367                                           const char *buf, size_t count)
 368 {
 369         int ret;
 370         unsigned long sectors;
 371         struct rnbd_clt_dev *dev;
 372
 373         dev = container_of(kobj, struct rnbd_clt_dev, kobj);
 374
 375         ret = kstrtoul(buf, 0, &sectors);
 376         if (ret)
 377                 return ret;
 378
 379         ret = rnbd_clt_resize_disk(dev, (size_t)sectors);
 380         if (ret)
 381                 return ret;
 382
 383         return count;
 384 }
 385
 386 static struct kobj_attribute rnbd_clt_resize_dev_attr =
 387         __ATTR(resize, 0644, rnbd_clt_resize_dev_show,
 388                rnbd_clt_resize_dev_store);
 389
 390 static ssize_t rnbd_clt_remap_dev_show(struct kobject *kobj,
 391                                         struct kobj_attribute *attr, char *page)
 392 {
 393         return sysfs_emit(page, "Usage: echo <1> > %s\n", attr->attr.name);
 394 }
 395
 396 static ssize_t rnbd_clt_remap_dev_store(struct kobject *kobj,
 397                                          struct kobj_attribute *attr,
 398                                          const char *buf, size_t count)
 399 {
 400         struct rnbd_clt_dev *dev;
 401         char *opt, *options;
 402         int err;
 403
 404         opt = kstrdup(buf, GFP_KERNEL);
 405         if (!opt)
 406                 return -ENOMEM;
 407
 408         options = strstrip(opt);
 409         dev = container_of(kobj, struct rnbd_clt_dev, kobj);
 410         if (!sysfs_streq(options, "1")) {
 411                 rnbd_clt_err(dev,
 412                               "remap_device: Invalid value: %s\n",
 413                               options);
 414                 err = -EINVAL;
 415                 goto out;
 416         }
 417         err = rnbd_clt_remap_device(dev);
 418         if (likely(!err))
 419                 err = count;
 420
 421 out:
 422         kfree(opt);
 423
 424         return err;
 425 }
 426
 427 static struct kobj_attribute rnbd_clt_remap_device_attr =
 428         __ATTR(remap_device, 0644, rnbd_clt_remap_dev_show,
 429                rnbd_clt_remap_dev_store);
 430
 431 static ssize_t session_show(struct kobject *kobj, struct kobj_attribute *attr,
 432                             char *page)
 433 {
 434         struct rnbd_clt_dev *dev;
 435
 436         dev = container_of(kobj, struct rnbd_clt_dev, kobj);
 437
 438         return sysfs_emit(page, "%s\n", dev->sess->sessname);
 439 }
 440
 441 static struct kobj_attribute rnbd_clt_session_attr =
 442         __ATTR_RO(session);
 443
 444 static struct attribute *rnbd_dev_attrs[] = {
 445         &rnbd_clt_unmap_device_attr.attr,
 446         &rnbd_clt_resize_dev_attr.attr,
 447         &rnbd_clt_remap_device_attr.attr,
 448         &rnbd_clt_mapping_path_attr.attr,
 449         &rnbd_clt_state_attr.attr,
 450         &rnbd_clt_session_attr.attr,
 451         &rnbd_clt_access_mode.attr,
 452         &rnbd_clt_nr_poll_queues.attr,
 453         NULL,
 454 };
 455
 456 void rnbd_clt_remove_dev_symlink(struct rnbd_clt_dev *dev)
 457 {
 458         /*
 459          * The module unload rnbd_client_exit path is racing with unmapping of
 460          * the last single device from the sysfs manually
 461          * i.e. rnbd_clt_unmap_dev_store() leading to a sysfs warning because
 462          * of sysfs link already was removed already.
 463          */
 464         if (dev->blk_symlink_name) {
 465                 if (try_module_get(THIS_MODULE)) {
 466                         sysfs_remove_link(rnbd_devs_kobj, dev->blk_symlink_name);
 467                         module_put(THIS_MODULE);
 468                 }
 469                 /* It should be freed always. */
 470                 kfree(dev->blk_symlink_name);
 471                 dev->blk_symlink_name = NULL;
 472         }
 473 }
 474
 475 static struct kobj_type rnbd_dev_ktype = {
 476         .sysfs_ops      = &kobj_sysfs_ops,
 477         .default_attrs  = rnbd_dev_attrs,
 478 };
 479
 480 static int rnbd_clt_add_dev_kobj(struct rnbd_clt_dev *dev)
 481 {
 482         int ret;
 483         struct kobject *gd_kobj = &disk_to_dev(dev->gd)->kobj;
 484
 485         ret = kobject_init_and_add(&dev->kobj, &rnbd_dev_ktype, gd_kobj, "%s",
 486                                    "rnbd");
 487         if (ret) {
 488                 rnbd_clt_err(dev, "Failed to create device sysfs dir, err: %d\n",
 489                               ret);
 490                 kobject_put(&dev->kobj);
 491         }
 492         kobject_uevent(gd_kobj, KOBJ_ONLINE);
 493
 494         return ret;
 495 }
 496
 497 static ssize_t rnbd_clt_map_device_show(struct kobject *kobj,
 498                                          struct kobj_attribute *attr,
 499                                          char *page)
 500 {
 501         return sysfs_emit(page,
 502                           "Usage: echo \"[dest_port=server port number] sessname=<name of the rtrs session> path=<[srcaddr@]dstaddr> [path=<[srcaddr@]dstaddr>] device_path=<full path on remote side> [access_mode=<ro|rw|migration>] [nr_poll_queues=<number of queues>]\" > %s\n\naddr ::= [ ip:<ipv4> | ip:<ipv6> | gid:<gid> ]\n",
 503                          attr->attr.name);
 504 }
 505
 506 static int rnbd_clt_get_path_name(struct rnbd_clt_dev *dev, char *buf,
 507                                    size_t len)
 508 {
 509         int ret;
 510         char pathname[NAME_MAX], *s;
 511
 512         strscpy(pathname, dev->pathname, sizeof(pathname));
 513         while ((s = strchr(pathname, '/')))
 514                 s[0] = '!';
 515
 516         ret = snprintf(buf, len, "%s@%s", pathname, dev->sess->sessname);
 517         if (ret >= len)
 518                 return -ENAMETOOLONG;
 519
 520         return 0;
 521 }
 522
 523 static int rnbd_clt_add_dev_symlink(struct rnbd_clt_dev *dev)
 524 {
 525         struct kobject *gd_kobj = &disk_to_dev(dev->gd)->kobj;
 526         int ret, len;
 527
 528         len = strlen(dev->pathname) + strlen(dev->sess->sessname) + 2;
 529         dev->blk_symlink_name = kzalloc(len, GFP_KERNEL);
 530         if (!dev->blk_symlink_name) {
 531                 rnbd_clt_err(dev, "Failed to allocate memory for blk_symlink_name\n");
 532                 return -ENOMEM;
 533         }
 534
 535         ret = rnbd_clt_get_path_name(dev, dev->blk_symlink_name,
 536                                       len);
 537         if (ret) {
 538                 rnbd_clt_err(dev, "Failed to get /sys/block symlink path, err: %d\n",
 539                               ret);
 540                 goto out_err;
 541         }
 542
 543         ret = sysfs_create_link(rnbd_devs_kobj, gd_kobj,
 544                                 dev->blk_symlink_name);
 545         if (ret) {
 546                 rnbd_clt_err(dev, "Creating /sys/block symlink failed, err: %d\n",
 547                               ret);
 548                 goto out_err;
 549         }
 550
 551         return 0;
 552
 553 out_err:
 554         kfree(dev->blk_symlink_name);
 555         dev->blk_symlink_name = NULL ;
 556         return ret;
 557 }
 558
 559 static ssize_t rnbd_clt_map_device_store(struct kobject *kobj,
 560                                           struct kobj_attribute *attr,
 561                                           const char *buf, size_t count)
 562 {
 563         struct rnbd_clt_dev *dev;
 564         struct rnbd_map_options opt;
 565         int ret;
 566         char pathname[NAME_MAX];
 567         char sessname[NAME_MAX];
 568         enum rnbd_access_mode access_mode = RNBD_ACCESS_RW;
 569         u16 port_nr = RTRS_PORT;
 570         u32 nr_poll_queues = 0;
 571
 572         struct sockaddr_storage *addrs;
 573         struct rtrs_addr paths[6];
 574         size_t path_cnt;
 575
 576         opt.sessname = sessname;
 577         opt.paths = paths;
 578         opt.path_cnt = &path_cnt;
 579         opt.pathname = pathname;
 580         opt.dest_port = &port_nr;
 581         opt.access_mode = &access_mode;
 582         opt.nr_poll_queues = &nr_poll_queues;
 583         addrs = kcalloc(ARRAY_SIZE(paths) * 2, sizeof(*addrs), GFP_KERNEL);
 584         if (!addrs)
 585                 return -ENOMEM;
 586
 587         for (path_cnt = 0; path_cnt < ARRAY_SIZE(paths); path_cnt++) {
 588                 paths[path_cnt].src = &addrs[path_cnt * 2];
 589                 paths[path_cnt].dst = &addrs[path_cnt * 2 + 1];
 590         }
 591
 592         ret = rnbd_clt_parse_map_options(buf, ARRAY_SIZE(paths), &opt);
 593         if (ret)
 594                 goto out;
 595
 596         pr_info("Mapping device %s on session %s, (access_mode: %s, nr_poll_queues: %d)\n",
 597                 pathname, sessname,
 598                 rnbd_access_mode_str(access_mode),
 599                 nr_poll_queues);
 600
 601         dev = rnbd_clt_map_device(sessname, paths, path_cnt, port_nr, pathname,
 602                                   access_mode, nr_poll_queues);
 603         if (IS_ERR(dev)) {
 604                 ret = PTR_ERR(dev);
 605                 goto out;
 606         }
 607
 608         ret = rnbd_clt_add_dev_kobj(dev);
 609         if (ret)
 610                 goto unmap_dev;
 611
 612         ret = rnbd_clt_add_dev_symlink(dev);
 613         if (ret)
 614                 goto unmap_dev;
 615
 616         kfree(addrs);
 617         return count;
 618
 619 unmap_dev:
 620         rnbd_clt_unmap_device(dev, true, NULL);
 621 out:
 622         kfree(addrs);
 623         return ret;
 624 }
 625
 626 static struct kobj_attribute rnbd_clt_map_device_attr =
 627         __ATTR(map_device, 0644,
 628                rnbd_clt_map_device_show, rnbd_clt_map_device_store);
 629
 630 static struct attribute *default_attrs[] = {
 631         &rnbd_clt_map_device_attr.attr,
 632         NULL,
 633 };
 634
 635 static struct attribute_group default_attr_group = {
 636         .attrs = default_attrs,
 637 };
 638
 639 static const struct attribute_group *default_attr_groups[] = {
 640         &default_attr_group,
 641         NULL,
 642 };
 643
 644 int rnbd_clt_create_sysfs_files(void)
 645 {
 646         int err;
 647
 648         rnbd_dev_class = class_create(THIS_MODULE, "rnbd-client");
 649         if (IS_ERR(rnbd_dev_class))
 650                 return PTR_ERR(rnbd_dev_class);
 651
 652         rnbd_dev = device_create_with_groups(rnbd_dev_class, NULL,
 653                                               MKDEV(0, 0), NULL,
 654                                               default_attr_groups, "ctl");
 655         if (IS_ERR(rnbd_dev)) {
 656                 err = PTR_ERR(rnbd_dev);
 657                 goto cls_destroy;
 658         }
 659         rnbd_devs_kobj = kobject_create_and_add("devices", &rnbd_dev->kobj);
 660         if (!rnbd_devs_kobj) {
 661                 err = -ENOMEM;
 662                 goto dev_destroy;
 663         }
 664
 665         return 0;
 666
 667 dev_destroy:
 668         device_destroy(rnbd_dev_class, MKDEV(0, 0));
 669 cls_destroy:
 670         class_destroy(rnbd_dev_class);
 671
 672         return err;
 673 }
 674
 675 void rnbd_clt_destroy_sysfs_files(void)
 676 {
 677         sysfs_remove_group(&rnbd_dev->kobj, &default_attr_group);
 678         kobject_del(rnbd_devs_kobj);
 679         kobject_put(rnbd_devs_kobj);
 680         device_destroy(rnbd_dev_class, MKDEV(0, 0));
 681         class_destroy(rnbd_dev_class);
 682 }