drivers/block/null_blk/main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Add configfs and memory store: Kyungchan Koh <kkc6196@fb.com> and
   4  * Shaohua Li <shli@fb.com>
   5  */
   6 #include <linux/module.h>
   7
   8 #include <linux/moduleparam.h>
   9 #include <linux/sched.h>
  10 #include <linux/fs.h>
  11 #include <linux/init.h>
  12 #include "null_blk.h"
  13
  14 #undef pr_fmt
  15 #define pr_fmt(fmt)     "null_blk: " fmt
  16
  17 #define FREE_BATCH              16
  18
  19 #define TICKS_PER_SEC           50ULL
  20 #define TIMER_INTERVAL          (NSEC_PER_SEC / TICKS_PER_SEC)
  21
  22 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
  23 static DECLARE_FAULT_ATTR(null_timeout_attr);
  24 static DECLARE_FAULT_ATTR(null_requeue_attr);
  25 static DECLARE_FAULT_ATTR(null_init_hctx_attr);
  26 #endif
  27
  28 static inline u64 mb_per_tick(int mbps)
  29 {
  30         return (1 << 20) / TICKS_PER_SEC * ((u64) mbps);
  31 }
  32
  33 /*
  34  * Status flags for nullb_device.
  35  *
  36  * CONFIGURED:  Device has been configured and turned on. Cannot reconfigure.
  37  * UP:          Device is currently on and visible in userspace.
  38  * THROTTLED:   Device is being throttled.
  39  * CACHE:       Device is using a write-back cache.
  40  */
  41 enum nullb_device_flags {
  42         NULLB_DEV_FL_CONFIGURED = 0,
  43         NULLB_DEV_FL_UP         = 1,
  44         NULLB_DEV_FL_THROTTLED  = 2,
  45         NULLB_DEV_FL_CACHE      = 3,
  46 };
  47
  48 #define MAP_SZ          ((PAGE_SIZE >> SECTOR_SHIFT) + 2)
  49 /*
  50  * nullb_page is a page in memory for nullb devices.
  51  *
  52  * @page:       The page holding the data.
  53  * @bitmap:     The bitmap represents which sector in the page has data.
  54  *              Each bit represents one block size. For example, sector 8
  55  *              will use the 7th bit
  56  * The highest 2 bits of bitmap are for special purpose. LOCK means the cache
  57  * page is being flushing to storage. FREE means the cache page is freed and
  58  * should be skipped from flushing to storage. Please see
  59  * null_make_cache_space
  60  */
  61 struct nullb_page {
  62         struct page *page;
  63         DECLARE_BITMAP(bitmap, MAP_SZ);
  64 };
  65 #define NULLB_PAGE_LOCK (MAP_SZ - 1)
  66 #define NULLB_PAGE_FREE (MAP_SZ - 2)
  67
  68 static LIST_HEAD(nullb_list);
  69 static struct mutex lock;
  70 static int null_major;
  71 static DEFINE_IDA(nullb_indexes);
  72 static struct blk_mq_tag_set tag_set;
  73
  74 enum {
  75         NULL_IRQ_NONE           = 0,
  76         NULL_IRQ_SOFTIRQ        = 1,
  77         NULL_IRQ_TIMER          = 2,
  78 };
  79
  80 static bool g_virt_boundary = false;
  81 module_param_named(virt_boundary, g_virt_boundary, bool, 0444);
  82 MODULE_PARM_DESC(virt_boundary, "Require a virtual boundary for the device. Default: False");
  83
  84 static int g_no_sched;
  85 module_param_named(no_sched, g_no_sched, int, 0444);
  86 MODULE_PARM_DESC(no_sched, "No io scheduler");
  87
  88 static int g_submit_queues = 1;
  89 module_param_named(submit_queues, g_submit_queues, int, 0444);
  90 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
  91
  92 static int g_poll_queues = 1;
  93 module_param_named(poll_queues, g_poll_queues, int, 0444);
  94 MODULE_PARM_DESC(poll_queues, "Number of IOPOLL submission queues");
  95
  96 static int g_home_node = NUMA_NO_NODE;
  97 module_param_named(home_node, g_home_node, int, 0444);
  98 MODULE_PARM_DESC(home_node, "Home node for the device");
  99
 100 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
 101 /*
 102  * For more details about fault injection, please refer to
 103  * Documentation/fault-injection/fault-injection.rst.
 104  */
 105 static char g_timeout_str[80];
 106 module_param_string(timeout, g_timeout_str, sizeof(g_timeout_str), 0444);
 107 MODULE_PARM_DESC(timeout, "Fault injection. timeout=<interval>,<probability>,<space>,<times>");
 108
 109 static char g_requeue_str[80];
 110 module_param_string(requeue, g_requeue_str, sizeof(g_requeue_str), 0444);
 111 MODULE_PARM_DESC(requeue, "Fault injection. requeue=<interval>,<probability>,<space>,<times>");
 112
 113 static char g_init_hctx_str[80];
 114 module_param_string(init_hctx, g_init_hctx_str, sizeof(g_init_hctx_str), 0444);
 115 MODULE_PARM_DESC(init_hctx, "Fault injection to fail hctx init. init_hctx=<interval>,<probability>,<space>,<times>");
 116 #endif
 117
 118 static int g_queue_mode = NULL_Q_MQ;
 119
 120 static int null_param_store_val(const char *str, int *val, int min, int max)
 121 {
 122         int ret, new_val;
 123
 124         ret = kstrtoint(str, 10, &new_val);
 125         if (ret)
 126                 return -EINVAL;
 127
 128         if (new_val < min || new_val > max)
 129                 return -EINVAL;
 130
 131         *val = new_val;
 132         return 0;
 133 }
 134
 135 static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
 136 {
 137         return null_param_store_val(str, &g_queue_mode, NULL_Q_BIO, NULL_Q_MQ);
 138 }
 139
 140 static const struct kernel_param_ops null_queue_mode_param_ops = {
 141         .set    = null_set_queue_mode,
 142         .get    = param_get_int,
 143 };
 144
 145 device_param_cb(queue_mode, &null_queue_mode_param_ops, &g_queue_mode, 0444);
 146 MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
 147
 148 static int g_gb = 250;
 149 module_param_named(gb, g_gb, int, 0444);
 150 MODULE_PARM_DESC(gb, "Size in GB");
 151
 152 static int g_bs = 512;
 153 module_param_named(bs, g_bs, int, 0444);
 154 MODULE_PARM_DESC(bs, "Block size (in bytes)");
 155
 156 static int g_max_sectors;
 157 module_param_named(max_sectors, g_max_sectors, int, 0444);
 158 MODULE_PARM_DESC(max_sectors, "Maximum size of a command (in 512B sectors)");
 159
 160 static unsigned int nr_devices = 1;
 161 module_param(nr_devices, uint, 0444);
 162 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
 163
 164 static bool g_blocking;
 165 module_param_named(blocking, g_blocking, bool, 0444);
 166 MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
 167
 168 static bool shared_tags;
 169 module_param(shared_tags, bool, 0444);
 170 MODULE_PARM_DESC(shared_tags, "Share tag set between devices for blk-mq");
 171
 172 static bool g_shared_tag_bitmap;
 173 module_param_named(shared_tag_bitmap, g_shared_tag_bitmap, bool, 0444);
 174 MODULE_PARM_DESC(shared_tag_bitmap, "Use shared tag bitmap for all submission queues for blk-mq");
 175
 176 static int g_irqmode = NULL_IRQ_SOFTIRQ;
 177
 178 static int null_set_irqmode(const char *str, const struct kernel_param *kp)
 179 {
 180         return null_param_store_val(str, &g_irqmode, NULL_IRQ_NONE,
 181                                         NULL_IRQ_TIMER);
 182 }
 183
 184 static const struct kernel_param_ops null_irqmode_param_ops = {
 185         .set    = null_set_irqmode,
 186         .get    = param_get_int,
 187 };
 188
 189 device_param_cb(irqmode, &null_irqmode_param_ops, &g_irqmode, 0444);
 190 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
 191
 192 static unsigned long g_completion_nsec = 10000;
 193 module_param_named(completion_nsec, g_completion_nsec, ulong, 0444);
 194 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
 195
 196 static int g_hw_queue_depth = 64;
 197 module_param_named(hw_queue_depth, g_hw_queue_depth, int, 0444);
 198 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
 199
 200 static bool g_use_per_node_hctx;
 201 module_param_named(use_per_node_hctx, g_use_per_node_hctx, bool, 0444);
 202 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
 203
 204 static bool g_memory_backed;
 205 module_param_named(memory_backed, g_memory_backed, bool, 0444);
 206 MODULE_PARM_DESC(memory_backed, "Create a memory-backed block device. Default: false");
 207
 208 static bool g_discard;
 209 module_param_named(discard, g_discard, bool, 0444);
 210 MODULE_PARM_DESC(discard, "Support discard operations (requires memory-backed null_blk device). Default: false");
 211
 212 static unsigned long g_cache_size;
 213 module_param_named(cache_size, g_cache_size, ulong, 0444);
 214 MODULE_PARM_DESC(mbps, "Cache size in MiB for memory-backed device. Default: 0 (none)");
 215
 216 static unsigned int g_mbps;
 217 module_param_named(mbps, g_mbps, uint, 0444);
 218 MODULE_PARM_DESC(mbps, "Limit maximum bandwidth (in MiB/s). Default: 0 (no limit)");
 219
 220 static bool g_zoned;
 221 module_param_named(zoned, g_zoned, bool, S_IRUGO);
 222 MODULE_PARM_DESC(zoned, "Make device as a host-managed zoned block device. Default: false");
 223
 224 static unsigned long g_zone_size = 256;
 225 module_param_named(zone_size, g_zone_size, ulong, S_IRUGO);
 226 MODULE_PARM_DESC(zone_size, "Zone size in MB when block device is zoned. Must be power-of-two: Default: 256");
 227
 228 static unsigned long g_zone_capacity;
 229 module_param_named(zone_capacity, g_zone_capacity, ulong, 0444);
 230 MODULE_PARM_DESC(zone_capacity, "Zone capacity in MB when block device is zoned. Can be less than or equal to zone size. Default: Zone size");
 231
 232 static unsigned int g_zone_nr_conv;
 233 module_param_named(zone_nr_conv, g_zone_nr_conv, uint, 0444);
 234 MODULE_PARM_DESC(zone_nr_conv, "Number of conventional zones when block device is zoned. Default: 0");
 235
 236 static unsigned int g_zone_max_open;
 237 module_param_named(zone_max_open, g_zone_max_open, uint, 0444);
 238 MODULE_PARM_DESC(zone_max_open, "Maximum number of open zones when block device is zoned. Default: 0 (no limit)");
 239
 240 static unsigned int g_zone_max_active;
 241 module_param_named(zone_max_active, g_zone_max_active, uint, 0444);
 242 MODULE_PARM_DESC(zone_max_active, "Maximum number of active zones when block device is zoned. Default: 0 (no limit)");
 243
 244 static struct nullb_device *null_alloc_dev(void);
 245 static void null_free_dev(struct nullb_device *dev);
 246 static void null_del_dev(struct nullb *nullb);
 247 static int null_add_dev(struct nullb_device *dev);
 248 static struct nullb *null_find_dev_by_name(const char *name);
 249 static void null_free_device_storage(struct nullb_device *dev, bool is_cache);
 250
 251 static inline struct nullb_device *to_nullb_device(struct config_item *item)
 252 {
 253         return item ? container_of(to_config_group(item), struct nullb_device, group) : NULL;
 254 }
 255
 256 static inline ssize_t nullb_device_uint_attr_show(unsigned int val, char *page)
 257 {
 258         return snprintf(page, PAGE_SIZE, "%u\n", val);
 259 }
 260
 261 static inline ssize_t nullb_device_ulong_attr_show(unsigned long val,
 262         char *page)
 263 {
 264         return snprintf(page, PAGE_SIZE, "%lu\n", val);
 265 }
 266
 267 static inline ssize_t nullb_device_bool_attr_show(bool val, char *page)
 268 {
 269         return snprintf(page, PAGE_SIZE, "%u\n", val);
 270 }
 271
 272 static ssize_t nullb_device_uint_attr_store(unsigned int *val,
 273         const char *page, size_t count)
 274 {
 275         unsigned int tmp;
 276         int result;
 277
 278         result = kstrtouint(page, 0, &tmp);
 279         if (result < 0)
 280                 return result;
 281
 282         *val = tmp;
 283         return count;
 284 }
 285
 286 static ssize_t nullb_device_ulong_attr_store(unsigned long *val,
 287         const char *page, size_t count)
 288 {
 289         int result;
 290         unsigned long tmp;
 291
 292         result = kstrtoul(page, 0, &tmp);
 293         if (result < 0)
 294                 return result;
 295
 296         *val = tmp;
 297         return count;
 298 }
 299
 300 static ssize_t nullb_device_bool_attr_store(bool *val, const char *page,
 301         size_t count)
 302 {
 303         bool tmp;
 304         int result;
 305
 306         result = kstrtobool(page,  &tmp);
 307         if (result < 0)
 308                 return result;
 309
 310         *val = tmp;
 311         return count;
 312 }
 313
 314 /* The following macro should only be used with TYPE = {uint, ulong, bool}. */
 315 #define NULLB_DEVICE_ATTR(NAME, TYPE, APPLY)                            \
 316 static ssize_t                                                          \
 317 nullb_device_##NAME##_show(struct config_item *item, char *page)        \
 318 {                                                                       \
 319         return nullb_device_##TYPE##_attr_show(                         \
 320                                 to_nullb_device(item)->NAME, page);     \
 321 }                                                                       \
 322 static ssize_t                                                          \
 323 nullb_device_##NAME##_store(struct config_item *item, const char *page, \
 324                             size_t count)                               \
 325 {                                                                       \
 326         int (*apply_fn)(struct nullb_device *dev, TYPE new_value) = APPLY;\
 327         struct nullb_device *dev = to_nullb_device(item);               \
 328         TYPE new_value = 0;                                             \
 329         int ret;                                                        \
 330                                                                         \
 331         ret = nullb_device_##TYPE##_attr_store(&new_value, page, count);\
 332         if (ret < 0)                                                    \
 333                 return ret;                                             \
 334         if (apply_fn)                                                   \
 335                 ret = apply_fn(dev, new_value);                         \
 336         else if (test_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags))        \
 337                 ret = -EBUSY;                                           \
 338         if (ret < 0)                                                    \
 339                 return ret;                                             \
 340         dev->NAME = new_value;                                          \
 341         return count;                                                   \
 342 }                                                                       \
 343 CONFIGFS_ATTR(nullb_device_, NAME);
 344
 345 static int nullb_update_nr_hw_queues(struct nullb_device *dev,
 346                                      unsigned int submit_queues,
 347                                      unsigned int poll_queues)
 348
 349 {
 350         struct blk_mq_tag_set *set;
 351         int ret, nr_hw_queues;
 352
 353         if (!dev->nullb)
 354                 return 0;
 355
 356         /*
 357          * Make sure at least one submit queue exists.
 358          */
 359         if (!submit_queues)
 360                 return -EINVAL;
 361
 362         /*
 363          * Make sure that null_init_hctx() does not access nullb->queues[] past
 364          * the end of that array.
 365          */
 366         if (submit_queues > nr_cpu_ids || poll_queues > g_poll_queues)
 367                 return -EINVAL;
 368
 369         /*
 370          * Keep previous and new queue numbers in nullb_device for reference in
 371          * the call back function null_map_queues().
 372          */
 373         dev->prev_submit_queues = dev->submit_queues;
 374         dev->prev_poll_queues = dev->poll_queues;
 375         dev->submit_queues = submit_queues;
 376         dev->poll_queues = poll_queues;
 377
 378         set = dev->nullb->tag_set;
 379         nr_hw_queues = submit_queues + poll_queues;
 380         blk_mq_update_nr_hw_queues(set, nr_hw_queues);
 381         ret = set->nr_hw_queues == nr_hw_queues ? 0 : -ENOMEM;
 382
 383         if (ret) {
 384                 /* on error, revert the queue numbers */
 385                 dev->submit_queues = dev->prev_submit_queues;
 386                 dev->poll_queues = dev->prev_poll_queues;
 387         }
 388
 389         return ret;
 390 }
 391
 392 static int nullb_apply_submit_queues(struct nullb_device *dev,
 393                                      unsigned int submit_queues)
 394 {
 395         return nullb_update_nr_hw_queues(dev, submit_queues, dev->poll_queues);
 396 }
 397
 398 static int nullb_apply_poll_queues(struct nullb_device *dev,
 399                                    unsigned int poll_queues)
 400 {
 401         return nullb_update_nr_hw_queues(dev, dev->submit_queues, poll_queues);
 402 }
 403
 404 NULLB_DEVICE_ATTR(size, ulong, NULL);
 405 NULLB_DEVICE_ATTR(completion_nsec, ulong, NULL);
 406 NULLB_DEVICE_ATTR(submit_queues, uint, nullb_apply_submit_queues);
 407 NULLB_DEVICE_ATTR(poll_queues, uint, nullb_apply_poll_queues);
 408 NULLB_DEVICE_ATTR(home_node, uint, NULL);
 409 NULLB_DEVICE_ATTR(queue_mode, uint, NULL);
 410 NULLB_DEVICE_ATTR(blocksize, uint, NULL);
 411 NULLB_DEVICE_ATTR(max_sectors, uint, NULL);
 412 NULLB_DEVICE_ATTR(irqmode, uint, NULL);
 413 NULLB_DEVICE_ATTR(hw_queue_depth, uint, NULL);
 414 NULLB_DEVICE_ATTR(index, uint, NULL);
 415 NULLB_DEVICE_ATTR(blocking, bool, NULL);
 416 NULLB_DEVICE_ATTR(use_per_node_hctx, bool, NULL);
 417 NULLB_DEVICE_ATTR(memory_backed, bool, NULL);
 418 NULLB_DEVICE_ATTR(discard, bool, NULL);
 419 NULLB_DEVICE_ATTR(mbps, uint, NULL);
 420 NULLB_DEVICE_ATTR(cache_size, ulong, NULL);
 421 NULLB_DEVICE_ATTR(zoned, bool, NULL);
 422 NULLB_DEVICE_ATTR(zone_size, ulong, NULL);
 423 NULLB_DEVICE_ATTR(zone_capacity, ulong, NULL);
 424 NULLB_DEVICE_ATTR(zone_nr_conv, uint, NULL);
 425 NULLB_DEVICE_ATTR(zone_max_open, uint, NULL);
 426 NULLB_DEVICE_ATTR(zone_max_active, uint, NULL);
 427 NULLB_DEVICE_ATTR(virt_boundary, bool, NULL);
 428 NULLB_DEVICE_ATTR(no_sched, bool, NULL);
 429 NULLB_DEVICE_ATTR(shared_tag_bitmap, bool, NULL);
 430
 431 static ssize_t nullb_device_power_show(struct config_item *item, char *page)
 432 {
 433         return nullb_device_bool_attr_show(to_nullb_device(item)->power, page);
 434 }
 435
 436 static ssize_t nullb_device_power_store(struct config_item *item,
 437                                      const char *page, size_t count)
 438 {
 439         struct nullb_device *dev = to_nullb_device(item);
 440         bool newp = false;
 441         ssize_t ret;
 442
 443         ret = nullb_device_bool_attr_store(&newp, page, count);
 444         if (ret < 0)
 445                 return ret;
 446
 447         if (!dev->power && newp) {
 448                 if (test_and_set_bit(NULLB_DEV_FL_UP, &dev->flags))
 449                         return count;
 450                 ret = null_add_dev(dev);
 451                 if (ret) {
 452                         clear_bit(NULLB_DEV_FL_UP, &dev->flags);
 453                         return ret;
 454                 }
 455
 456                 set_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
 457                 dev->power = newp;
 458         } else if (dev->power && !newp) {
 459                 if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
 460                         mutex_lock(&lock);
 461                         dev->power = newp;
 462                         null_del_dev(dev->nullb);
 463                         mutex_unlock(&lock);
 464                 }
 465                 clear_bit(NULLB_DEV_FL_CONFIGURED, &dev->flags);
 466         }
 467
 468         return count;
 469 }
 470
 471 CONFIGFS_ATTR(nullb_device_, power);
 472
 473 static ssize_t nullb_device_badblocks_show(struct config_item *item, char *page)
 474 {
 475         struct nullb_device *t_dev = to_nullb_device(item);
 476
 477         return badblocks_show(&t_dev->badblocks, page, 0);
 478 }
 479
 480 static ssize_t nullb_device_badblocks_store(struct config_item *item,
 481                                      const char *page, size_t count)
 482 {
 483         struct nullb_device *t_dev = to_nullb_device(item);
 484         char *orig, *buf, *tmp;
 485         u64 start, end;
 486         int ret;
 487
 488         orig = kstrndup(page, count, GFP_KERNEL);
 489         if (!orig)
 490                 return -ENOMEM;
 491
 492         buf = strstrip(orig);
 493
 494         ret = -EINVAL;
 495         if (buf[0] != '+' && buf[0] != '-')
 496                 goto out;
 497         tmp = strchr(&buf[1], '-');
 498         if (!tmp)
 499                 goto out;
 500         *tmp = '\0';
 501         ret = kstrtoull(buf + 1, 0, &start);
 502         if (ret)
 503                 goto out;
 504         ret = kstrtoull(tmp + 1, 0, &end);
 505         if (ret)
 506                 goto out;
 507         ret = -EINVAL;
 508         if (start > end)
 509                 goto out;
 510         /* enable badblocks */
 511         cmpxchg(&t_dev->badblocks.shift, -1, 0);
 512         if (buf[0] == '+')
 513                 ret = badblocks_set(&t_dev->badblocks, start,
 514                         end - start + 1, 1);
 515         else
 516                 ret = badblocks_clear(&t_dev->badblocks, start,
 517                         end - start + 1);
 518         if (ret == 0)
 519                 ret = count;
 520 out:
 521         kfree(orig);
 522         return ret;
 523 }
 524 CONFIGFS_ATTR(nullb_device_, badblocks);
 525
 526 static ssize_t nullb_device_zone_readonly_store(struct config_item *item,
 527                                                 const char *page, size_t count)
 528 {
 529         struct nullb_device *dev = to_nullb_device(item);
 530
 531         return zone_cond_store(dev, page, count, BLK_ZONE_COND_READONLY);
 532 }
 533 CONFIGFS_ATTR_WO(nullb_device_, zone_readonly);
 534
 535 static ssize_t nullb_device_zone_offline_store(struct config_item *item,
 536                                                const char *page, size_t count)
 537 {
 538         struct nullb_device *dev = to_nullb_device(item);
 539
 540         return zone_cond_store(dev, page, count, BLK_ZONE_COND_OFFLINE);
 541 }
 542 CONFIGFS_ATTR_WO(nullb_device_, zone_offline);
 543
 544 static struct configfs_attribute *nullb_device_attrs[] = {
 545         &nullb_device_attr_size,
 546         &nullb_device_attr_completion_nsec,
 547         &nullb_device_attr_submit_queues,
 548         &nullb_device_attr_poll_queues,
 549         &nullb_device_attr_home_node,
 550         &nullb_device_attr_queue_mode,
 551         &nullb_device_attr_blocksize,
 552         &nullb_device_attr_max_sectors,
 553         &nullb_device_attr_irqmode,
 554         &nullb_device_attr_hw_queue_depth,
 555         &nullb_device_attr_index,
 556         &nullb_device_attr_blocking,
 557         &nullb_device_attr_use_per_node_hctx,
 558         &nullb_device_attr_power,
 559         &nullb_device_attr_memory_backed,
 560         &nullb_device_attr_discard,
 561         &nullb_device_attr_mbps,
 562         &nullb_device_attr_cache_size,
 563         &nullb_device_attr_badblocks,
 564         &nullb_device_attr_zoned,
 565         &nullb_device_attr_zone_size,
 566         &nullb_device_attr_zone_capacity,
 567         &nullb_device_attr_zone_nr_conv,
 568         &nullb_device_attr_zone_max_open,
 569         &nullb_device_attr_zone_max_active,
 570         &nullb_device_attr_zone_readonly,
 571         &nullb_device_attr_zone_offline,
 572         &nullb_device_attr_virt_boundary,
 573         &nullb_device_attr_no_sched,
 574         &nullb_device_attr_shared_tag_bitmap,
 575         NULL,
 576 };
 577
 578 static void nullb_device_release(struct config_item *item)
 579 {
 580         struct nullb_device *dev = to_nullb_device(item);
 581
 582         null_free_device_storage(dev, false);
 583         null_free_dev(dev);
 584 }
 585
 586 static struct configfs_item_operations nullb_device_ops = {
 587         .release        = nullb_device_release,
 588 };
 589
 590 static const struct config_item_type nullb_device_type = {
 591         .ct_item_ops    = &nullb_device_ops,
 592         .ct_attrs       = nullb_device_attrs,
 593         .ct_owner       = THIS_MODULE,
 594 };
 595
 596 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
 597
 598 static void nullb_add_fault_config(struct nullb_device *dev)
 599 {
 600         fault_config_init(&dev->timeout_config, "timeout_inject");
 601         fault_config_init(&dev->requeue_config, "requeue_inject");
 602         fault_config_init(&dev->init_hctx_fault_config, "init_hctx_fault_inject");
 603
 604         configfs_add_default_group(&dev->timeout_config.group, &dev->group);
 605         configfs_add_default_group(&dev->requeue_config.group, &dev->group);
 606         configfs_add_default_group(&dev->init_hctx_fault_config.group, &dev->group);
 607 }
 608
 609 #else
 610
 611 static void nullb_add_fault_config(struct nullb_device *dev)
 612 {
 613 }
 614
 615 #endif
 616
 617 static struct
 618 config_group *nullb_group_make_group(struct config_group *group, const char *name)
 619 {
 620         struct nullb_device *dev;
 621
 622         if (null_find_dev_by_name(name))
 623                 return ERR_PTR(-EEXIST);
 624
 625         dev = null_alloc_dev();
 626         if (!dev)
 627                 return ERR_PTR(-ENOMEM);
 628
 629         config_group_init_type_name(&dev->group, name, &nullb_device_type);
 630         nullb_add_fault_config(dev);
 631
 632         return &dev->group;
 633 }
 634
 635 static void
 636 nullb_group_drop_item(struct config_group *group, struct config_item *item)
 637 {
 638         struct nullb_device *dev = to_nullb_device(item);
 639
 640         if (test_and_clear_bit(NULLB_DEV_FL_UP, &dev->flags)) {
 641                 mutex_lock(&lock);
 642                 dev->power = false;
 643                 null_del_dev(dev->nullb);
 644                 mutex_unlock(&lock);
 645         }
 646
 647         config_item_put(item);
 648 }
 649
 650 static ssize_t memb_group_features_show(struct config_item *item, char *page)
 651 {
 652         return snprintf(page, PAGE_SIZE,
 653                         "badblocks,blocking,blocksize,cache_size,"
 654                         "completion_nsec,discard,home_node,hw_queue_depth,"
 655                         "irqmode,max_sectors,mbps,memory_backed,no_sched,"
 656                         "poll_queues,power,queue_mode,shared_tag_bitmap,size,"
 657                         "submit_queues,use_per_node_hctx,virt_boundary,zoned,"
 658                         "zone_capacity,zone_max_active,zone_max_open,"
 659                         "zone_nr_conv,zone_offline,zone_readonly,zone_size\n");
 660 }
 661
 662 CONFIGFS_ATTR_RO(memb_group_, features);
 663
 664 static struct configfs_attribute *nullb_group_attrs[] = {
 665         &memb_group_attr_features,
 666         NULL,
 667 };
 668
 669 static struct configfs_group_operations nullb_group_ops = {
 670         .make_group     = nullb_group_make_group,
 671         .drop_item      = nullb_group_drop_item,
 672 };
 673
 674 static const struct config_item_type nullb_group_type = {
 675         .ct_group_ops   = &nullb_group_ops,
 676         .ct_attrs       = nullb_group_attrs,
 677         .ct_owner       = THIS_MODULE,
 678 };
 679
 680 static struct configfs_subsystem nullb_subsys = {
 681         .su_group = {
 682                 .cg_item = {
 683                         .ci_namebuf = "nullb",
 684                         .ci_type = &nullb_group_type,
 685                 },
 686         },
 687 };
 688
 689 static inline int null_cache_active(struct nullb *nullb)
 690 {
 691         return test_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
 692 }
 693
 694 static struct nullb_device *null_alloc_dev(void)
 695 {
 696         struct nullb_device *dev;
 697
 698         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 699         if (!dev)
 700                 return NULL;
 701
 702 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
 703         dev->timeout_config.attr = null_timeout_attr;
 704         dev->requeue_config.attr = null_requeue_attr;
 705         dev->init_hctx_fault_config.attr = null_init_hctx_attr;
 706 #endif
 707
 708         INIT_RADIX_TREE(&dev->data, GFP_ATOMIC);
 709         INIT_RADIX_TREE(&dev->cache, GFP_ATOMIC);
 710         if (badblocks_init(&dev->badblocks, 0)) {
 711                 kfree(dev);
 712                 return NULL;
 713         }
 714
 715         dev->size = g_gb * 1024;
 716         dev->completion_nsec = g_completion_nsec;
 717         dev->submit_queues = g_submit_queues;
 718         dev->prev_submit_queues = g_submit_queues;
 719         dev->poll_queues = g_poll_queues;
 720         dev->prev_poll_queues = g_poll_queues;
 721         dev->home_node = g_home_node;
 722         dev->queue_mode = g_queue_mode;
 723         dev->blocksize = g_bs;
 724         dev->max_sectors = g_max_sectors;
 725         dev->irqmode = g_irqmode;
 726         dev->hw_queue_depth = g_hw_queue_depth;
 727         dev->blocking = g_blocking;
 728         dev->memory_backed = g_memory_backed;
 729         dev->discard = g_discard;
 730         dev->cache_size = g_cache_size;
 731         dev->mbps = g_mbps;
 732         dev->use_per_node_hctx = g_use_per_node_hctx;
 733         dev->zoned = g_zoned;
 734         dev->zone_size = g_zone_size;
 735         dev->zone_capacity = g_zone_capacity;
 736         dev->zone_nr_conv = g_zone_nr_conv;
 737         dev->zone_max_open = g_zone_max_open;
 738         dev->zone_max_active = g_zone_max_active;
 739         dev->virt_boundary = g_virt_boundary;
 740         dev->no_sched = g_no_sched;
 741         dev->shared_tag_bitmap = g_shared_tag_bitmap;
 742         return dev;
 743 }
 744
 745 static void null_free_dev(struct nullb_device *dev)
 746 {
 747         if (!dev)
 748                 return;
 749
 750         null_free_zoned_dev(dev);
 751         badblocks_exit(&dev->badblocks);
 752         kfree(dev);
 753 }
 754
 755 static void put_tag(struct nullb_queue *nq, unsigned int tag)
 756 {
 757         clear_bit_unlock(tag, nq->tag_map);
 758
 759         if (waitqueue_active(&nq->wait))
 760                 wake_up(&nq->wait);
 761 }
 762
 763 static unsigned int get_tag(struct nullb_queue *nq)
 764 {
 765         unsigned int tag;
 766
 767         do {
 768                 tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
 769                 if (tag >= nq->queue_depth)
 770                         return -1U;
 771         } while (test_and_set_bit_lock(tag, nq->tag_map));
 772
 773         return tag;
 774 }
 775
 776 static void free_cmd(struct nullb_cmd *cmd)
 777 {
 778         put_tag(cmd->nq, cmd->tag);
 779 }
 780
 781 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer);
 782
 783 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
 784 {
 785         struct nullb_cmd *cmd;
 786         unsigned int tag;
 787
 788         tag = get_tag(nq);
 789         if (tag != -1U) {
 790                 cmd = &nq->cmds[tag];
 791                 cmd->tag = tag;
 792                 cmd->error = BLK_STS_OK;
 793                 cmd->nq = nq;
 794                 if (nq->dev->irqmode == NULL_IRQ_TIMER) {
 795                         hrtimer_init(&cmd->timer, CLOCK_MONOTONIC,
 796                                      HRTIMER_MODE_REL);
 797                         cmd->timer.function = null_cmd_timer_expired;
 798                 }
 799                 return cmd;
 800         }
 801
 802         return NULL;
 803 }
 804
 805 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, struct bio *bio)
 806 {
 807         struct nullb_cmd *cmd;
 808         DEFINE_WAIT(wait);
 809
 810         do {
 811                 /*
 812                  * This avoids multiple return statements, multiple calls to
 813                  * __alloc_cmd() and a fast path call to prepare_to_wait().
 814                  */
 815                 cmd = __alloc_cmd(nq);
 816                 if (cmd) {
 817                         cmd->bio = bio;
 818                         return cmd;
 819                 }
 820                 prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
 821                 io_schedule();
 822                 finish_wait(&nq->wait, &wait);
 823         } while (1);
 824 }
 825
 826 static void end_cmd(struct nullb_cmd *cmd)
 827 {
 828         int queue_mode = cmd->nq->dev->queue_mode;
 829
 830         switch (queue_mode)  {
 831         case NULL_Q_MQ:
 832                 blk_mq_end_request(cmd->rq, cmd->error);
 833                 return;
 834         case NULL_Q_BIO:
 835                 cmd->bio->bi_status = cmd->error;
 836                 bio_endio(cmd->bio);
 837                 break;
 838         }
 839
 840         free_cmd(cmd);
 841 }
 842
 843 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
 844 {
 845         end_cmd(container_of(timer, struct nullb_cmd, timer));
 846
 847         return HRTIMER_NORESTART;
 848 }
 849
 850 static void null_cmd_end_timer(struct nullb_cmd *cmd)
 851 {
 852         ktime_t kt = cmd->nq->dev->completion_nsec;
 853
 854         hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
 855 }
 856
 857 static void null_complete_rq(struct request *rq)
 858 {
 859         end_cmd(blk_mq_rq_to_pdu(rq));
 860 }
 861
 862 static struct nullb_page *null_alloc_page(void)
 863 {
 864         struct nullb_page *t_page;
 865
 866         t_page = kmalloc(sizeof(struct nullb_page), GFP_NOIO);
 867         if (!t_page)
 868                 return NULL;
 869
 870         t_page->page = alloc_pages(GFP_NOIO, 0);
 871         if (!t_page->page) {
 872                 kfree(t_page);
 873                 return NULL;
 874         }
 875
 876         memset(t_page->bitmap, 0, sizeof(t_page->bitmap));
 877         return t_page;
 878 }
 879
 880 static void null_free_page(struct nullb_page *t_page)
 881 {
 882         __set_bit(NULLB_PAGE_FREE, t_page->bitmap);
 883         if (test_bit(NULLB_PAGE_LOCK, t_page->bitmap))
 884                 return;
 885         __free_page(t_page->page);
 886         kfree(t_page);
 887 }
 888
 889 static bool null_page_empty(struct nullb_page *page)
 890 {
 891         int size = MAP_SZ - 2;
 892
 893         return find_first_bit(page->bitmap, size) == size;
 894 }
 895
 896 static void null_free_sector(struct nullb *nullb, sector_t sector,
 897         bool is_cache)
 898 {
 899         unsigned int sector_bit;
 900         u64 idx;
 901         struct nullb_page *t_page, *ret;
 902         struct radix_tree_root *root;
 903
 904         root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 905         idx = sector >> PAGE_SECTORS_SHIFT;
 906         sector_bit = (sector & SECTOR_MASK);
 907
 908         t_page = radix_tree_lookup(root, idx);
 909         if (t_page) {
 910                 __clear_bit(sector_bit, t_page->bitmap);
 911
 912                 if (null_page_empty(t_page)) {
 913                         ret = radix_tree_delete_item(root, idx, t_page);
 914                         WARN_ON(ret != t_page);
 915                         null_free_page(ret);
 916                         if (is_cache)
 917                                 nullb->dev->curr_cache -= PAGE_SIZE;
 918                 }
 919         }
 920 }
 921
 922 static struct nullb_page *null_radix_tree_insert(struct nullb *nullb, u64 idx,
 923         struct nullb_page *t_page, bool is_cache)
 924 {
 925         struct radix_tree_root *root;
 926
 927         root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 928
 929         if (radix_tree_insert(root, idx, t_page)) {
 930                 null_free_page(t_page);
 931                 t_page = radix_tree_lookup(root, idx);
 932                 WARN_ON(!t_page || t_page->page->index != idx);
 933         } else if (is_cache)
 934                 nullb->dev->curr_cache += PAGE_SIZE;
 935
 936         return t_page;
 937 }
 938
 939 static void null_free_device_storage(struct nullb_device *dev, bool is_cache)
 940 {
 941         unsigned long pos = 0;
 942         int nr_pages;
 943         struct nullb_page *ret, *t_pages[FREE_BATCH];
 944         struct radix_tree_root *root;
 945
 946         root = is_cache ? &dev->cache : &dev->data;
 947
 948         do {
 949                 int i;
 950
 951                 nr_pages = radix_tree_gang_lookup(root,
 952                                 (void **)t_pages, pos, FREE_BATCH);
 953
 954                 for (i = 0; i < nr_pages; i++) {
 955                         pos = t_pages[i]->page->index;
 956                         ret = radix_tree_delete_item(root, pos, t_pages[i]);
 957                         WARN_ON(ret != t_pages[i]);
 958                         null_free_page(ret);
 959                 }
 960
 961                 pos++;
 962         } while (nr_pages == FREE_BATCH);
 963
 964         if (is_cache)
 965                 dev->curr_cache = 0;
 966 }
 967
 968 static struct nullb_page *__null_lookup_page(struct nullb *nullb,
 969         sector_t sector, bool for_write, bool is_cache)
 970 {
 971         unsigned int sector_bit;
 972         u64 idx;
 973         struct nullb_page *t_page;
 974         struct radix_tree_root *root;
 975
 976         idx = sector >> PAGE_SECTORS_SHIFT;
 977         sector_bit = (sector & SECTOR_MASK);
 978
 979         root = is_cache ? &nullb->dev->cache : &nullb->dev->data;
 980         t_page = radix_tree_lookup(root, idx);
 981         WARN_ON(t_page && t_page->page->index != idx);
 982
 983         if (t_page && (for_write || test_bit(sector_bit, t_page->bitmap)))
 984                 return t_page;
 985
 986         return NULL;
 987 }
 988
 989 static struct nullb_page *null_lookup_page(struct nullb *nullb,
 990         sector_t sector, bool for_write, bool ignore_cache)
 991 {
 992         struct nullb_page *page = NULL;
 993
 994         if (!ignore_cache)
 995                 page = __null_lookup_page(nullb, sector, for_write, true);
 996         if (page)
 997                 return page;
 998         return __null_lookup_page(nullb, sector, for_write, false);
 999 }
1000
1001 static struct nullb_page *null_insert_page(struct nullb *nullb,
1002                                            sector_t sector, bool ignore_cache)
1003         __releases(&nullb->lock)
1004         __acquires(&nullb->lock)
1005 {
1006         u64 idx;
1007         struct nullb_page *t_page;
1008
1009         t_page = null_lookup_page(nullb, sector, true, ignore_cache);
1010         if (t_page)
1011                 return t_page;
1012
1013         spin_unlock_irq(&nullb->lock);
1014
1015         t_page = null_alloc_page();
1016         if (!t_page)
1017                 goto out_lock;
1018
1019         if (radix_tree_preload(GFP_NOIO))
1020                 goto out_freepage;
1021
1022         spin_lock_irq(&nullb->lock);
1023         idx = sector >> PAGE_SECTORS_SHIFT;
1024         t_page->page->index = idx;
1025         t_page = null_radix_tree_insert(nullb, idx, t_page, !ignore_cache);
1026         radix_tree_preload_end();
1027
1028         return t_page;
1029 out_freepage:
1030         null_free_page(t_page);
1031 out_lock:
1032         spin_lock_irq(&nullb->lock);
1033         return null_lookup_page(nullb, sector, true, ignore_cache);
1034 }
1035
1036 static int null_flush_cache_page(struct nullb *nullb, struct nullb_page *c_page)
1037 {
1038         int i;
1039         unsigned int offset;
1040         u64 idx;
1041         struct nullb_page *t_page, *ret;
1042         void *dst, *src;
1043
1044         idx = c_page->page->index;
1045
1046         t_page = null_insert_page(nullb, idx << PAGE_SECTORS_SHIFT, true);
1047
1048         __clear_bit(NULLB_PAGE_LOCK, c_page->bitmap);
1049         if (test_bit(NULLB_PAGE_FREE, c_page->bitmap)) {
1050                 null_free_page(c_page);
1051                 if (t_page && null_page_empty(t_page)) {
1052                         ret = radix_tree_delete_item(&nullb->dev->data,
1053                                 idx, t_page);
1054                         null_free_page(t_page);
1055                 }
1056                 return 0;
1057         }
1058
1059         if (!t_page)
1060                 return -ENOMEM;
1061
1062         src = kmap_local_page(c_page->page);
1063         dst = kmap_local_page(t_page->page);
1064
1065         for (i = 0; i < PAGE_SECTORS;
1066                         i += (nullb->dev->blocksize >> SECTOR_SHIFT)) {
1067                 if (test_bit(i, c_page->bitmap)) {
1068                         offset = (i << SECTOR_SHIFT);
1069                         memcpy(dst + offset, src + offset,
1070                                 nullb->dev->blocksize);
1071                         __set_bit(i, t_page->bitmap);
1072                 }
1073         }
1074
1075         kunmap_local(dst);
1076         kunmap_local(src);
1077
1078         ret = radix_tree_delete_item(&nullb->dev->cache, idx, c_page);
1079         null_free_page(ret);
1080         nullb->dev->curr_cache -= PAGE_SIZE;
1081
1082         return 0;
1083 }
1084
1085 static int null_make_cache_space(struct nullb *nullb, unsigned long n)
1086 {
1087         int i, err, nr_pages;
1088         struct nullb_page *c_pages[FREE_BATCH];
1089         unsigned long flushed = 0, one_round;
1090
1091 again:
1092         if ((nullb->dev->cache_size * 1024 * 1024) >
1093              nullb->dev->curr_cache + n || nullb->dev->curr_cache == 0)
1094                 return 0;
1095
1096         nr_pages = radix_tree_gang_lookup(&nullb->dev->cache,
1097                         (void **)c_pages, nullb->cache_flush_pos, FREE_BATCH);
1098         /*
1099          * nullb_flush_cache_page could unlock before using the c_pages. To
1100          * avoid race, we don't allow page free
1101          */
1102         for (i = 0; i < nr_pages; i++) {
1103                 nullb->cache_flush_pos = c_pages[i]->page->index;
1104                 /*
1105                  * We found the page which is being flushed to disk by other
1106                  * threads
1107                  */
1108                 if (test_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap))
1109                         c_pages[i] = NULL;
1110                 else
1111                         __set_bit(NULLB_PAGE_LOCK, c_pages[i]->bitmap);
1112         }
1113
1114         one_round = 0;
1115         for (i = 0; i < nr_pages; i++) {
1116                 if (c_pages[i] == NULL)
1117                         continue;
1118                 err = null_flush_cache_page(nullb, c_pages[i]);
1119                 if (err)
1120                         return err;
1121                 one_round++;
1122         }
1123         flushed += one_round << PAGE_SHIFT;
1124
1125         if (n > flushed) {
1126                 if (nr_pages == 0)
1127                         nullb->cache_flush_pos = 0;
1128                 if (one_round == 0) {
1129                         /* give other threads a chance */
1130                         spin_unlock_irq(&nullb->lock);
1131                         spin_lock_irq(&nullb->lock);
1132                 }
1133                 goto again;
1134         }
1135         return 0;
1136 }
1137
1138 static int copy_to_nullb(struct nullb *nullb, struct page *source,
1139         unsigned int off, sector_t sector, size_t n, bool is_fua)
1140 {
1141         size_t temp, count = 0;
1142         unsigned int offset;
1143         struct nullb_page *t_page;
1144
1145         while (count < n) {
1146                 temp = min_t(size_t, nullb->dev->blocksize, n - count);
1147
1148                 if (null_cache_active(nullb) && !is_fua)
1149                         null_make_cache_space(nullb, PAGE_SIZE);
1150
1151                 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
1152                 t_page = null_insert_page(nullb, sector,
1153                         !null_cache_active(nullb) || is_fua);
1154                 if (!t_page)
1155                         return -ENOSPC;
1156
1157                 memcpy_page(t_page->page, offset, source, off + count, temp);
1158
1159                 __set_bit(sector & SECTOR_MASK, t_page->bitmap);
1160
1161                 if (is_fua)
1162                         null_free_sector(nullb, sector, true);
1163
1164                 count += temp;
1165                 sector += temp >> SECTOR_SHIFT;
1166         }
1167         return 0;
1168 }
1169
1170 static int copy_from_nullb(struct nullb *nullb, struct page *dest,
1171         unsigned int off, sector_t sector, size_t n)
1172 {
1173         size_t temp, count = 0;
1174         unsigned int offset;
1175         struct nullb_page *t_page;
1176
1177         while (count < n) {
1178                 temp = min_t(size_t, nullb->dev->blocksize, n - count);
1179
1180                 offset = (sector & SECTOR_MASK) << SECTOR_SHIFT;
1181                 t_page = null_lookup_page(nullb, sector, false,
1182                         !null_cache_active(nullb));
1183
1184                 if (t_page)
1185                         memcpy_page(dest, off + count, t_page->page, offset,
1186                                     temp);
1187                 else
1188                         zero_user(dest, off + count, temp);
1189
1190                 count += temp;
1191                 sector += temp >> SECTOR_SHIFT;
1192         }
1193         return 0;
1194 }
1195
1196 static void nullb_fill_pattern(struct nullb *nullb, struct page *page,
1197                                unsigned int len, unsigned int off)
1198 {
1199         memset_page(page, off, 0xff, len);
1200 }
1201
1202 blk_status_t null_handle_discard(struct nullb_device *dev,
1203                                  sector_t sector, sector_t nr_sectors)
1204 {
1205         struct nullb *nullb = dev->nullb;
1206         size_t n = nr_sectors << SECTOR_SHIFT;
1207         size_t temp;
1208
1209         spin_lock_irq(&nullb->lock);
1210         while (n > 0) {
1211                 temp = min_t(size_t, n, dev->blocksize);
1212                 null_free_sector(nullb, sector, false);
1213                 if (null_cache_active(nullb))
1214                         null_free_sector(nullb, sector, true);
1215                 sector += temp >> SECTOR_SHIFT;
1216                 n -= temp;
1217         }
1218         spin_unlock_irq(&nullb->lock);
1219
1220         return BLK_STS_OK;
1221 }
1222
1223 static int null_handle_flush(struct nullb *nullb)
1224 {
1225         int err;
1226
1227         if (!null_cache_active(nullb))
1228                 return 0;
1229
1230         spin_lock_irq(&nullb->lock);
1231         while (true) {
1232                 err = null_make_cache_space(nullb,
1233                         nullb->dev->cache_size * 1024 * 1024);
1234                 if (err || nullb->dev->curr_cache == 0)
1235                         break;
1236         }
1237
1238         WARN_ON(!radix_tree_empty(&nullb->dev->cache));
1239         spin_unlock_irq(&nullb->lock);
1240         return err;
1241 }
1242
1243 static int null_transfer(struct nullb *nullb, struct page *page,
1244         unsigned int len, unsigned int off, bool is_write, sector_t sector,
1245         bool is_fua)
1246 {
1247         struct nullb_device *dev = nullb->dev;
1248         unsigned int valid_len = len;
1249         int err = 0;
1250
1251         if (!is_write) {
1252                 if (dev->zoned)
1253                         valid_len = null_zone_valid_read_len(nullb,
1254                                 sector, len);
1255
1256                 if (valid_len) {
1257                         err = copy_from_nullb(nullb, page, off,
1258                                 sector, valid_len);
1259                         off += valid_len;
1260                         len -= valid_len;
1261                 }
1262
1263                 if (len)
1264                         nullb_fill_pattern(nullb, page, len, off);
1265                 flush_dcache_page(page);
1266         } else {
1267                 flush_dcache_page(page);
1268                 err = copy_to_nullb(nullb, page, off, sector, len, is_fua);
1269         }
1270
1271         return err;
1272 }
1273
1274 static int null_handle_rq(struct nullb_cmd *cmd)
1275 {
1276         struct request *rq = cmd->rq;
1277         struct nullb *nullb = cmd->nq->dev->nullb;
1278         int err;
1279         unsigned int len;
1280         sector_t sector = blk_rq_pos(rq);
1281         struct req_iterator iter;
1282         struct bio_vec bvec;
1283
1284         spin_lock_irq(&nullb->lock);
1285         rq_for_each_segment(bvec, rq, iter) {
1286                 len = bvec.bv_len;
1287                 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1288                                      op_is_write(req_op(rq)), sector,
1289                                      rq->cmd_flags & REQ_FUA);
1290                 if (err) {
1291                         spin_unlock_irq(&nullb->lock);
1292                         return err;
1293                 }
1294                 sector += len >> SECTOR_SHIFT;
1295         }
1296         spin_unlock_irq(&nullb->lock);
1297
1298         return 0;
1299 }
1300
1301 static int null_handle_bio(struct nullb_cmd *cmd)
1302 {
1303         struct bio *bio = cmd->bio;
1304         struct nullb *nullb = cmd->nq->dev->nullb;
1305         int err;
1306         unsigned int len;
1307         sector_t sector = bio->bi_iter.bi_sector;
1308         struct bio_vec bvec;
1309         struct bvec_iter iter;
1310
1311         spin_lock_irq(&nullb->lock);
1312         bio_for_each_segment(bvec, bio, iter) {
1313                 len = bvec.bv_len;
1314                 err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset,
1315                                      op_is_write(bio_op(bio)), sector,
1316                                      bio->bi_opf & REQ_FUA);
1317                 if (err) {
1318                         spin_unlock_irq(&nullb->lock);
1319                         return err;
1320                 }
1321                 sector += len >> SECTOR_SHIFT;
1322         }
1323         spin_unlock_irq(&nullb->lock);
1324         return 0;
1325 }
1326
1327 static void null_stop_queue(struct nullb *nullb)
1328 {
1329         struct request_queue *q = nullb->q;
1330
1331         if (nullb->dev->queue_mode == NULL_Q_MQ)
1332                 blk_mq_stop_hw_queues(q);
1333 }
1334
1335 static void null_restart_queue_async(struct nullb *nullb)
1336 {
1337         struct request_queue *q = nullb->q;
1338
1339         if (nullb->dev->queue_mode == NULL_Q_MQ)
1340                 blk_mq_start_stopped_hw_queues(q, true);
1341 }
1342
1343 static inline blk_status_t null_handle_throttled(struct nullb_cmd *cmd)
1344 {
1345         struct nullb_device *dev = cmd->nq->dev;
1346         struct nullb *nullb = dev->nullb;
1347         blk_status_t sts = BLK_STS_OK;
1348         struct request *rq = cmd->rq;
1349
1350         if (!hrtimer_active(&nullb->bw_timer))
1351                 hrtimer_restart(&nullb->bw_timer);
1352
1353         if (atomic_long_sub_return(blk_rq_bytes(rq), &nullb->cur_bytes) < 0) {
1354                 null_stop_queue(nullb);
1355                 /* race with timer */
1356                 if (atomic_long_read(&nullb->cur_bytes) > 0)
1357                         null_restart_queue_async(nullb);
1358                 /* requeue request */
1359                 sts = BLK_STS_DEV_RESOURCE;
1360         }
1361         return sts;
1362 }
1363
1364 static inline blk_status_t null_handle_badblocks(struct nullb_cmd *cmd,
1365                                                  sector_t sector,
1366                                                  sector_t nr_sectors)
1367 {
1368         struct badblocks *bb = &cmd->nq->dev->badblocks;
1369         sector_t first_bad;
1370         int bad_sectors;
1371
1372         if (badblocks_check(bb, sector, nr_sectors, &first_bad, &bad_sectors))
1373                 return BLK_STS_IOERR;
1374
1375         return BLK_STS_OK;
1376 }
1377
1378 static inline blk_status_t null_handle_memory_backed(struct nullb_cmd *cmd,
1379                                                      enum req_op op,
1380                                                      sector_t sector,
1381                                                      sector_t nr_sectors)
1382 {
1383         struct nullb_device *dev = cmd->nq->dev;
1384         int err;
1385
1386         if (op == REQ_OP_DISCARD)
1387                 return null_handle_discard(dev, sector, nr_sectors);
1388
1389         if (dev->queue_mode == NULL_Q_BIO)
1390                 err = null_handle_bio(cmd);
1391         else
1392                 err = null_handle_rq(cmd);
1393
1394         return errno_to_blk_status(err);
1395 }
1396
1397 static void nullb_zero_read_cmd_buffer(struct nullb_cmd *cmd)
1398 {
1399         struct nullb_device *dev = cmd->nq->dev;
1400         struct bio *bio;
1401
1402         if (dev->memory_backed)
1403                 return;
1404
1405         if (dev->queue_mode == NULL_Q_BIO && bio_op(cmd->bio) == REQ_OP_READ) {
1406                 zero_fill_bio(cmd->bio);
1407         } else if (req_op(cmd->rq) == REQ_OP_READ) {
1408                 __rq_for_each_bio(bio, cmd->rq)
1409                         zero_fill_bio(bio);
1410         }
1411 }
1412
1413 static inline void nullb_complete_cmd(struct nullb_cmd *cmd)
1414 {
1415         /*
1416          * Since root privileges are required to configure the null_blk
1417          * driver, it is fine that this driver does not initialize the
1418          * data buffers of read commands. Zero-initialize these buffers
1419          * anyway if KMSAN is enabled to prevent that KMSAN complains
1420          * about null_blk not initializing read data buffers.
1421          */
1422         if (IS_ENABLED(CONFIG_KMSAN))
1423                 nullb_zero_read_cmd_buffer(cmd);
1424
1425         /* Complete IO by inline, softirq or timer */
1426         switch (cmd->nq->dev->irqmode) {
1427         case NULL_IRQ_SOFTIRQ:
1428                 switch (cmd->nq->dev->queue_mode) {
1429                 case NULL_Q_MQ:
1430                         blk_mq_complete_request(cmd->rq);
1431                         break;
1432                 case NULL_Q_BIO:
1433                         /*
1434                          * XXX: no proper submitting cpu information available.
1435                          */
1436                         end_cmd(cmd);
1437                         break;
1438                 }
1439                 break;
1440         case NULL_IRQ_NONE:
1441                 end_cmd(cmd);
1442                 break;
1443         case NULL_IRQ_TIMER:
1444                 null_cmd_end_timer(cmd);
1445                 break;
1446         }
1447 }
1448
1449 blk_status_t null_process_cmd(struct nullb_cmd *cmd, enum req_op op,
1450                               sector_t sector, unsigned int nr_sectors)
1451 {
1452         struct nullb_device *dev = cmd->nq->dev;
1453         blk_status_t ret;
1454
1455         if (dev->badblocks.shift != -1) {
1456                 ret = null_handle_badblocks(cmd, sector, nr_sectors);
1457                 if (ret != BLK_STS_OK)
1458                         return ret;
1459         }
1460
1461         if (dev->memory_backed)
1462                 return null_handle_memory_backed(cmd, op, sector, nr_sectors);
1463
1464         return BLK_STS_OK;
1465 }
1466
1467 static void null_handle_cmd(struct nullb_cmd *cmd, sector_t sector,
1468                             sector_t nr_sectors, enum req_op op)
1469 {
1470         struct nullb_device *dev = cmd->nq->dev;
1471         struct nullb *nullb = dev->nullb;
1472         blk_status_t sts;
1473
1474         if (op == REQ_OP_FLUSH) {
1475                 cmd->error = errno_to_blk_status(null_handle_flush(nullb));
1476                 goto out;
1477         }
1478
1479         if (dev->zoned)
1480                 sts = null_process_zoned_cmd(cmd, op, sector, nr_sectors);
1481         else
1482                 sts = null_process_cmd(cmd, op, sector, nr_sectors);
1483
1484         /* Do not overwrite errors (e.g. timeout errors) */
1485         if (cmd->error == BLK_STS_OK)
1486                 cmd->error = sts;
1487
1488 out:
1489         nullb_complete_cmd(cmd);
1490 }
1491
1492 static enum hrtimer_restart nullb_bwtimer_fn(struct hrtimer *timer)
1493 {
1494         struct nullb *nullb = container_of(timer, struct nullb, bw_timer);
1495         ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1496         unsigned int mbps = nullb->dev->mbps;
1497
1498         if (atomic_long_read(&nullb->cur_bytes) == mb_per_tick(mbps))
1499                 return HRTIMER_NORESTART;
1500
1501         atomic_long_set(&nullb->cur_bytes, mb_per_tick(mbps));
1502         null_restart_queue_async(nullb);
1503
1504         hrtimer_forward_now(&nullb->bw_timer, timer_interval);
1505
1506         return HRTIMER_RESTART;
1507 }
1508
1509 static void nullb_setup_bwtimer(struct nullb *nullb)
1510 {
1511         ktime_t timer_interval = ktime_set(0, TIMER_INTERVAL);
1512
1513         hrtimer_init(&nullb->bw_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1514         nullb->bw_timer.function = nullb_bwtimer_fn;
1515         atomic_long_set(&nullb->cur_bytes, mb_per_tick(nullb->dev->mbps));
1516         hrtimer_start(&nullb->bw_timer, timer_interval, HRTIMER_MODE_REL);
1517 }
1518
1519 static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
1520 {
1521         int index = 0;
1522
1523         if (nullb->nr_queues != 1)
1524                 index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
1525
1526         return &nullb->queues[index];
1527 }
1528
1529 static void null_submit_bio(struct bio *bio)
1530 {
1531         sector_t sector = bio->bi_iter.bi_sector;
1532         sector_t nr_sectors = bio_sectors(bio);
1533         struct nullb *nullb = bio->bi_bdev->bd_disk->private_data;
1534         struct nullb_queue *nq = nullb_to_queue(nullb);
1535
1536         null_handle_cmd(alloc_cmd(nq, bio), sector, nr_sectors, bio_op(bio));
1537 }
1538
1539 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
1540
1541 static bool should_timeout_request(struct request *rq)
1542 {
1543         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1544         struct nullb_device *dev = cmd->nq->dev;
1545
1546         return should_fail(&dev->timeout_config.attr, 1);
1547 }
1548
1549 static bool should_requeue_request(struct request *rq)
1550 {
1551         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1552         struct nullb_device *dev = cmd->nq->dev;
1553
1554         return should_fail(&dev->requeue_config.attr, 1);
1555 }
1556
1557 static bool should_init_hctx_fail(struct nullb_device *dev)
1558 {
1559         return should_fail(&dev->init_hctx_fault_config.attr, 1);
1560 }
1561
1562 #else
1563
1564 static bool should_timeout_request(struct request *rq)
1565 {
1566         return false;
1567 }
1568
1569 static bool should_requeue_request(struct request *rq)
1570 {
1571         return false;
1572 }
1573
1574 static bool should_init_hctx_fail(struct nullb_device *dev)
1575 {
1576         return false;
1577 }
1578
1579 #endif
1580
1581 static void null_map_queues(struct blk_mq_tag_set *set)
1582 {
1583         struct nullb *nullb = set->driver_data;
1584         int i, qoff;
1585         unsigned int submit_queues = g_submit_queues;
1586         unsigned int poll_queues = g_poll_queues;
1587
1588         if (nullb) {
1589                 struct nullb_device *dev = nullb->dev;
1590
1591                 /*
1592                  * Refer nr_hw_queues of the tag set to check if the expected
1593                  * number of hardware queues are prepared. If block layer failed
1594                  * to prepare them, use previous numbers of submit queues and
1595                  * poll queues to map queues.
1596                  */
1597                 if (set->nr_hw_queues ==
1598                     dev->submit_queues + dev->poll_queues) {
1599                         submit_queues = dev->submit_queues;
1600                         poll_queues = dev->poll_queues;
1601                 } else if (set->nr_hw_queues ==
1602                            dev->prev_submit_queues + dev->prev_poll_queues) {
1603                         submit_queues = dev->prev_submit_queues;
1604                         poll_queues = dev->prev_poll_queues;
1605                 } else {
1606                         pr_warn("tag set has unexpected nr_hw_queues: %d\n",
1607                                 set->nr_hw_queues);
1608                         WARN_ON_ONCE(true);
1609                         submit_queues = 1;
1610                         poll_queues = 0;
1611                 }
1612         }
1613
1614         for (i = 0, qoff = 0; i < set->nr_maps; i++) {
1615                 struct blk_mq_queue_map *map = &set->map[i];
1616
1617                 switch (i) {
1618                 case HCTX_TYPE_DEFAULT:
1619                         map->nr_queues = submit_queues;
1620                         break;
1621                 case HCTX_TYPE_READ:
1622                         map->nr_queues = 0;
1623                         continue;
1624                 case HCTX_TYPE_POLL:
1625                         map->nr_queues = poll_queues;
1626                         break;
1627                 }
1628                 map->queue_offset = qoff;
1629                 qoff += map->nr_queues;
1630                 blk_mq_map_queues(map);
1631         }
1632 }
1633
1634 static int null_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
1635 {
1636         struct nullb_queue *nq = hctx->driver_data;
1637         LIST_HEAD(list);
1638         int nr = 0;
1639         struct request *rq;
1640
1641         spin_lock(&nq->poll_lock);
1642         list_splice_init(&nq->poll_list, &list);
1643         list_for_each_entry(rq, &list, queuelist)
1644                 blk_mq_set_request_complete(rq);
1645         spin_unlock(&nq->poll_lock);
1646
1647         while (!list_empty(&list)) {
1648                 struct nullb_cmd *cmd;
1649                 struct request *req;
1650
1651                 req = list_first_entry(&list, struct request, queuelist);
1652                 list_del_init(&req->queuelist);
1653                 cmd = blk_mq_rq_to_pdu(req);
1654                 cmd->error = null_process_cmd(cmd, req_op(req), blk_rq_pos(req),
1655                                                 blk_rq_sectors(req));
1656                 if (!blk_mq_add_to_batch(req, iob, (__force int) cmd->error,
1657                                         blk_mq_end_request_batch))
1658                         end_cmd(cmd);
1659                 nr++;
1660         }
1661
1662         return nr;
1663 }
1664
1665 static enum blk_eh_timer_return null_timeout_rq(struct request *rq)
1666 {
1667         struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1668         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1669
1670         if (hctx->type == HCTX_TYPE_POLL) {
1671                 struct nullb_queue *nq = hctx->driver_data;
1672
1673                 spin_lock(&nq->poll_lock);
1674                 /* The request may have completed meanwhile. */
1675                 if (blk_mq_request_completed(rq)) {
1676                         spin_unlock(&nq->poll_lock);
1677                         return BLK_EH_DONE;
1678                 }
1679                 list_del_init(&rq->queuelist);
1680                 spin_unlock(&nq->poll_lock);
1681         }
1682
1683         pr_info("rq %p timed out\n", rq);
1684
1685         /*
1686          * If the device is marked as blocking (i.e. memory backed or zoned
1687          * device), the submission path may be blocked waiting for resources
1688          * and cause real timeouts. For these real timeouts, the submission
1689          * path will complete the request using blk_mq_complete_request().
1690          * Only fake timeouts need to execute blk_mq_complete_request() here.
1691          */
1692         cmd->error = BLK_STS_TIMEOUT;
1693         if (cmd->fake_timeout || hctx->type == HCTX_TYPE_POLL)
1694                 blk_mq_complete_request(rq);
1695         return BLK_EH_DONE;
1696 }
1697
1698 static blk_status_t null_queue_rq(struct blk_mq_hw_ctx *hctx,
1699                                   const struct blk_mq_queue_data *bd)
1700 {
1701         struct request *rq = bd->rq;
1702         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(rq);
1703         struct nullb_queue *nq = hctx->driver_data;
1704         sector_t nr_sectors = blk_rq_sectors(rq);
1705         sector_t sector = blk_rq_pos(rq);
1706         const bool is_poll = hctx->type == HCTX_TYPE_POLL;
1707
1708         might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1709
1710         if (!is_poll && nq->dev->irqmode == NULL_IRQ_TIMER) {
1711                 hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1712                 cmd->timer.function = null_cmd_timer_expired;
1713         }
1714         cmd->rq = rq;
1715         cmd->error = BLK_STS_OK;
1716         cmd->nq = nq;
1717         cmd->fake_timeout = should_timeout_request(rq) ||
1718                 blk_should_fake_timeout(rq->q);
1719
1720         if (should_requeue_request(rq)) {
1721                 /*
1722                  * Alternate between hitting the core BUSY path, and the
1723                  * driver driven requeue path
1724                  */
1725                 nq->requeue_selection++;
1726                 if (nq->requeue_selection & 1)
1727                         return BLK_STS_RESOURCE;
1728                 blk_mq_requeue_request(rq, true);
1729                 return BLK_STS_OK;
1730         }
1731
1732         if (test_bit(NULLB_DEV_FL_THROTTLED, &nq->dev->flags)) {
1733                 blk_status_t sts = null_handle_throttled(cmd);
1734
1735                 if (sts != BLK_STS_OK)
1736                         return sts;
1737         }
1738
1739         blk_mq_start_request(rq);
1740
1741         if (is_poll) {
1742                 spin_lock(&nq->poll_lock);
1743                 list_add_tail(&rq->queuelist, &nq->poll_list);
1744                 spin_unlock(&nq->poll_lock);
1745                 return BLK_STS_OK;
1746         }
1747         if (cmd->fake_timeout)
1748                 return BLK_STS_OK;
1749
1750         null_handle_cmd(cmd, sector, nr_sectors, req_op(rq));
1751         return BLK_STS_OK;
1752 }
1753
1754 static void null_queue_rqs(struct request **rqlist)
1755 {
1756         struct request *requeue_list = NULL;
1757         struct request **requeue_lastp = &requeue_list;
1758         struct blk_mq_queue_data bd = { };
1759         blk_status_t ret;
1760
1761         do {
1762                 struct request *rq = rq_list_pop(rqlist);
1763
1764                 bd.rq = rq;
1765                 ret = null_queue_rq(rq->mq_hctx, &bd);
1766                 if (ret != BLK_STS_OK)
1767                         rq_list_add_tail(&requeue_lastp, rq);
1768         } while (!rq_list_empty(*rqlist));
1769
1770         *rqlist = requeue_list;
1771 }
1772
1773 static void cleanup_queue(struct nullb_queue *nq)
1774 {
1775         bitmap_free(nq->tag_map);
1776         kfree(nq->cmds);
1777 }
1778
1779 static void cleanup_queues(struct nullb *nullb)
1780 {
1781         int i;
1782
1783         for (i = 0; i < nullb->nr_queues; i++)
1784                 cleanup_queue(&nullb->queues[i]);
1785
1786         kfree(nullb->queues);
1787 }
1788
1789 static void null_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
1790 {
1791         struct nullb_queue *nq = hctx->driver_data;
1792         struct nullb *nullb = nq->dev->nullb;
1793
1794         nullb->nr_queues--;
1795 }
1796
1797 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
1798 {
1799         init_waitqueue_head(&nq->wait);
1800         nq->queue_depth = nullb->queue_depth;
1801         nq->dev = nullb->dev;
1802         INIT_LIST_HEAD(&nq->poll_list);
1803         spin_lock_init(&nq->poll_lock);
1804 }
1805
1806 static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *driver_data,
1807                           unsigned int hctx_idx)
1808 {
1809         struct nullb *nullb = hctx->queue->queuedata;
1810         struct nullb_queue *nq;
1811
1812         if (should_init_hctx_fail(nullb->dev))
1813                 return -EFAULT;
1814
1815         nq = &nullb->queues[hctx_idx];
1816         hctx->driver_data = nq;
1817         null_init_queue(nullb, nq);
1818         nullb->nr_queues++;
1819
1820         return 0;
1821 }
1822
1823 static const struct blk_mq_ops null_mq_ops = {
1824         .queue_rq       = null_queue_rq,
1825         .queue_rqs      = null_queue_rqs,
1826         .complete       = null_complete_rq,
1827         .timeout        = null_timeout_rq,
1828         .poll           = null_poll,
1829         .map_queues     = null_map_queues,
1830         .init_hctx      = null_init_hctx,
1831         .exit_hctx      = null_exit_hctx,
1832 };
1833
1834 static void null_del_dev(struct nullb *nullb)
1835 {
1836         struct nullb_device *dev;
1837
1838         if (!nullb)
1839                 return;
1840
1841         dev = nullb->dev;
1842
1843         ida_simple_remove(&nullb_indexes, nullb->index);
1844
1845         list_del_init(&nullb->list);
1846
1847         del_gendisk(nullb->disk);
1848
1849         if (test_bit(NULLB_DEV_FL_THROTTLED, &nullb->dev->flags)) {
1850                 hrtimer_cancel(&nullb->bw_timer);
1851                 atomic_long_set(&nullb->cur_bytes, LONG_MAX);
1852                 null_restart_queue_async(nullb);
1853         }
1854
1855         put_disk(nullb->disk);
1856         if (dev->queue_mode == NULL_Q_MQ &&
1857             nullb->tag_set == &nullb->__tag_set)
1858                 blk_mq_free_tag_set(nullb->tag_set);
1859         cleanup_queues(nullb);
1860         if (null_cache_active(nullb))
1861                 null_free_device_storage(nullb->dev, true);
1862         kfree(nullb);
1863         dev->nullb = NULL;
1864 }
1865
1866 static void null_config_discard(struct nullb *nullb)
1867 {
1868         if (nullb->dev->discard == false)
1869                 return;
1870
1871         if (!nullb->dev->memory_backed) {
1872                 nullb->dev->discard = false;
1873                 pr_info("discard option is ignored without memory backing\n");
1874                 return;
1875         }
1876
1877         if (nullb->dev->zoned) {
1878                 nullb->dev->discard = false;
1879                 pr_info("discard option is ignored in zoned mode\n");
1880                 return;
1881         }
1882
1883         nullb->q->limits.discard_granularity = nullb->dev->blocksize;
1884         blk_queue_max_discard_sectors(nullb->q, UINT_MAX >> 9);
1885 }
1886
1887 static const struct block_device_operations null_bio_ops = {
1888         .owner          = THIS_MODULE,
1889         .submit_bio     = null_submit_bio,
1890         .report_zones   = null_report_zones,
1891 };
1892
1893 static const struct block_device_operations null_rq_ops = {
1894         .owner          = THIS_MODULE,
1895         .report_zones   = null_report_zones,
1896 };
1897
1898 static int setup_commands(struct nullb_queue *nq)
1899 {
1900         struct nullb_cmd *cmd;
1901         int i;
1902
1903         nq->cmds = kcalloc(nq->queue_depth, sizeof(*cmd), GFP_KERNEL);
1904         if (!nq->cmds)
1905                 return -ENOMEM;
1906
1907         nq->tag_map = bitmap_zalloc(nq->queue_depth, GFP_KERNEL);
1908         if (!nq->tag_map) {
1909                 kfree(nq->cmds);
1910                 return -ENOMEM;
1911         }
1912
1913         for (i = 0; i < nq->queue_depth; i++) {
1914                 cmd = &nq->cmds[i];
1915                 cmd->tag = -1U;
1916         }
1917
1918         return 0;
1919 }
1920
1921 static int setup_queues(struct nullb *nullb)
1922 {
1923         int nqueues = nr_cpu_ids;
1924
1925         if (g_poll_queues)
1926                 nqueues += g_poll_queues;
1927
1928         nullb->queues = kcalloc(nqueues, sizeof(struct nullb_queue),
1929                                 GFP_KERNEL);
1930         if (!nullb->queues)
1931                 return -ENOMEM;
1932
1933         nullb->queue_depth = nullb->dev->hw_queue_depth;
1934         return 0;
1935 }
1936
1937 static int init_driver_queues(struct nullb *nullb)
1938 {
1939         struct nullb_queue *nq;
1940         int i, ret = 0;
1941
1942         for (i = 0; i < nullb->dev->submit_queues; i++) {
1943                 nq = &nullb->queues[i];
1944
1945                 null_init_queue(nullb, nq);
1946
1947                 ret = setup_commands(nq);
1948                 if (ret)
1949                         return ret;
1950                 nullb->nr_queues++;
1951         }
1952         return 0;
1953 }
1954
1955 static int null_gendisk_register(struct nullb *nullb)
1956 {
1957         sector_t size = ((sector_t)nullb->dev->size * SZ_1M) >> SECTOR_SHIFT;
1958         struct gendisk *disk = nullb->disk;
1959
1960         set_capacity(disk, size);
1961
1962         disk->major             = null_major;
1963         disk->first_minor       = nullb->index;
1964         disk->minors            = 1;
1965         if (queue_is_mq(nullb->q))
1966                 disk->fops              = &null_rq_ops;
1967         else
1968                 disk->fops              = &null_bio_ops;
1969         disk->private_data      = nullb;
1970         strscpy_pad(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
1971
1972         if (nullb->dev->zoned) {
1973                 int ret = null_register_zoned_dev(nullb);
1974
1975                 if (ret)
1976                         return ret;
1977         }
1978
1979         return add_disk(disk);
1980 }
1981
1982 static int null_init_tag_set(struct nullb *nullb, struct blk_mq_tag_set *set)
1983 {
1984         unsigned int flags = BLK_MQ_F_SHOULD_MERGE;
1985         int hw_queues, numa_node;
1986         unsigned int queue_depth;
1987         int poll_queues;
1988
1989         if (nullb) {
1990                 hw_queues = nullb->dev->submit_queues;
1991                 poll_queues = nullb->dev->poll_queues;
1992                 queue_depth = nullb->dev->hw_queue_depth;
1993                 numa_node = nullb->dev->home_node;
1994                 if (nullb->dev->no_sched)
1995                         flags |= BLK_MQ_F_NO_SCHED;
1996                 if (nullb->dev->shared_tag_bitmap)
1997                         flags |= BLK_MQ_F_TAG_HCTX_SHARED;
1998                 if (nullb->dev->blocking)
1999                         flags |= BLK_MQ_F_BLOCKING;
2000         } else {
2001                 hw_queues = g_submit_queues;
2002                 poll_queues = g_poll_queues;
2003                 queue_depth = g_hw_queue_depth;
2004                 numa_node = g_home_node;
2005                 if (g_no_sched)
2006                         flags |= BLK_MQ_F_NO_SCHED;
2007                 if (g_shared_tag_bitmap)
2008                         flags |= BLK_MQ_F_TAG_HCTX_SHARED;
2009                 if (g_blocking)
2010                         flags |= BLK_MQ_F_BLOCKING;
2011         }
2012
2013         set->ops = &null_mq_ops;
2014         set->cmd_size   = sizeof(struct nullb_cmd);
2015         set->flags = flags;
2016         set->driver_data = nullb;
2017         set->nr_hw_queues = hw_queues;
2018         set->queue_depth = queue_depth;
2019         set->numa_node = numa_node;
2020         if (poll_queues) {
2021                 set->nr_hw_queues += poll_queues;
2022                 set->nr_maps = 3;
2023         } else {
2024                 set->nr_maps = 1;
2025         }
2026
2027         return blk_mq_alloc_tag_set(set);
2028 }
2029
2030 static int null_validate_conf(struct nullb_device *dev)
2031 {
2032         if (dev->queue_mode == NULL_Q_RQ) {
2033                 pr_err("legacy IO path is no longer available\n");
2034                 return -EINVAL;
2035         }
2036
2037         dev->blocksize = round_down(dev->blocksize, 512);
2038         dev->blocksize = clamp_t(unsigned int, dev->blocksize, 512, 4096);
2039
2040         if (dev->queue_mode == NULL_Q_MQ && dev->use_per_node_hctx) {
2041                 if (dev->submit_queues != nr_online_nodes)
2042                         dev->submit_queues = nr_online_nodes;
2043         } else if (dev->submit_queues > nr_cpu_ids)
2044                 dev->submit_queues = nr_cpu_ids;
2045         else if (dev->submit_queues == 0)
2046                 dev->submit_queues = 1;
2047         dev->prev_submit_queues = dev->submit_queues;
2048
2049         if (dev->poll_queues > g_poll_queues)
2050                 dev->poll_queues = g_poll_queues;
2051         dev->prev_poll_queues = dev->poll_queues;
2052
2053         dev->queue_mode = min_t(unsigned int, dev->queue_mode, NULL_Q_MQ);
2054         dev->irqmode = min_t(unsigned int, dev->irqmode, NULL_IRQ_TIMER);
2055
2056         /* Do memory allocation, so set blocking */
2057         if (dev->memory_backed)
2058                 dev->blocking = true;
2059         else /* cache is meaningless */
2060                 dev->cache_size = 0;
2061         dev->cache_size = min_t(unsigned long, ULONG_MAX / 1024 / 1024,
2062                                                 dev->cache_size);
2063         dev->mbps = min_t(unsigned int, 1024 * 40, dev->mbps);
2064         /* can not stop a queue */
2065         if (dev->queue_mode == NULL_Q_BIO)
2066                 dev->mbps = 0;
2067
2068         if (dev->zoned &&
2069             (!dev->zone_size || !is_power_of_2(dev->zone_size))) {
2070                 pr_err("zone_size must be power-of-two\n");
2071                 return -EINVAL;
2072         }
2073
2074         return 0;
2075 }
2076
2077 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
2078 static bool __null_setup_fault(struct fault_attr *attr, char *str)
2079 {
2080         if (!str[0])
2081                 return true;
2082
2083         if (!setup_fault_attr(attr, str))
2084                 return false;
2085
2086         attr->verbose = 0;
2087         return true;
2088 }
2089 #endif
2090
2091 static bool null_setup_fault(void)
2092 {
2093 #ifdef CONFIG_BLK_DEV_NULL_BLK_FAULT_INJECTION
2094         if (!__null_setup_fault(&null_timeout_attr, g_timeout_str))
2095                 return false;
2096         if (!__null_setup_fault(&null_requeue_attr, g_requeue_str))
2097                 return false;
2098         if (!__null_setup_fault(&null_init_hctx_attr, g_init_hctx_str))
2099                 return false;
2100 #endif
2101         return true;
2102 }
2103
2104 static int null_add_dev(struct nullb_device *dev)
2105 {
2106         struct nullb *nullb;
2107         int rv;
2108
2109         rv = null_validate_conf(dev);
2110         if (rv)
2111                 return rv;
2112
2113         nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, dev->home_node);
2114         if (!nullb) {
2115                 rv = -ENOMEM;
2116                 goto out;
2117         }
2118         nullb->dev = dev;
2119         dev->nullb = nullb;
2120
2121         spin_lock_init(&nullb->lock);
2122
2123         rv = setup_queues(nullb);
2124         if (rv)
2125                 goto out_free_nullb;
2126
2127         if (dev->queue_mode == NULL_Q_MQ) {
2128                 if (shared_tags) {
2129                         nullb->tag_set = &tag_set;
2130                         rv = 0;
2131                 } else {
2132                         nullb->tag_set = &nullb->__tag_set;
2133                         rv = null_init_tag_set(nullb, nullb->tag_set);
2134                 }
2135
2136                 if (rv)
2137                         goto out_cleanup_queues;
2138
2139                 nullb->tag_set->timeout = 5 * HZ;
2140                 nullb->disk = blk_mq_alloc_disk(nullb->tag_set, nullb);
2141                 if (IS_ERR(nullb->disk)) {
2142                         rv = PTR_ERR(nullb->disk);
2143                         goto out_cleanup_tags;
2144                 }
2145                 nullb->q = nullb->disk->queue;
2146         } else if (dev->queue_mode == NULL_Q_BIO) {
2147                 rv = -ENOMEM;
2148                 nullb->disk = blk_alloc_disk(nullb->dev->home_node);
2149                 if (!nullb->disk)
2150                         goto out_cleanup_queues;
2151
2152                 nullb->q = nullb->disk->queue;
2153                 rv = init_driver_queues(nullb);
2154                 if (rv)
2155                         goto out_cleanup_disk;
2156         }
2157
2158         if (dev->mbps) {
2159                 set_bit(NULLB_DEV_FL_THROTTLED, &dev->flags);
2160                 nullb_setup_bwtimer(nullb);
2161         }
2162
2163         if (dev->cache_size > 0) {
2164                 set_bit(NULLB_DEV_FL_CACHE, &nullb->dev->flags);
2165                 blk_queue_write_cache(nullb->q, true, true);
2166         }
2167
2168         if (dev->zoned) {
2169                 rv = null_init_zoned_dev(dev, nullb->q);
2170                 if (rv)
2171                         goto out_cleanup_disk;
2172         }
2173
2174         nullb->q->queuedata = nullb;
2175         blk_queue_flag_set(QUEUE_FLAG_NONROT, nullb->q);
2176
2177         mutex_lock(&lock);
2178         rv = ida_simple_get(&nullb_indexes, 0, 0, GFP_KERNEL);
2179         if (rv < 0) {
2180                 mutex_unlock(&lock);
2181                 goto out_cleanup_zone;
2182         }
2183         nullb->index = rv;
2184         dev->index = rv;
2185         mutex_unlock(&lock);
2186
2187         blk_queue_logical_block_size(nullb->q, dev->blocksize);
2188         blk_queue_physical_block_size(nullb->q, dev->blocksize);
2189         if (dev->max_sectors)
2190                 blk_queue_max_hw_sectors(nullb->q, dev->max_sectors);
2191
2192         if (dev->virt_boundary)
2193                 blk_queue_virt_boundary(nullb->q, PAGE_SIZE - 1);
2194
2195         null_config_discard(nullb);
2196
2197         if (config_item_name(&dev->group.cg_item)) {
2198                 /* Use configfs dir name as the device name */
2199                 snprintf(nullb->disk_name, sizeof(nullb->disk_name),
2200                          "%s", config_item_name(&dev->group.cg_item));
2201         } else {
2202                 sprintf(nullb->disk_name, "nullb%d", nullb->index);
2203         }
2204
2205         rv = null_gendisk_register(nullb);
2206         if (rv)
2207                 goto out_ida_free;
2208
2209         mutex_lock(&lock);
2210         list_add_tail(&nullb->list, &nullb_list);
2211         mutex_unlock(&lock);
2212
2213         pr_info("disk %s created\n", nullb->disk_name);
2214
2215         return 0;
2216
2217 out_ida_free:
2218         ida_free(&nullb_indexes, nullb->index);
2219 out_cleanup_zone:
2220         null_free_zoned_dev(dev);
2221 out_cleanup_disk:
2222         put_disk(nullb->disk);
2223 out_cleanup_tags:
2224         if (dev->queue_mode == NULL_Q_MQ && nullb->tag_set == &nullb->__tag_set)
2225                 blk_mq_free_tag_set(nullb->tag_set);
2226 out_cleanup_queues:
2227         cleanup_queues(nullb);
2228 out_free_nullb:
2229         kfree(nullb);
2230         dev->nullb = NULL;
2231 out:
2232         return rv;
2233 }
2234
2235 static struct nullb *null_find_dev_by_name(const char *name)
2236 {
2237         struct nullb *nullb = NULL, *nb;
2238
2239         mutex_lock(&lock);
2240         list_for_each_entry(nb, &nullb_list, list) {
2241                 if (strcmp(nb->disk_name, name) == 0) {
2242                         nullb = nb;
2243                         break;
2244                 }
2245         }
2246         mutex_unlock(&lock);
2247
2248         return nullb;
2249 }
2250
2251 static int null_create_dev(void)
2252 {
2253         struct nullb_device *dev;
2254         int ret;
2255
2256         dev = null_alloc_dev();
2257         if (!dev)
2258                 return -ENOMEM;
2259
2260         ret = null_add_dev(dev);
2261         if (ret) {
2262                 null_free_dev(dev);
2263                 return ret;
2264         }
2265
2266         return 0;
2267 }
2268
2269 static void null_destroy_dev(struct nullb *nullb)
2270 {
2271         struct nullb_device *dev = nullb->dev;
2272
2273         null_del_dev(nullb);
2274         null_free_device_storage(dev, false);
2275         null_free_dev(dev);
2276 }
2277
2278 static int __init null_init(void)
2279 {
2280         int ret = 0;
2281         unsigned int i;
2282         struct nullb *nullb;
2283
2284         if (g_bs > PAGE_SIZE) {
2285                 pr_warn("invalid block size\n");
2286                 pr_warn("defaults block size to %lu\n", PAGE_SIZE);
2287                 g_bs = PAGE_SIZE;
2288         }
2289
2290         if (g_home_node != NUMA_NO_NODE && g_home_node >= nr_online_nodes) {
2291                 pr_err("invalid home_node value\n");
2292                 g_home_node = NUMA_NO_NODE;
2293         }
2294
2295         if (!null_setup_fault())
2296                 return -EINVAL;
2297
2298         if (g_queue_mode == NULL_Q_RQ) {
2299                 pr_err("legacy IO path is no longer available\n");
2300                 return -EINVAL;
2301         }
2302
2303         if (g_queue_mode == NULL_Q_MQ && g_use_per_node_hctx) {
2304                 if (g_submit_queues != nr_online_nodes) {
2305                         pr_warn("submit_queues param is set to %u.\n",
2306                                 nr_online_nodes);
2307                         g_submit_queues = nr_online_nodes;
2308                 }
2309         } else if (g_submit_queues > nr_cpu_ids) {
2310                 g_submit_queues = nr_cpu_ids;
2311         } else if (g_submit_queues <= 0) {
2312                 g_submit_queues = 1;
2313         }
2314
2315         if (g_queue_mode == NULL_Q_MQ && shared_tags) {
2316                 ret = null_init_tag_set(NULL, &tag_set);
2317                 if (ret)
2318                         return ret;
2319         }
2320
2321         config_group_init(&nullb_subsys.su_group);
2322         mutex_init(&nullb_subsys.su_mutex);
2323
2324         ret = configfs_register_subsystem(&nullb_subsys);
2325         if (ret)
2326                 goto err_tagset;
2327
2328         mutex_init(&lock);
2329
2330         null_major = register_blkdev(0, "nullb");
2331         if (null_major < 0) {
2332                 ret = null_major;
2333                 goto err_conf;
2334         }
2335
2336         for (i = 0; i < nr_devices; i++) {
2337                 ret = null_create_dev();
2338                 if (ret)
2339                         goto err_dev;
2340         }
2341
2342         pr_info("module loaded\n");
2343         return 0;
2344
2345 err_dev:
2346         while (!list_empty(&nullb_list)) {
2347                 nullb = list_entry(nullb_list.next, struct nullb, list);
2348                 null_destroy_dev(nullb);
2349         }
2350         unregister_blkdev(null_major, "nullb");
2351 err_conf:
2352         configfs_unregister_subsystem(&nullb_subsys);
2353 err_tagset:
2354         if (g_queue_mode == NULL_Q_MQ && shared_tags)
2355                 blk_mq_free_tag_set(&tag_set);
2356         return ret;
2357 }
2358
2359 static void __exit null_exit(void)
2360 {
2361         struct nullb *nullb;
2362
2363         configfs_unregister_subsystem(&nullb_subsys);
2364
2365         unregister_blkdev(null_major, "nullb");
2366
2367         mutex_lock(&lock);
2368         while (!list_empty(&nullb_list)) {
2369                 nullb = list_entry(nullb_list.next, struct nullb, list);
2370                 null_destroy_dev(nullb);
2371         }
2372         mutex_unlock(&lock);
2373
2374         if (g_queue_mode == NULL_Q_MQ && shared_tags)
2375                 blk_mq_free_tag_set(&tag_set);
2376 }
2377
2378 module_init(null_init);
2379 module_exit(null_exit);
2380
2381 MODULE_AUTHOR("Jens Axboe <axboe@kernel.dk>");
2382 MODULE_LICENSE("GPL");