drivers/block/zram/zram_drv.c

   1 /*
   2  * Compressed RAM block device
   3  *
   4  * Copyright (C) 2008, 2009, 2010  Nitin Gupta
   5  *               2012, 2013 Minchan Kim
   6  *
   7  * This code is released using a dual license strategy: BSD/GPL
   8  * You can choose the licence that better fits your requirements.
   9  *
  10  * Released under the terms of 3-clause BSD License
  11  * Released under the terms of GNU General Public License Version 2.0
  12  *
  13  */
  14
  15 #define KMSG_COMPONENT "zram"
  16 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  17
  18 #include <linux/module.h>
  19 #include <linux/kernel.h>
  20 #include <linux/bio.h>
  21 #include <linux/bitops.h>
  22 #include <linux/blkdev.h>
  23 #include <linux/buffer_head.h>
  24 #include <linux/device.h>
  25 #include <linux/genhd.h>
  26 #include <linux/highmem.h>
  27 #include <linux/slab.h>
  28 #include <linux/backing-dev.h>
  29 #include <linux/string.h>
  30 #include <linux/vmalloc.h>
  31 #include <linux/err.h>
  32 #include <linux/idr.h>
  33 #include <linux/sysfs.h>
  34
  35 #include "zram_drv.h"
  36
  37 static DEFINE_IDR(zram_index_idr);
  38 /* idr index must be protected */
  39 static DEFINE_MUTEX(zram_index_mutex);
  40
  41 static int zram_major;
  42 static const char *default_compressor = "lzo";
  43
  44 /* Module params (documentation at end) */
  45 static unsigned int num_devices = 1;
  46
  47 static inline void deprecated_attr_warn(const char *name)
  48 {
  49         pr_warn_once("%d (%s) Attribute %s (and others) will be removed. %s\n",
  50                         task_pid_nr(current),
  51                         current->comm,
  52                         name,
  53                         "See zram documentation.");
  54 }
  55
  56 #define ZRAM_ATTR_RO(name)                                              \
  57 static ssize_t name##_show(struct device *d,                            \
  58                                 struct device_attribute *attr, char *b) \
  59 {                                                                       \
  60         struct zram *zram = dev_to_zram(d);                             \
  61                                                                         \
  62         deprecated_attr_warn(__stringify(name));                        \
  63         return scnprintf(b, PAGE_SIZE, "%llu\n",                        \
  64                 (u64)atomic64_read(&zram->stats.name));                 \
  65 }                                                                       \
  66 static DEVICE_ATTR_RO(name);
  67
  68 static inline bool init_done(struct zram *zram)
  69 {
  70         return zram->disksize;
  71 }
  72
  73 static inline struct zram *dev_to_zram(struct device *dev)
  74 {
  75         return (struct zram *)dev_to_disk(dev)->private_data;
  76 }
  77
  78 /* flag operations require table entry bit_spin_lock() being held */
  79 static int zram_test_flag(struct zram_meta *meta, u32 index,
  80                         enum zram_pageflags flag)
  81 {
  82         return meta->table[index].value & BIT(flag);
  83 }
  84
  85 static void zram_set_flag(struct zram_meta *meta, u32 index,
  86                         enum zram_pageflags flag)
  87 {
  88         meta->table[index].value |= BIT(flag);
  89 }
  90
  91 static void zram_clear_flag(struct zram_meta *meta, u32 index,
  92                         enum zram_pageflags flag)
  93 {
  94         meta->table[index].value &= ~BIT(flag);
  95 }
  96
  97 static size_t zram_get_obj_size(struct zram_meta *meta, u32 index)
  98 {
  99         return meta->table[index].value & (BIT(ZRAM_FLAG_SHIFT) - 1);
 100 }
 101
 102 static void zram_set_obj_size(struct zram_meta *meta,
 103                                         u32 index, size_t size)
 104 {
 105         unsigned long flags = meta->table[index].value >> ZRAM_FLAG_SHIFT;
 106
 107         meta->table[index].value = (flags << ZRAM_FLAG_SHIFT) | size;
 108 }
 109
 110 static inline bool is_partial_io(struct bio_vec *bvec)
 111 {
 112         return bvec->bv_len != PAGE_SIZE;
 113 }
 114
 115 static void zram_revalidate_disk(struct zram *zram)
 116 {
 117         revalidate_disk(zram->disk);
 118         /* revalidate_disk reset the BDI_CAP_STABLE_WRITES so set again */
 119         zram->disk->queue->backing_dev_info.capabilities |=
 120                 BDI_CAP_STABLE_WRITES;
 121 }
 122
 123 /*
 124  * Check if request is within bounds and aligned on zram logical blocks.
 125  */
 126 static inline bool valid_io_request(struct zram *zram,
 127                 sector_t start, unsigned int size)
 128 {
 129         u64 end, bound;
 130
 131         /* unaligned request */
 132         if (unlikely(start & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
 133                 return false;
 134         if (unlikely(size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
 135                 return false;
 136
 137         end = start + (size >> SECTOR_SHIFT);
 138         bound = zram->disksize >> SECTOR_SHIFT;
 139         /* out of range range */
 140         if (unlikely(start >= bound || end > bound || start > end))
 141                 return false;
 142
 143         /* I/O request is valid */
 144         return true;
 145 }
 146
 147 static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
 148 {
 149         if (*offset + bvec->bv_len >= PAGE_SIZE)
 150                 (*index)++;
 151         *offset = (*offset + bvec->bv_len) % PAGE_SIZE;
 152 }
 153
 154 static inline void update_used_max(struct zram *zram,
 155                                         const unsigned long pages)
 156 {
 157         unsigned long old_max, cur_max;
 158
 159         old_max = atomic_long_read(&zram->stats.max_used_pages);
 160
 161         do {
 162                 cur_max = old_max;
 163                 if (pages > cur_max)
 164                         old_max = atomic_long_cmpxchg(
 165                                 &zram->stats.max_used_pages, cur_max, pages);
 166         } while (old_max != cur_max);
 167 }
 168
 169 static bool page_zero_filled(void *ptr)
 170 {
 171         unsigned int pos;
 172         unsigned long *page;
 173
 174         page = (unsigned long *)ptr;
 175
 176         for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
 177                 if (page[pos])
 178                         return false;
 179         }
 180
 181         return true;
 182 }
 183
 184 static void handle_zero_page(struct bio_vec *bvec)
 185 {
 186         struct page *page = bvec->bv_page;
 187         void *user_mem;
 188
 189         user_mem = kmap_atomic(page);
 190         if (is_partial_io(bvec))
 191                 memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
 192         else
 193                 clear_page(user_mem);
 194         kunmap_atomic(user_mem);
 195
 196         flush_dcache_page(page);
 197 }
 198
 199 static ssize_t initstate_show(struct device *dev,
 200                 struct device_attribute *attr, char *buf)
 201 {
 202         u32 val;
 203         struct zram *zram = dev_to_zram(dev);
 204
 205         down_read(&zram->init_lock);
 206         val = init_done(zram);
 207         up_read(&zram->init_lock);
 208
 209         return scnprintf(buf, PAGE_SIZE, "%u\n", val);
 210 }
 211
 212 static ssize_t disksize_show(struct device *dev,
 213                 struct device_attribute *attr, char *buf)
 214 {
 215         struct zram *zram = dev_to_zram(dev);
 216
 217         return scnprintf(buf, PAGE_SIZE, "%llu\n", zram->disksize);
 218 }
 219
 220 static ssize_t orig_data_size_show(struct device *dev,
 221                 struct device_attribute *attr, char *buf)
 222 {
 223         struct zram *zram = dev_to_zram(dev);
 224
 225         deprecated_attr_warn("orig_data_size");
 226         return scnprintf(buf, PAGE_SIZE, "%llu\n",
 227                 (u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
 228 }
 229
 230 static ssize_t mem_used_total_show(struct device *dev,
 231                 struct device_attribute *attr, char *buf)
 232 {
 233         u64 val = 0;
 234         struct zram *zram = dev_to_zram(dev);
 235
 236         deprecated_attr_warn("mem_used_total");
 237         down_read(&zram->init_lock);
 238         if (init_done(zram)) {
 239                 struct zram_meta *meta = zram->meta;
 240                 val = zs_get_total_pages(meta->mem_pool);
 241         }
 242         up_read(&zram->init_lock);
 243
 244         return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
 245 }
 246
 247 static ssize_t mem_limit_show(struct device *dev,
 248                 struct device_attribute *attr, char *buf)
 249 {
 250         u64 val;
 251         struct zram *zram = dev_to_zram(dev);
 252
 253         deprecated_attr_warn("mem_limit");
 254         down_read(&zram->init_lock);
 255         val = zram->limit_pages;
 256         up_read(&zram->init_lock);
 257
 258         return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
 259 }
 260
 261 static ssize_t mem_limit_store(struct device *dev,
 262                 struct device_attribute *attr, const char *buf, size_t len)
 263 {
 264         u64 limit;
 265         char *tmp;
 266         struct zram *zram = dev_to_zram(dev);
 267
 268         limit = memparse(buf, &tmp);
 269         if (buf == tmp) /* no chars parsed, invalid input */
 270                 return -EINVAL;
 271
 272         down_write(&zram->init_lock);
 273         zram->limit_pages = PAGE_ALIGN(limit) >> PAGE_SHIFT;
 274         up_write(&zram->init_lock);
 275
 276         return len;
 277 }
 278
 279 static ssize_t mem_used_max_show(struct device *dev,
 280                 struct device_attribute *attr, char *buf)
 281 {
 282         u64 val = 0;
 283         struct zram *zram = dev_to_zram(dev);
 284
 285         deprecated_attr_warn("mem_used_max");
 286         down_read(&zram->init_lock);
 287         if (init_done(zram))
 288                 val = atomic_long_read(&zram->stats.max_used_pages);
 289         up_read(&zram->init_lock);
 290
 291         return scnprintf(buf, PAGE_SIZE, "%llu\n", val << PAGE_SHIFT);
 292 }
 293
 294 static ssize_t mem_used_max_store(struct device *dev,
 295                 struct device_attribute *attr, const char *buf, size_t len)
 296 {
 297         int err;
 298         unsigned long val;
 299         struct zram *zram = dev_to_zram(dev);
 300
 301         err = kstrtoul(buf, 10, &val);
 302         if (err || val != 0)
 303                 return -EINVAL;
 304
 305         down_read(&zram->init_lock);
 306         if (init_done(zram)) {
 307                 struct zram_meta *meta = zram->meta;
 308                 atomic_long_set(&zram->stats.max_used_pages,
 309                                 zs_get_total_pages(meta->mem_pool));
 310         }
 311         up_read(&zram->init_lock);
 312
 313         return len;
 314 }
 315
 316 /*
 317  * We switched to per-cpu streams and this attr is not needed anymore.
 318  * However, we will keep it around for some time, because:
 319  * a) we may revert per-cpu streams in the future
 320  * b) it's visible to user space and we need to follow our 2 years
 321  *    retirement rule; but we already have a number of 'soon to be
 322  *    altered' attrs, so max_comp_streams need to wait for the next
 323  *    layoff cycle.
 324  */
 325 static ssize_t max_comp_streams_show(struct device *dev,
 326                 struct device_attribute *attr, char *buf)
 327 {
 328         return scnprintf(buf, PAGE_SIZE, "%d\n", num_online_cpus());
 329 }
 330
 331 static ssize_t max_comp_streams_store(struct device *dev,
 332                 struct device_attribute *attr, const char *buf, size_t len)
 333 {
 334         return len;
 335 }
 336
 337 static ssize_t comp_algorithm_show(struct device *dev,
 338                 struct device_attribute *attr, char *buf)
 339 {
 340         size_t sz;
 341         struct zram *zram = dev_to_zram(dev);
 342
 343         down_read(&zram->init_lock);
 344         sz = zcomp_available_show(zram->compressor, buf);
 345         up_read(&zram->init_lock);
 346
 347         return sz;
 348 }
 349
 350 static ssize_t comp_algorithm_store(struct device *dev,
 351                 struct device_attribute *attr, const char *buf, size_t len)
 352 {
 353         struct zram *zram = dev_to_zram(dev);
 354         char compressor[CRYPTO_MAX_ALG_NAME];
 355         size_t sz;
 356
 357         strlcpy(compressor, buf, sizeof(compressor));
 358         /* ignore trailing newline */
 359         sz = strlen(compressor);
 360         if (sz > 0 && compressor[sz - 1] == '\n')
 361                 compressor[sz - 1] = 0x00;
 362
 363         if (!zcomp_available_algorithm(compressor))
 364                 return -EINVAL;
 365
 366         down_write(&zram->init_lock);
 367         if (init_done(zram)) {
 368                 up_write(&zram->init_lock);
 369                 pr_info("Can't change algorithm for initialized device\n");
 370                 return -EBUSY;
 371         }
 372
 373         strlcpy(zram->compressor, compressor, sizeof(compressor));
 374         up_write(&zram->init_lock);
 375         return len;
 376 }
 377
 378 static ssize_t compact_store(struct device *dev,
 379                 struct device_attribute *attr, const char *buf, size_t len)
 380 {
 381         struct zram *zram = dev_to_zram(dev);
 382         struct zram_meta *meta;
 383
 384         down_read(&zram->init_lock);
 385         if (!init_done(zram)) {
 386                 up_read(&zram->init_lock);
 387                 return -EINVAL;
 388         }
 389
 390         meta = zram->meta;
 391         zs_compact(meta->mem_pool);
 392         up_read(&zram->init_lock);
 393
 394         return len;
 395 }
 396
 397 static ssize_t io_stat_show(struct device *dev,
 398                 struct device_attribute *attr, char *buf)
 399 {
 400         struct zram *zram = dev_to_zram(dev);
 401         ssize_t ret;
 402
 403         down_read(&zram->init_lock);
 404         ret = scnprintf(buf, PAGE_SIZE,
 405                         "%8llu %8llu %8llu %8llu\n",
 406                         (u64)atomic64_read(&zram->stats.failed_reads),
 407                         (u64)atomic64_read(&zram->stats.failed_writes),
 408                         (u64)atomic64_read(&zram->stats.invalid_io),
 409                         (u64)atomic64_read(&zram->stats.notify_free));
 410         up_read(&zram->init_lock);
 411
 412         return ret;
 413 }
 414
 415 static ssize_t mm_stat_show(struct device *dev,
 416                 struct device_attribute *attr, char *buf)
 417 {
 418         struct zram *zram = dev_to_zram(dev);
 419         struct zs_pool_stats pool_stats;
 420         u64 orig_size, mem_used = 0;
 421         long max_used;
 422         ssize_t ret;
 423
 424         memset(&pool_stats, 0x00, sizeof(struct zs_pool_stats));
 425
 426         down_read(&zram->init_lock);
 427         if (init_done(zram)) {
 428                 mem_used = zs_get_total_pages(zram->meta->mem_pool);
 429                 zs_pool_stats(zram->meta->mem_pool, &pool_stats);
 430         }
 431
 432         orig_size = atomic64_read(&zram->stats.pages_stored);
 433         max_used = atomic_long_read(&zram->stats.max_used_pages);
 434
 435         ret = scnprintf(buf, PAGE_SIZE,
 436                         "%8llu %8llu %8llu %8lu %8ld %8llu %8lu\n",
 437                         orig_size << PAGE_SHIFT,
 438                         (u64)atomic64_read(&zram->stats.compr_data_size),
 439                         mem_used << PAGE_SHIFT,
 440                         zram->limit_pages << PAGE_SHIFT,
 441                         max_used << PAGE_SHIFT,
 442                         (u64)atomic64_read(&zram->stats.zero_pages),
 443                         atomic_long_read(&pool_stats.pages_compacted));
 444         up_read(&zram->init_lock);
 445
 446         return ret;
 447 }
 448
 449 static ssize_t debug_stat_show(struct device *dev,
 450                 struct device_attribute *attr, char *buf)
 451 {
 452         int version = 1;
 453         struct zram *zram = dev_to_zram(dev);
 454         ssize_t ret;
 455
 456         down_read(&zram->init_lock);
 457         ret = scnprintf(buf, PAGE_SIZE,
 458                         "version: %d\n%8llu\n",
 459                         version,
 460                         (u64)atomic64_read(&zram->stats.writestall));
 461         up_read(&zram->init_lock);
 462
 463         return ret;
 464 }
 465
 466 static DEVICE_ATTR_RO(io_stat);
 467 static DEVICE_ATTR_RO(mm_stat);
 468 static DEVICE_ATTR_RO(debug_stat);
 469 ZRAM_ATTR_RO(num_reads);
 470 ZRAM_ATTR_RO(num_writes);
 471 ZRAM_ATTR_RO(failed_reads);
 472 ZRAM_ATTR_RO(failed_writes);
 473 ZRAM_ATTR_RO(invalid_io);
 474 ZRAM_ATTR_RO(notify_free);
 475 ZRAM_ATTR_RO(zero_pages);
 476 ZRAM_ATTR_RO(compr_data_size);
 477
 478 static inline bool zram_meta_get(struct zram *zram)
 479 {
 480         if (atomic_inc_not_zero(&zram->refcount))
 481                 return true;
 482         return false;
 483 }
 484
 485 static inline void zram_meta_put(struct zram *zram)
 486 {
 487         atomic_dec(&zram->refcount);
 488 }
 489
 490 static void zram_meta_free(struct zram_meta *meta, u64 disksize)
 491 {
 492         size_t num_pages = disksize >> PAGE_SHIFT;
 493         size_t index;
 494
 495         /* Free all pages that are still in this zram device */
 496         for (index = 0; index < num_pages; index++) {
 497                 unsigned long handle = meta->table[index].handle;
 498
 499                 if (!handle)
 500                         continue;
 501
 502                 zs_free(meta->mem_pool, handle);
 503         }
 504
 505         zs_destroy_pool(meta->mem_pool);
 506         vfree(meta->table);
 507         kfree(meta);
 508 }
 509
 510 static struct zram_meta *zram_meta_alloc(char *pool_name, u64 disksize)
 511 {
 512         size_t num_pages;
 513         struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
 514
 515         if (!meta)
 516                 return NULL;
 517
 518         num_pages = disksize >> PAGE_SHIFT;
 519         meta->table = vzalloc(num_pages * sizeof(*meta->table));
 520         if (!meta->table) {
 521                 pr_err("Error allocating zram address table\n");
 522                 goto out_error;
 523         }
 524
 525         meta->mem_pool = zs_create_pool(pool_name);
 526         if (!meta->mem_pool) {
 527                 pr_err("Error creating memory pool\n");
 528                 goto out_error;
 529         }
 530
 531         return meta;
 532
 533 out_error:
 534         vfree(meta->table);
 535         kfree(meta);
 536         return NULL;
 537 }
 538
 539 /*
 540  * To protect concurrent access to the same index entry,
 541  * caller should hold this table index entry's bit_spinlock to
 542  * indicate this index entry is accessing.
 543  */
 544 static void zram_free_page(struct zram *zram, size_t index)
 545 {
 546         struct zram_meta *meta = zram->meta;
 547         unsigned long handle = meta->table[index].handle;
 548
 549         if (unlikely(!handle)) {
 550                 /*
 551                  * No memory is allocated for zero filled pages.
 552                  * Simply clear zero page flag.
 553                  */
 554                 if (zram_test_flag(meta, index, ZRAM_ZERO)) {
 555                         zram_clear_flag(meta, index, ZRAM_ZERO);
 556                         atomic64_dec(&zram->stats.zero_pages);
 557                 }
 558                 return;
 559         }
 560
 561         zs_free(meta->mem_pool, handle);
 562
 563         atomic64_sub(zram_get_obj_size(meta, index),
 564                         &zram->stats.compr_data_size);
 565         atomic64_dec(&zram->stats.pages_stored);
 566
 567         meta->table[index].handle = 0;
 568         zram_set_obj_size(meta, index, 0);
 569 }
 570
 571 static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
 572 {
 573         int ret = 0;
 574         unsigned char *cmem;
 575         struct zram_meta *meta = zram->meta;
 576         unsigned long handle;
 577         unsigned int size;
 578
 579         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 580         handle = meta->table[index].handle;
 581         size = zram_get_obj_size(meta, index);
 582
 583         if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
 584                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 585                 memset(mem, 0, PAGE_SIZE);
 586                 return 0;
 587         }
 588
 589         cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
 590         if (size == PAGE_SIZE) {
 591                 memcpy(mem, cmem, PAGE_SIZE);
 592         } else {
 593                 struct zcomp_strm *zstrm = zcomp_stream_get(zram->comp);
 594
 595                 ret = zcomp_decompress(zstrm, cmem, size, mem);
 596                 zcomp_stream_put(zram->comp);
 597         }
 598         zs_unmap_object(meta->mem_pool, handle);
 599         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 600
 601         /* Should NEVER happen. Return bio error if it does. */
 602         if (unlikely(ret)) {
 603                 pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
 604                 return ret;
 605         }
 606
 607         return 0;
 608 }
 609
 610 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
 611                           u32 index, int offset)
 612 {
 613         int ret;
 614         struct page *page;
 615         unsigned char *user_mem, *uncmem = NULL;
 616         struct zram_meta *meta = zram->meta;
 617         page = bvec->bv_page;
 618
 619         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 620         if (unlikely(!meta->table[index].handle) ||
 621                         zram_test_flag(meta, index, ZRAM_ZERO)) {
 622                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 623                 handle_zero_page(bvec);
 624                 return 0;
 625         }
 626         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 627
 628         if (is_partial_io(bvec))
 629                 /* Use  a temporary buffer to decompress the page */
 630                 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
 631
 632         user_mem = kmap_atomic(page);
 633         if (!is_partial_io(bvec))
 634                 uncmem = user_mem;
 635
 636         if (!uncmem) {
 637                 pr_err("Unable to allocate temp memory\n");
 638                 ret = -ENOMEM;
 639                 goto out_cleanup;
 640         }
 641
 642         ret = zram_decompress_page(zram, uncmem, index);
 643         /* Should NEVER happen. Return bio error if it does. */
 644         if (unlikely(ret))
 645                 goto out_cleanup;
 646
 647         if (is_partial_io(bvec))
 648                 memcpy(user_mem + bvec->bv_offset, uncmem + offset,
 649                                 bvec->bv_len);
 650
 651         flush_dcache_page(page);
 652         ret = 0;
 653 out_cleanup:
 654         kunmap_atomic(user_mem);
 655         if (is_partial_io(bvec))
 656                 kfree(uncmem);
 657         return ret;
 658 }
 659
 660 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
 661                            int offset)
 662 {
 663         int ret = 0;
 664         unsigned int clen;
 665         unsigned long handle = 0;
 666         struct page *page;
 667         unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
 668         struct zram_meta *meta = zram->meta;
 669         struct zcomp_strm *zstrm = NULL;
 670         unsigned long alloced_pages;
 671
 672         page = bvec->bv_page;
 673         if (is_partial_io(bvec)) {
 674                 /*
 675                  * This is a partial IO. We need to read the full page
 676                  * before to write the changes.
 677                  */
 678                 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
 679                 if (!uncmem) {
 680                         ret = -ENOMEM;
 681                         goto out;
 682                 }
 683                 ret = zram_decompress_page(zram, uncmem, index);
 684                 if (ret)
 685                         goto out;
 686         }
 687
 688 compress_again:
 689         user_mem = kmap_atomic(page);
 690         if (is_partial_io(bvec)) {
 691                 memcpy(uncmem + offset, user_mem + bvec->bv_offset,
 692                        bvec->bv_len);
 693                 kunmap_atomic(user_mem);
 694                 user_mem = NULL;
 695         } else {
 696                 uncmem = user_mem;
 697         }
 698
 699         if (page_zero_filled(uncmem)) {
 700                 if (user_mem)
 701                         kunmap_atomic(user_mem);
 702                 /* Free memory associated with this sector now. */
 703                 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 704                 zram_free_page(zram, index);
 705                 zram_set_flag(meta, index, ZRAM_ZERO);
 706                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 707
 708                 atomic64_inc(&zram->stats.zero_pages);
 709                 ret = 0;
 710                 goto out;
 711         }
 712
 713         zstrm = zcomp_stream_get(zram->comp);
 714         ret = zcomp_compress(zstrm, uncmem, &clen);
 715         if (!is_partial_io(bvec)) {
 716                 kunmap_atomic(user_mem);
 717                 user_mem = NULL;
 718                 uncmem = NULL;
 719         }
 720
 721         if (unlikely(ret)) {
 722                 pr_err("Compression failed! err=%d\n", ret);
 723                 goto out;
 724         }
 725
 726         src = zstrm->buffer;
 727         if (unlikely(clen > max_zpage_size)) {
 728                 clen = PAGE_SIZE;
 729                 if (is_partial_io(bvec))
 730                         src = uncmem;
 731         }
 732
 733         /*
 734          * handle allocation has 2 paths:
 735          * a) fast path is executed with preemption disabled (for
 736          *  per-cpu streams) and has __GFP_DIRECT_RECLAIM bit clear,
 737          *  since we can't sleep;
 738          * b) slow path enables preemption and attempts to allocate
 739          *  the page with __GFP_DIRECT_RECLAIM bit set. we have to
 740          *  put per-cpu compression stream and, thus, to re-do
 741          *  the compression once handle is allocated.
 742          *
 743          * if we have a 'non-null' handle here then we are coming
 744          * from the slow path and handle has already been allocated.
 745          */
 746         if (!handle)
 747                 handle = zs_malloc(meta->mem_pool, clen,
 748                                 __GFP_KSWAPD_RECLAIM |
 749                                 __GFP_NOWARN |
 750                                 __GFP_HIGHMEM |
 751                                 __GFP_MOVABLE);
 752         if (!handle) {
 753                 zcomp_stream_put(zram->comp);
 754                 zstrm = NULL;
 755
 756                 atomic64_inc(&zram->stats.writestall);
 757
 758                 handle = zs_malloc(meta->mem_pool, clen,
 759                                 GFP_NOIO | __GFP_HIGHMEM |
 760                                 __GFP_MOVABLE);
 761                 if (handle)
 762                         goto compress_again;
 763
 764                 pr_err("Error allocating memory for compressed page: %u, size=%u\n",
 765                         index, clen);
 766                 ret = -ENOMEM;
 767                 goto out;
 768         }
 769
 770         alloced_pages = zs_get_total_pages(meta->mem_pool);
 771         update_used_max(zram, alloced_pages);
 772
 773         if (zram->limit_pages && alloced_pages > zram->limit_pages) {
 774                 zs_free(meta->mem_pool, handle);
 775                 ret = -ENOMEM;
 776                 goto out;
 777         }
 778
 779         cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
 780
 781         if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
 782                 src = kmap_atomic(page);
 783                 memcpy(cmem, src, PAGE_SIZE);
 784                 kunmap_atomic(src);
 785         } else {
 786                 memcpy(cmem, src, clen);
 787         }
 788
 789         zcomp_stream_put(zram->comp);
 790         zstrm = NULL;
 791         zs_unmap_object(meta->mem_pool, handle);
 792
 793         /*
 794          * Free memory associated with this sector
 795          * before overwriting unused sectors.
 796          */
 797         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 798         zram_free_page(zram, index);
 799
 800         meta->table[index].handle = handle;
 801         zram_set_obj_size(meta, index, clen);
 802         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 803
 804         /* Update stats */
 805         atomic64_add(clen, &zram->stats.compr_data_size);
 806         atomic64_inc(&zram->stats.pages_stored);
 807 out:
 808         if (zstrm)
 809                 zcomp_stream_put(zram->comp);
 810         if (is_partial_io(bvec))
 811                 kfree(uncmem);
 812         return ret;
 813 }
 814
 815 /*
 816  * zram_bio_discard - handler on discard request
 817  * @index: physical block index in PAGE_SIZE units
 818  * @offset: byte offset within physical block
 819  */
 820 static void zram_bio_discard(struct zram *zram, u32 index,
 821                              int offset, struct bio *bio)
 822 {
 823         size_t n = bio->bi_iter.bi_size;
 824         struct zram_meta *meta = zram->meta;
 825
 826         /*
 827          * zram manages data in physical block size units. Because logical block
 828          * size isn't identical with physical block size on some arch, we
 829          * could get a discard request pointing to a specific offset within a
 830          * certain physical block.  Although we can handle this request by
 831          * reading that physiclal block and decompressing and partially zeroing
 832          * and re-compressing and then re-storing it, this isn't reasonable
 833          * because our intent with a discard request is to save memory.  So
 834          * skipping this logical block is appropriate here.
 835          */
 836         if (offset) {
 837                 if (n <= (PAGE_SIZE - offset))
 838                         return;
 839
 840                 n -= (PAGE_SIZE - offset);
 841                 index++;
 842         }
 843
 844         while (n >= PAGE_SIZE) {
 845                 bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 846                 zram_free_page(zram, index);
 847                 bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 848                 atomic64_inc(&zram->stats.notify_free);
 849                 index++;
 850                 n -= PAGE_SIZE;
 851         }
 852 }
 853
 854 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
 855                         int offset, bool is_write)
 856 {
 857         unsigned long start_time = jiffies;
 858         int rw_acct = is_write ? REQ_OP_WRITE : REQ_OP_READ;
 859         int ret;
 860
 861         generic_start_io_acct(rw_acct, bvec->bv_len >> SECTOR_SHIFT,
 862                         &zram->disk->part0);
 863
 864         if (!is_write) {
 865                 atomic64_inc(&zram->stats.num_reads);
 866                 ret = zram_bvec_read(zram, bvec, index, offset);
 867         } else {
 868                 atomic64_inc(&zram->stats.num_writes);
 869                 ret = zram_bvec_write(zram, bvec, index, offset);
 870         }
 871
 872         generic_end_io_acct(rw_acct, &zram->disk->part0, start_time);
 873
 874         if (unlikely(ret)) {
 875                 if (!is_write)
 876                         atomic64_inc(&zram->stats.failed_reads);
 877                 else
 878                         atomic64_inc(&zram->stats.failed_writes);
 879         }
 880
 881         return ret;
 882 }
 883
 884 static void __zram_make_request(struct zram *zram, struct bio *bio)
 885 {
 886         int offset;
 887         u32 index;
 888         struct bio_vec bvec;
 889         struct bvec_iter iter;
 890
 891         index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
 892         offset = (bio->bi_iter.bi_sector &
 893                   (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
 894
 895         if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
 896                 zram_bio_discard(zram, index, offset, bio);
 897                 bio_endio(bio);
 898                 return;
 899         }
 900
 901         bio_for_each_segment(bvec, bio, iter) {
 902                 int max_transfer_size = PAGE_SIZE - offset;
 903
 904                 if (bvec.bv_len > max_transfer_size) {
 905                         /*
 906                          * zram_bvec_rw() can only make operation on a single
 907                          * zram page. Split the bio vector.
 908                          */
 909                         struct bio_vec bv;
 910
 911                         bv.bv_page = bvec.bv_page;
 912                         bv.bv_len = max_transfer_size;
 913                         bv.bv_offset = bvec.bv_offset;
 914
 915                         if (zram_bvec_rw(zram, &bv, index, offset,
 916                                          op_is_write(bio_op(bio))) < 0)
 917                                 goto out;
 918
 919                         bv.bv_len = bvec.bv_len - max_transfer_size;
 920                         bv.bv_offset += max_transfer_size;
 921                         if (zram_bvec_rw(zram, &bv, index + 1, 0,
 922                                          op_is_write(bio_op(bio))) < 0)
 923                                 goto out;
 924                 } else
 925                         if (zram_bvec_rw(zram, &bvec, index, offset,
 926                                          op_is_write(bio_op(bio))) < 0)
 927                                 goto out;
 928
 929                 update_position(&index, &offset, &bvec);
 930         }
 931
 932         bio_endio(bio);
 933         return;
 934
 935 out:
 936         bio_io_error(bio);
 937 }
 938
 939 /*
 940  * Handler function for all zram I/O requests.
 941  */
 942 static blk_qc_t zram_make_request(struct request_queue *queue, struct bio *bio)
 943 {
 944         struct zram *zram = queue->queuedata;
 945
 946         if (unlikely(!zram_meta_get(zram)))
 947                 goto error;
 948
 949         blk_queue_split(queue, &bio, queue->bio_split);
 950
 951         if (!valid_io_request(zram, bio->bi_iter.bi_sector,
 952                                         bio->bi_iter.bi_size)) {
 953                 atomic64_inc(&zram->stats.invalid_io);
 954                 goto put_zram;
 955         }
 956
 957         __zram_make_request(zram, bio);
 958         zram_meta_put(zram);
 959         return BLK_QC_T_NONE;
 960 put_zram:
 961         zram_meta_put(zram);
 962 error:
 963         bio_io_error(bio);
 964         return BLK_QC_T_NONE;
 965 }
 966
 967 static void zram_slot_free_notify(struct block_device *bdev,
 968                                 unsigned long index)
 969 {
 970         struct zram *zram;
 971         struct zram_meta *meta;
 972
 973         zram = bdev->bd_disk->private_data;
 974         meta = zram->meta;
 975
 976         bit_spin_lock(ZRAM_ACCESS, &meta->table[index].value);
 977         zram_free_page(zram, index);
 978         bit_spin_unlock(ZRAM_ACCESS, &meta->table[index].value);
 979         atomic64_inc(&zram->stats.notify_free);
 980 }
 981
 982 static int zram_rw_page(struct block_device *bdev, sector_t sector,
 983                        struct page *page, bool is_write)
 984 {
 985         int offset, err = -EIO;
 986         u32 index;
 987         struct zram *zram;
 988         struct bio_vec bv;
 989
 990         zram = bdev->bd_disk->private_data;
 991         if (unlikely(!zram_meta_get(zram)))
 992                 goto out;
 993
 994         if (!valid_io_request(zram, sector, PAGE_SIZE)) {
 995                 atomic64_inc(&zram->stats.invalid_io);
 996                 err = -EINVAL;
 997                 goto put_zram;
 998         }
 999
1000         index = sector >> SECTORS_PER_PAGE_SHIFT;
1001         offset = sector & (SECTORS_PER_PAGE - 1) << SECTOR_SHIFT;
1002
1003         bv.bv_page = page;
1004         bv.bv_len = PAGE_SIZE;
1005         bv.bv_offset = 0;
1006
1007         err = zram_bvec_rw(zram, &bv, index, offset, is_write);
1008 put_zram:
1009         zram_meta_put(zram);
1010 out:
1011         /*
1012          * If I/O fails, just return error(ie, non-zero) without
1013          * calling page_endio.
1014          * It causes resubmit the I/O with bio request by upper functions
1015          * of rw_page(e.g., swap_readpage, __swap_writepage) and
1016          * bio->bi_end_io does things to handle the error
1017          * (e.g., SetPageError, set_page_dirty and extra works).
1018          */
1019         if (err == 0)
1020                 page_endio(page, is_write, 0);
1021         return err;
1022 }
1023
1024 static void zram_reset_device(struct zram *zram)
1025 {
1026         struct zram_meta *meta;
1027         struct zcomp *comp;
1028         u64 disksize;
1029
1030         down_write(&zram->init_lock);
1031
1032         zram->limit_pages = 0;
1033
1034         if (!init_done(zram)) {
1035                 up_write(&zram->init_lock);
1036                 return;
1037         }
1038
1039         meta = zram->meta;
1040         comp = zram->comp;
1041         disksize = zram->disksize;
1042         /*
1043          * Refcount will go down to 0 eventually and r/w handler
1044          * cannot handle further I/O so it will bail out by
1045          * check zram_meta_get.
1046          */
1047         zram_meta_put(zram);
1048         /*
1049          * We want to free zram_meta in process context to avoid
1050          * deadlock between reclaim path and any other locks.
1051          */
1052         wait_event(zram->io_done, atomic_read(&zram->refcount) == 0);
1053
1054         /* Reset stats */
1055         memset(&zram->stats, 0, sizeof(zram->stats));
1056         zram->disksize = 0;
1057
1058         set_capacity(zram->disk, 0);
1059         part_stat_set_all(&zram->disk->part0, 0);
1060
1061         up_write(&zram->init_lock);
1062         /* I/O operation under all of CPU are done so let's free */
1063         zram_meta_free(meta, disksize);
1064         zcomp_destroy(comp);
1065 }
1066
1067 static ssize_t disksize_store(struct device *dev,
1068                 struct device_attribute *attr, const char *buf, size_t len)
1069 {
1070         u64 disksize;
1071         struct zcomp *comp;
1072         struct zram_meta *meta;
1073         struct zram *zram = dev_to_zram(dev);
1074         int err;
1075
1076         disksize = memparse(buf, NULL);
1077         if (!disksize)
1078                 return -EINVAL;
1079
1080         disksize = PAGE_ALIGN(disksize);
1081         meta = zram_meta_alloc(zram->disk->disk_name, disksize);
1082         if (!meta)
1083                 return -ENOMEM;
1084
1085         comp = zcomp_create(zram->compressor);
1086         if (IS_ERR(comp)) {
1087                 pr_err("Cannot initialise %s compressing backend\n",
1088                                 zram->compressor);
1089                 err = PTR_ERR(comp);
1090                 goto out_free_meta;
1091         }
1092
1093         down_write(&zram->init_lock);
1094         if (init_done(zram)) {
1095                 pr_info("Cannot change disksize for initialized device\n");
1096                 err = -EBUSY;
1097                 goto out_destroy_comp;
1098         }
1099
1100         init_waitqueue_head(&zram->io_done);
1101         atomic_set(&zram->refcount, 1);
1102         zram->meta = meta;
1103         zram->comp = comp;
1104         zram->disksize = disksize;
1105         set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
1106         zram_revalidate_disk(zram);
1107         up_write(&zram->init_lock);
1108
1109         return len;
1110
1111 out_destroy_comp:
1112         up_write(&zram->init_lock);
1113         zcomp_destroy(comp);
1114 out_free_meta:
1115         zram_meta_free(meta, disksize);
1116         return err;
1117 }
1118
1119 static ssize_t reset_store(struct device *dev,
1120                 struct device_attribute *attr, const char *buf, size_t len)
1121 {
1122         int ret;
1123         unsigned short do_reset;
1124         struct zram *zram;
1125         struct block_device *bdev;
1126
1127         ret = kstrtou16(buf, 10, &do_reset);
1128         if (ret)
1129                 return ret;
1130
1131         if (!do_reset)
1132                 return -EINVAL;
1133
1134         zram = dev_to_zram(dev);
1135         bdev = bdget_disk(zram->disk, 0);
1136         if (!bdev)
1137                 return -ENOMEM;
1138
1139         mutex_lock(&bdev->bd_mutex);
1140         /* Do not reset an active device or claimed device */
1141         if (bdev->bd_openers || zram->claim) {
1142                 mutex_unlock(&bdev->bd_mutex);
1143                 bdput(bdev);
1144                 return -EBUSY;
1145         }
1146
1147         /* From now on, anyone can't open /dev/zram[0-9] */
1148         zram->claim = true;
1149         mutex_unlock(&bdev->bd_mutex);
1150
1151         /* Make sure all the pending I/O are finished */
1152         fsync_bdev(bdev);
1153         zram_reset_device(zram);
1154         zram_revalidate_disk(zram);
1155         bdput(bdev);
1156
1157         mutex_lock(&bdev->bd_mutex);
1158         zram->claim = false;
1159         mutex_unlock(&bdev->bd_mutex);
1160
1161         return len;
1162 }
1163
1164 static int zram_open(struct block_device *bdev, fmode_t mode)
1165 {
1166         int ret = 0;
1167         struct zram *zram;
1168
1169         WARN_ON(!mutex_is_locked(&bdev->bd_mutex));
1170
1171         zram = bdev->bd_disk->private_data;
1172         /* zram was claimed to reset so open request fails */
1173         if (zram->claim)
1174                 ret = -EBUSY;
1175
1176         return ret;
1177 }
1178
1179 static const struct block_device_operations zram_devops = {
1180         .open = zram_open,
1181         .swap_slot_free_notify = zram_slot_free_notify,
1182         .rw_page = zram_rw_page,
1183         .owner = THIS_MODULE
1184 };
1185
1186 static DEVICE_ATTR_WO(compact);
1187 static DEVICE_ATTR_RW(disksize);
1188 static DEVICE_ATTR_RO(initstate);
1189 static DEVICE_ATTR_WO(reset);
1190 static DEVICE_ATTR_RO(orig_data_size);
1191 static DEVICE_ATTR_RO(mem_used_total);
1192 static DEVICE_ATTR_RW(mem_limit);
1193 static DEVICE_ATTR_RW(mem_used_max);
1194 static DEVICE_ATTR_RW(max_comp_streams);
1195 static DEVICE_ATTR_RW(comp_algorithm);
1196
1197 static struct attribute *zram_disk_attrs[] = {
1198         &dev_attr_disksize.attr,
1199         &dev_attr_initstate.attr,
1200         &dev_attr_reset.attr,
1201         &dev_attr_num_reads.attr,
1202         &dev_attr_num_writes.attr,
1203         &dev_attr_failed_reads.attr,
1204         &dev_attr_failed_writes.attr,
1205         &dev_attr_compact.attr,
1206         &dev_attr_invalid_io.attr,
1207         &dev_attr_notify_free.attr,
1208         &dev_attr_zero_pages.attr,
1209         &dev_attr_orig_data_size.attr,
1210         &dev_attr_compr_data_size.attr,
1211         &dev_attr_mem_used_total.attr,
1212         &dev_attr_mem_limit.attr,
1213         &dev_attr_mem_used_max.attr,
1214         &dev_attr_max_comp_streams.attr,
1215         &dev_attr_comp_algorithm.attr,
1216         &dev_attr_io_stat.attr,
1217         &dev_attr_mm_stat.attr,
1218         &dev_attr_debug_stat.attr,
1219         NULL,
1220 };
1221
1222 static struct attribute_group zram_disk_attr_group = {
1223         .attrs = zram_disk_attrs,
1224 };
1225
1226 static const struct attribute_group *zram_disk_attr_groups[] = {
1227         &zram_disk_attr_group,
1228         NULL,
1229 };
1230
1231 /*
1232  * Allocate and initialize new zram device. the function returns
1233  * '>= 0' device_id upon success, and negative value otherwise.
1234  */
1235 static int zram_add(void)
1236 {
1237         struct zram *zram;
1238         struct request_queue *queue;
1239         int ret, device_id;
1240
1241         zram = kzalloc(sizeof(struct zram), GFP_KERNEL);
1242         if (!zram)
1243                 return -ENOMEM;
1244
1245         ret = idr_alloc(&zram_index_idr, zram, 0, 0, GFP_KERNEL);
1246         if (ret < 0)
1247                 goto out_free_dev;
1248         device_id = ret;
1249
1250         init_rwsem(&zram->init_lock);
1251
1252         queue = blk_alloc_queue(GFP_KERNEL);
1253         if (!queue) {
1254                 pr_err("Error allocating disk queue for device %d\n",
1255                         device_id);
1256                 ret = -ENOMEM;
1257                 goto out_free_idr;
1258         }
1259
1260         blk_queue_make_request(queue, zram_make_request);
1261
1262         /* gendisk structure */
1263         zram->disk = alloc_disk(1);
1264         if (!zram->disk) {
1265                 pr_err("Error allocating disk structure for device %d\n",
1266                         device_id);
1267                 ret = -ENOMEM;
1268                 goto out_free_queue;
1269         }
1270
1271         zram->disk->major = zram_major;
1272         zram->disk->first_minor = device_id;
1273         zram->disk->fops = &zram_devops;
1274         zram->disk->queue = queue;
1275         zram->disk->queue->queuedata = zram;
1276         zram->disk->private_data = zram;
1277         snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
1278
1279         /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
1280         set_capacity(zram->disk, 0);
1281         /* zram devices sort of resembles non-rotational disks */
1282         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
1283         queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, zram->disk->queue);
1284         /*
1285          * To ensure that we always get PAGE_SIZE aligned
1286          * and n*PAGE_SIZED sized I/O requests.
1287          */
1288         blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
1289         blk_queue_logical_block_size(zram->disk->queue,
1290                                         ZRAM_LOGICAL_BLOCK_SIZE);
1291         blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
1292         blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
1293         zram->disk->queue->limits.discard_granularity = PAGE_SIZE;
1294         zram->disk->queue->limits.max_sectors = SECTORS_PER_PAGE;
1295         zram->disk->queue->limits.chunk_sectors = 0;
1296         blk_queue_max_discard_sectors(zram->disk->queue, UINT_MAX);
1297         /*
1298          * zram_bio_discard() will clear all logical blocks if logical block
1299          * size is identical with physical block size(PAGE_SIZE). But if it is
1300          * different, we will skip discarding some parts of logical blocks in
1301          * the part of the request range which isn't aligned to physical block
1302          * size.  So we can't ensure that all discarded logical blocks are
1303          * zeroed.
1304          */
1305         if (ZRAM_LOGICAL_BLOCK_SIZE == PAGE_SIZE)
1306                 zram->disk->queue->limits.discard_zeroes_data = 1;
1307         else
1308                 zram->disk->queue->limits.discard_zeroes_data = 0;
1309         queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, zram->disk->queue);
1310
1311         disk_to_dev(zram->disk)->groups = zram_disk_attr_groups;
1312         add_disk(zram->disk);
1313
1314         strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
1315         zram->meta = NULL;
1316
1317         pr_info("Added device: %s\n", zram->disk->disk_name);
1318         return device_id;
1319
1320 out_free_queue:
1321         blk_cleanup_queue(queue);
1322 out_free_idr:
1323         idr_remove(&zram_index_idr, device_id);
1324 out_free_dev:
1325         kfree(zram);
1326         return ret;
1327 }
1328
1329 static int zram_remove(struct zram *zram)
1330 {
1331         struct block_device *bdev;
1332
1333         bdev = bdget_disk(zram->disk, 0);
1334         if (!bdev)
1335                 return -ENOMEM;
1336
1337         mutex_lock(&bdev->bd_mutex);
1338         if (bdev->bd_openers || zram->claim) {
1339                 mutex_unlock(&bdev->bd_mutex);
1340                 bdput(bdev);
1341                 return -EBUSY;
1342         }
1343
1344         zram->claim = true;
1345         mutex_unlock(&bdev->bd_mutex);
1346
1347         /* Make sure all the pending I/O are finished */
1348         fsync_bdev(bdev);
1349         zram_reset_device(zram);
1350         bdput(bdev);
1351
1352         pr_info("Removed device: %s\n", zram->disk->disk_name);
1353
1354         blk_cleanup_queue(zram->disk->queue);
1355         del_gendisk(zram->disk);
1356         put_disk(zram->disk);
1357         kfree(zram);
1358         return 0;
1359 }
1360
1361 /* zram-control sysfs attributes */
1362 static ssize_t hot_add_show(struct class *class,
1363                         struct class_attribute *attr,
1364                         char *buf)
1365 {
1366         int ret;
1367
1368         mutex_lock(&zram_index_mutex);
1369         ret = zram_add();
1370         mutex_unlock(&zram_index_mutex);
1371
1372         if (ret < 0)
1373                 return ret;
1374         return scnprintf(buf, PAGE_SIZE, "%d\n", ret);
1375 }
1376
1377 static ssize_t hot_remove_store(struct class *class,
1378                         struct class_attribute *attr,
1379                         const char *buf,
1380                         size_t count)
1381 {
1382         struct zram *zram;
1383         int ret, dev_id;
1384
1385         /* dev_id is gendisk->first_minor, which is `int' */
1386         ret = kstrtoint(buf, 10, &dev_id);
1387         if (ret)
1388                 return ret;
1389         if (dev_id < 0)
1390                 return -EINVAL;
1391
1392         mutex_lock(&zram_index_mutex);
1393
1394         zram = idr_find(&zram_index_idr, dev_id);
1395         if (zram) {
1396                 ret = zram_remove(zram);
1397                 if (!ret)
1398                         idr_remove(&zram_index_idr, dev_id);
1399         } else {
1400                 ret = -ENODEV;
1401         }
1402
1403         mutex_unlock(&zram_index_mutex);
1404         return ret ? ret : count;
1405 }
1406
1407 /*
1408  * NOTE: hot_add attribute is not the usual read-only sysfs attribute. In a
1409  * sense that reading from this file does alter the state of your system -- it
1410  * creates a new un-initialized zram device and returns back this device's
1411  * device_id (or an error code if it fails to create a new device).
1412  */
1413 static struct class_attribute zram_control_class_attrs[] = {
1414         __ATTR(hot_add, 0400, hot_add_show, NULL),
1415         __ATTR_WO(hot_remove),
1416         __ATTR_NULL,
1417 };
1418
1419 static struct class zram_control_class = {
1420         .name           = "zram-control",
1421         .owner          = THIS_MODULE,
1422         .class_attrs    = zram_control_class_attrs,
1423 };
1424
1425 static int zram_remove_cb(int id, void *ptr, void *data)
1426 {
1427         zram_remove(ptr);
1428         return 0;
1429 }
1430
1431 static void destroy_devices(void)
1432 {
1433         class_unregister(&zram_control_class);
1434         idr_for_each(&zram_index_idr, &zram_remove_cb, NULL);
1435         idr_destroy(&zram_index_idr);
1436         unregister_blkdev(zram_major, "zram");
1437 }
1438
1439 static int __init zram_init(void)
1440 {
1441         int ret;
1442
1443         ret = class_register(&zram_control_class);
1444         if (ret) {
1445                 pr_err("Unable to register zram-control class\n");
1446                 return ret;
1447         }
1448
1449         zram_major = register_blkdev(0, "zram");
1450         if (zram_major <= 0) {
1451                 pr_err("Unable to get major number\n");
1452                 class_unregister(&zram_control_class);
1453                 return -EBUSY;
1454         }
1455
1456         while (num_devices != 0) {
1457                 mutex_lock(&zram_index_mutex);
1458                 ret = zram_add();
1459                 mutex_unlock(&zram_index_mutex);
1460                 if (ret < 0)
1461                         goto out_error;
1462                 num_devices--;
1463         }
1464
1465         return 0;
1466
1467 out_error:
1468         destroy_devices();
1469         return ret;
1470 }
1471
1472 static void __exit zram_exit(void)
1473 {
1474         destroy_devices();
1475 }
1476
1477 module_init(zram_init);
1478 module_exit(zram_exit);
1479
1480 module_param(num_devices, uint, 0);
1481 MODULE_PARM_DESC(num_devices, "Number of pre-created zram devices");
1482
1483 MODULE_LICENSE("Dual BSD/GPL");
1484 MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
1485 MODULE_DESCRIPTION("Compressed RAM Block Device");