drivers/md/bcache/request.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Main bcache entry point - handle a read or a write request and decide what to
   4  * do with it; the make_request functions are called by the block layer.
   5  *
   6  * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
   7  * Copyright 2012 Google, Inc.
   8  */
   9
  10 #include "bcache.h"
  11 #include "btree.h"
  12 #include "debug.h"
  13 #include "request.h"
  14 #include "writeback.h"
  15
  16 #include <linux/module.h>
  17 #include <linux/hash.h>
  18 #include <linux/random.h>
  19 #include <linux/backing-dev.h>
  20
  21 #include <trace/events/bcache.h>
  22
  23 #define CUTOFF_CACHE_ADD        95
  24 #define CUTOFF_CACHE_READA      90
  25
  26 struct kmem_cache *bch_search_cache;
  27
  28 static void bch_data_insert_start(struct closure *cl);
  29
  30 static unsigned int cache_mode(struct cached_dev *dc)
  31 {
  32         return BDEV_CACHE_MODE(&dc->sb);
  33 }
  34
  35 static bool verify(struct cached_dev *dc)
  36 {
  37         return dc->verify;
  38 }
  39
  40 static void bio_csum(struct bio *bio, struct bkey *k)
  41 {
  42         struct bio_vec bv;
  43         struct bvec_iter iter;
  44         uint64_t csum = 0;
  45
  46         bio_for_each_segment(bv, bio, iter) {
  47                 void *d = kmap(bv.bv_page) + bv.bv_offset;
  48
  49                 csum = bch_crc64_update(csum, d, bv.bv_len);
  50                 kunmap(bv.bv_page);
  51         }
  52
  53         k->ptr[KEY_PTRS(k)] = csum & (~0ULL >> 1);
  54 }
  55
  56 /* Insert data into cache */
  57
  58 static void bch_data_insert_keys(struct closure *cl)
  59 {
  60         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
  61         atomic_t *journal_ref = NULL;
  62         struct bkey *replace_key = op->replace ? &op->replace_key : NULL;
  63         int ret;
  64
  65         /*
  66          * If we're looping, might already be waiting on
  67          * another journal write - can't wait on more than one journal write at
  68          * a time
  69          *
  70          * XXX: this looks wrong
  71          */
  72 #if 0
  73         while (atomic_read(&s->cl.remaining) & CLOSURE_WAITING)
  74                 closure_sync(&s->cl);
  75 #endif
  76
  77         if (!op->replace)
  78                 journal_ref = bch_journal(op->c, &op->insert_keys,
  79                                           op->flush_journal ? cl : NULL);
  80
  81         ret = bch_btree_insert(op->c, &op->insert_keys,
  82                                journal_ref, replace_key);
  83         if (ret == -ESRCH) {
  84                 op->replace_collision = true;
  85         } else if (ret) {
  86                 op->status              = BLK_STS_RESOURCE;
  87                 op->insert_data_done    = true;
  88         }
  89
  90         if (journal_ref)
  91                 atomic_dec_bug(journal_ref);
  92
  93         if (!op->insert_data_done) {
  94                 continue_at(cl, bch_data_insert_start, op->wq);
  95                 return;
  96         }
  97
  98         bch_keylist_free(&op->insert_keys);
  99         closure_return(cl);
 100 }
 101
 102 static int bch_keylist_realloc(struct keylist *l, unsigned int u64s,
 103                                struct cache_set *c)
 104 {
 105         size_t oldsize = bch_keylist_nkeys(l);
 106         size_t newsize = oldsize + u64s;
 107
 108         /*
 109          * The journalling code doesn't handle the case where the keys to insert
 110          * is bigger than an empty write: If we just return -ENOMEM here,
 111          * bch_data_insert_keys() will insert the keys created so far
 112          * and finish the rest when the keylist is empty.
 113          */
 114         if (newsize * sizeof(uint64_t) > block_bytes(c) - sizeof(struct jset))
 115                 return -ENOMEM;
 116
 117         return __bch_keylist_realloc(l, u64s);
 118 }
 119
 120 static void bch_data_invalidate(struct closure *cl)
 121 {
 122         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
 123         struct bio *bio = op->bio;
 124
 125         pr_debug("invalidating %i sectors from %llu",
 126                  bio_sectors(bio), (uint64_t) bio->bi_iter.bi_sector);
 127
 128         while (bio_sectors(bio)) {
 129                 unsigned int sectors = min(bio_sectors(bio),
 130                                        1U << (KEY_SIZE_BITS - 1));
 131
 132                 if (bch_keylist_realloc(&op->insert_keys, 2, op->c))
 133                         goto out;
 134
 135                 bio->bi_iter.bi_sector  += sectors;
 136                 bio->bi_iter.bi_size    -= sectors << 9;
 137
 138                 bch_keylist_add(&op->insert_keys,
 139                                 &KEY(op->inode,
 140                                      bio->bi_iter.bi_sector,
 141                                      sectors));
 142         }
 143
 144         op->insert_data_done = true;
 145         /* get in bch_data_insert() */
 146         bio_put(bio);
 147 out:
 148         continue_at(cl, bch_data_insert_keys, op->wq);
 149 }
 150
 151 static void bch_data_insert_error(struct closure *cl)
 152 {
 153         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
 154
 155         /*
 156          * Our data write just errored, which means we've got a bunch of keys to
 157          * insert that point to data that wasn't successfully written.
 158          *
 159          * We don't have to insert those keys but we still have to invalidate
 160          * that region of the cache - so, if we just strip off all the pointers
 161          * from the keys we'll accomplish just that.
 162          */
 163
 164         struct bkey *src = op->insert_keys.keys, *dst = op->insert_keys.keys;
 165
 166         while (src != op->insert_keys.top) {
 167                 struct bkey *n = bkey_next(src);
 168
 169                 SET_KEY_PTRS(src, 0);
 170                 memmove(dst, src, bkey_bytes(src));
 171
 172                 dst = bkey_next(dst);
 173                 src = n;
 174         }
 175
 176         op->insert_keys.top = dst;
 177
 178         bch_data_insert_keys(cl);
 179 }
 180
 181 static void bch_data_insert_endio(struct bio *bio)
 182 {
 183         struct closure *cl = bio->bi_private;
 184         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
 185
 186         if (bio->bi_status) {
 187                 /* TODO: We could try to recover from this. */
 188                 if (op->writeback)
 189                         op->status = bio->bi_status;
 190                 else if (!op->replace)
 191                         set_closure_fn(cl, bch_data_insert_error, op->wq);
 192                 else
 193                         set_closure_fn(cl, NULL, NULL);
 194         }
 195
 196         bch_bbio_endio(op->c, bio, bio->bi_status, "writing data to cache");
 197 }
 198
 199 static void bch_data_insert_start(struct closure *cl)
 200 {
 201         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
 202         struct bio *bio = op->bio, *n;
 203
 204         if (op->bypass)
 205                 return bch_data_invalidate(cl);
 206
 207         if (atomic_sub_return(bio_sectors(bio), &op->c->sectors_to_gc) < 0)
 208                 wake_up_gc(op->c);
 209
 210         /*
 211          * Journal writes are marked REQ_PREFLUSH; if the original write was a
 212          * flush, it'll wait on the journal write.
 213          */
 214         bio->bi_opf &= ~(REQ_PREFLUSH|REQ_FUA);
 215
 216         do {
 217                 unsigned int i;
 218                 struct bkey *k;
 219                 struct bio_set *split = &op->c->bio_split;
 220
 221                 /* 1 for the device pointer and 1 for the chksum */
 222                 if (bch_keylist_realloc(&op->insert_keys,
 223                                         3 + (op->csum ? 1 : 0),
 224                                         op->c)) {
 225                         continue_at(cl, bch_data_insert_keys, op->wq);
 226                         return;
 227                 }
 228
 229                 k = op->insert_keys.top;
 230                 bkey_init(k);
 231                 SET_KEY_INODE(k, op->inode);
 232                 SET_KEY_OFFSET(k, bio->bi_iter.bi_sector);
 233
 234                 if (!bch_alloc_sectors(op->c, k, bio_sectors(bio),
 235                                        op->write_point, op->write_prio,
 236                                        op->writeback))
 237                         goto err;
 238
 239                 n = bio_next_split(bio, KEY_SIZE(k), GFP_NOIO, split);
 240
 241                 n->bi_end_io    = bch_data_insert_endio;
 242                 n->bi_private   = cl;
 243
 244                 if (op->writeback) {
 245                         SET_KEY_DIRTY(k, true);
 246
 247                         for (i = 0; i < KEY_PTRS(k); i++)
 248                                 SET_GC_MARK(PTR_BUCKET(op->c, k, i),
 249                                             GC_MARK_DIRTY);
 250                 }
 251
 252                 SET_KEY_CSUM(k, op->csum);
 253                 if (KEY_CSUM(k))
 254                         bio_csum(n, k);
 255
 256                 trace_bcache_cache_insert(k);
 257                 bch_keylist_push(&op->insert_keys);
 258
 259                 bio_set_op_attrs(n, REQ_OP_WRITE, 0);
 260                 bch_submit_bbio(n, op->c, k, 0);
 261         } while (n != bio);
 262
 263         op->insert_data_done = true;
 264         continue_at(cl, bch_data_insert_keys, op->wq);
 265         return;
 266 err:
 267         /* bch_alloc_sectors() blocks if s->writeback = true */
 268         BUG_ON(op->writeback);
 269
 270         /*
 271          * But if it's not a writeback write we'd rather just bail out if
 272          * there aren't any buckets ready to write to - it might take awhile and
 273          * we might be starving btree writes for gc or something.
 274          */
 275
 276         if (!op->replace) {
 277                 /*
 278                  * Writethrough write: We can't complete the write until we've
 279                  * updated the index. But we don't want to delay the write while
 280                  * we wait for buckets to be freed up, so just invalidate the
 281                  * rest of the write.
 282                  */
 283                 op->bypass = true;
 284                 return bch_data_invalidate(cl);
 285         } else {
 286                 /*
 287                  * From a cache miss, we can just insert the keys for the data
 288                  * we have written or bail out if we didn't do anything.
 289                  */
 290                 op->insert_data_done = true;
 291                 bio_put(bio);
 292
 293                 if (!bch_keylist_empty(&op->insert_keys))
 294                         continue_at(cl, bch_data_insert_keys, op->wq);
 295                 else
 296                         closure_return(cl);
 297         }
 298 }
 299
 300 /**
 301  * bch_data_insert - stick some data in the cache
 302  * @cl: closure pointer.
 303  *
 304  * This is the starting point for any data to end up in a cache device; it could
 305  * be from a normal write, or a writeback write, or a write to a flash only
 306  * volume - it's also used by the moving garbage collector to compact data in
 307  * mostly empty buckets.
 308  *
 309  * It first writes the data to the cache, creating a list of keys to be inserted
 310  * (if the data had to be fragmented there will be multiple keys); after the
 311  * data is written it calls bch_journal, and after the keys have been added to
 312  * the next journal write they're inserted into the btree.
 313  *
 314  * It inserts the data in s->cache_bio; bi_sector is used for the key offset,
 315  * and op->inode is used for the key inode.
 316  *
 317  * If s->bypass is true, instead of inserting the data it invalidates the
 318  * region of the cache represented by s->cache_bio and op->inode.
 319  */
 320 void bch_data_insert(struct closure *cl)
 321 {
 322         struct data_insert_op *op = container_of(cl, struct data_insert_op, cl);
 323
 324         trace_bcache_write(op->c, op->inode, op->bio,
 325                            op->writeback, op->bypass);
 326
 327         bch_keylist_init(&op->insert_keys);
 328         bio_get(op->bio);
 329         bch_data_insert_start(cl);
 330 }
 331
 332 /* Congested? */
 333
 334 unsigned int bch_get_congested(struct cache_set *c)
 335 {
 336         int i;
 337         long rand;
 338
 339         if (!c->congested_read_threshold_us &&
 340             !c->congested_write_threshold_us)
 341                 return 0;
 342
 343         i = (local_clock_us() - c->congested_last_us) / 1024;
 344         if (i < 0)
 345                 return 0;
 346
 347         i += atomic_read(&c->congested);
 348         if (i >= 0)
 349                 return 0;
 350
 351         i += CONGESTED_MAX;
 352
 353         if (i > 0)
 354                 i = fract_exp_two(i, 6);
 355
 356         rand = get_random_int();
 357         i -= bitmap_weight(&rand, BITS_PER_LONG);
 358
 359         return i > 0 ? i : 1;
 360 }
 361
 362 static void add_sequential(struct task_struct *t)
 363 {
 364         ewma_add(t->sequential_io_avg,
 365                  t->sequential_io, 8, 0);
 366
 367         t->sequential_io = 0;
 368 }
 369
 370 static struct hlist_head *iohash(struct cached_dev *dc, uint64_t k)
 371 {
 372         return &dc->io_hash[hash_64(k, RECENT_IO_BITS)];
 373 }
 374
 375 static bool check_should_bypass(struct cached_dev *dc, struct bio *bio)
 376 {
 377         struct cache_set *c = dc->disk.c;
 378         unsigned int mode = cache_mode(dc);
 379         unsigned int sectors, congested = bch_get_congested(c);
 380         struct task_struct *task = current;
 381         struct io *i;
 382
 383         if (test_bit(BCACHE_DEV_DETACHING, &dc->disk.flags) ||
 384             c->gc_stats.in_use > CUTOFF_CACHE_ADD ||
 385             (bio_op(bio) == REQ_OP_DISCARD))
 386                 goto skip;
 387
 388         if (mode == CACHE_MODE_NONE ||
 389             (mode == CACHE_MODE_WRITEAROUND &&
 390              op_is_write(bio_op(bio))))
 391                 goto skip;
 392
 393         /*
 394          * If the bio is for read-ahead or background IO, bypass it or
 395          * not depends on the following situations,
 396          * - If the IO is for meta data, always cache it and no bypass
 397          * - If the IO is not meta data, check dc->cache_reada_policy,
 398          *      BCH_CACHE_READA_ALL: cache it and not bypass
 399          *      BCH_CACHE_READA_META_ONLY: not cache it and bypass
 400          * That is, read-ahead request for metadata always get cached
 401          * (eg, for gfs2 or xfs).
 402          */
 403         if ((bio->bi_opf & (REQ_RAHEAD|REQ_BACKGROUND))) {
 404                 if (!(bio->bi_opf & (REQ_META|REQ_PRIO)) &&
 405                     (dc->cache_readahead_policy != BCH_CACHE_READA_ALL))
 406                         goto skip;
 407         }
 408
 409         if (bio->bi_iter.bi_sector & (c->sb.block_size - 1) ||
 410             bio_sectors(bio) & (c->sb.block_size - 1)) {
 411                 pr_debug("skipping unaligned io");
 412                 goto skip;
 413         }
 414
 415         if (bypass_torture_test(dc)) {
 416                 if ((get_random_int() & 3) == 3)
 417                         goto skip;
 418                 else
 419                         goto rescale;
 420         }
 421
 422         if (!congested && !dc->sequential_cutoff)
 423                 goto rescale;
 424
 425         spin_lock(&dc->io_lock);
 426
 427         hlist_for_each_entry(i, iohash(dc, bio->bi_iter.bi_sector), hash)
 428                 if (i->last == bio->bi_iter.bi_sector &&
 429                     time_before(jiffies, i->jiffies))
 430                         goto found;
 431
 432         i = list_first_entry(&dc->io_lru, struct io, lru);
 433
 434         add_sequential(task);
 435         i->sequential = 0;
 436 found:
 437         if (i->sequential + bio->bi_iter.bi_size > i->sequential)
 438                 i->sequential   += bio->bi_iter.bi_size;
 439
 440         i->last                  = bio_end_sector(bio);
 441         i->jiffies               = jiffies + msecs_to_jiffies(5000);
 442         task->sequential_io      = i->sequential;
 443
 444         hlist_del(&i->hash);
 445         hlist_add_head(&i->hash, iohash(dc, i->last));
 446         list_move_tail(&i->lru, &dc->io_lru);
 447
 448         spin_unlock(&dc->io_lock);
 449
 450         sectors = max(task->sequential_io,
 451                       task->sequential_io_avg) >> 9;
 452
 453         if (dc->sequential_cutoff &&
 454             sectors >= dc->sequential_cutoff >> 9) {
 455                 trace_bcache_bypass_sequential(bio);
 456                 goto skip;
 457         }
 458
 459         if (congested && sectors >= congested) {
 460                 trace_bcache_bypass_congested(bio);
 461                 goto skip;
 462         }
 463
 464 rescale:
 465         bch_rescale_priorities(c, bio_sectors(bio));
 466         return false;
 467 skip:
 468         bch_mark_sectors_bypassed(c, dc, bio_sectors(bio));
 469         return true;
 470 }
 471
 472 /* Cache lookup */
 473
 474 struct search {
 475         /* Stack frame for bio_complete */
 476         struct closure          cl;
 477
 478         struct bbio             bio;
 479         struct bio              *orig_bio;
 480         struct bio              *cache_miss;
 481         struct bcache_device    *d;
 482
 483         unsigned int            insert_bio_sectors;
 484         unsigned int            recoverable:1;
 485         unsigned int            write:1;
 486         unsigned int            read_dirty_data:1;
 487         unsigned int            cache_missed:1;
 488
 489         unsigned long           start_time;
 490
 491         struct btree_op         op;
 492         struct data_insert_op   iop;
 493 };
 494
 495 static void bch_cache_read_endio(struct bio *bio)
 496 {
 497         struct bbio *b = container_of(bio, struct bbio, bio);
 498         struct closure *cl = bio->bi_private;
 499         struct search *s = container_of(cl, struct search, cl);
 500
 501         /*
 502          * If the bucket was reused while our bio was in flight, we might have
 503          * read the wrong data. Set s->error but not error so it doesn't get
 504          * counted against the cache device, but we'll still reread the data
 505          * from the backing device.
 506          */
 507
 508         if (bio->bi_status)
 509                 s->iop.status = bio->bi_status;
 510         else if (!KEY_DIRTY(&b->key) &&
 511                  ptr_stale(s->iop.c, &b->key, 0)) {
 512                 atomic_long_inc(&s->iop.c->cache_read_races);
 513                 s->iop.status = BLK_STS_IOERR;
 514         }
 515
 516         bch_bbio_endio(s->iop.c, bio, bio->bi_status, "reading from cache");
 517 }
 518
 519 /*
 520  * Read from a single key, handling the initial cache miss if the key starts in
 521  * the middle of the bio
 522  */
 523 static int cache_lookup_fn(struct btree_op *op, struct btree *b, struct bkey *k)
 524 {
 525         struct search *s = container_of(op, struct search, op);
 526         struct bio *n, *bio = &s->bio.bio;
 527         struct bkey *bio_key;
 528         unsigned int ptr;
 529
 530         if (bkey_cmp(k, &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0)) <= 0)
 531                 return MAP_CONTINUE;
 532
 533         if (KEY_INODE(k) != s->iop.inode ||
 534             KEY_START(k) > bio->bi_iter.bi_sector) {
 535                 unsigned int bio_sectors = bio_sectors(bio);
 536                 unsigned int sectors = KEY_INODE(k) == s->iop.inode
 537                         ? min_t(uint64_t, INT_MAX,
 538                                 KEY_START(k) - bio->bi_iter.bi_sector)
 539                         : INT_MAX;
 540                 int ret = s->d->cache_miss(b, s, bio, sectors);
 541
 542                 if (ret != MAP_CONTINUE)
 543                         return ret;
 544
 545                 /* if this was a complete miss we shouldn't get here */
 546                 BUG_ON(bio_sectors <= sectors);
 547         }
 548
 549         if (!KEY_SIZE(k))
 550                 return MAP_CONTINUE;
 551
 552         /* XXX: figure out best pointer - for multiple cache devices */
 553         ptr = 0;
 554
 555         PTR_BUCKET(b->c, k, ptr)->prio = INITIAL_PRIO;
 556
 557         if (KEY_DIRTY(k))
 558                 s->read_dirty_data = true;
 559
 560         n = bio_next_split(bio, min_t(uint64_t, INT_MAX,
 561                                       KEY_OFFSET(k) - bio->bi_iter.bi_sector),
 562                            GFP_NOIO, &s->d->bio_split);
 563
 564         bio_key = &container_of(n, struct bbio, bio)->key;
 565         bch_bkey_copy_single_ptr(bio_key, k, ptr);
 566
 567         bch_cut_front(&KEY(s->iop.inode, n->bi_iter.bi_sector, 0), bio_key);
 568         bch_cut_back(&KEY(s->iop.inode, bio_end_sector(n), 0), bio_key);
 569
 570         n->bi_end_io    = bch_cache_read_endio;
 571         n->bi_private   = &s->cl;
 572
 573         /*
 574          * The bucket we're reading from might be reused while our bio
 575          * is in flight, and we could then end up reading the wrong
 576          * data.
 577          *
 578          * We guard against this by checking (in cache_read_endio()) if
 579          * the pointer is stale again; if so, we treat it as an error
 580          * and reread from the backing device (but we don't pass that
 581          * error up anywhere).
 582          */
 583
 584         __bch_submit_bbio(n, b->c);
 585         return n == bio ? MAP_DONE : MAP_CONTINUE;
 586 }
 587
 588 static void cache_lookup(struct closure *cl)
 589 {
 590         struct search *s = container_of(cl, struct search, iop.cl);
 591         struct bio *bio = &s->bio.bio;
 592         struct cached_dev *dc;
 593         int ret;
 594
 595         bch_btree_op_init(&s->op, -1);
 596
 597         ret = bch_btree_map_keys(&s->op, s->iop.c,
 598                                  &KEY(s->iop.inode, bio->bi_iter.bi_sector, 0),
 599                                  cache_lookup_fn, MAP_END_KEY);
 600         if (ret == -EAGAIN) {
 601                 continue_at(cl, cache_lookup, bcache_wq);
 602                 return;
 603         }
 604
 605         /*
 606          * We might meet err when searching the btree, If that happens, we will
 607          * get negative ret, in this scenario we should not recover data from
 608          * backing device (when cache device is dirty) because we don't know
 609          * whether bkeys the read request covered are all clean.
 610          *
 611          * And after that happened, s->iop.status is still its initial value
 612          * before we submit s->bio.bio
 613          */
 614         if (ret < 0) {
 615                 BUG_ON(ret == -EINTR);
 616                 if (s->d && s->d->c &&
 617                                 !UUID_FLASH_ONLY(&s->d->c->uuids[s->d->id])) {
 618                         dc = container_of(s->d, struct cached_dev, disk);
 619                         if (dc && atomic_read(&dc->has_dirty))
 620                                 s->recoverable = false;
 621                 }
 622                 if (!s->iop.status)
 623                         s->iop.status = BLK_STS_IOERR;
 624         }
 625
 626         closure_return(cl);
 627 }
 628
 629 /* Common code for the make_request functions */
 630
 631 static void request_endio(struct bio *bio)
 632 {
 633         struct closure *cl = bio->bi_private;
 634
 635         if (bio->bi_status) {
 636                 struct search *s = container_of(cl, struct search, cl);
 637
 638                 s->iop.status = bio->bi_status;
 639                 /* Only cache read errors are recoverable */
 640                 s->recoverable = false;
 641         }
 642
 643         bio_put(bio);
 644         closure_put(cl);
 645 }
 646
 647 static void backing_request_endio(struct bio *bio)
 648 {
 649         struct closure *cl = bio->bi_private;
 650
 651         if (bio->bi_status) {
 652                 struct search *s = container_of(cl, struct search, cl);
 653                 struct cached_dev *dc = container_of(s->d,
 654                                                      struct cached_dev, disk);
 655                 /*
 656                  * If a bio has REQ_PREFLUSH for writeback mode, it is
 657                  * speically assembled in cached_dev_write() for a non-zero
 658                  * write request which has REQ_PREFLUSH. we don't set
 659                  * s->iop.status by this failure, the status will be decided
 660                  * by result of bch_data_insert() operation.
 661                  */
 662                 if (unlikely(s->iop.writeback &&
 663                              bio->bi_opf & REQ_PREFLUSH)) {
 664                         pr_err("Can't flush %s: returned bi_status %i",
 665                                 dc->backing_dev_name, bio->bi_status);
 666                 } else {
 667                         /* set to orig_bio->bi_status in bio_complete() */
 668                         s->iop.status = bio->bi_status;
 669                 }
 670                 s->recoverable = false;
 671                 /* should count I/O error for backing device here */
 672                 bch_count_backing_io_errors(dc, bio);
 673         }
 674
 675         bio_put(bio);
 676         closure_put(cl);
 677 }
 678
 679 static void bio_complete(struct search *s)
 680 {
 681         if (s->orig_bio) {
 682                 generic_end_io_acct(s->d->disk->queue, bio_op(s->orig_bio),
 683                                     &s->d->disk->part0, s->start_time);
 684
 685                 trace_bcache_request_end(s->d, s->orig_bio);
 686                 s->orig_bio->bi_status = s->iop.status;
 687                 bio_endio(s->orig_bio);
 688                 s->orig_bio = NULL;
 689         }
 690 }
 691
 692 static void do_bio_hook(struct search *s,
 693                         struct bio *orig_bio,
 694                         bio_end_io_t *end_io_fn)
 695 {
 696         struct bio *bio = &s->bio.bio;
 697
 698         bio_init(bio, NULL, 0);
 699         __bio_clone_fast(bio, orig_bio);
 700         /*
 701          * bi_end_io can be set separately somewhere else, e.g. the
 702          * variants in,
 703          * - cache_bio->bi_end_io from cached_dev_cache_miss()
 704          * - n->bi_end_io from cache_lookup_fn()
 705          */
 706         bio->bi_end_io          = end_io_fn;
 707         bio->bi_private         = &s->cl;
 708
 709         bio_cnt_set(bio, 3);
 710 }
 711
 712 static void search_free(struct closure *cl)
 713 {
 714         struct search *s = container_of(cl, struct search, cl);
 715
 716         atomic_dec(&s->d->c->search_inflight);
 717
 718         if (s->iop.bio)
 719                 bio_put(s->iop.bio);
 720
 721         bio_complete(s);
 722         closure_debug_destroy(cl);
 723         mempool_free(s, &s->d->c->search);
 724 }
 725
 726 static inline struct search *search_alloc(struct bio *bio,
 727                                           struct bcache_device *d)
 728 {
 729         struct search *s;
 730
 731         s = mempool_alloc(&d->c->search, GFP_NOIO);
 732
 733         closure_init(&s->cl, NULL);
 734         do_bio_hook(s, bio, request_endio);
 735         atomic_inc(&d->c->search_inflight);
 736
 737         s->orig_bio             = bio;
 738         s->cache_miss           = NULL;
 739         s->cache_missed         = 0;
 740         s->d                    = d;
 741         s->recoverable          = 1;
 742         s->write                = op_is_write(bio_op(bio));
 743         s->read_dirty_data      = 0;
 744         s->start_time           = jiffies;
 745
 746         s->iop.c                = d->c;
 747         s->iop.bio              = NULL;
 748         s->iop.inode            = d->id;
 749         s->iop.write_point      = hash_long((unsigned long) current, 16);
 750         s->iop.write_prio       = 0;
 751         s->iop.status           = 0;
 752         s->iop.flags            = 0;
 753         s->iop.flush_journal    = op_is_flush(bio->bi_opf);
 754         s->iop.wq               = bcache_wq;
 755
 756         return s;
 757 }
 758
 759 /* Cached devices */
 760
 761 static void cached_dev_bio_complete(struct closure *cl)
 762 {
 763         struct search *s = container_of(cl, struct search, cl);
 764         struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
 765
 766         search_free(cl);
 767         cached_dev_put(dc);
 768 }
 769
 770 /* Process reads */
 771
 772 static void cached_dev_cache_miss_done(struct closure *cl)
 773 {
 774         struct search *s = container_of(cl, struct search, cl);
 775
 776         if (s->iop.replace_collision)
 777                 bch_mark_cache_miss_collision(s->iop.c, s->d);
 778
 779         if (s->iop.bio)
 780                 bio_free_pages(s->iop.bio);
 781
 782         cached_dev_bio_complete(cl);
 783 }
 784
 785 static void cached_dev_read_error(struct closure *cl)
 786 {
 787         struct search *s = container_of(cl, struct search, cl);
 788         struct bio *bio = &s->bio.bio;
 789
 790         /*
 791          * If read request hit dirty data (s->read_dirty_data is true),
 792          * then recovery a failed read request from cached device may
 793          * get a stale data back. So read failure recovery is only
 794          * permitted when read request hit clean data in cache device,
 795          * or when cache read race happened.
 796          */
 797         if (s->recoverable && !s->read_dirty_data) {
 798                 /* Retry from the backing device: */
 799                 trace_bcache_read_retry(s->orig_bio);
 800
 801                 s->iop.status = 0;
 802                 do_bio_hook(s, s->orig_bio, backing_request_endio);
 803
 804                 /* XXX: invalidate cache */
 805
 806                 /* I/O request sent to backing device */
 807                 closure_bio_submit(s->iop.c, bio, cl);
 808         }
 809
 810         continue_at(cl, cached_dev_cache_miss_done, NULL);
 811 }
 812
 813 static void cached_dev_read_done(struct closure *cl)
 814 {
 815         struct search *s = container_of(cl, struct search, cl);
 816         struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
 817
 818         /*
 819          * We had a cache miss; cache_bio now contains data ready to be inserted
 820          * into the cache.
 821          *
 822          * First, we copy the data we just read from cache_bio's bounce buffers
 823          * to the buffers the original bio pointed to:
 824          */
 825
 826         if (s->iop.bio) {
 827                 bio_reset(s->iop.bio);
 828                 s->iop.bio->bi_iter.bi_sector =
 829                         s->cache_miss->bi_iter.bi_sector;
 830                 bio_copy_dev(s->iop.bio, s->cache_miss);
 831                 s->iop.bio->bi_iter.bi_size = s->insert_bio_sectors << 9;
 832                 bch_bio_map(s->iop.bio, NULL);
 833
 834                 bio_copy_data(s->cache_miss, s->iop.bio);
 835
 836                 bio_put(s->cache_miss);
 837                 s->cache_miss = NULL;
 838         }
 839
 840         if (verify(dc) && s->recoverable && !s->read_dirty_data)
 841                 bch_data_verify(dc, s->orig_bio);
 842
 843         bio_complete(s);
 844
 845         if (s->iop.bio &&
 846             !test_bit(CACHE_SET_STOPPING, &s->iop.c->flags)) {
 847                 BUG_ON(!s->iop.replace);
 848                 closure_call(&s->iop.cl, bch_data_insert, NULL, cl);
 849         }
 850
 851         continue_at(cl, cached_dev_cache_miss_done, NULL);
 852 }
 853
 854 static void cached_dev_read_done_bh(struct closure *cl)
 855 {
 856         struct search *s = container_of(cl, struct search, cl);
 857         struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
 858
 859         bch_mark_cache_accounting(s->iop.c, s->d,
 860                                   !s->cache_missed, s->iop.bypass);
 861         trace_bcache_read(s->orig_bio, !s->cache_missed, s->iop.bypass);
 862
 863         if (s->iop.status)
 864                 continue_at_nobarrier(cl, cached_dev_read_error, bcache_wq);
 865         else if (s->iop.bio || verify(dc))
 866                 continue_at_nobarrier(cl, cached_dev_read_done, bcache_wq);
 867         else
 868                 continue_at_nobarrier(cl, cached_dev_bio_complete, NULL);
 869 }
 870
 871 static int cached_dev_cache_miss(struct btree *b, struct search *s,
 872                                  struct bio *bio, unsigned int sectors)
 873 {
 874         int ret = MAP_CONTINUE;
 875         unsigned int reada = 0;
 876         struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
 877         struct bio *miss, *cache_bio;
 878
 879         s->cache_missed = 1;
 880
 881         if (s->cache_miss || s->iop.bypass) {
 882                 miss = bio_next_split(bio, sectors, GFP_NOIO, &s->d->bio_split);
 883                 ret = miss == bio ? MAP_DONE : MAP_CONTINUE;
 884                 goto out_submit;
 885         }
 886
 887         if (!(bio->bi_opf & REQ_RAHEAD) &&
 888             !(bio->bi_opf & (REQ_META|REQ_PRIO)) &&
 889             s->iop.c->gc_stats.in_use < CUTOFF_CACHE_READA)
 890                 reada = min_t(sector_t, dc->readahead >> 9,
 891                               get_capacity(bio->bi_disk) - bio_end_sector(bio));
 892
 893         s->insert_bio_sectors = min(sectors, bio_sectors(bio) + reada);
 894
 895         s->iop.replace_key = KEY(s->iop.inode,
 896                                  bio->bi_iter.bi_sector + s->insert_bio_sectors,
 897                                  s->insert_bio_sectors);
 898
 899         ret = bch_btree_insert_check_key(b, &s->op, &s->iop.replace_key);
 900         if (ret)
 901                 return ret;
 902
 903         s->iop.replace = true;
 904
 905         miss = bio_next_split(bio, sectors, GFP_NOIO, &s->d->bio_split);
 906
 907         /* btree_search_recurse()'s btree iterator is no good anymore */
 908         ret = miss == bio ? MAP_DONE : -EINTR;
 909
 910         cache_bio = bio_alloc_bioset(GFP_NOWAIT,
 911                         DIV_ROUND_UP(s->insert_bio_sectors, PAGE_SECTORS),
 912                         &dc->disk.bio_split);
 913         if (!cache_bio)
 914                 goto out_submit;
 915
 916         cache_bio->bi_iter.bi_sector    = miss->bi_iter.bi_sector;
 917         bio_copy_dev(cache_bio, miss);
 918         cache_bio->bi_iter.bi_size      = s->insert_bio_sectors << 9;
 919
 920         cache_bio->bi_end_io    = backing_request_endio;
 921         cache_bio->bi_private   = &s->cl;
 922
 923         bch_bio_map(cache_bio, NULL);
 924         if (bch_bio_alloc_pages(cache_bio, __GFP_NOWARN|GFP_NOIO))
 925                 goto out_put;
 926
 927         if (reada)
 928                 bch_mark_cache_readahead(s->iop.c, s->d);
 929
 930         s->cache_miss   = miss;
 931         s->iop.bio      = cache_bio;
 932         bio_get(cache_bio);
 933         /* I/O request sent to backing device */
 934         closure_bio_submit(s->iop.c, cache_bio, &s->cl);
 935
 936         return ret;
 937 out_put:
 938         bio_put(cache_bio);
 939 out_submit:
 940         miss->bi_end_io         = backing_request_endio;
 941         miss->bi_private        = &s->cl;
 942         /* I/O request sent to backing device */
 943         closure_bio_submit(s->iop.c, miss, &s->cl);
 944         return ret;
 945 }
 946
 947 static void cached_dev_read(struct cached_dev *dc, struct search *s)
 948 {
 949         struct closure *cl = &s->cl;
 950
 951         closure_call(&s->iop.cl, cache_lookup, NULL, cl);
 952         continue_at(cl, cached_dev_read_done_bh, NULL);
 953 }
 954
 955 /* Process writes */
 956
 957 static void cached_dev_write_complete(struct closure *cl)
 958 {
 959         struct search *s = container_of(cl, struct search, cl);
 960         struct cached_dev *dc = container_of(s->d, struct cached_dev, disk);
 961
 962         up_read_non_owner(&dc->writeback_lock);
 963         cached_dev_bio_complete(cl);
 964 }
 965
 966 static void cached_dev_write(struct cached_dev *dc, struct search *s)
 967 {
 968         struct closure *cl = &s->cl;
 969         struct bio *bio = &s->bio.bio;
 970         struct bkey start = KEY(dc->disk.id, bio->bi_iter.bi_sector, 0);
 971         struct bkey end = KEY(dc->disk.id, bio_end_sector(bio), 0);
 972
 973         bch_keybuf_check_overlapping(&s->iop.c->moving_gc_keys, &start, &end);
 974
 975         down_read_non_owner(&dc->writeback_lock);
 976         if (bch_keybuf_check_overlapping(&dc->writeback_keys, &start, &end)) {
 977                 /*
 978                  * We overlap with some dirty data undergoing background
 979                  * writeback, force this write to writeback
 980                  */
 981                 s->iop.bypass = false;
 982                 s->iop.writeback = true;
 983         }
 984
 985         /*
 986          * Discards aren't _required_ to do anything, so skipping if
 987          * check_overlapping returned true is ok
 988          *
 989          * But check_overlapping drops dirty keys for which io hasn't started,
 990          * so we still want to call it.
 991          */
 992         if (bio_op(bio) == REQ_OP_DISCARD)
 993                 s->iop.bypass = true;
 994
 995         if (should_writeback(dc, s->orig_bio,
 996                              cache_mode(dc),
 997                              s->iop.bypass)) {
 998                 s->iop.bypass = false;
 999                 s->iop.writeback = true;
1000         }
1001
1002         if (s->iop.bypass) {
1003                 s->iop.bio = s->orig_bio;
1004                 bio_get(s->iop.bio);
1005
1006                 if (bio_op(bio) == REQ_OP_DISCARD &&
1007                     !blk_queue_discard(bdev_get_queue(dc->bdev)))
1008                         goto insert_data;
1009
1010                 /* I/O request sent to backing device */
1011                 bio->bi_end_io = backing_request_endio;
1012                 closure_bio_submit(s->iop.c, bio, cl);
1013
1014         } else if (s->iop.writeback) {
1015                 bch_writeback_add(dc);
1016                 s->iop.bio = bio;
1017
1018                 if (bio->bi_opf & REQ_PREFLUSH) {
1019                         /*
1020                          * Also need to send a flush to the backing
1021                          * device.
1022                          */
1023                         struct bio *flush;
1024
1025                         flush = bio_alloc_bioset(GFP_NOIO, 0,
1026                                                  &dc->disk.bio_split);
1027                         if (!flush) {
1028                                 s->iop.status = BLK_STS_RESOURCE;
1029                                 goto insert_data;
1030                         }
1031                         bio_copy_dev(flush, bio);
1032                         flush->bi_end_io = backing_request_endio;
1033                         flush->bi_private = cl;
1034                         flush->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
1035                         /* I/O request sent to backing device */
1036                         closure_bio_submit(s->iop.c, flush, cl);
1037                 }
1038         } else {
1039                 s->iop.bio = bio_clone_fast(bio, GFP_NOIO, &dc->disk.bio_split);
1040                 /* I/O request sent to backing device */
1041                 bio->bi_end_io = backing_request_endio;
1042                 closure_bio_submit(s->iop.c, bio, cl);
1043         }
1044
1045 insert_data:
1046         closure_call(&s->iop.cl, bch_data_insert, NULL, cl);
1047         continue_at(cl, cached_dev_write_complete, NULL);
1048 }
1049
1050 static void cached_dev_nodata(struct closure *cl)
1051 {
1052         struct search *s = container_of(cl, struct search, cl);
1053         struct bio *bio = &s->bio.bio;
1054
1055         if (s->iop.flush_journal)
1056                 bch_journal_meta(s->iop.c, cl);
1057
1058         /* If it's a flush, we send the flush to the backing device too */
1059         bio->bi_end_io = backing_request_endio;
1060         closure_bio_submit(s->iop.c, bio, cl);
1061
1062         continue_at(cl, cached_dev_bio_complete, NULL);
1063 }
1064
1065 struct detached_dev_io_private {
1066         struct bcache_device    *d;
1067         unsigned long           start_time;
1068         bio_end_io_t            *bi_end_io;
1069         void                    *bi_private;
1070 };
1071
1072 static void detached_dev_end_io(struct bio *bio)
1073 {
1074         struct detached_dev_io_private *ddip;
1075
1076         ddip = bio->bi_private;
1077         bio->bi_end_io = ddip->bi_end_io;
1078         bio->bi_private = ddip->bi_private;
1079
1080         generic_end_io_acct(ddip->d->disk->queue, bio_op(bio),
1081                             &ddip->d->disk->part0, ddip->start_time);
1082
1083         if (bio->bi_status) {
1084                 struct cached_dev *dc = container_of(ddip->d,
1085                                                      struct cached_dev, disk);
1086                 /* should count I/O error for backing device here */
1087                 bch_count_backing_io_errors(dc, bio);
1088         }
1089
1090         kfree(ddip);
1091         bio->bi_end_io(bio);
1092 }
1093
1094 static void detached_dev_do_request(struct bcache_device *d, struct bio *bio)
1095 {
1096         struct detached_dev_io_private *ddip;
1097         struct cached_dev *dc = container_of(d, struct cached_dev, disk);
1098
1099         /*
1100          * no need to call closure_get(&dc->disk.cl),
1101          * because upper layer had already opened bcache device,
1102          * which would call closure_get(&dc->disk.cl)
1103          */
1104         ddip = kzalloc(sizeof(struct detached_dev_io_private), GFP_NOIO);
1105         ddip->d = d;
1106         ddip->start_time = jiffies;
1107         ddip->bi_end_io = bio->bi_end_io;
1108         ddip->bi_private = bio->bi_private;
1109         bio->bi_end_io = detached_dev_end_io;
1110         bio->bi_private = ddip;
1111
1112         if ((bio_op(bio) == REQ_OP_DISCARD) &&
1113             !blk_queue_discard(bdev_get_queue(dc->bdev)))
1114                 bio->bi_end_io(bio);
1115         else
1116                 generic_make_request(bio);
1117 }
1118
1119 static void quit_max_writeback_rate(struct cache_set *c,
1120                                     struct cached_dev *this_dc)
1121 {
1122         int i;
1123         struct bcache_device *d;
1124         struct cached_dev *dc;
1125
1126         /*
1127          * mutex bch_register_lock may compete with other parallel requesters,
1128          * or attach/detach operations on other backing device. Waiting to
1129          * the mutex lock may increase I/O request latency for seconds or more.
1130          * To avoid such situation, if mutext_trylock() failed, only writeback
1131          * rate of current cached device is set to 1, and __update_write_back()
1132          * will decide writeback rate of other cached devices (remember now
1133          * c->idle_counter is 0 already).
1134          */
1135         if (mutex_trylock(&bch_register_lock)) {
1136                 for (i = 0; i < c->devices_max_used; i++) {
1137                         if (!c->devices[i])
1138                                 continue;
1139
1140                         if (UUID_FLASH_ONLY(&c->uuids[i]))
1141                                 continue;
1142
1143                         d = c->devices[i];
1144                         dc = container_of(d, struct cached_dev, disk);
1145                         /*
1146                          * set writeback rate to default minimum value,
1147                          * then let update_writeback_rate() to decide the
1148                          * upcoming rate.
1149                          */
1150                         atomic_long_set(&dc->writeback_rate.rate, 1);
1151                 }
1152                 mutex_unlock(&bch_register_lock);
1153         } else
1154                 atomic_long_set(&this_dc->writeback_rate.rate, 1);
1155 }
1156
1157 /* Cached devices - read & write stuff */
1158
1159 static blk_qc_t cached_dev_make_request(struct request_queue *q,
1160                                         struct bio *bio)
1161 {
1162         struct search *s;
1163         struct bcache_device *d = bio->bi_disk->private_data;
1164         struct cached_dev *dc = container_of(d, struct cached_dev, disk);
1165         int rw = bio_data_dir(bio);
1166
1167         if (unlikely((d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags)) ||
1168                      dc->io_disable)) {
1169                 bio->bi_status = BLK_STS_IOERR;
1170                 bio_endio(bio);
1171                 return BLK_QC_T_NONE;
1172         }
1173
1174         if (likely(d->c)) {
1175                 if (atomic_read(&d->c->idle_counter))
1176                         atomic_set(&d->c->idle_counter, 0);
1177                 /*
1178                  * If at_max_writeback_rate of cache set is true and new I/O
1179                  * comes, quit max writeback rate of all cached devices
1180                  * attached to this cache set, and set at_max_writeback_rate
1181                  * to false.
1182                  */
1183                 if (unlikely(atomic_read(&d->c->at_max_writeback_rate) == 1)) {
1184                         atomic_set(&d->c->at_max_writeback_rate, 0);
1185                         quit_max_writeback_rate(d->c, dc);
1186                 }
1187         }
1188
1189         generic_start_io_acct(q,
1190                               bio_op(bio),
1191                               bio_sectors(bio),
1192                               &d->disk->part0);
1193
1194         bio_set_dev(bio, dc->bdev);
1195         bio->bi_iter.bi_sector += dc->sb.data_offset;
1196
1197         if (cached_dev_get(dc)) {
1198                 s = search_alloc(bio, d);
1199                 trace_bcache_request_start(s->d, bio);
1200
1201                 if (!bio->bi_iter.bi_size) {
1202                         /*
1203                          * can't call bch_journal_meta from under
1204                          * generic_make_request
1205                          */
1206                         continue_at_nobarrier(&s->cl,
1207                                               cached_dev_nodata,
1208                                               bcache_wq);
1209                 } else {
1210                         s->iop.bypass = check_should_bypass(dc, bio);
1211
1212                         if (rw)
1213                                 cached_dev_write(dc, s);
1214                         else
1215                                 cached_dev_read(dc, s);
1216                 }
1217         } else
1218                 /* I/O request sent to backing device */
1219                 detached_dev_do_request(d, bio);
1220
1221         return BLK_QC_T_NONE;
1222 }
1223
1224 static int cached_dev_ioctl(struct bcache_device *d, fmode_t mode,
1225                             unsigned int cmd, unsigned long arg)
1226 {
1227         struct cached_dev *dc = container_of(d, struct cached_dev, disk);
1228
1229         if (dc->io_disable)
1230                 return -EIO;
1231
1232         return __blkdev_driver_ioctl(dc->bdev, mode, cmd, arg);
1233 }
1234
1235 static int cached_dev_congested(void *data, int bits)
1236 {
1237         struct bcache_device *d = data;
1238         struct cached_dev *dc = container_of(d, struct cached_dev, disk);
1239         struct request_queue *q = bdev_get_queue(dc->bdev);
1240         int ret = 0;
1241
1242         if (bdi_congested(q->backing_dev_info, bits))
1243                 return 1;
1244
1245         if (cached_dev_get(dc)) {
1246                 unsigned int i;
1247                 struct cache *ca;
1248
1249                 for_each_cache(ca, d->c, i) {
1250                         q = bdev_get_queue(ca->bdev);
1251                         ret |= bdi_congested(q->backing_dev_info, bits);
1252                 }
1253
1254                 cached_dev_put(dc);
1255         }
1256
1257         return ret;
1258 }
1259
1260 void bch_cached_dev_request_init(struct cached_dev *dc)
1261 {
1262         struct gendisk *g = dc->disk.disk;
1263
1264         g->queue->make_request_fn               = cached_dev_make_request;
1265         g->queue->backing_dev_info->congested_fn = cached_dev_congested;
1266         dc->disk.cache_miss                     = cached_dev_cache_miss;
1267         dc->disk.ioctl                          = cached_dev_ioctl;
1268 }
1269
1270 /* Flash backed devices */
1271
1272 static int flash_dev_cache_miss(struct btree *b, struct search *s,
1273                                 struct bio *bio, unsigned int sectors)
1274 {
1275         unsigned int bytes = min(sectors, bio_sectors(bio)) << 9;
1276
1277         swap(bio->bi_iter.bi_size, bytes);
1278         zero_fill_bio(bio);
1279         swap(bio->bi_iter.bi_size, bytes);
1280
1281         bio_advance(bio, bytes);
1282
1283         if (!bio->bi_iter.bi_size)
1284                 return MAP_DONE;
1285
1286         return MAP_CONTINUE;
1287 }
1288
1289 static void flash_dev_nodata(struct closure *cl)
1290 {
1291         struct search *s = container_of(cl, struct search, cl);
1292
1293         if (s->iop.flush_journal)
1294                 bch_journal_meta(s->iop.c, cl);
1295
1296         continue_at(cl, search_free, NULL);
1297 }
1298
1299 static blk_qc_t flash_dev_make_request(struct request_queue *q,
1300                                              struct bio *bio)
1301 {
1302         struct search *s;
1303         struct closure *cl;
1304         struct bcache_device *d = bio->bi_disk->private_data;
1305
1306         if (unlikely(d->c && test_bit(CACHE_SET_IO_DISABLE, &d->c->flags))) {
1307                 bio->bi_status = BLK_STS_IOERR;
1308                 bio_endio(bio);
1309                 return BLK_QC_T_NONE;
1310         }
1311
1312         generic_start_io_acct(q, bio_op(bio), bio_sectors(bio), &d->disk->part0);
1313
1314         s = search_alloc(bio, d);
1315         cl = &s->cl;
1316         bio = &s->bio.bio;
1317
1318         trace_bcache_request_start(s->d, bio);
1319
1320         if (!bio->bi_iter.bi_size) {
1321                 /*
1322                  * can't call bch_journal_meta from under
1323                  * generic_make_request
1324                  */
1325                 continue_at_nobarrier(&s->cl,
1326                                       flash_dev_nodata,
1327                                       bcache_wq);
1328                 return BLK_QC_T_NONE;
1329         } else if (bio_data_dir(bio)) {
1330                 bch_keybuf_check_overlapping(&s->iop.c->moving_gc_keys,
1331                                         &KEY(d->id, bio->bi_iter.bi_sector, 0),
1332                                         &KEY(d->id, bio_end_sector(bio), 0));
1333
1334                 s->iop.bypass           = (bio_op(bio) == REQ_OP_DISCARD) != 0;
1335                 s->iop.writeback        = true;
1336                 s->iop.bio              = bio;
1337
1338                 closure_call(&s->iop.cl, bch_data_insert, NULL, cl);
1339         } else {
1340                 closure_call(&s->iop.cl, cache_lookup, NULL, cl);
1341         }
1342
1343         continue_at(cl, search_free, NULL);
1344         return BLK_QC_T_NONE;
1345 }
1346
1347 static int flash_dev_ioctl(struct bcache_device *d, fmode_t mode,
1348                            unsigned int cmd, unsigned long arg)
1349 {
1350         return -ENOTTY;
1351 }
1352
1353 static int flash_dev_congested(void *data, int bits)
1354 {
1355         struct bcache_device *d = data;
1356         struct request_queue *q;
1357         struct cache *ca;
1358         unsigned int i;
1359         int ret = 0;
1360
1361         for_each_cache(ca, d->c, i) {
1362                 q = bdev_get_queue(ca->bdev);
1363                 ret |= bdi_congested(q->backing_dev_info, bits);
1364         }
1365
1366         return ret;
1367 }
1368
1369 void bch_flash_dev_request_init(struct bcache_device *d)
1370 {
1371         struct gendisk *g = d->disk;
1372
1373         g->queue->make_request_fn               = flash_dev_make_request;
1374         g->queue->backing_dev_info->congested_fn = flash_dev_congested;
1375         d->cache_miss                           = flash_dev_cache_miss;
1376         d->ioctl                                = flash_dev_ioctl;
1377 }
1378
1379 void bch_request_exit(void)
1380 {
1381         kmem_cache_destroy(bch_search_cache);
1382 }
1383
1384 int __init bch_request_init(void)
1385 {
1386         bch_search_cache = KMEM_CACHE(search, 0);
1387         if (!bch_search_cache)
1388                 return -ENOMEM;
1389
1390         return 0;
1391 }